diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.h b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
index 1ad881f673a97821c67f5aa840cec1745485784b..440293f5beaaa24a2637d49cdedf390746ff8662 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
@@ -133,8 +133,8 @@ public:
     /// ``CheckOptions``. If the corresponding key is not present, returns
     /// \p Default.
     template <typename T>
-    typename std::enable_if<std::is_integral<T>::value, T>::type
-    get(StringRef LocalName, T Default) const {
+    std::enable_if_t<std::is_integral<T>::value, T> get(StringRef LocalName,
+                                                        T Default) const {
       std::string Value = get(LocalName, "");
       T Result = Default;
       if (!Value.empty())
@@ -150,7 +150,7 @@ public:
     /// present, falls back to get global option. If global option is not
     /// present either, returns Default.
     template <typename T>
-    typename std::enable_if<std::is_integral<T>::value, T>::type
+    std::enable_if_t<std::is_integral<T>::value, T>
     getLocalOrGlobal(StringRef LocalName, T Default) const {
       std::string Value = getLocalOrGlobal(LocalName, "");
       T Result = Default;
diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
index 5a705376cd4b08d3bc4e430ef31f94912eef7e3d..00793009ed39056f8c65953cf95e044b9173c92e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
@@ -152,6 +152,13 @@ static std::string getCondVarNames(const Stmt *Cond) {
   return Result;
 }
 
+static bool isKnownFalse(const Expr &Cond, const ASTContext &Ctx) {
+  bool Result = false;
+  if (Cond.EvaluateAsBooleanCondition(Result, Ctx))
+    return !Result;
+  return false;
+}
+
 void InfiniteLoopCheck::registerMatchers(MatchFinder *Finder) {
   const auto LoopCondition = allOf(
       hasCondition(
@@ -170,6 +177,9 @@ void InfiniteLoopCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *LoopStmt = Result.Nodes.getNodeAs<Stmt>("loop-stmt");
   const auto *Func = Result.Nodes.getNodeAs<FunctionDecl>("func");
 
+  if (isKnownFalse(*Cond, *Result.Context))
+    return;
+
   bool ShouldHaveConditionVariables = true;
   if (const auto *While = dyn_cast<WhileStmt>(LoopStmt)) {
     if (const VarDecl *LoopVarDecl = While->getConditionVariable()) {
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 4d4d15b0efaeff162659086ea2505e960b87a92e..6fc6c5d21c24e54ea76521bd284e34a586e398d5 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -1473,6 +1473,7 @@ private:
     }
     Output.HasMore = Incomplete;
     Output.Context = CCContextKind;
+    Output.CompletionRange = ReplacedRange;
     return Output;
   }
 
diff --git a/clang-tools-extra/clangd/CodeComplete.h b/clang-tools-extra/clangd/CodeComplete.h
index fd01dfd80b0e6d2abcf311c3cd54e6dd0f0050d0..3b3dca3ee8c5347dda407eacfaabd8b621ab5890 100644
--- a/clang-tools-extra/clangd/CodeComplete.h
+++ b/clang-tools-extra/clangd/CodeComplete.h
@@ -216,6 +216,11 @@ struct CodeCompleteResult {
   std::vector<CodeCompletion> Completions;
   bool HasMore = false;
   CodeCompletionContext::Kind Context = CodeCompletionContext::CCC_Other;
+  // The text that is being directly completed.
+  // Example: foo.pb^ -> foo.push_back()
+  //              ~~
+  // Typically matches the textEdit.range of Completions, but not guaranteed to.
+  llvm::Optional<Range> CompletionRange;
   // Usually the source will be parsed with a real C++ parser.
   // But heuristics may be used instead if e.g. the preamble is not ready.
   bool RanParser = true;
diff --git a/clang-tools-extra/clangd/Shutdown.h b/clang-tools-extra/clangd/Shutdown.h
index 3097f6a3e63c72c658147883e0294a0d0f55f0fc..94ada3171f319efc8add0ec0c821eca3218cc04f 100644
--- a/clang-tools-extra/clangd/Shutdown.h
+++ b/clang-tools-extra/clangd/Shutdown.h
@@ -66,7 +66,7 @@ bool shutdownRequested();
 /// requested (which interrupts IO), we'll fail rather than retry.
 template <typename Fun, typename Ret = decltype(std::declval<Fun>()())>
 Ret retryAfterSignalUnlessShutdown(
-    const typename std::enable_if<true, Ret>::type &Fail, // Suppress deduction.
+    const std::enable_if_t<true, Ret> &Fail, // Suppress deduction.
     const Fun &F) {
   Ret Res;
   do {
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 8863d9f278d73021d5093f8a5fc15ff7fe131e4d..67b8abb722be07d4908c3d86decd018d4a234039 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -2134,6 +2134,7 @@ TEST(GuessCompletionPrefix, Filters) {
            "some text [[scope::more::]][[identif]]^ier",
            "some text [[scope::]][[mor]]^e::identifier",
            "weird case foo::[[::bar::]][[baz]]^",
+           "/* [[]][[]]^ */",
        }) {
     Annotations F(Case);
     auto Offset = cantFail(positionToOffset(F.code(), F.point()));
@@ -2675,6 +2676,28 @@ TEST(CompletionTest, NoCrashWithIncompleteLambda) {
   EXPECT_THAT(Signatures, Contains(Sig("x() -> auto")));
 }
 
+TEST(CompletionTest, CompletionRange) {
+  const char *WithRange = "auto x = [[abc]]^";
+  auto Completions = completions(WithRange);
+  EXPECT_EQ(Completions.CompletionRange, Annotations(WithRange).range());
+  Completions = completionsNoCompile(WithRange);
+  EXPECT_EQ(Completions.CompletionRange, Annotations(WithRange).range());
+
+  const char *EmptyRange = "auto x = [[]]^";
+  Completions = completions(EmptyRange);
+  EXPECT_EQ(Completions.CompletionRange, Annotations(EmptyRange).range());
+  Completions = completionsNoCompile(EmptyRange);
+  EXPECT_EQ(Completions.CompletionRange, Annotations(EmptyRange).range());
+
+  // Sema doesn't trigger at all here, while the no-sema completion runs
+  // heuristics as normal and reports a range. It'd be nice to be consistent.
+  const char *NoCompletion = "/* [[]]^ */";
+  Completions = completions(NoCompletion);
+  EXPECT_EQ(Completions.CompletionRange, llvm::None);
+  Completions = completionsNoCompile(NoCompletion);
+  EXPECT_EQ(Completions.CompletionRange, Annotations(NoCompletion).range());
+}
+
 TEST(NoCompileCompletionTest, Basic) {
   auto Results = completionsNoCompile(R"cpp(
     void func() {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-infinite-loop.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone-infinite-loop.cpp
index d89bdadf02125ff46bbdc232e84158e768f86114..427b5f0272b94ad8dc2690c7d58b8aabf3e08b60 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone-infinite-loop.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-infinite-loop.cpp
@@ -354,3 +354,12 @@ void lambda_capture() {
     (*p)++;
   } while (i < Limit);
 }
+
+void evaluatable(bool CondVar) {
+  for (; false && CondVar;) {
+  }
+  while (false && CondVar) {
+  }
+  do {
+  } while (false && CondVar);
+}
diff --git a/clang/include/clang/AST/ASTTypeTraits.h b/clang/include/clang/AST/ASTTypeTraits.h
index 1a12281d039d814d7795947096e2838ef968eb58..777ad2fec349bf92993e9499fbcb59551c78448a 100644
--- a/clang/include/clang/AST/ASTTypeTraits.h
+++ b/clang/include/clang/AST/ASTTypeTraits.h
@@ -465,22 +465,22 @@ private:
 
 template <typename T>
 struct DynTypedNode::BaseConverter<
-    T, typename std::enable_if<std::is_base_of<Decl, T>::value>::type>
+    T, std::enable_if_t<std::is_base_of<Decl, T>::value>>
     : public DynCastPtrConverter<T, Decl> {};
 
 template <typename T>
 struct DynTypedNode::BaseConverter<
-    T, typename std::enable_if<std::is_base_of<Stmt, T>::value>::type>
+    T, std::enable_if_t<std::is_base_of<Stmt, T>::value>>
     : public DynCastPtrConverter<T, Stmt> {};
 
 template <typename T>
 struct DynTypedNode::BaseConverter<
-    T, typename std::enable_if<std::is_base_of<Type, T>::value>::type>
+    T, std::enable_if_t<std::is_base_of<Type, T>::value>>
     : public DynCastPtrConverter<T, Type> {};
 
 template <typename T>
 struct DynTypedNode::BaseConverter<
-    T, typename std::enable_if<std::is_base_of<OMPClause, T>::value>::type>
+    T, std::enable_if_t<std::is_base_of<OMPClause, T>::value>>
     : public DynCastPtrConverter<T, OMPClause> {};
 
 template <>
diff --git a/clang/include/clang/AST/CanonicalType.h b/clang/include/clang/AST/CanonicalType.h
index 64ec1c0ce471e7b4e45a4994e6daebbd3ca35d3c..31b14c0d39c3da95e70329d61026fe96aeb484a4 100644
--- a/clang/include/clang/AST/CanonicalType.h
+++ b/clang/include/clang/AST/CanonicalType.h
@@ -74,7 +74,7 @@ public:
   /// canonical type pointers.
   template <typename U>
   CanQual(const CanQual<U> &Other,
-          typename std::enable_if<std::is_base_of<T, U>::value, int>::type = 0);
+          std::enable_if_t<std::is_base_of<T, U>::value, int> = 0);
 
   /// Retrieve the underlying type pointer, which refers to a
   /// canonical type.
diff --git a/clang/include/clang/AST/DataCollection.h b/clang/include/clang/AST/DataCollection.h
index 37f101793ecc8c61e6b4813b0ce7c751854461d9..14d1bc188623a9360142091695c2ecaee8a794ab 100644
--- a/clang/include/clang/AST/DataCollection.h
+++ b/clang/include/clang/AST/DataCollection.h
@@ -50,10 +50,9 @@ template <class T> void addDataToConsumer(T &DataConsumer, const QualType &QT) {
 }
 
 template <class T, class Type>
-typename std::enable_if<
-    std::is_integral<Type>::value || std::is_enum<Type>::value ||
-    std::is_convertible<Type, size_t>::value // for llvm::hash_code
-    >::type
+std::enable_if_t<std::is_integral<Type>::value || std::is_enum<Type>::value ||
+                 std::is_convertible<Type, size_t>::value // for llvm::hash_code
+                 >
 addDataToConsumer(T &DataConsumer, Type Data) {
   DataConsumer.update(StringRef(reinterpret_cast<char *>(&Data), sizeof(Data)));
 }
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 5d9174f6e6272b7c0941c063648cd086ca907f1a..fcdb0b992134e41e7a612139adb77d88c3a64e3c 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -5282,10 +5282,9 @@ class GenericSelectionExpr final
   template <bool Const> class AssociationTy {
     friend class GenericSelectionExpr;
     template <bool OtherConst> friend class AssociationIteratorTy;
-    using ExprPtrTy =
-        typename std::conditional<Const, const Expr *, Expr *>::type;
-    using TSIPtrTy = typename std::conditional<Const, const TypeSourceInfo *,
-                                               TypeSourceInfo *>::type;
+    using ExprPtrTy = std::conditional_t<Const, const Expr *, Expr *>;
+    using TSIPtrTy =
+        std::conditional_t<Const, const TypeSourceInfo *, TypeSourceInfo *>;
     ExprPtrTy E;
     TSIPtrTy TSI;
     bool Selected;
@@ -5327,10 +5326,9 @@ class GenericSelectionExpr final
     //    const Association &Assoc = *It++; // Oops, Assoc is dangling.
     using BaseTy = typename AssociationIteratorTy::iterator_facade_base;
     using StmtPtrPtrTy =
-        typename std::conditional<Const, const Stmt *const *, Stmt **>::type;
-    using TSIPtrPtrTy =
-        typename std::conditional<Const, const TypeSourceInfo *const *,
-                                  TypeSourceInfo **>::type;
+        std::conditional_t<Const, const Stmt *const *, Stmt **>;
+    using TSIPtrPtrTy = std::conditional_t<Const, const TypeSourceInfo *const *,
+                                           TypeSourceInfo **>;
     StmtPtrPtrTy E; // = nullptr; FIXME: Once support for gcc 4.8 is dropped.
     TSIPtrPtrTy TSI; // Kept in sync with E.
     unsigned Offset = 0, SelectedOffset = 0;
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 19a84bd00a36dec33411913cd064d8471f5e6fcb..f103530457ee376b9db507a3a1c004bd9d39dea3 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1995,6 +1995,46 @@ public:
   }
 };
 
+/// This represents 'relaxed' clause in the '#pragma omp atomic'
+/// directives.
+///
+/// \code
+/// #pragma omp atomic relaxed
+/// \endcode
+/// In this example directive '#pragma omp atomic' has 'relaxed' clause.
+class OMPRelaxedClause final : public OMPClause {
+public:
+  /// Build 'relaxed' clause.
+  ///
+  /// \param StartLoc Starting location of the clause.
+  /// \param EndLoc Ending location of the clause.
+  OMPRelaxedClause(SourceLocation StartLoc, SourceLocation EndLoc)
+      : OMPClause(OMPC_relaxed, StartLoc, EndLoc) {}
+
+  /// Build an empty clause.
+  OMPRelaxedClause()
+      : OMPClause(OMPC_relaxed, SourceLocation(), SourceLocation()) {}
+
+  child_range children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == OMPC_relaxed;
+  }
+};
+
 /// This represents clause 'private' in the '#pragma omp ...' directives.
 ///
 /// \code
@@ -6571,7 +6611,7 @@ public:
 template<class ImplClass, template <typename> class Ptr, typename RetTy>
 class OMPClauseVisitorBase {
 public:
-#define PTR(CLASS) typename Ptr<CLASS>::type
+#define PTR(CLASS) Ptr<CLASS>
 #define DISPATCH(CLASS) \
   return static_cast<ImplClass*>(this)->Visit##CLASS(static_cast<PTR(CLASS)>(S))
 
@@ -6594,12 +6634,11 @@ public:
 #undef DISPATCH
 };
 
-template <typename T>
-using const_ptr = typename std::add_pointer<typename std::add_const<T>::type>;
+template <typename T> using const_ptr = std::add_pointer_t<std::add_const_t<T>>;
 
-template<class ImplClass, typename RetTy = void>
-class OMPClauseVisitor :
-      public OMPClauseVisitorBase <ImplClass, std::add_pointer, RetTy> {};
+template <class ImplClass, typename RetTy = void>
+class OMPClauseVisitor
+    : public OMPClauseVisitorBase<ImplClass, std::add_pointer_t, RetTy> {};
 template<class ImplClass, typename RetTy = void>
 class ConstOMPClauseVisitor :
       public OMPClauseVisitorBase <ImplClass, const_ptr, RetTy> {};
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 4ef9528dcf5bb8c076bb421bb3a9c8c183495705..29b2c3541002090c82fcf8acb5d65fb965a1dcad 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -340,11 +340,11 @@ private:
   (has_same_member_pointer_type<decltype(                                      \
                                     &RecursiveASTVisitor::Traverse##NAME),     \
                                 decltype(&Derived::Traverse##NAME)>::value     \
-       ? static_cast<typename std::conditional<                                \
+       ? static_cast<std::conditional_t<                                       \
              has_same_member_pointer_type<                                     \
                  decltype(&RecursiveASTVisitor::Traverse##NAME),               \
                  decltype(&Derived::Traverse##NAME)>::value,                   \
-             Derived &, RecursiveASTVisitor &>::type>(*this)                   \
+             Derived &, RecursiveASTVisitor &>>(*this)                         \
              .Traverse##NAME(static_cast<CLASS *>(VAR), QUEUE)                 \
        : getDerived().Traverse##NAME(static_cast<CLASS *>(VAR)))
 
@@ -3136,6 +3136,11 @@ bool RecursiveASTVisitor<Derived>::VisitOMPReleaseClause(OMPReleaseClause *) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPRelaxedClause(OMPRelaxedClause *) {
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPThreadsClause(OMPThreadsClause *) {
   return true;
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index 696eca1aa6431c67c7d46c515f3f312c72cfafdb..ff6aad056fcc94477f153f0ea0d884ca7c1109ec 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -516,8 +516,8 @@ public:
   /// Requires \c T to be derived from \c From.
   template <typename From>
   Matcher(const Matcher<From> &Other,
-          typename std::enable_if<std::is_base_of<From, T>::value &&
-                               !std::is_same<From, T>::value>::type * = nullptr)
+          std::enable_if_t<std::is_base_of<From, T>::value &&
+                           !std::is_same<From, T>::value> * = nullptr)
       : Implementation(restrictMatcher(Other.Implementation)) {
     assert(Implementation.getSupportedKind().isSame(
         ast_type_traits::ASTNodeKind::getFromNodeKind<T>()));
@@ -528,9 +528,8 @@ public:
   /// The resulting matcher is not strict, i.e. ignores qualifiers.
   template <typename TypeT>
   Matcher(const Matcher<TypeT> &Other,
-          typename std::enable_if<
-            std::is_same<T, QualType>::value &&
-            std::is_same<TypeT, Type>::value>::type* = nullptr)
+          std::enable_if_t<std::is_same<T, QualType>::value &&
+                           std::is_same<TypeT, Type>::value> * = nullptr)
       : Implementation(new TypeToQualType<TypeT>(Other)) {}
 
   /// Convert \c this into a \c Matcher<T> by applying dyn_cast<> to the
diff --git a/clang/include/clang/Analysis/CFG.h b/clang/include/clang/Analysis/CFG.h
index 93de3178e661dadc7dd4af889bddca3794378c62..43fb523c863a54ec7211d26da5e98d079d40bf57 100644
--- a/clang/include/clang/Analysis/CFG.h
+++ b/clang/include/clang/Analysis/CFG.h
@@ -624,10 +624,10 @@ class CFGBlock {
     template <bool IsOtherConst> friend class ElementRefImpl;
 
     using CFGBlockPtr =
-        typename std::conditional<IsConst, const CFGBlock *, CFGBlock *>::type;
+        std::conditional_t<IsConst, const CFGBlock *, CFGBlock *>;
 
-    using CFGElementPtr = typename std::conditional<IsConst, const CFGElement *,
-                                                    CFGElement *>::type;
+    using CFGElementPtr =
+        std::conditional_t<IsConst, const CFGElement *, CFGElement *>;
 
   protected:
     CFGBlockPtr Parent;
@@ -675,15 +675,14 @@ class CFGBlock {
     friend class ElementRefIterator;
 
     using CFGBlockRef =
-        typename std::conditional<IsConst, const CFGBlock *, CFGBlock *>::type;
+        std::conditional_t<IsConst, const CFGBlock *, CFGBlock *>;
 
-    using UnderlayingIteratorTy = typename std::conditional<
+    using UnderlayingIteratorTy = std::conditional_t<
         IsConst,
-        typename std::conditional<IsReverse,
-                                  ElementList::const_reverse_iterator,
-                                  ElementList::const_iterator>::type,
-        typename std::conditional<IsReverse, ElementList::reverse_iterator,
-                                  ElementList::iterator>::type>::type;
+        std::conditional_t<IsReverse, ElementList::const_reverse_iterator,
+                           ElementList::const_iterator>,
+        std::conditional_t<IsReverse, ElementList::reverse_iterator,
+                           ElementList::iterator>>;
 
     using IteratorTraits = typename std::iterator_traits<UnderlayingIteratorTy>;
     using ElementRef = typename CFGBlock::ElementRefImpl<IsConst>;
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 48c0df49e32dc43a1d761f2521c60257e842051b..fa450724ddd476ad424fad47e70187b8024d5bdf 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -63,7 +63,6 @@ CODEGENOPT(ExperimentalNewPassManager, 1, 0) ///< Enables the new, experimental
 CODEGENOPT(DebugPassManager, 1, 0) ///< Prints debug information for the new
                                    ///< pass manager.
 CODEGENOPT(DisableRedZone    , 1, 0) ///< Set when -mno-red-zone is enabled.
-CODEGENOPT(EnableDebugEntryValues, 1, 0) ///< Emit call site parameter dbg info
 CODEGENOPT(IndirectTlsSegRefs, 1, 0) ///< Set when -mno-tls-direct-seg-refs
                                      ///< is specified.
 CODEGENOPT(DisableTailCalls  , 1, 0) ///< Do not emit tail calls.
diff --git a/clang/include/clang/Basic/Diagnostic.h b/clang/include/clang/Basic/Diagnostic.h
index 377c3d56eafed648975e74eb9803a131b7a367e6..6bc8a143a26de2937ed16d214e83f6d27b4a3299 100644
--- a/clang/include/clang/Basic/Diagnostic.h
+++ b/clang/include/clang/Basic/Diagnostic.h
@@ -1217,9 +1217,7 @@ inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB, int I) {
 // We use enable_if here to prevent that this overload is selected for
 // pointers or other arguments that are implicitly convertible to bool.
 template <typename T>
-inline
-typename std::enable_if<std::is_same<T, bool>::value,
-                        const DiagnosticBuilder &>::type
+inline std::enable_if_t<std::is_same<T, bool>::value, const DiagnosticBuilder &>
 operator<<(const DiagnosticBuilder &DB, T I) {
   DB.AddTaggedVal(I, DiagnosticsEngine::ak_sint);
   return DB;
@@ -1249,9 +1247,9 @@ inline const DiagnosticBuilder &operator<<(const DiagnosticBuilder &DB,
 // other arguments that derive from DeclContext (e.g., RecordDecls) will not
 // match.
 template <typename T>
-inline typename std::enable_if<
-    std::is_same<typename std::remove_const<T>::type, DeclContext>::value,
-    const DiagnosticBuilder &>::type
+inline std::enable_if_t<
+    std::is_same<std::remove_const_t<T>, DeclContext>::value,
+    const DiagnosticBuilder &>
 operator<<(const DiagnosticBuilder &DB, T *DC) {
   DB.AddTaggedVal(reinterpret_cast<intptr_t>(DC),
                   DiagnosticsEngine::ak_declcontext);
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 31f73490f4e331d8a61a963210427468c7a97999..7eb3bc274cf050a2539d2ef756380b5f33346bfa 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9696,7 +9696,9 @@ def note_omp_atomic_capture: Note<
 def err_omp_atomic_several_clauses : Error<
   "directive '#pragma omp atomic' cannot contain more than one 'read', 'write', 'update' or 'capture' clause">;
 def err_omp_several_mem_order_clauses : Error<
-  "directive '#pragma omp %0' cannot contain more than one %select{'seq_cst', |}1'acq_rel', 'acquire' or 'release' clause">;
+  "directive '#pragma omp %0' cannot contain more than one %select{'seq_cst', 'relaxed', |}1'acq_rel', 'acquire' or 'release' clause">;
+def err_omp_atomic_incompatible_mem_order_clause : Error<
+  "directive '#pragma omp atomic%select{ %0|}1' cannot be used with '%2' clause">;
 def note_omp_previous_mem_order_clause : Note<
   "'%0' clause used here">;
 def err_omp_target_contains_not_only_teams : Error<
@@ -9850,10 +9852,10 @@ def err_omp_requires_clause_redeclaration : Error <
   "Only one %0 clause can appear on a requires directive in a single translation unit">;
 def note_omp_requires_previous_clause : Note <
   "%0 clause previously used here">;
-def err_omp_target_before_requires : Error <
-  "target region encountered before requires directive with '%0' clause">;
-def note_omp_requires_encountered_target : Note <
-  "target previously encountered here">;
+def err_omp_directive_before_requires : Error <
+  "'%0' region encountered before requires directive with '%1' clause">;
+def note_omp_requires_encountered_directive : Note <
+  "'%0' previously encountered here">;
 def err_omp_invalid_scope : Error <
   "'#pragma omp %0' directive must appear only in file scope">;
 def note_omp_invalid_length_on_this_ptr_mapping : Note <
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 48c96d687f0564a1dd89c03befe79da90f6b0e1d..126d67d6203c3771035e4b12c82ba956765aa5ed 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -132,6 +132,7 @@ LANGOPT(DWARFExceptions   , 1, 0, "dwarf exception handling")
 LANGOPT(SjLjExceptions    , 1, 0, "setjmp-longjump exception handling")
 LANGOPT(SEHExceptions     , 1, 0, "SEH .xdata exception handling")
 LANGOPT(WasmExceptions    , 1, 0, "WebAssembly exception handling")
+LANGOPT(IgnoreExceptions  , 1, 0, "ignore exceptions")
 LANGOPT(ExternCNoUnwind   , 1, 0, "Assume extern C functions don't unwind")
 LANGOPT(TraditionalCPP    , 1, 0, "traditional CPP emulation")
 LANGOPT(RTTI              , 1, 1, "run-time type information")
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 19fcf7cfac58cd9244aa60610b2c5f8210194da4..13b3438fec2c85ca78f67e701d22e891cb19a8b3 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -260,6 +260,7 @@ OPENMP_CLAUSE(seq_cst, OMPSeqCstClause)
 OPENMP_CLAUSE(acq_rel, OMPAcqRelClause)
 OPENMP_CLAUSE(acquire, OMPAcquireClause)
 OPENMP_CLAUSE(release, OMPReleaseClause)
+OPENMP_CLAUSE(relaxed, OMPRelaxedClause)
 OPENMP_CLAUSE(depend, OMPDependClause)
 OPENMP_CLAUSE(device, OMPDeviceClause)
 OPENMP_CLAUSE(threads, OMPThreadsClause)
@@ -495,6 +496,7 @@ OPENMP_ATOMIC_CLAUSE(seq_cst)
 OPENMP_ATOMIC_CLAUSE(acq_rel)
 OPENMP_ATOMIC_CLAUSE(acquire)
 OPENMP_ATOMIC_CLAUSE(release)
+OPENMP_ATOMIC_CLAUSE(relaxed)
 
 // Clauses allowed for OpenMP directive 'target'.
 OPENMP_TARGET_CLAUSE(if)
diff --git a/clang/include/clang/Basic/PartialDiagnostic.h b/clang/include/clang/Basic/PartialDiagnostic.h
index c413cc2ffdf8598800530345fcb5f0bc9038f559..107d621f0dec55a4194ecb2124e7fcbdc0a1e2d5 100644
--- a/clang/include/clang/Basic/PartialDiagnostic.h
+++ b/clang/include/clang/Basic/PartialDiagnostic.h
@@ -378,10 +378,9 @@ public:
   // so that we only match those arguments that are (statically) DeclContexts;
   // other arguments that derive from DeclContext (e.g., RecordDecls) will not
   // match.
-  template<typename T>
-  friend inline
-  typename std::enable_if<std::is_same<T, DeclContext>::value,
-                          const PartialDiagnostic &>::type
+  template <typename T>
+  friend inline std::enable_if_t<std::is_same<T, DeclContext>::value,
+                                 const PartialDiagnostic &>
   operator<<(const PartialDiagnostic &PD, T *DC) {
     PD.AddTaggedVal(reinterpret_cast<intptr_t>(DC),
                     DiagnosticsEngine::ak_declcontext);
diff --git a/clang/include/clang/CodeGen/CGFunctionInfo.h b/clang/include/clang/CodeGen/CGFunctionInfo.h
index 2a41ab9eece7ff69c28561632e82d687ba1e3b06..588c96afe402feadd8c871b8756e266ce65cfea5 100644
--- a/clang/include/clang/CodeGen/CGFunctionInfo.h
+++ b/clang/include/clang/CodeGen/CGFunctionInfo.h
@@ -88,6 +88,7 @@ private:
   Kind TheKind;
   bool PaddingInReg : 1;
   bool InAllocaSRet : 1;    // isInAlloca()
+  bool InAllocaIndirect : 1;// isInAlloca()
   bool IndirectByVal : 1;   // isIndirect()
   bool IndirectRealign : 1; // isIndirect()
   bool SRetAfterThis : 1;   // isIndirect()
@@ -110,8 +111,8 @@ private:
 
 public:
   ABIArgInfo(Kind K = Direct)
-      : TypeData(nullptr), PaddingType(nullptr), DirectOffset(0),
-        TheKind(K), PaddingInReg(false), InAllocaSRet(false),
+      : TypeData(nullptr), PaddingType(nullptr), DirectOffset(0), TheKind(K),
+        PaddingInReg(false), InAllocaSRet(false), InAllocaIndirect(false),
         IndirectByVal(false), IndirectRealign(false), SRetAfterThis(false),
         InReg(false), CanBeFlattened(false), SignExt(false) {}
 
@@ -185,9 +186,10 @@ public:
     AI.setInReg(true);
     return AI;
   }
-  static ABIArgInfo getInAlloca(unsigned FieldIndex) {
+  static ABIArgInfo getInAlloca(unsigned FieldIndex, bool Indirect = false) {
     auto AI = ABIArgInfo(InAlloca);
     AI.setInAllocaFieldIndex(FieldIndex);
+    AI.setInAllocaIndirect(Indirect);
     return AI;
   }
   static ABIArgInfo getExpand() {
@@ -380,6 +382,15 @@ public:
     AllocaFieldIndex = FieldIndex;
   }
 
+  unsigned getInAllocaIndirect() const {
+    assert(isInAlloca() && "Invalid kind!");
+    return InAllocaIndirect;
+  }
+  void setInAllocaIndirect(bool Indirect) {
+    assert(isInAlloca() && "Invalid kind!");
+    InAllocaIndirect = Indirect;
+  }
+
   /// Return true if this field of an inalloca struct should be returned
   /// to implement a struct return calling convention.
   bool getInAllocaSRet() const {
diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td
index 733a1080be52dc53840af96a94a6ae33ba7d3301..8dcf28e34810b115072bb2637c6371164a40b33d 100644
--- a/clang/include/clang/Driver/CC1Options.td
+++ b/clang/include/clang/Driver/CC1Options.td
@@ -390,8 +390,6 @@ def flto_visibility_public_std:
 def flto_unit: Flag<["-"], "flto-unit">,
     HelpText<"Emit IR to support LTO unit features (CFI, whole program vtable opt)">;
 def fno_lto_unit: Flag<["-"], "fno-lto-unit">;
-def femit_debug_entry_values : Flag<["-"], "femit-debug-entry-values">,
-    HelpText<"Enables debug info about call site parameter's entry values">;
 def fdebug_pass_manager : Flag<["-"], "fdebug-pass-manager">,
     HelpText<"Prints debug information for the new pass manager">;
 def fno_debug_pass_manager : Flag<["-"], "fno-debug-pass-manager">,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 209e6a17f30046af0037a24e495051745747cce2..65f2e9b0273a9939729121e54bbad25a227da46e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -932,6 +932,8 @@ def fseh_exceptions : Flag<["-"], "fseh-exceptions">, Group<f_Group>,
   Flags<[CC1Option]>, HelpText<"Use SEH style exceptions">;
 def fwasm_exceptions : Flag<["-"], "fwasm-exceptions">, Group<f_Group>,
   Flags<[CC1Option]>, HelpText<"Use WebAssembly style exceptions">;
+def fignore_exceptions : Flag<["-"], "fignore-exceptions">, Group<f_Group>, Flags<[CC1Option]>,
+  HelpText<"Enable support for ignoring exception handling constructs">;
 def fexcess_precision_EQ : Joined<["-"], "fexcess-precision=">,
     Group<clang_ignored_gcc_optimization_f_Group>;
 def : Flag<["-"], "fexpensive-optimizations">, Group<clang_ignored_gcc_optimization_f_Group>;
diff --git a/clang/include/clang/Driver/XRayArgs.h b/clang/include/clang/Driver/XRayArgs.h
index fa2583f4b9664c29233a4046997ec4bcfa793db2..96098bf629cd232c173924ac093ecb0569e116b4 100644
--- a/clang/include/clang/Driver/XRayArgs.h
+++ b/clang/include/clang/Driver/XRayArgs.h
@@ -30,6 +30,7 @@ class XRayArgs {
   bool XRayAlwaysEmitCustomEvents = false;
   bool XRayAlwaysEmitTypedEvents = false;
   bool XRayRT = true;
+  bool XRayIgnoreLoops = false;
 
 public:
   /// Parses the XRay arguments from an argument list.
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index a274102ceb11e2a732ea03812c223e273f7c43d0..f1a8b98e5efdcceb02cde8faf0b8cdd1a18b5dc5 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -677,6 +677,24 @@ class Sema;
       StdInitializerListElement = V;
     }
 
+    /// Form an "implicit" conversion sequence from nullptr_t to bool, for a
+    /// direct-initialization of a bool object from nullptr_t.
+    static ImplicitConversionSequence getNullptrToBool(QualType SourceType,
+                                                       QualType DestType,
+                                                       bool NeedLValToRVal) {
+      ImplicitConversionSequence ICS;
+      ICS.setStandard();
+      ICS.Standard.setAsIdentityConversion();
+      ICS.Standard.setFromType(SourceType);
+      if (NeedLValToRVal)
+        ICS.Standard.First = ICK_Lvalue_To_Rvalue;
+      ICS.Standard.setToType(0, SourceType);
+      ICS.Standard.Second = ICK_Boolean_Conversion;
+      ICS.Standard.setToType(1, DestType);
+      ICS.Standard.setToType(2, DestType);
+      return ICS;
+    }
+
     // The result of a comparison between implicit conversion
     // sequences. Use Sema::CompareImplicitConversionSequences to
     // actually perform the comparison.
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 1b9f3b95c25fec3f3149a6770bd9253d8cbcb8fc..7099e692a3b3f58b9470576de8ea3672ae5852a1 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -10337,6 +10337,9 @@ public:
   /// Called on well-formed 'release' clause.
   OMPClause *ActOnOpenMPReleaseClause(SourceLocation StartLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'relaxed' clause.
+  OMPClause *ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
   /// Called on well-formed 'threads' clause.
   OMPClause *ActOnOpenMPThreadsClause(SourceLocation StartLoc,
                                       SourceLocation EndLoc);
diff --git a/clang/include/clang/Tooling/Refactoring/RefactoringOptions.h b/clang/include/clang/Tooling/Refactoring/RefactoringOptions.h
index f25f526e146cc1c85028f6b0f608d74133d1be35..84122b111ee1704392ba83cae090e711d3153b0a 100644
--- a/clang/include/clang/Tooling/Refactoring/RefactoringOptions.h
+++ b/clang/include/clang/Tooling/Refactoring/RefactoringOptions.h
@@ -20,8 +20,8 @@ namespace clang {
 namespace tooling {
 
 /// A refactoring option that stores a value of type \c T.
-template <typename T, typename = typename std::enable_if<
-                          traits::IsValidOptionType<T>::value>::type>
+template <typename T,
+          typename = std::enable_if_t<traits::IsValidOptionType<T>::value>>
 class OptionalRefactoringOption : public RefactoringOption {
 public:
   void passToVisitor(RefactoringOptionVisitor &Visitor) final override {
@@ -39,8 +39,8 @@ protected:
 };
 
 /// A required refactoring option that stores a value of type \c T.
-template <typename T, typename = typename std::enable_if<
-                          traits::IsValidOptionType<T>::value>::type>
+template <typename T,
+          typename = std::enable_if_t<traits::IsValidOptionType<T>::value>>
 class RequiredRefactoringOption : public OptionalRefactoringOption<T> {
 public:
   using ValueType = T;
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index dba23e05371db09a2967ee92dd0c78a84286974f..2e46a2b4e9839230e73cbd7cab9fa84d5f60815f 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -10059,6 +10059,8 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
     return true;
   else if (isa<PragmaDetectMismatchDecl>(D))
     return true;
+  else if (isa<OMPRequiresDecl>(D))
+    return true;
   else if (isa<OMPThreadPrivateDecl>(D))
     return !D->getDeclContext()->isDependentContext();
   else if (isa<OMPAllocateDecl>(D))
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 6aad4543b41c1d70618971f8a45e989280bb0423..49058ceaab25cb909f4ae51c8250f400d9eaaa2e 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -204,9 +204,7 @@ namespace clang {
 
     // Wrapper for an overload set.
     template <typename ToDeclT> struct CallOverloadedCreateFun {
-      template <typename... Args>
-      auto operator()(Args &&... args)
-          -> decltype(ToDeclT::Create(std::forward<Args>(args)...)) {
+      template <typename... Args> decltype(auto) operator()(Args &&... args) {
         return ToDeclT::Create(std::forward<Args>(args)...);
       }
     };
@@ -650,7 +648,7 @@ namespace clang {
 
     template<typename IIter, typename OIter>
     Error ImportArrayChecked(IIter Ibegin, IIter Iend, OIter Obegin) {
-      using ItemT = typename std::remove_reference<decltype(*Obegin)>::type;
+      using ItemT = std::remove_reference_t<decltype(*Obegin)>;
       for (; Ibegin != Iend; ++Ibegin, ++Obegin) {
         Expected<ItemT> ToOrErr = import(*Ibegin);
         if (!ToOrErr)
diff --git a/clang/lib/AST/Interp/Boolean.h b/clang/lib/AST/Interp/Boolean.h
index 3e6c8b5da9f056c08a6bff1feb4808f918e1b01d..2baa717311bc479c0e99b2b05f419afe83fdd82f 100644
--- a/clang/lib/AST/Interp/Boolean.h
+++ b/clang/lib/AST/Interp/Boolean.h
@@ -85,14 +85,13 @@ class Boolean {
   static Boolean max(unsigned NumBits) { return Boolean(true); }
 
   template <typename T>
-  static typename std::enable_if<std::is_integral<T>::value, Boolean>::type
-  from(T Value) {
+  static std::enable_if_t<std::is_integral<T>::value, Boolean> from(T Value) {
     return Boolean(Value != 0);
   }
 
   template <unsigned SrcBits, bool SrcSign>
-  static typename std::enable_if<SrcBits != 0, Boolean>::type from(
-      Integral<SrcBits, SrcSign> Value) {
+  static std::enable_if_t<SrcBits != 0, Boolean>
+  from(Integral<SrcBits, SrcSign> Value) {
     return Boolean(!Value.isZero());
   }
 
diff --git a/clang/lib/AST/Interp/Integral.h b/clang/lib/AST/Interp/Integral.h
index 7cc788070de84eced91ee81e5962386585baecf4..46cd611ee3892aac69268fd0b44bebdbaeccc275 100644
--- a/clang/lib/AST/Interp/Integral.h
+++ b/clang/lib/AST/Interp/Integral.h
@@ -156,13 +156,12 @@ public:
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_integral<T>::value, Integral>::type
-  from(T Value) {
+  static std::enable_if_t<std::is_integral<T>::value, Integral> from(T Value) {
     return Integral(Value);
   }
 
   template <unsigned SrcBits, bool SrcSign>
-  static typename std::enable_if<SrcBits != 0, Integral>::type
+  static std::enable_if_t<SrcBits != 0, Integral>
   from(Integral<SrcBits, SrcSign> Value) {
     return Integral(Value.V);
   }
@@ -206,52 +205,52 @@ public:
 
 private:
   template <typename T>
-  static typename std::enable_if<std::is_signed<T>::value, bool>::type
-  CheckAddUB(T A, T B, T &R) {
+  static std::enable_if_t<std::is_signed<T>::value, bool> CheckAddUB(T A, T B,
+                                                                     T &R) {
     return llvm::AddOverflow<T>(A, B, R);
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_unsigned<T>::value, bool>::type
-  CheckAddUB(T A, T B, T &R) {
+  static std::enable_if_t<std::is_unsigned<T>::value, bool> CheckAddUB(T A, T B,
+                                                                       T &R) {
     R = A + B;
     return false;
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_signed<T>::value, bool>::type
-  CheckSubUB(T A, T B, T &R) {
+  static std::enable_if_t<std::is_signed<T>::value, bool> CheckSubUB(T A, T B,
+                                                                     T &R) {
     return llvm::SubOverflow<T>(A, B, R);
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_unsigned<T>::value, bool>::type
-  CheckSubUB(T A, T B, T &R) {
+  static std::enable_if_t<std::is_unsigned<T>::value, bool> CheckSubUB(T A, T B,
+                                                                       T &R) {
     R = A - B;
     return false;
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_signed<T>::value, bool>::type
-  CheckMulUB(T A, T B, T &R) {
+  static std::enable_if_t<std::is_signed<T>::value, bool> CheckMulUB(T A, T B,
+                                                                     T &R) {
     return llvm::MulOverflow<T>(A, B, R);
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_unsigned<T>::value, bool>::type
-  CheckMulUB(T A, T B, T &R) {
+  static std::enable_if_t<std::is_unsigned<T>::value, bool> CheckMulUB(T A, T B,
+                                                                       T &R) {
     R = A * B;
     return false;
   }
 
   template <typename T, T Min, T Max>
-  static typename std::enable_if<std::is_signed<T>::value, bool>::type
+  static std::enable_if_t<std::is_signed<T>::value, bool>
   CheckRange(int64_t V) {
     return Min <= V && V <= Max;
   }
 
   template <typename T, T Min, T Max>
-  static typename std::enable_if<std::is_unsigned<T>::value, bool>::type
+  static std::enable_if_t<std::is_unsigned<T>::value, bool>
   CheckRange(int64_t V) {
     return V >= 0 && static_cast<uint64_t>(V) <= Max;
   }
diff --git a/clang/lib/AST/Interp/Source.h b/clang/lib/AST/Interp/Source.h
index e591c3399d7c1dc5d15f5e78666ea23a0eb99607..19c652b7331a12d05c61fe6a373f2139726b7bd9 100644
--- a/clang/lib/AST/Interp/Source.h
+++ b/clang/lib/AST/Interp/Source.h
@@ -56,14 +56,14 @@ private:
 
   /// Helper to decode a value or a pointer.
   template <typename T>
-  static typename std::enable_if<!std::is_pointer<T>::value, T>::type
+  static std::enable_if_t<!std::is_pointer<T>::value, T>
   ReadHelper(const char *Ptr) {
     using namespace llvm::support;
     return endian::read<T, endianness::native, 1>(Ptr);
   }
 
   template <typename T>
-  static typename std::enable_if<std::is_pointer<T>::value, T>::type
+  static std::enable_if_t<std::is_pointer<T>::value, T>
   ReadHelper(const char *Ptr) {
     using namespace llvm::support;
     auto Punned = endian::read<uintptr_t, endianness::native, 1>(Ptr);
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 2947665981aff6f39f8b61ea417b88f07bf502b5..6eac98250c8ff737af3148f3410b2f06b6136b0c 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -119,6 +119,7 @@ const OMPClauseWithPreInit *OMPClauseWithPreInit::get(const OMPClause *C) {
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_depend:
   case OMPC_threads:
   case OMPC_simd:
@@ -196,6 +197,7 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_depend:
   case OMPC_device:
   case OMPC_threads:
@@ -1352,6 +1354,10 @@ void OMPClausePrinter::VisitOMPReleaseClause(OMPReleaseClause *) {
   OS << "release";
 }
 
+void OMPClausePrinter::VisitOMPRelaxedClause(OMPRelaxedClause *) {
+  OS << "relaxed";
+}
+
 void OMPClausePrinter::VisitOMPThreadsClause(OMPThreadsClause *) {
   OS << "threads";
 }
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index b303851a75c6f588c7a768df465748886778fe9c..14ddc13ce561d76304496a086797c38da89d57d8 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -524,6 +524,8 @@ void OMPClauseProfiler::VisitOMPAcquireClause(const OMPAcquireClause *) {}
 
 void OMPClauseProfiler::VisitOMPReleaseClause(const OMPReleaseClause *) {}
 
+void OMPClauseProfiler::VisitOMPRelaxedClause(const OMPRelaxedClause *) {}
+
 void OMPClauseProfiler::VisitOMPThreadsClause(const OMPThreadsClause *) {}
 
 void OMPClauseProfiler::VisitOMPSIMDClause(const OMPSIMDClause *) {}
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index 4c1ea8995f9f66713e78679b8ad851f3b2343589..8091625703bc50dba0b808994d85699b493bf6f2 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -720,10 +720,10 @@ private:
   // These sorts of call expressions don't have a common superclass,
   // hence strict duck-typing.
   template <typename CallLikeExpr,
-            typename = typename std::enable_if<
-                std::is_same<CallLikeExpr, CallExpr>::value ||
-                std::is_same<CallLikeExpr, CXXConstructExpr>::value ||
-                std::is_same<CallLikeExpr, ObjCMessageExpr>::value>>
+            typename = std::enable_if_t<
+                std::is_base_of<CallExpr, CallLikeExpr>::value ||
+                std::is_base_of<CXXConstructExpr, CallLikeExpr>::value ||
+                std::is_base_of<ObjCMessageExpr, CallLikeExpr>::value>>
   void findConstructionContextsForArguments(CallLikeExpr *E) {
     for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
       Expr *Arg = E->getArg(i);
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index 48f4106b6bae80169423945910e364c897a1015f..252083f377d9480f3070aeb3db48cd607877616f 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -1249,8 +1249,7 @@ static StringRef ClassifyDiagnostic(const ValueDecl *VD) {
 }
 
 template <typename AttrTy>
-static typename std::enable_if<!has_arg_iterator_range<AttrTy>::value,
-                               StringRef>::type
+static std::enable_if_t<!has_arg_iterator_range<AttrTy>::value, StringRef>
 ClassifyDiagnostic(const AttrTy *A) {
   if (const ValueDecl *VD = getValueDecl(A->getArg()))
     return ClassifyDiagnostic(VD);
@@ -1258,8 +1257,7 @@ ClassifyDiagnostic(const AttrTy *A) {
 }
 
 template <typename AttrTy>
-static typename std::enable_if<has_arg_iterator_range<AttrTy>::value,
-                               StringRef>::type
+static std::enable_if_t<has_arg_iterator_range<AttrTy>::value, StringRef>
 ClassifyDiagnostic(const AttrTy *A) {
   for (const auto *Arg : A->args()) {
     if (const ValueDecl *VD = getValueDecl(Arg))
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 124723552af4c9564c520959c6de162cbdfa58d3..f13364e5378bb2ff380eccbeadff83eb7c0c077b 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -217,6 +217,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind,
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_device:
   case OMPC_threads:
   case OMPC_simd:
@@ -432,6 +433,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_device:
   case OMPC_threads:
   case OMPC_simd:
@@ -598,7 +600,7 @@ bool clang::isAllowedClauseForDirective(OpenMPDirectiveKind DKind,
     break;
   case OMPD_atomic:
     if (OpenMPVersion < 50 && (CKind == OMPC_acq_rel || CKind == OMPC_acquire ||
-                               CKind == OMPC_release))
+                               CKind == OMPC_release || CKind == OMPC_relaxed))
       return false;
     switch (CKind) {
 #define OPENMP_ATOMIC_CLAUSE(Name)                                             \
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 48e26459e94f46093a4ca84bfeb8ef8727f0ca06..033a2898c5a8f9894db4c4b8321decd8afbf229d 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -481,7 +481,6 @@ static void initTargetOptions(llvm::TargetOptions &Options,
   Options.DebuggerTuning = CodeGenOpts.getDebuggerTuning();
   Options.EmitStackSizeSection = CodeGenOpts.StackSizeSection;
   Options.EmitAddrsig = CodeGenOpts.Addrsig;
-  Options.EnableDebugEntryValues = CodeGenOpts.EnableDebugEntryValues;
   Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection;
 
   Options.MCOptions.SplitDwarfFile = CodeGenOpts.SplitDwarfFile;
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index f6028cc65209707b0faaf980be1e149f7978cdce..508a868d45a708e903560afc126df53596209edb 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2370,6 +2370,9 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       auto FieldIndex = ArgI.getInAllocaFieldIndex();
       Address V =
           Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
+      if (ArgI.getInAllocaIndirect())
+        V = Address(Builder.CreateLoad(V),
+                    getContext().getTypeAlignInChars(Ty));
       ArgVals.push_back(ParamValue::forIndirect(V));
       break;
     }
@@ -4091,18 +4094,39 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       assert(NumIRArgs == 0);
       assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
       if (I->isAggregate()) {
-        // Replace the placeholder with the appropriate argument slot GEP.
         Address Addr = I->hasLValue()
                            ? I->getKnownLValue().getAddress(*this)
                            : I->getKnownRValue().getAggregateAddress();
         llvm::Instruction *Placeholder =
             cast<llvm::Instruction>(Addr.getPointer());
-        CGBuilderTy::InsertPoint IP = Builder.saveIP();
-        Builder.SetInsertPoint(Placeholder);
-        Addr =
-            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
-        Builder.restoreIP(IP);
+
+        if (!ArgInfo.getInAllocaIndirect()) {
+          // Replace the placeholder with the appropriate argument slot GEP.
+          CGBuilderTy::InsertPoint IP = Builder.saveIP();
+          Builder.SetInsertPoint(Placeholder);
+          Addr = Builder.CreateStructGEP(ArgMemory,
+                                         ArgInfo.getInAllocaFieldIndex());
+          Builder.restoreIP(IP);
+        } else {
+          // For indirect things such as overaligned structs, replace the
+          // placeholder with a regular aggregate temporary alloca. Store the
+          // address of this alloca into the struct.
+          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
+          Address ArgSlot = Builder.CreateStructGEP(
+              ArgMemory, ArgInfo.getInAllocaFieldIndex());
+          Builder.CreateStore(Addr.getPointer(), ArgSlot);
+        }
         deferPlaceholderReplacement(Placeholder, Addr.getPointer());
+      } else if (ArgInfo.getInAllocaIndirect()) {
+        // Make a temporary alloca and store the address of it into the argument
+        // struct.
+        Address Addr = CreateMemTempWithoutCast(
+            I->Ty, getContext().getTypeAlignInChars(I->Ty),
+            "indirect-arg-temp");
+        I->copyInto(*this, Addr);
+        Address ArgSlot =
+            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
+        Builder.CreateStore(Addr.getPointer(), ArgSlot);
       } else {
         // Store the RValue into the argument struct.
         Address Addr =
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index e171082942f6e123a9ad0a25e1ce295d4fc33039..f706ffb92a0e83652a1e1a34b5063227efb751de 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -4861,8 +4861,7 @@ llvm::DINode::DIFlags CGDebugInfo::getCallSiteRelatedAttrs() const {
       (CGM.getCodeGenOpts().getDebuggerTuning() == llvm::DebuggerKind::LLDB ||
        CGM.getCodeGenOpts().getDebuggerTuning() == llvm::DebuggerKind::GDB);
 
-  if (!SupportsDWARFv4Ext && CGM.getCodeGenOpts().DwarfVersion < 5 &&
-      !CGM.getCodeGenOpts().EnableDebugEntryValues)
+  if (!SupportsDWARFv4Ext && CGM.getCodeGenOpts().DwarfVersion < 5)
     return llvm::DINode::FlagZero;
 
   return llvm::DINode::FlagAllCallsDescribed;
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 4eaa306199a526dea4ed2010b4a9892d0884ed1b..822c903660744160523a7f9a6bb125d45b30b9ba 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -2540,5 +2540,5 @@ void CodeGenModule::EmitOMPDeclareMapper(const OMPDeclareMapperDecl *D,
 }
 
 void CodeGenModule::EmitOMPRequiresDecl(const OMPRequiresDecl *D) {
-  getOpenMPRuntime().checkArchForUnifiedAddressing(D);
+  getOpenMPRuntime().processRequiresDirective(D);
 }
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index fffd9897270e175810b3d3834f9cfc25933f16b4..a542c3d85a8479917a1a31eea875cad8c9f2f8e8 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -703,12 +703,12 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() {
   assert(EHStack.requiresLandingPad());
   assert(!EHStack.empty());
 
-  // If exceptions are disabled and SEH is not in use, then there is no invoke
-  // destination. SEH "works" even if exceptions are off. In practice, this
-  // means that C++ destructors and other EH cleanups don't run, which is
+  // If exceptions are disabled/ignored and SEH is not in use, then there is no
+  // invoke destination. SEH "works" even if exceptions are off. In practice,
+  // this means that C++ destructors and other EH cleanups don't run, which is
   // consistent with MSVC's behavior.
   const LangOptions &LO = CGM.getLangOpts();
-  if (!LO.Exceptions) {
+  if (!LO.Exceptions || LO.IgnoreExceptions) {
     if (!LO.Borland && !LO.MicrosoftExt)
       return nullptr;
     if (!currentFunctionUsesSEHTry())
@@ -751,7 +751,9 @@ llvm::BasicBlock *CodeGenFunction::getInvokeDestImpl() {
 
 llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
   assert(EHStack.requiresLandingPad());
-
+  assert(!CGM.getLangOpts().IgnoreExceptions &&
+         "LandingPad should not be emitted when -fignore-exceptions are in "
+         "effect.");
   EHScope &innermostEHScope = *EHStack.find(EHStack.getInnermostEHScope());
   switch (innermostEHScope.getKind()) {
   case EHScope::Terminate:
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 23d49b23a3b4a2bd164aff9eaa509724be173063..c3e2e1e0a5d9f158904a285f1479e5a07c3161f0 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/BitmaskEnum.h"
+#include "clang/Basic/OpenMPKinds.h"
 #include "clang/CodeGen/ConstantInitBuilder.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SetOperations.h"
@@ -30,6 +31,7 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
@@ -9784,16 +9786,33 @@ void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
          " Expected target-based directive.");
 }
 
-void CGOpenMPRuntime::checkArchForUnifiedAddressing(
-    const OMPRequiresDecl *D) {
+void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {
   for (const OMPClause *Clause : D->clauselists()) {
     if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
       HasRequiresUnifiedSharedMemory = true;
-      break;
+    } else if (const auto *AC =
+                   dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {
+      switch (AC->getAtomicDefaultMemOrderKind()) {
+      case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
+        RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
+        break;
+      case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
+        RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
+        break;
+      case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
+        RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
+        break;
+      case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:
+        break;
+      }
     }
   }
 }
 
+llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
+  return RequiresAtomicOrdering;
+}
+
 bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
                                                        LangAS &AS) {
   if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index b1a59b49fe4a912748155c1b1f3615d99e023e89..f559e0d225749f26b77b89c5333271ac41732625 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -26,6 +26,7 @@
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/AtomicOrdering.h"
 
 namespace llvm {
 class ArrayType;
@@ -80,11 +81,10 @@ public:
   template <typename Callable>
   RegionCodeGenTy(
       Callable &&CodeGen,
-      typename std::enable_if<
-          !std::is_same<typename std::remove_reference<Callable>::type,
-                        RegionCodeGenTy>::value>::type * = nullptr)
+      std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
+                                     RegionCodeGenTy>::value> * = nullptr)
       : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
-        Callback(CallbackFn<typename std::remove_reference<Callable>::type>),
+        Callback(CallbackFn<std::remove_reference_t<Callable>>),
         PrePostAction(nullptr) {}
   void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
   void operator()(CodeGenFunction &CGF) const;
@@ -705,6 +705,9 @@ private:
   /// directive is present.
   bool HasRequiresUnifiedSharedMemory = false;
 
+  /// Atomic ordering from the omp requires directive.
+  llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
+
   /// Flag for keeping track of weather a target region has been emitted.
   bool HasEmittedTargetRegion = false;
 
@@ -1701,7 +1704,10 @@ public:
 
   /// Perform check on requires decl to ensure that target architecture
   /// supports unified addressing
-  virtual void checkArchForUnifiedAddressing(const OMPRequiresDecl *D);
+  virtual void processRequiresDirective(const OMPRequiresDecl *D);
+
+  /// Gets default memory ordering as specified in requires directive.
+  llvm::AtomicOrdering getDefaultMemoryOrdering() const;
 
   /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
   /// the predefined allocator and translates it into the corresponding address
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
index d00d84b79cfec8bf1ad110242d7fd9f12851b3aa..867bfb0727367b8f4d7c1278c3c6ddb219d1bd13 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
@@ -4962,7 +4962,7 @@ static CudaArch getCudaArch(CodeGenModule &CGM) {
 
 /// Check to see if target architecture supports unified addressing which is
 /// a restriction for OpenMP requires clause "unified_shared_memory".
-void CGOpenMPRuntimeNVPTX::checkArchForUnifiedAddressing(
+void CGOpenMPRuntimeNVPTX::processRequiresDirective(
     const OMPRequiresDecl *D) {
   for (const OMPClause *Clause : D->clauselists()) {
     if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
@@ -5017,7 +5017,7 @@ void CGOpenMPRuntimeNVPTX::checkArchForUnifiedAddressing(
       }
     }
   }
-  CGOpenMPRuntime::checkArchForUnifiedAddressing(D);
+  CGOpenMPRuntime::processRequiresDirective(D);
 }
 
 /// Get number of SMs and number of blocks per SM.
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
index 4159af0a622f768440f44d1a0d1d9ea87d63b7b8..834adb3782a09bf9d77377354ae95fe115caca92 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.h
@@ -395,7 +395,7 @@ public:
 
   /// Perform check on requires decl to ensure that target architecture
   /// supports unified addressing
-  void checkArchForUnifiedAddressing(const OMPRequiresDecl *D) override;
+  void processRequiresDirective(const OMPRequiresDecl *D) override;
 
   /// Returns default address space for the constant firstprivates, __constant__
   /// address space by default.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index bc71c27fd95f7416ec5f5e72138399802c36978e..7181374a73fce92a628b84b8888528001e9e4d05 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -25,6 +25,7 @@
 #include "clang/Basic/PrettyStackTrace.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/Support/AtomicOrdering.h"
 using namespace clang;
 using namespace CodeGen;
 using namespace llvm::omp;
@@ -4494,6 +4495,7 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_shared:
   case OMPC_linear:
   case OMPC_aligned:
@@ -4541,26 +4543,53 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
 
 void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
   llvm::AtomicOrdering AO = llvm::AtomicOrdering::Monotonic;
-  if (S.getSingleClause<OMPSeqCstClause>())
+  bool MemOrderingSpecified = false;
+  if (S.getSingleClause<OMPSeqCstClause>()) {
     AO = llvm::AtomicOrdering::SequentiallyConsistent;
-  else if (S.getSingleClause<OMPAcqRelClause>())
+    MemOrderingSpecified = true;
+  } else if (S.getSingleClause<OMPAcqRelClause>()) {
     AO = llvm::AtomicOrdering::AcquireRelease;
-  else if (S.getSingleClause<OMPAcquireClause>())
+    MemOrderingSpecified = true;
+  } else if (S.getSingleClause<OMPAcquireClause>()) {
     AO = llvm::AtomicOrdering::Acquire;
-  else if (S.getSingleClause<OMPReleaseClause>())
+    MemOrderingSpecified = true;
+  } else if (S.getSingleClause<OMPReleaseClause>()) {
     AO = llvm::AtomicOrdering::Release;
+    MemOrderingSpecified = true;
+  } else if (S.getSingleClause<OMPRelaxedClause>()) {
+    AO = llvm::AtomicOrdering::Monotonic;
+    MemOrderingSpecified = true;
+  }
   OpenMPClauseKind Kind = OMPC_unknown;
   for (const OMPClause *C : S.clauses()) {
-    // Find first clause (skip seq_cst|acq_rel|aqcuire|release clause, if it is
-    // first).
+    // Find first clause (skip seq_cst|acq_rel|aqcuire|release|relaxed clause,
+    // if it is first).
     if (C->getClauseKind() != OMPC_seq_cst &&
         C->getClauseKind() != OMPC_acq_rel &&
         C->getClauseKind() != OMPC_acquire &&
-        C->getClauseKind() != OMPC_release) {
+        C->getClauseKind() != OMPC_release &&
+        C->getClauseKind() != OMPC_relaxed) {
       Kind = C->getClauseKind();
       break;
     }
   }
+  if (!MemOrderingSpecified) {
+    llvm::AtomicOrdering DefaultOrder =
+        CGM.getOpenMPRuntime().getDefaultMemoryOrdering();
+    if (DefaultOrder == llvm::AtomicOrdering::Monotonic ||
+        DefaultOrder == llvm::AtomicOrdering::SequentiallyConsistent ||
+        (DefaultOrder == llvm::AtomicOrdering::AcquireRelease &&
+         Kind == OMPC_capture)) {
+      AO = DefaultOrder;
+    } else if (DefaultOrder == llvm::AtomicOrdering::AcquireRelease) {
+      if (Kind == OMPC_unknown || Kind == OMPC_update || Kind == OMPC_write) {
+        AO = llvm::AtomicOrdering::Release;
+      } else if (Kind == OMPC_read) {
+        assert(Kind == OMPC_read && "Unexpected atomic kind.");
+        AO = llvm::AtomicOrdering::Acquire;
+      }
+    }
+  }
 
   const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
   if (const auto *FE = dyn_cast<FullExpr>(CS))
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 113f6f5683b62d86d7e51d17cd286670e3cd8283..8ad27e19c5e2c5bc1f76e82b3631dd9e7f42defa 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -820,23 +820,25 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
           if (ShouldXRayInstrumentFunction())
             Fn->addFnAttr("xray-log-args",
                           llvm::utostr(LogArgs->getArgumentCount()));
-        if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
-                XRayInstrKind::FunctionExit)) {
-          Fn->addFnAttr("xray-skip-exit");
-        }
-        if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
-                XRayInstrKind::FunctionEntry)) {
-          Fn->addFnAttr("xray-skip-entry");
-        }
       }
     } else {
       if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
         Fn->addFnAttr(
             "xray-instruction-threshold",
             llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
-      if (CGM.getCodeGenOpts().XRayIgnoreLoops) {
+    }
+
+    if (ShouldXRayInstrumentFunction()) {
+      if (CGM.getCodeGenOpts().XRayIgnoreLoops)
         Fn->addFnAttr("xray-ignore-loops");
-      }
+
+      if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
+              XRayInstrKind::FunctionExit))
+        Fn->addFnAttr("xray-skip-exit");
+
+      if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
+              XRayInstrKind::FunctionEntry))
+        Fn->addFnAttr("xray-skip-entry");
     }
 
     unsigned Count, Offset;
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 1e5609d799a986da6fcbc7ee60347dd1158ee9e2..232a4cb219f511c8f835aa07a54e59ba567f7fd7 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -1702,6 +1702,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
   bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
 
   Ty = useFirstFieldIfTransparentUnion(Ty);
+  TypeInfo TI = getContext().getTypeInfo(Ty);
 
   // Check with the C++ ABI first.
   const RecordType *RT = Ty->getAs<RecordType>();
@@ -1751,7 +1752,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
     bool NeedsPadding = false;
     bool InReg;
     if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
-      unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32;
+      unsigned SizeInRegs = (TI.Width + 31) / 32;
       SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
       llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
       if (InReg)
@@ -1761,14 +1762,19 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
     }
     llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;
 
+    // Pass over-aligned aggregates on Windows indirectly. This behavior was
+    // added in MSVC 2015.
+    if (IsWin32StructABI && TI.AlignIsRequired && TI.Align > 32)
+      return getIndirectResult(Ty, /*ByVal=*/false, State);
+
     // Expand small (<= 128-bit) record types when we know that the stack layout
     // of those arguments will match the struct. This is important because the
     // LLVM backend isn't smart enough to remove byval, which inhibits many
     // optimizations.
     // Don't do this for the MCU if there are still free integer registers
     // (see X86_64 ABI for full explanation).
-    if (getContext().getTypeSize(Ty) <= 4 * 32 &&
-        (!IsMCUABI || State.FreeRegs == 0) && canExpandIndirectArgument(Ty))
+    if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&
+        canExpandIndirectArgument(Ty))
       return ABIArgInfo::getExpandWithPadding(
           IsFastCall || IsVectorCall || IsRegCall, PaddingType);
 
@@ -1776,14 +1782,24 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
   }
 
   if (const VectorType *VT = Ty->getAs<VectorType>()) {
+    // On Windows, vectors are passed directly if registers are available, or
+    // indirectly if not. This avoids the need to align argument memory. Pass
+    // user-defined vector types larger than 512 bits indirectly for simplicity.
+    if (IsWin32StructABI) {
+      if (TI.Width <= 512 && State.FreeSSERegs > 0) {
+        --State.FreeSSERegs;
+        return ABIArgInfo::getDirectInReg();
+      }
+      return getIndirectResult(Ty, /*ByVal=*/false, State);
+    }
+
     // On Darwin, some vectors are passed in memory, we handle this by passing
     // it as an i8/i16/i32/i64.
     if (IsDarwinVectorABI) {
-      uint64_t Size = getContext().getTypeSize(Ty);
-      if ((Size == 8 || Size == 16 || Size == 32) ||
-          (Size == 64 && VT->getNumElements() == 1))
-        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
-                                                            Size));
+      if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
+          (TI.Width == 64 && VT->getNumElements() == 1))
+        return ABIArgInfo::getDirect(
+            llvm::IntegerType::get(getVMContext(), TI.Width));
     }
 
     if (IsX86_MMXType(CGT.ConvertType(Ty)))
@@ -1813,9 +1829,10 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
   CCState State(FI);
   if (IsMCUABI)
     State.FreeRegs = 3;
-  else if (State.CC == llvm::CallingConv::X86_FastCall)
+  else if (State.CC == llvm::CallingConv::X86_FastCall) {
     State.FreeRegs = 2;
-  else if (State.CC == llvm::CallingConv::X86_VectorCall) {
+    State.FreeSSERegs = 3;
+  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
     State.FreeRegs = 2;
     State.FreeSSERegs = 6;
   } else if (FI.getHasRegParm())
@@ -1823,6 +1840,11 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
   else if (State.CC == llvm::CallingConv::X86_RegCall) {
     State.FreeRegs = 5;
     State.FreeSSERegs = 8;
+  } else if (IsWin32StructABI) {
+    // Since MSVC 2015, the first three SSE vectors have been passed in
+    // registers. The rest are passed indirectly.
+    State.FreeRegs = DefaultNumRegisterParameters;
+    State.FreeSSERegs = 3;
   } else
     State.FreeRegs = DefaultNumRegisterParameters;
 
@@ -1869,16 +1891,25 @@ X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
                                    CharUnits &StackOffset, ABIArgInfo &Info,
                                    QualType Type) const {
   // Arguments are always 4-byte-aligned.
-  CharUnits FieldAlign = CharUnits::fromQuantity(4);
+  CharUnits WordSize = CharUnits::fromQuantity(4);
+  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
 
-  assert(StackOffset.isMultipleOf(FieldAlign) && "unaligned inalloca struct");
-  Info = ABIArgInfo::getInAlloca(FrameFields.size());
-  FrameFields.push_back(CGT.ConvertTypeForMem(Type));
-  StackOffset += getContext().getTypeSizeInChars(Type);
+  // sret pointers and indirect things will require an extra pointer
+  // indirection, unless they are byval. Most things are byval, and will not
+  // require this indirection.
+  bool IsIndirect = false;
+  if (Info.isIndirect() && !Info.getIndirectByVal())
+    IsIndirect = true;
+  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
+  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
+  if (IsIndirect)
+    LLTy = LLTy->getPointerTo(0);
+  FrameFields.push_back(LLTy);
+  StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type);
 
   // Insert padding bytes to respect alignment.
   CharUnits FieldEnd = StackOffset;
-  StackOffset = FieldEnd.alignTo(FieldAlign);
+  StackOffset = FieldEnd.alignTo(WordSize);
   if (StackOffset != FieldEnd) {
     CharUnits NumBytes = StackOffset - FieldEnd;
     llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
@@ -1892,16 +1923,12 @@ static bool isArgInAlloca(const ABIArgInfo &Info) {
   switch (Info.getKind()) {
   case ABIArgInfo::InAlloca:
     return true;
-  case ABIArgInfo::Indirect:
-    assert(Info.getIndirectByVal());
-    return true;
   case ABIArgInfo::Ignore:
     return false;
+  case ABIArgInfo::Indirect:
   case ABIArgInfo::Direct:
   case ABIArgInfo::Extend:
-    if (Info.getInReg())
-      return false;
-    return true;
+    return !Info.getInReg();
   case ABIArgInfo::Expand:
   case ABIArgInfo::CoerceAndExpand:
     // These are aggregate types which are never passed in registers when
@@ -1935,8 +1962,7 @@ void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
 
   // Put the sret parameter into the inalloca struct if it's in memory.
   if (Ret.isIndirect() && !Ret.getInReg()) {
-    CanQualType PtrTy = getContext().getPointerType(FI.getReturnType());
-    addFieldToArgStruct(FrameFields, StackOffset, Ret, PtrTy);
+    addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
     // On Windows, the hidden sret parameter is always returned in eax.
     Ret.setInAllocaSRet(IsWin32StructABI);
   }
@@ -7486,13 +7512,10 @@ void TCETargetCodeGenInfo::setTargetAttributes(
 namespace {
 
 class HexagonABIInfo : public ABIInfo {
-
-
 public:
   HexagonABIInfo(CodeGenTypes &CGT) : ABIInfo(CGT) {}
 
 private:
-
   ABIArgInfo classifyReturnType(QualType RetTy) const;
   ABIArgInfo classifyArgumentType(QualType RetTy) const;
 
@@ -7505,14 +7528,14 @@ private:
 class HexagonTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   HexagonTargetCodeGenInfo(CodeGenTypes &CGT)
-    :TargetCodeGenInfo(new HexagonABIInfo(CGT)) {}
+    : TargetCodeGenInfo(new HexagonABIInfo(CGT)) {}
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
     return 29;
   }
 };
 
-}
+} // namespace
 
 void HexagonABIInfo::computeInfo(CGFunctionInfo &FI) const {
   if (!getCXXABI().classifyReturnType(FI))
@@ -7527,8 +7550,8 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const {
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
       Ty = EnumTy->getDecl()->getIntegerType();
 
-    return (Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty)
-                                          : ABIArgInfo::getDirect());
+    return Ty->isPromotableIntegerType() ? ABIArgInfo::getExtend(Ty)
+                                         : ABIArgInfo::getDirect();
   }
 
   if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
@@ -7539,53 +7562,56 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty) const {
     return ABIArgInfo::getIgnore();
 
   uint64_t Size = getContext().getTypeSize(Ty);
-  if (Size > 64)
-    return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
+  if (Size <= 64) {
     // Pass in the smallest viable integer type.
-  else if (Size > 32)
-      return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext()));
-  else if (Size > 16)
-      return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
-  else if (Size > 8)
-      return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
-  else
-      return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
+    if (!llvm::isPowerOf2_64(Size))
+      Size = llvm::NextPowerOf2(Size);
+    return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
+  }
+  return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
 }
 
 ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
   if (RetTy->isVoidType())
     return ABIArgInfo::getIgnore();
 
-  // Large vector types should be returned via memory.
-  if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 64)
-    return getNaturalAlignIndirect(RetTy);
+  const TargetInfo &T = CGT.getTarget();
+  uint64_t Size = getContext().getTypeSize(RetTy);
+
+  if (const auto *VecTy = RetTy->getAs<VectorType>()) {
+    // HVX vectors are returned in vector registers or register pairs.
+    if (T.hasFeature("hvx")) {
+      assert(T.hasFeature("hvx-length64b") || T.hasFeature("hvx-length128b"));
+      uint64_t VecSize = T.hasFeature("hvx-length64b") ? 64*8 : 128*8;
+      if (Size == VecSize || Size == 2*VecSize)
+        return ABIArgInfo::getDirectInReg();
+    }
+
+    // Large vector types should be returned via memory.
+    if (Size > 64)
+      return getNaturalAlignIndirect(RetTy);
+  }
 
   if (!isAggregateTypeForABI(RetTy)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
       RetTy = EnumTy->getDecl()->getIntegerType();
 
-    return (RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy)
-                                             : ABIArgInfo::getDirect());
+    return RetTy->isPromotableIntegerType() ? ABIArgInfo::getExtend(RetTy)
+                                            : ABIArgInfo::getDirect();
   }
 
   if (isEmptyRecord(getContext(), RetTy, true))
     return ABIArgInfo::getIgnore();
 
-  // Aggregates <= 8 bytes are returned in r0; other aggregates
+  // Aggregates <= 8 bytes are returned in registers, other aggregates
   // are returned indirectly.
-  uint64_t Size = getContext().getTypeSize(RetTy);
   if (Size <= 64) {
     // Return in the smallest viable integer type.
-    if (Size <= 8)
-      return ABIArgInfo::getDirect(llvm::Type::getInt8Ty(getVMContext()));
-    if (Size <= 16)
-      return ABIArgInfo::getDirect(llvm::Type::getInt16Ty(getVMContext()));
-    if (Size <= 32)
-      return ABIArgInfo::getDirect(llvm::Type::getInt32Ty(getVMContext()));
-    return ABIArgInfo::getDirect(llvm::Type::getInt64Ty(getVMContext()));
+    if (!llvm::isPowerOf2_64(Size))
+      Size = llvm::NextPowerOf2(Size);
+    return ABIArgInfo::getDirect(llvm::Type::getIntNTy(getVMContext(), Size));
   }
-
   return getNaturalAlignIndirect(RetTy, /*ByVal=*/true);
 }
 
diff --git a/clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp b/clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp
index 2cae847e7657b1b4233e9024205232c52fcf454e..7864fb76d160d88a3ffe9142c2fa442e875d5320 100644
--- a/clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp
+++ b/clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp
@@ -43,24 +43,32 @@ namespace {
 class DirectoryWatcherMac : public clang::DirectoryWatcher {
 public:
   DirectoryWatcherMac(
-      FSEventStreamRef EventStream,
+      dispatch_queue_t Queue, FSEventStreamRef EventStream,
       std::function<void(llvm::ArrayRef<DirectoryWatcher::Event>, bool)>
           Receiver,
       llvm::StringRef WatchedDirPath)
-      : EventStream(EventStream), Receiver(Receiver),
+      : Queue(Queue), EventStream(EventStream), Receiver(Receiver),
         WatchedDirPath(WatchedDirPath) {}
 
   ~DirectoryWatcherMac() override {
-    stopFSEventStream(EventStream);
-    EventStream = nullptr;
-    // Now it's safe to use Receiver as the only other concurrent use would have
-    // been in EventStream processing.
-    Receiver(DirectoryWatcher::Event(
-                 DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""),
-             false);
+    // FSEventStreamStop and Invalidate must be called after Start and
+    // SetDispatchQueue to follow FSEvents API contract. The call to Receiver
+    // also uses Queue to not race with the initial scan.
+    dispatch_sync(Queue, ^{
+      stopFSEventStream(EventStream);
+      EventStream = nullptr;
+      Receiver(
+          DirectoryWatcher::Event(
+              DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""),
+          false);
+    });
+
+    // Balance initial creation.
+    dispatch_release(Queue);
   }
 
 private:
+  dispatch_queue_t Queue;
   FSEventStreamRef EventStream;
   std::function<void(llvm::ArrayRef<Event>, bool)> Receiver;
   const std::string WatchedDirPath;
@@ -217,7 +225,7 @@ llvm::Expected<std::unique_ptr<DirectoryWatcher>> clang::DirectoryWatcher::creat
   assert(EventStream && "EventStream expected to be non-null");
 
   std::unique_ptr<DirectoryWatcher> Result =
-      std::make_unique<DirectoryWatcherMac>(EventStream, Receiver, Path);
+      std::make_unique<DirectoryWatcherMac>(Queue, EventStream, Receiver, Path);
 
   // We need to copy the data so the lifetime is ok after a const copy is made
   // for the block.
@@ -230,10 +238,6 @@ llvm::Expected<std::unique_ptr<DirectoryWatcher>> clang::DirectoryWatcher::creat
     // inital scan and handling events ONLY AFTER the scan finishes.
     FSEventStreamSetDispatchQueue(EventStream, Queue);
     FSEventStreamStart(EventStream);
-    // We need to decrement the ref count for Queue as initialize() will return
-    // and FSEvents has incremented it. Since we have to wait for FSEvents to
-    // take ownership it's the easiest to do it here rather than main thread.
-    dispatch_release(Queue);
     Receiver(getAsFileEvents(scanDirectory(CopiedPath)), /*IsInitial=*/true);
   };
 
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index c231d053fe27ed316f4abe97af951737cbd2f2bf..854db1f4f84781e09d3cd02a0ed6eac1a67dbb55 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -436,12 +436,11 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
     Features.push_back("-relax");
 
   // GCC Compatibility: -mno-save-restore is default, unless -msave-restore is
-  // specified...
-  if (Args.hasFlag(options::OPT_msave_restore, options::OPT_mno_save_restore, false)) {
-    // ... but we don't support -msave-restore, so issue a warning.
-    D.Diag(diag::warn_drv_clang_unsupported)
-      << Args.getLastArg(options::OPT_msave_restore)->getAsString(Args);
-  }
+  // specified.
+  if (Args.hasFlag(options::OPT_msave_restore, options::OPT_mno_save_restore, false))
+    Features.push_back("+save-restore");
+  else
+    Features.push_back("-save-restore");
 
   // EPI requires rv64g.
   // FIXME - We need a parser of the RISCV march.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 57659f7b228d708e9d2e7d55c42621df428417df..7362a9e3cc256786cd5f7e7b8382dc2dc9892251 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -463,6 +463,11 @@ static void addExceptionArgs(const ArgList &Args, types::ID InputType,
     }
   }
 
+  // OPT_fignore_exceptions means exception could still be thrown,
+  // but no clean up or catch would happen in current module.
+  // So we do not set EH to false.
+  Args.AddLastArg(CmdArgs, options::OPT_fignore_exceptions);
+
   if (EH)
     CmdArgs.push_back("-fexceptions");
 }
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index 66c3959648aca90d87e9497286009443592a24e8..54c15685d3898f8cafe260df00c8856deb59fbe9 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -101,6 +101,10 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
                     options::OPT_fnoxray_link_deps, true))
     XRayRT = false;
 
+  if (Args.hasFlag(options::OPT_fxray_ignore_loops,
+                   options::OPT_fno_xray_ignore_loops, false))
+    XRayIgnoreLoops = true;
+
   auto Bundles =
       Args.getAllArgValues(options::OPT_fxray_instrumentation_bundle);
   if (Bundles.empty())
@@ -197,6 +201,9 @@ void XRayArgs::addArgs(const ToolChain &TC, const ArgList &Args,
   if (XRayAlwaysEmitTypedEvents)
     CmdArgs.push_back("-fxray-always-emit-typedevents");
 
+  if (XRayIgnoreLoops)
+    CmdArgs.push_back("-fxray-ignore-loops");
+
   CmdArgs.push_back(Args.MakeArgString(Twine(XRayInstructionThresholdOption) +
                                        Twine(InstructionThreshold)));
 
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index d7d5efaf345db129c8dd038f0b628fa809d81697..ec40bd6af410dff828732d718b3096748499bfd8 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -792,16 +792,6 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
 
   Opts.DisableLLVMPasses = Args.hasArg(OPT_disable_llvm_passes);
   Opts.DisableLifetimeMarkers = Args.hasArg(OPT_disable_lifetimemarkers);
-
-  const llvm::Triple::ArchType DebugEntryValueArchs[] = {
-      llvm::Triple::x86, llvm::Triple::x86_64, llvm::Triple::aarch64,
-      llvm::Triple::arm, llvm::Triple::armeb};
-
-  llvm::Triple T(TargetOpts.Triple);
-  if (Opts.OptimizationLevel > 0 && Opts.hasReducedDebugInfo() &&
-      llvm::is_contained(DebugEntryValueArchs, T.getArch()))
-    Opts.EnableDebugEntryValues = Args.hasArg(OPT_femit_debug_entry_values);
-
   Opts.DisableO0ImplyOptNone = Args.hasArg(OPT_disable_O0_optnone);
   Opts.DisableRedZone = Args.hasArg(OPT_disable_red_zone);
   Opts.IndirectTlsSegRefs = Args.hasArg(OPT_mno_tls_direct_seg_refs);
@@ -1099,8 +1089,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
       Args.hasArg(OPT_fxray_always_emit_typedevents);
   Opts.XRayInstructionThreshold =
       getLastArgIntValue(Args, OPT_fxray_instruction_threshold_EQ, 200, Diags);
-  Opts.XRayIgnoreLoops =
-      Args.hasArg(OPT_fxray_ignore_loops, OPT_fno_xray_ignore_loops, false);
+  Opts.XRayIgnoreLoops = Args.hasArg(OPT_fxray_ignore_loops);
 
   auto XRayInstrBundles =
       Args.getAllArgValues(OPT_fxray_instrumentation_bundle);
@@ -2784,6 +2773,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   if (Args.hasArg(OPT_fno_threadsafe_statics))
     Opts.ThreadsafeStatics = 0;
   Opts.Exceptions = Args.hasArg(OPT_fexceptions);
+  Opts.IgnoreExceptions = Args.hasArg(OPT_fignore_exceptions);
   Opts.ObjCExceptions = Args.hasArg(OPT_fobjc_exceptions);
   Opts.CXXExceptions = Args.hasArg(OPT_fcxx_exceptions);
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 1a7916e5adb865ff7f60b17b787f3db9c3fcf6c7..fbabe92977c9d308c825d34d9727ff528d8af481 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -1473,7 +1473,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       Actions.EndOpenMPClause();
     }
     // Consume final annot_pragma_openmp_end
-    if (Clauses.size() == 0) {
+    if (Clauses.empty()) {
       Diag(Tok, diag::err_omp_expected_clause)
           << getOpenMPDirectiveName(OMPD_requires);
       ConsumeAnnotationToken();
@@ -2090,7 +2090,7 @@ bool Parser::ParseOpenMPSimpleVarList(
 ///       nogroup-clause | num_tasks-clause | hint-clause | to-clause |
 ///       from-clause | is_device_ptr-clause | task_reduction-clause |
 ///       in_reduction-clause | allocator-clause | allocate-clause |
-///       acq_rel-clause | acquire-clause | release-clause
+///       acq_rel-clause | acquire-clause | release-clause | relaxed-clause
 ///
 OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
                                      OpenMPClauseKind CKind, bool FirstClause) {
@@ -2202,6 +2202,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_threads:
   case OMPC_simd:
   case OMPC_nogroup:
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 273b4e3c6f09c4f89f0acf4294ae918d6cb99386..cf3fa3855cc8731c40859409d34a93e193338e8b 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -225,8 +225,7 @@ static bool checkAttributeAtMostNumArgs(Sema &S, const ParsedAttr &AL,
 /// A helper function to provide Attribute Location for the Attr types
 /// AND the ParsedAttr.
 template <typename AttrInfo>
-static typename std::enable_if<std::is_base_of<Attr, AttrInfo>::value,
-                               SourceLocation>::type
+static std::enable_if_t<std::is_base_of<Attr, AttrInfo>::value, SourceLocation>
 getAttrLoc(const AttrInfo &AL) {
   return AL.getLocation();
 }
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 61b7c166239f20ce4219eac51316759aba35fd43..61bb1f053ce2e2229d2fe3254fe1f8ffcc66ca89 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -4420,16 +4420,20 @@ static void TryListInitialization(Sema &S,
     // direct-list-initialization and copy-initialization otherwise.
     // We can't use InitListChecker for this, because it always performs
     // copy-initialization. This only matters if we might use an 'explicit'
-    // conversion operator, so we only need to handle the cases where the source
-    // is of record type.
-    if (InitList->getInit(0)->getType()->isRecordType()) {
+    // conversion operator, or for the special case conversion of nullptr_t to
+    // bool, so we only need to handle those cases.
+    //
+    // FIXME: Why not do this in all cases?
+    Expr *Init = InitList->getInit(0);
+    if (Init->getType()->isRecordType() ||
+        (Init->getType()->isNullPtrType() && DestType->isBooleanType())) {
       InitializationKind SubKind =
           Kind.getKind() == InitializationKind::IK_DirectList
               ? InitializationKind::CreateDirect(Kind.getLocation(),
                                                  InitList->getLBraceLoc(),
                                                  InitList->getRBraceLoc())
               : Kind;
-      Expr *SubInit[1] = { InitList->getInit(0) };
+      Expr *SubInit[1] = { Init };
       Sequence.InitializeFrom(S, Entity, SubKind, SubInit,
                               /*TopLevelOfInitList*/true,
                               TreatUnavailableAsInvalid);
@@ -5854,6 +5858,19 @@ void InitializationSequence::InitializeFrom(Sema &S,
     return;
   }
 
+  //    - Otherwise, if the initialization is direct-initialization, the source
+  //    type is std::nullptr_t, and the destination type is bool, the initial
+  //    value of the object being initialized is false.
+  if (!SourceType.isNull() && SourceType->isNullPtrType() &&
+      DestType->isBooleanType() &&
+      Kind.getKind() == InitializationKind::IK_Direct) {
+    AddConversionSequenceStep(
+        ImplicitConversionSequence::getNullptrToBool(SourceType, DestType,
+                                                     Initializer->isGLValue()),
+        DestType);
+    return;
+  }
+
   //    - Otherwise, the initial value of the object being initialized is the
   //      (possibly converted) value of the initializer expression. Standard
   //      conversions (Clause 4) will be used, if necessary, to convert the
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 2e3c75fed37017cda6ec52c43e7f712cf6d56031..bd1c4777bc0d587abe791bdc3b6937a15a373498 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -271,6 +271,7 @@ private:
       nullptr};
   /// Vector of previously encountered target directives
   SmallVector<SourceLocation, 2> TargetLocations;
+  SourceLocation AtomicLocation;
 
 public:
   explicit DSAStackTy(Sema &S) : SemaRef(S) {}
@@ -555,7 +556,7 @@ public:
 
   /// Checks if the defined 'requires' directive has specified type of clause.
   template <typename ClauseType>
-  bool hasRequiresDeclWithClause() {
+  bool hasRequiresDeclWithClause() const {
     return llvm::any_of(RequiresDecls, [](const OMPRequiresDecl *D) {
       return llvm::any_of(D->clauselists(), [](const OMPClause *C) {
         return isa<ClauseType>(C);
@@ -590,6 +591,18 @@ public:
     TargetLocations.push_back(LocStart);
   }
 
+  /// Add location for the first encountered atomicc directive.
+  void addAtomicDirectiveLoc(SourceLocation Loc) {
+    if (AtomicLocation.isInvalid())
+      AtomicLocation = Loc;
+  }
+
+  /// Returns the location of the first encountered atomic directive in the
+  /// module.
+  SourceLocation getAtomicDirectiveLoc() const {
+    return AtomicLocation;
+  }
+
   // Return previously encountered target region locations.
   ArrayRef<SourceLocation> getEncounteredTargetLocs() const {
     return TargetLocations;
@@ -2830,18 +2843,26 @@ OMPRequiresDecl *Sema::CheckOMPRequiresDecl(SourceLocation Loc,
   /// current compilation unit.
   ArrayRef<SourceLocation> TargetLocations =
       DSAStack->getEncounteredTargetLocs();
-  if (!TargetLocations.empty()) {
+  SourceLocation AtomicLoc = DSAStack->getAtomicDirectiveLoc();
+  if (!TargetLocations.empty() || !AtomicLoc.isInvalid()) {
     for (const OMPClause *CNew : ClauseList) {
       // Check if any of the requires clauses affect target regions.
       if (isa<OMPUnifiedSharedMemoryClause>(CNew) ||
           isa<OMPUnifiedAddressClause>(CNew) ||
           isa<OMPReverseOffloadClause>(CNew) ||
           isa<OMPDynamicAllocatorsClause>(CNew)) {
-        Diag(Loc, diag::err_omp_target_before_requires)
-            << getOpenMPClauseName(CNew->getClauseKind());
+        Diag(Loc, diag::err_omp_directive_before_requires)
+            << "target" << getOpenMPClauseName(CNew->getClauseKind());
         for (SourceLocation TargetLoc : TargetLocations) {
-          Diag(TargetLoc, diag::note_omp_requires_encountered_target);
+          Diag(TargetLoc, diag::note_omp_requires_encountered_directive)
+              << "target";
         }
+      } else if (!AtomicLoc.isInvalid() &&
+                 isa<OMPAtomicDefaultMemOrderClause>(CNew)) {
+        Diag(Loc, diag::err_omp_directive_before_requires)
+            << "atomic" << getOpenMPClauseName(CNew->getClauseKind());
+        Diag(AtomicLoc, diag::note_omp_requires_encountered_directive)
+            << "atomic";
       }
     }
   }
@@ -4990,6 +5011,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
       case OMPC_acq_rel:
       case OMPC_acquire:
       case OMPC_release:
+      case OMPC_relaxed:
       case OMPC_depend:
       case OMPC_threads:
       case OMPC_simd:
@@ -8925,6 +8947,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
                                             Stmt *AStmt,
                                             SourceLocation StartLoc,
                                             SourceLocation EndLoc) {
+  // Register location of the first atomic directive.
+  DSAStack->addAtomicDirectiveLoc(StartLoc);
   if (!AStmt)
     return StmtError();
 
@@ -8955,7 +8979,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
     if (C->getClauseKind() == OMPC_seq_cst ||
         C->getClauseKind() == OMPC_acq_rel ||
         C->getClauseKind() == OMPC_acquire ||
-        C->getClauseKind() == OMPC_release) {
+        C->getClauseKind() == OMPC_release ||
+        C->getClauseKind() == OMPC_relaxed) {
       if (MemOrderKind != OMPC_unknown) {
         Diag(C->getBeginLoc(), diag::err_omp_several_mem_order_clauses)
             << getOpenMPDirectiveName(OMPD_atomic) << 0
@@ -8968,6 +8993,28 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
       }
     }
   }
+  // OpenMP 5.0, 2.17.7 atomic Construct, Restrictions
+  // If atomic-clause is read then memory-order-clause must not be acq_rel or
+  // release.
+  // If atomic-clause is write then memory-order-clause must not be acq_rel or
+  // acquire.
+  // If atomic-clause is update or not present then memory-order-clause must not
+  // be acq_rel or acquire.
+  if ((AtomicKind == OMPC_read &&
+       (MemOrderKind == OMPC_acq_rel || MemOrderKind == OMPC_release)) ||
+      ((AtomicKind == OMPC_write || AtomicKind == OMPC_update ||
+        AtomicKind == OMPC_unknown) &&
+       (MemOrderKind == OMPC_acq_rel || MemOrderKind == OMPC_acquire))) {
+    SourceLocation Loc = AtomicKindLoc;
+    if (AtomicKind == OMPC_unknown)
+      Loc = StartLoc;
+    Diag(Loc, diag::err_omp_atomic_incompatible_mem_order_clause)
+        << getOpenMPClauseName(AtomicKind)
+        << (AtomicKind == OMPC_unknown ? 1 : 0)
+        << getOpenMPClauseName(MemOrderKind);
+    Diag(MemOrderLoc, diag::note_omp_previous_mem_order_clause)
+        << getOpenMPClauseName(MemOrderKind);
+  }
 
   Stmt *Body = CS->getCapturedStmt();
   if (auto *EWC = dyn_cast<ExprWithCleanups>(Body))
@@ -10896,6 +10943,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_depend:
   case OMPC_threads:
   case OMPC_simd:
@@ -11611,6 +11659,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_depend:
   case OMPC_threads:
   case OMPC_simd:
@@ -12036,6 +12085,7 @@ OMPClause *Sema::ActOnOpenMPSimpleClause(
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_depend:
   case OMPC_device:
   case OMPC_threads:
@@ -12239,6 +12289,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_depend:
   case OMPC_device:
   case OMPC_threads:
@@ -12421,6 +12472,9 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_release:
     Res = ActOnOpenMPReleaseClause(StartLoc, EndLoc);
     break;
+  case OMPC_relaxed:
+    Res = ActOnOpenMPRelaxedClause(StartLoc, EndLoc);
+    break;
   case OMPC_threads:
     Res = ActOnOpenMPThreadsClause(StartLoc, EndLoc);
     break;
@@ -12549,6 +12603,11 @@ OMPClause *Sema::ActOnOpenMPReleaseClause(SourceLocation StartLoc,
   return new (Context) OMPReleaseClause(StartLoc, EndLoc);
 }
 
+OMPClause *Sema::ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
+                                          SourceLocation EndLoc) {
+  return new (Context) OMPRelaxedClause(StartLoc, EndLoc);
+}
+
 OMPClause *Sema::ActOnOpenMPThreadsClause(SourceLocation StartLoc,
                                           SourceLocation EndLoc) {
   return new (Context) OMPThreadsClause(StartLoc, EndLoc);
@@ -12708,6 +12767,7 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
   case OMPC_acq_rel:
   case OMPC_acquire:
   case OMPC_release:
+  case OMPC_relaxed:
   case OMPC_device:
   case OMPC_threads:
   case OMPC_simd:
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 003d9bb3a97dc92380fa0f6d0d8fe1df550671d7..a10b7cd7648278f82962cec32f8156fc2ab888b2 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -230,7 +230,6 @@ bool StandardConversionSequence::isPointerConversionToBool() const {
        getFromType()->isMemberPointerType() ||
        getFromType()->isObjCObjectPointerType() ||
        getFromType()->isBlockPointerType() ||
-       getFromType()->isNullPtrType() ||
        First == ICK_Array_To_Pointer || First == ICK_Function_To_Pointer))
     return true;
 
@@ -328,9 +327,8 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
       goto FloatingIntegralConversion;
     if (FromType->isIntegralOrUnscopedEnumerationType())
       goto IntegralConversion;
-    // Boolean conversions can be from pointers and pointers to members
-    // [conv.bool], and those aren't considered narrowing conversions.
-    return NK_Not_Narrowing;
+    // -- from a pointer type or pointer-to-member type to bool, or
+    return NK_Type_Narrowing;
 
   // -- from a floating-point type to an integer type, or
   //
@@ -1848,8 +1846,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
              (FromType->isArithmeticType() ||
               FromType->isAnyPointerType() ||
               FromType->isBlockPointerType() ||
-              FromType->isMemberPointerType() ||
-              FromType->isNullPtrType())) {
+              FromType->isMemberPointerType())) {
     // Boolean conversions (C++ 4.12).
     SCS.Second = ICK_Boolean_Conversion;
     FromType = S.Context.BoolTy;
@@ -5438,6 +5435,17 @@ Sema::PerformObjectArgumentInitialization(Expr *From,
 /// expression From to bool (C++0x [conv]p3).
 static ImplicitConversionSequence
 TryContextuallyConvertToBool(Sema &S, Expr *From) {
+  // C++ [dcl.init]/17.8:
+  //   - Otherwise, if the initialization is direct-initialization, the source
+  //     type is std::nullptr_t, and the destination type is bool, the initial
+  //     value of the object being initialized is false.
+  if (From->getType()->isNullPtrType())
+    return ImplicitConversionSequence::getNullptrToBool(From->getType(),
+                                                        S.Context.BoolTy,
+                                                        From->isGLValue());
+
+  // All other direct-initialization of bool is equivalent to an implicit
+  // conversion to bool in which explicit conversions are permitted.
   return TryImplicitConversion(S, From, S.Context.BoolTy,
                                /*SuppressUserConversions=*/false,
                                AllowedExplicit::Conversions,
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 7451bf62cafeb4d55545db096c28ba8c9e7cfaa9..24019bf7975bca5d8ba635aa72e8b9facaf72982 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -2747,8 +2747,8 @@ CheckDeducedArgumentConstraints(Sema& S, TemplateDeclT *Template,
 
 /// Complete template argument deduction for a partial specialization.
 template <typename T>
-static typename std::enable_if<IsPartialSpecialization<T>::value,
-                               Sema::TemplateDeductionResult>::type
+static std::enable_if_t<IsPartialSpecialization<T>::value,
+                        Sema::TemplateDeductionResult>
 FinishTemplateArgumentDeduction(
     Sema &S, T *Partial, bool IsPartialOrdering,
     const TemplateArgumentList &TemplateArgs,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index dffaf893862798747e8054f1800c7bb8069151f6..3f3915b8c44bf461fcd59fbbc15615ad72edce88 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -8825,6 +8825,13 @@ TreeTransform<Derived>::TransformOMPReleaseClause(OMPReleaseClause *C) {
   return C;
 }
 
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPRelaxedClause(OMPRelaxedClause *C) {
+  // No need to rebuild this clause, no template-dependent parameters.
+  return C;
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPThreadsClause(OMPThreadsClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index c9553de8143e33cd704f51380a418e9742b85f86..a1161d26483877112138176f3c87aa31fda5fa7a 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11675,6 +11675,9 @@ OMPClause *OMPClauseReader::readClause() {
   case OMPC_release:
     C = new (Context) OMPReleaseClause();
     break;
+  case OMPC_relaxed:
+    C = new (Context) OMPRelaxedClause();
+    break;
   case OMPC_threads:
     C = new (Context) OMPThreadsClause();
     break;
@@ -11943,6 +11946,8 @@ void OMPClauseReader::VisitOMPAcquireClause(OMPAcquireClause *) {}
 
 void OMPClauseReader::VisitOMPReleaseClause(OMPReleaseClause *) {}
 
+void OMPClauseReader::VisitOMPRelaxedClause(OMPRelaxedClause *) {}
+
 void OMPClauseReader::VisitOMPThreadsClause(OMPThreadsClause *) {}
 
 void OMPClauseReader::VisitOMPSIMDClause(OMPSIMDClause *) {}
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 093b69ab19d0309697516e6715e7a39d918d5ac8..3d47274079faec6fd5d6c4c6b51e1fbac0478c6f 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2840,7 +2840,8 @@ static bool isConsumerInterestedIn(ASTContext &Ctx, Decl *D, bool HasBody) {
       isa<PragmaDetectMismatchDecl>(D))
     return true;
   if (isa<OMPThreadPrivateDecl>(D) || isa<OMPDeclareReductionDecl>(D) ||
-      isa<OMPDeclareMapperDecl>(D) || isa<OMPAllocateDecl>(D))
+      isa<OMPDeclareMapperDecl>(D) || isa<OMPAllocateDecl>(D) ||
+      isa<OMPRequiresDecl>(D))
     return !D->getDeclContext()->isFunctionOrMethod();
   if (const auto *Var = dyn_cast<VarDecl>(D))
     return Var->isFileVarDecl() &&
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index eb79ba20ddf7a5ab900662579101f7a3ea8d15c0..f935a69769bf9449a1a69ac8a8798e3a38cabe4b 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6153,6 +6153,8 @@ void OMPClauseWriter::VisitOMPAcquireClause(OMPAcquireClause *) {}
 
 void OMPClauseWriter::VisitOMPReleaseClause(OMPReleaseClause *) {}
 
+void OMPClauseWriter::VisitOMPRelaxedClause(OMPRelaxedClause *) {}
+
 void OMPClauseWriter::VisitOMPThreadsClause(OMPThreadsClause *) {}
 
 void OMPClauseWriter::VisitOMPSIMDClause(OMPSIMDClause *) {}
diff --git a/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
index d471c23b83bfcfabbe05fe12a0aee8cecb7aa8c6..76fa56406443a1f4c314f3a07ce1dd1ede8c84ba 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
@@ -52,18 +52,16 @@ public:
                         BugReporter &BR) const;
 };
 
-auto callsName(const char *FunctionName)
-    -> decltype(callee(functionDecl())) {
+decltype(auto) callsName(const char *FunctionName) {
   return callee(functionDecl(hasName(FunctionName)));
 }
 
-auto equalsBoundArgDecl(int ArgIdx, const char *DeclName)
-    -> decltype(hasArgument(0, expr())) {
+decltype(auto) equalsBoundArgDecl(int ArgIdx, const char *DeclName) {
   return hasArgument(ArgIdx, ignoringParenCasts(declRefExpr(
                                  to(varDecl(equalsBoundNode(DeclName))))));
 }
 
-auto bindAssignmentToDecl(const char *DeclName) -> decltype(hasLHS(expr())) {
+decltype(auto) bindAssignmentToDecl(const char *DeclName) {
   return hasLHS(ignoringParenImpCasts(
                          declRefExpr(to(varDecl().bind(DeclName)))));
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp b/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
index 5b9895c338d81e9b548911d29941eb8418f1b257..90d69b81305c870c4629ae3cb3f1e3b84b1f5314 100644
--- a/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
@@ -55,8 +55,7 @@ static void emitDiagnostics(const BoundNodes &Nodes,
     CE->getSourceRange());
 }
 
-static auto hasTypePointingTo(DeclarationMatcher DeclM)
-    -> decltype(hasType(pointerType())) {
+static decltype(auto) hasTypePointingTo(DeclarationMatcher DeclM) {
   return hasType(pointerType(pointee(hasDeclaration(DeclM))));
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
index d2371fe60d21d1c168032d9e413ff8ea1209aa0a..9d587c5856500ce242a847480aac91387e8ccf25 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp
@@ -100,8 +100,7 @@ static inline std::vector<llvm::StringRef> toRefs(std::vector<std::string> V) {
   return std::vector<llvm::StringRef>(V.begin(), V.end());
 }
 
-static auto callsNames(std::vector<std::string> FunctionNames)
-    -> decltype(callee(functionDecl())) {
+static decltype(auto) callsNames(std::vector<std::string> FunctionNames) {
   return callee(functionDecl(hasAnyName(toRefs(FunctionNames))));
 }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
index 586d9d3af2a642925ceacaf9d3f1ab44f636f41c..a3bfac97e40a77eefb283381772f2b6ccc1b40e6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerSortingChecker.cpp
@@ -54,7 +54,7 @@ static void emitDiagnostics(const BoundNodes &Match, const Decl *D,
                      OS.str(), Location, Range);
 }
 
-auto callsName(const char *FunctionName) -> decltype(callee(functionDecl())) {
+decltype(auto) callsName(const char *FunctionName) {
   return callee(functionDecl(hasName(FunctionName)));
 }
 
diff --git a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
index 2f713a12eee0db19f0ff645807efcf371e3a9749..4af204474494fb72ee023b184fbca347e4b6d122 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
@@ -51,11 +51,10 @@ using CheckerNameLT = FullNameLT<CheckerRegistry::CheckerInfo>;
 } // end of anonymous namespace
 
 template <class CheckerOrPackageInfoList>
-static
-    typename std::conditional<std::is_const<CheckerOrPackageInfoList>::value,
-                              typename CheckerOrPackageInfoList::const_iterator,
-                              typename CheckerOrPackageInfoList::iterator>::type
-    binaryFind(CheckerOrPackageInfoList &Collection, StringRef FullName) {
+static std::conditional_t<std::is_const<CheckerOrPackageInfoList>::value,
+                          typename CheckerOrPackageInfoList::const_iterator,
+                          typename CheckerOrPackageInfoList::iterator>
+binaryFind(CheckerOrPackageInfoList &Collection, StringRef FullName) {
 
   using CheckerOrPackage = typename CheckerOrPackageInfoList::value_type;
   using CheckerOrPackageFullNameLT = FullNameLT<CheckerOrPackage>;
diff --git a/clang/lib/Tooling/ASTDiff/ASTDiff.cpp b/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
index df00183d417b45fdf3112117990a70a9ab2ddd19..ec44ff18a2ce3fe5707eb87dedc92573934b25b6 100644
--- a/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
+++ b/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
@@ -117,12 +117,12 @@ public:
   Impl(SyntaxTree *Parent, Stmt *N, ASTContext &AST);
   template <class T>
   Impl(SyntaxTree *Parent,
-       typename std::enable_if<std::is_base_of<Stmt, T>::value, T>::type *Node,
+       std::enable_if_t<std::is_base_of<Stmt, T>::value, T> *Node,
        ASTContext &AST)
       : Impl(Parent, dyn_cast<Stmt>(Node), AST) {}
   template <class T>
   Impl(SyntaxTree *Parent,
-       typename std::enable_if<std::is_base_of<Decl, T>::value, T>::type *Node,
+       std::enable_if_t<std::is_base_of<Decl, T>::value, T> *Node,
        ASTContext &AST)
       : Impl(Parent, dyn_cast<Decl>(Node), AST) {}
 
diff --git a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp
index 4436cb0aac607d4fa63e8e475a059943108541fa..eac9ac0e827982c38728e9653b9b53d88251bbff 100644
--- a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-0x.cpp
@@ -180,9 +180,9 @@ void shrink_int() {
   Agg<bool> b2 = {1};  // OK
   Agg<bool> b3 = {-1};  // expected-error {{ cannot be narrowed }} expected-note {{silence}}
 
-  // Conversions from pointers to booleans aren't narrowing conversions.
+  // Conversions from pointers to booleans are narrowing conversions.
   Agg<bool>* ptr = &b1;
-  Agg<bool> b = {ptr};  // OK
+  Agg<bool> b = {ptr};  // expected-error {{ cannot be narrowed }} expected-note {{silence}}
 
   Agg<short> ce1 = { Convert<int>(100000) }; // expected-error {{constant expression evaluates to 100000 which cannot be narrowed to type 'short'}} expected-note {{silence}} expected-warning {{changes value from 100000 to -31072}}
   Agg<char> ce2 = { ConvertVar<short>() }; // expected-error {{non-constant-expression cannot be narrowed from type 'short' to 'char'}} expected-note {{silence}}
@@ -240,3 +240,13 @@ void test_narrowed(Value<sizeof(int)> vi, Value<sizeof(double)> vd) {
   int &ir = check_narrowed<double>(vd);
   float &fr = check_narrowed<int>(vi);
 }
+
+// * from a pointer type or a pointer-to-member type to bool.
+void P1957R2(void *a, int *b, Agg<int> *c, int Agg<int>::*d) {
+  Agg<bool> ta = {a}; // expected-error {{cannot be narrowed}} expected-note {{}}
+  Agg<bool> tb = {b}; // expected-error {{cannot be narrowed}} expected-note {{}}
+  Agg<bool> tc = {c}; // expected-error {{cannot be narrowed}} expected-note {{}}
+  Agg<bool> td = {d}; // expected-error {{cannot be narrowed}} expected-note {{}}
+}
+template<bool> struct BoolParam {};
+BoolParam<&P1957R2> bp; // expected-error {{not allowed in a converted constant expression}}
diff --git a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-cxx11-nowarn.cpp b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-cxx11-nowarn.cpp
index d4d8198d4fc83f19cb5bf3a3e3c81e292d3589a9..d701ec964605232fbb9f01f61a8d14265499ee52 100644
--- a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-cxx11-nowarn.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.list/p7-cxx11-nowarn.cpp
@@ -163,10 +163,6 @@ void shrink_int() {
   Agg<bool> b2 = {1};  // OK
   Agg<bool> b3 = {-1};  // expected-warning {{ cannot be narrowed }} expected-note {{silence}}
 
-  // Conversions from pointers to booleans aren't narrowing conversions.
-  Agg<bool>* ptr = &b1;
-  Agg<bool> b = {ptr};  // OK
-
   Agg<short> ce1 = { Convert<int>(100000) }; // expected-warning {{constant expression evaluates to 100000 which cannot be narrowed to type 'short'}} expected-note {{silence}} expected-warning {{changes value from 100000 to -31072}}
   Agg<char> ce2 = { ConvertVar<short>() }; // expected-warning {{non-constant-expression cannot be narrowed from type 'short' to 'char'}} expected-note {{silence}}
 }
diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp
index 52129844c4188b4a5bd95f38873477254fefaa3f..d55427f5de8cb202c24cb795bc7fb805fee6f537 100644
--- a/clang/test/CXX/drs/dr14xx.cpp
+++ b/clang/test/CXX/drs/dr14xx.cpp
@@ -8,6 +8,15 @@
 // expected-no-diagnostics
 #endif
 
+namespace dr1423 { // dr1423: 11
+#if __cplusplus >= 201103L
+  bool b1 = nullptr; // expected-error {{cannot initialize}}
+  bool b2(nullptr); // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+  bool b3 = {nullptr}; // expected-error {{cannot initialize}}
+  bool b4{nullptr}; // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+#endif
+}
+
 // dr1425: na abi
 
 namespace dr1460 { // dr1460: 3.5
diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp
index 7a0adb5fe406b3882ab16835d4c9450ac27bf3a9..4c4ed7767ecd2c6b216ae88a8669708db4a4d981 100644
--- a/clang/test/CXX/drs/dr6xx.cpp
+++ b/clang/test/CXX/drs/dr6xx.cpp
@@ -585,10 +585,10 @@ namespace dr652 { // dr652: yes
 // dr653 FIXME: add codegen test
 
 #if __cplusplus >= 201103L
-namespace dr654 { // dr654: yes
+namespace dr654 { // dr654: sup 1423
   void f() {
     if (nullptr) {} // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
-    bool b = nullptr; // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+    bool b = nullptr; // expected-error {{cannot initialize a variable of type 'bool' with an rvalue of type 'nullptr_t'}}
     if (nullptr == 0) {}
     if (nullptr != 0) {}
     if (nullptr <= 0) {} // expected-error {{invalid operands}}
diff --git a/clang/test/CXX/expr/expr.const/p3-0x.cpp b/clang/test/CXX/expr/expr.const/p3-0x.cpp
index 731e0c312fa13f1a284ca9c727f05101ad3e01a9..8daca7a565f31d9182644c9f0d846645a5e93ee0 100644
--- a/clang/test/CXX/expr/expr.const/p3-0x.cpp
+++ b/clang/test/CXX/expr/expr.const/p3-0x.cpp
@@ -95,7 +95,7 @@ Val<decltype(&noexcept_true), &noexcept_false> add_noexcept;
 using Int = A<1.0>; // expected-error {{conversion from 'double' to 'unsigned char' is not allowed in a converted constant expression}}
 enum B : bool {
   True = &a, // expected-error {{conversion from 'bool (*)(int)' to 'bool' is not allowed in a converted constant expression}}
-  False = nullptr // expected-error {{conversion from 'nullptr_t' to 'bool' is not allowed in a converted constant expression}}
+  False = 0.0, // expected-error {{conversion from 'double' to 'bool' is not allowed in a converted constant expression}}
 };
 void c() {
   // Note, promoted type of switch is 'int'.
diff --git a/clang/test/CodeGen/debug-info-extern-call.c b/clang/test/CodeGen/debug-info-extern-call.c
index da3764f7359eab0b70e63d35d3168c5f81039696..072e578b58986c2d02500fbec911c1bfc4ab30da 100644
--- a/clang/test/CodeGen/debug-info-extern-call.c
+++ b/clang/test/CodeGen/debug-info-extern-call.c
@@ -1,7 +1,7 @@
 // When entry values are emitted, expect a subprogram for extern decls so that
 // the dwarf generator can describe call site parameters at extern call sites.
 //
-// RUN: %clang -Xclang -femit-debug-entry-values -g -O2 -target x86_64-none-linux-gnu -S -emit-llvm %s -o - \
+// RUN: %clang -g -O2 -target x86_64-none-linux-gnu -S -emit-llvm %s -o - \
 // RUN:   | FileCheck %s -check-prefix=DECLS-FOR-EXTERN
 
 // Similarly, when the debugger tuning is gdb, expect a subprogram for extern
diff --git a/clang/test/CodeGen/hexagon-hvx-abi.c b/clang/test/CodeGen/hexagon-hvx-abi.c
new file mode 100644
index 0000000000000000000000000000000000000000..cd70f57e4ffca7dc1b9cb2b9d6a99c7f2336b952
--- /dev/null
+++ b/clang/test/CodeGen/hexagon-hvx-abi.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -triple hexagon -emit-llvm -target-cpu hexagonv66 -target-feature +hvxv66 -target-feature +hvx-length64b -o - %s | FileCheck %s --check-prefix CHECK-HVX64
+// RUN: %clang_cc1 -triple hexagon -emit-llvm -target-cpu hexagonv66 -target-feature +hvxv66 -target-feature +hvx-length128b -o - %s | FileCheck %s --check-prefix CHECK-HVX128
+
+typedef long HVX_Vector __attribute__((__vector_size__(__HVX_LENGTH__)))
+  __attribute__((aligned(__HVX_LENGTH__)));
+typedef long HVX_VectorPair __attribute__((__vector_size__(2*__HVX_LENGTH__)))
+  __attribute__((aligned(__HVX_LENGTH__)));
+
+// CHECK-HVX64: define {{.*}} <16 x i32> @foo(<16 x i32> %a, <32 x i32> %b)
+// CHECK-HVX128: define {{.*}} <32 x i32> @foo(<32 x i32> %a, <64 x i32> %b) 
+HVX_Vector foo(HVX_Vector a, HVX_VectorPair b) {
+  return a;
+}
+
+// CHECK-HVX64: define {{.*}} <32 x i32> @bar(<16 x i32> %a, <32 x i32> %b)
+// CHECK-HVX128: define {{.*}} <64 x i32> @bar(<32 x i32> %a, <64 x i32> %b)
+HVX_VectorPair bar(HVX_Vector a, HVX_VectorPair b) {
+  return b;
+}
+
diff --git a/clang/test/CodeGen/ignore-exceptions.cpp b/clang/test/CodeGen/ignore-exceptions.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c2770005cb73e18bc8b8f2c3cbc57ecebce60051
--- /dev/null
+++ b/clang/test/CodeGen/ignore-exceptions.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 %s -triple powerpc64-linux -fexceptions -fcxx-exceptions -fignore-exceptions -emit-llvm -o - | FileCheck %s
+
+struct A {
+  ~A(){}
+};
+
+void f(void) {
+// CHECK-NOT: personality i8* bitcast (i32 (...)* @__gcc_personality_v0 to i8*)
+  A a;
+  try {
+    throw 1;
+  } catch(...) {
+  }
+// CHECK:  %a = alloca %struct.A, align 1
+// CHECK:  %exception = call i8* @__cxa_allocate_exception(i64 4) #1
+// CHECK:  %0 = bitcast i8* %exception to i32*
+// CHECK:  store i32 1, i32* %0, align 16
+// CHECK:  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #2
+// CHECK:  unreachable
+
+// CHECK-NOT: invoke
+// CHECK-NOT: landingpad
+// CHECK-NOT: __cxa_begin_catch
+// CHECK-NOT: __cxa_end_catch
+}
diff --git a/clang/test/CodeGen/x86_32-arguments-win32.c b/clang/test/CodeGen/x86_32-arguments-win32.c
index 65e25f32c2502d5ac5bbf6b256c14fb77a8e5e16..33a6216a62bd71d8ce28966ce16d83ebf49535ba 100644
--- a/clang/test/CodeGen/x86_32-arguments-win32.c
+++ b/clang/test/CodeGen/x86_32-arguments-win32.c
@@ -46,3 +46,47 @@ struct s6 {
 struct s6 f6_1(void) { while (1) {} }
 void f6_2(struct s6 a0) {}
 
+
+// MSVC passes up to three vectors in registers, and the rest indirectly.  We
+// (arbitrarily) pass oversized vectors indirectly, since that is the safest way
+// to do it.
+typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
+typedef float __m256 __attribute__((__vector_size__(32), __aligned__(32)));
+typedef float __m512 __attribute__((__vector_size__(64), __aligned__(64)));
+typedef float __m1024 __attribute__((__vector_size__(128), __aligned__(128)));
+
+__m128 gv128;
+__m256 gv256;
+__m512 gv512;
+__m1024 gv1024;
+
+void receive_vec_128(__m128 x, __m128 y, __m128 z, __m128 w, __m128 q) {
+  gv128 = x + y + z + w + q;
+}
+void receive_vec_256(__m256 x, __m256 y, __m256 z, __m256 w, __m256 q) {
+  gv256 = x + y + z + w + q;
+}
+void receive_vec_512(__m512 x, __m512 y, __m512 z, __m512 w, __m512 q) {
+  gv512 = x + y + z + w + q;
+}
+void receive_vec_1024(__m1024 x, __m1024 y, __m1024 z, __m1024 w, __m1024 q) {
+  gv1024 = x + y + z + w + q;
+}
+// CHECK-LABEL: define dso_local void @receive_vec_128(<4 x float> inreg %x, <4 x float> inreg %y, <4 x float> inreg %z, <4 x float>* %0, <4 x float>* %1)
+// CHECK-LABEL: define dso_local void @receive_vec_256(<8 x float> inreg %x, <8 x float> inreg %y, <8 x float> inreg %z, <8 x float>* %0, <8 x float>* %1)
+// CHECK-LABEL: define dso_local void @receive_vec_512(<16 x float> inreg %x, <16 x float> inreg %y, <16 x float> inreg %z, <16 x float>* %0, <16 x float>* %1)
+// CHECK-LABEL: define dso_local void @receive_vec_1024(<32 x float>* %0, <32 x float>* %1, <32 x float>* %2, <32 x float>* %3, <32 x float>* %4)
+
+void pass_vec_128() {
+  __m128 z = {0};
+  receive_vec_128(z, z, z, z, z);
+}
+
+// CHECK-LABEL: define dso_local void @pass_vec_128()
+// CHECK: call void @receive_vec_128(<4 x float> inreg %{{[^,)]*}}, <4 x float> inreg %{{[^,)]*}}, <4 x float> inreg %{{[^,)]*}}, <4 x float>* %{{[^,)]*}}, <4 x float>* %{{[^,)]*}})
+
+
+void __fastcall fastcall_indirect_vec(__m128 x, __m128 y, __m128 z, __m128 w, int edx, __m128 q) {
+  gv128 = x + y + z + w + q;
+}
+// CHECK-LABEL: define dso_local x86_fastcallcc void @"\01@fastcall_indirect_vec@84"(<4 x float> inreg %x, <4 x float> inreg %y, <4 x float> inreg %z, <4 x float>* inreg %0, i32 inreg %edx, <4 x float>* %1)
diff --git a/clang/test/CodeGen/xray-attributes-skip-entry-exit.cpp b/clang/test/CodeGen/xray-attributes-skip-entry-exit.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e23262fd5dbc9e3addebe63643ef9167f4354aa3
--- /dev/null
+++ b/clang/test/CodeGen/xray-attributes-skip-entry-exit.cpp
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -fxray-instrument \
+// RUN:     -fxray-instrumentation-bundle=function-entry -x c++ \
+// RUN:     -std=c++11 -triple x86_64-unknown-unknown -emit-llvm -o - %s \
+// RUN:     | FileCheck --check-prefixes CHECK,SKIPEXIT %s
+// RUN: %clang_cc1 -fxray-instrument \
+// RUN:     -fxray-instrumentation-bundle=function-exit -x c++ \
+// RUN:     -std=c++11 -triple x86_64-unknown-unknown -emit-llvm -o - %s \
+// RUN:     | FileCheck --check-prefixes CHECK,SKIPENTRY %s
+// RUN: %clang_cc1 -fxray-instrument \
+// RUN:     -fxray-instrumentation-bundle=function-entry,function-exit -x c++ \
+// RUN:     -std=c++11 -triple x86_64-unknown-unknown -emit-llvm -o - %s \
+// RUN:     | FileCheck --check-prefixes CHECK,NOSKIPENTRY,NOSKIPEXIT %s
+
+// CHECK: define void @_Z13justAFunctionv() #[[ATTR:[0-9]+]] {
+void justAFunction() {
+}
+
+// SKIPENTRY: attributes #[[ATTR]] = {{.*}} "xray-skip-entry" {{.*}}
+// SKIPEXIT: attributes #[[ATTR]] = {{.*}} "xray-skip-exit" {{.*}}
+
+// NOSKIPENTRY-NOT: attributes #[[ATTR]] = {{.*}} "xray-skip-entry" {{.*}}
+// NOSKIPEXIT-NOT: attributes #[[ATTR]] = {{.*}} "xray-skip-exit" {{.*}}
diff --git a/clang/test/CodeGen/xray-ignore-loops.cpp b/clang/test/CodeGen/xray-ignore-loops.cpp
index 17ea2afbeeceff45dc54988ba5b6de11f9e79326..ed918b8f9b310f1937665ad1c1a6d0af93a344be 100644
--- a/clang/test/CodeGen/xray-ignore-loops.cpp
+++ b/clang/test/CodeGen/xray-ignore-loops.cpp
@@ -4,5 +4,5 @@ int foo() {
   return 1;
 }
 
-// CHECK: define i32 @_Z3foov() #[[ATTRS:[0-9]+]] {
+// CHECK: define{{.*}} i32 @_Z3foov() #[[ATTRS:[0-9]+]] {
 // CHECK-DAG: attributes #[[ATTRS]] = {{.*}} "xray-ignore-loops" {{.*}}
diff --git a/clang/test/CodeGenCXX/dbg-info-all-calls-described.cpp b/clang/test/CodeGenCXX/dbg-info-all-calls-described.cpp
index 667c2469b55eacc7005048ee23c43118b15a0010..540ef02c908aff06f59bb210e60e0eeb92ed7163 100644
--- a/clang/test/CodeGenCXX/dbg-info-all-calls-described.cpp
+++ b/clang/test/CodeGenCXX/dbg-info-all-calls-described.cpp
@@ -15,22 +15,22 @@
 // RUN: | FileCheck %s -check-prefix=HAS-ATTR \
 // RUN:     -implicit-check-not=DISubprogram -implicit-check-not=DIFlagAllCallsDescribed
 
-// Supported: DWARF4 + GDB tuning by using '-femit-debug-entry-values'
-// RUN: %clang_cc1 -femit-debug-entry-values -emit-llvm -triple x86_64-linux-gnu \
+// Supported: DWARF4 + GDB tuning, -O1
+// RUN: %clang_cc1 -emit-llvm -triple x86_64-linux-gnu \
 // RUN:   %s -o - -O1 -disable-llvm-passes -debugger-tuning=gdb \
 // RUN:   -debug-info-kind=standalone -dwarf-version=4 \
 // RUN: | FileCheck %s -check-prefix=HAS-ATTR \
 // RUN:     -implicit-check-not=DIFlagAllCallsDescribed
 
-// Supported: DWARF4 + LLDB tuning by using '-femit-debug-entry-values'
-// RUN: %clang_cc1 -femit-debug-entry-values -emit-llvm -triple x86_64-linux-gnu \
+// Supported: DWARF4 + LLDB tuning, -O1
+// RUN: %clang_cc1 -emit-llvm -triple x86_64-linux-gnu \
 // RUN:   %s -o - -O1 -disable-llvm-passes -debugger-tuning=lldb \
 // RUN:   -debug-info-kind=standalone -dwarf-version=4 \
 // RUN: | FileCheck %s -check-prefix=HAS-ATTR \
 // RUN:     -implicit-check-not=DIFlagAllCallsDescribed
 
-// Unsupported: -O0 + '-femit-debug-entry-values'
-// RUN: %clang_cc1 -femit-debug-entry-values -emit-llvm -triple x86_64-linux-gnu \
+// Unsupported: -O0
+// RUN: %clang_cc1 -emit-llvm -triple x86_64-linux-gnu \
 // RUN:   %s -o - -O0 -disable-llvm-passes -debugger-tuning=gdb \
 // RUN:   -debug-info-kind=standalone -dwarf-version=4 \
 // RUN: | FileCheck %s -check-prefix=NO-ATTR
diff --git a/clang/test/CodeGenCXX/inalloca-overaligned.cpp b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..910f0d92895e7591eb4627705b47b5734c4a6208
--- /dev/null
+++ b/clang/test/CodeGenCXX/inalloca-overaligned.cpp
@@ -0,0 +1,52 @@
+// RUN: %clang_cc1 -fms-extensions -w -triple i386-pc-win32 -emit-llvm -o - %s | FileCheck %s
+
+// PR44395
+// MSVC passes overaligned types indirectly since MSVC 2015. Make sure that
+// works with inalloca.
+
+// FIXME: Pass non-trivial *and* overaligned types indirectly. Right now the C++
+// ABI rules say to use inalloca, and they take precedence, so it's not easy to
+// implement this.
+
+
+struct NonTrivial {
+  NonTrivial();
+  NonTrivial(const NonTrivial &o);
+  int x;
+};
+
+struct __declspec(align(64)) OverAligned {
+  OverAligned();
+  int buf[16];
+};
+
+extern int gvi32;
+
+int receive_inalloca_overaligned(NonTrivial nt, OverAligned o) {
+  return nt.x + o.buf[0];
+}
+
+// CHECK-LABEL: define dso_local i32 @"?receive_inalloca_overaligned@@Y{{.*}}"
+// CHECK-SAME: (<{ %struct.NonTrivial, %struct.OverAligned* }>* inalloca %0)
+
+int pass_inalloca_overaligned() {
+  gvi32 = receive_inalloca_overaligned(NonTrivial(), OverAligned());
+  return gvi32;
+}
+
+// CHECK-LABEL: define dso_local i32 @"?pass_inalloca_overaligned@@Y{{.*}}"
+// CHECK: [[TMP:%[^ ]*]] = alloca %struct.OverAligned, align 64
+// CHECK: call i8* @llvm.stacksave()
+// CHECK: alloca inalloca <{ %struct.NonTrivial, %struct.OverAligned* }>
+
+// Construct OverAligned into TMP.
+// CHECK: call x86_thiscallcc %struct.OverAligned* @"??0OverAligned@@QAE@XZ"(%struct.OverAligned* [[TMP]])
+
+// Construct NonTrivial into the GEP.
+// CHECK: [[GEP:%[^ ]*]] = getelementptr inbounds <{ %struct.NonTrivial, %struct.OverAligned* }>, <{ %struct.NonTrivial, %struct.OverAligned* }>* %{{.*}}, i32 0, i32 0
+// CHECK: call x86_thiscallcc %struct.NonTrivial* @"??0NonTrivial@@QAE@XZ"(%struct.NonTrivial* [[GEP]])
+
+// Store the address of an OverAligned temporary into the struct.
+// CHECK: getelementptr inbounds <{ %struct.NonTrivial, %struct.OverAligned* }>, <{ %struct.NonTrivial, %struct.OverAligned* }>* %{{.*}}, i32 0, i32 1
+// CHECK: store %struct.OverAligned* [[TMP]], %struct.OverAligned** %{{.*}}, align 4
+// CHECK: call i32 @"?receive_inalloca_overaligned@@Y{{.*}}"(<{ %struct.NonTrivial, %struct.OverAligned* }>* inalloca %argmem)
diff --git a/clang/test/CodeGenCXX/inalloca-vector.cpp b/clang/test/CodeGenCXX/inalloca-vector.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f3d7f81e94439a701282c159eea32832b99f7322
--- /dev/null
+++ b/clang/test/CodeGenCXX/inalloca-vector.cpp
@@ -0,0 +1,79 @@
+// RUN: %clang_cc1 -w -triple i686-pc-win32 -emit-llvm -o - %s | FileCheck %s
+
+// PR44395
+// MSVC passes up to three vectors in registers, and the rest indirectly. Check
+// that both are compatible with an inalloca prototype.
+
+struct NonTrivial {
+  NonTrivial();
+  NonTrivial(const NonTrivial &o);
+  unsigned handle;
+};
+
+typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
+__m128 gv128;
+
+// nt, w, and q will be in the inalloca pack.
+void receive_vec_128(NonTrivial nt, __m128 x, __m128 y, __m128 z, __m128 w, __m128 q) {
+  gv128 = x + y + z + w + q;
+}
+// CHECK-LABEL: define dso_local void  @"?receive_vec_128@@YAXUNonTrivial@@T__m128@@1111@Z"
+// CHECK-SAME: (<4 x float> inreg %x,
+// CHECK-SAME: <4 x float> inreg %y,
+// CHECK-SAME: <4 x float> inreg %z,
+// CHECK-SAME: <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* inalloca %0)
+
+void pass_vec_128() {
+  __m128 z = {0};
+  receive_vec_128(NonTrivial(), z, z, z, z, z);
+}
+// CHECK-LABEL: define dso_local void @"?pass_vec_128@@YAXXZ"()
+// CHECK: getelementptr inbounds <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>, <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* %{{[^,]*}}, i32 0, i32 0
+// CHECK: call x86_thiscallcc %struct.NonTrivial* @"??0NonTrivial@@QAE@XZ"(%struct.NonTrivial* %{{.*}})
+
+// Store q, store temp alloca.
+// CHECK: store <4 x float> %{{[^,]*}}, <4 x float>* %{{[^,]*}}, align 16
+// CHECK: getelementptr inbounds <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>, <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* %{{[^,]*}}, i32 0, i32 1
+// CHECK: store <4 x float>* %{{[^,]*}}, <4 x float>** %{{[^,]*}}, align 4
+
+// Store w, store temp alloca.
+// CHECK: store <4 x float> %{{[^,]*}}, <4 x float>* %{{[^,]*}}, align 16
+// CHECK: getelementptr inbounds <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>, <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* %{{[^,]*}}, i32 0, i32 2
+// CHECK: store <4 x float>* %{{[^,]*}}, <4 x float>** %{{[^,]*}}, align 4
+
+// CHECK: call void @"?receive_vec_128@@YAXUNonTrivial@@T__m128@@1111@Z"
+// CHECK-SAME: (<4 x float> inreg %{{[^,]*}},
+// CHECK-SAME: <4 x float> inreg %{{[^,]*}},
+// CHECK-SAME: <4 x float> inreg %{{[^,]*}},
+// CHECK-SAME: <{ %struct.NonTrivial, <4 x float>*, <4 x float>* }>* inalloca %{{[^,]*}})
+
+// w will be passed indirectly by register, and q will be passed indirectly, but
+// the pointer will be in memory.
+void __fastcall fastcall_receive_vec(__m128 x, __m128 y, __m128 z, __m128 w, int edx, __m128 q, NonTrivial nt) {
+  gv128 = x + y + z + w + q;
+}
+// CHECK-LABEL: define dso_local x86_fastcallcc void @"?fastcall_receive_vec@@Y{{[^"]*}}"
+// CHECK-SAME: (<4 x float> inreg %x,
+// CHECK-SAME: <4 x float> inreg %y,
+// CHECK-SAME: <4 x float> inreg %z,
+// CHECK-SAME: <4 x float>* inreg %0,
+// CHECK-SAME: i32 inreg %edx,
+// CHECK-SAME: <{ <4 x float>*, %struct.NonTrivial }>* inalloca %1)
+
+
+void __vectorcall vectorcall_receive_vec(double xmm0, double xmm1, double xmm2,
+                                         __m128 x, __m128 y, __m128 z,
+                                         __m128 w, int edx, __m128 q, NonTrivial nt) {
+  gv128 = x + y + z + w + q;
+}
+// FIXME: Enable these checks, clang generates wrong IR.
+// CHECK-LABEL: define dso_local x86_vectorcallcc void @"?vectorcall_receive_vec@@Y{{[^"]*}}"
+// CHECKX-SAME: (double inreg %xmm0,
+// CHECKX-SAME: double inreg %xmm1,
+// CHECKX-SAME: double inreg %xmm2,
+// CHECKX-SAME: <4 x float> inreg %x,
+// CHECKX-SAME: <4 x float> inreg %y,
+// CHECKX-SAME: <4 x float> inreg %z,
+// CHECKX-SAME: <4 x float>* inreg %0,
+// CHECKX-SAME: i32 inreg %edx,
+// CHECKX-SAME: <{ <4 x float>*, %struct.NonTrivial }>* inalloca %1)
diff --git a/clang/test/CodeGenCXX/nullptr.cpp b/clang/test/CodeGenCXX/nullptr.cpp
index 823c0d7d18a730c8af1bd89290a6004a905bb1ec..ab47282569fd61712c9cec708e1aa882927929fc 100644
--- a/clang/test/CodeGenCXX/nullptr.cpp
+++ b/clang/test/CodeGenCXX/nullptr.cpp
@@ -32,7 +32,7 @@ union U {
 // CHECK: load
 // CHECK-NOT: load
 // CHECK: ret i1 false
-bool pr23833_a(U &u) { return u.b; }
+bool pr23833_a(U &u) { return bool(u.b); }
 
 // CHECK-LABEL: define {{.*}}pr23833_b
 // CHECK: store
diff --git a/clang/test/Driver/XRay/xray-ignore-loops-flags.cpp b/clang/test/Driver/XRay/xray-ignore-loops-flags.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..476786fc639a3b98d5eb1d4287ec14c7d98d1e66
--- /dev/null
+++ b/clang/test/Driver/XRay/xray-ignore-loops-flags.cpp
@@ -0,0 +1,10 @@
+// This test ensures that when we invoke the clang compiler, that the -cc1
+// options include the -fxray-ignore-loops flag we provide in the
+// invocation.
+//
+// RUN: %clang -fxray-instrument -fxray-ignore-loops -target x86_64-linux- -### \
+// RUN:     -x c++ -std=c++11 -emit-llvm -c -o - %s 2>&1 \
+// RUN:     | FileCheck %s
+// CHECK:  -fxray-ignore-loops
+//
+// REQUIRES: x86_64 || x86_64h
diff --git a/clang/test/Driver/aix-as.c b/clang/test/Driver/aix-as.c
index 4f67d1ba90b702bad50132f2eaa73d0c54126fe4..cb3053f5acd317672978174657a09b52a4e08fb6 100644
--- a/clang/test/Driver/aix-as.c
+++ b/clang/test/Driver/aix-as.c
@@ -23,7 +23,6 @@
 // CHECK-AS64: "-u" 
 // CHECK-AS64: "-many"
 
-
 // Check powerpc-ibm-aix7.1.0.0, 32-bit. -Xassembler <arg> option. 
 // RUN: %clang -no-canonical-prefixes %s -### -c -o %t.o 2>&1 \
 // RUN:         -Xassembler -w \
diff --git a/clang/test/Driver/crash-report.c b/clang/test/Driver/crash-report.c
index 3a77a21c62b47e56fc1ccf8724a07c5fb98f7810..ceb16cb6e18a775c07aa2e4cec44b1f63624abc7 100644
--- a/clang/test/Driver/crash-report.c
+++ b/clang/test/Driver/crash-report.c
@@ -21,12 +21,20 @@
 // RUN: cat %t/crash-report-*.c | FileCheck --check-prefix=CHECKSRC %s
 // RUN: cat %t/crash-report-*.sh | FileCheck --check-prefix=CHECKSH %s
 
+// RUN: env TMPDIR=%t TEMP=%t TMP=%t RC_DEBUG_OPTIONS=1                  \
+// RUN:  CC_PRINT_HEADERS=1 CC_LOG_DIAGNOSTICS=1                         \
+// RUN:  not %clang %s @%t.rsp -DFATAL 2>&1 | FileCheck %s
+// RUN: cat %t/crash-report-*.c | FileCheck --check-prefix=CHECKSRC %s
+// RUN: cat %t/crash-report-*.sh | FileCheck --check-prefix=CHECKSH %s
+
 // REQUIRES: crash-recovery
 
 #ifdef PARSER
 #pragma clang __debug parser_crash
 #elif CRASH
 #pragma clang __debug crash
+#elif FATAL
+#pragma clang __debug llvm_fatal_error
 #endif
 
 // CHECK: Preprocessed source(s) and associated run script(s) are located at:
diff --git a/clang/test/Driver/riscv-features.c b/clang/test/Driver/riscv-features.c
index c5e0158b0ca1fe532ab9e4a82a6f408b80f1bcd1..8c0daf85e41f8428c2d6c8a07bf645c6990ee6a7 100644
--- a/clang/test/Driver/riscv-features.c
+++ b/clang/test/Driver/riscv-features.c
@@ -16,9 +16,10 @@
 // RUN: %clang -target riscv32-unknown-elf -### %s -msave-restore 2>&1 | FileCheck %s -check-prefix=SAVE-RESTORE
 // RUN: %clang -target riscv32-unknown-elf -### %s -mno-save-restore 2>&1 | FileCheck %s -check-prefix=NO-SAVE-RESTORE
 
-// SAVE-RESTORE: warning: the clang compiler does not support '-msave-restore'
-// NO-SAVE-RESTORE-NOT: warning: the clang compiler does not support
-// DEFAULT-NOT: warning: the clang compiler does not support
+// SAVE-RESTORE: "-target-feature" "+save-restore"
+// NO-SAVE-RESTORE: "-target-feature" "-save-restore"
+// DEFAULT: "-target-feature" "-save-restore"
+// DEFAULT-NOT: "-target-feature" "+save-restore"
 
 // RUN: %clang -target riscv32-linux -### %s -fsyntax-only 2>&1 \
 // RUN:   | FileCheck %s -check-prefix=DEFAULT-LINUX
diff --git a/clang/test/OpenMP/atomic_ast_print.cpp b/clang/test/OpenMP/atomic_ast_print.cpp
index 4dd64a75f2c906516f5bb933bfa33930024514ae..5d8e92c1476568409ad669e001eb331cb2730a13 100644
--- a/clang/test/OpenMP/atomic_ast_print.cpp
+++ b/clang/test/OpenMP/atomic_ast_print.cpp
@@ -44,13 +44,13 @@ T foo(T argc) {
     a = b;
     b++;
   }
-#pragma omp atomic acq_rel
+#pragma omp atomic
   a++;
-#pragma omp atomic read acq_rel
+#pragma omp atomic read
   a = argc;
-#pragma omp atomic acq_rel write
+#pragma omp atomic write
   a = argc + argc;
-#pragma omp atomic update acq_rel
+#pragma omp atomic update
   a = a + argc;
 #pragma omp atomic acq_rel capture
   a = b++;
@@ -59,13 +59,13 @@ T foo(T argc) {
     a = b;
     b++;
   }
-#pragma omp atomic acquire
+#pragma omp atomic
   a++;
 #pragma omp atomic read acquire
   a = argc;
-#pragma omp atomic acquire write
+#pragma omp atomic write
   a = argc + argc;
-#pragma omp atomic update acquire
+#pragma omp atomic update
   a = a + argc;
 #pragma omp atomic acquire capture
   a = b++;
@@ -76,7 +76,7 @@ T foo(T argc) {
   }
 #pragma omp atomic release
   a++;
-#pragma omp atomic read release
+#pragma omp atomic read
   a = argc;
 #pragma omp atomic release write
   a = argc + argc;
@@ -89,6 +89,21 @@ T foo(T argc) {
     a = b;
     b++;
   }
+#pragma omp atomic relaxed
+  a++;
+#pragma omp atomic read
+  a = argc;
+#pragma omp atomic relaxed write
+  a = argc + argc;
+#pragma omp atomic update relaxed
+  a = a + argc;
+#pragma omp atomic relaxed capture
+  a = b++;
+#pragma omp atomic capture relaxed
+  {
+    a = b;
+    b++;
+  }
   return T();
 }
 
@@ -123,13 +138,13 @@ T foo(T argc) {
 // CHECK-NEXT: a = b;
 // CHECK-NEXT: b++;
 // CHECK-NEXT: }
-// CHECK-NEXT: #pragma omp atomic acq_rel
+// CHECK-NEXT: #pragma omp atomic
 // CHECK-NEXT: a++;
-// CHECK-NEXT: #pragma omp atomic read acq_rel
+// CHECK-NEXT: #pragma omp atomic read
 // CHECK-NEXT: a = argc;
-// CHECK-NEXT: #pragma omp atomic acq_rel write
+// CHECK-NEXT: #pragma omp atomic write
 // CHECK-NEXT: a = argc + argc;
-// CHECK-NEXT: #pragma omp atomic update acq_rel
+// CHECK-NEXT: #pragma omp atomic update
 // CHECK-NEXT: a = a + argc;
 // CHECK-NEXT: #pragma omp atomic acq_rel capture
 // CHECK-NEXT: a = b++;
@@ -138,13 +153,13 @@ T foo(T argc) {
 // CHECK-NEXT: a = b;
 // CHECK-NEXT: b++;
 // CHECK-NEXT: }
-// CHECK-NEXT: #pragma omp atomic acquire
+// CHECK-NEXT: #pragma omp atomic
 // CHECK-NEXT: a++;
 // CHECK-NEXT: #pragma omp atomic read acquire
 // CHECK-NEXT: a = argc;
-// CHECK-NEXT: #pragma omp atomic acquire write
+// CHECK-NEXT: #pragma omp atomic write
 // CHECK-NEXT: a = argc + argc;
-// CHECK-NEXT: #pragma omp atomic update acquire
+// CHECK-NEXT: #pragma omp atomic update
 // CHECK-NEXT: a = a + argc;
 // CHECK-NEXT: #pragma omp atomic acquire capture
 // CHECK-NEXT: a = b++;
@@ -155,7 +170,7 @@ T foo(T argc) {
 // CHECK-NEXT: }
 // CHECK-NEXT: #pragma omp atomic release
 // CHECK-NEXT: a++;
-// CHECK-NEXT: #pragma omp atomic read release
+// CHECK-NEXT: #pragma omp atomic read
 // CHECK-NEXT: a = argc;
 // CHECK-NEXT: #pragma omp atomic release write
 // CHECK-NEXT: a = argc + argc;
@@ -168,6 +183,21 @@ T foo(T argc) {
 // CHECK-NEXT: a = b;
 // CHECK-NEXT: b++;
 // CHECK-NEXT: }
+// CHECK-NEXT: #pragma omp atomic relaxed
+// CHECK-NEXT: a++;
+// CHECK-NEXT: #pragma omp atomic read
+// CHECK-NEXT: a = argc;
+// CHECK-NEXT: #pragma omp atomic relaxed write
+// CHECK-NEXT: a = argc + argc;
+// CHECK-NEXT: #pragma omp atomic update relaxed
+// CHECK-NEXT: a = a + argc;
+// CHECK-NEXT: #pragma omp atomic relaxed capture
+// CHECK-NEXT: a = b++;
+// CHECK-NEXT: #pragma omp atomic capture relaxed
+// CHECK-NEXT: {
+// CHECK-NEXT: a = b;
+// CHECK-NEXT: b++;
+// CHECK-NEXT: }
 // CHECK: int a = int();
 // CHECK-NEXT: #pragma omp atomic
 // CHECK-NEXT: a++;
@@ -199,13 +229,13 @@ T foo(T argc) {
 // CHECK-NEXT: a = b;
 // CHECK-NEXT: b++;
 // CHECK-NEXT: }
-// CHECK-NEXT: #pragma omp atomic acq_rel
+// CHECK-NEXT: #pragma omp atomic
 // CHECK-NEXT: a++;
-// CHECK-NEXT: #pragma omp atomic read acq_rel
+// CHECK-NEXT: #pragma omp atomic read
 // CHECK-NEXT: a = argc;
-// CHECK-NEXT: #pragma omp atomic acq_rel write
+// CHECK-NEXT: #pragma omp atomic write
 // CHECK-NEXT: a = argc + argc;
-// CHECK-NEXT: #pragma omp atomic update acq_rel
+// CHECK-NEXT: #pragma omp atomic update
 // CHECK-NEXT: a = a + argc;
 // CHECK-NEXT: #pragma omp atomic acq_rel capture
 // CHECK-NEXT: a = b++;
@@ -214,13 +244,13 @@ T foo(T argc) {
 // CHECK-NEXT: a = b;
 // CHECK-NEXT: b++;
 // CHECK-NEXT: }
-// CHECK-NEXT: #pragma omp atomic acquire
+// CHECK-NEXT: #pragma omp atomic
 // CHECK-NEXT: a++;
 // CHECK-NEXT: #pragma omp atomic read acquire
 // CHECK-NEXT: a = argc;
-// CHECK-NEXT: #pragma omp atomic acquire write
+// CHECK-NEXT: #pragma omp atomic write
 // CHECK-NEXT: a = argc + argc;
-// CHECK-NEXT: #pragma omp atomic update acquire
+// CHECK-NEXT: #pragma omp atomic update
 // CHECK-NEXT: a = a + argc;
 // CHECK-NEXT: #pragma omp atomic acquire capture
 // CHECK-NEXT: a = b++;
@@ -231,7 +261,7 @@ T foo(T argc) {
 // CHECK-NEXT: }
 // CHECK-NEXT: #pragma omp atomic release
 // CHECK-NEXT: a++;
-// CHECK-NEXT: #pragma omp atomic read release
+// CHECK-NEXT: #pragma omp atomic read
 // CHECK-NEXT: a = argc;
 // CHECK-NEXT: #pragma omp atomic release write
 // CHECK-NEXT: a = argc + argc;
@@ -244,6 +274,21 @@ T foo(T argc) {
 // CHECK-NEXT: a = b;
 // CHECK-NEXT: b++;
 // CHECK-NEXT: }
+// CHECK-NEXT: #pragma omp atomic relaxed
+// CHECK-NEXT: a++;
+// CHECK-NEXT: #pragma omp atomic read
+// CHECK-NEXT: a = argc;
+// CHECK-NEXT: #pragma omp atomic relaxed write
+// CHECK-NEXT: a = argc + argc;
+// CHECK-NEXT: #pragma omp atomic update relaxed
+// CHECK-NEXT: a = a + argc;
+// CHECK-NEXT: #pragma omp atomic relaxed capture
+// CHECK-NEXT: a = b++;
+// CHECK-NEXT: #pragma omp atomic capture relaxed
+// CHECK-NEXT: {
+// CHECK-NEXT: a = b;
+// CHECK-NEXT: b++;
+// CHECK-NEXT: }
 
 int main(int argc, char **argv) {
   int b = 0;
@@ -279,13 +324,13 @@ int main(int argc, char **argv) {
     a = b;
     b++;
   }
-#pragma omp atomic acq_rel
+#pragma omp atomic
   a++;
-#pragma omp atomic read acq_rel
+#pragma omp atomic read
   a = argc;
-#pragma omp atomic acq_rel write
+#pragma omp atomic write
   a = argc + argc;
-#pragma omp atomic update acq_rel
+#pragma omp atomic update
   a = a + argc;
 #pragma omp atomic acq_rel capture
   a = b++;
@@ -294,13 +339,13 @@ int main(int argc, char **argv) {
     a = b;
     b++;
   }
-#pragma omp atomic acquire
+#pragma omp atomic
   a++;
 #pragma omp atomic read acquire
   a = argc;
-#pragma omp atomic acquire write
+#pragma omp atomic write
   a = argc + argc;
-#pragma omp atomic update acquire
+#pragma omp atomic update
   a = a + argc;
 #pragma omp atomic acquire capture
   a = b++;
@@ -311,7 +356,7 @@ int main(int argc, char **argv) {
   }
 #pragma omp atomic release
   a++;
-#pragma omp atomic read release
+#pragma omp atomic read
   a = argc;
 #pragma omp atomic release write
   a = argc + argc;
@@ -324,6 +369,21 @@ int main(int argc, char **argv) {
     a = b;
     b++;
   }
+#pragma omp atomic relaxed
+  a++;
+#pragma omp atomic read
+  a = argc;
+#pragma omp atomic relaxed write
+  a = argc + argc;
+#pragma omp atomic update relaxed
+  a = a + argc;
+#pragma omp atomic relaxed capture
+  a = b++;
+#pragma omp atomic capture relaxed
+  {
+    a = b;
+    b++;
+  }
   // CHECK-NEXT: #pragma omp atomic
   // CHECK-NEXT: a++;
   // CHECK-NEXT: #pragma omp atomic read
@@ -354,13 +414,13 @@ int main(int argc, char **argv) {
   // CHECK-NEXT: a = b;
   // CHECK-NEXT: b++;
   // CHECK-NEXT: }
-  // CHECK-NEXT: #pragma omp atomic acq_rel
+  // CHECK-NEXT: #pragma omp atomic
   // CHECK-NEXT: a++;
-  // CHECK-NEXT: #pragma omp atomic read acq_rel
+  // CHECK-NEXT: #pragma omp atomic read
   // CHECK-NEXT: a = argc;
-  // CHECK-NEXT: #pragma omp atomic acq_rel write
+  // CHECK-NEXT: #pragma omp atomic write
   // CHECK-NEXT: a = argc + argc;
-  // CHECK-NEXT: #pragma omp atomic update acq_rel
+  // CHECK-NEXT: #pragma omp atomic update
   // CHECK-NEXT: a = a + argc;
   // CHECK-NEXT: #pragma omp atomic acq_rel capture
   // CHECK-NEXT: a = b++;
@@ -369,13 +429,13 @@ int main(int argc, char **argv) {
   // CHECK-NEXT: a = b;
   // CHECK-NEXT: b++;
   // CHECK-NEXT: }
-  // CHECK-NEXT: #pragma omp atomic acquire
+  // CHECK-NEXT: #pragma omp atomic
   // CHECK-NEXT: a++;
   // CHECK-NEXT: #pragma omp atomic read acquire
   // CHECK-NEXT: a = argc;
-  // CHECK-NEXT: #pragma omp atomic acquire write
+  // CHECK-NEXT: #pragma omp atomic write
   // CHECK-NEXT: a = argc + argc;
-  // CHECK-NEXT: #pragma omp atomic update acquire
+  // CHECK-NEXT: #pragma omp atomic update
   // CHECK-NEXT: a = a + argc;
   // CHECK-NEXT: #pragma omp atomic acquire capture
   // CHECK-NEXT: a = b++;
@@ -386,7 +446,7 @@ int main(int argc, char **argv) {
   // CHECK-NEXT: }
   // CHECK-NEXT: #pragma omp atomic release
   // CHECK-NEXT: a++;
-  // CHECK-NEXT: #pragma omp atomic read release
+  // CHECK-NEXT: #pragma omp atomic read
   // CHECK-NEXT: a = argc;
   // CHECK-NEXT: #pragma omp atomic release write
   // CHECK-NEXT: a = argc + argc;
@@ -399,6 +459,21 @@ int main(int argc, char **argv) {
   // CHECK-NEXT: a = b;
   // CHECK-NEXT: b++;
   // CHECK-NEXT: }
+  // CHECK-NEXT: #pragma omp atomic relaxed
+  // CHECK-NEXT: a++;
+  // CHECK-NEXT: #pragma omp atomic read
+  // CHECK-NEXT: a = argc;
+  // CHECK-NEXT: #pragma omp atomic relaxed write
+  // CHECK-NEXT: a = argc + argc;
+  // CHECK-NEXT: #pragma omp atomic update relaxed
+  // CHECK-NEXT: a = a + argc;
+  // CHECK-NEXT: #pragma omp atomic relaxed capture
+  // CHECK-NEXT: a = b++;
+  // CHECK-NEXT: #pragma omp atomic capture relaxed
+  // CHECK-NEXT: {
+  // CHECK-NEXT: a = b;
+  // CHECK-NEXT: b++;
+  // CHECK-NEXT: }
   return foo(a);
 }
 
diff --git a/clang/test/OpenMP/atomic_capture_codegen.cpp b/clang/test/OpenMP/atomic_capture_codegen.cpp
index 3164c4f419f52dea04e4d296723f082c299fd899..d7b8748bbc24228e29ea46323707019d34eec6b6 100644
--- a/clang/test/OpenMP/atomic_capture_codegen.cpp
+++ b/clang/test/OpenMP/atomic_capture_codegen.cpp
@@ -843,7 +843,7 @@ int main() {
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
 // CHECK: store i32 [[NEW_VAL]], i32* @{{.+}},
-#pragma omp atomic capture
+#pragma omp atomic relaxed capture
   iv = bfx4.a = bfx4.a * ldv;
 // CHECK: [[EXPR:%.+]] = load x86_fp80, x86_fp80* @{{.+}}
 // CHECK: [[PREV_VALUE:%.+]] = load atomic i8, i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2) monotonic
@@ -874,7 +874,7 @@ int main() {
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
 // CHECK: store i32 [[CAST]], i32* @{{.+}},
-#pragma omp atomic capture
+#pragma omp atomic capture relaxed
   {iv = bfx4_packed.a; bfx4_packed.a -= ldv;}
 // CHECK: [[EXPR:%.+]] = load x86_fp80, x86_fp80* @{{.+}}
 // CHECK: [[PREV_VALUE:%.+]] = load atomic i64, i64* bitcast (%struct.BitFields4* @{{.+}} to i64*) monotonic
diff --git a/clang/test/OpenMP/atomic_messages.cpp b/clang/test/OpenMP/atomic_messages.cpp
index d595689e4d15c1558927c46709852c85990761ec..c44405178c8418fbcaf0e3ba166530b1411efc1a 100644
--- a/clang/test/OpenMP/atomic_messages.cpp
+++ b/clang/test/OpenMP/atomic_messages.cpp
@@ -728,16 +728,16 @@ int seq_cst() {
 template <class T>
 T acq_rel() {
   T a = 0, b = 0;
-// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic' cannot be used with 'acq_rel' clause}} omp50-note@+1 {{'acq_rel' clause used here}}
 #pragma omp atomic acq_rel
   // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
   ;
-// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'acq_rel' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
-#pragma omp atomic acq_rel seq_cst
-  a += b;
+// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'acq_rel' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}} omp50-error@+1 2 {{directive '#pragma omp atomic read' cannot be used with 'acq_rel' clause}} omp50-note@+1 2 {{'acq_rel' clause used here}}
+#pragma omp atomic read acq_rel seq_cst
+  a = b;
 
-// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic update' cannot be used with 'acq_rel' clause}} omp50-note@+1 {{'acq_rel' clause used here}}
 #pragma omp atomic update acq_rel
   // expected-error@+2 {{the statement for 'atomic update' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
@@ -749,16 +749,16 @@ T acq_rel() {
 int acq_rel() {
   int a = 0, b = 0;
 // Test for atomic acq_rel
-// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
-#pragma omp atomic acq_rel
-  // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic write' cannot be used with 'acq_rel' clause}} omp50-note@+1 {{'acq_rel' clause used here}}
+#pragma omp atomic acq_rel write
+  // expected-error@+2 {{the statement for 'atomic write' must be an expression statement of form 'x = expr;', where x is a lvalue expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
   ;
-// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
+// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
 #pragma omp atomic seq_cst acq_rel
   a += b;
 
-// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic update' cannot be used with 'acq_rel' clause}} omp50-note@+1 {{'acq_rel' clause used here}}
 #pragma omp atomic update acq_rel
   // expected-error@+2 {{the statement for 'atomic update' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
@@ -770,16 +770,16 @@ int acq_rel() {
 template <class T>
 T acquire() {
   T a = 0, b = 0;
-// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic' cannot be used with 'acquire' clause}} omp50-note@+1 {{'acquire' clause used here}}
 #pragma omp atomic acquire
   // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
   ;
-// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'acquire' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
+// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'acquire' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}} omp50-error@+1 2 {{directive '#pragma omp atomic' cannot be used with 'acquire' clause}} omp50-note@+1 2 {{'acquire' clause used here}}
 #pragma omp atomic acquire seq_cst
   a += b;
 
-// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic update' cannot be used with 'acquire' clause}} omp50-note@+1 {{'acquire' clause used here}}
 #pragma omp atomic update acquire
   // expected-error@+2 {{the statement for 'atomic update' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
@@ -791,16 +791,16 @@ T acquire() {
 int acquire() {
   int a = 0, b = 0;
 // Test for atomic acquire
-// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
-#pragma omp atomic acquire
-  // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic write' cannot be used with 'acquire' clause}} omp50-note@+1 {{'acquire' clause used here}}
+#pragma omp atomic write acquire
+  // expected-error@+2 {{the statement for 'atomic write' must be an expression statement of form 'x = expr;', where x is a lvalue expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
   ;
-// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
+// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
 #pragma omp atomic seq_cst acquire
   a += b;
 
-// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}}
+// omp45-error@+1 {{unexpected OpenMP clause 'acquire' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic update' cannot be used with 'acquire' clause}} omp50-note@+1 {{'acquire' clause used here}}
 #pragma omp atomic update acquire
   // expected-error@+2 {{the statement for 'atomic update' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
@@ -817,7 +817,7 @@ T release() {
   // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
   ;
-// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'release' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'release' in directive '#pragma omp atomic'}}
+// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'release' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'release' in directive '#pragma omp atomic'}}
 #pragma omp atomic release seq_cst
   a += b;
 
@@ -833,12 +833,12 @@ T release() {
 int release() {
   int a = 0, b = 0;
 // Test for atomic release
-// omp45-error@+1 {{unexpected OpenMP clause 'release' in directive '#pragma omp atomic'}}
-#pragma omp atomic release
-  // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
+// omp45-error@+1 {{unexpected OpenMP clause 'release' in directive '#pragma omp atomic'}} omp50-error@+1 {{directive '#pragma omp atomic read' cannot be used with 'release' clause}} omp50-note@+1 {{'release' clause used here}}
+#pragma omp atomic read release
+  // expected-error@+2 {{the statement for 'atomic read' must be an expression statement of form 'v = x;', where v and x are both lvalue expressions with scalar type}}
   // expected-note@+1 {{expected an expression statement}}
   ;
-// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'release' in directive '#pragma omp atomic'}}
+// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'release' in directive '#pragma omp atomic'}}
 #pragma omp atomic seq_cst release
   a += b;
 
@@ -851,6 +851,48 @@ int release() {
  return release<int>(); // omp50-note {{in instantiation of function template specialization 'release<int>' requested here}}
 }
 
+template <class T>
+T relaxed() {
+  T a = 0, b = 0;
+// omp45-error@+1 {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp atomic'}}
+#pragma omp atomic relaxed
+  // expected-error@+2 {{the statement for 'atomic' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
+  // expected-note@+1 {{expected an expression statement}}
+  ;
+// omp50-error@+1 2 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 2 {{'relaxed' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp atomic'}}
+#pragma omp atomic relaxed seq_cst
+  a += b;
+
+// omp45-error@+1 {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp atomic'}}
+#pragma omp atomic update relaxed
+  // expected-error@+2 {{the statement for 'atomic update' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
+  // expected-note@+1 {{expected an expression statement}}
+  ;
+
+  return T();
+}
+
+int relaxed() {
+  int a = 0, b = 0;
+// Test for atomic relaxed
+// omp45-error@+1 {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp atomic'}}
+#pragma omp atomic read relaxed
+  // expected-error@+2 {{the statement for 'atomic read' must be an expression statement of form 'v = x;', where v and x are both lvalue expressions with scalar type}}
+  // expected-note@+1 {{expected an expression statement}}
+  ;
+// omp50-error@+1 {{directive '#pragma omp atomic' cannot contain more than one 'seq_cst', 'relaxed', 'acq_rel', 'acquire' or 'release' clause}} omp50-note@+1 {{'seq_cst' clause used here}} omp45-error@+1 {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp atomic'}}
+#pragma omp atomic seq_cst relaxed
+  a += b;
+
+// omp45-error@+1 {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp atomic'}}
+#pragma omp atomic update relaxed
+  // expected-error@+2 {{the statement for 'atomic update' must be an expression statement of form '++x;', '--x;', 'x++;', 'x--;', 'x binop= expr;', 'x = x binop expr' or 'x = expr binop x', where x is an l-value expression with scalar type}}
+  // expected-note@+1 {{expected an expression statement}}
+  ;
+
+ return relaxed<int>(); // omp50-note {{in instantiation of function template specialization 'relaxed<int>' requested here}}
+}
+
 template <class T>
 T mixed() {
   T a, b = T();
diff --git a/clang/test/OpenMP/atomic_read_codegen.c b/clang/test/OpenMP/atomic_read_codegen.c
index 814975e38ea27a4935306b41fd4d739953d61491..94e212f72447782488d089a8f254d99ccbbaf5ad 100644
--- a/clang/test/OpenMP/atomic_read_codegen.c
+++ b/clang/test/OpenMP/atomic_read_codegen.c
@@ -304,7 +304,7 @@ int main() {
 // CHECK: [[ASHR:%.+]] = ashr i8 [[SHL]], 7
 // CHECK: sext i8 [[ASHR]] to i32
 // CHECK: store x86_fp80
-#pragma omp atomic read
+#pragma omp atomic relaxed read
   ldv = bfx4_packed.a;
 // CHECK: [[LD:%.+]] = load atomic i64, i64* bitcast (%struct.BitFields4* @bfx4 to i64*) monotonic
 // CHECK: store i64 [[LD]], i64* [[LDTEMP:%.+]]
@@ -312,7 +312,7 @@ int main() {
 // CHECK: [[SHL:%.+]] = shl i64 [[LD]], 40
 // CHECK: [[ASHR:%.+]] = ashr i64 [[SHL]], 57
 // CHECK: store x86_fp80
-#pragma omp atomic read release
+#pragma omp atomic read relaxed
   ldv = bfx4.b;
 // CHECK: [[LD:%.+]] = load atomic i8, i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @bfx4_packed, i32 0, i32 0, i64 2) acquire
 // CHECK: store i8 [[LD]], i8* [[LDTEMP:%.+]]
@@ -323,14 +323,13 @@ int main() {
 // CHECK: store x86_fp80
 #pragma omp atomic read acquire
   ldv = bfx4_packed.b;
-// CHECK: [[LD:%.+]] = load atomic i64, i64* bitcast (<2 x float>* @{{.+}} to i64*) acquire
+// CHECK: [[LD:%.+]] = load atomic i64, i64* bitcast (<2 x float>* @{{.+}} to i64*) monotonic
 // CHECK: [[BITCAST:%.+]] = bitcast <2 x float>* [[LDTEMP:%.+]] to i64*
 // CHECK: store i64 [[LD]], i64* [[BITCAST]]
 // CHECK: [[LD:%.+]] = load <2 x float>, <2 x float>* [[LDTEMP]]
 // CHECK: extractelement <2 x float> [[LD]]
-// CHECK: call{{.*}} @__kmpc_flush(
 // CHECK: store i64
-#pragma omp atomic read acq_rel
+#pragma omp atomic read
   ulv = float2x.x;
 // CHECK: call{{.*}} i{{[0-9]+}} @llvm.read_register
 // CHECK: call{{.*}} @__kmpc_flush(
diff --git a/clang/test/OpenMP/atomic_update_codegen.cpp b/clang/test/OpenMP/atomic_update_codegen.cpp
index fb900753898c533d9344cce1ccbeacbcd11a3c11..a2b6f70540aa3c6f56919af64bc77df9ab6a3829 100644
--- a/clang/test/OpenMP/atomic_update_codegen.cpp
+++ b/clang/test/OpenMP/atomic_update_codegen.cpp
@@ -809,10 +809,10 @@ int main() {
 // CHECK: [[FAIL_SUCCESS:%.+]] = extractvalue { i8, i1 } [[RES]], 1
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
-#pragma omp atomic update
+#pragma omp atomic relaxed update
   bfx4_packed.a -= ldv;
 // CHECK: [[EXPR:%.+]] = load x86_fp80, x86_fp80* @{{.+}}
-// CHECK: [[PREV_VALUE:%.+]] = load atomic i64, i64* bitcast (%struct.BitFields4* @{{.+}} to i64*) acquire
+// CHECK: [[PREV_VALUE:%.+]] = load atomic i64, i64* bitcast (%struct.BitFields4* @{{.+}} to i64*) monotonic
 // CHECK: br label %[[CONT:.+]]
 // CHECK: [[CONT]]
 // CHECK: [[OLD_BF_VALUE:%.+]] = phi i64 [ [[PREV_VALUE]], %[[EXIT]] ], [ [[FAILED_OLD_VAL:%.+]], %[[CONT]] ]
@@ -831,15 +831,15 @@ int main() {
 // CHECK: [[VAL:%.+]] = or i64 [[BF_CLEAR]], [[BF_VALUE]]
 // CHECK: store i64 [[VAL]], i64* [[TEMP1]]
 // CHECK: [[NEW_BF_VALUE:%.+]] = load i64, i64* [[TEMP1]]
-// CHECK: [[RES:%.+]] = cmpxchg i64* bitcast (%struct.BitFields4* @{{.+}} to i64*), i64 [[OLD_BF_VALUE]], i64 [[NEW_BF_VALUE]] acquire acquire
+// CHECK: [[RES:%.+]] = cmpxchg i64* bitcast (%struct.BitFields4* @{{.+}} to i64*), i64 [[OLD_BF_VALUE]], i64 [[NEW_BF_VALUE]] monotonic monotonic
 // CHECK: [[FAILED_OLD_VAL]] = extractvalue { i64, i1 } [[RES]], 0
 // CHECK: [[FAIL_SUCCESS:%.+]] = extractvalue { i64, i1 } [[RES]], 1
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
-#pragma omp atomic acquire
+#pragma omp atomic
   bfx4.b /= ldv;
 // CHECK: [[EXPR:%.+]] = load x86_fp80, x86_fp80* @{{.+}}
-// CHECK: [[PREV_VALUE:%.+]] = load atomic i8, i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2) acquire
+// CHECK: [[PREV_VALUE:%.+]] = load atomic i8, i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2) monotonic
 // CHECK: br label %[[CONT:.+]]
 // CHECK: [[CONT]]
 // CHECK: [[OLD_BF_VALUE:%.+]] = phi i8 [ [[PREV_VALUE]], %[[EXIT]] ], [ [[FAILED_OLD_VAL:%.+]], %[[CONT]] ]
@@ -861,17 +861,16 @@ int main() {
 // CHECK: or i8 [[BF_CLEAR]], [[BF_VALUE]]
 // CHECK: store i8 %{{.+}}, i8* [[BITCAST1]]
 // CHECK: [[NEW_BF_VALUE:%.+]] = load i8, i8* [[BITCAST1]]
-// CHECK: [[RES:%.+]] = cmpxchg i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2), i8 [[OLD_BF_VALUE]], i8 [[NEW_BF_VALUE]] acq_rel acquire
+// CHECK: [[RES:%.+]] = cmpxchg i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2), i8 [[OLD_BF_VALUE]], i8 [[NEW_BF_VALUE]] monotonic monotonic
 // CHECK: [[FAILED_OLD_VAL]] = extractvalue { i8, i1 } [[RES]], 0
 // CHECK: [[FAIL_SUCCESS:%.+]] = extractvalue { i8, i1 } [[RES]], 1
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
-// CHECK: call{{.*}} @__kmpc_flush(
-#pragma omp atomic update acq_rel
+#pragma omp atomic update relaxed
   bfx4_packed.b += ldv;
 // CHECK: load i64, i64*
 // CHECK: [[EXPR:%.+]] = uitofp i64 %{{.+}} to float
-// CHECK: [[I64VAL:%.+]] = load atomic i64, i64* bitcast (<2 x float>* [[DEST:@.+]] to i64*) acquire
+// CHECK: [[I64VAL:%.+]] = load atomic i64, i64* bitcast (<2 x float>* [[DEST:@.+]] to i64*) monotonic
 // CHECK: br label %[[CONT:.+]]
 // CHECK: [[CONT]]
 // CHECK: [[OLD_I64:%.+]] = phi i64 [ [[I64VAL]], %{{.+}} ], [ [[FAILED_I64_OLD_VAL:%.+]], %[[CONT]] ]
@@ -886,13 +885,12 @@ int main() {
 // CHECK: [[NEW_VEC_VAL:%.+]] = insertelement <2 x float> [[VEC_VAL]], float [[VEC_ITEM_VAL]], i64 0
 // CHECK: store <2 x float> [[NEW_VEC_VAL]], <2 x float>* [[TEMP]]
 // CHECK: [[NEW_I64:%.+]] = load i64, i64* [[BITCAST]]
-// CHECK: [[RES:%.+]] = cmpxchg i64* bitcast (<2 x float>* [[DEST]] to i64*), i64 [[OLD_I64]], i64 [[NEW_I64]] acq_rel acquire
+// CHECK: [[RES:%.+]] = cmpxchg i64* bitcast (<2 x float>* [[DEST]] to i64*), i64 [[OLD_I64]], i64 [[NEW_I64]] monotonic monotonic
 // CHECK: [[FAILED_I64_OLD_VAL:%.+]] = extractvalue { i64, i1 } [[RES]], 0
 // CHECK: [[FAIL_SUCCESS:%.+]] = extractvalue { i64, i1 } [[RES]], 1
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
-// CHECK: call{{.*}} @__kmpc_flush(
-#pragma omp atomic acq_rel
+#pragma omp atomic relaxed
   float2x.x = ulv - float2x.x;
 // CHECK: [[EXPR:%.+]] = load double, double* @{{.+}},
 // CHECK: [[OLD_VAL:%.+]] = call i32 @llvm.read_register.i32([[REG:metadata ![0-9]+]])
diff --git a/clang/test/OpenMP/atomic_write_codegen.c b/clang/test/OpenMP/atomic_write_codegen.c
index f8a74e8c6edd27f095e7083938c3318a018c07a1..3cbaf2752448b1112b67093eb87bfa6e26fa81b8 100644
--- a/clang/test/OpenMP/atomic_write_codegen.c
+++ b/clang/test/OpenMP/atomic_write_codegen.c
@@ -88,7 +88,7 @@ int main() {
  __imag(civ) = 1;
 // CHECK: load i8, i8*
 // CHECK: store atomic i8{{.*}}monotonic
-#pragma omp atomic write acquire
+#pragma omp atomic write
   bx = bv;
 // CHECK: load i8, i8*
 // CHECK: store atomic i8{{.*}}release
@@ -470,7 +470,7 @@ int main() {
   bfx4.b = ldv;
 // CHECK: load x86_fp80, x86_fp80* @{{.+}}
 // CHECK: [[NEW_VAL:%.+]] = fptosi x86_fp80 %{{.+}} to i64
-// CHECK: [[PREV_VALUE:%.+]] = load atomic i8, i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2) acquire
+// CHECK: [[PREV_VALUE:%.+]] = load atomic i8, i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2) monotonic
 // CHECK: br label %[[CONT:.+]]
 // CHECK: [[CONT]]
 // CHECK: [[OLD_BF_VALUE:%.+]] = phi i8 [ [[PREV_VALUE]], %[[EXIT]] ], [ [[FAILED_OLD_VAL:%.+]], %[[CONT]] ]
@@ -481,16 +481,16 @@ int main() {
 // CHECK: or i8 [[BF_CLEAR]], [[BF_VALUE]]
 // CHECK: store i8 %{{.+}}, i8* [[LDTEMP:%.+]]
 // CHECK: [[NEW_BF_VALUE:%.+]] = load i8, i8* [[LDTEMP]]
-// CHECK: [[RES:%.+]] = cmpxchg i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2), i8 [[OLD_BF_VALUE]], i8 [[NEW_BF_VALUE]] acquire acquire
+// CHECK: [[RES:%.+]] = cmpxchg i8* getelementptr inbounds (%struct.BitFields4_packed, %struct.BitFields4_packed* @{{.+}}, i32 0, i32 0, i64 2), i8 [[OLD_BF_VALUE]], i8 [[NEW_BF_VALUE]] monotonic monotonic
 // CHECK: [[FAILED_OLD_VAL]] = extractvalue { i8, i1 } [[RES]], 0
 // CHECK: [[FAIL_SUCCESS:%.+]] = extractvalue { i8, i1 } [[RES]], 1
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
-#pragma omp atomic write acquire
+#pragma omp atomic relaxed write
   bfx4_packed.b = ldv;
 // CHECK: load i64, i64*
 // CHECK: [[VEC_ITEM_VAL:%.+]] = uitofp i64 %{{.+}} to float
-// CHECK: [[I64VAL:%.+]] = load atomic i64, i64* bitcast (<2 x float>* [[DEST:@.+]] to i64*) acquire
+// CHECK: [[I64VAL:%.+]] = load atomic i64, i64* bitcast (<2 x float>* [[DEST:@.+]] to i64*) monotonic
 // CHECK: br label %[[CONT:.+]]
 // CHECK: [[CONT]]
 // CHECK: [[OLD_I64:%.+]] = phi i64 [ [[I64VAL]], %{{.+}} ], [ [[FAILED_I64_OLD_VAL:%.+]], %[[CONT]] ]
@@ -500,13 +500,12 @@ int main() {
 // CHECK: [[NEW_VEC_VAL:%.+]] = insertelement <2 x float> [[VEC_VAL]], float [[VEC_ITEM_VAL]], i64 0
 // CHECK: store <2 x float> [[NEW_VEC_VAL]], <2 x float>* [[LDTEMP]]
 // CHECK: [[NEW_I64:%.+]] = load i64, i64* [[BITCAST]]
-// CHECK: [[RES:%.+]] = cmpxchg i64* bitcast (<2 x float>* [[DEST]] to i64*), i64 [[OLD_I64]], i64 [[NEW_I64]] acq_rel acquire
+// CHECK: [[RES:%.+]] = cmpxchg i64* bitcast (<2 x float>* [[DEST]] to i64*), i64 [[OLD_I64]], i64 [[NEW_I64]] monotonic monotonic
 // CHECK: [[FAILED_I64_OLD_VAL:%.+]] = extractvalue { i64, i1 } [[RES]], 0
 // CHECK: [[FAIL_SUCCESS:%.+]] = extractvalue { i64, i1 } [[RES]], 1
 // CHECK: br i1 [[FAIL_SUCCESS]], label %[[EXIT:.+]], label %[[CONT]]
 // CHECK: [[EXIT]]
-// CHECK: call{{.*}} @__kmpc_flush(
-#pragma omp atomic write acq_rel
+#pragma omp atomic write relaxed
   float2x.x = ulv;
 // CHECK: call i32 @llvm.read_register.i32(
 // CHECK: sitofp i32 %{{.+}} to double
diff --git a/clang/test/OpenMP/flush_messages.cpp b/clang/test/OpenMP/flush_messages.cpp
index 48a70f3f8dcb5364fbfea5d34901cf028cbe2718..51497249a8f386dae6eb2307c6accdaf05970303 100644
--- a/clang/test/OpenMP/flush_messages.cpp
+++ b/clang/test/OpenMP/flush_messages.cpp
@@ -138,6 +138,8 @@ label1 : {
 #pragma omp flush acq_rel // omp45-error {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp flush'}}
 #pragma omp flush acquire // omp45-error {{unexpected OpenMP clause 'acquire' in directive '#pragma omp flush'}}
 #pragma omp flush release // omp45-error {{unexpected OpenMP clause 'release' in directive '#pragma omp flush'}}
+#pragma omp flush relaxed // expected-error {{unexpected OpenMP clause 'relaxed' in directive '#pragma omp flush'}}
+#pragma omp flush seq_cst // expected-error {{unexpected OpenMP clause 'seq_cst' in directive '#pragma omp flush'}}
 #pragma omp flush acq_rel acquire // omp45-error {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp flush'}} omp45-error {{unexpected OpenMP clause 'acquire' in directive '#pragma omp flush'}} omp50-error {{directive '#pragma omp flush' cannot contain more than one 'acq_rel', 'acquire' or 'release' clause}} omp50-note {{'acq_rel' clause used here}}
 #pragma omp flush release acquire // omp45-error {{unexpected OpenMP clause 'release' in directive '#pragma omp flush'}} omp45-error {{unexpected OpenMP clause 'acquire' in directive '#pragma omp flush'}} omp50-error {{directive '#pragma omp flush' cannot contain more than one 'acq_rel', 'acquire' or 'release' clause}} omp50-note {{'release' clause used here}}
 #pragma omp flush acq_rel (argc) // omp45-error {{unexpected OpenMP clause 'acq_rel' in directive '#pragma omp flush'}} omp50-error {{'flush' directive with memory order clause 'acq_rel' cannot have the list}} omp50-note {{memory order clause 'acq_rel' is specified here}}
diff --git a/clang/test/OpenMP/requires_acq_rel_codegen.cpp b/clang/test/OpenMP/requires_acq_rel_codegen.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b8ba01b0cafe025508344c7166cc227580839b12
--- /dev/null
+++ b/clang/test/OpenMP/requires_acq_rel_codegen.cpp
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 %s -triple x86_64-apple-darwin10 -x c++ -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s -triple x86_64-apple-darwin10
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -verify %s -triple x86_64-apple-darwin10 -x c++ -emit-llvm -o -| FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd %s -fopenmp-version=50 -x c++ -emit-llvm -triple x86_64-apple-darwin10 -o -| FileCheck %s --check-prefix SIMD-ONLY0
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s -triple x86_64-apple-darwin10
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -verify %s -emit-llvm -x c++ -emit-llvm -triple x86_64-apple-darwin10 -o -| FileCheck %s --check-prefix SIMD-ONLY0
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+#pragma omp requires atomic_default_mem_order(acq_rel)
+
+// CHECK-LABEL: foo
+void foo() {
+  int a = 0, b = 0;
+// CHECK: load atomic i32,{{.*}}acquire
+#pragma omp atomic read
+  a = b;
+// CHECK: store atomic i32{{.*}}release
+#pragma omp atomic write
+  a = b;
+// CHECK: atomicrmw add i32{{.*}}release
+#pragma omp atomic
+  a += 1;
+// CHECK: atomicrmw add i32{{.*}}release
+#pragma omp atomic update
+  a += 1;
+// CHECK: atomicrmw add i32{{.*}}acq_rel
+#pragma omp atomic capture
+  {
+    b = a;
+    a += 1;
+  }
+}
+
+#endif
diff --git a/clang/test/OpenMP/requires_default_atomic_mem_order_messages.cpp b/clang/test/OpenMP/requires_default_atomic_mem_order_messages.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..19f6ede043d85fcda073815020bb6b44c49448a0
--- /dev/null
+++ b/clang/test/OpenMP/requires_default_atomic_mem_order_messages.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100  %s
+
+void foo2() {
+  int a;
+  #pragma omp atomic update // expected-note 3 {{'atomic' previously encountered here}}
+    a = a + 1;
+}
+
+#pragma omp requires atomic_default_mem_order(seq_cst) // expected-error {{'atomic' region encountered before requires directive with 'atomic_default_mem_order' clause}} expected-note 2 {{atomic_default_mem_order clause previously used here}}
+#pragma omp requires atomic_default_mem_order(acq_rel) // expected-error {{'atomic' region encountered before requires directive with 'atomic_default_mem_order' clause}} expected-error {{Only one atomic_default_mem_order clause can appear on a requires directive in a single translation unit}}
+#pragma omp requires atomic_default_mem_order(relaxed) // expected-error {{'atomic' region encountered before requires directive with 'atomic_default_mem_order' clause}} expected-error {{Only one atomic_default_mem_order clause can appear on a requires directive in a single translation unit}}
+#pragma omp requires atomic_default_mem_order(release) // expected-error {{expected 'seq_cst', 'acq_rel' or 'relaxed' in OpenMP clause 'atomic_default_mem_order'}} expected-error {{expected at least one clause on '#pragma omp requires' directive}}
diff --git a/clang/test/OpenMP/requires_relaxed_codegen.cpp b/clang/test/OpenMP/requires_relaxed_codegen.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e92b55e8f1379f35203a254c7083278f8196867d
--- /dev/null
+++ b/clang/test/OpenMP/requires_relaxed_codegen.cpp
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 %s -triple x86_64-apple-darwin10 -x c++ -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s -triple x86_64-apple-darwin10
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -verify %s -triple x86_64-apple-darwin10 -x c++ -emit-llvm -o -| FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd %s -fopenmp-version=50 -x c++ -emit-llvm -triple x86_64-apple-darwin10 -o -| FileCheck %s --check-prefix SIMD-ONLY0
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s -triple x86_64-apple-darwin10
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -verify %s -emit-llvm -x c++ -emit-llvm -triple x86_64-apple-darwin10 -o -| FileCheck %s --check-prefix SIMD-ONLY0
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+#pragma omp requires atomic_default_mem_order(relaxed)
+
+// CHECK-LABEL: foo
+void foo() {
+  int a = 0, b = 0;
+// CHECK: load atomic i32,{{.*}}monotonic
+#pragma omp atomic read
+  a = b;
+// CHECK: store atomic i32{{.*}}monotonic
+#pragma omp atomic write
+  a = b;
+// CHECK: atomicrmw add i32{{.*}}monotonic
+#pragma omp atomic
+  a += 1;
+// CHECK: atomicrmw add i32{{.*}}monotonic
+#pragma omp atomic update
+  a += 1;
+// CHECK: atomicrmw add i32{{.*}}monotonic
+#pragma omp atomic capture
+  {
+    b = a;
+    a += 1;
+  }
+}
+
+#endif
diff --git a/clang/test/OpenMP/requires_seq_cst_codegen.cpp b/clang/test/OpenMP/requires_seq_cst_codegen.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c2f02665d153eb3fc64ed9ac19061593d63102a7
--- /dev/null
+++ b/clang/test/OpenMP/requires_seq_cst_codegen.cpp
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 %s -triple x86_64-apple-darwin10 -x c++ -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s -triple x86_64-apple-darwin10
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -std=c++11 -include-pch %t -verify %s -triple x86_64-apple-darwin10 -x c++ -emit-llvm -o -| FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd %s -fopenmp-version=50 -x c++ -emit-llvm -triple x86_64-apple-darwin10 -o -| FileCheck %s --check-prefix SIMD-ONLY0
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -emit-pch -o %t %s -triple x86_64-apple-darwin10
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -std=c++11 -include-pch %t -verify %s -emit-llvm -x c++ -emit-llvm -triple x86_64-apple-darwin10 -o -| FileCheck %s --check-prefix SIMD-ONLY0
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+#pragma omp requires atomic_default_mem_order(seq_cst)
+
+// CHECK-LABEL: foo
+void foo() {
+  int a = 0, b = 0;
+// CHECK: load atomic i32,{{.*}}seq_cst
+#pragma omp atomic read
+  a = b;
+// CHECK: store atomic i32{{.*}}seq_cst
+#pragma omp atomic write
+  a = b;
+// CHECK: atomicrmw add i32{{.*}}seq_cst
+#pragma omp atomic
+  a += 1;
+// CHECK: atomicrmw add i32{{.*}}seq_cst
+#pragma omp atomic update
+  a += 1;
+// CHECK: atomicrmw add i32{{.*}}seq_cst
+#pragma omp atomic capture
+  {
+    b = a;
+    a += 1;
+  }
+}
+
+#endif
diff --git a/clang/test/OpenMP/requires_target_messages.cpp b/clang/test/OpenMP/requires_target_messages.cpp
index ef65d98fed9c017bed921926d54521c7e8cee1b5..93f318ea1bb739da47c4d4cf3440b83d41e2c6d0 100644
--- a/clang/test/OpenMP/requires_target_messages.cpp
+++ b/clang/test/OpenMP/requires_target_messages.cpp
@@ -2,14 +2,14 @@
 
 void foo2() {
   int a;
-  #pragma omp target // expected-note 4 {{target previously encountered here}}
+  #pragma omp target // expected-note 4 {{'target' previously encountered here}}
   {
     a = a + 1;
   }
 }
 
 #pragma omp requires atomic_default_mem_order(seq_cst)
-#pragma omp requires unified_address //expected-error {{target region encountered before requires directive with 'unified_address' clause}}
-#pragma omp requires unified_shared_memory //expected-error {{target region encountered before requires directive with 'unified_shared_memory' clause}}
-#pragma omp requires reverse_offload //expected-error {{target region encountered before requires directive with 'reverse_offload' clause}}
-#pragma omp requires dynamic_allocators //expected-error {{target region encountered before requires directive with 'dynamic_allocators' clause}}
+#pragma omp requires unified_address //expected-error {{'target' region encountered before requires directive with 'unified_address' clause}}
+#pragma omp requires unified_shared_memory //expected-error {{'target' region encountered before requires directive with 'unified_shared_memory' clause}}
+#pragma omp requires reverse_offload //expected-error {{'target' region encountered before requires directive with 'reverse_offload' clause}}
+#pragma omp requires dynamic_allocators //expected-error {{'target' region encountered before requires directive with 'dynamic_allocators' clause}}
diff --git a/clang/test/SemaCXX/conversion.cpp b/clang/test/SemaCXX/conversion.cpp
index dcd64fa2ec8aeaa0ce986864b098ffdd9691b0d8..67bfdf5532b5dc939f5b235d6d792c9522921212 100644
--- a/clang/test/SemaCXX/conversion.cpp
+++ b/clang/test/SemaCXX/conversion.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -Wconversion -std=c++11 -verify %s
-// RUN: %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -Wconversion -std=c++11 %s 2>&1 | FileCheck %s
+// RUN: not %clang_cc1 -triple x86_64-apple-darwin -fsyntax-only -Wconversion -std=c++11 %s 2>&1 | FileCheck %s
 
 #include <stddef.h>
 
@@ -129,9 +129,9 @@ namespace test6 {
 
 namespace test7 {
   bool fun() {
-    bool x = nullptr; // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+    bool x = nullptr; // expected-error {{cannot initialize}}
     if (nullptr) {} // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
-    return nullptr; // expected-warning {{implicit conversion of nullptr constant to 'bool'}}
+    return nullptr; // expected-error {{cannot initialize}}
   }
 }
 
@@ -198,14 +198,12 @@ namespace test8 {
   }
 }
 
-// Don't warn on a nullptr to bool conversion when the nullptr is the return
-// type of a function.
 namespace test9 {
   typedef decltype(nullptr) nullptr_t;
   nullptr_t EXIT();
 
   bool test() {
-    return EXIT();
+    return EXIT(); // expected-error {{cannot initialize}}
   }
 }
 
@@ -273,10 +271,10 @@ void function1(const char* str) {
   CHECK13(check_str_null_13(str));
 }
 
-bool some_bool_function(bool);
+bool some_bool_function(bool); // expected-note {{no known conversion}}
 void function2() {
-  CHECK13(some_bool_function(nullptr));  // expected-warning{{implicit conversion of nullptr constant to 'bool'}}
-  CHECK13(some_bool_function(NULL));  // expected-warning{{implicit conversion of NULL constant to 'bool'}}
+  CHECK13(some_bool_function(nullptr));  // expected-error {{no matching function}}
+  CHECK13(some_bool_function(NULL));  // expected-warning {{implicit conversion of NULL constant to 'bool'}}
 }
 
 #define run_check_nullptr_13(str) \
diff --git a/clang/test/SemaCXX/nullptr.cpp b/clang/test/SemaCXX/nullptr.cpp
index 9a092910b6f9772f4f90e7a2f033758072dc5b85..23ea383c3e87c832c53edbc7bf9a11775d4a42bd 100644
--- a/clang/test/SemaCXX/nullptr.cpp
+++ b/clang/test/SemaCXX/nullptr.cpp
@@ -25,7 +25,7 @@ nullptr_t f(nullptr_t null)
   pf = null;
   void (A::*pmf)() = nullptr;
   pmf = null;
-  bool b = nullptr;
+  bool b = nullptr; // expected-error {{cannot initialize}}
 
   // Can't convert nullptr to integral implicitly.
   uintptr_t i = nullptr; // expected-error {{cannot initialize}}
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index b551e9f4cf829627e2be2d8efca874afae098649..6d1a67f2a4fada07eb2bed5856ef7ca28bcd2312 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -36,6 +36,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
@@ -69,7 +70,7 @@ static void LLVMErrorHandler(void *UserData, const std::string &Message,
   // We cannot recover from llvm errors.  When reporting a fatal error, exit
   // with status 70 to generate crash diagnostics.  For BSD systems this is
   // defined as an internal software error.  Otherwise, exit with status 1.
-  exit(GenCrashDiag ? 70 : 1);
+  llvm::sys::Process::Exit(GenCrashDiag ? 70 : 1);
 }
 
 #ifdef CLANG_HAVE_RLIMITS
diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index ce390b9f3b17fd4f2cec79429440358da5605f7a..110f1dc70223bfd6d0cb4e5a46f5f4aa15f25d2f 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -46,6 +46,7 @@
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TargetRegistry.h"
@@ -555,7 +556,7 @@ static void LLVMErrorHandler(void *UserData, const std::string &Message,
   Diags.Report(diag::err_fe_error_backend) << Message;
 
   // We cannot recover from llvm errors.
-  exit(1);
+  sys::Process::Exit(1);
 }
 
 int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 12479c7abb84e7bab1bb7e5195ee3339f3f77e87..6f32240fe6e48b8ea621c35883f04f1008e51a34 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2241,6 +2241,8 @@ void OMPClauseEnqueue::VisitOMPAcquireClause(const OMPAcquireClause *) {}
 
 void OMPClauseEnqueue::VisitOMPReleaseClause(const OMPReleaseClause *) {}
 
+void OMPClauseEnqueue::VisitOMPRelaxedClause(const OMPRelaxedClause *) {}
+
 void OMPClauseEnqueue::VisitOMPThreadsClause(const OMPThreadsClause *) {}
 
 void OMPClauseEnqueue::VisitOMPSIMDClause(const OMPSIMDClause *) {}
diff --git a/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp b/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
index 510ade453b0a06e92581ccbcb2fd77a7cc64b10b..650c0fc497644a148d9018aa1b8ce868133cac12 100644
--- a/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
+++ b/clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
@@ -449,3 +449,42 @@ TEST(DirectoryWatcherTest, InvalidatedWatcher) {
 
   checkEventualResultWithTimeout(TestConsumer);
 }
+
+TEST(DirectoryWatcherTest, InvalidatedWatcherAsync) {
+  DirectoryWatcherTestFixture fixture;
+  fixture.addFile("a");
+
+  // This test is checking that we get the initial notification for 'a' before
+  // the final 'invalidated'. Strictly speaking, we do not care whether 'a' is
+  // processed or not, only that it is neither racing with, nor after
+  // 'invalidated'. In practice, it is always processed in our implementations.
+  VerifyingConsumer TestConsumer{
+      {{EventKind::Modified, "a"}},
+      {{EventKind::WatcherGotInvalidated, ""}},
+      // We have to ignore these as it's a race between the test process
+      // which is scanning the directory and kernel which is sending
+      // notification.
+      {{EventKind::Modified, "a"}},
+  };
+
+  // A counter that can help detect data races on the event receiver,
+  // particularly if used with TSan. Expected count will be 2 or 3 depending on
+  // whether we get the kernel event or not (see comment above).
+  unsigned Count = 0;
+  {
+    llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
+        DirectoryWatcher::create(
+            fixture.TestWatchedDir,
+            [&TestConsumer,
+             &Count](llvm::ArrayRef<DirectoryWatcher::Event> Events,
+                     bool IsInitial) {
+              Count += 1;
+              TestConsumer.consume(Events, IsInitial);
+            },
+            /*waitForInitialSync=*/false);
+    ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
+  } // DW is destructed here.
+
+  checkEventualResultWithTimeout(TestConsumer);
+  ASSERT_TRUE(Count == 2u || Count == 3u);
+}
diff --git a/clang/unittests/Tooling/ASTSelectionTest.cpp b/clang/unittests/Tooling/ASTSelectionTest.cpp
index 7ad5148213ce3d88b500be9e535e6189b5f76ca1..88988ef4478753e6c95f925ca9cc9ad882588553 100644
--- a/clang/unittests/Tooling/ASTSelectionTest.cpp
+++ b/clang/unittests/Tooling/ASTSelectionTest.cpp
@@ -101,22 +101,22 @@ void checkDeclName(const SelectedASTNode &Node, StringRef Name) {
 }
 
 template <typename T>
-const SelectedASTNode &
-checkNode(const SelectedASTNode &StmtNode, SourceSelectionKind SelectionKind,
-          unsigned NumChildren = 0,
-          typename std::enable_if<std::is_base_of<Stmt, T>::value, T>::type
-              *StmtOverloadChecker = nullptr) {
+const SelectedASTNode &checkNode(
+    const SelectedASTNode &StmtNode, SourceSelectionKind SelectionKind,
+    unsigned NumChildren = 0,
+    std::enable_if_t<std::is_base_of<Stmt, T>::value, T> *StmtOverloadChecker =
+        nullptr) {
   checkNodeImpl(isa<T>(StmtNode.Node.get<Stmt>()), StmtNode, SelectionKind,
                 NumChildren);
   return StmtNode;
 }
 
 template <typename T>
-const SelectedASTNode &
-checkNode(const SelectedASTNode &DeclNode, SourceSelectionKind SelectionKind,
-          unsigned NumChildren = 0, StringRef Name = "",
-          typename std::enable_if<std::is_base_of<Decl, T>::value, T>::type
-              *DeclOverloadChecker = nullptr) {
+const SelectedASTNode &checkNode(
+    const SelectedASTNode &DeclNode, SourceSelectionKind SelectionKind,
+    unsigned NumChildren = 0, StringRef Name = "",
+    std::enable_if_t<std::is_base_of<Decl, T>::value, T> *DeclOverloadChecker =
+        nullptr) {
   checkNodeImpl(isa<T>(DeclNode.Node.get<Decl>()), DeclNode, SelectionKind,
                 NumChildren);
   if (!Name.empty())
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index d4c6175a8c1b59263d519364d145a3634637e14f..3e9210f861abfe31e3efe13b045b972fb859c4b6 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -1504,7 +1504,7 @@ accessible?</td>
     <td><a href="https://wg21.link/cwg244">244</a></td>
     <td>CD1</td>
     <td>Destructor lookup</td>
-    <td class="partial" align="center">Partial</td>
+    <td class="unreleased" align="center">Clang 11</td>
   </tr>
   <tr id="245">
     <td><a href="https://wg21.link/cwg245">245</a></td>
@@ -3967,7 +3967,7 @@ and <I>POD class</I></td>
     <td><a href="https://wg21.link/cwg654">654</a></td>
     <td>CD1</td>
     <td>Conversions to and from <TT>nullptr_t</TT></td>
-    <td class="full" align="center">Yes</td>
+    <td class="unreleased" align="center">Superseded by <a href="#1423">1423</a></td>
   </tr>
   <tr id="655">
     <td><a href="https://wg21.link/cwg655">655</a></td>
@@ -8353,7 +8353,7 @@ and <I>POD class</I></td>
     <td><a href="https://wg21.link/cwg1423">1423</a></td>
     <td>CD3</td>
     <td>Convertibility of <TT>nullptr</TT> to <TT>bool</TT></td>
-    <td class="none" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 11</td>
   </tr>
   <tr id="1424">
     <td><a href="https://wg21.link/cwg1424">1424</a></td>
diff --git a/compiler-rt/include/fuzzer/FuzzedDataProvider.h b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
index 3e069eba69b46229aa765d36db84197698a5b42a..b0a37737af75c7d95713b4bb49ff9117a8e35e0f 100644
--- a/compiler-rt/include/fuzzer/FuzzedDataProvider.h
+++ b/compiler-rt/include/fuzzer/FuzzedDataProvider.h
@@ -237,6 +237,18 @@ class FuzzedDataProvider {
     return result + range * ConsumeProbability<T>();
   }
 
+  // Writes |num_bytes| of input data to the given destination pointer. If there
+  // is not enough data left, writes all remaining bytes. Return value is the
+  // number of bytes written.
+  // In general, it's better to avoid using this function, but it may be useful
+  // in cases when it's necessary to fill a certain buffer or object with
+  // fuzzing data.
+  size_t ConsumeData(void *destination, size_t num_bytes) {
+    num_bytes = std::min(num_bytes, remaining_bytes_);
+    CopyAndAdvance(destination, num_bytes);
+    return num_bytes;
+  }
+
   // Reports the remaining bytes available for fuzzed input.
   size_t remaining_bytes() { return remaining_bytes_; }
 
@@ -244,6 +256,11 @@ class FuzzedDataProvider {
   FuzzedDataProvider(const FuzzedDataProvider &) = delete;
   FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete;
 
+  void CopyAndAdvance(void *destination, size_t num_bytes) {
+    std::memcpy(destination, data_ptr_, num_bytes);
+    Advance(num_bytes);
+  }
+
   void Advance(size_t num_bytes) {
     if (num_bytes > remaining_bytes_)
       abort();
@@ -253,7 +270,7 @@ class FuzzedDataProvider {
   }
 
   template <typename T>
-  std::vector<T> ConsumeBytes(size_t size, size_t num_bytes_to_consume) {
+  std::vector<T> ConsumeBytes(size_t size, size_t num_bytes) {
     static_assert(sizeof(T) == sizeof(uint8_t), "Incompatible data type.");
 
     // The point of using the size-based constructor below is to increase the
@@ -264,13 +281,12 @@ class FuzzedDataProvider {
     // To increase the odds even more, we also call |shrink_to_fit| below.
     std::vector<T> result(size);
     if (size == 0) {
-      if (num_bytes_to_consume != 0)
+      if (num_bytes != 0)
         abort();
       return result;
     }
 
-    std::memcpy(result.data(), data_ptr_, num_bytes_to_consume);
-    Advance(num_bytes_to_consume);
+    CopyAndAdvance(result.data(), num_bytes);
 
     // Even though |shrink_to_fit| is also implementation specific, we expect it
     // to provide an additional assurance in case vector's constructor allocated
diff --git a/compiler-rt/lib/builtins/powitf2.c b/compiler-rt/lib/builtins/powitf2.c
index fcbdb4c2ee2a68e5f3ec78da783d2605430039ab..141a3a0ea727f4c711bc2968d7ae132506cbc3ba 100644
--- a/compiler-rt/lib/builtins/powitf2.c
+++ b/compiler-rt/lib/builtins/powitf2.c
@@ -10,9 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "int_lib.h"
+#define QUAD_PRECISION
+#include "fp_lib.h"
 
-#if _ARCH_PPC
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
 
 // Returns: a ^ b
 
diff --git a/compiler-rt/lib/fuzzer/tests/FuzzedDataProviderUnittest.cpp b/compiler-rt/lib/fuzzer/tests/FuzzedDataProviderUnittest.cpp
index 222283434eb8e1b8376cf835cea8d84fadc185c6..5eb46533d98a0d1e9fabd32e24398a36b30a746b 100644
--- a/compiler-rt/lib/fuzzer/tests/FuzzedDataProviderUnittest.cpp
+++ b/compiler-rt/lib/fuzzer/tests/FuzzedDataProviderUnittest.cpp
@@ -399,6 +399,25 @@ TEST(FuzzedDataProvider, ConsumeFloatingPoint) {
                                        -13.37, 31.337));
 }
 
+TEST(FuzzedDataProvider, ConsumeData) {
+  FuzzedDataProvider DataProv(Data, sizeof(Data));
+  uint8_t Buffer[10] = {};
+  EXPECT_EQ(sizeof(Buffer), DataProv.ConsumeData(Buffer, sizeof(Buffer)));
+  std::vector<uint8_t> Expected(Data, Data + sizeof(Buffer));
+  EXPECT_EQ(Expected, std::vector<uint8_t>(Buffer, Buffer + sizeof(Buffer)));
+
+  EXPECT_EQ(size_t(2), DataProv.ConsumeData(Buffer, 2));
+  Expected[0] = Data[sizeof(Buffer)];
+  Expected[1] = Data[sizeof(Buffer) + 1];
+  EXPECT_EQ(Expected, std::vector<uint8_t>(Buffer, Buffer + sizeof(Buffer)));
+
+  // Exhaust the buffer.
+  EXPECT_EQ(std::vector<uint8_t>(Data + 12, Data + sizeof(Data)),
+            DataProv.ConsumeRemainingBytes<uint8_t>());
+  EXPECT_EQ(size_t(0), DataProv.ConsumeData(Buffer, sizeof(Buffer)));
+  EXPECT_EQ(Expected, std::vector<uint8_t>(Buffer, Buffer + sizeof(Buffer)));
+}
+
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
diff --git a/compiler-rt/lib/scudo/standalone/size_class_map.h b/compiler-rt/lib/scudo/standalone/size_class_map.h
index 46f53ae51fbaaf12549f2118023ce593c74c4ddb..3bbd165289e6c1b78d3577c69e4d81fc07f6c0bb 100644
--- a/compiler-rt/lib/scudo/standalone/size_class_map.h
+++ b/compiler-rt/lib/scudo/standalone/size_class_map.h
@@ -24,7 +24,6 @@ inline uptr scaledLog2(uptr Size, uptr ZeroLog, uptr LogBits) {
 
 template <typename Config> struct SizeClassMapBase {
   static u32 getMaxCachedHint(uptr Size) {
-    DCHECK_LE(Size, MaxSize);
     DCHECK_NE(Size, 0);
     u32 N;
     // Force a 32-bit division if the template parameters allow for it.
@@ -95,10 +94,17 @@ public:
       return (Size + MinSize - 1) >> Config::MinSizeLog;
     return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S);
   }
+
+  static u32 getMaxCachedHint(uptr Size) {
+    DCHECK_LE(Size, MaxSize);
+    return Base::getMaxCachedHint(Size);
+  }
 };
 
 template <typename Config>
 class TableSizeClassMap : public SizeClassMapBase<Config> {
+  typedef SizeClassMapBase<Config> Base;
+
   static const u8 S = Config::NumBits - 1;
   static const uptr M = (1UL << S) - 1;
   static const uptr ClassesSize =
@@ -119,9 +125,9 @@ class TableSizeClassMap : public SizeClassMapBase<Config> {
     constexpr static u8 computeClassId(uptr Size) {
       for (uptr i = 0; i != ClassesSize; ++i) {
         if (Size <= Config::Classes[i])
-          return i + 1;
+          return static_cast<u8>(i + 1);
       }
-      return -1;
+      return static_cast<u8>(-1);
     }
 
     constexpr static uptr getTableSize() {
@@ -156,8 +162,10 @@ public:
     return Table.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
   }
 
-  static void print() {}
-  static void validate() {}
+  static u32 getMaxCachedHint(uptr Size) {
+    DCHECK_LE(Size, MaxSize);
+    return Base::getMaxCachedHint(Size);
+  }
 };
 
 struct AndroidSizeClassConfig {
diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
index d4ba7d7138abb863ca435c62bd113bc6a4b1e967..8b2bc6ecbd5b607b0360e723529618593cc44a4e 100644
--- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp
@@ -268,10 +268,26 @@ TEST(ScudoWrappersCTest, MallocIterateBoundary) {
   const size_t BlockDelta = FIRST_32_SECOND_64(8U, 16U);
   const size_t SpecialSize = PageSize - BlockDelta;
 
-  void *P = malloc(SpecialSize);
-  EXPECT_NE(P, nullptr);
-  BoundaryP = reinterpret_cast<uintptr_t>(P);
-  const uintptr_t Block = BoundaryP - BlockDelta;
+  // We aren't guaranteed that any size class is exactly a page wide. So we need
+  // to keep making allocations until we succeed.
+  //
+  // With a 16-byte block alignment and 4096-byte page size, each allocation has
+  // a probability of (1 - (16/4096)) of failing to meet the alignment
+  // requirements, and the probability of failing 65536 times is
+  // (1 - (16/4096))^65536 < 10^-112. So if we still haven't succeeded after
+  // 65536 tries, give up.
+  uintptr_t Block;
+  void *P = nullptr;
+  for (unsigned I = 0; I != 65536; ++I) {
+    void *PrevP = P;
+    P = malloc(SpecialSize);
+    EXPECT_NE(P, nullptr);
+    *reinterpret_cast<void **>(P) = PrevP;
+    BoundaryP = reinterpret_cast<uintptr_t>(P);
+    Block = BoundaryP - BlockDelta;
+    if ((Block & (PageSize - 1)) == 0U)
+      break;
+  }
   EXPECT_EQ((Block & (PageSize - 1)), 0U);
 
   Count = 0U;
@@ -281,7 +297,11 @@ TEST(ScudoWrappersCTest, MallocIterateBoundary) {
   malloc_enable();
   EXPECT_EQ(Count, 1U);
 
-  free(P);
+  while (P) {
+    void *NextP = *reinterpret_cast<void **>(P);
+    free(P);
+    P = NextP;
+  }
 }
 
 // We expect heap operations within a disable/enable scope to deadlock.
diff --git a/compiler-rt/lib/scudo/standalone/tsd_shared.h b/compiler-rt/lib/scudo/standalone/tsd_shared.h
index 1626732c70f49e4646eefce2c441532c1ad77865..cf5453d202086f758851de8fc1bdf6006976e033 100644
--- a/compiler-rt/lib/scudo/standalone/tsd_shared.h
+++ b/compiler-rt/lib/scudo/standalone/tsd_shared.h
@@ -79,7 +79,7 @@ template <class Allocator, u32 MaxTSDCount> struct TSDRegistrySharedT {
   }
 
   void enable() {
-    for (s32 I = NumberOfTSDs - 1; I >= 0; I--)
+    for (s32 I = static_cast<s32>(NumberOfTSDs - 1); I >= 0; I--)
       TSDs[I].unlock();
     Mutex.unlock();
   }
diff --git a/compiler-rt/test/builtins/Unit/powitf2_test.c b/compiler-rt/test/builtins/Unit/powitf2_test.c
index 70bc6dae0d2de2a9812cfa991a8568ceca5e0169..fd25f38d517741a19194db63a0c58f1b1faf134f 100644
--- a/compiler-rt/test/builtins/Unit/powitf2_test.c
+++ b/compiler-rt/test/builtins/Unit/powitf2_test.c
@@ -14,7 +14,7 @@
 
 #include <stdio.h>
 
-#if _ARCH_PPC
+#if __LDBL_MANT_DIG__ == 113
 
 #include "int_lib.h"
 #include <math.h>
@@ -37,7 +37,7 @@ int test__powitf2(long double a, si_int b, long double expected)
 
 int main()
 {
-#if _ARCH_PPC
+#if __LDBL_MANT_DIG__ == 113
     if (test__powitf2(0, 0, 1))
         return 1;
     if (test__powitf2(1, 0, 1))
diff --git a/libcxx/include/memory b/libcxx/include/memory
index 34c3e0c0d8d1e6bce699b98d47e7bc83e3dde514..821f371eb0b047f45e8ac75ee04d909e50695fd2 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -1695,7 +1695,7 @@ struct _LIBCPP_TEMPLATE_VIS allocator_traits
         static
         typename enable_if
         <
-            is_trivially_move_constructible<_DestTp>::value &&
+            is_trivially_copy_constructible<_DestTp>::value &&
             is_same<_RawSourceTp, _RawDestTp>::value &&
             (__is_default_allocator<allocator_type>::value ||
              !__has_construct<allocator_type, _DestTp*, _SourceTp&>::value),
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp
index 2729e59029bc3a070aa2cad801a6a8b47acee2bc..d5df253603c1a0f716b09ad68a771b0d03906964 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.path/path.itr/iterator_db.pass.cpp
@@ -11,6 +11,9 @@
 
 // MODULES_DEFINES: _LIBCPP_DEBUG=0
 
+// This test requires debug mode, which the library on macOS doesn't have.
+// UNSUPPORTED: with_system_cxx_lib=macosx
+
 // <filesystem>
 
 // class path
diff --git a/libcxx/test/libcxx/thread/futures/futures.promise/set_exception.pass.cpp b/libcxx/test/libcxx/thread/futures/futures.promise/set_exception.pass.cpp
index b3621a4e697d0bfd7e69fe62c66199d9f5f06f72..37ef0d1f04be8b4c88822805252f7380632485de 100644
--- a/libcxx/test/libcxx/thread/futures/futures.promise/set_exception.pass.cpp
+++ b/libcxx/test/libcxx/thread/futures/futures.promise/set_exception.pass.cpp
@@ -12,8 +12,8 @@
 
 // MODULES_DEFINES: _LIBCPP_DEBUG=0
 
-// Can't test the system lib because this test enables debug mode
-// UNSUPPORTED: with_system_cxx_lib
+// This test requires debug mode, which the library on macOS doesn't have.
+// UNSUPPORTED: with_system_cxx_lib=macosx
 
 // <future>
 
diff --git a/libcxx/test/libcxx/thread/futures/futures.promise/set_exception_at_thread_exit.pass.cpp b/libcxx/test/libcxx/thread/futures/futures.promise/set_exception_at_thread_exit.pass.cpp
index 49a41362db9a5f00f83fc77fdad34a8da86419be..54778777056c4b030b35aea213557e0c0ff98612 100644
--- a/libcxx/test/libcxx/thread/futures/futures.promise/set_exception_at_thread_exit.pass.cpp
+++ b/libcxx/test/libcxx/thread/futures/futures.promise/set_exception_at_thread_exit.pass.cpp
@@ -12,8 +12,8 @@
 
 // MODULES_DEFINES: _LIBCPP_DEBUG=0
 
-// Can't test the system lib because this test enables debug mode
-// UNSUPPORTED: with_system_cxx_lib
+// This test requires debug mode, which the library on macOS doesn't have.
+// UNSUPPORTED: with_system_cxx_lib=macosx
 
 // <future>
 
diff --git a/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.fail.cpp b/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.fail.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b38f24c3be2547b626f1d8ee9a2fe20406fdb48a
--- /dev/null
+++ b/libcxx/test/std/containers/sequences/vector/vector.cons/copy.move_only.fail.cpp
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure that a std::vector containing move-only types can't be copied.
+
+// UNSUPPORTED: c++98, c++03
+// REQUIRES: verify-support
+
+#include <vector>
+
+struct move_only
+{
+    move_only() = default;
+    move_only(move_only&&) = default;
+    move_only& operator=(move_only&&) = default;
+};
+
+int main(int, char**)
+{
+    std::vector<move_only> v;
+    std::vector<move_only> copy = v; // expected-error@memory:* {{call to implicitly-deleted copy constructor of 'move_only'}}
+    return 0;
+}
diff --git a/libcxx/test/std/language.support/support.types/nullptr_t.pass.cpp b/libcxx/test/std/language.support/support.types/nullptr_t.pass.cpp
index 14ab3c65d02451ccdb7b26e7cbbd154ed353b2d9..b12e5c00638bf34d25af4846774c69129cf1ae55 100644
--- a/libcxx/test/std/language.support/support.types/nullptr_t.pass.cpp
+++ b/libcxx/test/std/language.support/support.types/nullptr_t.pass.cpp
@@ -56,14 +56,6 @@ void test_comparisons()
 #pragma clang diagnostic ignored "-Wnull-conversion"
 #endif
 void test_nullptr_conversions() {
-// GCC does not accept this due to CWG Defect #1423
-// http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#1423
-#if defined(__clang__)
-    {
-        bool b = nullptr;
-        assert(!b);
-    }
-#endif
     {
         bool b(nullptr);
         assert(!b);
diff --git a/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll b/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..839890ee1d9a7b8c4a9f064878b26f9c92419ac8
--- /dev/null
+++ b/lld/test/wasm/lto/Inputs/libcall-truncsfhf2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define half @__truncsfhf2(float) {
+  ret half 0.0
+}
diff --git a/lld/test/wasm/lto/libcall-truncsfhf2.ll b/lld/test/wasm/lto/libcall-truncsfhf2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a427acd139d8e7a1c682c2321cf7e4526a8767b5
--- /dev/null
+++ b/lld/test/wasm/lto/libcall-truncsfhf2.ll
@@ -0,0 +1,20 @@
+; RUN: llvm-as %s -o %t.o
+; RUN: llvm-as %p/Inputs/libcall-truncsfhf2.ll -o %t.truncsfhf2.o
+; RUN: rm -f %t.a
+; RUN: llvm-ar rcs %t.a %t.truncsfhf2.o
+; RUN: not wasm-ld --export-all %t.o %t.a -o %t.wasm 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+@g_float = global float 0.0
+@g_half = global half 0.0
+
+define void @_start() {
+  %val1 = load float, float* @g_float
+  %v0 = fptrunc float %val1 to half
+  store half %v0, half* @g_half
+  ret void
+}
+
+; CHECK: wasm-ld: error: {{.*}}truncsfhf2.o: attempt to add bitcode file after LTO.
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 2d96062739dc8114d79f298573e63c6a399751b6..d71a2b4f083da33eb69b1c6a888ccfe1f07a4ec4 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -536,7 +536,15 @@ static Symbol *createBitcodeSymbol(const std::vector<bool> &keptComdats,
   return symtab->addDefinedData(name, flags, &f, nullptr, 0, 0);
 }
 
+bool BitcodeFile::doneLTO = false;
+
 void BitcodeFile::parse() {
+  if (doneLTO) {
+    error(toString(mb.getBufferIdentifier()) +
+          ": attempt to add bitcode file after LTO.");
+    return;
+  }
+
   obj = check(lto::InputFile::create(MemoryBufferRef(
       mb.getBuffer(), saver.save(archiveName + mb.getBufferIdentifier()))));
   Triple t(obj->getTargetTriple());
diff --git a/lld/wasm/InputFiles.h b/lld/wasm/InputFiles.h
index d5aebf46b47741fc0c08ef8f97d0bff696f3c1a8..bf4a4ec99abf55d4e84961c58b38957e5e6f6326 100644
--- a/lld/wasm/InputFiles.h
+++ b/lld/wasm/InputFiles.h
@@ -160,6 +160,10 @@ public:
 
   void parse();
   std::unique_ptr<llvm::lto::InputFile> obj;
+
+  // Set to true once LTO is complete in order prevent further bitcode objects
+  // being added.
+  static bool doneLTO;
 };
 
 inline bool isBitcode(MemoryBufferRef mb) {
diff --git a/lld/wasm/SymbolTable.cpp b/lld/wasm/SymbolTable.cpp
index 3c1acbd2c68fd5c5addcea5a1a7e14983aa9d60b..ce74b6ad7ea54e9f5178005a933e6ad961f4b37e 100644
--- a/lld/wasm/SymbolTable.cpp
+++ b/lld/wasm/SymbolTable.cpp
@@ -65,6 +65,9 @@ void SymbolTable::addFile(InputFile *file) {
 // Because all bitcode files that the program consists of are passed
 // to the compiler at once, it can do whole-program optimization.
 void SymbolTable::addCombinedLTOObject() {
+  // Prevent further LTO objects being included
+  BitcodeFile::doneLTO = true;
+
   if (bitcodeFiles.empty())
     return;
 
diff --git a/lldb/include/lldb/DataFormatters/FormatClasses.h b/lldb/include/lldb/DataFormatters/FormatClasses.h
index 344a2d70ac536c7e139e05ee84cac409c4fb8467..41b9539803aaae56aaf3833ecabefb9847b54794 100644
--- a/lldb/include/lldb/DataFormatters/FormatClasses.h
+++ b/lldb/include/lldb/DataFormatters/FormatClasses.h
@@ -126,7 +126,7 @@ public:
 
   TypeNameSpecifierImpl(CompilerType type) : m_is_regex(false), m_type() {
     if (type.IsValid()) {
-      m_type.m_type_name.assign(type.GetConstTypeName().GetCString());
+      m_type.m_type_name.assign(type.GetTypeName().GetCString());
       m_type.m_compiler_type = type;
     }
   }
diff --git a/lldb/include/lldb/Interpreter/CommandCompletions.h b/lldb/include/lldb/Interpreter/CommandCompletions.h
index 275cc7e7c145d5c3d941cd2f19da304333e9781b..a678a1173c36433d0f2e707011dacb303018bf2f 100644
--- a/lldb/include/lldb/Interpreter/CommandCompletions.h
+++ b/lldb/include/lldb/Interpreter/CommandCompletions.h
@@ -23,13 +23,6 @@ namespace lldb_private {
 class TildeExpressionResolver;
 class CommandCompletions {
 public:
-  // This is the command completion callback that is used to complete the
-  // argument of the option it is bound to (in the OptionDefinition table
-  // below).  Return the total number of matches.
-  typedef void (*CompletionCallback)(CommandInterpreter &interpreter,
-                                     CompletionRequest &request,
-                                     // A search filter to limit the search...
-                                     lldb_private::SearchFilter *searcher);
   enum CommonCompletionTypes {
     eNoCompletion = 0u,
     eSourceFileCompletion = (1u << 0),
@@ -47,11 +40,6 @@ public:
     eCustomCompletion = (1u << 9)
   };
 
-  struct CommonCompletionElement {
-    uint32_t type;
-    CompletionCallback callback;
-  };
-
   static bool InvokeCommonCompletionCallbacks(
       CommandInterpreter &interpreter, uint32_t completion_mask,
       lldb_private::CompletionRequest &request, SearchFilter *searcher);
@@ -93,98 +81,6 @@ public:
 
   static void VariablePath(CommandInterpreter &interpreter,
                            CompletionRequest &request, SearchFilter *searcher);
-
-  // The Completer class is a convenient base class for building searchers that
-  // go along with the SearchFilter passed to the standard Completer functions.
-  class Completer : public Searcher {
-  public:
-    Completer(CommandInterpreter &interpreter, CompletionRequest &request);
-
-    ~Completer() override;
-
-    CallbackReturn SearchCallback(SearchFilter &filter, SymbolContext &context,
-                                  Address *addr) override = 0;
-
-    lldb::SearchDepth GetDepth() override = 0;
-
-    virtual void DoCompletion(SearchFilter *filter) = 0;
-
-  protected:
-    CommandInterpreter &m_interpreter;
-    CompletionRequest &m_request;
-
-  private:
-    DISALLOW_COPY_AND_ASSIGN(Completer);
-  };
-
-  // SourceFileCompleter implements the source file completer
-  class SourceFileCompleter : public Completer {
-  public:
-    SourceFileCompleter(CommandInterpreter &interpreter,
-                        bool include_support_files, CompletionRequest &request);
-
-    lldb::SearchDepth GetDepth() override;
-
-    Searcher::CallbackReturn SearchCallback(SearchFilter &filter,
-                                            SymbolContext &context,
-                                            Address *addr) override;
-
-    void DoCompletion(SearchFilter *filter) override;
-
-  private:
-    bool m_include_support_files;
-    FileSpecList m_matching_files;
-    const char *m_file_name;
-    const char *m_dir_name;
-
-    DISALLOW_COPY_AND_ASSIGN(SourceFileCompleter);
-  };
-
-  // ModuleCompleter implements the module completer
-  class ModuleCompleter : public Completer {
-  public:
-    ModuleCompleter(CommandInterpreter &interpreter,
-                    CompletionRequest &request);
-
-    lldb::SearchDepth GetDepth() override;
-
-    Searcher::CallbackReturn SearchCallback(SearchFilter &filter,
-                                            SymbolContext &context,
-                                            Address *addr) override;
-
-    void DoCompletion(SearchFilter *filter) override;
-
-  private:
-    const char *m_file_name;
-    const char *m_dir_name;
-
-    DISALLOW_COPY_AND_ASSIGN(ModuleCompleter);
-  };
-
-  // SymbolCompleter implements the symbol completer
-  class SymbolCompleter : public Completer {
-  public:
-    SymbolCompleter(CommandInterpreter &interpreter,
-                    CompletionRequest &request);
-
-    lldb::SearchDepth GetDepth() override;
-
-    Searcher::CallbackReturn SearchCallback(SearchFilter &filter,
-                                            SymbolContext &context,
-                                            Address *addr) override;
-
-    void DoCompletion(SearchFilter *filter) override;
-
-  private:
-    RegularExpression m_regex;
-    typedef std::set<ConstString> collection;
-    collection m_match_set;
-
-    DISALLOW_COPY_AND_ASSIGN(SymbolCompleter);
-  };
-
-private:
-  static CommonCompletionElement g_common_completions[];
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 3885283e8ecf98e7286e12136945b6e78e9c104b..8399166b4dd423fffa3a1daa1172c3a16bb454ce 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -149,10 +149,6 @@ public:
 
   TypeSystem *GetTypeSystem() const { return m_type_system; }
 
-  ConstString GetConstQualifiedTypeName() const;
-
-  ConstString GetConstTypeName() const;
-
   ConstString GetTypeName() const;
 
   ConstString GetDisplayTypeName() const;
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index 811b99028f636fcf738bdc0a82406c625278a631..188b276b7c48582b057d1288723694fbdeb3bd3d 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -199,6 +199,8 @@ public:
 
   virtual ConstString GetTypeName(lldb::opaque_compiler_type_t type) = 0;
 
+  virtual ConstString GetDisplayTypeName(lldb::opaque_compiler_type_t type) = 0;
+
   virtual uint32_t
   GetTypeInfo(lldb::opaque_compiler_type_t type,
               CompilerType *pointee_or_element_compiler_type) = 0;
diff --git a/lldb/packages/Python/lldbsuite/__init__.py b/lldb/packages/Python/lldbsuite/__init__.py
index 9efa05c7f454e68a78633f75e3e5e2ca43de05f9..195b2683f7b4d34638fa8004e2dfcc43e4d5c9fb 100644
--- a/lldb/packages/Python/lldbsuite/__init__.py
+++ b/lldb/packages/Python/lldbsuite/__init__.py
@@ -22,11 +22,4 @@ lldb_root = find_lldb_root()
 
 # lldbsuite.lldb_test_src_root refers to the root of the python test case tree
 # (i.e. the actual unit tests).
-lldb_test_root = os.path.join(
-    lldb_root,
-    "packages",
-    "Python",
-    "lldbsuite",
-    "test")
-# TODO(rupprecht): update the above definition after moving test cases:
-# lldb_test_root = os.path.join(lldb_root, "test", "API")
+lldb_test_root = os.path.join(lldb_root, "test", "API")
diff --git a/lldb/packages/Python/lldbsuite/test/decorators.py b/lldb/packages/Python/lldbsuite/test/decorators.py
index ec17cb7c25697b44104798570fa2e2ef51c0cf65..39cda3a81954d08af9f481e4db687f913175ccc1 100644
--- a/lldb/packages/Python/lldbsuite/test/decorators.py
+++ b/lldb/packages/Python/lldbsuite/test/decorators.py
@@ -694,7 +694,7 @@ def skipUnlessHasCallSiteInfo(func):
 
         f = tempfile.NamedTemporaryFile()
         cmd = "echo 'int main() {}' | " \
-              "%s -g -glldb -O1 -Xclang -femit-debug-entry-values -S -emit-llvm -x c -o %s -" % (compiler_path, f.name)
+              "%s -g -glldb -O1 -S -emit-llvm -x c -o %s -" % (compiler_path, f.name)
         if os.popen(cmd).close() is not None:
             return "Compiler can't compile with call site info enabled"
 
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
deleted file mode 100644
index db8fa57abb910f0100355c2e47a6bb638c39311a..0000000000000000000000000000000000000000
--- a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-CXX_SOURCES := main.cpp
-CXXFLAGS_EXTRAS := -O2 -glldb -Xclang -femit-debug-entry-values
-include Makefile.rules
diff --git a/lldb/source/API/SystemInitializerFull.cpp b/lldb/source/API/SystemInitializerFull.cpp
index b14a9a2ccd28ed764b94a0776b3427fdb8fa7a35..cde820bbb76b6ad61845b5ab729b6d35e6302a5f 100644
--- a/lldb/source/API/SystemInitializerFull.cpp
+++ b/lldb/source/API/SystemInitializerFull.cpp
@@ -60,7 +60,6 @@ LLDB_PLUGIN_DECLARE(PlatformNetBSD);
 LLDB_PLUGIN_DECLARE(PlatformOpenBSD);
 LLDB_PLUGIN_DECLARE(PlatformWindows);
 LLDB_PLUGIN_DECLARE(PlatformAndroid);
-LLDB_PLUGIN_DECLARE(PlatformRemoteiOS);
 LLDB_PLUGIN_DECLARE(PlatformMacOSX);
 LLDB_PLUGIN_DECLARE(TypeSystemClang);
 LLDB_PLUGIN_DECLARE(ArchitectureArm);
@@ -187,7 +186,6 @@ llvm::Error SystemInitializerFull::Initialize() {
   LLDB_PLUGIN_INITIALIZE(PlatformOpenBSD);
   LLDB_PLUGIN_INITIALIZE(PlatformWindows);
   LLDB_PLUGIN_INITIALIZE(PlatformAndroid);
-  LLDB_PLUGIN_INITIALIZE(PlatformRemoteiOS);
   LLDB_PLUGIN_INITIALIZE(PlatformMacOSX);
 
   // Initialize LLVM and Clang
@@ -363,7 +361,6 @@ void SystemInitializerFull::Terminate() {
   LLDB_PLUGIN_TERMINATE(PlatformOpenBSD);
   LLDB_PLUGIN_TERMINATE(PlatformWindows);
   LLDB_PLUGIN_TERMINATE(PlatformAndroid);
-  LLDB_PLUGIN_TERMINATE(PlatformRemoteiOS);
   LLDB_PLUGIN_TERMINATE(PlatformMacOSX);
 
   LLDB_PLUGIN_TERMINATE(ObjectFileBreakpad);
diff --git a/lldb/source/Commands/CommandCompletions.cpp b/lldb/source/Commands/CommandCompletions.cpp
index e5f29115a8a73ea578773875b3321f8dbe420a5b..bc3a2ec4bd9e148811062a60d0bc32aab47e6bc4 100644
--- a/lldb/source/Commands/CommandCompletions.cpp
+++ b/lldb/source/Commands/CommandCompletions.cpp
@@ -27,19 +27,17 @@
 
 using namespace lldb_private;
 
-CommandCompletions::CommonCompletionElement
-    CommandCompletions::g_common_completions[] = {
-        {eCustomCompletion, nullptr},
-        {eSourceFileCompletion, CommandCompletions::SourceFiles},
-        {eDiskFileCompletion, CommandCompletions::DiskFiles},
-        {eDiskDirectoryCompletion, CommandCompletions::DiskDirectories},
-        {eSymbolCompletion, CommandCompletions::Symbols},
-        {eModuleCompletion, CommandCompletions::Modules},
-        {eSettingsNameCompletion, CommandCompletions::SettingsNames},
-        {ePlatformPluginCompletion, CommandCompletions::PlatformPluginNames},
-        {eArchitectureCompletion, CommandCompletions::ArchitectureNames},
-        {eVariablePathCompletion, CommandCompletions::VariablePath},
-        {eNoCompletion, nullptr} // This one has to be last in the list.
+// This is the command completion callback that is used to complete the
+// argument of the option it is bound to (in the OptionDefinition table
+// below).
+typedef void (*CompletionCallback)(CommandInterpreter &interpreter,
+                                   CompletionRequest &request,
+                                   // A search filter to limit the search...
+                                   lldb_private::SearchFilter *searcher);
+
+struct CommonCompletionElement {
+  uint32_t type;
+  CompletionCallback callback;
 };
 
 bool CommandCompletions::InvokeCommonCompletionCallbacks(
@@ -47,27 +45,238 @@ bool CommandCompletions::InvokeCommonCompletionCallbacks(
     CompletionRequest &request, SearchFilter *searcher) {
   bool handled = false;
 
-  if (completion_mask & eCustomCompletion)
-    return false;
+  const CommonCompletionElement common_completions[] = {
+      {eSourceFileCompletion, CommandCompletions::SourceFiles},
+      {eDiskFileCompletion, CommandCompletions::DiskFiles},
+      {eDiskDirectoryCompletion, CommandCompletions::DiskDirectories},
+      {eSymbolCompletion, CommandCompletions::Symbols},
+      {eModuleCompletion, CommandCompletions::Modules},
+      {eSettingsNameCompletion, CommandCompletions::SettingsNames},
+      {ePlatformPluginCompletion, CommandCompletions::PlatformPluginNames},
+      {eArchitectureCompletion, CommandCompletions::ArchitectureNames},
+      {eVariablePathCompletion, CommandCompletions::VariablePath},
+      {eNoCompletion, nullptr} // This one has to be last in the list.
+  };
 
   for (int i = 0;; i++) {
-    if (g_common_completions[i].type == eNoCompletion)
+    if (common_completions[i].type == eNoCompletion)
       break;
-    else if ((g_common_completions[i].type & completion_mask) ==
-                 g_common_completions[i].type &&
-             g_common_completions[i].callback != nullptr) {
+    else if ((common_completions[i].type & completion_mask) ==
+                 common_completions[i].type &&
+             common_completions[i].callback != nullptr) {
       handled = true;
-      g_common_completions[i].callback(interpreter, request, searcher);
+      common_completions[i].callback(interpreter, request, searcher);
     }
   }
   return handled;
 }
 
+namespace {
+// The Completer class is a convenient base class for building searchers that
+// go along with the SearchFilter passed to the standard Completer functions.
+class Completer : public Searcher {
+public:
+  Completer(CommandInterpreter &interpreter, CompletionRequest &request)
+      : m_interpreter(interpreter), m_request(request) {}
+
+  ~Completer() override = default;
+
+  CallbackReturn SearchCallback(SearchFilter &filter, SymbolContext &context,
+                                Address *addr) override = 0;
+
+  lldb::SearchDepth GetDepth() override = 0;
+
+  virtual void DoCompletion(SearchFilter *filter) = 0;
+
+protected:
+  CommandInterpreter &m_interpreter;
+  CompletionRequest &m_request;
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(Completer);
+};
+} // namespace
+
+// SourceFileCompleter implements the source file completer
+namespace {
+class SourceFileCompleter : public Completer {
+public:
+  SourceFileCompleter(CommandInterpreter &interpreter,
+                      CompletionRequest &request)
+      : Completer(interpreter, request), m_matching_files() {
+    FileSpec partial_spec(m_request.GetCursorArgumentPrefix());
+    m_file_name = partial_spec.GetFilename().GetCString();
+    m_dir_name = partial_spec.GetDirectory().GetCString();
+  }
+
+  lldb::SearchDepth GetDepth() override { return lldb::eSearchDepthCompUnit; }
+
+  Searcher::CallbackReturn SearchCallback(SearchFilter &filter,
+                                          SymbolContext &context,
+                                          Address *addr) override {
+    if (context.comp_unit != nullptr) {
+      const char *cur_file_name =
+          context.comp_unit->GetPrimaryFile().GetFilename().GetCString();
+      const char *cur_dir_name =
+          context.comp_unit->GetPrimaryFile().GetDirectory().GetCString();
+
+      bool match = false;
+      if (m_file_name && cur_file_name &&
+          strstr(cur_file_name, m_file_name) == cur_file_name)
+        match = true;
+
+      if (match && m_dir_name && cur_dir_name &&
+          strstr(cur_dir_name, m_dir_name) != cur_dir_name)
+        match = false;
+
+      if (match) {
+        m_matching_files.AppendIfUnique(context.comp_unit->GetPrimaryFile());
+      }
+    }
+    return Searcher::eCallbackReturnContinue;
+  }
+
+  void DoCompletion(SearchFilter *filter) override {
+    filter->Search(*this);
+    // Now convert the filelist to completions:
+    for (size_t i = 0; i < m_matching_files.GetSize(); i++) {
+      m_request.AddCompletion(
+          m_matching_files.GetFileSpecAtIndex(i).GetFilename().GetCString());
+    }
+  }
+
+private:
+  FileSpecList m_matching_files;
+  const char *m_file_name;
+  const char *m_dir_name;
+
+  DISALLOW_COPY_AND_ASSIGN(SourceFileCompleter);
+};
+} // namespace
+
+static bool regex_chars(const char comp) {
+  return llvm::StringRef("[](){}+.*|^$\\?").contains(comp);
+}
+
+namespace {
+class SymbolCompleter : public Completer {
+
+public:
+  SymbolCompleter(CommandInterpreter &interpreter, CompletionRequest &request)
+      : Completer(interpreter, request) {
+    std::string regex_str;
+    if (!m_request.GetCursorArgumentPrefix().empty()) {
+      regex_str.append("^");
+      regex_str.append(std::string(m_request.GetCursorArgumentPrefix()));
+    } else {
+      // Match anything since the completion string is empty
+      regex_str.append(".");
+    }
+    std::string::iterator pos =
+        find_if(regex_str.begin() + 1, regex_str.end(), regex_chars);
+    while (pos < regex_str.end()) {
+      pos = regex_str.insert(pos, '\\');
+      pos = find_if(pos + 2, regex_str.end(), regex_chars);
+    }
+    m_regex = RegularExpression(regex_str);
+  }
+
+  lldb::SearchDepth GetDepth() override { return lldb::eSearchDepthModule; }
+
+  Searcher::CallbackReturn SearchCallback(SearchFilter &filter,
+                                          SymbolContext &context,
+                                          Address *addr) override {
+    if (context.module_sp) {
+      SymbolContextList sc_list;
+      const bool include_symbols = true;
+      const bool include_inlines = true;
+      context.module_sp->FindFunctions(m_regex, include_symbols,
+                                       include_inlines, sc_list);
+
+      SymbolContext sc;
+      // Now add the functions & symbols to the list - only add if unique:
+      for (uint32_t i = 0; i < sc_list.GetSize(); i++) {
+        if (sc_list.GetContextAtIndex(i, sc)) {
+          ConstString func_name = sc.GetFunctionName(Mangled::ePreferDemangled);
+          // Ensure that the function name matches the regex. This is more than
+          // a sanity check. It is possible that the demangled function name
+          // does not start with the prefix, for example when it's in an
+          // anonymous namespace.
+          if (!func_name.IsEmpty() && m_regex.Execute(func_name.GetStringRef()))
+            m_match_set.insert(func_name);
+        }
+      }
+    }
+    return Searcher::eCallbackReturnContinue;
+  }
+
+  void DoCompletion(SearchFilter *filter) override {
+    filter->Search(*this);
+    collection::iterator pos = m_match_set.begin(), end = m_match_set.end();
+    for (pos = m_match_set.begin(); pos != end; pos++)
+      m_request.AddCompletion((*pos).GetCString());
+  }
+
+private:
+  RegularExpression m_regex;
+  typedef std::set<ConstString> collection;
+  collection m_match_set;
+
+  DISALLOW_COPY_AND_ASSIGN(SymbolCompleter);
+};
+} // namespace
+
+namespace {
+class ModuleCompleter : public Completer {
+public:
+  ModuleCompleter(CommandInterpreter &interpreter, CompletionRequest &request)
+      : Completer(interpreter, request) {
+    FileSpec partial_spec(m_request.GetCursorArgumentPrefix());
+    m_file_name = partial_spec.GetFilename().GetCString();
+    m_dir_name = partial_spec.GetDirectory().GetCString();
+  }
+
+  lldb::SearchDepth GetDepth() override { return lldb::eSearchDepthModule; }
+
+  Searcher::CallbackReturn SearchCallback(SearchFilter &filter,
+                                          SymbolContext &context,
+                                          Address *addr) override {
+    if (context.module_sp) {
+      const char *cur_file_name =
+          context.module_sp->GetFileSpec().GetFilename().GetCString();
+      const char *cur_dir_name =
+          context.module_sp->GetFileSpec().GetDirectory().GetCString();
+
+      bool match = false;
+      if (m_file_name && cur_file_name &&
+          strstr(cur_file_name, m_file_name) == cur_file_name)
+        match = true;
+
+      if (match && m_dir_name && cur_dir_name &&
+          strstr(cur_dir_name, m_dir_name) != cur_dir_name)
+        match = false;
+
+      if (match) {
+        m_request.AddCompletion(cur_file_name);
+      }
+    }
+    return Searcher::eCallbackReturnContinue;
+  }
+
+  void DoCompletion(SearchFilter *filter) override { filter->Search(*this); }
+
+private:
+  const char *m_file_name;
+  const char *m_dir_name;
+
+  DISALLOW_COPY_AND_ASSIGN(ModuleCompleter);
+};
+} // namespace
+
 void CommandCompletions::SourceFiles(CommandInterpreter &interpreter,
                                      CompletionRequest &request,
                                      SearchFilter *searcher) {
-  // Find some way to switch "include support files..."
-  SourceFileCompleter completer(interpreter, false, request);
+  SourceFileCompleter completer(interpreter, request);
 
   if (searcher == nullptr) {
     lldb::TargetSP target_sp = interpreter.GetDebugger().GetSelectedTarget();
@@ -322,186 +531,3 @@ void CommandCompletions::VariablePath(CommandInterpreter &interpreter,
                                       SearchFilter *searcher) {
   Variable::AutoComplete(interpreter.GetExecutionContext(), request);
 }
-
-CommandCompletions::Completer::Completer(CommandInterpreter &interpreter,
-                                         CompletionRequest &request)
-    : m_interpreter(interpreter), m_request(request) {}
-
-CommandCompletions::Completer::~Completer() = default;
-
-// SourceFileCompleter
-
-CommandCompletions::SourceFileCompleter::SourceFileCompleter(
-    CommandInterpreter &interpreter, bool include_support_files,
-    CompletionRequest &request)
-    : CommandCompletions::Completer(interpreter, request),
-      m_include_support_files(include_support_files), m_matching_files() {
-  FileSpec partial_spec(m_request.GetCursorArgumentPrefix());
-  m_file_name = partial_spec.GetFilename().GetCString();
-  m_dir_name = partial_spec.GetDirectory().GetCString();
-}
-
-lldb::SearchDepth CommandCompletions::SourceFileCompleter::GetDepth() {
-  return lldb::eSearchDepthCompUnit;
-}
-
-Searcher::CallbackReturn
-CommandCompletions::SourceFileCompleter::SearchCallback(SearchFilter &filter,
-                                                        SymbolContext &context,
-                                                        Address *addr) {
-  if (context.comp_unit != nullptr) {
-    if (m_include_support_files) {
-      FileSpecList supporting_files = context.comp_unit->GetSupportFiles();
-      for (size_t sfiles = 0; sfiles < supporting_files.GetSize(); sfiles++) {
-        const FileSpec &sfile_spec =
-            supporting_files.GetFileSpecAtIndex(sfiles);
-        const char *sfile_file_name = sfile_spec.GetFilename().GetCString();
-        const char *sfile_dir_name = sfile_spec.GetFilename().GetCString();
-        bool match = false;
-        if (m_file_name && sfile_file_name &&
-            strstr(sfile_file_name, m_file_name) == sfile_file_name)
-          match = true;
-        if (match && m_dir_name && sfile_dir_name &&
-            strstr(sfile_dir_name, m_dir_name) != sfile_dir_name)
-          match = false;
-
-        if (match) {
-          m_matching_files.AppendIfUnique(sfile_spec);
-        }
-      }
-    } else {
-      const char *cur_file_name =
-          context.comp_unit->GetPrimaryFile().GetFilename().GetCString();
-      const char *cur_dir_name =
-          context.comp_unit->GetPrimaryFile().GetDirectory().GetCString();
-
-      bool match = false;
-      if (m_file_name && cur_file_name &&
-          strstr(cur_file_name, m_file_name) == cur_file_name)
-        match = true;
-
-      if (match && m_dir_name && cur_dir_name &&
-          strstr(cur_dir_name, m_dir_name) != cur_dir_name)
-        match = false;
-
-      if (match) {
-        m_matching_files.AppendIfUnique(context.comp_unit->GetPrimaryFile());
-      }
-    }
-  }
-  return Searcher::eCallbackReturnContinue;
-}
-
-void CommandCompletions::SourceFileCompleter::DoCompletion(
-    SearchFilter *filter) {
-  filter->Search(*this);
-  // Now convert the filelist to completions:
-  for (size_t i = 0; i < m_matching_files.GetSize(); i++) {
-    m_request.AddCompletion(
-        m_matching_files.GetFileSpecAtIndex(i).GetFilename().GetCString());
-  }
-}
-
-// SymbolCompleter
-
-static bool regex_chars(const char comp) {
-  return llvm::StringRef("[](){}+.*|^$\\?").contains(comp);
-}
-
-CommandCompletions::SymbolCompleter::SymbolCompleter(
-    CommandInterpreter &interpreter, CompletionRequest &request)
-    : CommandCompletions::Completer(interpreter, request) {
-  std::string regex_str;
-  if (!m_request.GetCursorArgumentPrefix().empty()) {
-    regex_str.append("^");
-    regex_str.append(std::string(m_request.GetCursorArgumentPrefix()));
-  } else {
-    // Match anything since the completion string is empty
-    regex_str.append(".");
-  }
-  std::string::iterator pos =
-      find_if(regex_str.begin() + 1, regex_str.end(), regex_chars);
-  while (pos < regex_str.end()) {
-    pos = regex_str.insert(pos, '\\');
-    pos = find_if(pos + 2, regex_str.end(), regex_chars);
-  }
-  m_regex = RegularExpression(regex_str);
-}
-
-lldb::SearchDepth CommandCompletions::SymbolCompleter::GetDepth() {
-  return lldb::eSearchDepthModule;
-}
-
-Searcher::CallbackReturn CommandCompletions::SymbolCompleter::SearchCallback(
-    SearchFilter &filter, SymbolContext &context, Address *addr) {
-  if (context.module_sp) {
-    SymbolContextList sc_list;
-    const bool include_symbols = true;
-    const bool include_inlines = true;
-    context.module_sp->FindFunctions(m_regex, include_symbols, include_inlines,
-                                     sc_list);
-
-    SymbolContext sc;
-    // Now add the functions & symbols to the list - only add if unique:
-    for (uint32_t i = 0; i < sc_list.GetSize(); i++) {
-      if (sc_list.GetContextAtIndex(i, sc)) {
-        ConstString func_name = sc.GetFunctionName(Mangled::ePreferDemangled);
-        // Ensure that the function name matches the regex. This is more than a
-        // sanity check. It is possible that the demangled function name does
-        // not start with the prefix, for example when it's in an anonymous
-        // namespace.
-        if (!func_name.IsEmpty() && m_regex.Execute(func_name.GetStringRef()))
-          m_match_set.insert(func_name);
-      }
-    }
-  }
-  return Searcher::eCallbackReturnContinue;
-}
-
-void CommandCompletions::SymbolCompleter::DoCompletion(SearchFilter *filter) {
-  filter->Search(*this);
-  collection::iterator pos = m_match_set.begin(), end = m_match_set.end();
-  for (pos = m_match_set.begin(); pos != end; pos++)
-    m_request.AddCompletion((*pos).GetCString());
-}
-
-// ModuleCompleter
-CommandCompletions::ModuleCompleter::ModuleCompleter(
-    CommandInterpreter &interpreter, CompletionRequest &request)
-    : CommandCompletions::Completer(interpreter, request) {
-  FileSpec partial_spec(m_request.GetCursorArgumentPrefix());
-  m_file_name = partial_spec.GetFilename().GetCString();
-  m_dir_name = partial_spec.GetDirectory().GetCString();
-}
-
-lldb::SearchDepth CommandCompletions::ModuleCompleter::GetDepth() {
-  return lldb::eSearchDepthModule;
-}
-
-Searcher::CallbackReturn CommandCompletions::ModuleCompleter::SearchCallback(
-    SearchFilter &filter, SymbolContext &context, Address *addr) {
-  if (context.module_sp) {
-    const char *cur_file_name =
-        context.module_sp->GetFileSpec().GetFilename().GetCString();
-    const char *cur_dir_name =
-        context.module_sp->GetFileSpec().GetDirectory().GetCString();
-
-    bool match = false;
-    if (m_file_name && cur_file_name &&
-        strstr(cur_file_name, m_file_name) == cur_file_name)
-      match = true;
-
-    if (match && m_dir_name && cur_dir_name &&
-        strstr(cur_dir_name, m_dir_name) != cur_dir_name)
-      match = false;
-
-    if (match) {
-      m_request.AddCompletion(cur_file_name);
-    }
-  }
-  return Searcher::eCallbackReturnContinue;
-}
-
-void CommandCompletions::ModuleCompleter::DoCompletion(SearchFilter *filter) {
-  filter->Search(*this);
-}
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index 80e41c91e68b284e7309413bc9ba9ed6d245d55b..71ceef7798d3147f2f63aacb6525589398ecad27 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -1650,13 +1650,13 @@ bool ValueObject::GetDeclaration(Declaration &decl) {
 }
 
 ConstString ValueObject::GetTypeName() {
-  return GetCompilerType().GetConstTypeName();
+  return GetCompilerType().GetTypeName();
 }
 
 ConstString ValueObject::GetDisplayTypeName() { return GetTypeName(); }
 
 ConstString ValueObject::GetQualifiedTypeName() {
-  return GetCompilerType().GetConstQualifiedTypeName();
+  return GetCompilerType().GetTypeName();
 }
 
 LanguageType ValueObject::GetObjectRuntimeLanguage() {
diff --git a/lldb/source/Core/ValueObjectChild.cpp b/lldb/source/Core/ValueObjectChild.cpp
index 778bcecf05d3fb36bcc8a680c3d77af07f7b8731..6205ed32c615ac182610b36c2aae759eafe5bee4 100644
--- a/lldb/source/Core/ValueObjectChild.cpp
+++ b/lldb/source/Core/ValueObjectChild.cpp
@@ -70,14 +70,14 @@ static void AdjustForBitfieldness(ConstString &name,
 
 ConstString ValueObjectChild::GetTypeName() {
   if (m_type_name.IsEmpty()) {
-    m_type_name = GetCompilerType().GetConstTypeName();
+    m_type_name = GetCompilerType().GetTypeName();
     AdjustForBitfieldness(m_type_name, m_bitfield_bit_size);
   }
   return m_type_name;
 }
 
 ConstString ValueObjectChild::GetQualifiedTypeName() {
-  ConstString qualified_name = GetCompilerType().GetConstTypeName();
+  ConstString qualified_name = GetCompilerType().GetTypeName();
   AdjustForBitfieldness(qualified_name, m_bitfield_bit_size);
   return qualified_name;
 }
diff --git a/lldb/source/Core/ValueObjectConstResult.cpp b/lldb/source/Core/ValueObjectConstResult.cpp
index 80d7783ea267341eebc9ca98ac09911c66eb0e7f..8d84f8e62ccc5da87180171235d63a2f9d146672 100644
--- a/lldb/source/Core/ValueObjectConstResult.cpp
+++ b/lldb/source/Core/ValueObjectConstResult.cpp
@@ -228,7 +228,7 @@ size_t ValueObjectConstResult::CalculateNumChildren(uint32_t max) {
 
 ConstString ValueObjectConstResult::GetTypeName() {
   if (m_type_name.IsEmpty())
-    m_type_name = GetCompilerType().GetConstTypeName();
+    m_type_name = GetCompilerType().GetTypeName();
   return m_type_name;
 }
 
diff --git a/lldb/source/Core/ValueObjectMemory.cpp b/lldb/source/Core/ValueObjectMemory.cpp
index b1fd5138668299a39ae82202511021b9b09eb836..91b2c6084928a6eee24d6f5cf046557d744b2793 100644
--- a/lldb/source/Core/ValueObjectMemory.cpp
+++ b/lldb/source/Core/ValueObjectMemory.cpp
@@ -117,7 +117,7 @@ CompilerType ValueObjectMemory::GetCompilerTypeImpl() {
 ConstString ValueObjectMemory::GetTypeName() {
   if (m_type_sp)
     return m_type_sp->GetName();
-  return m_compiler_type.GetConstTypeName();
+  return m_compiler_type.GetTypeName();
 }
 
 ConstString ValueObjectMemory::GetDisplayTypeName() {
diff --git a/lldb/source/Core/ValueObjectRegister.cpp b/lldb/source/Core/ValueObjectRegister.cpp
index bd1e2359afae6f34ddef88833a775bf3959f5b7c..ec87c38fb367927bcaf2c1f3e81bd991f7bddf70 100644
--- a/lldb/source/Core/ValueObjectRegister.cpp
+++ b/lldb/source/Core/ValueObjectRegister.cpp
@@ -219,7 +219,7 @@ CompilerType ValueObjectRegister::GetCompilerTypeImpl() {
 
 ConstString ValueObjectRegister::GetTypeName() {
   if (m_type_name.IsEmpty())
-    m_type_name = GetCompilerType().GetConstTypeName();
+    m_type_name = GetCompilerType().GetTypeName();
   return m_type_name;
 }
 
diff --git a/lldb/source/DataFormatters/FormatManager.cpp b/lldb/source/DataFormatters/FormatManager.cpp
index 0230c84e992e9b555be0e58eb5b651d7a29fe011..23ad6c67f8876c67afcad984884c2ab61a7ce0f6 100644
--- a/lldb/source/DataFormatters/FormatManager.cpp
+++ b/lldb/source/DataFormatters/FormatManager.cpp
@@ -179,7 +179,7 @@ void FormatManager::GetPossibleMatches(
     bool did_strip_ptr, bool did_strip_ref, bool did_strip_typedef,
     bool root_level) {
   compiler_type = compiler_type.GetTypeForFormatters();
-  ConstString type_name(compiler_type.GetConstTypeName());
+  ConstString type_name(compiler_type.GetTypeName());
   if (valobj.GetBitfieldBitSize() > 0) {
     StreamString sstring;
     sstring.Printf("%s:%d", type_name.AsCString(), valobj.GetBitfieldBitSize());
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index bae489128fd1db700fcfd1f24dc1fed7a8b3d5c8..0f89550984acd0a4fb02993eb5721bddace80b24 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -895,7 +895,7 @@ size_t AppleObjCRuntimeV2::GetByteOffsetForIvar(CompilerType &parent_ast_type,
                                                 const char *ivar_name) {
   uint32_t ivar_offset = LLDB_INVALID_IVAR_OFFSET;
 
-  ConstString class_name = parent_ast_type.GetConstTypeName();
+  ConstString class_name = parent_ast_type.GetTypeName();
   if (!class_name.IsEmpty() && ivar_name && ivar_name[0]) {
     // Make the objective C V2 mangled name for the ivar offset from the class
     // name and ivar name
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
index a9e48e2cabfe0ecc99c0dfaf6a4c54f3e542e42d..459e02e557450900b0b87027f5d11b927fbcf200 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
@@ -397,7 +397,7 @@ ObjCLanguageRuntime::GetRuntimeType(CompilerType base_type) {
   if (!class_type)
     return llvm::None;
 
-  ConstString class_name(class_type.GetConstTypeName());
+  ConstString class_name(class_type.GetTypeName());
   if (!class_name)
     return llvm::None;
 
diff --git a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
index 3b273896cb59850f36bb00b0f6bd328b5c01fc58..d3f620385d97d0017945fa719e1d902d20edcfe9 100644
--- a/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
+++ b/lldb/source/Plugins/ObjectFile/ELF/ObjectFileELF.h
@@ -328,9 +328,6 @@ private:
   /// section index 0 is never valid).
   lldb::user_id_t GetSectionIndexByName(const char *name);
 
-  // Returns the ID of the first section that has the given type.
-  lldb::user_id_t GetSectionIndexByType(unsigned type);
-
   /// Returns the section header with the given id or NULL.
   const ELFSectionHeaderInfo *GetSectionHeaderByIndex(lldb::user_id_t id);
 
diff --git a/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt b/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt
index e8b0f31d373634615b5747a38d5df1d6dbd23fc5..91d7901011bc4af887721e799150a3f3dfed28ae 100644
--- a/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt
+++ b/lldb/source/Plugins/OperatingSystem/Python/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_lldb_library(lldbPluginOSPython PLUGIN
+add_lldb_library(lldbPluginOperatingSystemPython PLUGIN
   OperatingSystemPython.cpp
 
   LINK_LIBS
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
index 6cf96eded7aa5c38a33e8058767c25bcb14fbb32..c62940f35e5c757c18df16438569fcbda6f34aed 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "PlatformMacOSX.h"
+#include "PlatformRemoteiOS.h"
 #if defined(__APPLE__)
 #include "PlatformAppleTVSimulator.h"
 #include "PlatformAppleWatchSimulator.h"
@@ -44,6 +45,7 @@ static uint32_t g_initialize_count = 0;
 
 void PlatformMacOSX::Initialize() {
   PlatformDarwin::Initialize();
+  PlatformRemoteiOS::Initialize();
 #if defined(__APPLE__)
   PlatformiOSSimulator::Initialize();
   PlatformDarwinKernel::Initialize();
@@ -82,6 +84,7 @@ void PlatformMacOSX::Terminate() {
   PlatformDarwinKernel::Terminate();
   PlatformiOSSimulator::Terminate();
 #endif
+  PlatformRemoteiOS::Terminate();
   PlatformDarwin::Terminate();
 }
 
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index db03595e970b9f2a1d47fbdd0c7afb5cb628c38b..049975a3547b80583ae9da32de1662dac359f9ba 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -586,6 +586,10 @@ public:
 
   ConstString GetTypeName(lldb::opaque_compiler_type_t type) override;
 
+  ConstString GetDisplayTypeName(lldb::opaque_compiler_type_t type) override {
+    return GetTypeName(type);
+  }
+
   uint32_t GetTypeInfo(lldb::opaque_compiler_type_t type,
                        CompilerType *pointee_or_element_compiler_type) override;
 
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index a0f38981e2db98a6b03ff7910124daa21eeb35cf..1507f2c3121b38e996f0ba9a0898dee132eacdd0 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -268,19 +268,6 @@ size_t CompilerType::GetPointerByteSize() const {
   return 0;
 }
 
-ConstString CompilerType::GetConstQualifiedTypeName() const {
-  return GetConstTypeName();
-}
-
-ConstString CompilerType::GetConstTypeName() const {
-  if (IsValid()) {
-    ConstString type_name(GetTypeName());
-    if (type_name)
-      return type_name;
-  }
-  return ConstString("<invalid>");
-}
-
 ConstString CompilerType::GetTypeName() const {
   if (IsValid()) {
     return m_type_system->GetTypeName(m_type);
@@ -288,7 +275,11 @@ ConstString CompilerType::GetTypeName() const {
   return ConstString("<invalid>");
 }
 
-ConstString CompilerType::GetDisplayTypeName() const { return GetTypeName(); }
+ConstString CompilerType::GetDisplayTypeName() const {
+  if (IsValid())
+    return m_type_system->GetDisplayTypeName(m_type);
+  return ConstString("<invalid>");
+}
 
 uint32_t CompilerType::GetTypeInfo(
     CompilerType *pointee_or_element_compiler_type) const {
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index b6d86801d230956105b2fd3d81cfc5a341488cb4..09c0e16387937deedba5f55291936a2e619315bf 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -303,7 +303,7 @@ void Type::Dump(Stream *s, bool show_context) {
 
 ConstString Type::GetName() {
   if (!m_name)
-    m_name = GetForwardCompilerType().GetConstTypeName();
+    m_name = GetForwardCompilerType().GetTypeName();
   return m_name;
 }
 
@@ -657,7 +657,7 @@ CompilerType Type::GetForwardCompilerType() {
 }
 
 ConstString Type::GetQualifiedName() {
-  return GetForwardCompilerType().GetConstTypeName();
+  return GetForwardCompilerType().GetTypeName();
 }
 
 bool Type::GetTypeScopeAndBasename(const llvm::StringRef& name,
diff --git a/lldb/packages/Python/lldbsuite/test/android/platform/Makefile b/lldb/test/API/android/platform/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/android/platform/Makefile
rename to lldb/test/API/android/platform/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/android/platform/TestDefaultCacheLineSize.py b/lldb/test/API/android/platform/TestDefaultCacheLineSize.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/android/platform/TestDefaultCacheLineSize.py
rename to lldb/test/API/android/platform/TestDefaultCacheLineSize.py
diff --git a/lldb/packages/Python/lldbsuite/test/android/platform/main.cpp b/lldb/test/API/android/platform/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/android/platform/main.cpp
rename to lldb/test/API/android/platform/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/Makefile b/lldb/test/API/api/check_public_api_headers/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/Makefile
rename to lldb/test/API/api/check_public_api_headers/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/TestPublicAPIHeaders.py b/lldb/test/API/api/check_public_api_headers/TestPublicAPIHeaders.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/TestPublicAPIHeaders.py
rename to lldb/test/API/api/check_public_api_headers/TestPublicAPIHeaders.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/main.cpp.template b/lldb/test/API/api/check_public_api_headers/main.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/check_public_api_headers/main.cpp.template
rename to lldb/test/API/api/check_public_api_headers/main.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/api/command-return-object/Makefile b/lldb/test/API/api/command-return-object/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/command-return-object/Makefile
rename to lldb/test/API/api/command-return-object/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/api/command-return-object/TestSBCommandReturnObject.py b/lldb/test/API/api/command-return-object/TestSBCommandReturnObject.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/command-return-object/TestSBCommandReturnObject.py
rename to lldb/test/API/api/command-return-object/TestSBCommandReturnObject.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/command-return-object/main.cpp b/lldb/test/API/api/command-return-object/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/command-return-object/main.cpp
rename to lldb/test/API/api/command-return-object/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/api/listeners/Makefile b/lldb/test/API/api/listeners/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/listeners/Makefile
rename to lldb/test/API/api/listeners/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/api/listeners/TestListener.py b/lldb/test/API/api/listeners/TestListener.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/listeners/TestListener.py
rename to lldb/test/API/api/listeners/TestListener.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/listeners/main.c b/lldb/test/API/api/listeners/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/listeners/main.c
rename to lldb/test/API/api/listeners/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/api/log/TestAPILog.py b/lldb/test/API/api/log/TestAPILog.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/log/TestAPILog.py
rename to lldb/test/API/api/log/TestAPILog.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/.categories b/lldb/test/API/api/multiple-debuggers/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/.categories
rename to lldb/test/API/api/multiple-debuggers/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/Makefile b/lldb/test/API/api/multiple-debuggers/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/Makefile
rename to lldb/test/API/api/multiple-debuggers/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/TestMultipleDebuggers.py b/lldb/test/API/api/multiple-debuggers/TestMultipleDebuggers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/TestMultipleDebuggers.py
rename to lldb/test/API/api/multiple-debuggers/TestMultipleDebuggers.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/multi-process-driver.cpp b/lldb/test/API/api/multiple-debuggers/multi-process-driver.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/multi-process-driver.cpp
rename to lldb/test/API/api/multiple-debuggers/multi-process-driver.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/testprog.cpp b/lldb/test/API/api/multiple-debuggers/testprog.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-debuggers/testprog.cpp
rename to lldb/test/API/api/multiple-debuggers/testprog.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-targets/Makefile b/lldb/test/API/api/multiple-targets/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-targets/Makefile
rename to lldb/test/API/api/multiple-targets/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-targets/TestMultipleTargets.py b/lldb/test/API/api/multiple-targets/TestMultipleTargets.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-targets/TestMultipleTargets.py
rename to lldb/test/API/api/multiple-targets/TestMultipleTargets.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/multiple-targets/main.cpp b/lldb/test/API/api/multiple-targets/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multiple-targets/main.cpp
rename to lldb/test/API/api/multiple-targets/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/Makefile b/lldb/test/API/api/multithreaded/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/Makefile
rename to lldb/test/API/api/multithreaded/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/TestMultithreaded.py b/lldb/test/API/api/multithreaded/TestMultithreaded.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/TestMultithreaded.py
rename to lldb/test/API/api/multithreaded/TestMultithreaded.py
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/common.h b/lldb/test/API/api/multithreaded/common.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/common.h
rename to lldb/test/API/api/multithreaded/common.h
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/driver.cpp.template b/lldb/test/API/api/multithreaded/driver.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/driver.cpp.template
rename to lldb/test/API/api/multithreaded/driver.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/inferior.cpp b/lldb/test/API/api/multithreaded/inferior.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/inferior.cpp
rename to lldb/test/API/api/multithreaded/inferior.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/listener_test.cpp.template b/lldb/test/API/api/multithreaded/listener_test.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/listener_test.cpp.template
rename to lldb/test/API/api/multithreaded/listener_test.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/test_breakpoint_callback.cpp.template b/lldb/test/API/api/multithreaded/test_breakpoint_callback.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/test_breakpoint_callback.cpp.template
rename to lldb/test/API/api/multithreaded/test_breakpoint_callback.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/test_listener_event_description.cpp.template b/lldb/test/API/api/multithreaded/test_listener_event_description.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/test_listener_event_description.cpp.template
rename to lldb/test/API/api/multithreaded/test_listener_event_description.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/test_listener_event_process_state.cpp.template b/lldb/test/API/api/multithreaded/test_listener_event_process_state.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/test_listener_event_process_state.cpp.template
rename to lldb/test/API/api/multithreaded/test_listener_event_process_state.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/api/multithreaded/test_listener_resume.cpp.template b/lldb/test/API/api/multithreaded/test_listener_resume.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/api/multithreaded/test_listener_resume.cpp.template
rename to lldb/test/API/api/multithreaded/test_listener_resume.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/Makefile b/lldb/test/API/arm/breakpoint-it/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/Makefile
rename to lldb/test/API/arm/breakpoint-it/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/TestBreakpointIt.py b/lldb/test/API/arm/breakpoint-it/TestBreakpointIt.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/TestBreakpointIt.py
rename to lldb/test/API/arm/breakpoint-it/TestBreakpointIt.py
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/main.c b/lldb/test/API/arm/breakpoint-it/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/main.c
rename to lldb/test/API/arm/breakpoint-it/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/Makefile b/lldb/test/API/arm/breakpoint-thumb-codesection/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/Makefile
rename to lldb/test/API/arm/breakpoint-thumb-codesection/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py b/lldb/test/API/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py
rename to lldb/test/API/arm/breakpoint-thumb-codesection/TestBreakpointThumbCodesection.py
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/main.c b/lldb/test/API/arm/breakpoint-thumb-codesection/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/main.c
rename to lldb/test/API/arm/breakpoint-thumb-codesection/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/TestEmulations.py b/lldb/test/API/arm/emulation/TestEmulations.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/TestEmulations.py
rename to lldb/test/API/arm/emulation/TestEmulations.py
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-10-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-10-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-10-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-10-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-11-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-11-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-11-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-11-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-12-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-12-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-12-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-12-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-4-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-4-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-4-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-4-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-5-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-5-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-5-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-5-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-5-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-5-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-5-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-5-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-6-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-6-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-6-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-6-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-6-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-6-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-6-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-6-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-7-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-7-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-7-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-7-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-7-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-7-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-7-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-7-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-8-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-8-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-8-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-8-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-8-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-8-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-8-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-8-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-9-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-add-9-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-add-9-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-add-9-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-bic-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-bic-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-bic-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-bic-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-bic-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-bic-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-bic-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-bic-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldmia-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldmia-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldmia-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldmia-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldmia-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldmia-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldmia-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldmia-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldmia-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldmia-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldmia-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldmia-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldmia-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-10-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-10-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-10-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-10-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-11-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-11-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-11-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-11-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-12-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-12-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-12-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-12-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-4-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-4-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-4-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-4-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-5-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-5-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-5-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-5-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-5-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-5-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-5-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-5-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-6-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-6-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-6-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-6-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-6-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-6-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-6-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-6-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-7-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-7-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-7-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-7-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-7-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-7-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-7-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-7-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-8-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-8-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-8-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-8-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-8-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-8-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-8-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-8-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-9-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldr-9-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldr-9-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldr-9-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrd-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldrd-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrd-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldrd-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrd-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldrd-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrd-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldrd-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrd-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldrd-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrd-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldrd-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrh-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldrh-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrh-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldrh-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrsh-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldrsh-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrsh-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldrsh-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrsh-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-ldrsh-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-ldrsh-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-ldrsh-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-10-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-10-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-10-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-10-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-11-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-11-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-11-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-11-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-12-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-12-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-12-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-12-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-13-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-13-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-13-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-13-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-14-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-14-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-14-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-14-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-15-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-15-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-15-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-15-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-16-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-16-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-16-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-16-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-17-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-17-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-17-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-17-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-18-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-18-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-18-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-18-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-19-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-19-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-19-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-19-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-20-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-20-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-20-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-20-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-21-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-21-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-21-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-21-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-22-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-22-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-22-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-22-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-23-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-23-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-23-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-23-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-24-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-24-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-24-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-24-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-25-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-25-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-25-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-25-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-26-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-26-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-26-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-26-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-27-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-27-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-27-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-27-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-28-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-28-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-28-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-28-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-29-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-29-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-29-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-29-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-30-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-30-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-30-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-30-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-31-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-31-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-31-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-31-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-4-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-4-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-4-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-4-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-5-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-5-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-5-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-5-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-5-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-5-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-5-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-5-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-6-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-6-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-6-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-6-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-6-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-6-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-6-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-6-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-7-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-7-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-7-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-7-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-8-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-8-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-8-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-8-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-9-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mov-9-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mov-9-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mov-9-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-moveq-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-moveq-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-moveq-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-moveq-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-movs-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-movs-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-movs-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-movs-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-4-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-4-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-4-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-4-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-mvn-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-mvn-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-mvn-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-pop-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-pop-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-pop-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-pop-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-pop-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-pop-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-pop-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-pop-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-pop-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-pop-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-pop-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-push-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-push-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-push-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-push-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-push-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-push-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-push-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-push-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-push-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-push-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-push-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-push-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-push-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-4-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-4-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-4-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-4-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-5-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-str-5-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-str-5-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-str-5-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strb-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-strb-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strb-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-strb-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strb-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-strb-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strb-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-strb-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strbt-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-strbt-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strbt-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-strbt-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strd-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-strd-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strd-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-strd-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strt-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-strt-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-strt-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-strt-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-10-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-10-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-10-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-10-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-2-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-2-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-2-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-2-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-3-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-3-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-3-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-3-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-4-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-4-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-4-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-4-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-5-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-5-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-5-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-5-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-5-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-5-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-5-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-5-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-6-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-6-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-6-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-6-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-6-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-6-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-6-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-6-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-8-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-8-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-8-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-8-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-9-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-sub-9-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-sub-9-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-sub-9-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-1-arm.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-1-arm.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-1-arm.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-1-arm.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-10-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-10-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-10-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-10-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-4-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-4-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-4-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-4-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-5-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-5-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-5-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-5-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-6-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-6-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-6-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-6-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-8-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-8-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-8-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-8-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-9-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-subs-9-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-subs-9-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-subs-9-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpop-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-vpop-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpop-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-vpop-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpop-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-vpop-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpop-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-vpop-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpop-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-vpop-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpop-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-vpop-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpush-1-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-vpush-1-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpush-1-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-vpush-1-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpush-2-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-vpush-2-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpush-2-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-vpush-2-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpush-3-thumb.dat b/lldb/test/API/arm/emulation/new-test-files/test-vpush-3-thumb.dat
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/arm/emulation/new-test-files/test-vpush-3-thumb.dat
rename to lldb/test/API/arm/emulation/new-test-files/test-vpush-3-thumb.dat
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/continue/Makefile b/lldb/test/API/benchmarks/continue/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/continue/Makefile
rename to lldb/test/API/benchmarks/continue/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/continue/TestBenchmarkContinue.py b/lldb/test/API/benchmarks/continue/TestBenchmarkContinue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/continue/TestBenchmarkContinue.py
rename to lldb/test/API/benchmarks/continue/TestBenchmarkContinue.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/continue/main.cpp b/lldb/test/API/benchmarks/continue/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/continue/main.cpp
rename to lldb/test/API/benchmarks/continue/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/expression/Makefile b/lldb/test/API/benchmarks/expression/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/expression/Makefile
rename to lldb/test/API/benchmarks/expression/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/expression/TestExpressionCmd.py b/lldb/test/API/benchmarks/expression/TestExpressionCmd.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/expression/TestExpressionCmd.py
rename to lldb/test/API/benchmarks/expression/TestExpressionCmd.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/expression/TestRepeatedExprs.py b/lldb/test/API/benchmarks/expression/TestRepeatedExprs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/expression/TestRepeatedExprs.py
rename to lldb/test/API/benchmarks/expression/TestRepeatedExprs.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/expression/main.cpp b/lldb/test/API/benchmarks/expression/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/expression/main.cpp
rename to lldb/test/API/benchmarks/expression/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/frame_variable/TestFrameVariableResponse.py b/lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/frame_variable/TestFrameVariableResponse.py
rename to lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/libcxxlist/Makefile b/lldb/test/API/benchmarks/libcxxlist/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/libcxxlist/Makefile
rename to lldb/test/API/benchmarks/libcxxlist/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/libcxxlist/TestBenchmarkLibcxxList.py b/lldb/test/API/benchmarks/libcxxlist/TestBenchmarkLibcxxList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/libcxxlist/TestBenchmarkLibcxxList.py
rename to lldb/test/API/benchmarks/libcxxlist/TestBenchmarkLibcxxList.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/libcxxlist/main.cpp b/lldb/test/API/benchmarks/libcxxlist/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/libcxxlist/main.cpp
rename to lldb/test/API/benchmarks/libcxxlist/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/libcxxmap/Makefile b/lldb/test/API/benchmarks/libcxxmap/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/libcxxmap/Makefile
rename to lldb/test/API/benchmarks/libcxxmap/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/libcxxmap/TestBenchmarkLibcxxMap.py b/lldb/test/API/benchmarks/libcxxmap/TestBenchmarkLibcxxMap.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/libcxxmap/TestBenchmarkLibcxxMap.py
rename to lldb/test/API/benchmarks/libcxxmap/TestBenchmarkLibcxxMap.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/libcxxmap/main.cpp b/lldb/test/API/benchmarks/libcxxmap/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/libcxxmap/main.cpp
rename to lldb/test/API/benchmarks/libcxxmap/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/startup/TestStartupDelays.py b/lldb/test/API/benchmarks/startup/TestStartupDelays.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/startup/TestStartupDelays.py
rename to lldb/test/API/benchmarks/startup/TestStartupDelays.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/stepping/TestSteppingSpeed.py b/lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/stepping/TestSteppingSpeed.py
rename to lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py
diff --git a/lldb/packages/Python/lldbsuite/test/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py b/lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py
rename to lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/add-dsym/uuid/Makefile b/lldb/test/API/commands/add-dsym/uuid/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/add-dsym/uuid/Makefile
rename to lldb/test/API/commands/add-dsym/uuid/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/add-dsym/uuid/TestAddDsymCommand.py b/lldb/test/API/commands/add-dsym/uuid/TestAddDsymCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/add-dsym/uuid/TestAddDsymCommand.py
rename to lldb/test/API/commands/add-dsym/uuid/TestAddDsymCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/add-dsym/uuid/main.cpp.template b/lldb/test/API/commands/add-dsym/uuid/main.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/add-dsym/uuid/main.cpp.template
rename to lldb/test/API/commands/add-dsym/uuid/main.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/commands/apropos/basic/TestApropos.py b/lldb/test/API/commands/apropos/basic/TestApropos.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/apropos/basic/TestApropos.py
rename to lldb/test/API/commands/apropos/basic/TestApropos.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/apropos/with-process/Makefile b/lldb/test/API/commands/apropos/with-process/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/apropos/with-process/Makefile
rename to lldb/test/API/commands/apropos/with-process/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/apropos/with-process/TestAproposWithProcess.py b/lldb/test/API/commands/apropos/with-process/TestAproposWithProcess.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/apropos/with-process/TestAproposWithProcess.py
rename to lldb/test/API/commands/apropos/with-process/TestAproposWithProcess.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/apropos/with-process/main.cpp b/lldb/test/API/commands/apropos/with-process/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/apropos/with-process/main.cpp
rename to lldb/test/API/commands/apropos/with-process/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/breakpoint/command/list/TestBreakpointCommandList.py b/lldb/test/API/commands/breakpoint/command/list/TestBreakpointCommandList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/breakpoint/command/list/TestBreakpointCommandList.py
rename to lldb/test/API/commands/breakpoint/command/list/TestBreakpointCommandList.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/breakpoint/command/list/a.yaml b/lldb/test/API/commands/breakpoint/command/list/a.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/breakpoint/command/list/a.yaml
rename to lldb/test/API/commands/breakpoint/command/list/a.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/.categories b/lldb/test/API/commands/command/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/.categories
rename to lldb/test/API/commands/command/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/delete/TestCommandDelete.py b/lldb/test/API/commands/command/delete/TestCommandDelete.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/delete/TestCommandDelete.py
rename to lldb/test/API/commands/command/delete/TestCommandDelete.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/history/TestCommandHistory.py b/lldb/test/API/commands/command/history/TestCommandHistory.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/history/TestCommandHistory.py
rename to lldb/test/API/commands/command/history/TestCommandHistory.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/invalid-args/TestInvalidArgsCommand.py b/lldb/test/API/commands/command/invalid-args/TestInvalidArgsCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/invalid-args/TestInvalidArgsCommand.py
rename to lldb/test/API/commands/command/invalid-args/TestInvalidArgsCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/nested_alias/Makefile b/lldb/test/API/commands/command/nested_alias/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/nested_alias/Makefile
rename to lldb/test/API/commands/command/nested_alias/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/nested_alias/TestNestedAlias.py b/lldb/test/API/commands/command/nested_alias/TestNestedAlias.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/nested_alias/TestNestedAlias.py
rename to lldb/test/API/commands/command/nested_alias/TestNestedAlias.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/nested_alias/main.cpp b/lldb/test/API/commands/command/nested_alias/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/nested_alias/main.cpp
rename to lldb/test/API/commands/command/nested_alias/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/Makefile b/lldb/test/API/commands/command/script/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/Makefile
rename to lldb/test/API/commands/command/script/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/TestCommandScript.py b/lldb/test/API/commands/command/script/TestCommandScript.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/TestCommandScript.py
rename to lldb/test/API/commands/command/script/TestCommandScript.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/bug11569.py b/lldb/test/API/commands/command/script/bug11569.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/bug11569.py
rename to lldb/test/API/commands/command/script/bug11569.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/callables.py b/lldb/test/API/commands/command/script/callables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/callables.py
rename to lldb/test/API/commands/command/script/callables.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/decorated.py b/lldb/test/API/commands/command/script/decorated.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/decorated.py
rename to lldb/test/API/commands/command/script/decorated.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/Makefile b/lldb/test/API/commands/command/script/import/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/Makefile
rename to lldb/test/API/commands/command/script/import/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/TestImport.py b/lldb/test/API/commands/command/script/import/TestImport.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/TestImport.py
rename to lldb/test/API/commands/command/script/import/TestImport.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/bar/bar.py b/lldb/test/API/commands/command/script/import/bar/bar.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/bar/bar.py
rename to lldb/test/API/commands/command/script/import/bar/bar.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/bar/barutil.py b/lldb/test/API/commands/command/script/import/bar/barutil.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/bar/barutil.py
rename to lldb/test/API/commands/command/script/import/bar/barutil.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/dummymodule.py b/lldb/test/API/commands/command/script/import/dummymodule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/dummymodule.py
rename to lldb/test/API/commands/command/script/import/dummymodule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/foo/bar/foobar.py b/lldb/test/API/commands/command/script/import/foo/bar/foobar.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/foo/bar/foobar.py
rename to lldb/test/API/commands/command/script/import/foo/bar/foobar.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/foo/foo.py b/lldb/test/API/commands/command/script/import/foo/foo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/foo/foo.py
rename to lldb/test/API/commands/command/script/import/foo/foo.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/foo/foo2.py b/lldb/test/API/commands/command/script/import/foo/foo2.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/foo/foo2.py
rename to lldb/test/API/commands/command/script/import/foo/foo2.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/main.c b/lldb/test/API/commands/command/script/import/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/main.c
rename to lldb/test/API/commands/command/script/import/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/Makefile b/lldb/test/API/commands/command/script/import/rdar-12586188/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/Makefile
rename to lldb/test/API/commands/command/script/import/rdar-12586188/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/TestRdar12586188.py b/lldb/test/API/commands/command/script/import/rdar-12586188/TestRdar12586188.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/TestRdar12586188.py
rename to lldb/test/API/commands/command/script/import/rdar-12586188/TestRdar12586188.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/fail12586188.py b/lldb/test/API/commands/command/script/import/rdar-12586188/fail12586188.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/fail12586188.py
rename to lldb/test/API/commands/command/script/import/rdar-12586188/fail12586188.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/fail212586188.py b/lldb/test/API/commands/command/script/import/rdar-12586188/fail212586188.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/rdar-12586188/fail212586188.py
rename to lldb/test/API/commands/command/script/import/rdar-12586188/fail212586188.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/thepackage/TPunitA.py b/lldb/test/API/commands/command/script/import/thepackage/TPunitA.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/thepackage/TPunitA.py
rename to lldb/test/API/commands/command/script/import/thepackage/TPunitA.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/thepackage/TPunitB.py b/lldb/test/API/commands/command/script/import/thepackage/TPunitB.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/thepackage/TPunitB.py
rename to lldb/test/API/commands/command/script/import/thepackage/TPunitB.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/import/thepackage/__init__.py b/lldb/test/API/commands/command/script/import/thepackage/__init__.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/import/thepackage/__init__.py
rename to lldb/test/API/commands/command/script/import/thepackage/__init__.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/main.cpp b/lldb/test/API/commands/command/script/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/main.cpp
rename to lldb/test/API/commands/command/script/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/mysto.py b/lldb/test/API/commands/command/script/mysto.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/mysto.py
rename to lldb/test/API/commands/command/script/mysto.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/py_import b/lldb/test/API/commands/command/script/py_import
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/py_import
rename to lldb/test/API/commands/command/script/py_import
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script/welcome.py b/lldb/test/API/commands/command/script/welcome.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script/welcome.py
rename to lldb/test/API/commands/command/script/welcome.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script_alias/TestCommandScriptAlias.py b/lldb/test/API/commands/command/script_alias/TestCommandScriptAlias.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script_alias/TestCommandScriptAlias.py
rename to lldb/test/API/commands/command/script_alias/TestCommandScriptAlias.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/script_alias/tcsacmd.py b/lldb/test/API/commands/command/script_alias/tcsacmd.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/script_alias/tcsacmd.py
rename to lldb/test/API/commands/command/script_alias/tcsacmd.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/source/.lldb b/lldb/test/API/commands/command/source/.lldb
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/source/.lldb
rename to lldb/test/API/commands/command/source/.lldb
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/source/TestCommandSource.py b/lldb/test/API/commands/command/source/TestCommandSource.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/source/TestCommandSource.py
rename to lldb/test/API/commands/command/source/TestCommandSource.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/source/commands.txt b/lldb/test/API/commands/command/source/commands.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/source/commands.txt
rename to lldb/test/API/commands/command/source/commands.txt
diff --git a/lldb/packages/Python/lldbsuite/test/commands/command/source/my.py b/lldb/test/API/commands/command/source/my.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/command/source/my.py
rename to lldb/test/API/commands/command/source/my.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/Makefile b/lldb/test/API/commands/disassemble/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/Makefile
rename to lldb/test/API/commands/disassemble/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/TestDisassembleBreakpoint.py b/lldb/test/API/commands/disassemble/basic/TestDisassembleBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/TestDisassembleBreakpoint.py
rename to lldb/test/API/commands/disassemble/basic/TestDisassembleBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/TestFrameDisassemble.py b/lldb/test/API/commands/disassemble/basic/TestFrameDisassemble.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/TestFrameDisassemble.py
rename to lldb/test/API/commands/disassemble/basic/TestFrameDisassemble.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/main.cpp b/lldb/test/API/commands/disassemble/basic/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/disassemble/basic/main.cpp
rename to lldb/test/API/commands/disassemble/basic/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/.categories b/lldb/test/API/commands/expression/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/.categories
rename to lldb/test/API/commands/expression/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/anonymous-struct/Makefile b/lldb/test/API/commands/expression/anonymous-struct/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/anonymous-struct/Makefile
rename to lldb/test/API/commands/expression/anonymous-struct/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py b/lldb/test/API/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py
rename to lldb/test/API/commands/expression/anonymous-struct/TestCallUserAnonTypedef.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/anonymous-struct/main.cpp b/lldb/test/API/commands/expression/anonymous-struct/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/anonymous-struct/main.cpp
rename to lldb/test/API/commands/expression/anonymous-struct/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/argument_passing_restrictions/Makefile b/lldb/test/API/commands/expression/argument_passing_restrictions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/argument_passing_restrictions/Makefile
rename to lldb/test/API/commands/expression/argument_passing_restrictions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/argument_passing_restrictions/TestArgumentPassingRestrictions.py b/lldb/test/API/commands/expression/argument_passing_restrictions/TestArgumentPassingRestrictions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/argument_passing_restrictions/TestArgumentPassingRestrictions.py
rename to lldb/test/API/commands/expression/argument_passing_restrictions/TestArgumentPassingRestrictions.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/argument_passing_restrictions/main.cpp b/lldb/test/API/commands/expression/argument_passing_restrictions/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/argument_passing_restrictions/main.cpp
rename to lldb/test/API/commands/expression/argument_passing_restrictions/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/calculator_mode/TestCalculatorMode.py b/lldb/test/API/commands/expression/calculator_mode/TestCalculatorMode.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/calculator_mode/TestCalculatorMode.py
rename to lldb/test/API/commands/expression/calculator_mode/TestCalculatorMode.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/Makefile b/lldb/test/API/commands/expression/call-function/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-function/Makefile
rename to lldb/test/API/commands/expression/call-function/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallBuiltinFunction.py b/lldb/test/API/commands/expression/call-function/TestCallBuiltinFunction.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallBuiltinFunction.py
rename to lldb/test/API/commands/expression/call-function/TestCallBuiltinFunction.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallStdStringFunction.py b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallStdStringFunction.py
rename to lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallStopAndContinue.py b/lldb/test/API/commands/expression/call-function/TestCallStopAndContinue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallStopAndContinue.py
rename to lldb/test/API/commands/expression/call-function/TestCallStopAndContinue.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallUserDefinedFunction.py b/lldb/test/API/commands/expression/call-function/TestCallUserDefinedFunction.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallUserDefinedFunction.py
rename to lldb/test/API/commands/expression/call-function/TestCallUserDefinedFunction.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/main.cpp b/lldb/test/API/commands/expression/call-function/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-function/main.cpp
rename to lldb/test/API/commands/expression/call-function/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-restarts/Makefile b/lldb/test/API/commands/expression/call-restarts/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-restarts/Makefile
rename to lldb/test/API/commands/expression/call-restarts/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-restarts/TestCallThatRestarts.py b/lldb/test/API/commands/expression/call-restarts/TestCallThatRestarts.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-restarts/TestCallThatRestarts.py
rename to lldb/test/API/commands/expression/call-restarts/TestCallThatRestarts.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-restarts/lotta-signals.c b/lldb/test/API/commands/expression/call-restarts/lotta-signals.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-restarts/lotta-signals.c
rename to lldb/test/API/commands/expression/call-restarts/lotta-signals.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/Makefile b/lldb/test/API/commands/expression/call-throws/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/Makefile
rename to lldb/test/API/commands/expression/call-throws/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/TestCallThatThrows.py b/lldb/test/API/commands/expression/call-throws/TestCallThatThrows.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/TestCallThatThrows.py
rename to lldb/test/API/commands/expression/call-throws/TestCallThatThrows.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/call-throws.m b/lldb/test/API/commands/expression/call-throws/call-throws.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/call-throws.m
rename to lldb/test/API/commands/expression/call-throws/call-throws.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/cast_int_to_anonymous_enum/Makefile b/lldb/test/API/commands/expression/cast_int_to_anonymous_enum/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/cast_int_to_anonymous_enum/Makefile
rename to lldb/test/API/commands/expression/cast_int_to_anonymous_enum/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/cast_int_to_anonymous_enum/TestCastIntToAnonymousEnum.py b/lldb/test/API/commands/expression/cast_int_to_anonymous_enum/TestCastIntToAnonymousEnum.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/cast_int_to_anonymous_enum/TestCastIntToAnonymousEnum.py
rename to lldb/test/API/commands/expression/cast_int_to_anonymous_enum/TestCastIntToAnonymousEnum.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/cast_int_to_anonymous_enum/main.cpp b/lldb/test/API/commands/expression/cast_int_to_anonymous_enum/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/cast_int_to_anonymous_enum/main.cpp
rename to lldb/test/API/commands/expression/cast_int_to_anonymous_enum/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/char/Makefile b/lldb/test/API/commands/expression/char/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/char/Makefile
rename to lldb/test/API/commands/expression/char/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/char/TestExprsChar.py b/lldb/test/API/commands/expression/char/TestExprsChar.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/char/TestExprsChar.py
rename to lldb/test/API/commands/expression/char/TestExprsChar.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/char/main.cpp b/lldb/test/API/commands/expression/char/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/char/main.cpp
rename to lldb/test/API/commands/expression/char/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/class_template_specialization_empty_pack/Makefile b/lldb/test/API/commands/expression/class_template_specialization_empty_pack/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/class_template_specialization_empty_pack/Makefile
rename to lldb/test/API/commands/expression/class_template_specialization_empty_pack/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/class_template_specialization_empty_pack/TestClassTemplateSpecializationParametersHandling.py b/lldb/test/API/commands/expression/class_template_specialization_empty_pack/TestClassTemplateSpecializationParametersHandling.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/class_template_specialization_empty_pack/TestClassTemplateSpecializationParametersHandling.py
rename to lldb/test/API/commands/expression/class_template_specialization_empty_pack/TestClassTemplateSpecializationParametersHandling.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/class_template_specialization_empty_pack/main.cpp b/lldb/test/API/commands/expression/class_template_specialization_empty_pack/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/class_template_specialization_empty_pack/main.cpp
rename to lldb/test/API/commands/expression/class_template_specialization_empty_pack/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/TestCodegenCrashTypedefDeclNotInDeclContext.py b/lldb/test/API/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/TestCodegenCrashTypedefDeclNotInDeclContext.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/TestCodegenCrashTypedefDeclNotInDeclContext.py
rename to lldb/test/API/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/TestCodegenCrashTypedefDeclNotInDeclContext.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/main.cpp b/lldb/test/API/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/main.cpp
rename to lldb/test/API/commands/expression/codegen-crash-typedefdecl-not-in_declcontext/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-incomplete-record/TestCompletionCrashIncompleteRecord.py b/lldb/test/API/commands/expression/completion-crash-incomplete-record/TestCompletionCrashIncompleteRecord.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-incomplete-record/TestCompletionCrashIncompleteRecord.py
rename to lldb/test/API/commands/expression/completion-crash-incomplete-record/TestCompletionCrashIncompleteRecord.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-incomplete-record/main.cpp b/lldb/test/API/commands/expression/completion-crash-incomplete-record/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-incomplete-record/main.cpp
rename to lldb/test/API/commands/expression/completion-crash-incomplete-record/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-invalid-iterator/Makefile b/lldb/test/API/commands/expression/completion-crash-invalid-iterator/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-invalid-iterator/Makefile
rename to lldb/test/API/commands/expression/completion-crash-invalid-iterator/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-invalid-iterator/TestInvalidIteratorCompletionCrash.py b/lldb/test/API/commands/expression/completion-crash-invalid-iterator/TestInvalidIteratorCompletionCrash.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-invalid-iterator/TestInvalidIteratorCompletionCrash.py
rename to lldb/test/API/commands/expression/completion-crash-invalid-iterator/TestInvalidIteratorCompletionCrash.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-invalid-iterator/main.cpp b/lldb/test/API/commands/expression/completion-crash-invalid-iterator/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-crash-invalid-iterator/main.cpp
rename to lldb/test/API/commands/expression/completion-crash-invalid-iterator/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-in-lambda-and-unnamed-class/TestCompletionInLambdaAndUnnamedClass.py b/lldb/test/API/commands/expression/completion-in-lambda-and-unnamed-class/TestCompletionInLambdaAndUnnamedClass.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-in-lambda-and-unnamed-class/TestCompletionInLambdaAndUnnamedClass.py
rename to lldb/test/API/commands/expression/completion-in-lambda-and-unnamed-class/TestCompletionInLambdaAndUnnamedClass.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion-in-lambda-and-unnamed-class/main.cpp b/lldb/test/API/commands/expression/completion-in-lambda-and-unnamed-class/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion-in-lambda-and-unnamed-class/main.cpp
rename to lldb/test/API/commands/expression/completion-in-lambda-and-unnamed-class/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion/.categories b/lldb/test/API/commands/expression/completion/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion/.categories
rename to lldb/test/API/commands/expression/completion/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion/Makefile b/lldb/test/API/commands/expression/completion/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion/Makefile
rename to lldb/test/API/commands/expression/completion/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion/TestExprCompletion.py b/lldb/test/API/commands/expression/completion/TestExprCompletion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion/TestExprCompletion.py
rename to lldb/test/API/commands/expression/completion/TestExprCompletion.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion/main.cpp b/lldb/test/API/commands/expression/completion/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion/main.cpp
rename to lldb/test/API/commands/expression/completion/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/completion/other.cpp b/lldb/test/API/commands/expression/completion/other.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/completion/other.cpp
rename to lldb/test/API/commands/expression/completion/other.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/Makefile b/lldb/test/API/commands/expression/context-object-objc/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/Makefile
rename to lldb/test/API/commands/expression/context-object-objc/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/TestContextObjectObjc.py b/lldb/test/API/commands/expression/context-object-objc/TestContextObjectObjc.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/TestContextObjectObjc.py
rename to lldb/test/API/commands/expression/context-object-objc/TestContextObjectObjc.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/main.m b/lldb/test/API/commands/expression/context-object-objc/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/main.m
rename to lldb/test/API/commands/expression/context-object-objc/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object/Makefile b/lldb/test/API/commands/expression/context-object/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/context-object/Makefile
rename to lldb/test/API/commands/expression/context-object/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object/TestContextObject.py b/lldb/test/API/commands/expression/context-object/TestContextObject.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/context-object/TestContextObject.py
rename to lldb/test/API/commands/expression/context-object/TestContextObject.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object/main.cpp b/lldb/test/API/commands/expression/context-object/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/context-object/main.cpp
rename to lldb/test/API/commands/expression/context-object/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py b/lldb/test/API/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py
rename to lldb/test/API/commands/expression/deleting-implicit-copy-constructor/TestDeletingImplicitCopyConstructor.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/deleting-implicit-copy-constructor/main.cpp b/lldb/test/API/commands/expression/deleting-implicit-copy-constructor/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/deleting-implicit-copy-constructor/main.cpp
rename to lldb/test/API/commands/expression/deleting-implicit-copy-constructor/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/Makefile b/lldb/test/API/commands/expression/diagnostics/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/Makefile
rename to lldb/test/API/commands/expression/diagnostics/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/TestExprDiagnostics.py b/lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/TestExprDiagnostics.py
rename to lldb/test/API/commands/expression/diagnostics/TestExprDiagnostics.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/main.cpp b/lldb/test/API/commands/expression/diagnostics/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/main.cpp
rename to lldb/test/API/commands/expression/diagnostics/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/dollar-in-variable/TestDollarInVariable.py b/lldb/test/API/commands/expression/dollar-in-variable/TestDollarInVariable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/dollar-in-variable/TestDollarInVariable.py
rename to lldb/test/API/commands/expression/dollar-in-variable/TestDollarInVariable.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/dollar-in-variable/main.c b/lldb/test/API/commands/expression/dollar-in-variable/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/dollar-in-variable/main.c
rename to lldb/test/API/commands/expression/dollar-in-variable/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/Makefile b/lldb/test/API/commands/expression/dont_allow_jit/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/Makefile
rename to lldb/test/API/commands/expression/dont_allow_jit/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/TestAllowJIT.py b/lldb/test/API/commands/expression/dont_allow_jit/TestAllowJIT.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/TestAllowJIT.py
rename to lldb/test/API/commands/expression/dont_allow_jit/TestAllowJIT.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/main.c b/lldb/test/API/commands/expression/dont_allow_jit/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/main.c
rename to lldb/test/API/commands/expression/dont_allow_jit/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/entry-bp/Makefile b/lldb/test/API/commands/expression/entry-bp/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/entry-bp/Makefile
rename to lldb/test/API/commands/expression/entry-bp/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/entry-bp/TestExprEntryBP.py b/lldb/test/API/commands/expression/entry-bp/TestExprEntryBP.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/entry-bp/TestExprEntryBP.py
rename to lldb/test/API/commands/expression/entry-bp/TestExprEntryBP.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/entry-bp/main.c b/lldb/test/API/commands/expression/entry-bp/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/entry-bp/main.c
rename to lldb/test/API/commands/expression/entry-bp/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/expr-in-syscall/Makefile b/lldb/test/API/commands/expression/expr-in-syscall/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/expr-in-syscall/Makefile
rename to lldb/test/API/commands/expression/expr-in-syscall/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/expr-in-syscall/TestExpressionInSyscall.py b/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
rename to lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/expr-in-syscall/main.cpp b/lldb/test/API/commands/expression/expr-in-syscall/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/expr-in-syscall/main.cpp
rename to lldb/test/API/commands/expression/expr-in-syscall/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/fixits/Makefile b/lldb/test/API/commands/expression/fixits/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/fixits/Makefile
rename to lldb/test/API/commands/expression/fixits/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/fixits/TestFixIts.py b/lldb/test/API/commands/expression/fixits/TestFixIts.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/fixits/TestFixIts.py
rename to lldb/test/API/commands/expression/fixits/TestFixIts.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/fixits/main.cpp b/lldb/test/API/commands/expression/fixits/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/fixits/main.cpp
rename to lldb/test/API/commands/expression/fixits/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/formatters/Makefile b/lldb/test/API/commands/expression/formatters/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/formatters/Makefile
rename to lldb/test/API/commands/expression/formatters/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/formatters/TestFormatters.py b/lldb/test/API/commands/expression/formatters/TestFormatters.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/formatters/TestFormatters.py
rename to lldb/test/API/commands/expression/formatters/TestFormatters.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/formatters/foosynth.py b/lldb/test/API/commands/expression/formatters/foosynth.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/formatters/foosynth.py
rename to lldb/test/API/commands/expression/formatters/foosynth.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/formatters/formatters.py b/lldb/test/API/commands/expression/formatters/formatters.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/formatters/formatters.py
rename to lldb/test/API/commands/expression/formatters/formatters.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/formatters/main.cpp b/lldb/test/API/commands/expression/formatters/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/formatters/main.cpp
rename to lldb/test/API/commands/expression/formatters/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/function_template_specialization_temp_args/Makefile b/lldb/test/API/commands/expression/function_template_specialization_temp_args/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/function_template_specialization_temp_args/Makefile
rename to lldb/test/API/commands/expression/function_template_specialization_temp_args/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/function_template_specialization_temp_args/TestFunctionTemplateSpecializationTempArgs.py b/lldb/test/API/commands/expression/function_template_specialization_temp_args/TestFunctionTemplateSpecializationTempArgs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/function_template_specialization_temp_args/TestFunctionTemplateSpecializationTempArgs.py
rename to lldb/test/API/commands/expression/function_template_specialization_temp_args/TestFunctionTemplateSpecializationTempArgs.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/function_template_specialization_temp_args/main.cpp b/lldb/test/API/commands/expression/function_template_specialization_temp_args/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/function_template_specialization_temp_args/main.cpp
rename to lldb/test/API/commands/expression/function_template_specialization_temp_args/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ignore-artificial-constructors/TestIgnoreArtificialConstructors.py b/lldb/test/API/commands/expression/ignore-artificial-constructors/TestIgnoreArtificialConstructors.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ignore-artificial-constructors/TestIgnoreArtificialConstructors.py
rename to lldb/test/API/commands/expression/ignore-artificial-constructors/TestIgnoreArtificialConstructors.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ignore-artificial-constructors/main.cpp b/lldb/test/API/commands/expression/ignore-artificial-constructors/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ignore-artificial-constructors/main.cpp
rename to lldb/test/API/commands/expression/ignore-artificial-constructors/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/Makefile b/lldb/test/API/commands/expression/import-std-module/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/Makefile
rename to lldb/test/API/commands/expression/import-std-module/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/TestImportStdModule.py b/lldb/test/API/commands/expression/import-std-module/basic/TestImportStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/TestImportStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/basic/TestImportStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/main.cpp b/lldb/test/API/commands/expression/import-std-module/basic/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/basic/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/Makefile b/lldb/test/API/commands/expression/import-std-module/conflicts/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/Makefile
rename to lldb/test/API/commands/expression/import-std-module/conflicts/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py b/lldb/test/API/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py
rename to lldb/test/API/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/main.cpp b/lldb/test/API/commands/expression/import-std-module/conflicts/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/conflicts/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/Makefile b/lldb/test/API/commands/expression/import-std-module/deque-basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/Makefile
rename to lldb/test/API/commands/expression/import-std-module/deque-basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/TestDequeFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/deque-basic/TestDequeFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/TestDequeFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/deque-basic/TestDequeFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/main.cpp b/lldb/test/API/commands/expression/import-std-module/deque-basic/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/deque-basic/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDequeFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/deque-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/Makefile b/lldb/test/API/commands/expression/import-std-module/empty-module/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/Makefile
rename to lldb/test/API/commands/expression/import-std-module/empty-module/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/TestEmptyStdModule.py b/lldb/test/API/commands/expression/import-std-module/empty-module/TestEmptyStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/TestEmptyStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/empty-module/TestEmptyStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/main.cpp b/lldb/test/API/commands/expression/import-std-module/empty-module/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/empty-module/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/algorithm b/lldb/test/API/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/algorithm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/algorithm
rename to lldb/test/API/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/algorithm
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/module.modulemap b/lldb/test/API/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/module.modulemap
rename to lldb/test/API/commands/expression/import-std-module/empty-module/root/usr/include/c++/v1/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/root/usr/include/libc_header.h b/lldb/test/API/commands/expression/import-std-module/empty-module/root/usr/include/libc_header.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/empty-module/root/usr/include/libc_header.h
rename to lldb/test/API/commands/expression/import-std-module/empty-module/root/usr/include/libc_header.h
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardListFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardListFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardListFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardListFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/forward_list-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/forward_list-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list/Makefile b/lldb/test/API/commands/expression/import-std-module/forward_list/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list/Makefile
rename to lldb/test/API/commands/expression/import-std-module/forward_list/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list/TestForwardListFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/forward_list/TestForwardListFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list/TestForwardListFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/forward_list/TestForwardListFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list/main.cpp b/lldb/test/API/commands/expression/import-std-module/forward_list/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/forward_list/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/list-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/list-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentListFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentListFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentListFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentListFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/list-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/list-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list/Makefile b/lldb/test/API/commands/expression/import-std-module/list/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list/Makefile
rename to lldb/test/API/commands/expression/import-std-module/list/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list/TestListFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/list/TestListFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list/TestListFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/list/TestListFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list/main.cpp b/lldb/test/API/commands/expression/import-std-module/list/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/list/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/Makefile b/lldb/test/API/commands/expression/import-std-module/no-std-module/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/Makefile
rename to lldb/test/API/commands/expression/import-std-module/no-std-module/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py b/lldb/test/API/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/main.cpp b/lldb/test/API/commands/expression/import-std-module/no-std-module/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/no-std-module/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/Makefile b/lldb/test/API/commands/expression/import-std-module/queue/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/Makefile
rename to lldb/test/API/commands/expression/import-std-module/queue/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/TestQueueFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/queue/TestQueueFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/TestQueueFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/queue/TestQueueFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/main.cpp b/lldb/test/API/commands/expression/import-std-module/queue/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/queue/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/Makefile b/lldb/test/API/commands/expression/import-std-module/shared_ptr/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/Makefile
rename to lldb/test/API/commands/expression/import-std-module/shared_ptr/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/main.cpp b/lldb/test/API/commands/expression/import-std-module/shared_ptr/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/shared_ptr/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/Makefile b/lldb/test/API/commands/expression/import-std-module/stack/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/Makefile
rename to lldb/test/API/commands/expression/import-std-module/stack/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/TestStackFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/stack/TestStackFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/TestStackFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/stack/TestStackFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/main.cpp b/lldb/test/API/commands/expression/import-std-module/stack/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/stack/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/Makefile b/lldb/test/API/commands/expression/import-std-module/sysroot/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/Makefile
rename to lldb/test/API/commands/expression/import-std-module/sysroot/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py b/lldb/test/API/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py
rename to lldb/test/API/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/main.cpp b/lldb/test/API/commands/expression/import-std-module/sysroot/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/sysroot/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/algorithm b/lldb/test/API/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/algorithm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/algorithm
rename to lldb/test/API/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/algorithm
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/module.modulemap b/lldb/test/API/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/module.modulemap
rename to lldb/test/API/commands/expression/import-std-module/sysroot/root/usr/include/c++/v1/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/root/usr/include/libc_header.h b/lldb/test/API/commands/expression/import-std-module/sysroot/root/usr/include/libc_header.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/root/usr/include/libc_header.h
rename to lldb/test/API/commands/expression/import-std-module/sysroot/root/usr/include/libc_header.h
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py b/lldb/test/API/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py
rename to lldb/test/API/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/unique_ptr-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/unique_ptr-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/Makefile b/lldb/test/API/commands/expression/import-std-module/unique_ptr/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/Makefile
rename to lldb/test/API/commands/expression/import-std-module/unique_ptr/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/unique_ptr/TestUniquePtrFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/main.cpp b/lldb/test/API/commands/expression/import-std-module/unique_ptr/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/unique_ptr/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/Makefile b/lldb/test/API/commands/expression/import-std-module/vector-bool/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/Makefile
rename to lldb/test/API/commands/expression/import-std-module/vector-bool/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/TestVectorBoolFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/vector-bool/TestVectorBoolFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/TestVectorBoolFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/vector-bool/TestVectorBoolFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/main.cpp b/lldb/test/API/commands/expression/import-std-module/vector-bool/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/vector-bool/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/vector-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/vector-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVectorFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVectorFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVectorFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVectorFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/vector-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/vector-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/Makefile b/lldb/test/API/commands/expression/import-std-module/vector-of-vectors/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/Makefile
rename to lldb/test/API/commands/expression/import-std-module/vector-of-vectors/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectorsFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectorsFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectorsFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectorsFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/main.cpp b/lldb/test/API/commands/expression/import-std-module/vector-of-vectors/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/vector-of-vectors/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector/Makefile b/lldb/test/API/commands/expression/import-std-module/vector/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector/Makefile
rename to lldb/test/API/commands/expression/import-std-module/vector/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector/TestVectorFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/vector/TestVectorFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector/TestVectorFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/vector/TestVectorFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector/main.cpp b/lldb/test/API/commands/expression/import-std-module/vector/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/vector/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile
rename to lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/main.cpp b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/Makefile b/lldb/test/API/commands/expression/import-std-module/weak_ptr/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/Makefile
rename to lldb/test/API/commands/expression/import-std-module/weak_ptr/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
rename to lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/main.cpp b/lldb/test/API/commands/expression/import-std-module/weak_ptr/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/main.cpp
rename to lldb/test/API/commands/expression/import-std-module/weak_ptr/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/Makefile b/lldb/test/API/commands/expression/import_builtin_fileid/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/Makefile
rename to lldb/test/API/commands/expression/import_builtin_fileid/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/TestImportBuiltinFileID.py b/lldb/test/API/commands/expression/import_builtin_fileid/TestImportBuiltinFileID.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/TestImportBuiltinFileID.py
rename to lldb/test/API/commands/expression/import_builtin_fileid/TestImportBuiltinFileID.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/main.m b/lldb/test/API/commands/expression/import_builtin_fileid/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/main.m
rename to lldb/test/API/commands/expression/import_builtin_fileid/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/inline-namespace/Makefile b/lldb/test/API/commands/expression/inline-namespace/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/inline-namespace/Makefile
rename to lldb/test/API/commands/expression/inline-namespace/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/inline-namespace/TestInlineNamespace.py b/lldb/test/API/commands/expression/inline-namespace/TestInlineNamespace.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/inline-namespace/TestInlineNamespace.py
rename to lldb/test/API/commands/expression/inline-namespace/TestInlineNamespace.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/inline-namespace/main.cpp b/lldb/test/API/commands/expression/inline-namespace/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/inline-namespace/main.cpp
rename to lldb/test/API/commands/expression/inline-namespace/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/invalid-args/TestInvalidArgsExpression.py b/lldb/test/API/commands/expression/invalid-args/TestInvalidArgsExpression.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/invalid-args/TestInvalidArgsExpression.py
rename to lldb/test/API/commands/expression/invalid-args/TestInvalidArgsExpression.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter-phi-nodes/Makefile b/lldb/test/API/commands/expression/ir-interpreter-phi-nodes/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter-phi-nodes/Makefile
rename to lldb/test/API/commands/expression/ir-interpreter-phi-nodes/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter-phi-nodes/TestIRInterpreterPHINodes.py b/lldb/test/API/commands/expression/ir-interpreter-phi-nodes/TestIRInterpreterPHINodes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter-phi-nodes/TestIRInterpreterPHINodes.py
rename to lldb/test/API/commands/expression/ir-interpreter-phi-nodes/TestIRInterpreterPHINodes.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter-phi-nodes/main.cpp b/lldb/test/API/commands/expression/ir-interpreter-phi-nodes/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter-phi-nodes/main.cpp
rename to lldb/test/API/commands/expression/ir-interpreter-phi-nodes/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter/Makefile b/lldb/test/API/commands/expression/ir-interpreter/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter/Makefile
rename to lldb/test/API/commands/expression/ir-interpreter/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter/TestIRInterpreter.py b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter/TestIRInterpreter.py
rename to lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter/main.c b/lldb/test/API/commands/expression/ir-interpreter/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/ir-interpreter/main.c
rename to lldb/test/API/commands/expression/ir-interpreter/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/Makefile b/lldb/test/API/commands/expression/issue_11588/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/Makefile
rename to lldb/test/API/commands/expression/issue_11588/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/Test11588.py b/lldb/test/API/commands/expression/issue_11588/Test11588.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/Test11588.py
rename to lldb/test/API/commands/expression/issue_11588/Test11588.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/main.cpp b/lldb/test/API/commands/expression/issue_11588/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/main.cpp
rename to lldb/test/API/commands/expression/issue_11588/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/s11588.py b/lldb/test/API/commands/expression/issue_11588/s11588.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/issue_11588/s11588.py
rename to lldb/test/API/commands/expression/issue_11588/s11588.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/macros/Makefile b/lldb/test/API/commands/expression/macros/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/macros/Makefile
rename to lldb/test/API/commands/expression/macros/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/macros/TestMacros.py b/lldb/test/API/commands/expression/macros/TestMacros.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/macros/TestMacros.py
rename to lldb/test/API/commands/expression/macros/TestMacros.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/macros/macro1.h b/lldb/test/API/commands/expression/macros/macro1.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/macros/macro1.h
rename to lldb/test/API/commands/expression/macros/macro1.h
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/macros/macro2.h b/lldb/test/API/commands/expression/macros/macro2.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/macros/macro2.h
rename to lldb/test/API/commands/expression/macros/macro2.h
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/macros/main.cpp b/lldb/test/API/commands/expression/macros/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/macros/main.cpp
rename to lldb/test/API/commands/expression/macros/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/multiline-completion/Makefile b/lldb/test/API/commands/expression/multiline-completion/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/multiline-completion/Makefile
rename to lldb/test/API/commands/expression/multiline-completion/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/multiline-completion/TestMultilineCompletion.py b/lldb/test/API/commands/expression/multiline-completion/TestMultilineCompletion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/multiline-completion/TestMultilineCompletion.py
rename to lldb/test/API/commands/expression/multiline-completion/TestMultilineCompletion.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/multiline-completion/main.c b/lldb/test/API/commands/expression/multiline-completion/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/multiline-completion/main.c
rename to lldb/test/API/commands/expression/multiline-completion/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/multiline-navigation/TestMultilineNavigation.py b/lldb/test/API/commands/expression/multiline-navigation/TestMultilineNavigation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/multiline-navigation/TestMultilineNavigation.py
rename to lldb/test/API/commands/expression/multiline-navigation/TestMultilineNavigation.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_cpp_and_c/Makefile b/lldb/test/API/commands/expression/namespace_local_var_same_name_cpp_and_c/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_cpp_and_c/Makefile
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_cpp_and_c/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_cpp_and_c/TestNamespaceLocalVarSameNameCppAndC.py b/lldb/test/API/commands/expression/namespace_local_var_same_name_cpp_and_c/TestNamespaceLocalVarSameNameCppAndC.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_cpp_and_c/TestNamespaceLocalVarSameNameCppAndC.py
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_cpp_and_c/TestNamespaceLocalVarSameNameCppAndC.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_cpp_and_c/main.cpp b/lldb/test/API/commands/expression/namespace_local_var_same_name_cpp_and_c/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_cpp_and_c/main.cpp
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_cpp_and_c/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/Makefile b/lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/Makefile
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/TestNamespaceLocalVarSameNameObjC.py b/lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/TestNamespaceLocalVarSameNameObjC.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/TestNamespaceLocalVarSameNameObjC.py
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/TestNamespaceLocalVarSameNameObjC.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/main.mm b/lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/main.mm
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/util.mm b/lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/util.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/util.mm
rename to lldb/test/API/commands/expression/namespace_local_var_same_name_obj_c/util.mm
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/no-deadlock/Makefile b/lldb/test/API/commands/expression/no-deadlock/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/no-deadlock/Makefile
rename to lldb/test/API/commands/expression/no-deadlock/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/no-deadlock/TestExprDoesntBlock.py b/lldb/test/API/commands/expression/no-deadlock/TestExprDoesntBlock.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/no-deadlock/TestExprDoesntBlock.py
rename to lldb/test/API/commands/expression/no-deadlock/TestExprDoesntBlock.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/no-deadlock/locking.cpp b/lldb/test/API/commands/expression/no-deadlock/locking.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/no-deadlock/locking.cpp
rename to lldb/test/API/commands/expression/no-deadlock/locking.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/options/Makefile b/lldb/test/API/commands/expression/options/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/options/Makefile
rename to lldb/test/API/commands/expression/options/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/options/TestExprOptions.py b/lldb/test/API/commands/expression/options/TestExprOptions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/options/TestExprOptions.py
rename to lldb/test/API/commands/expression/options/TestExprOptions.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/options/foo.cpp b/lldb/test/API/commands/expression/options/foo.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/options/foo.cpp
rename to lldb/test/API/commands/expression/options/foo.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/options/main.cpp b/lldb/test/API/commands/expression/options/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/options/main.cpp
rename to lldb/test/API/commands/expression/options/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/Makefile b/lldb/test/API/commands/expression/persist_objc_pointeetype/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/Makefile
rename to lldb/test/API/commands/expression/persist_objc_pointeetype/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/TestPersistObjCPointeeType.py b/lldb/test/API/commands/expression/persist_objc_pointeetype/TestPersistObjCPointeeType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/TestPersistObjCPointeeType.py
rename to lldb/test/API/commands/expression/persist_objc_pointeetype/TestPersistObjCPointeeType.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/main.m b/lldb/test/API/commands/expression/persist_objc_pointeetype/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/main.m
rename to lldb/test/API/commands/expression/persist_objc_pointeetype/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_ptr_update/Makefile b/lldb/test/API/commands/expression/persistent_ptr_update/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_ptr_update/Makefile
rename to lldb/test/API/commands/expression/persistent_ptr_update/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_ptr_update/TestPersistentPtrUpdate.py b/lldb/test/API/commands/expression/persistent_ptr_update/TestPersistentPtrUpdate.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_ptr_update/TestPersistentPtrUpdate.py
rename to lldb/test/API/commands/expression/persistent_ptr_update/TestPersistentPtrUpdate.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_ptr_update/main.c b/lldb/test/API/commands/expression/persistent_ptr_update/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_ptr_update/main.c
rename to lldb/test/API/commands/expression/persistent_ptr_update/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/Makefile b/lldb/test/API/commands/expression/persistent_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/Makefile
rename to lldb/test/API/commands/expression/persistent_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/TestNestedPersistentTypes.py b/lldb/test/API/commands/expression/persistent_types/TestNestedPersistentTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/TestNestedPersistentTypes.py
rename to lldb/test/API/commands/expression/persistent_types/TestNestedPersistentTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/TestPersistentTypes.py b/lldb/test/API/commands/expression/persistent_types/TestPersistentTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/TestPersistentTypes.py
rename to lldb/test/API/commands/expression/persistent_types/TestPersistentTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/main.c b/lldb/test/API/commands/expression/persistent_types/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_types/main.c
rename to lldb/test/API/commands/expression/persistent_types/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_variables/Makefile b/lldb/test/API/commands/expression/persistent_variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_variables/Makefile
rename to lldb/test/API/commands/expression/persistent_variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_variables/TestPersistentVariables.py b/lldb/test/API/commands/expression/persistent_variables/TestPersistentVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_variables/TestPersistentVariables.py
rename to lldb/test/API/commands/expression/persistent_variables/TestPersistentVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persistent_variables/main.c b/lldb/test/API/commands/expression/persistent_variables/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/persistent_variables/main.c
rename to lldb/test/API/commands/expression/persistent_variables/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/Makefile b/lldb/test/API/commands/expression/po_verbosity/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/Makefile
rename to lldb/test/API/commands/expression/po_verbosity/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/TestPoVerbosity.py b/lldb/test/API/commands/expression/po_verbosity/TestPoVerbosity.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/TestPoVerbosity.py
rename to lldb/test/API/commands/expression/po_verbosity/TestPoVerbosity.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/main.m b/lldb/test/API/commands/expression/po_verbosity/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/main.m
rename to lldb/test/API/commands/expression/po_verbosity/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/pr35310/Makefile b/lldb/test/API/commands/expression/pr35310/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/pr35310/Makefile
rename to lldb/test/API/commands/expression/pr35310/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/pr35310/TestExprsBug35310.py b/lldb/test/API/commands/expression/pr35310/TestExprsBug35310.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/pr35310/TestExprsBug35310.py
rename to lldb/test/API/commands/expression/pr35310/TestExprsBug35310.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/pr35310/main.cpp b/lldb/test/API/commands/expression/pr35310/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/pr35310/main.cpp
rename to lldb/test/API/commands/expression/pr35310/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_8638051/Makefile b/lldb/test/API/commands/expression/radar_8638051/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_8638051/Makefile
rename to lldb/test/API/commands/expression/radar_8638051/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_8638051/Test8638051.py b/lldb/test/API/commands/expression/radar_8638051/Test8638051.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_8638051/Test8638051.py
rename to lldb/test/API/commands/expression/radar_8638051/Test8638051.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_8638051/main.c b/lldb/test/API/commands/expression/radar_8638051/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_8638051/main.c
rename to lldb/test/API/commands/expression/radar_8638051/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_9531204/Makefile b/lldb/test/API/commands/expression/radar_9531204/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_9531204/Makefile
rename to lldb/test/API/commands/expression/radar_9531204/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_9531204/TestPrintfAfterUp.py b/lldb/test/API/commands/expression/radar_9531204/TestPrintfAfterUp.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_9531204/TestPrintfAfterUp.py
rename to lldb/test/API/commands/expression/radar_9531204/TestPrintfAfterUp.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_9531204/main.c b/lldb/test/API/commands/expression/radar_9531204/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_9531204/main.c
rename to lldb/test/API/commands/expression/radar_9531204/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_9673664/Makefile b/lldb/test/API/commands/expression/radar_9673664/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_9673664/Makefile
rename to lldb/test/API/commands/expression/radar_9673664/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_9673664/TestExprHelpExamples.py b/lldb/test/API/commands/expression/radar_9673664/TestExprHelpExamples.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_9673664/TestExprHelpExamples.py
rename to lldb/test/API/commands/expression/radar_9673664/TestExprHelpExamples.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/radar_9673664/main.c b/lldb/test/API/commands/expression/radar_9673664/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/radar_9673664/main.c
rename to lldb/test/API/commands/expression/radar_9673664/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/rdar42038760/TestScalarURem.py b/lldb/test/API/commands/expression/rdar42038760/TestScalarURem.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/rdar42038760/TestScalarURem.py
rename to lldb/test/API/commands/expression/rdar42038760/TestScalarURem.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/rdar42038760/main.c b/lldb/test/API/commands/expression/rdar42038760/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/rdar42038760/main.c
rename to lldb/test/API/commands/expression/rdar42038760/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/rdar44436068/Test128BitsInteger.py b/lldb/test/API/commands/expression/rdar44436068/Test128BitsInteger.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/rdar44436068/Test128BitsInteger.py
rename to lldb/test/API/commands/expression/rdar44436068/Test128BitsInteger.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/rdar44436068/main.c b/lldb/test/API/commands/expression/rdar44436068/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/rdar44436068/main.c
rename to lldb/test/API/commands/expression/rdar44436068/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/regression-access-function-template-in-record/TestRegressionAccessFunctionTemplateInRecord.py b/lldb/test/API/commands/expression/regression-access-function-template-in-record/TestRegressionAccessFunctionTemplateInRecord.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/regression-access-function-template-in-record/TestRegressionAccessFunctionTemplateInRecord.py
rename to lldb/test/API/commands/expression/regression-access-function-template-in-record/TestRegressionAccessFunctionTemplateInRecord.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/regression-access-function-template-in-record/main.cpp b/lldb/test/API/commands/expression/regression-access-function-template-in-record/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/regression-access-function-template-in-record/main.cpp
rename to lldb/test/API/commands/expression/regression-access-function-template-in-record/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/save_jit_objects/Makefile b/lldb/test/API/commands/expression/save_jit_objects/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/save_jit_objects/Makefile
rename to lldb/test/API/commands/expression/save_jit_objects/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/save_jit_objects/TestSaveJITObjects.py b/lldb/test/API/commands/expression/save_jit_objects/TestSaveJITObjects.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/save_jit_objects/TestSaveJITObjects.py
rename to lldb/test/API/commands/expression/save_jit_objects/TestSaveJITObjects.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/save_jit_objects/main.c b/lldb/test/API/commands/expression/save_jit_objects/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/save_jit_objects/main.c
rename to lldb/test/API/commands/expression/save_jit_objects/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/Makefile b/lldb/test/API/commands/expression/scoped_enums/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/Makefile
rename to lldb/test/API/commands/expression/scoped_enums/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/TestScopedEnumType.py b/lldb/test/API/commands/expression/scoped_enums/TestScopedEnumType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/TestScopedEnumType.py
rename to lldb/test/API/commands/expression/scoped_enums/TestScopedEnumType.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/main.cpp b/lldb/test/API/commands/expression/scoped_enums/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/main.cpp
rename to lldb/test/API/commands/expression/scoped_enums/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/static-initializers/Makefile b/lldb/test/API/commands/expression/static-initializers/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/static-initializers/Makefile
rename to lldb/test/API/commands/expression/static-initializers/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/static-initializers/TestStaticInitializers.py b/lldb/test/API/commands/expression/static-initializers/TestStaticInitializers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/static-initializers/TestStaticInitializers.py
rename to lldb/test/API/commands/expression/static-initializers/TestStaticInitializers.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/static-initializers/main.cpp b/lldb/test/API/commands/expression/static-initializers/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/static-initializers/main.cpp
rename to lldb/test/API/commands/expression/static-initializers/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/test/Makefile b/lldb/test/API/commands/expression/test/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/test/Makefile
rename to lldb/test/API/commands/expression/test/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/test/TestExprs.py b/lldb/test/API/commands/expression/test/TestExprs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/test/TestExprs.py
rename to lldb/test/API/commands/expression/test/TestExprs.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/test/TestExprs2.py b/lldb/test/API/commands/expression/test/TestExprs2.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/test/TestExprs2.py
rename to lldb/test/API/commands/expression/test/TestExprs2.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/test/main.cpp b/lldb/test/API/commands/expression/test/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/test/main.cpp
rename to lldb/test/API/commands/expression/test/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/timeout/Makefile b/lldb/test/API/commands/expression/timeout/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/timeout/Makefile
rename to lldb/test/API/commands/expression/timeout/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/timeout/TestCallWithTimeout.py b/lldb/test/API/commands/expression/timeout/TestCallWithTimeout.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/timeout/TestCallWithTimeout.py
rename to lldb/test/API/commands/expression/timeout/TestCallWithTimeout.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/timeout/wait-a-while.cpp b/lldb/test/API/commands/expression/timeout/wait-a-while.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/timeout/wait-a-while.cpp
rename to lldb/test/API/commands/expression/timeout/wait-a-while.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/top-level/Makefile b/lldb/test/API/commands/expression/top-level/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/top-level/Makefile
rename to lldb/test/API/commands/expression/top-level/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/top-level/TestTopLevelExprs.py b/lldb/test/API/commands/expression/top-level/TestTopLevelExprs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/top-level/TestTopLevelExprs.py
rename to lldb/test/API/commands/expression/top-level/TestTopLevelExprs.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/top-level/dummy.cpp b/lldb/test/API/commands/expression/top-level/dummy.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/top-level/dummy.cpp
rename to lldb/test/API/commands/expression/top-level/dummy.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/top-level/main.cpp b/lldb/test/API/commands/expression/top-level/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/top-level/main.cpp
rename to lldb/test/API/commands/expression/top-level/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/top-level/test.cpp b/lldb/test/API/commands/expression/top-level/test.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/top-level/test.cpp
rename to lldb/test/API/commands/expression/top-level/test.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/Makefile b/lldb/test/API/commands/expression/two-files/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/two-files/Makefile
rename to lldb/test/API/commands/expression/two-files/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/TestObjCTypeQueryFromOtherCompileUnit.py b/lldb/test/API/commands/expression/two-files/TestObjCTypeQueryFromOtherCompileUnit.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/two-files/TestObjCTypeQueryFromOtherCompileUnit.py
rename to lldb/test/API/commands/expression/two-files/TestObjCTypeQueryFromOtherCompileUnit.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/foo.m b/lldb/test/API/commands/expression/two-files/foo.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/two-files/foo.m
rename to lldb/test/API/commands/expression/two-files/foo.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/main.m b/lldb/test/API/commands/expression/two-files/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/two-files/main.m
rename to lldb/test/API/commands/expression/two-files/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/unicode-in-variable/TestUnicodeInVariable.py b/lldb/test/API/commands/expression/unicode-in-variable/TestUnicodeInVariable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/unicode-in-variable/TestUnicodeInVariable.py
rename to lldb/test/API/commands/expression/unicode-in-variable/TestUnicodeInVariable.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/unicode-in-variable/main.cpp b/lldb/test/API/commands/expression/unicode-in-variable/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/unicode-in-variable/main.cpp
rename to lldb/test/API/commands/expression/unicode-in-variable/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/unwind_expression/Makefile b/lldb/test/API/commands/expression/unwind_expression/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/unwind_expression/Makefile
rename to lldb/test/API/commands/expression/unwind_expression/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/unwind_expression/TestUnwindExpression.py b/lldb/test/API/commands/expression/unwind_expression/TestUnwindExpression.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/unwind_expression/TestUnwindExpression.py
rename to lldb/test/API/commands/expression/unwind_expression/TestUnwindExpression.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/unwind_expression/main.cpp b/lldb/test/API/commands/expression/unwind_expression/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/unwind_expression/main.cpp
rename to lldb/test/API/commands/expression/unwind_expression/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/vector_of_enums/Makefile b/lldb/test/API/commands/expression/vector_of_enums/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/vector_of_enums/Makefile
rename to lldb/test/API/commands/expression/vector_of_enums/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/vector_of_enums/TestVectorOfEnums.py b/lldb/test/API/commands/expression/vector_of_enums/TestVectorOfEnums.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/vector_of_enums/TestVectorOfEnums.py
rename to lldb/test/API/commands/expression/vector_of_enums/TestVectorOfEnums.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/vector_of_enums/main.cpp b/lldb/test/API/commands/expression/vector_of_enums/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/vector_of_enums/main.cpp
rename to lldb/test/API/commands/expression/vector_of_enums/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/Makefile b/lldb/test/API/commands/expression/weak_symbols/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/Makefile
rename to lldb/test/API/commands/expression/weak_symbols/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/TestWeakSymbols.py b/lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/TestWeakSymbols.py
rename to lldb/test/API/commands/expression/weak_symbols/TestWeakSymbols.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/dylib.c b/lldb/test/API/commands/expression/weak_symbols/dylib.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/dylib.c
rename to lldb/test/API/commands/expression/weak_symbols/dylib.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/dylib.h b/lldb/test/API/commands/expression/weak_symbols/dylib.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/dylib.h
rename to lldb/test/API/commands/expression/weak_symbols/dylib.h
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/main.c b/lldb/test/API/commands/expression/weak_symbols/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/main.c
rename to lldb/test/API/commands/expression/weak_symbols/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/module.modulemap b/lldb/test/API/commands/expression/weak_symbols/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/module.modulemap
rename to lldb/test/API/commands/expression/weak_symbols/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/xvalue/Makefile b/lldb/test/API/commands/expression/xvalue/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/xvalue/Makefile
rename to lldb/test/API/commands/expression/xvalue/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/xvalue/TestXValuePrinting.py b/lldb/test/API/commands/expression/xvalue/TestXValuePrinting.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/xvalue/TestXValuePrinting.py
rename to lldb/test/API/commands/expression/xvalue/TestXValuePrinting.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/xvalue/main.cpp b/lldb/test/API/commands/expression/xvalue/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/expression/xvalue/main.cpp
rename to lldb/test/API/commands/expression/xvalue/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/array/Makefile b/lldb/test/API/commands/frame/diagnose/array/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/array/Makefile
rename to lldb/test/API/commands/frame/diagnose/array/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/array/TestArray.py b/lldb/test/API/commands/frame/diagnose/array/TestArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/array/TestArray.py
rename to lldb/test/API/commands/frame/diagnose/array/TestArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/array/main.c b/lldb/test/API/commands/frame/diagnose/array/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/array/main.c
rename to lldb/test/API/commands/frame/diagnose/array/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/bad-reference/Makefile b/lldb/test/API/commands/frame/diagnose/bad-reference/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/bad-reference/Makefile
rename to lldb/test/API/commands/frame/diagnose/bad-reference/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/bad-reference/TestBadReference.py b/lldb/test/API/commands/frame/diagnose/bad-reference/TestBadReference.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/bad-reference/TestBadReference.py
rename to lldb/test/API/commands/frame/diagnose/bad-reference/TestBadReference.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/bad-reference/main.cpp b/lldb/test/API/commands/frame/diagnose/bad-reference/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/bad-reference/main.cpp
rename to lldb/test/API/commands/frame/diagnose/bad-reference/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/complicated-expression/Makefile b/lldb/test/API/commands/frame/diagnose/complicated-expression/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/complicated-expression/Makefile
rename to lldb/test/API/commands/frame/diagnose/complicated-expression/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/complicated-expression/TestComplicatedExpression.py b/lldb/test/API/commands/frame/diagnose/complicated-expression/TestComplicatedExpression.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/complicated-expression/TestComplicatedExpression.py
rename to lldb/test/API/commands/frame/diagnose/complicated-expression/TestComplicatedExpression.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/complicated-expression/main.c b/lldb/test/API/commands/frame/diagnose/complicated-expression/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/complicated-expression/main.c
rename to lldb/test/API/commands/frame/diagnose/complicated-expression/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-argument/Makefile b/lldb/test/API/commands/frame/diagnose/dereference-argument/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-argument/Makefile
rename to lldb/test/API/commands/frame/diagnose/dereference-argument/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-argument/TestDiagnoseDereferenceArgument.py b/lldb/test/API/commands/frame/diagnose/dereference-argument/TestDiagnoseDereferenceArgument.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-argument/TestDiagnoseDereferenceArgument.py
rename to lldb/test/API/commands/frame/diagnose/dereference-argument/TestDiagnoseDereferenceArgument.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-argument/main.c b/lldb/test/API/commands/frame/diagnose/dereference-argument/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-argument/main.c
rename to lldb/test/API/commands/frame/diagnose/dereference-argument/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-function-return/Makefile b/lldb/test/API/commands/frame/diagnose/dereference-function-return/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-function-return/Makefile
rename to lldb/test/API/commands/frame/diagnose/dereference-function-return/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py b/lldb/test/API/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py
rename to lldb/test/API/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-function-return/main.c b/lldb/test/API/commands/frame/diagnose/dereference-function-return/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-function-return/main.c
rename to lldb/test/API/commands/frame/diagnose/dereference-function-return/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-this/Makefile b/lldb/test/API/commands/frame/diagnose/dereference-this/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-this/Makefile
rename to lldb/test/API/commands/frame/diagnose/dereference-this/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-this/TestDiagnoseDereferenceThis.py b/lldb/test/API/commands/frame/diagnose/dereference-this/TestDiagnoseDereferenceThis.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-this/TestDiagnoseDereferenceThis.py
rename to lldb/test/API/commands/frame/diagnose/dereference-this/TestDiagnoseDereferenceThis.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-this/main.cpp b/lldb/test/API/commands/frame/diagnose/dereference-this/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/dereference-this/main.cpp
rename to lldb/test/API/commands/frame/diagnose/dereference-this/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/inheritance/Makefile b/lldb/test/API/commands/frame/diagnose/inheritance/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/inheritance/Makefile
rename to lldb/test/API/commands/frame/diagnose/inheritance/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/inheritance/TestDiagnoseInheritance.py b/lldb/test/API/commands/frame/diagnose/inheritance/TestDiagnoseInheritance.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/inheritance/TestDiagnoseInheritance.py
rename to lldb/test/API/commands/frame/diagnose/inheritance/TestDiagnoseInheritance.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/inheritance/main.cpp b/lldb/test/API/commands/frame/diagnose/inheritance/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/inheritance/main.cpp
rename to lldb/test/API/commands/frame/diagnose/inheritance/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/local-variable/Makefile b/lldb/test/API/commands/frame/diagnose/local-variable/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/local-variable/Makefile
rename to lldb/test/API/commands/frame/diagnose/local-variable/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/local-variable/TestLocalVariable.py b/lldb/test/API/commands/frame/diagnose/local-variable/TestLocalVariable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/local-variable/TestLocalVariable.py
rename to lldb/test/API/commands/frame/diagnose/local-variable/TestLocalVariable.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/local-variable/main.c b/lldb/test/API/commands/frame/diagnose/local-variable/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/local-variable/main.c
rename to lldb/test/API/commands/frame/diagnose/local-variable/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/virtual-method-call/Makefile b/lldb/test/API/commands/frame/diagnose/virtual-method-call/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/virtual-method-call/Makefile
rename to lldb/test/API/commands/frame/diagnose/virtual-method-call/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/virtual-method-call/TestDiagnoseDereferenceVirtualMethodCall.py b/lldb/test/API/commands/frame/diagnose/virtual-method-call/TestDiagnoseDereferenceVirtualMethodCall.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/virtual-method-call/TestDiagnoseDereferenceVirtualMethodCall.py
rename to lldb/test/API/commands/frame/diagnose/virtual-method-call/TestDiagnoseDereferenceVirtualMethodCall.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/virtual-method-call/main.cpp b/lldb/test/API/commands/frame/diagnose/virtual-method-call/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/diagnose/virtual-method-call/main.cpp
rename to lldb/test/API/commands/frame/diagnose/virtual-method-call/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/Makefile b/lldb/test/API/commands/frame/language/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/Makefile
rename to lldb/test/API/commands/frame/language/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/TestGuessLanguage.py b/lldb/test/API/commands/frame/language/TestGuessLanguage.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/TestGuessLanguage.py
rename to lldb/test/API/commands/frame/language/TestGuessLanguage.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/main.cpp b/lldb/test/API/commands/frame/language/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/main.cpp
rename to lldb/test/API/commands/frame/language/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/other-2.cpp b/lldb/test/API/commands/frame/language/other-2.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/other-2.cpp
rename to lldb/test/API/commands/frame/language/other-2.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/other.cpp b/lldb/test/API/commands/frame/language/other.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/other.cpp
rename to lldb/test/API/commands/frame/language/other.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/other.h b/lldb/test/API/commands/frame/language/other.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/other.h
rename to lldb/test/API/commands/frame/language/other.h
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/somefunc.c b/lldb/test/API/commands/frame/language/somefunc.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/language/somefunc.c
rename to lldb/test/API/commands/frame/language/somefunc.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/Makefile b/lldb/test/API/commands/frame/recognizer/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/Makefile
rename to lldb/test/API/commands/frame/recognizer/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/TestFrameRecognizer.py b/lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/TestFrameRecognizer.py
rename to lldb/test/API/commands/frame/recognizer/TestFrameRecognizer.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/main.m b/lldb/test/API/commands/frame/recognizer/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/main.m
rename to lldb/test/API/commands/frame/recognizer/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/recognizer.py b/lldb/test/API/commands/frame/recognizer/recognizer.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/recognizer.py
rename to lldb/test/API/commands/frame/recognizer/recognizer.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/select/Makefile b/lldb/test/API/commands/frame/select/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/select/Makefile
rename to lldb/test/API/commands/frame/select/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/select/TestFrameSelect.py b/lldb/test/API/commands/frame/select/TestFrameSelect.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/select/TestFrameSelect.py
rename to lldb/test/API/commands/frame/select/TestFrameSelect.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/select/main.cpp b/lldb/test/API/commands/frame/select/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/select/main.cpp
rename to lldb/test/API/commands/frame/select/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/var-scope/TestFrameVariableScope.py b/lldb/test/API/commands/frame/var-scope/TestFrameVariableScope.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/var-scope/TestFrameVariableScope.py
rename to lldb/test/API/commands/frame/var-scope/TestFrameVariableScope.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/var-scope/main.c b/lldb/test/API/commands/frame/var-scope/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/var-scope/main.c
rename to lldb/test/API/commands/frame/var-scope/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/var/Makefile b/lldb/test/API/commands/frame/var/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/var/Makefile
rename to lldb/test/API/commands/frame/var/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/var/TestFrameVar.py b/lldb/test/API/commands/frame/var/TestFrameVar.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/var/TestFrameVar.py
rename to lldb/test/API/commands/frame/var/TestFrameVar.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/var/main.c b/lldb/test/API/commands/frame/var/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/frame/var/main.c
rename to lldb/test/API/commands/frame/var/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basic/Makefile b/lldb/test/API/commands/gui/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/gui/basic/Makefile
rename to lldb/test/API/commands/gui/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basic/TestGuiBasic.py b/lldb/test/API/commands/gui/basic/TestGuiBasic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/gui/basic/TestGuiBasic.py
rename to lldb/test/API/commands/gui/basic/TestGuiBasic.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/basic/main.c b/lldb/test/API/commands/gui/basic/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/gui/basic/main.c
rename to lldb/test/API/commands/gui/basic/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/gui/invalid-args/TestInvalidArgsGui.py b/lldb/test/API/commands/gui/invalid-args/TestInvalidArgsGui.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/gui/invalid-args/TestInvalidArgsGui.py
rename to lldb/test/API/commands/gui/invalid-args/TestInvalidArgsGui.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/help/TestHelp.py b/lldb/test/API/commands/help/TestHelp.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/help/TestHelp.py
rename to lldb/test/API/commands/help/TestHelp.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/log/basic/Makefile b/lldb/test/API/commands/log/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/log/basic/Makefile
rename to lldb/test/API/commands/log/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/log/basic/TestLogging.py b/lldb/test/API/commands/log/basic/TestLogging.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/log/basic/TestLogging.py
rename to lldb/test/API/commands/log/basic/TestLogging.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/log/basic/main.cpp b/lldb/test/API/commands/log/basic/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/log/basic/main.cpp
rename to lldb/test/API/commands/log/basic/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/log/invalid-args/TestInvalidArgsLog.py b/lldb/test/API/commands/log/invalid-args/TestInvalidArgsLog.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/log/invalid-args/TestInvalidArgsLog.py
rename to lldb/test/API/commands/log/invalid-args/TestInvalidArgsLog.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/platform/basic/TestPlatformCommand.py b/lldb/test/API/commands/platform/basic/TestPlatformCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/platform/basic/TestPlatformCommand.py
rename to lldb/test/API/commands/platform/basic/TestPlatformCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/platform/basic/TestPlatformPython.py b/lldb/test/API/commands/platform/basic/TestPlatformPython.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/platform/basic/TestPlatformPython.py
rename to lldb/test/API/commands/platform/basic/TestPlatformPython.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/platform/process/Makefile b/lldb/test/API/commands/platform/process/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/platform/process/Makefile
rename to lldb/test/API/commands/platform/process/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/platform/process/TestProcessList.py b/lldb/test/API/commands/platform/process/TestProcessList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/platform/process/TestProcessList.py
rename to lldb/test/API/commands/platform/process/TestProcessList.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/platform/process/main.cpp b/lldb/test/API/commands/platform/process/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/platform/process/main.cpp
rename to lldb/test/API/commands/platform/process/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach-resume/Makefile b/lldb/test/API/commands/process/attach-resume/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach-resume/Makefile
rename to lldb/test/API/commands/process/attach-resume/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach-resume/TestAttachResume.py b/lldb/test/API/commands/process/attach-resume/TestAttachResume.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach-resume/TestAttachResume.py
rename to lldb/test/API/commands/process/attach-resume/TestAttachResume.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach-resume/main.cpp b/lldb/test/API/commands/process/attach-resume/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach-resume/main.cpp
rename to lldb/test/API/commands/process/attach-resume/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/Makefile b/lldb/test/API/commands/process/attach/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/Makefile
rename to lldb/test/API/commands/process/attach/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/TestProcessAttach.py b/lldb/test/API/commands/process/attach/TestProcessAttach.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/TestProcessAttach.py
rename to lldb/test/API/commands/process/attach/TestProcessAttach.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/Makefile b/lldb/test/API/commands/process/attach/attach_denied/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/Makefile
rename to lldb/test/API/commands/process/attach/attach_denied/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/TestAttachDenied.py b/lldb/test/API/commands/process/attach/attach_denied/TestAttachDenied.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/TestAttachDenied.py
rename to lldb/test/API/commands/process/attach/attach_denied/TestAttachDenied.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/entitlements.plist b/lldb/test/API/commands/process/attach/attach_denied/entitlements.plist
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/entitlements.plist
rename to lldb/test/API/commands/process/attach/attach_denied/entitlements.plist
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/main.cpp b/lldb/test/API/commands/process/attach/attach_denied/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/attach_denied/main.cpp
rename to lldb/test/API/commands/process/attach/attach_denied/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/attach/main.cpp b/lldb/test/API/commands/process/attach/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/attach/main.cpp
rename to lldb/test/API/commands/process/attach/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/Makefile b/lldb/test/API/commands/process/launch-with-shellexpand/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/Makefile
rename to lldb/test/API/commands/process/launch-with-shellexpand/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py b/lldb/test/API/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py
rename to lldb/test/API/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file1.txt b/lldb/test/API/commands/process/launch-with-shellexpand/file1.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file1.txt
rename to lldb/test/API/commands/process/launch-with-shellexpand/file1.txt
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file2.txt b/lldb/test/API/commands/process/launch-with-shellexpand/file2.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file2.txt
rename to lldb/test/API/commands/process/launch-with-shellexpand/file2.txt
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file3.txt b/lldb/test/API/commands/process/launch-with-shellexpand/file3.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file3.txt
rename to lldb/test/API/commands/process/launch-with-shellexpand/file3.txt
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file4.txy b/lldb/test/API/commands/process/launch-with-shellexpand/file4.txy
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file4.txy
rename to lldb/test/API/commands/process/launch-with-shellexpand/file4.txy
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file5.tyx b/lldb/test/API/commands/process/launch-with-shellexpand/file5.tyx
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/file5.tyx
rename to lldb/test/API/commands/process/launch-with-shellexpand/file5.tyx
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/foo bar b/lldb/test/API/commands/process/launch-with-shellexpand/foo bar
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/foo bar
rename to lldb/test/API/commands/process/launch-with-shellexpand/foo bar
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/main.cpp b/lldb/test/API/commands/process/launch-with-shellexpand/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch-with-shellexpand/main.cpp
rename to lldb/test/API/commands/process/launch-with-shellexpand/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch/Makefile b/lldb/test/API/commands/process/launch/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch/Makefile
rename to lldb/test/API/commands/process/launch/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch/TestProcessLaunch.py b/lldb/test/API/commands/process/launch/TestProcessLaunch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch/TestProcessLaunch.py
rename to lldb/test/API/commands/process/launch/TestProcessLaunch.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch/input-file.txt b/lldb/test/API/commands/process/launch/input-file.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch/input-file.txt
rename to lldb/test/API/commands/process/launch/input-file.txt
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch/main.cpp b/lldb/test/API/commands/process/launch/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch/main.cpp
rename to lldb/test/API/commands/process/launch/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch/print_cwd.cpp b/lldb/test/API/commands/process/launch/print_cwd.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch/print_cwd.cpp
rename to lldb/test/API/commands/process/launch/print_cwd.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/process/launch/print_env.cpp b/lldb/test/API/commands/process/launch/print_env.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/process/launch/print_env.cpp
rename to lldb/test/API/commands/process/launch/print_env.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/quit/TestQuit.py b/lldb/test/API/commands/quit/TestQuit.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/quit/TestQuit.py
rename to lldb/test/API/commands/quit/TestQuit.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/Makefile b/lldb/test/API/commands/register/register/intel_xtended_registers/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/Makefile
rename to lldb/test/API/commands/register/register/intel_xtended_registers/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/TestMPXRegisters.py b/lldb/test/API/commands/register/register/intel_xtended_registers/TestMPXRegisters.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/TestMPXRegisters.py
rename to lldb/test/API/commands/register/register/intel_xtended_registers/TestMPXRegisters.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/main.cpp b/lldb/test/API/commands/register/register/intel_xtended_registers/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/main.cpp
rename to lldb/test/API/commands/register/register/intel_xtended_registers/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile b/lldb/test/API/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile
rename to lldb/test/API/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/TestBoundViolation.py b/lldb/test/API/commands/register/register/intel_xtended_registers/mpx_bound_violation/TestBoundViolation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/TestBoundViolation.py
rename to lldb/test/API/commands/register/register/intel_xtended_registers/mpx_bound_violation/TestBoundViolation.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/main.cpp b/lldb/test/API/commands/register/register/intel_xtended_registers/mpx_bound_violation/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/main.cpp
rename to lldb/test/API/commands/register/register/intel_xtended_registers/mpx_bound_violation/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_offset_intersection/Makefile b/lldb/test/API/commands/register/register/intel_xtended_registers/mpx_offset_intersection/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_offset_intersection/Makefile
rename to lldb/test/API/commands/register/register/intel_xtended_registers/mpx_offset_intersection/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_offset_intersection/TestMPXOffsetIntersection.py b/lldb/test/API/commands/register/register/intel_xtended_registers/mpx_offset_intersection/TestMPXOffsetIntersection.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_offset_intersection/TestMPXOffsetIntersection.py
rename to lldb/test/API/commands/register/register/intel_xtended_registers/mpx_offset_intersection/TestMPXOffsetIntersection.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_offset_intersection/main.cpp b/lldb/test/API/commands/register/register/intel_xtended_registers/mpx_offset_intersection/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_offset_intersection/main.cpp
rename to lldb/test/API/commands/register/register/intel_xtended_registers/mpx_offset_intersection/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/Makefile b/lldb/test/API/commands/register/register/register_command/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/Makefile
rename to lldb/test/API/commands/register/register/register_command/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/TestRegisters.py b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/TestRegisters.py
rename to lldb/test/API/commands/register/register/register_command/TestRegisters.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/a.cpp b/lldb/test/API/commands/register/register/register_command/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/a.cpp
rename to lldb/test/API/commands/register/register/register_command/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/main.cpp b/lldb/test/API/commands/register/register/register_command/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/register/register/register_command/main.cpp
rename to lldb/test/API/commands/register/register/register_command/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/reproducer/invalid-args/TestInvalidArgsReproducer.py b/lldb/test/API/commands/reproducer/invalid-args/TestInvalidArgsReproducer.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/reproducer/invalid-args/TestInvalidArgsReproducer.py
rename to lldb/test/API/commands/reproducer/invalid-args/TestInvalidArgsReproducer.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/settings/Makefile b/lldb/test/API/commands/settings/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/settings/Makefile
rename to lldb/test/API/commands/settings/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/settings/TestSettings.py b/lldb/test/API/commands/settings/TestSettings.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/settings/TestSettings.py
rename to lldb/test/API/commands/settings/TestSettings.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/settings/main.cpp b/lldb/test/API/commands/settings/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/settings/main.cpp
rename to lldb/test/API/commands/settings/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/settings/quoting/Makefile b/lldb/test/API/commands/settings/quoting/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/settings/quoting/Makefile
rename to lldb/test/API/commands/settings/quoting/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/settings/quoting/TestQuoting.py b/lldb/test/API/commands/settings/quoting/TestQuoting.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/settings/quoting/TestQuoting.py
rename to lldb/test/API/commands/settings/quoting/TestQuoting.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/settings/quoting/main.c b/lldb/test/API/commands/settings/quoting/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/settings/quoting/main.c
rename to lldb/test/API/commands/settings/quoting/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/source/info/TestSourceInfo.py b/lldb/test/API/commands/source/info/TestSourceInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/source/info/TestSourceInfo.py
rename to lldb/test/API/commands/source/info/TestSourceInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/source/info/main.cpp b/lldb/test/API/commands/source/info/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/source/info/main.cpp
rename to lldb/test/API/commands/source/info/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/source/info/second.cpp b/lldb/test/API/commands/source/info/second.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/source/info/second.cpp
rename to lldb/test/API/commands/source/info/second.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/statistics/basic/TestStats.py b/lldb/test/API/commands/statistics/basic/TestStats.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/statistics/basic/TestStats.py
rename to lldb/test/API/commands/statistics/basic/TestStats.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/statistics/basic/main.c b/lldb/test/API/commands/statistics/basic/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/statistics/basic/main.c
rename to lldb/test/API/commands/statistics/basic/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/auto-install-main-executable/Makefile b/lldb/test/API/commands/target/auto-install-main-executable/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/auto-install-main-executable/Makefile
rename to lldb/test/API/commands/target/auto-install-main-executable/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py b/lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
rename to lldb/test/API/commands/target/auto-install-main-executable/TestAutoInstallMainExecutable.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/auto-install-main-executable/main.cpp b/lldb/test/API/commands/target/auto-install-main-executable/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/auto-install-main-executable/main.cpp
rename to lldb/test/API/commands/target/auto-install-main-executable/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/Makefile b/lldb/test/API/commands/target/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/Makefile
rename to lldb/test/API/commands/target/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/TestTargetCommand.py b/lldb/test/API/commands/target/basic/TestTargetCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/TestTargetCommand.py
rename to lldb/test/API/commands/target/basic/TestTargetCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/a.c b/lldb/test/API/commands/target/basic/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/a.c
rename to lldb/test/API/commands/target/basic/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/b.c b/lldb/test/API/commands/target/basic/b.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/b.c
rename to lldb/test/API/commands/target/basic/b.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/c.c b/lldb/test/API/commands/target/basic/c.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/c.c
rename to lldb/test/API/commands/target/basic/c.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/globals.c b/lldb/test/API/commands/target/basic/globals.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/globals.c
rename to lldb/test/API/commands/target/basic/globals.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/basic/invalid_core_file b/lldb/test/API/commands/target/basic/invalid_core_file
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/basic/invalid_core_file
rename to lldb/test/API/commands/target/basic/invalid_core_file
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-deps/Makefile b/lldb/test/API/commands/target/create-deps/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-deps/Makefile
rename to lldb/test/API/commands/target/create-deps/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-deps/TestTargetCreateDeps.py b/lldb/test/API/commands/target/create-deps/TestTargetCreateDeps.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-deps/TestTargetCreateDeps.py
rename to lldb/test/API/commands/target/create-deps/TestTargetCreateDeps.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-deps/a.cpp b/lldb/test/API/commands/target/create-deps/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-deps/a.cpp
rename to lldb/test/API/commands/target/create-deps/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-deps/main.cpp b/lldb/test/API/commands/target/create-deps/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-deps/main.cpp
rename to lldb/test/API/commands/target/create-deps/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-no-such-arch/Makefile b/lldb/test/API/commands/target/create-no-such-arch/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-no-such-arch/Makefile
rename to lldb/test/API/commands/target/create-no-such-arch/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-no-such-arch/TestNoSuchArch.py b/lldb/test/API/commands/target/create-no-such-arch/TestNoSuchArch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-no-such-arch/TestNoSuchArch.py
rename to lldb/test/API/commands/target/create-no-such-arch/TestNoSuchArch.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/create-no-such-arch/main.cpp b/lldb/test/API/commands/target/create-no-such-arch/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/create-no-such-arch/main.cpp
rename to lldb/test/API/commands/target/create-no-such-arch/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/dump-symtab-demangle/TestDumpSymtabDemangle.py b/lldb/test/API/commands/target/dump-symtab-demangle/TestDumpSymtabDemangle.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/dump-symtab-demangle/TestDumpSymtabDemangle.py
rename to lldb/test/API/commands/target/dump-symtab-demangle/TestDumpSymtabDemangle.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/dump-symtab-demangle/a.yaml b/lldb/test/API/commands/target/dump-symtab-demangle/a.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/dump-symtab-demangle/a.yaml
rename to lldb/test/API/commands/target/dump-symtab-demangle/a.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/Makefile b/lldb/test/API/commands/target/stop-hooks/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/Makefile
rename to lldb/test/API/commands/target/stop-hooks/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/TestStopHooks.py b/lldb/test/API/commands/target/stop-hooks/TestStopHooks.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/TestStopHooks.py
rename to lldb/test/API/commands/target/stop-hooks/TestStopHooks.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/main.c b/lldb/test/API/commands/target/stop-hooks/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/main.c
rename to lldb/test/API/commands/target/stop-hooks/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/version/TestVersion.py b/lldb/test/API/commands/version/TestVersion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/version/TestVersion.py
rename to lldb/test/API/commands/version/TestVersion.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/.categories b/lldb/test/API/commands/watchpoints/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/.categories
rename to lldb/test/API/commands/watchpoints/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchlocation/Makefile b/lldb/test/API/commands/watchpoints/hello_watchlocation/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchlocation/Makefile
rename to lldb/test/API/commands/watchpoints/hello_watchlocation/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchlocation/TestWatchLocation.py b/lldb/test/API/commands/watchpoints/hello_watchlocation/TestWatchLocation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchlocation/TestWatchLocation.py
rename to lldb/test/API/commands/watchpoints/hello_watchlocation/TestWatchLocation.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchlocation/main.cpp b/lldb/test/API/commands/watchpoints/hello_watchlocation/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchlocation/main.cpp
rename to lldb/test/API/commands/watchpoints/hello_watchlocation/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchpoint/Makefile b/lldb/test/API/commands/watchpoints/hello_watchpoint/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchpoint/Makefile
rename to lldb/test/API/commands/watchpoints/hello_watchpoint/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchpoint/TestMyFirstWatchpoint.py b/lldb/test/API/commands/watchpoints/hello_watchpoint/TestMyFirstWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchpoint/TestMyFirstWatchpoint.py
rename to lldb/test/API/commands/watchpoints/hello_watchpoint/TestMyFirstWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchpoint/main.c b/lldb/test/API/commands/watchpoints/hello_watchpoint/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/hello_watchpoint/main.c
rename to lldb/test/API/commands/watchpoints/hello_watchpoint/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multi_watchpoint_slots/Makefile b/lldb/test/API/commands/watchpoints/multi_watchpoint_slots/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multi_watchpoint_slots/Makefile
rename to lldb/test/API/commands/watchpoints/multi_watchpoint_slots/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multi_watchpoint_slots/TestWatchpointMultipleSlots.py b/lldb/test/API/commands/watchpoints/multi_watchpoint_slots/TestWatchpointMultipleSlots.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multi_watchpoint_slots/TestWatchpointMultipleSlots.py
rename to lldb/test/API/commands/watchpoints/multi_watchpoint_slots/TestWatchpointMultipleSlots.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multi_watchpoint_slots/main.c b/lldb/test/API/commands/watchpoints/multi_watchpoint_slots/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multi_watchpoint_slots/main.c
rename to lldb/test/API/commands/watchpoints/multi_watchpoint_slots/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_hits/Makefile b/lldb/test/API/commands/watchpoints/multiple_hits/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_hits/Makefile
rename to lldb/test/API/commands/watchpoints/multiple_hits/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_hits/TestMultipleHits.py b/lldb/test/API/commands/watchpoints/multiple_hits/TestMultipleHits.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_hits/TestMultipleHits.py
rename to lldb/test/API/commands/watchpoints/multiple_hits/TestMultipleHits.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_hits/main.cpp b/lldb/test/API/commands/watchpoints/multiple_hits/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_hits/main.cpp
rename to lldb/test/API/commands/watchpoints/multiple_hits/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_threads/Makefile b/lldb/test/API/commands/watchpoints/multiple_threads/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_threads/Makefile
rename to lldb/test/API/commands/watchpoints/multiple_threads/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_threads/TestWatchpointMultipleThreads.py b/lldb/test/API/commands/watchpoints/multiple_threads/TestWatchpointMultipleThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_threads/TestWatchpointMultipleThreads.py
rename to lldb/test/API/commands/watchpoints/multiple_threads/TestWatchpointMultipleThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_threads/main.cpp b/lldb/test/API/commands/watchpoints/multiple_threads/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/multiple_threads/main.cpp
rename to lldb/test/API/commands/watchpoints/multiple_threads/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/step_over_watchpoint/Makefile b/lldb/test/API/commands/watchpoints/step_over_watchpoint/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/step_over_watchpoint/Makefile
rename to lldb/test/API/commands/watchpoints/step_over_watchpoint/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/step_over_watchpoint/TestStepOverWatchpoint.py b/lldb/test/API/commands/watchpoints/step_over_watchpoint/TestStepOverWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/step_over_watchpoint/TestStepOverWatchpoint.py
rename to lldb/test/API/commands/watchpoints/step_over_watchpoint/TestStepOverWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/step_over_watchpoint/main.c b/lldb/test/API/commands/watchpoints/step_over_watchpoint/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/step_over_watchpoint/main.c
rename to lldb/test/API/commands/watchpoints/step_over_watchpoint/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/variable_out_of_scope/Makefile b/lldb/test/API/commands/watchpoints/variable_out_of_scope/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/variable_out_of_scope/Makefile
rename to lldb/test/API/commands/watchpoints/variable_out_of_scope/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/variable_out_of_scope/TestWatchedVarHitWhenInScope.py b/lldb/test/API/commands/watchpoints/variable_out_of_scope/TestWatchedVarHitWhenInScope.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/variable_out_of_scope/TestWatchedVarHitWhenInScope.py
rename to lldb/test/API/commands/watchpoints/variable_out_of_scope/TestWatchedVarHitWhenInScope.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/variable_out_of_scope/main.c b/lldb/test/API/commands/watchpoints/variable_out_of_scope/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/variable_out_of_scope/main.c
rename to lldb/test/API/commands/watchpoints/variable_out_of_scope/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_commands/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/TestWatchpointCommands.py b/lldb/test/API/commands/watchpoints/watchpoint_commands/TestWatchpointCommands.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/TestWatchpointCommands.py
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/TestWatchpointCommands.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_commands/command/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/command/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandLLDB.py b/lldb/test/API/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandLLDB.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandLLDB.py
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandLLDB.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandPython.py b/lldb/test/API/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandPython.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandPython.py
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/command/TestWatchpointCommandPython.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/main.cpp b/lldb/test/API/commands/watchpoints/watchpoint_commands/command/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/main.cpp
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/command/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/watchpoint_command.py b/lldb/test/API/commands/watchpoints/watchpoint_commands/command/watchpoint_command.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/command/watchpoint_command.py
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/command/watchpoint_command.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/condition/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_commands/condition/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/condition/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/condition/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/condition/TestWatchpointConditionCmd.py b/lldb/test/API/commands/watchpoints/watchpoint_commands/condition/TestWatchpointConditionCmd.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/condition/TestWatchpointConditionCmd.py
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/condition/TestWatchpointConditionCmd.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/condition/main.cpp b/lldb/test/API/commands/watchpoints/watchpoint_commands/condition/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/condition/main.cpp
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/condition/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/main.c b/lldb/test/API/commands/watchpoints/watchpoint_commands/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_commands/main.c
rename to lldb/test/API/commands/watchpoints/watchpoint_commands/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_disable/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_disable/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_disable/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_disable/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_disable/TestWatchpointDisable.py b/lldb/test/API/commands/watchpoints/watchpoint_disable/TestWatchpointDisable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_disable/TestWatchpointDisable.py
rename to lldb/test/API/commands/watchpoints/watchpoint_disable/TestWatchpointDisable.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_disable/main.c b/lldb/test/API/commands/watchpoints/watchpoint_disable/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_disable/main.c
rename to lldb/test/API/commands/watchpoints/watchpoint_disable/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_events/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_events/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_events/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_events/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py b/lldb/test/API/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py
rename to lldb/test/API/commands/watchpoints/watchpoint_events/TestWatchpointEvents.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_events/main.c b/lldb/test/API/commands/watchpoints/watchpoint_events/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_events/main.c
rename to lldb/test/API/commands/watchpoints/watchpoint_events/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_on_vectors/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_on_vectors/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_on_vectors/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_on_vectors/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_on_vectors/TestValueOfVectorVariable.py b/lldb/test/API/commands/watchpoints/watchpoint_on_vectors/TestValueOfVectorVariable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_on_vectors/TestValueOfVectorVariable.py
rename to lldb/test/API/commands/watchpoints/watchpoint_on_vectors/TestValueOfVectorVariable.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_on_vectors/main.c b/lldb/test/API/commands/watchpoints/watchpoint_on_vectors/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_on_vectors/main.c
rename to lldb/test/API/commands/watchpoints/watchpoint_on_vectors/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_set_command/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_set_command/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_set_command/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_set_command/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_set_command/TestWatchLocationWithWatchSet.py b/lldb/test/API/commands/watchpoints/watchpoint_set_command/TestWatchLocationWithWatchSet.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_set_command/TestWatchLocationWithWatchSet.py
rename to lldb/test/API/commands/watchpoints/watchpoint_set_command/TestWatchLocationWithWatchSet.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_set_command/main.cpp b/lldb/test/API/commands/watchpoints/watchpoint_set_command/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_set_command/main.cpp
rename to lldb/test/API/commands/watchpoints/watchpoint_set_command/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_size/Makefile b/lldb/test/API/commands/watchpoints/watchpoint_size/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_size/Makefile
rename to lldb/test/API/commands/watchpoints/watchpoint_size/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_size/TestWatchpointSizes.py b/lldb/test/API/commands/watchpoints/watchpoint_size/TestWatchpointSizes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_size/TestWatchpointSizes.py
rename to lldb/test/API/commands/watchpoints/watchpoint_size/TestWatchpointSizes.py
diff --git a/lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_size/main.c b/lldb/test/API/commands/watchpoints/watchpoint_size/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/commands/watchpoints/watchpoint_size/main.c
rename to lldb/test/API/commands/watchpoints/watchpoint_size/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/driver/batch_mode/Makefile b/lldb/test/API/driver/batch_mode/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/driver/batch_mode/Makefile
rename to lldb/test/API/driver/batch_mode/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/driver/batch_mode/TestBatchMode.py b/lldb/test/API/driver/batch_mode/TestBatchMode.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/driver/batch_mode/TestBatchMode.py
rename to lldb/test/API/driver/batch_mode/TestBatchMode.py
diff --git a/lldb/packages/Python/lldbsuite/test/driver/batch_mode/main.c b/lldb/test/API/driver/batch_mode/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/driver/batch_mode/main.c
rename to lldb/test/API/driver/batch_mode/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/abbreviation/.categories b/lldb/test/API/functionalities/abbreviation/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/abbreviation/.categories
rename to lldb/test/API/functionalities/abbreviation/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/abbreviation/TestAbbreviations.py b/lldb/test/API/functionalities/abbreviation/TestAbbreviations.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/abbreviation/TestAbbreviations.py
rename to lldb/test/API/functionalities/abbreviation/TestAbbreviations.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/abbreviation/TestCommonShortSpellings.py b/lldb/test/API/functionalities/abbreviation/TestCommonShortSpellings.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/abbreviation/TestCommonShortSpellings.py
rename to lldb/test/API/functionalities/abbreviation/TestCommonShortSpellings.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/alias/.categories b/lldb/test/API/functionalities/alias/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/alias/.categories
rename to lldb/test/API/functionalities/alias/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/archives/Makefile b/lldb/test/API/functionalities/archives/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/archives/Makefile
rename to lldb/test/API/functionalities/archives/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/archives/README b/lldb/test/API/functionalities/archives/README
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/archives/README
rename to lldb/test/API/functionalities/archives/README
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/archives/TestBSDArchives.py b/lldb/test/API/functionalities/archives/TestBSDArchives.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/archives/TestBSDArchives.py
rename to lldb/test/API/functionalities/archives/TestBSDArchives.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/archives/a.c b/lldb/test/API/functionalities/archives/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/archives/a.c
rename to lldb/test/API/functionalities/archives/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/archives/b.c b/lldb/test/API/functionalities/archives/b.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/archives/b.c
rename to lldb/test/API/functionalities/archives/b.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/archives/main.c b/lldb/test/API/functionalities/archives/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/archives/main.c
rename to lldb/test/API/functionalities/archives/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/asan/Makefile b/lldb/test/API/functionalities/asan/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/asan/Makefile
rename to lldb/test/API/functionalities/asan/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/asan/TestMemoryHistory.py b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/asan/TestMemoryHistory.py
rename to lldb/test/API/functionalities/asan/TestMemoryHistory.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/asan/TestReportData.py b/lldb/test/API/functionalities/asan/TestReportData.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/asan/TestReportData.py
rename to lldb/test/API/functionalities/asan/TestReportData.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/asan/main.c b/lldb/test/API/functionalities/asan/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/asan/main.c
rename to lldb/test/API/functionalities/asan/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/avoids-fd-leak/Makefile b/lldb/test/API/functionalities/avoids-fd-leak/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/avoids-fd-leak/Makefile
rename to lldb/test/API/functionalities/avoids-fd-leak/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/avoids-fd-leak/TestFdLeak.py b/lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/avoids-fd-leak/TestFdLeak.py
rename to lldb/test/API/functionalities/avoids-fd-leak/TestFdLeak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/avoids-fd-leak/main.c b/lldb/test/API/functionalities/avoids-fd-leak/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/avoids-fd-leak/main.c
rename to lldb/test/API/functionalities/avoids-fd-leak/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/backticks/.categories b/lldb/test/API/functionalities/backticks/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/backticks/.categories
rename to lldb/test/API/functionalities/backticks/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/backticks/TestBackticksWithoutATarget.py b/lldb/test/API/functionalities/backticks/TestBackticksWithoutATarget.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/backticks/TestBackticksWithoutATarget.py
rename to lldb/test/API/functionalities/backticks/TestBackticksWithoutATarget.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/address_breakpoints/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/Makefile
rename to lldb/test/API/functionalities/breakpoint/address_breakpoints/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/TestAddressBreakpoints.py b/lldb/test/API/functionalities/breakpoint/address_breakpoints/TestAddressBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/TestAddressBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/address_breakpoints/TestAddressBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/TestBadAddressBreakpoints.py b/lldb/test/API/functionalities/breakpoint/address_breakpoints/TestBadAddressBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/TestBadAddressBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/address_breakpoints/TestBadAddressBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/main.c b/lldb/test/API/functionalities/breakpoint/address_breakpoints/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/main.c
rename to lldb/test/API/functionalities/breakpoint/address_breakpoints/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/Makefile b/lldb/test/API/functionalities/breakpoint/auto_continue/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/Makefile
rename to lldb/test/API/functionalities/breakpoint/auto_continue/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/TestBreakpointAutoContinue.py b/lldb/test/API/functionalities/breakpoint/auto_continue/TestBreakpointAutoContinue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/TestBreakpointAutoContinue.py
rename to lldb/test/API/functionalities/breakpoint/auto_continue/TestBreakpointAutoContinue.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/main.c b/lldb/test/API/functionalities/breakpoint/auto_continue/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/main.c
rename to lldb/test/API/functionalities/breakpoint/auto_continue/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/TestBreakpointByLineAndColumn.py b/lldb/test/API/functionalities/breakpoint/breakpoint_by_line_and_column/TestBreakpointByLineAndColumn.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/TestBreakpointByLineAndColumn.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_by_line_and_column/TestBreakpointByLineAndColumn.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_by_line_and_column/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_by_line_and_column/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py b/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/source.lldb b/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/source.lldb
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_callback_command_source/source.lldb
rename to lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/source.lldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_command/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestBreakpointCommandsFromPython.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommandsFromPython.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestBreakpointCommandsFromPython.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommandsFromPython.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestRegexpBreakCommand.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestRegexpBreakCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/TestRegexpBreakCommand.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/TestRegexpBreakCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/a.c b/lldb/test/API/functionalities/breakpoint/breakpoint_command/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/a.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/b.c b/lldb/test/API/functionalities/breakpoint/breakpoint_command/b.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/b.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/b.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/bktptcmd.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/bktptcmd.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/bktptcmd.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/bktptcmd.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_command/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/side_effect.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/side_effect.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_command/side_effect.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_command/side_effect.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_conditions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_conditions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/TestBreakpointConditions.py b/lldb/test/API/functionalities/breakpoint/breakpoint_conditions/TestBreakpointConditions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/TestBreakpointConditions.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_conditions/TestBreakpointConditions.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_conditions/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_conditions/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_hit_count/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_hit_count/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_hit_count/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_hit_count/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_hit_count/TestBreakpointHitCount.py b/lldb/test/API/functionalities/breakpoint/breakpoint_hit_count/TestBreakpointHitCount.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_hit_count/TestBreakpointHitCount.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_hit_count/TestBreakpointHitCount.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_hit_count/main.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_hit_count/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_hit_count/main.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_hit_count/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_ids/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_ids/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/TestBreakpointIDs.py b/lldb/test/API/functionalities/breakpoint/breakpoint_ids/TestBreakpointIDs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/TestBreakpointIDs.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_ids/TestBreakpointIDs.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/main.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_ids/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/main.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_ids/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py b/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/TestBreakpointIgnoreCount.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ignore_count/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_ignore_count/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_in_delayslot/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_in_delayslot/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_in_delayslot/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_in_delayslot/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_in_delayslot/TestAvoidBreakpointInDelaySlot.py b/lldb/test/API/functionalities/breakpoint/breakpoint_in_delayslot/TestAvoidBreakpointInDelaySlot.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_in_delayslot/TestAvoidBreakpointInDelaySlot.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_in_delayslot/TestAvoidBreakpointInDelaySlot.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_in_delayslot/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_in_delayslot/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_in_delayslot/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_in_delayslot/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_language/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_language/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/TestBreakpointLanguage.py b/lldb/test/API/functionalities/breakpoint/breakpoint_language/TestBreakpointLanguage.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/TestBreakpointLanguage.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_language/TestBreakpointLanguage.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/a.c b/lldb/test/API/functionalities/breakpoint/breakpoint_language/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/a.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_language/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/b.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_language/b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/b.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_language/b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/main.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_language/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_language/main.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_language/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_locations/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_locations/TestBreakpointLocations.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_locations/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_names/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_names/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_names/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_names/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py b/lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_names/TestBreakpointNames.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_names/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_names/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_names/main.c
rename to lldb/test/API/functionalities/breakpoint/breakpoint_names/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_options/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_options/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py b/lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/foo.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_options/foo.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/foo.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_options/foo.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/main.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_options/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_options/main.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_options/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_set_restart/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_set_restart/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_set_restart/Makefile
rename to lldb/test/API/functionalities/breakpoint/breakpoint_set_restart/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_set_restart/TestBreakpointSetRestart.py b/lldb/test/API/functionalities/breakpoint/breakpoint_set_restart/TestBreakpointSetRestart.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_set_restart/TestBreakpointSetRestart.py
rename to lldb/test/API/functionalities/breakpoint/breakpoint_set_restart/TestBreakpointSetRestart.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_set_restart/main.cpp b/lldb/test/API/functionalities/breakpoint/breakpoint_set_restart/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_set_restart/main.cpp
rename to lldb/test/API/functionalities/breakpoint/breakpoint_set_restart/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/comp_dir_symlink/Makefile b/lldb/test/API/functionalities/breakpoint/comp_dir_symlink/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/comp_dir_symlink/Makefile
rename to lldb/test/API/functionalities/breakpoint/comp_dir_symlink/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/comp_dir_symlink/TestCompDirSymLink.py b/lldb/test/API/functionalities/breakpoint/comp_dir_symlink/TestCompDirSymLink.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/comp_dir_symlink/TestCompDirSymLink.py
rename to lldb/test/API/functionalities/breakpoint/comp_dir_symlink/TestCompDirSymLink.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/comp_dir_symlink/main.cpp b/lldb/test/API/functionalities/breakpoint/comp_dir_symlink/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/comp_dir_symlink/main.cpp
rename to lldb/test/API/functionalities/breakpoint/comp_dir_symlink/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/Makefile
rename to lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/TestConsecutiveBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/main.cpp b/lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/main.cpp
rename to lldb/test/API/functionalities/breakpoint/consecutive_breakpoints/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/Makefile b/lldb/test/API/functionalities/breakpoint/cpp/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/Makefile
rename to lldb/test/API/functionalities/breakpoint/cpp/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/TestCPPBreakpointLocations.py b/lldb/test/API/functionalities/breakpoint/cpp/TestCPPBreakpointLocations.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/TestCPPBreakpointLocations.py
rename to lldb/test/API/functionalities/breakpoint/cpp/TestCPPBreakpointLocations.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/main.cpp b/lldb/test/API/functionalities/breakpoint/cpp/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/main.cpp
rename to lldb/test/API/functionalities/breakpoint/cpp/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp_exception/Makefile b/lldb/test/API/functionalities/breakpoint/cpp_exception/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp_exception/Makefile
rename to lldb/test/API/functionalities/breakpoint/cpp_exception/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp_exception/TestCPPExceptionBreakpoint.py b/lldb/test/API/functionalities/breakpoint/cpp_exception/TestCPPExceptionBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp_exception/TestCPPExceptionBreakpoint.py
rename to lldb/test/API/functionalities/breakpoint/cpp_exception/TestCPPExceptionBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp_exception/main.cpp b/lldb/test/API/functionalities/breakpoint/cpp_exception/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp_exception/main.cpp
rename to lldb/test/API/functionalities/breakpoint/cpp_exception/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/debugbreak/Makefile b/lldb/test/API/functionalities/breakpoint/debugbreak/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/debugbreak/Makefile
rename to lldb/test/API/functionalities/breakpoint/debugbreak/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/debugbreak/TestDebugBreak.py b/lldb/test/API/functionalities/breakpoint/debugbreak/TestDebugBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/debugbreak/TestDebugBreak.py
rename to lldb/test/API/functionalities/breakpoint/debugbreak/TestDebugBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/debugbreak/main.c b/lldb/test/API/functionalities/breakpoint/debugbreak/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/debugbreak/main.c
rename to lldb/test/API/functionalities/breakpoint/debugbreak/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/Makefile
rename to lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/TestBreakpointsWithNoTargets.py b/lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/TestBreakpointsWithNoTargets.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/TestBreakpointsWithNoTargets.py
rename to lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/TestBreakpointsWithNoTargets.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/main.c b/lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/main.c
rename to lldb/test/API/functionalities/breakpoint/dummy_target_breakpoints/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/Makefile b/lldb/test/API/functionalities/breakpoint/global_constructor/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/Makefile
rename to lldb/test/API/functionalities/breakpoint/global_constructor/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py b/lldb/test/API/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py
rename to lldb/test/API/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/foo.cpp b/lldb/test/API/functionalities/breakpoint/global_constructor/foo.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/foo.cpp
rename to lldb/test/API/functionalities/breakpoint/global_constructor/foo.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/foo.h b/lldb/test/API/functionalities/breakpoint/global_constructor/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/foo.h
rename to lldb/test/API/functionalities/breakpoint/global_constructor/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/main.cpp b/lldb/test/API/functionalities/breakpoint/global_constructor/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/global_constructor/main.cpp
rename to lldb/test/API/functionalities/breakpoint/global_constructor/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/Makefile b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/Makefile
rename to lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
rename to lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/main.cpp b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/main.cpp
rename to lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/inlined_breakpoints/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/Makefile
rename to lldb/test/API/functionalities/breakpoint/inlined_breakpoints/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/TestInlinedBreakpoints.py b/lldb/test/API/functionalities/breakpoint/inlined_breakpoints/TestInlinedBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/TestInlinedBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/inlined_breakpoints/TestInlinedBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/basic_type.cpp b/lldb/test/API/functionalities/breakpoint/inlined_breakpoints/basic_type.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/basic_type.cpp
rename to lldb/test/API/functionalities/breakpoint/inlined_breakpoints/basic_type.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/int.cpp b/lldb/test/API/functionalities/breakpoint/inlined_breakpoints/int.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/inlined_breakpoints/int.cpp
rename to lldb/test/API/functionalities/breakpoint/inlined_breakpoints/int.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/Makefile b/lldb/test/API/functionalities/breakpoint/move_nearest/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/Makefile
rename to lldb/test/API/functionalities/breakpoint/move_nearest/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/TestMoveNearest.py b/lldb/test/API/functionalities/breakpoint/move_nearest/TestMoveNearest.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/TestMoveNearest.py
rename to lldb/test/API/functionalities/breakpoint/move_nearest/TestMoveNearest.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/foo.cpp b/lldb/test/API/functionalities/breakpoint/move_nearest/foo.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/foo.cpp
rename to lldb/test/API/functionalities/breakpoint/move_nearest/foo.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/foo.h b/lldb/test/API/functionalities/breakpoint/move_nearest/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/foo.h
rename to lldb/test/API/functionalities/breakpoint/move_nearest/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/main.cpp b/lldb/test/API/functionalities/breakpoint/move_nearest/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/move_nearest/main.cpp
rename to lldb/test/API/functionalities/breakpoint/move_nearest/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/Makefile b/lldb/test/API/functionalities/breakpoint/objc/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/Makefile
rename to lldb/test/API/functionalities/breakpoint/objc/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/TestObjCBreakpoints.py b/lldb/test/API/functionalities/breakpoint/objc/TestObjCBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/TestObjCBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/objc/TestObjCBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/main.m b/lldb/test/API/functionalities/breakpoint/objc/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/main.m
rename to lldb/test/API/functionalities/breakpoint/objc/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/Makefile
rename to lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/TestRequireHWBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/main.c b/lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/main.c
rename to lldb/test/API/functionalities/breakpoint/require_hw_breakpoints/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/Makefile b/lldb/test/API/functionalities/breakpoint/scripted_bkpt/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/Makefile
rename to lldb/test/API/functionalities/breakpoint/scripted_bkpt/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py b/lldb/test/API/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py
rename to lldb/test/API/functionalities/breakpoint/scripted_bkpt/TestScriptedResolver.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/main.c b/lldb/test/API/functionalities/breakpoint/scripted_bkpt/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/main.c
rename to lldb/test/API/functionalities/breakpoint/scripted_bkpt/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/resolver.py b/lldb/test/API/functionalities/breakpoint/scripted_bkpt/resolver.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/resolver.py
rename to lldb/test/API/functionalities/breakpoint/scripted_bkpt/resolver.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/Makefile b/lldb/test/API/functionalities/breakpoint/serialize/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/Makefile
rename to lldb/test/API/functionalities/breakpoint/serialize/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/TestBreakpointSerialization.py b/lldb/test/API/functionalities/breakpoint/serialize/TestBreakpointSerialization.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/TestBreakpointSerialization.py
rename to lldb/test/API/functionalities/breakpoint/serialize/TestBreakpointSerialization.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/main.c b/lldb/test/API/functionalities/breakpoint/serialize/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/main.c
rename to lldb/test/API/functionalities/breakpoint/serialize/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/resolver.py b/lldb/test/API/functionalities/breakpoint/serialize/resolver.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/resolver.py
rename to lldb/test/API/functionalities/breakpoint/serialize/resolver.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/side_effect.py b/lldb/test/API/functionalities/breakpoint/serialize/side_effect.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/serialize/side_effect.py
rename to lldb/test/API/functionalities/breakpoint/serialize/side_effect.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/Makefile b/lldb/test/API/functionalities/breakpoint/source_regexp/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/Makefile
rename to lldb/test/API/functionalities/breakpoint/source_regexp/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/TestSourceRegexBreakpoints.py b/lldb/test/API/functionalities/breakpoint/source_regexp/TestSourceRegexBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/TestSourceRegexBreakpoints.py
rename to lldb/test/API/functionalities/breakpoint/source_regexp/TestSourceRegexBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/a.c b/lldb/test/API/functionalities/breakpoint/source_regexp/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/a.c
rename to lldb/test/API/functionalities/breakpoint/source_regexp/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/a.h b/lldb/test/API/functionalities/breakpoint/source_regexp/a.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/a.h
rename to lldb/test/API/functionalities/breakpoint/source_regexp/a.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/main.c b/lldb/test/API/functionalities/breakpoint/source_regexp/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/source_regexp/main.c
rename to lldb/test/API/functionalities/breakpoint/source_regexp/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/Makefile b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/Makefile
rename to lldb/test/API/functionalities/breakpoint/step_over_breakpoint/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
rename to lldb/test/API/functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/main.cpp b/lldb/test/API/functionalities/breakpoint/step_over_breakpoint/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/main.cpp
rename to lldb/test/API/functionalities/breakpoint/step_over_breakpoint/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/completion/.categories b/lldb/test/API/functionalities/completion/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/completion/.categories
rename to lldb/test/API/functionalities/completion/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/completion/Makefile b/lldb/test/API/functionalities/completion/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/completion/Makefile
rename to lldb/test/API/functionalities/completion/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py b/lldb/test/API/functionalities/completion/TestCompletion.py
similarity index 86%
rename from lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py
rename to lldb/test/API/functionalities/completion/TestCompletion.py
index 26c70c1e7fc95041dcd9bdbbd1ed022b5c448eec..9e15b5d3f557b16209589108ef16baf0fae9a255 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py
+++ b/lldb/test/API/functionalities/completion/TestCompletion.py
@@ -27,17 +27,14 @@ class CommandLineCompletionTestCase(TestBase):
         except:
             pass
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_at(self):
         """Test that 'at' completes to 'attach '."""
         self.complete_from_to('at', 'attach ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_de(self):
         """Test that 'de' completes to 'detach '."""
         self.complete_from_to('de', 'detach ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_frame_variable(self):
         self.build()
         self.main_source = "main.cpp"
@@ -76,95 +73,79 @@ class CommandLineCompletionTestCase(TestBase):
         self.complete_from_to('frame variable ptr_container->Mem',
                               'frame variable ptr_container->MemberVar')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_process_attach_dash_dash_con(self):
         """Test that 'process attach --con' completes to 'process attach --continue '."""
         self.complete_from_to(
             'process attach --con',
             'process attach --continue ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_process_launch_arch(self):
         self.complete_from_to('process launch --arch ',
                               ['mips',
                                'arm64'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_ambiguous_long_opt(self):
         self.completions_match('breakpoint modify --th',
                                ['--thread-id',
                                 '--thread-index',
                                 '--thread-name'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_plugin_load(self):
         self.complete_from_to('plugin load ', [])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_log_enable(self):
         self.complete_from_to('log enable ll', ['lldb'])
         self.complete_from_to('log enable dw', ['dwarf'])
         self.complete_from_to('log enable lldb al', ['all'])
         self.complete_from_to('log enable lldb sym', ['symbol'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_log_enable(self):
         self.complete_from_to('log disable ll', ['lldb'])
         self.complete_from_to('log disable dw', ['dwarf'])
         self.complete_from_to('log disable lldb al', ['all'])
         self.complete_from_to('log disable lldb sym', ['symbol'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_log_list(self):
         self.complete_from_to('log list ll', ['lldb'])
         self.complete_from_to('log list dw', ['dwarf'])
         self.complete_from_to('log list ll', ['lldb'])
         self.complete_from_to('log list lldb dwa', ['dwarf'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_quoted_command(self):
         self.complete_from_to('"set',
                               ['"settings" '])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_quoted_arg_with_quoted_command(self):
         self.complete_from_to('"settings" "repl',
                               ['"replace" '])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_quoted_arg_without_quoted_command(self):
         self.complete_from_to('settings "repl',
                               ['"replace" '])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_single_quote_command(self):
         self.complete_from_to("'set",
                               ["'settings' "])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_terminated_quote_command(self):
         # This should not crash, but we don't get any
         # reasonable completions from this.
         self.complete_from_to("'settings'", [])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_process_launch_arch_arm(self):
         self.complete_from_to('process launch --arch arm',
                               ['arm64'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_target_symbols_add_shlib(self):
         # Doesn't seem to work, but at least it shouldn't crash.
         self.complete_from_to('target symbols add --shlib ', [])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_log_file(self):
         # Complete in our source directory which contains a 'main.cpp' file.
         src_dir =  os.path.dirname(os.path.realpath(__file__)) + '/'
         self.complete_from_to('log enable lldb expr -f ' + src_dir,
                               ['main.cpp'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_log_dir(self):
         # Complete our source directory.
         src_dir =  os.path.dirname(os.path.realpath(__file__))
@@ -172,140 +153,118 @@ class CommandLineCompletionTestCase(TestBase):
                               [src_dir + os.sep], turn_off_re_match=True)
 
     # <rdar://problem/11052829>
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_infinite_loop_while_completing(self):
         """Test that 'process print hello\' completes to itself and does not infinite loop."""
         self.complete_from_to('process print hello\\', 'process print hello\\',
                               turn_off_re_match=True)
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_watchpoint_co(self):
         """Test that 'watchpoint co' completes to 'watchpoint command '."""
         self.complete_from_to('watchpoint co', 'watchpoint command ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_watchpoint_command_space(self):
         """Test that 'watchpoint command ' completes to ['add', 'delete', 'list']."""
         self.complete_from_to(
             'watchpoint command ', [
                 'add', 'delete', 'list'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_watchpoint_command_a(self):
         """Test that 'watchpoint command a' completes to 'watchpoint command add '."""
         self.complete_from_to(
             'watchpoint command a',
             'watchpoint command add ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_watchpoint_set_ex(self):
         """Test that 'watchpoint set ex' completes to 'watchpoint set expression '."""
         self.complete_from_to(
             'watchpoint set ex',
             'watchpoint set expression ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_watchpoint_set_var(self):
         """Test that 'watchpoint set var' completes to 'watchpoint set variable '."""
         self.complete_from_to('watchpoint set var', 'watchpoint set variable ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_help_fi(self):
         """Test that 'help fi' completes to ['file', 'finish']."""
         self.complete_from_to(
             'help fi', [
                 'file', 'finish'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_help_watchpoint_s(self):
         """Test that 'help watchpoint s' completes to 'help watchpoint set '."""
         self.complete_from_to('help watchpoint s', 'help watchpoint set ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_append_target_er(self):
         """Test that 'settings append target.er' completes to 'settings append target.error-path'."""
         self.complete_from_to(
             'settings append target.er',
             'settings append target.error-path')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_insert_after_target_en(self):
         """Test that 'settings insert-after target.env' completes to 'settings insert-after target.env-vars'."""
         self.complete_from_to(
             'settings insert-after target.env',
             'settings insert-after target.env-vars')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_insert_before_target_en(self):
         """Test that 'settings insert-before target.env' completes to 'settings insert-before target.env-vars'."""
         self.complete_from_to(
             'settings insert-before target.env',
             'settings insert-before target.env-vars')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_replace_target_ru(self):
         """Test that 'settings replace target.ru' completes to 'settings replace target.run-args'."""
         self.complete_from_to(
             'settings replace target.ru',
             'settings replace target.run-args')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_show_term(self):
         self.complete_from_to(
             'settings show term-',
             'settings show term-width')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_list_term(self):
         self.complete_from_to(
             'settings list term-',
             'settings list term-width')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_remove_term(self):
         self.complete_from_to(
             'settings remove term-',
             'settings remove term-width')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_s(self):
         """Test that 'settings s' completes to ['set', 'show']."""
         self.complete_from_to(
             'settings s', [
                 'set', 'show'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_th(self):
         """Test that 'settings set thread-f' completes to 'settings set thread-format'."""
         self.complete_from_to('settings set thread-f', 'settings set thread-format')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_s_dash(self):
         """Test that 'settings set --g' completes to 'settings set --global'."""
         self.complete_from_to('settings set --g', 'settings set --global')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_clear_th(self):
         """Test that 'settings clear thread-f' completes to 'settings clear thread-format'."""
         self.complete_from_to(
             'settings clear thread-f',
             'settings clear thread-format')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_ta(self):
         """Test that 'settings set ta' completes to 'settings set target.'."""
         self.complete_from_to(
             'settings set target.ma',
             'settings set target.max-')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_target_exec(self):
         """Test that 'settings set target.exec' completes to 'settings set target.exec-search-paths '."""
         self.complete_from_to(
             'settings set target.exec',
             'settings set target.exec-search-paths')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_target_pr(self):
         """Test that 'settings set target.pr' completes to [
         'target.prefer-dynamic-value', 'target.process.']."""
@@ -313,21 +272,18 @@ class CommandLineCompletionTestCase(TestBase):
                               ['target.prefer-dynamic-value',
                                'target.process.'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_target_process(self):
         """Test that 'settings set target.process' completes to 'settings set target.process.'."""
         self.complete_from_to(
             'settings set target.process',
             'settings set target.process.')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_target_process_dot(self):
         """Test that 'settings set target.process.t' completes to 'settings set target.process.thread.'."""
         self.complete_from_to(
             'settings set target.process.t',
             'settings set target.process.thread.')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_settings_set_target_process_thread_dot(self):
         """Test that 'settings set target.process.thread.' completes to [
         'target.process.thread.step-avoid-regexp', 'target.process.thread.trace-thread']."""
@@ -335,7 +291,6 @@ class CommandLineCompletionTestCase(TestBase):
                               ['target.process.thread.step-avoid-regexp',
                                'target.process.thread.trace-thread'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_target_space(self):
         """Test that 'target ' completes to ['create', 'delete', 'list',
         'modules', 'select', 'stop-hook', 'variable']."""
@@ -348,12 +303,24 @@ class CommandLineCompletionTestCase(TestBase):
                                'stop-hook',
                                'variable'])
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
+    def test_target_modules_dump_line_table(self):
+        """Tests source file completion by completing the line-table argument."""
+        self.build()
+        self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+        self.complete_from_to('target modules dump line-table main.cp',
+                              ['main.cpp'])
+
+    def test_target_modules_load_aout(self):
+        """Tests modules completion by completing the target modules load argument."""
+        self.build()
+        self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+        self.complete_from_to('target modules load a.ou',
+                              ['a.out'])
+
     def test_target_create_dash_co(self):
         """Test that 'target create --co' completes to 'target variable --core '."""
         self.complete_from_to('target create --co', 'target create --core ')
 
-    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_target_va(self):
         """Test that 'target va' completes to 'target variable '."""
         self.complete_from_to('target va', 'target variable ')
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/completion/main.cpp b/lldb/test/API/functionalities/completion/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/completion/main.cpp
rename to lldb/test/API/functionalities/completion/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/.lldb b/lldb/test/API/functionalities/conditional_break/.lldb
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/.lldb
rename to lldb/test/API/functionalities/conditional_break/.lldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/Makefile b/lldb/test/API/functionalities/conditional_break/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/Makefile
rename to lldb/test/API/functionalities/conditional_break/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/TestConditionalBreak.py b/lldb/test/API/functionalities/conditional_break/TestConditionalBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/TestConditionalBreak.py
rename to lldb/test/API/functionalities/conditional_break/TestConditionalBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/conditional_break.py b/lldb/test/API/functionalities/conditional_break/conditional_break.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/conditional_break.py
rename to lldb/test/API/functionalities/conditional_break/conditional_break.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/main.c b/lldb/test/API/functionalities/conditional_break/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/conditional_break/main.c
rename to lldb/test/API/functionalities/conditional_break/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/.categories b/lldb/test/API/functionalities/darwin_log/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/.categories
rename to lldb/test/API/functionalities/darwin_log/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/basic/Makefile b/lldb/test/API/functionalities/darwin_log/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/basic/Makefile
rename to lldb/test/API/functionalities/darwin_log/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/basic/TestDarwinLogBasic.py b/lldb/test/API/functionalities/darwin_log/basic/TestDarwinLogBasic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/basic/TestDarwinLogBasic.py
rename to lldb/test/API/functionalities/darwin_log/basic/TestDarwinLogBasic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/basic/main.c b/lldb/test/API/functionalities/darwin_log/basic/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/basic/main.c
rename to lldb/test/API/functionalities/darwin_log/basic/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/common/darwin_log_common.h b/lldb/test/API/functionalities/darwin_log/common/darwin_log_common.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/common/darwin_log_common.h
rename to lldb/test/API/functionalities/darwin_log/common/darwin_log_common.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity-chain/Makefile b/lldb/test/API/functionalities/darwin_log/filter/exact_match/activity-chain/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity-chain/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/activity-chain/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity-chain/TestDarwinLogFilterMatchActivityChain.py b/lldb/test/API/functionalities/darwin_log/filter/exact_match/activity-chain/TestDarwinLogFilterMatchActivityChain.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity-chain/TestDarwinLogFilterMatchActivityChain.py
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/activity-chain/TestDarwinLogFilterMatchActivityChain.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity-chain/main.c b/lldb/test/API/functionalities/darwin_log/filter/exact_match/activity-chain/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity-chain/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/activity-chain/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity/Makefile b/lldb/test/API/functionalities/darwin_log/filter/exact_match/activity/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/activity/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity/TestDarwinLogFilterMatchActivity.py b/lldb/test/API/functionalities/darwin_log/filter/exact_match/activity/TestDarwinLogFilterMatchActivity.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity/TestDarwinLogFilterMatchActivity.py
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/activity/TestDarwinLogFilterMatchActivity.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity/main.c b/lldb/test/API/functionalities/darwin_log/filter/exact_match/activity/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/activity/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/activity/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/category/Makefile b/lldb/test/API/functionalities/darwin_log/filter/exact_match/category/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/category/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/category/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/category/TestDarwinLogFilterMatchCategory.py b/lldb/test/API/functionalities/darwin_log/filter/exact_match/category/TestDarwinLogFilterMatchCategory.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/category/TestDarwinLogFilterMatchCategory.py
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/category/TestDarwinLogFilterMatchCategory.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/category/main.c b/lldb/test/API/functionalities/darwin_log/filter/exact_match/category/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/category/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/category/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/message/Makefile b/lldb/test/API/functionalities/darwin_log/filter/exact_match/message/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/message/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/message/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/message/TestDarwinLogFilterMatchMessage.py b/lldb/test/API/functionalities/darwin_log/filter/exact_match/message/TestDarwinLogFilterMatchMessage.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/message/TestDarwinLogFilterMatchMessage.py
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/message/TestDarwinLogFilterMatchMessage.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/message/main.c b/lldb/test/API/functionalities/darwin_log/filter/exact_match/message/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/message/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/message/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/subsystem/Makefile b/lldb/test/API/functionalities/darwin_log/filter/exact_match/subsystem/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/subsystem/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/subsystem/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/subsystem/TestDarwinLogFilterMatchSubsystem.py b/lldb/test/API/functionalities/darwin_log/filter/exact_match/subsystem/TestDarwinLogFilterMatchSubsystem.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/subsystem/TestDarwinLogFilterMatchSubsystem.py
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/subsystem/TestDarwinLogFilterMatchSubsystem.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/subsystem/main.c b/lldb/test/API/functionalities/darwin_log/filter/exact_match/subsystem/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/exact_match/subsystem/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/exact_match/subsystem/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity-chain/Makefile b/lldb/test/API/functionalities/darwin_log/filter/regex/activity-chain/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity-chain/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/regex/activity-chain/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity-chain/TestDarwinLogFilterRegexActivityChain.py b/lldb/test/API/functionalities/darwin_log/filter/regex/activity-chain/TestDarwinLogFilterRegexActivityChain.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity-chain/TestDarwinLogFilterRegexActivityChain.py
rename to lldb/test/API/functionalities/darwin_log/filter/regex/activity-chain/TestDarwinLogFilterRegexActivityChain.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity-chain/main.c b/lldb/test/API/functionalities/darwin_log/filter/regex/activity-chain/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity-chain/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/regex/activity-chain/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity/Makefile b/lldb/test/API/functionalities/darwin_log/filter/regex/activity/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/regex/activity/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity/TestDarwinLogFilterRegexActivity.py b/lldb/test/API/functionalities/darwin_log/filter/regex/activity/TestDarwinLogFilterRegexActivity.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity/TestDarwinLogFilterRegexActivity.py
rename to lldb/test/API/functionalities/darwin_log/filter/regex/activity/TestDarwinLogFilterRegexActivity.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity/main.c b/lldb/test/API/functionalities/darwin_log/filter/regex/activity/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/activity/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/regex/activity/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/category/Makefile b/lldb/test/API/functionalities/darwin_log/filter/regex/category/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/category/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/regex/category/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/category/TestDarwinLogFilterRegexCategory.py b/lldb/test/API/functionalities/darwin_log/filter/regex/category/TestDarwinLogFilterRegexCategory.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/category/TestDarwinLogFilterRegexCategory.py
rename to lldb/test/API/functionalities/darwin_log/filter/regex/category/TestDarwinLogFilterRegexCategory.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/category/main.c b/lldb/test/API/functionalities/darwin_log/filter/regex/category/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/category/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/regex/category/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/message/Makefile b/lldb/test/API/functionalities/darwin_log/filter/regex/message/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/message/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/regex/message/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/message/TestDarwinLogFilterRegexMessage.py b/lldb/test/API/functionalities/darwin_log/filter/regex/message/TestDarwinLogFilterRegexMessage.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/message/TestDarwinLogFilterRegexMessage.py
rename to lldb/test/API/functionalities/darwin_log/filter/regex/message/TestDarwinLogFilterRegexMessage.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/message/main.c b/lldb/test/API/functionalities/darwin_log/filter/regex/message/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/message/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/regex/message/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/subsystem/Makefile b/lldb/test/API/functionalities/darwin_log/filter/regex/subsystem/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/subsystem/Makefile
rename to lldb/test/API/functionalities/darwin_log/filter/regex/subsystem/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/subsystem/TestDarwinLogFilterRegexSubsystem.py b/lldb/test/API/functionalities/darwin_log/filter/regex/subsystem/TestDarwinLogFilterRegexSubsystem.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/subsystem/TestDarwinLogFilterRegexSubsystem.py
rename to lldb/test/API/functionalities/darwin_log/filter/regex/subsystem/TestDarwinLogFilterRegexSubsystem.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/subsystem/main.c b/lldb/test/API/functionalities/darwin_log/filter/regex/subsystem/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/filter/regex/subsystem/main.c
rename to lldb/test/API/functionalities/darwin_log/filter/regex/subsystem/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/format/Makefile b/lldb/test/API/functionalities/darwin_log/format/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/format/Makefile
rename to lldb/test/API/functionalities/darwin_log/format/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/format/TestDarwinLogMessageFormat.py b/lldb/test/API/functionalities/darwin_log/format/TestDarwinLogMessageFormat.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/format/TestDarwinLogMessageFormat.py
rename to lldb/test/API/functionalities/darwin_log/format/TestDarwinLogMessageFormat.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/format/main.c b/lldb/test/API/functionalities/darwin_log/format/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/format/main.c
rename to lldb/test/API/functionalities/darwin_log/format/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/debug/Makefile b/lldb/test/API/functionalities/darwin_log/source/debug/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/debug/Makefile
rename to lldb/test/API/functionalities/darwin_log/source/debug/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/debug/TestDarwinLogSourceDebug.py b/lldb/test/API/functionalities/darwin_log/source/debug/TestDarwinLogSourceDebug.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/debug/TestDarwinLogSourceDebug.py
rename to lldb/test/API/functionalities/darwin_log/source/debug/TestDarwinLogSourceDebug.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/debug/main.c b/lldb/test/API/functionalities/darwin_log/source/debug/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/debug/main.c
rename to lldb/test/API/functionalities/darwin_log/source/debug/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/info/Makefile b/lldb/test/API/functionalities/darwin_log/source/info/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/info/Makefile
rename to lldb/test/API/functionalities/darwin_log/source/info/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/info/TestDarwinLogSourceInfo.py b/lldb/test/API/functionalities/darwin_log/source/info/TestDarwinLogSourceInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/info/TestDarwinLogSourceInfo.py
rename to lldb/test/API/functionalities/darwin_log/source/info/TestDarwinLogSourceInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/info/main.c b/lldb/test/API/functionalities/darwin_log/source/info/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/darwin_log/source/info/main.c
rename to lldb/test/API/functionalities/darwin_log/source/info/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/.categories b/lldb/test/API/functionalities/data-formatter/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/.categories
rename to lldb/test/API/functionalities/data-formatter/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/array_typedef/Makefile b/lldb/test/API/functionalities/data-formatter/array_typedef/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/array_typedef/Makefile
rename to lldb/test/API/functionalities/data-formatter/array_typedef/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/array_typedef/TestArrayTypedef.py b/lldb/test/API/functionalities/data-formatter/array_typedef/TestArrayTypedef.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/array_typedef/TestArrayTypedef.py
rename to lldb/test/API/functionalities/data-formatter/array_typedef/TestArrayTypedef.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/array_typedef/main.cpp b/lldb/test/API/functionalities/data-formatter/array_typedef/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/array_typedef/main.cpp
rename to lldb/test/API/functionalities/data-formatter/array_typedef/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/Makefile b/lldb/test/API/functionalities/data-formatter/boolreference/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/Makefile
rename to lldb/test/API/functionalities/data-formatter/boolreference/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py b/lldb/test/API/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py
rename to lldb/test/API/functionalities/data-formatter/boolreference/TestFormattersBoolRefPtr.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/main.mm b/lldb/test/API/functionalities/data-formatter/boolreference/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/main.mm
rename to lldb/test/API/functionalities/data-formatter/boolreference/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/Makefile b/lldb/test/API/functionalities/data-formatter/compactvectors/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/Makefile
rename to lldb/test/API/functionalities/data-formatter/compactvectors/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/TestCompactVectors.py b/lldb/test/API/functionalities/data-formatter/compactvectors/TestCompactVectors.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/TestCompactVectors.py
rename to lldb/test/API/functionalities/data-formatter/compactvectors/TestCompactVectors.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/main.cpp b/lldb/test/API/functionalities/data-formatter/compactvectors/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/main.cpp
rename to lldb/test/API/functionalities/data-formatter/compactvectors/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-advanced/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-advanced/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py b/lldb/test/API/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-advanced/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-advanced/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-caching/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-caching/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/TestDataFormatterCaching.py b/lldb/test/API/functionalities/data-formatter/data-formatter-caching/TestDataFormatterCaching.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/TestDataFormatterCaching.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-caching/TestDataFormatterCaching.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/a.c b/lldb/test/API/functionalities/data-formatter/data-formatter-caching/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/a.c
rename to lldb/test/API/functionalities/data-formatter/data-formatter-caching/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/b.c b/lldb/test/API/functionalities/data-formatter/data-formatter-caching/b.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-caching/b.c
rename to lldb/test/API/functionalities/data-formatter/data-formatter-caching/b.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-categories/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-categories/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-categories/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-categories/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py b/lldb/test/API/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-categories/TestDataFormatterCategories.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-categories/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-categories/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-categories/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-categories/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-cpp/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-cpp/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-cpp/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-cpp/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-cpp/TestDataFormatterCpp.py b/lldb/test/API/functionalities/data-formatter/data-formatter-cpp/TestDataFormatterCpp.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-cpp/TestDataFormatterCpp.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-cpp/TestDataFormatterCpp.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-cpp/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-cpp/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-cpp/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-cpp/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-disabling/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-disabling/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-disabling/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-disabling/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-disabling/TestDataFormatterDisabling.py b/lldb/test/API/functionalities/data-formatter/data-formatter-disabling/TestDataFormatterDisabling.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-disabling/TestDataFormatterDisabling.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-disabling/TestDataFormatterDisabling.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-disabling/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-disabling/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-disabling/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-disabling/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-enum-format/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-enum-format/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-enum-format/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-enum-format/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-enum-format/TestDataFormatterEnumFormat.py b/lldb/test/API/functionalities/data-formatter/data-formatter-enum-format/TestDataFormatterEnumFormat.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-enum-format/TestDataFormatterEnumFormat.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-enum-format/TestDataFormatterEnumFormat.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-enum-format/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-enum-format/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-enum-format/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-enum-format/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-globals/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-globals/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-globals/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-globals/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-globals/TestDataFormatterGlobals.py b/lldb/test/API/functionalities/data-formatter/data-formatter-globals/TestDataFormatterGlobals.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-globals/TestDataFormatterGlobals.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-globals/TestDataFormatterGlobals.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-globals/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-globals/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-globals/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-globals/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-named-summaries/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-named-summaries/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-named-summaries/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-named-summaries/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-named-summaries/TestDataFormatterNamedSummaries.py b/lldb/test/API/functionalities/data-formatter/data-formatter-named-summaries/TestDataFormatterNamedSummaries.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-named-summaries/TestDataFormatterNamedSummaries.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-named-summaries/TestDataFormatterNamedSummaries.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-named-summaries/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-named-summaries/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-named-summaries/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-named-summaries/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/.categories b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/.categories
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/ObjCDataFormatterTestCase.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/ObjCDataFormatterTestCase.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/ObjCDataFormatterTestCase.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/ObjCDataFormatterTestCase.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCCF.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCCF.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCCF.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCCF.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCExpr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCExpr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCKVO.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCKVO.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCKVO.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCKVO.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSBundle.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSBundle.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSBundle.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSBundle.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSContainer.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSContainer.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSContainer.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSContainer.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSData.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSData.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSData.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSData.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSDate.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSError.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSError.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSError.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSError.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSURL.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSURL.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSURL.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCNSURL.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCPlain.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCPlain.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCPlain.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjCPlain.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjNSException.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjNSException.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjNSException.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/TestDataFormatterObjNSException.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/cmtime/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/cmtime/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/cmtime/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/cmtime/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/cmtime/TestDataFormatterCMTime.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/cmtime/TestDataFormatterCMTime.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/cmtime/TestDataFormatterCMTime.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/cmtime/TestDataFormatterCMTime.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/cmtime/main.m b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/cmtime/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/cmtime/main.m
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/cmtime/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/main.m b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/main.m
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/TestDataFormatterNSIndexPath.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsindexpath/TestDataFormatterNSIndexPath.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/TestDataFormatterNSIndexPath.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsindexpath/TestDataFormatterNSIndexPath.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/main.m b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsindexpath/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/main.m
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsindexpath/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/TestDataFormatterNSString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsstring/TestDataFormatterNSString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/TestDataFormatterNSString.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsstring/TestDataFormatterNSString.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/main.m b/lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsstring/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/main.m
rename to lldb/test/API/functionalities/data-formatter/data-formatter-objc/nsstring/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-proper-plurals/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-proper-plurals/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/TestFormattersOneIsSingular.py b/lldb/test/API/functionalities/data-formatter/data-formatter-proper-plurals/TestFormattersOneIsSingular.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/TestFormattersOneIsSingular.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-proper-plurals/TestFormattersOneIsSingular.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/main.m b/lldb/test/API/functionalities/data-formatter/data-formatter-proper-plurals/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/main.m
rename to lldb/test/API/functionalities/data-formatter/data-formatter-proper-plurals/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-ptr-to-array/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-ptr-to-array/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-ptr-to-array/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-ptr-to-array/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-ptr-to-array/TestPtrToArrayFormatting.py b/lldb/test/API/functionalities/data-formatter/data-formatter-ptr-to-array/TestPtrToArrayFormatting.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-ptr-to-array/TestPtrToArrayFormatting.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-ptr-to-array/TestPtrToArrayFormatting.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-ptr-to-array/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-ptr-to-array/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-ptr-to-array/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-ptr-to-array/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/TestDataFormatterPythonSynth.py b/lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/TestDataFormatterPythonSynth.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/TestDataFormatterPythonSynth.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/TestDataFormatterPythonSynth.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/fooSynthProvider.py b/lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/fooSynthProvider.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/fooSynthProvider.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/fooSynthProvider.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/ftsp.py b/lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/ftsp.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/ftsp.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/ftsp.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-python-synth/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-python-synth/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-script/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-script/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-script/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-script/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-script/TestDataFormatterScript.py b/lldb/test/API/functionalities/data-formatter/data-formatter-script/TestDataFormatterScript.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-script/TestDataFormatterScript.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-script/TestDataFormatterScript.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-script/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-script/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-script/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-script/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/TestDataFormatterSkipSummary.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-skip-summary/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-smart-array/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-smart-array/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-smart-array/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-smart-array/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-smart-array/TestDataFormatterSmartArray.py b/lldb/test/API/functionalities/data-formatter/data-formatter-smart-array/TestDataFormatterSmartArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-smart-array/TestDataFormatterSmartArray.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-smart-array/TestDataFormatterSmartArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-smart-array/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-smart-array/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-smart-array/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-smart-array/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/TestDataFormatterLibcxxBitset.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/TestDataFormatterLibcxxBitset.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/TestDataFormatterLibcxxBitset.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/TestDataFormatterLibcxxBitset.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/bitset/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/TestDataFormatterLibcxxForwardList.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/TestDataFormatterLibcxxForwardList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/TestDataFormatterLibcxxForwardList.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/TestDataFormatterLibcxxForwardList.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/forward_list/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/TestLibCxxFunction.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/function/TestLibCxxFunction.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/TestLibCxxFunction.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/function/TestLibCxxFunction.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/function/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/function/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/TestInitializerList.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/TestDataFormatterLibccIterator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/TestDataFormatterLibccIterator.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/TestDataFormatterLibccIterator.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/TestDataFormatterLibccIterator.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/TestDataFormatterLibcxxList.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/TestDataFormatterLibcxxList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/TestDataFormatterLibcxxList.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/TestDataFormatterLibcxxList.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/TestDataFormatterLibcxxListLoop.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/TestDataFormatterLibcxxListLoop.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/TestDataFormatterLibcxxListLoop.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/TestDataFormatterLibcxxListLoop.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/list/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/TestDataFormatterLibccMultiMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/TestDataFormatterLibccMultiMap.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/TestDataFormatterLibccMultiMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/TestDataFormatterLibccMultiMap.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/TestDataFormatterLibcxxMultiSet.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/TestDataFormatterLibcxxMultiSet.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/TestDataFormatterLibcxxMultiSet.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/TestDataFormatterLibcxxMultiSet.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/TestDataFormatterLibcxxOptional.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/optional/TestDataFormatterLibcxxOptional.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/TestDataFormatterLibcxxOptional.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/optional/TestDataFormatterLibcxxOptional.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/optional/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/optional/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/TestDataFormatterLibcxxQueue.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/queue/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/TestDataFormatterLibcxxSet.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/set/TestDataFormatterLibcxxSet.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/TestDataFormatterLibcxxSet.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/set/TestDataFormatterLibcxxSet.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/set/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/set/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/TestDataFormatterLibcxxTuple.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/tuple/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/TestDataFormatterUnordered.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/TestDataFormatterUnordered.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/TestDataFormatterUnordered.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/TestDataFormatterUnordered.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/TestDataFormatterLibcxxVariant.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/variant/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/TestDataFormatterStdIterator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/TestDataFormatterStdIterator.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/TestDataFormatterStdIterator.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/TestDataFormatterStdIterator.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/TestDataFormatterStdList.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/TestDataFormatterStdList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/TestDataFormatterStdList.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/TestDataFormatterStdList.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/TestDataFormatterStdTuple.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/tuple/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synth/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-synth/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synth/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synth/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synth/TestDataFormatterSynth.py b/lldb/test/API/functionalities/data-formatter/data-formatter-synth/TestDataFormatterSynth.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synth/TestDataFormatterSynth.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synth/TestDataFormatterSynth.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synth/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-synth/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synth/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synth/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/TestDataFormatterSynthType.py b/lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/TestDataFormatterSynthType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/TestDataFormatterSynthType.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/TestDataFormatterSynthType.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/myIntSynthProvider.py b/lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/myIntSynthProvider.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthtype/myIntSynthProvider.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthtype/myIntSynthProvider.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-synthval/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthval/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/TestDataFormatterSynthVal.py b/lldb/test/API/functionalities/data-formatter/data-formatter-synthval/TestDataFormatterSynthVal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/TestDataFormatterSynthVal.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthval/TestDataFormatterSynthVal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-synthval/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthval/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/myIntSynthProvider.py b/lldb/test/API/functionalities/data-formatter/data-formatter-synthval/myIntSynthProvider.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-synthval/myIntSynthProvider.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-synthval/myIntSynthProvider.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/dump_dynamic/TestDumpDynamic.py b/lldb/test/API/functionalities/data-formatter/dump_dynamic/TestDumpDynamic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/dump_dynamic/TestDumpDynamic.py
rename to lldb/test/API/functionalities/data-formatter/dump_dynamic/TestDumpDynamic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/dump_dynamic/main.cpp b/lldb/test/API/functionalities/data-formatter/dump_dynamic/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/dump_dynamic/main.cpp
rename to lldb/test/API/functionalities/data-formatter/dump_dynamic/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/format-propagation/Makefile b/lldb/test/API/functionalities/data-formatter/format-propagation/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/format-propagation/Makefile
rename to lldb/test/API/functionalities/data-formatter/format-propagation/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/format-propagation/TestFormatPropagation.py b/lldb/test/API/functionalities/data-formatter/format-propagation/TestFormatPropagation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/format-propagation/TestFormatPropagation.py
rename to lldb/test/API/functionalities/data-formatter/format-propagation/TestFormatPropagation.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/format-propagation/main.cpp b/lldb/test/API/functionalities/data-formatter/format-propagation/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/format-propagation/main.cpp
rename to lldb/test/API/functionalities/data-formatter/format-propagation/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/frameformat_smallstruct/Makefile b/lldb/test/API/functionalities/data-formatter/frameformat_smallstruct/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/frameformat_smallstruct/Makefile
rename to lldb/test/API/functionalities/data-formatter/frameformat_smallstruct/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/frameformat_smallstruct/TestFrameFormatSmallStruct.py b/lldb/test/API/functionalities/data-formatter/frameformat_smallstruct/TestFrameFormatSmallStruct.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/frameformat_smallstruct/TestFrameFormatSmallStruct.py
rename to lldb/test/API/functionalities/data-formatter/frameformat_smallstruct/TestFrameFormatSmallStruct.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/frameformat_smallstruct/main.cpp b/lldb/test/API/functionalities/data-formatter/frameformat_smallstruct/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/frameformat_smallstruct/main.cpp
rename to lldb/test/API/functionalities/data-formatter/frameformat_smallstruct/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/hexcaps/Makefile b/lldb/test/API/functionalities/data-formatter/hexcaps/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/hexcaps/Makefile
rename to lldb/test/API/functionalities/data-formatter/hexcaps/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/hexcaps/TestDataFormatterHexCaps.py b/lldb/test/API/functionalities/data-formatter/hexcaps/TestDataFormatterHexCaps.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/hexcaps/TestDataFormatterHexCaps.py
rename to lldb/test/API/functionalities/data-formatter/hexcaps/TestDataFormatterHexCaps.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/hexcaps/main.cpp b/lldb/test/API/functionalities/data-formatter/hexcaps/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/hexcaps/main.cpp
rename to lldb/test/API/functionalities/data-formatter/hexcaps/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/language_category_updates/Makefile b/lldb/test/API/functionalities/data-formatter/language_category_updates/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/language_category_updates/Makefile
rename to lldb/test/API/functionalities/data-formatter/language_category_updates/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/language_category_updates/TestDataFormatterLanguageCategoryUpdates.py b/lldb/test/API/functionalities/data-formatter/language_category_updates/TestDataFormatterLanguageCategoryUpdates.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/language_category_updates/TestDataFormatterLanguageCategoryUpdates.py
rename to lldb/test/API/functionalities/data-formatter/language_category_updates/TestDataFormatterLanguageCategoryUpdates.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/language_category_updates/main.cpp b/lldb/test/API/functionalities/data-formatter/language_category_updates/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/language_category_updates/main.cpp
rename to lldb/test/API/functionalities/data-formatter/language_category_updates/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/Makefile b/lldb/test/API/functionalities/data-formatter/nsarraysynth/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/Makefile
rename to lldb/test/API/functionalities/data-formatter/nsarraysynth/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/TestNSArraySynthetic.py b/lldb/test/API/functionalities/data-formatter/nsarraysynth/TestNSArraySynthetic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/TestNSArraySynthetic.py
rename to lldb/test/API/functionalities/data-formatter/nsarraysynth/TestNSArraySynthetic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/main.m b/lldb/test/API/functionalities/data-formatter/nsarraysynth/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/main.m
rename to lldb/test/API/functionalities/data-formatter/nsarraysynth/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/Makefile b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/Makefile
rename to lldb/test/API/functionalities/data-formatter/nsdictionarysynth/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
rename to lldb/test/API/functionalities/data-formatter/nsdictionarysynth/TestNSDictionarySynthetic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/main.m b/lldb/test/API/functionalities/data-formatter/nsdictionarysynth/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/main.m
rename to lldb/test/API/functionalities/data-formatter/nsdictionarysynth/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/Makefile b/lldb/test/API/functionalities/data-formatter/nssetsynth/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/Makefile
rename to lldb/test/API/functionalities/data-formatter/nssetsynth/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/TestNSSetSynthetic.py b/lldb/test/API/functionalities/data-formatter/nssetsynth/TestNSSetSynthetic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/TestNSSetSynthetic.py
rename to lldb/test/API/functionalities/data-formatter/nssetsynth/TestNSSetSynthetic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/main.m b/lldb/test/API/functionalities/data-formatter/nssetsynth/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/main.m
rename to lldb/test/API/functionalities/data-formatter/nssetsynth/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/Makefile b/lldb/test/API/functionalities/data-formatter/ostypeformatting/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/Makefile
rename to lldb/test/API/functionalities/data-formatter/ostypeformatting/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/TestFormattersOsType.py b/lldb/test/API/functionalities/data-formatter/ostypeformatting/TestFormattersOsType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/TestFormattersOsType.py
rename to lldb/test/API/functionalities/data-formatter/ostypeformatting/TestFormattersOsType.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/main.mm b/lldb/test/API/functionalities/data-formatter/ostypeformatting/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/main.mm
rename to lldb/test/API/functionalities/data-formatter/ostypeformatting/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/parray/Makefile b/lldb/test/API/functionalities/data-formatter/parray/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/parray/Makefile
rename to lldb/test/API/functionalities/data-formatter/parray/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/parray/TestPrintArray.py b/lldb/test/API/functionalities/data-formatter/parray/TestPrintArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/parray/TestPrintArray.py
rename to lldb/test/API/functionalities/data-formatter/parray/TestPrintArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/parray/main.cpp b/lldb/test/API/functionalities/data-formatter/parray/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/parray/main.cpp
rename to lldb/test/API/functionalities/data-formatter/parray/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/Makefile b/lldb/test/API/functionalities/data-formatter/poarray/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/Makefile
rename to lldb/test/API/functionalities/data-formatter/poarray/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/TestPrintObjectArray.py b/lldb/test/API/functionalities/data-formatter/poarray/TestPrintObjectArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/TestPrintObjectArray.py
rename to lldb/test/API/functionalities/data-formatter/poarray/TestPrintObjectArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/main.mm b/lldb/test/API/functionalities/data-formatter/poarray/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/main.mm
rename to lldb/test/API/functionalities/data-formatter/poarray/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/Makefile b/lldb/test/API/functionalities/data-formatter/ptr_ref_typedef/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/Makefile
rename to lldb/test/API/functionalities/data-formatter/ptr_ref_typedef/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/TestPtrRef2Typedef.py b/lldb/test/API/functionalities/data-formatter/ptr_ref_typedef/TestPtrRef2Typedef.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/TestPtrRef2Typedef.py
rename to lldb/test/API/functionalities/data-formatter/ptr_ref_typedef/TestPtrRef2Typedef.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/main.cpp b/lldb/test/API/functionalities/data-formatter/ptr_ref_typedef/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/main.cpp
rename to lldb/test/API/functionalities/data-formatter/ptr_ref_typedef/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/Makefile b/lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/Makefile
rename to lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/TestPyObjSynthProvider.py b/lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/TestPyObjSynthProvider.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/TestPyObjSynthProvider.py
rename to lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/TestPyObjSynthProvider.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/main.cpp b/lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/main.cpp
rename to lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/provider.py b/lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/provider.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/pyobjsynthprovider/provider.py
rename to lldb/test/API/functionalities/data-formatter/pyobjsynthprovider/provider.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/refpointer-recursion/Makefile b/lldb/test/API/functionalities/data-formatter/refpointer-recursion/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/refpointer-recursion/Makefile
rename to lldb/test/API/functionalities/data-formatter/refpointer-recursion/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/refpointer-recursion/TestDataFormatterRefPtrRecursion.py b/lldb/test/API/functionalities/data-formatter/refpointer-recursion/TestDataFormatterRefPtrRecursion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/refpointer-recursion/TestDataFormatterRefPtrRecursion.py
rename to lldb/test/API/functionalities/data-formatter/refpointer-recursion/TestDataFormatterRefPtrRecursion.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/refpointer-recursion/main.cpp b/lldb/test/API/functionalities/data-formatter/refpointer-recursion/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/refpointer-recursion/main.cpp
rename to lldb/test/API/functionalities/data-formatter/refpointer-recursion/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/setvaluefromcstring/TestSetValueFromCString.py b/lldb/test/API/functionalities/data-formatter/setvaluefromcstring/TestSetValueFromCString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/setvaluefromcstring/TestSetValueFromCString.py
rename to lldb/test/API/functionalities/data-formatter/setvaluefromcstring/TestSetValueFromCString.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/setvaluefromcstring/main.m b/lldb/test/API/functionalities/data-formatter/setvaluefromcstring/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/setvaluefromcstring/main.m
rename to lldb/test/API/functionalities/data-formatter/setvaluefromcstring/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/stringprinter/TestStringPrinter.py b/lldb/test/API/functionalities/data-formatter/stringprinter/TestStringPrinter.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/stringprinter/TestStringPrinter.py
rename to lldb/test/API/functionalities/data-formatter/stringprinter/TestStringPrinter.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/stringprinter/main.cpp b/lldb/test/API/functionalities/data-formatter/stringprinter/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/stringprinter/main.cpp
rename to lldb/test/API/functionalities/data-formatter/stringprinter/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/summary-string-onfail/Makefile b/lldb/test/API/functionalities/data-formatter/summary-string-onfail/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/summary-string-onfail/Makefile
rename to lldb/test/API/functionalities/data-formatter/summary-string-onfail/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/summary-string-onfail/Test-rdar-9974002.py b/lldb/test/API/functionalities/data-formatter/summary-string-onfail/Test-rdar-9974002.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/summary-string-onfail/Test-rdar-9974002.py
rename to lldb/test/API/functionalities/data-formatter/summary-string-onfail/Test-rdar-9974002.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/summary-string-onfail/main.cpp b/lldb/test/API/functionalities/data-formatter/summary-string-onfail/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/summary-string-onfail/main.cpp
rename to lldb/test/API/functionalities/data-formatter/summary-string-onfail/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/Makefile b/lldb/test/API/functionalities/data-formatter/synthcapping/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/Makefile
rename to lldb/test/API/functionalities/data-formatter/synthcapping/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/TestSyntheticCapping.py b/lldb/test/API/functionalities/data-formatter/synthcapping/TestSyntheticCapping.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/TestSyntheticCapping.py
rename to lldb/test/API/functionalities/data-formatter/synthcapping/TestSyntheticCapping.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/fooSynthProvider.py b/lldb/test/API/functionalities/data-formatter/synthcapping/fooSynthProvider.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/fooSynthProvider.py
rename to lldb/test/API/functionalities/data-formatter/synthcapping/fooSynthProvider.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/main.cpp b/lldb/test/API/functionalities/data-formatter/synthcapping/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthcapping/main.cpp
rename to lldb/test/API/functionalities/data-formatter/synthcapping/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/Makefile b/lldb/test/API/functionalities/data-formatter/synthupdate/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/Makefile
rename to lldb/test/API/functionalities/data-formatter/synthupdate/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/TestSyntheticFilterRecompute.py b/lldb/test/API/functionalities/data-formatter/synthupdate/TestSyntheticFilterRecompute.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/TestSyntheticFilterRecompute.py
rename to lldb/test/API/functionalities/data-formatter/synthupdate/TestSyntheticFilterRecompute.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/main.m b/lldb/test/API/functionalities/data-formatter/synthupdate/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/main.m
rename to lldb/test/API/functionalities/data-formatter/synthupdate/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_arg/TestTypeSummaryListArg.py b/lldb/test/API/functionalities/data-formatter/type_summary_list_arg/TestTypeSummaryListArg.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_arg/TestTypeSummaryListArg.py
rename to lldb/test/API/functionalities/data-formatter/type_summary_list_arg/TestTypeSummaryListArg.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/Makefile b/lldb/test/API/functionalities/data-formatter/type_summary_list_script/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/Makefile
rename to lldb/test/API/functionalities/data-formatter/type_summary_list_script/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/TestTypeSummaryListScript.py b/lldb/test/API/functionalities/data-formatter/type_summary_list_script/TestTypeSummaryListScript.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/TestTypeSummaryListScript.py
rename to lldb/test/API/functionalities/data-formatter/type_summary_list_script/TestTypeSummaryListScript.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/main.cpp b/lldb/test/API/functionalities/data-formatter/type_summary_list_script/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/main.cpp
rename to lldb/test/API/functionalities/data-formatter/type_summary_list_script/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/tslsformatters.py b/lldb/test/API/functionalities/data-formatter/type_summary_list_script/tslsformatters.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/type_summary_list_script/tslsformatters.py
rename to lldb/test/API/functionalities/data-formatter/type_summary_list_script/tslsformatters.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/TestTypedefArray.py b/lldb/test/API/functionalities/data-formatter/typedef_array/TestTypedefArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/TestTypedefArray.py
rename to lldb/test/API/functionalities/data-formatter/typedef_array/TestTypedefArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp b/lldb/test/API/functionalities/data-formatter/typedef_array/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/typedef_array/main.cpp
rename to lldb/test/API/functionalities/data-formatter/typedef_array/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/user-format-vs-summary/Makefile b/lldb/test/API/functionalities/data-formatter/user-format-vs-summary/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/user-format-vs-summary/Makefile
rename to lldb/test/API/functionalities/data-formatter/user-format-vs-summary/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/user-format-vs-summary/TestUserFormatVsSummary.py b/lldb/test/API/functionalities/data-formatter/user-format-vs-summary/TestUserFormatVsSummary.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/user-format-vs-summary/TestUserFormatVsSummary.py
rename to lldb/test/API/functionalities/data-formatter/user-format-vs-summary/TestUserFormatVsSummary.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/user-format-vs-summary/main.cpp b/lldb/test/API/functionalities/data-formatter/user-format-vs-summary/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/user-format-vs-summary/main.cpp
rename to lldb/test/API/functionalities/data-formatter/user-format-vs-summary/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/var-in-aggregate-misuse/Makefile b/lldb/test/API/functionalities/data-formatter/var-in-aggregate-misuse/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/var-in-aggregate-misuse/Makefile
rename to lldb/test/API/functionalities/data-formatter/var-in-aggregate-misuse/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/var-in-aggregate-misuse/TestVarInAggregateMisuse.py b/lldb/test/API/functionalities/data-formatter/var-in-aggregate-misuse/TestVarInAggregateMisuse.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/var-in-aggregate-misuse/TestVarInAggregateMisuse.py
rename to lldb/test/API/functionalities/data-formatter/var-in-aggregate-misuse/TestVarInAggregateMisuse.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/var-in-aggregate-misuse/main.cpp b/lldb/test/API/functionalities/data-formatter/var-in-aggregate-misuse/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/var-in-aggregate-misuse/main.cpp
rename to lldb/test/API/functionalities/data-formatter/var-in-aggregate-misuse/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/Makefile b/lldb/test/API/functionalities/data-formatter/varscript_formatting/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/Makefile
rename to lldb/test/API/functionalities/data-formatter/varscript_formatting/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/TestDataFormatterVarScriptFormatting.py b/lldb/test/API/functionalities/data-formatter/varscript_formatting/TestDataFormatterVarScriptFormatting.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/TestDataFormatterVarScriptFormatting.py
rename to lldb/test/API/functionalities/data-formatter/varscript_formatting/TestDataFormatterVarScriptFormatting.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/helperfunc.py b/lldb/test/API/functionalities/data-formatter/varscript_formatting/helperfunc.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/helperfunc.py
rename to lldb/test/API/functionalities/data-formatter/varscript_formatting/helperfunc.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/main.cpp b/lldb/test/API/functionalities/data-formatter/varscript_formatting/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/varscript_formatting/main.cpp
rename to lldb/test/API/functionalities/data-formatter/varscript_formatting/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/vector-types/Makefile b/lldb/test/API/functionalities/data-formatter/vector-types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/vector-types/Makefile
rename to lldb/test/API/functionalities/data-formatter/vector-types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py b/lldb/test/API/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py
rename to lldb/test/API/functionalities/data-formatter/vector-types/TestVectorTypesFormatting.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/vector-types/main.cpp b/lldb/test/API/functionalities/data-formatter/vector-types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/vector-types/main.cpp
rename to lldb/test/API/functionalities/data-formatter/vector-types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/Makefile b/lldb/test/API/functionalities/dead-strip/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/Makefile
rename to lldb/test/API/functionalities/dead-strip/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/TestDeadStrip.py b/lldb/test/API/functionalities/dead-strip/TestDeadStrip.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/TestDeadStrip.py
rename to lldb/test/API/functionalities/dead-strip/TestDeadStrip.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/cmds.txt b/lldb/test/API/functionalities/dead-strip/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/cmds.txt
rename to lldb/test/API/functionalities/dead-strip/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/main.c b/lldb/test/API/functionalities/dead-strip/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dead-strip/main.c
rename to lldb/test/API/functionalities/dead-strip/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/deleted-executable/Makefile b/lldb/test/API/functionalities/deleted-executable/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/deleted-executable/Makefile
rename to lldb/test/API/functionalities/deleted-executable/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/deleted-executable/TestDeletedExecutable.py b/lldb/test/API/functionalities/deleted-executable/TestDeletedExecutable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/deleted-executable/TestDeletedExecutable.py
rename to lldb/test/API/functionalities/deleted-executable/TestDeletedExecutable.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/deleted-executable/main.cpp b/lldb/test/API/functionalities/deleted-executable/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/deleted-executable/main.cpp
rename to lldb/test/API/functionalities/deleted-executable/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dynamic_value_child_count/Makefile b/lldb/test/API/functionalities/dynamic_value_child_count/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dynamic_value_child_count/Makefile
rename to lldb/test/API/functionalities/dynamic_value_child_count/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dynamic_value_child_count/TestDynamicValueChildCount.py b/lldb/test/API/functionalities/dynamic_value_child_count/TestDynamicValueChildCount.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dynamic_value_child_count/TestDynamicValueChildCount.py
rename to lldb/test/API/functionalities/dynamic_value_child_count/TestDynamicValueChildCount.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/dynamic_value_child_count/pass-to-base.cpp b/lldb/test/API/functionalities/dynamic_value_child_count/pass-to-base.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/dynamic_value_child_count/pass-to-base.cpp
rename to lldb/test/API/functionalities/dynamic_value_child_count/pass-to-base.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/exec/Makefile b/lldb/test/API/functionalities/exec/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/exec/Makefile
rename to lldb/test/API/functionalities/exec/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/exec/TestExec.py b/lldb/test/API/functionalities/exec/TestExec.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/exec/TestExec.py
rename to lldb/test/API/functionalities/exec/TestExec.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/exec/main.cpp b/lldb/test/API/functionalities/exec/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/exec/main.cpp
rename to lldb/test/API/functionalities/exec/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/exec/secondprog.cpp b/lldb/test/API/functionalities/exec/secondprog.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/exec/secondprog.cpp
rename to lldb/test/API/functionalities/exec/secondprog.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/Makefile b/lldb/test/API/functionalities/fat_archives/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/Makefile
rename to lldb/test/API/functionalities/fat_archives/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/TestFatArchives.py b/lldb/test/API/functionalities/fat_archives/TestFatArchives.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/TestFatArchives.py
rename to lldb/test/API/functionalities/fat_archives/TestFatArchives.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/a.c b/lldb/test/API/functionalities/fat_archives/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/a.c
rename to lldb/test/API/functionalities/fat_archives/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/a.h b/lldb/test/API/functionalities/fat_archives/a.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/a.h
rename to lldb/test/API/functionalities/fat_archives/a.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/main.c b/lldb/test/API/functionalities/fat_archives/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/fat_archives/main.c
rename to lldb/test/API/functionalities/fat_archives/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/float-display/Makefile b/lldb/test/API/functionalities/float-display/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/float-display/Makefile
rename to lldb/test/API/functionalities/float-display/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/float-display/TestFloatDisplay.py b/lldb/test/API/functionalities/float-display/TestFloatDisplay.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/float-display/TestFloatDisplay.py
rename to lldb/test/API/functionalities/float-display/TestFloatDisplay.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/float-display/main.c b/lldb/test/API/functionalities/float-display/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/float-display/main.c
rename to lldb/test/API/functionalities/float-display/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestArmRegisterDefinition.py b/lldb/test/API/functionalities/gdb_remote_client/TestArmRegisterDefinition.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestArmRegisterDefinition.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestArmRegisterDefinition.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestGDBRemoteClient.py b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestGDBRemoteClient.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestGDBRemoteLoad.py b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteLoad.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestGDBRemoteLoad.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteLoad.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestJLink6Armv7RegisterDefinition.py b/lldb/test/API/functionalities/gdb_remote_client/TestJLink6Armv7RegisterDefinition.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestJLink6Armv7RegisterDefinition.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestJLink6Armv7RegisterDefinition.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestNestedRegDefinitions.py b/lldb/test/API/functionalities/gdb_remote_client/TestNestedRegDefinitions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestNestedRegDefinitions.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestNestedRegDefinitions.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestNoGPacketSupported.py b/lldb/test/API/functionalities/gdb_remote_client/TestNoGPacketSupported.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestNoGPacketSupported.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestNoGPacketSupported.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestNoWatchpointSupportInfo.py b/lldb/test/API/functionalities/gdb_remote_client/TestNoWatchpointSupportInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestNoWatchpointSupportInfo.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestNoWatchpointSupportInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestPlatformClient.py b/lldb/test/API/functionalities/gdb_remote_client/TestPlatformClient.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestPlatformClient.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestPlatformClient.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestRecognizeBreakpoint.py b/lldb/test/API/functionalities/gdb_remote_client/TestRecognizeBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestRecognizeBreakpoint.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestRecognizeBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestRegDefinitionInParts.py b/lldb/test/API/functionalities/gdb_remote_client/TestRegDefinitionInParts.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestRegDefinitionInParts.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestRegDefinitionInParts.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestRestartBug.py b/lldb/test/API/functionalities/gdb_remote_client/TestRestartBug.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestRestartBug.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestRestartBug.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestStopPCs.py b/lldb/test/API/functionalities/gdb_remote_client/TestStopPCs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestStopPCs.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestStopPCs.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestTargetXMLArch.py b/lldb/test/API/functionalities/gdb_remote_client/TestTargetXMLArch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestTargetXMLArch.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestTargetXMLArch.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestThreadSelectionBug.py b/lldb/test/API/functionalities/gdb_remote_client/TestThreadSelectionBug.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestThreadSelectionBug.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestThreadSelectionBug.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestWriteMemory.py b/lldb/test/API/functionalities/gdb_remote_client/TestWriteMemory.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/TestWriteMemory.py
rename to lldb/test/API/functionalities/gdb_remote_client/TestWriteMemory.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/a.yaml b/lldb/test/API/functionalities/gdb_remote_client/a.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/a.yaml
rename to lldb/test/API/functionalities/gdb_remote_client/a.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/basic_eh_frame.yaml b/lldb/test/API/functionalities/gdb_remote_client/basic_eh_frame.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/basic_eh_frame.yaml
rename to lldb/test/API/functionalities/gdb_remote_client/basic_eh_frame.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/gdbclientutils.py b/lldb/test/API/functionalities/gdb_remote_client/gdbclientutils.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/gdbclientutils.py
rename to lldb/test/API/functionalities/gdb_remote_client/gdbclientutils.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/operating_system.py b/lldb/test/API/functionalities/gdb_remote_client/operating_system.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/operating_system.py
rename to lldb/test/API/functionalities/gdb_remote_client/operating_system.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/operating_system_2.py b/lldb/test/API/functionalities/gdb_remote_client/operating_system_2.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/gdb_remote_client/operating_system_2.py
rename to lldb/test/API/functionalities/gdb_remote_client/operating_system_2.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/history/TestHistoryRecall.py b/lldb/test/API/functionalities/history/TestHistoryRecall.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/history/TestHistoryRecall.py
rename to lldb/test/API/functionalities/history/TestHistoryRecall.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-assert/Makefile b/lldb/test/API/functionalities/inferior-assert/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-assert/Makefile
rename to lldb/test/API/functionalities/inferior-assert/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-assert/TestInferiorAssert.py b/lldb/test/API/functionalities/inferior-assert/TestInferiorAssert.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-assert/TestInferiorAssert.py
rename to lldb/test/API/functionalities/inferior-assert/TestInferiorAssert.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-assert/main.c b/lldb/test/API/functionalities/inferior-assert/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-assert/main.c
rename to lldb/test/API/functionalities/inferior-assert/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/Makefile b/lldb/test/API/functionalities/inferior-changed/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/Makefile
rename to lldb/test/API/functionalities/inferior-changed/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/TestInferiorChanged.py b/lldb/test/API/functionalities/inferior-changed/TestInferiorChanged.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/TestInferiorChanged.py
rename to lldb/test/API/functionalities/inferior-changed/TestInferiorChanged.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/main.c b/lldb/test/API/functionalities/inferior-changed/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/main.c
rename to lldb/test/API/functionalities/inferior-changed/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/main2.c b/lldb/test/API/functionalities/inferior-changed/main2.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-changed/main2.c
rename to lldb/test/API/functionalities/inferior-changed/main2.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/Makefile b/lldb/test/API/functionalities/inferior-crashing/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/Makefile
rename to lldb/test/API/functionalities/inferior-crashing/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/TestInferiorCrashing.py b/lldb/test/API/functionalities/inferior-crashing/TestInferiorCrashing.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/TestInferiorCrashing.py
rename to lldb/test/API/functionalities/inferior-crashing/TestInferiorCrashing.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/TestInferiorCrashingStep.py b/lldb/test/API/functionalities/inferior-crashing/TestInferiorCrashingStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/TestInferiorCrashingStep.py
rename to lldb/test/API/functionalities/inferior-crashing/TestInferiorCrashingStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/main.c b/lldb/test/API/functionalities/inferior-crashing/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/main.c
rename to lldb/test/API/functionalities/inferior-crashing/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/Makefile b/lldb/test/API/functionalities/inferior-crashing/recursive-inferior/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/Makefile
rename to lldb/test/API/functionalities/inferior-crashing/recursive-inferior/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferior.py b/lldb/test/API/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferior.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferior.py
rename to lldb/test/API/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferior.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py b/lldb/test/API/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py
rename to lldb/test/API/functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/main.c b/lldb/test/API/functionalities/inferior-crashing/recursive-inferior/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/main.c
rename to lldb/test/API/functionalities/inferior-crashing/recursive-inferior/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/Makefile b/lldb/test/API/functionalities/inline-stepping/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/Makefile
rename to lldb/test/API/functionalities/inline-stepping/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/TestInlineStepping.py b/lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/TestInlineStepping.py
rename to lldb/test/API/functionalities/inline-stepping/TestInlineStepping.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/calling.cpp b/lldb/test/API/functionalities/inline-stepping/calling.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/calling.cpp
rename to lldb/test/API/functionalities/inline-stepping/calling.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/Makefile b/lldb/test/API/functionalities/jitloader_gdb/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/Makefile
rename to lldb/test/API/functionalities/jitloader_gdb/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/TestJITLoaderGDB.py b/lldb/test/API/functionalities/jitloader_gdb/TestJITLoaderGDB.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/TestJITLoaderGDB.py
rename to lldb/test/API/functionalities/jitloader_gdb/TestJITLoaderGDB.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/main.c b/lldb/test/API/functionalities/jitloader_gdb/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/main.c
rename to lldb/test/API/functionalities/jitloader_gdb/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/simple.c b/lldb/test/API/functionalities/jitloader_gdb/simple.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb/simple.c
rename to lldb/test/API/functionalities/jitloader_gdb/simple.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/lazy-loading/Makefile b/lldb/test/API/functionalities/lazy-loading/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/lazy-loading/Makefile
rename to lldb/test/API/functionalities/lazy-loading/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/lazy-loading/TestLazyLoading.py b/lldb/test/API/functionalities/lazy-loading/TestLazyLoading.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/lazy-loading/TestLazyLoading.py
rename to lldb/test/API/functionalities/lazy-loading/TestLazyLoading.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/lazy-loading/main.cpp b/lldb/test/API/functionalities/lazy-loading/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/lazy-loading/main.cpp
rename to lldb/test/API/functionalities/lazy-loading/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/.categories b/lldb/test/API/functionalities/load_unload/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/.categories
rename to lldb/test/API/functionalities/load_unload/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/Makefile b/lldb/test/API/functionalities/load_unload/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/Makefile
rename to lldb/test/API/functionalities/load_unload/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/TestLoadUnload.py b/lldb/test/API/functionalities/load_unload/TestLoadUnload.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/TestLoadUnload.py
rename to lldb/test/API/functionalities/load_unload/TestLoadUnload.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/a.cpp b/lldb/test/API/functionalities/load_unload/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/a.cpp
rename to lldb/test/API/functionalities/load_unload/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/b.cpp b/lldb/test/API/functionalities/load_unload/b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/b.cpp
rename to lldb/test/API/functionalities/load_unload/b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/c.cpp b/lldb/test/API/functionalities/load_unload/c.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/c.cpp
rename to lldb/test/API/functionalities/load_unload/c.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/cmds.txt b/lldb/test/API/functionalities/load_unload/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/cmds.txt
rename to lldb/test/API/functionalities/load_unload/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/d.cpp b/lldb/test/API/functionalities/load_unload/d.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/d.cpp
rename to lldb/test/API/functionalities/load_unload/d.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/hidden/Makefile b/lldb/test/API/functionalities/load_unload/hidden/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/hidden/Makefile
rename to lldb/test/API/functionalities/load_unload/hidden/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/hidden/d.cpp b/lldb/test/API/functionalities/load_unload/hidden/d.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/hidden/d.cpp
rename to lldb/test/API/functionalities/load_unload/hidden/d.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_unload/main.cpp b/lldb/test/API/functionalities/load_unload/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_unload/main.cpp
rename to lldb/test/API/functionalities/load_unload/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/.categories b/lldb/test/API/functionalities/load_using_paths/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/.categories
rename to lldb/test/API/functionalities/load_using_paths/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/Makefile b/lldb/test/API/functionalities/load_using_paths/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/Makefile
rename to lldb/test/API/functionalities/load_using_paths/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/TestLoadUsingPaths.py b/lldb/test/API/functionalities/load_using_paths/TestLoadUsingPaths.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/TestLoadUsingPaths.py
rename to lldb/test/API/functionalities/load_using_paths/TestLoadUsingPaths.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/hidden/Makefile b/lldb/test/API/functionalities/load_using_paths/hidden/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/hidden/Makefile
rename to lldb/test/API/functionalities/load_using_paths/hidden/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/hidden/d.cpp b/lldb/test/API/functionalities/load_using_paths/hidden/d.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/hidden/d.cpp
rename to lldb/test/API/functionalities/load_using_paths/hidden/d.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/main.cpp b/lldb/test/API/functionalities/load_using_paths/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/load_using_paths/main.cpp
rename to lldb/test/API/functionalities/load_using_paths/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/longjmp/Makefile b/lldb/test/API/functionalities/longjmp/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/longjmp/Makefile
rename to lldb/test/API/functionalities/longjmp/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/longjmp/TestLongjmp.py b/lldb/test/API/functionalities/longjmp/TestLongjmp.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/longjmp/TestLongjmp.py
rename to lldb/test/API/functionalities/longjmp/TestLongjmp.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/longjmp/main.c b/lldb/test/API/functionalities/longjmp/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/longjmp/main.c
rename to lldb/test/API/functionalities/longjmp/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory-region/Makefile b/lldb/test/API/functionalities/memory-region/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory-region/Makefile
rename to lldb/test/API/functionalities/memory-region/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory-region/TestMemoryRegion.py b/lldb/test/API/functionalities/memory-region/TestMemoryRegion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory-region/TestMemoryRegion.py
rename to lldb/test/API/functionalities/memory-region/TestMemoryRegion.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory-region/main.cpp b/lldb/test/API/functionalities/memory-region/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory-region/main.cpp
rename to lldb/test/API/functionalities/memory-region/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/Makefile b/lldb/test/API/functionalities/memory/cache/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/Makefile
rename to lldb/test/API/functionalities/memory/cache/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/TestMemoryCache.py b/lldb/test/API/functionalities/memory/cache/TestMemoryCache.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/TestMemoryCache.py
rename to lldb/test/API/functionalities/memory/cache/TestMemoryCache.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/main.cpp b/lldb/test/API/functionalities/memory/cache/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/cache/main.cpp
rename to lldb/test/API/functionalities/memory/cache/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/find/Makefile b/lldb/test/API/functionalities/memory/find/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/find/Makefile
rename to lldb/test/API/functionalities/memory/find/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/find/TestMemoryFind.py b/lldb/test/API/functionalities/memory/find/TestMemoryFind.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/find/TestMemoryFind.py
rename to lldb/test/API/functionalities/memory/find/TestMemoryFind.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/find/main.cpp b/lldb/test/API/functionalities/memory/find/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/find/main.cpp
rename to lldb/test/API/functionalities/memory/find/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/read/Makefile b/lldb/test/API/functionalities/memory/read/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/read/Makefile
rename to lldb/test/API/functionalities/memory/read/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/read/TestMemoryRead.py b/lldb/test/API/functionalities/memory/read/TestMemoryRead.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/read/TestMemoryRead.py
rename to lldb/test/API/functionalities/memory/read/TestMemoryRead.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/memory/read/main.cpp b/lldb/test/API/functionalities/memory/read/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/memory/read/main.cpp
rename to lldb/test/API/functionalities/memory/read/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/Makefile b/lldb/test/API/functionalities/mtc/simple/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/Makefile
rename to lldb/test/API/functionalities/mtc/simple/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/TestMTCSimple.py b/lldb/test/API/functionalities/mtc/simple/TestMTCSimple.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/TestMTCSimple.py
rename to lldb/test/API/functionalities/mtc/simple/TestMTCSimple.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/main.m b/lldb/test/API/functionalities/mtc/simple/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/main.m
rename to lldb/test/API/functionalities/mtc/simple/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/multidebugger_commands/TestMultipleDebuggersCommands.py b/lldb/test/API/functionalities/multidebugger_commands/TestMultipleDebuggersCommands.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/multidebugger_commands/TestMultipleDebuggersCommands.py
rename to lldb/test/API/functionalities/multidebugger_commands/TestMultipleDebuggersCommands.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/multiword-commands/TestMultiWordCommands.py b/lldb/test/API/functionalities/multiword-commands/TestMultiWordCommands.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/multiword-commands/TestMultiWordCommands.py
rename to lldb/test/API/functionalities/multiword-commands/TestMultiWordCommands.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/non-overlapping-index-variable-i/Makefile b/lldb/test/API/functionalities/non-overlapping-index-variable-i/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/non-overlapping-index-variable-i/Makefile
rename to lldb/test/API/functionalities/non-overlapping-index-variable-i/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/non-overlapping-index-variable-i/TestIndexVariable.py b/lldb/test/API/functionalities/non-overlapping-index-variable-i/TestIndexVariable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/non-overlapping-index-variable-i/TestIndexVariable.py
rename to lldb/test/API/functionalities/non-overlapping-index-variable-i/TestIndexVariable.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/non-overlapping-index-variable-i/main.cpp b/lldb/test/API/functionalities/non-overlapping-index-variable-i/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/non-overlapping-index-variable-i/main.cpp
rename to lldb/test/API/functionalities/non-overlapping-index-variable-i/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/TestImageListMultiArchitecture.py b/lldb/test/API/functionalities/object-file/TestImageListMultiArchitecture.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/TestImageListMultiArchitecture.py
rename to lldb/test/API/functionalities/object-file/TestImageListMultiArchitecture.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-clang-3.3 b/lldb/test/API/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-clang-3.3
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-clang-3.3
rename to lldb/test/API/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-clang-3.3
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-gcc-4.7.3 b/lldb/test/API/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-gcc-4.7.3
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-gcc-4.7.3
rename to lldb/test/API/functionalities/object-file/bin/hello-freebsd-10.0-x86_64-gcc-4.7.3
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-netbsd-6.1-x86_64-gcc-4.5.3 b/lldb/test/API/functionalities/object-file/bin/hello-netbsd-6.1-x86_64-gcc-4.5.3
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-netbsd-6.1-x86_64-gcc-4.5.3
rename to lldb/test/API/functionalities/object-file/bin/hello-netbsd-6.1-x86_64-gcc-4.5.3
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-clang-3.5pre b/lldb/test/API/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-clang-3.5pre
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-clang-3.5pre
rename to lldb/test/API/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-clang-3.5pre
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-gcc-4.8.2 b/lldb/test/API/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-gcc-4.8.2
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-gcc-4.8.2
rename to lldb/test/API/functionalities/object-file/bin/hello-ubuntu-14.04-x86_64-gcc-4.8.2
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-unknown-kalimba_arch4-kcc-36 b/lldb/test/API/functionalities/object-file/bin/hello-unknown-kalimba_arch4-kcc-36
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-unknown-kalimba_arch4-kcc-36
rename to lldb/test/API/functionalities/object-file/bin/hello-unknown-kalimba_arch4-kcc-36
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-unknown-kalimba_arch5-kcc-39 b/lldb/test/API/functionalities/object-file/bin/hello-unknown-kalimba_arch5-kcc-39
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello-unknown-kalimba_arch5-kcc-39
rename to lldb/test/API/functionalities/object-file/bin/hello-unknown-kalimba_arch5-kcc-39
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello.c b/lldb/test/API/functionalities/object-file/bin/hello.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello.c
rename to lldb/test/API/functionalities/object-file/bin/hello.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello.cpp b/lldb/test/API/functionalities/object-file/bin/hello.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/object-file/bin/hello.cpp
rename to lldb/test/API/functionalities/object-file/bin/hello.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/optimized_code/Makefile b/lldb/test/API/functionalities/optimized_code/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/optimized_code/Makefile
rename to lldb/test/API/functionalities/optimized_code/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/optimized_code/TestNoASanExceptionAfterEvalOP_piece.py b/lldb/test/API/functionalities/optimized_code/TestNoASanExceptionAfterEvalOP_piece.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/optimized_code/TestNoASanExceptionAfterEvalOP_piece.py
rename to lldb/test/API/functionalities/optimized_code/TestNoASanExceptionAfterEvalOP_piece.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/optimized_code/main.cpp b/lldb/test/API/functionalities/optimized_code/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/optimized_code/main.cpp
rename to lldb/test/API/functionalities/optimized_code/main.cpp
diff --git a/lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile b/lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..ab505a68412623d2321bcfa6d35e23bb3bef3694
--- /dev/null
+++ b/lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+CXXFLAGS_EXTRAS := -O2 -glldb
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py b/lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py
rename to lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp b/lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp
rename to lldb/test/API/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/paths/TestPaths.py b/lldb/test/API/functionalities/paths/TestPaths.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/paths/TestPaths.py
rename to lldb/test/API/functionalities/paths/TestPaths.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/command_plugin/Makefile b/lldb/test/API/functionalities/plugins/command_plugin/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/command_plugin/Makefile
rename to lldb/test/API/functionalities/plugins/command_plugin/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/command_plugin/TestPluginCommands.py b/lldb/test/API/functionalities/plugins/command_plugin/TestPluginCommands.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/command_plugin/TestPluginCommands.py
rename to lldb/test/API/functionalities/plugins/command_plugin/TestPluginCommands.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/command_plugin/plugin.cpp.template b/lldb/test/API/functionalities/plugins/command_plugin/plugin.cpp.template
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/command_plugin/plugin.cpp.template
rename to lldb/test/API/functionalities/plugins/command_plugin/plugin.cpp.template
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/Makefile b/lldb/test/API/functionalities/plugins/python_os_plugin/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/Makefile
rename to lldb/test/API/functionalities/plugins/python_os_plugin/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py b/lldb/test/API/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py
rename to lldb/test/API/functionalities/plugins/python_os_plugin/TestPythonOSPlugin.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/main.c b/lldb/test/API/functionalities/plugins/python_os_plugin/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/main.c
rename to lldb/test/API/functionalities/plugins/python_os_plugin/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/operating_system.py b/lldb/test/API/functionalities/plugins/python_os_plugin/operating_system.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/operating_system.py
rename to lldb/test/API/functionalities/plugins/python_os_plugin/operating_system.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/operating_system2.py b/lldb/test/API/functionalities/plugins/python_os_plugin/operating_system2.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/plugins/python_os_plugin/operating_system2.py
rename to lldb/test/API/functionalities/plugins/python_os_plugin/operating_system2.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/TestLinuxCore.py b/lldb/test/API/functionalities/postmortem/elf-core/TestLinuxCore.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/TestLinuxCore.py
rename to lldb/test/API/functionalities/postmortem/elf-core/TestLinuxCore.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/altmain.c b/lldb/test/API/functionalities/postmortem/elf-core/altmain.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/altmain.c
rename to lldb/test/API/functionalities/postmortem/elf-core/altmain.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/altmain.core b/lldb/test/API/functionalities/postmortem/elf-core/altmain.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/altmain.core
rename to lldb/test/API/functionalities/postmortem/elf-core/altmain.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/altmain.out b/lldb/test/API/functionalities/postmortem/elf-core/altmain.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/altmain.out
rename to lldb/test/API/functionalities/postmortem/elf-core/altmain.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/fpr_sse.cpp b/lldb/test/API/functionalities/postmortem/elf-core/fpr_sse.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/fpr_sse.cpp
rename to lldb/test/API/functionalities/postmortem/elf-core/fpr_sse.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/TestGCore.py b/lldb/test/API/functionalities/postmortem/elf-core/gcore/TestGCore.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/TestGCore.py
rename to lldb/test/API/functionalities/postmortem/elf-core/gcore/TestGCore.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/linux-i386.core b/lldb/test/API/functionalities/postmortem/elf-core/gcore/linux-i386.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/linux-i386.core
rename to lldb/test/API/functionalities/postmortem/elf-core/gcore/linux-i386.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/linux-x86_64.core b/lldb/test/API/functionalities/postmortem/elf-core/gcore/linux-x86_64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/linux-x86_64.core
rename to lldb/test/API/functionalities/postmortem/elf-core/gcore/linux-x86_64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/main.cpp b/lldb/test/API/functionalities/postmortem/elf-core/gcore/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/main.cpp
rename to lldb/test/API/functionalities/postmortem/elf-core/gcore/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/main.mk b/lldb/test/API/functionalities/postmortem/elf-core/gcore/main.mk
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/main.mk
rename to lldb/test/API/functionalities/postmortem/elf-core/gcore/main.mk
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/make-core.sh b/lldb/test/API/functionalities/postmortem/elf-core/gcore/make-core.sh
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/gcore/make-core.sh
rename to lldb/test/API/functionalities/postmortem/elf-core/gcore/make-core.sh
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-arm.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-arm.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-arm.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-arm.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-fpr_sse_i386.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-fpr_sse_i386.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-fpr_sse_i386.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-fpr_sse_i386.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-fpr_sse_x86_64.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-fpr_sse_x86_64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-fpr_sse_x86_64.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-fpr_sse_x86_64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-i386.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-i386.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-i386.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-i386.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-i386.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-i386.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-i386.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-i386.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabi64.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-mips64el-gnuabin32.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-mipsel-gnuabio32.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-ppc64le.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-ppc64le.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-ppc64le.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-ppc64le.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-ppc64le.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-ppc64le.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-ppc64le.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-ppc64le.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-s390x.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-s390x.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-s390x.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-s390x.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-s390x.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-s390x.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-s390x.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-s390x.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-x86_64.core b/lldb/test/API/functionalities/postmortem/elf-core/linux-x86_64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-x86_64.core
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-x86_64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-x86_64.out b/lldb/test/API/functionalities/postmortem/elf-core/linux-x86_64.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/linux-x86_64.out
rename to lldb/test/API/functionalities/postmortem/elf-core/linux-x86_64.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/main.c b/lldb/test/API/functionalities/postmortem/elf-core/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/main.c
rename to lldb/test/API/functionalities/postmortem/elf-core/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/make-core.sh b/lldb/test/API/functionalities/postmortem/elf-core/make-core.sh
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/make-core.sh
rename to lldb/test/API/functionalities/postmortem/elf-core/make-core.sh
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/TestLinuxCoreThreads.py b/lldb/test/API/functionalities/postmortem/elf-core/thread_crash/TestLinuxCoreThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/TestLinuxCoreThreads.py
rename to lldb/test/API/functionalities/postmortem/elf-core/thread_crash/TestLinuxCoreThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/linux-i386.core b/lldb/test/API/functionalities/postmortem/elf-core/thread_crash/linux-i386.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/linux-i386.core
rename to lldb/test/API/functionalities/postmortem/elf-core/thread_crash/linux-i386.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/linux-x86_64.core b/lldb/test/API/functionalities/postmortem/elf-core/thread_crash/linux-x86_64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/linux-x86_64.core
rename to lldb/test/API/functionalities/postmortem/elf-core/thread_crash/linux-x86_64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/main.cpp b/lldb/test/API/functionalities/postmortem/elf-core/thread_crash/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/main.cpp
rename to lldb/test/API/functionalities/postmortem/elf-core/thread_crash/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/main.mk b/lldb/test/API/functionalities/postmortem/elf-core/thread_crash/main.mk
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/main.mk
rename to lldb/test/API/functionalities/postmortem/elf-core/thread_crash/main.mk
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/make-core.sh b/lldb/test/API/functionalities/postmortem/elf-core/thread_crash/make-core.sh
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/elf-core/thread_crash/make-core.sh
rename to lldb/test/API/functionalities/postmortem/elf-core/thread_crash/make-core.sh
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/mach-core/TestMachCore.py b/lldb/test/API/functionalities/postmortem/mach-core/TestMachCore.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/mach-core/TestMachCore.py
rename to lldb/test/API/functionalities/postmortem/mach-core/TestMachCore.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/mach-core/operating_system.py b/lldb/test/API/functionalities/postmortem/mach-core/operating_system.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/mach-core/operating_system.py
rename to lldb/test/API/functionalities/postmortem/mach-core/operating_system.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/mach-core/test.core.yaml b/lldb/test/API/functionalities/postmortem/mach-core/test.core.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/mach-core/test.core.yaml
rename to lldb/test/API/functionalities/postmortem/mach-core/test.core.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/TestMiniDumpNew.py b/lldb/test/API/functionalities/postmortem/minidump-new/TestMiniDumpNew.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/TestMiniDumpNew.py
rename to lldb/test/API/functionalities/postmortem/minidump-new/TestMiniDumpNew.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/TestMiniDumpUUID.py b/lldb/test/API/functionalities/postmortem/minidump-new/TestMiniDumpUUID.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/TestMiniDumpUUID.py
rename to lldb/test/API/functionalities/postmortem/minidump-new/TestMiniDumpUUID.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/arm-linux.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/arm-linux.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/arm-linux.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/arm-linux.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/arm-macos.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/arm-macos.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/arm-macos.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/arm-macos.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/arm64-macos.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/arm64-macos.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/arm64-macos.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/arm64-macos.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/install_breakpad.cpp b/lldb/test/API/functionalities/postmortem/minidump-new/install_breakpad.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/install_breakpad.cpp
rename to lldb/test/API/functionalities/postmortem/minidump-new/install_breakpad.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/libuuidmatch.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/libuuidmatch.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/libuuidmatch.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/libuuidmatch.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/libuuidmismatch.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/libuuidmismatch.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/libuuidmismatch.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/libuuidmismatch.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-match.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-match.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-match.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-match.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-mismatch.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-mismatch.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-mismatch.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-partial-uuids-mismatch.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-same-uuids.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-same-uuids.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-same-uuids.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-same-uuids.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-16.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-16.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-16.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-16.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-20.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-20.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-20.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-20.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-zero.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-zero.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-zero.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-elf-build-id-zero.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-no-age.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-no-age.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-no-age.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-no-age.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-with-age.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-with-age.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-uuids-with-age.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-uuids-with-age.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-zero-uuids.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-zero-uuids.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-arm-zero-uuids.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-arm-zero-uuids.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64 b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64.cpp b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64.cpp
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64.dmp b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64.dmp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64.dmp
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64.dmp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.cpp b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.cpp
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.dmp b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.dmp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.dmp
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_not_crashed.dmp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_null_signal.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_null_signal.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/linux-x86_64_null_signal.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/linux-x86_64_null_signal.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/macos-arm-uuids-no-age.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/macos-arm-uuids-no-age.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/macos-arm-uuids-no-age.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/macos-arm-uuids-no-age.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/makefile.txt b/lldb/test/API/functionalities/postmortem/minidump-new/makefile.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/makefile.txt
rename to lldb/test/API/functionalities/postmortem/minidump-new/makefile.txt
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/regions-linux-map.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/regions-linux-map.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/regions-linux-map.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/regions-linux-map.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/relative_module_name.yaml b/lldb/test/API/functionalities/postmortem/minidump-new/relative_module_name.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump-new/relative_module_name.yaml
rename to lldb/test/API/functionalities/postmortem/minidump-new/relative_module_name.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/Makefile b/lldb/test/API/functionalities/postmortem/minidump/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/Makefile
rename to lldb/test/API/functionalities/postmortem/minidump/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/TestMiniDump.py b/lldb/test/API/functionalities/postmortem/minidump/TestMiniDump.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/TestMiniDump.py
rename to lldb/test/API/functionalities/postmortem/minidump/TestMiniDump.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/fizzbuzz.cpp b/lldb/test/API/functionalities/postmortem/minidump/fizzbuzz.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/fizzbuzz.cpp
rename to lldb/test/API/functionalities/postmortem/minidump/fizzbuzz.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/fizzbuzz.syms b/lldb/test/API/functionalities/postmortem/minidump/fizzbuzz.syms
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/fizzbuzz.syms
rename to lldb/test/API/functionalities/postmortem/minidump/fizzbuzz.syms
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/fizzbuzz_no_heap.dmp b/lldb/test/API/functionalities/postmortem/minidump/fizzbuzz_no_heap.dmp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/fizzbuzz_no_heap.dmp
rename to lldb/test/API/functionalities/postmortem/minidump/fizzbuzz_no_heap.dmp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/main.cpp b/lldb/test/API/functionalities/postmortem/minidump/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/minidump/main.cpp
rename to lldb/test/API/functionalities/postmortem/minidump/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64 b/lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64
rename to lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64.core b/lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64.core
rename to lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.aarch64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64 b/lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64
rename to lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64.core b/lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64.core
rename to lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.amd64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.c b/lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.c
rename to lldb/test/API/functionalities/postmortem/netbsd-core/1lwp_SIGSEGV.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64 b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64.core b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64.core
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.aarch64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64 b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64.core b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64.core
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.amd64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.c b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.c
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_process_SIGSEGV.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64 b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64.core b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64.core
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.aarch64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64 b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64.core b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64.core
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.amd64.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.c b/lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.c
rename to lldb/test/API/functionalities/postmortem/netbsd-core/2lwp_t2_SIGSEGV.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/GNUmakefile b/lldb/test/API/functionalities/postmortem/netbsd-core/GNUmakefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/GNUmakefile
rename to lldb/test/API/functionalities/postmortem/netbsd-core/GNUmakefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/TestNetBSDCore.py b/lldb/test/API/functionalities/postmortem/netbsd-core/TestNetBSDCore.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/netbsd-core/TestNetBSDCore.py
rename to lldb/test/API/functionalities/postmortem/netbsd-core/TestNetBSDCore.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/wow64_minidump/TestWow64MiniDump.py b/lldb/test/API/functionalities/postmortem/wow64_minidump/TestWow64MiniDump.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/wow64_minidump/TestWow64MiniDump.py
rename to lldb/test/API/functionalities/postmortem/wow64_minidump/TestWow64MiniDump.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/wow64_minidump/fizzbuzz.cpp b/lldb/test/API/functionalities/postmortem/wow64_minidump/fizzbuzz.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/wow64_minidump/fizzbuzz.cpp
rename to lldb/test/API/functionalities/postmortem/wow64_minidump/fizzbuzz.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/postmortem/wow64_minidump/fizzbuzz_wow64.dmp b/lldb/test/API/functionalities/postmortem/wow64_minidump/fizzbuzz_wow64.dmp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/postmortem/wow64_minidump/fizzbuzz_wow64.dmp
rename to lldb/test/API/functionalities/postmortem/wow64_minidump/fizzbuzz_wow64.dmp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/Makefile b/lldb/test/API/functionalities/pre_run_dylibs/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/Makefile
rename to lldb/test/API/functionalities/pre_run_dylibs/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/TestPreRunDylibs.py b/lldb/test/API/functionalities/pre_run_dylibs/TestPreRunDylibs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/TestPreRunDylibs.py
rename to lldb/test/API/functionalities/pre_run_dylibs/TestPreRunDylibs.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/foo.cpp b/lldb/test/API/functionalities/pre_run_dylibs/foo.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/foo.cpp
rename to lldb/test/API/functionalities/pre_run_dylibs/foo.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/foo.h b/lldb/test/API/functionalities/pre_run_dylibs/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/foo.h
rename to lldb/test/API/functionalities/pre_run_dylibs/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/main.cpp b/lldb/test/API/functionalities/pre_run_dylibs/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/pre_run_dylibs/main.cpp
rename to lldb/test/API/functionalities/pre_run_dylibs/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/process_group/Makefile b/lldb/test/API/functionalities/process_group/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/process_group/Makefile
rename to lldb/test/API/functionalities/process_group/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/process_group/TestChangeProcessGroup.py b/lldb/test/API/functionalities/process_group/TestChangeProcessGroup.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/process_group/TestChangeProcessGroup.py
rename to lldb/test/API/functionalities/process_group/TestChangeProcessGroup.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/process_group/main.c b/lldb/test/API/functionalities/process_group/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/process_group/main.c
rename to lldb/test/API/functionalities/process_group/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/process_save_core/Makefile b/lldb/test/API/functionalities/process_save_core/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/process_save_core/Makefile
rename to lldb/test/API/functionalities/process_save_core/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/process_save_core/TestProcessSaveCore.py b/lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/process_save_core/TestProcessSaveCore.py
rename to lldb/test/API/functionalities/process_save_core/TestProcessSaveCore.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/process_save_core/main.cpp b/lldb/test/API/functionalities/process_save_core/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/process_save_core/main.cpp
rename to lldb/test/API/functionalities/process_save_core/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ptr_refs/Makefile b/lldb/test/API/functionalities/ptr_refs/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ptr_refs/Makefile
rename to lldb/test/API/functionalities/ptr_refs/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ptr_refs/TestPtrRefs.py b/lldb/test/API/functionalities/ptr_refs/TestPtrRefs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ptr_refs/TestPtrRefs.py
rename to lldb/test/API/functionalities/ptr_refs/TestPtrRefs.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ptr_refs/main.c b/lldb/test/API/functionalities/ptr_refs/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ptr_refs/main.c
rename to lldb/test/API/functionalities/ptr_refs/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/recursion/Makefile b/lldb/test/API/functionalities/recursion/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/recursion/Makefile
rename to lldb/test/API/functionalities/recursion/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/recursion/TestValueObjectRecursion.py b/lldb/test/API/functionalities/recursion/TestValueObjectRecursion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/recursion/TestValueObjectRecursion.py
rename to lldb/test/API/functionalities/recursion/TestValueObjectRecursion.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/recursion/main.cpp b/lldb/test/API/functionalities/recursion/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/recursion/main.cpp
rename to lldb/test/API/functionalities/recursion/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/rerun/Makefile b/lldb/test/API/functionalities/rerun/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/rerun/Makefile
rename to lldb/test/API/functionalities/rerun/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/rerun/TestRerun.py b/lldb/test/API/functionalities/rerun/TestRerun.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/rerun/TestRerun.py
rename to lldb/test/API/functionalities/rerun/TestRerun.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/rerun/main.cpp b/lldb/test/API/functionalities/rerun/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/rerun/main.cpp
rename to lldb/test/API/functionalities/rerun/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/return-value/Makefile b/lldb/test/API/functionalities/return-value/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/return-value/Makefile
rename to lldb/test/API/functionalities/return-value/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/return-value/TestReturnValue.py b/lldb/test/API/functionalities/return-value/TestReturnValue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/return-value/TestReturnValue.py
rename to lldb/test/API/functionalities/return-value/TestReturnValue.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/return-value/call-func.cpp b/lldb/test/API/functionalities/return-value/call-func.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/return-value/call-func.cpp
rename to lldb/test/API/functionalities/return-value/call-func.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/set-data/Makefile b/lldb/test/API/functionalities/set-data/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/set-data/Makefile
rename to lldb/test/API/functionalities/set-data/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/set-data/TestSetData.py b/lldb/test/API/functionalities/set-data/TestSetData.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/set-data/TestSetData.py
rename to lldb/test/API/functionalities/set-data/TestSetData.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/set-data/main.m b/lldb/test/API/functionalities/set-data/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/set-data/main.m
rename to lldb/test/API/functionalities/set-data/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/show_location/TestShowLocationDwarf5.py b/lldb/test/API/functionalities/show_location/TestShowLocationDwarf5.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/show_location/TestShowLocationDwarf5.py
rename to lldb/test/API/functionalities/show_location/TestShowLocationDwarf5.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/show_location/a.yaml b/lldb/test/API/functionalities/show_location/a.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/show_location/a.yaml
rename to lldb/test/API/functionalities/show_location/a.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/Makefile b/lldb/test/API/functionalities/signal/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/Makefile
rename to lldb/test/API/functionalities/signal/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/TestSendSignal.py b/lldb/test/API/functionalities/signal/TestSendSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/TestSendSignal.py
rename to lldb/test/API/functionalities/signal/TestSendSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-abrt/Makefile b/lldb/test/API/functionalities/signal/handle-abrt/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-abrt/Makefile
rename to lldb/test/API/functionalities/signal/handle-abrt/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-abrt/TestHandleAbort.py b/lldb/test/API/functionalities/signal/handle-abrt/TestHandleAbort.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-abrt/TestHandleAbort.py
rename to lldb/test/API/functionalities/signal/handle-abrt/TestHandleAbort.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-abrt/main.c b/lldb/test/API/functionalities/signal/handle-abrt/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-abrt/main.c
rename to lldb/test/API/functionalities/signal/handle-abrt/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-segv/Makefile b/lldb/test/API/functionalities/signal/handle-segv/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-segv/Makefile
rename to lldb/test/API/functionalities/signal/handle-segv/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-segv/TestHandleSegv.py b/lldb/test/API/functionalities/signal/handle-segv/TestHandleSegv.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-segv/TestHandleSegv.py
rename to lldb/test/API/functionalities/signal/handle-segv/TestHandleSegv.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-segv/main.c b/lldb/test/API/functionalities/signal/handle-segv/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/handle-segv/main.c
rename to lldb/test/API/functionalities/signal/handle-segv/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/main.c b/lldb/test/API/functionalities/signal/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/main.c
rename to lldb/test/API/functionalities/signal/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/raise/Makefile b/lldb/test/API/functionalities/signal/raise/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/raise/Makefile
rename to lldb/test/API/functionalities/signal/raise/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/raise/TestRaise.py b/lldb/test/API/functionalities/signal/raise/TestRaise.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/raise/TestRaise.py
rename to lldb/test/API/functionalities/signal/raise/TestRaise.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/signal/raise/main.c b/lldb/test/API/functionalities/signal/raise/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/signal/raise/main.c
rename to lldb/test/API/functionalities/signal/raise/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/source-map/TestTargetSourceMap.py b/lldb/test/API/functionalities/source-map/TestTargetSourceMap.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/source-map/TestTargetSourceMap.py
rename to lldb/test/API/functionalities/source-map/TestTargetSourceMap.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/source-map/Trivial/main.c b/lldb/test/API/functionalities/source-map/Trivial/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/source-map/Trivial/main.c
rename to lldb/test/API/functionalities/source-map/Trivial/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/source-map/a.yaml b/lldb/test/API/functionalities/source-map/a.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/source-map/a.yaml
rename to lldb/test/API/functionalities/source-map/a.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/stats_api/Makefile b/lldb/test/API/functionalities/stats_api/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/stats_api/Makefile
rename to lldb/test/API/functionalities/stats_api/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/stats_api/TestStatisticsAPI.py b/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/stats_api/TestStatisticsAPI.py
rename to lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/stats_api/main.c b/lldb/test/API/functionalities/stats_api/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/stats_api/main.c
rename to lldb/test/API/functionalities/stats_api/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/Makefile b/lldb/test/API/functionalities/step-avoids-no-debug/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/Makefile
rename to lldb/test/API/functionalities/step-avoids-no-debug/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/TestStepNoDebug.py b/lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/TestStepNoDebug.py
rename to lldb/test/API/functionalities/step-avoids-no-debug/TestStepNoDebug.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/with-debug.c b/lldb/test/API/functionalities/step-avoids-no-debug/with-debug.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/with-debug.c
rename to lldb/test/API/functionalities/step-avoids-no-debug/with-debug.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/without-debug.c b/lldb/test/API/functionalities/step-avoids-no-debug/without-debug.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step-avoids-no-debug/without-debug.c
rename to lldb/test/API/functionalities/step-avoids-no-debug/without-debug.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Makefile b/lldb/test/API/functionalities/step_scripted/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Makefile
rename to lldb/test/API/functionalities/step_scripted/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Steps.py b/lldb/test/API/functionalities/step_scripted/Steps.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Steps.py
rename to lldb/test/API/functionalities/step_scripted/Steps.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/TestStepScripted.py b/lldb/test/API/functionalities/step_scripted/TestStepScripted.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/TestStepScripted.py
rename to lldb/test/API/functionalities/step_scripted/TestStepScripted.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/main.c b/lldb/test/API/functionalities/step_scripted/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/main.c
rename to lldb/test/API/functionalities/step_scripted/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile b/lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/TestAmbiguousTailCallSeq1.py b/lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq1/TestAmbiguousTailCallSeq1.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/TestAmbiguousTailCallSeq1.py
rename to lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq1/TestAmbiguousTailCallSeq1.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/main.cpp b/lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq1/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq1/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile b/lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/TestAmbiguousTailCallSeq2.py b/lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq2/TestAmbiguousTailCallSeq2.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/TestAmbiguousTailCallSeq2.py
rename to lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq2/TestAmbiguousTailCallSeq2.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/main.cpp b/lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq2/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/ambiguous_tail_call_seq2/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/Makefile b/lldb/test/API/functionalities/tail_call_frames/cross_dso/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/One.mk b/lldb/test/API/functionalities/tail_call_frames/cross_dso/One.mk
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/One.mk
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/One.mk
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/One/One.c b/lldb/test/API/functionalities/tail_call_frames/cross_dso/One/One.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/One/One.c
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/One/One.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/TestCrossDSOTailCalls.py b/lldb/test/API/functionalities/tail_call_frames/cross_dso/TestCrossDSOTailCalls.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/TestCrossDSOTailCalls.py
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/TestCrossDSOTailCalls.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/Two.mk b/lldb/test/API/functionalities/tail_call_frames/cross_dso/Two.mk
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/Two.mk
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/Two.mk
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/Two/Two.c b/lldb/test/API/functionalities/tail_call_frames/cross_dso/Two/Two.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/Two/Two.c
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/Two/Two.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/main.c b/lldb/test/API/functionalities/tail_call_frames/cross_dso/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/main.c
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/shared.h b/lldb/test/API/functionalities/tail_call_frames/cross_dso/shared.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_dso/shared.h
rename to lldb/test/API/functionalities/tail_call_frames/cross_dso/shared.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/Makefile b/lldb/test/API/functionalities/tail_call_frames/cross_object/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/cross_object/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/One.c b/lldb/test/API/functionalities/tail_call_frames/cross_object/One.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/One.c
rename to lldb/test/API/functionalities/tail_call_frames/cross_object/One.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/TestCrossObjectTailCalls.py b/lldb/test/API/functionalities/tail_call_frames/cross_object/TestCrossObjectTailCalls.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/TestCrossObjectTailCalls.py
rename to lldb/test/API/functionalities/tail_call_frames/cross_object/TestCrossObjectTailCalls.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/Two.c b/lldb/test/API/functionalities/tail_call_frames/cross_object/Two.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/Two.c
rename to lldb/test/API/functionalities/tail_call_frames/cross_object/Two.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/main.c b/lldb/test/API/functionalities/tail_call_frames/cross_object/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/main.c
rename to lldb/test/API/functionalities/tail_call_frames/cross_object/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/shared.h b/lldb/test/API/functionalities/tail_call_frames/cross_object/shared.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/cross_object/shared.h
rename to lldb/test/API/functionalities/tail_call_frames/cross_object/shared.h
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/Makefile b/lldb/test/API/functionalities/tail_call_frames/disambiguate_call_site/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_call_site/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/TestDisambiguateCallSite.py b/lldb/test/API/functionalities/tail_call_frames/disambiguate_call_site/TestDisambiguateCallSite.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/TestDisambiguateCallSite.py
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_call_site/TestDisambiguateCallSite.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/main.cpp b/lldb/test/API/functionalities/tail_call_frames/disambiguate_call_site/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_call_site/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile b/lldb/test/API/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/TestDisambiguatePathsToCommonSink.py b/lldb/test/API/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/TestDisambiguatePathsToCommonSink.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/TestDisambiguatePathsToCommonSink.py
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/TestDisambiguatePathsToCommonSink.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/main.cpp b/lldb/test/API/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile b/lldb/test/API/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/TestDisambiguateTailCallSeq.py b/lldb/test/API/functionalities/tail_call_frames/disambiguate_tail_call_seq/TestDisambiguateTailCallSeq.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/TestDisambiguateTailCallSeq.py
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_tail_call_seq/TestDisambiguateTailCallSeq.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/main.cpp b/lldb/test/API/functionalities/tail_call_frames/disambiguate_tail_call_seq/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/disambiguate_tail_call_seq/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile b/lldb/test/API/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/TestInliningAndTailCalls.py b/lldb/test/API/functionalities/tail_call_frames/inlining_and_tail_calls/TestInliningAndTailCalls.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/TestInliningAndTailCalls.py
rename to lldb/test/API/functionalities/tail_call_frames/inlining_and_tail_calls/TestInliningAndTailCalls.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/main.cpp b/lldb/test/API/functionalities/tail_call_frames/inlining_and_tail_calls/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/inlining_and_tail_calls/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/Makefile b/lldb/test/API/functionalities/tail_call_frames/sbapi_support/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/sbapi_support/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/TestTailCallFrameSBAPI.py b/lldb/test/API/functionalities/tail_call_frames/sbapi_support/TestTailCallFrameSBAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/TestTailCallFrameSBAPI.py
rename to lldb/test/API/functionalities/tail_call_frames/sbapi_support/TestTailCallFrameSBAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/main.cpp b/lldb/test/API/functionalities/tail_call_frames/sbapi_support/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/sbapi_support/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/Makefile b/lldb/test/API/functionalities/tail_call_frames/thread_step_out_message/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/thread_step_out_message/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/TestArtificialFrameStepOutMessage.py b/lldb/test/API/functionalities/tail_call_frames/thread_step_out_message/TestArtificialFrameStepOutMessage.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/TestArtificialFrameStepOutMessage.py
rename to lldb/test/API/functionalities/tail_call_frames/thread_step_out_message/TestArtificialFrameStepOutMessage.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/main.cpp b/lldb/test/API/functionalities/tail_call_frames/thread_step_out_message/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/thread_step_out_message/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/Makefile b/lldb/test/API/functionalities/tail_call_frames/thread_step_out_or_return/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/thread_step_out_or_return/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/TestSteppingOutWithArtificialFrames.py b/lldb/test/API/functionalities/tail_call_frames/thread_step_out_or_return/TestSteppingOutWithArtificialFrames.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/TestSteppingOutWithArtificialFrames.py
rename to lldb/test/API/functionalities/tail_call_frames/thread_step_out_or_return/TestSteppingOutWithArtificialFrames.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/main.cpp b/lldb/test/API/functionalities/tail_call_frames/thread_step_out_or_return/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/thread_step_out_or_return/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/Makefile b/lldb/test/API/functionalities/tail_call_frames/unambiguous_sequence/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/Makefile
rename to lldb/test/API/functionalities/tail_call_frames/unambiguous_sequence/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/TestUnambiguousTailCalls.py b/lldb/test/API/functionalities/tail_call_frames/unambiguous_sequence/TestUnambiguousTailCalls.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/TestUnambiguousTailCalls.py
rename to lldb/test/API/functionalities/tail_call_frames/unambiguous_sequence/TestUnambiguousTailCalls.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/main.cpp b/lldb/test/API/functionalities/tail_call_frames/unambiguous_sequence/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/main.cpp
rename to lldb/test/API/functionalities/tail_call_frames/unambiguous_sequence/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target-new-solib-notifications/Makefile b/lldb/test/API/functionalities/target-new-solib-notifications/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target-new-solib-notifications/Makefile
rename to lldb/test/API/functionalities/target-new-solib-notifications/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target-new-solib-notifications/TestModuleLoadedNotifys.py b/lldb/test/API/functionalities/target-new-solib-notifications/TestModuleLoadedNotifys.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target-new-solib-notifications/TestModuleLoadedNotifys.py
rename to lldb/test/API/functionalities/target-new-solib-notifications/TestModuleLoadedNotifys.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target-new-solib-notifications/main.cpp b/lldb/test/API/functionalities/target-new-solib-notifications/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target-new-solib-notifications/main.cpp
rename to lldb/test/API/functionalities/target-new-solib-notifications/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target_var/Makefile b/lldb/test/API/functionalities/target_var/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target_var/Makefile
rename to lldb/test/API/functionalities/target_var/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target_var/TestTargetVar.py b/lldb/test/API/functionalities/target_var/TestTargetVar.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target_var/TestTargetVar.py
rename to lldb/test/API/functionalities/target_var/TestTargetVar.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target_var/globals.c b/lldb/test/API/functionalities/target_var/globals.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target_var/globals.c
rename to lldb/test/API/functionalities/target_var/globals.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/target_var/globals.ll b/lldb/test/API/functionalities/target_var/globals.ll
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/target_var/globals.ll
rename to lldb/test/API/functionalities/target_var/globals.ll
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/testid/TestTestId.py b/lldb/test/API/functionalities/testid/TestTestId.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/testid/TestTestId.py
rename to lldb/test/API/functionalities/testid/TestTestId.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/Makefile b/lldb/test/API/functionalities/thread/backtrace_all/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/Makefile
rename to lldb/test/API/functionalities/thread/backtrace_all/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/ParallelTask.cpp b/lldb/test/API/functionalities/thread/backtrace_all/ParallelTask.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/ParallelTask.cpp
rename to lldb/test/API/functionalities/thread/backtrace_all/ParallelTask.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/TestBacktraceAll.py b/lldb/test/API/functionalities/thread/backtrace_all/TestBacktraceAll.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/TestBacktraceAll.py
rename to lldb/test/API/functionalities/thread/backtrace_all/TestBacktraceAll.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/Makefile b/lldb/test/API/functionalities/thread/backtrace_limit/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/Makefile
rename to lldb/test/API/functionalities/thread/backtrace_limit/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/TestBacktraceLimit.py b/lldb/test/API/functionalities/thread/backtrace_limit/TestBacktraceLimit.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/TestBacktraceLimit.py
rename to lldb/test/API/functionalities/thread/backtrace_limit/TestBacktraceLimit.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/main.cpp b/lldb/test/API/functionalities/thread/backtrace_limit/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/main.cpp
rename to lldb/test/API/functionalities/thread/backtrace_limit/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/break_after_join/Makefile b/lldb/test/API/functionalities/thread/break_after_join/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/break_after_join/Makefile
rename to lldb/test/API/functionalities/thread/break_after_join/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/break_after_join/TestBreakAfterJoin.py b/lldb/test/API/functionalities/thread/break_after_join/TestBreakAfterJoin.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/break_after_join/TestBreakAfterJoin.py
rename to lldb/test/API/functionalities/thread/break_after_join/TestBreakAfterJoin.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/break_after_join/main.cpp b/lldb/test/API/functionalities/thread/break_after_join/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/break_after_join/main.cpp
rename to lldb/test/API/functionalities/thread/break_after_join/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/Makefile b/lldb/test/API/functionalities/thread/concurrent_events/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/Makefile
rename to lldb/test/API/functionalities/thread/concurrent_events/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentBreakpointDelayBreakpointOneSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentBreakpointDelayBreakpointOneSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentBreakpointDelayBreakpointOneSignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentBreakpointDelayBreakpointOneSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentBreakpointOneDelayBreakpointThreads.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentBreakpointOneDelayBreakpointThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentBreakpointOneDelayBreakpointThreads.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentBreakpointOneDelayBreakpointThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentBreakpointsDelayedBreakpointOneWatchpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentBreakpointsDelayedBreakpointOneWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentBreakpointsDelayedBreakpointOneWatchpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentBreakpointsDelayedBreakpointOneWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithSignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpointBreakpointSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpointBreakpointSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpointBreakpointSignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentCrashWithWatchpointBreakpointSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelaySignalBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelaySignalBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelaySignalBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelaySignalBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelaySignalWatch.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelaySignalWatch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelaySignalWatch.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelaySignalWatch.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelayWatchBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelayWatchBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelayWatchBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelayWatchBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointSignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointWatchpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointWatchpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentDelayedCrashWithBreakpointWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentNWatchNBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentNWatchNBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentNWatchNBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentNWatchNBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalDelayBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalDelayBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalDelayBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalDelayBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalDelayWatch.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalDelayWatch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalDelayWatch.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalDelayWatch.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalNWatchNBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalNWatchNBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalNWatchNBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalNWatchNBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalWatch.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalWatch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalWatch.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalWatch.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalWatchBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalWatchBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentSignalWatchBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentSignalWatchBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointThreads.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointThreads.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneDelaySignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneDelaySignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneDelaySignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneDelaySignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneWatchpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneWatchpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoBreakpointsOneWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointThreads.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointThreads.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneBreakpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneBreakpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneDelayBreakpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneDelayBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneDelayBreakpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneDelayBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneSignal.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneSignal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneSignal.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentTwoWatchpointsOneSignal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchBreak.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchBreak.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchBreakDelay.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchBreakDelay.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchBreakDelay.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchBreakDelay.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchpointDelayWatchpointOneBreakpoint.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchpointDelayWatchpointOneBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchpointDelayWatchpointOneBreakpoint.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchpointDelayWatchpointOneBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchpointWithDelayWatchpointThreads.py b/lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchpointWithDelayWatchpointThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentWatchpointWithDelayWatchpointThreads.py
rename to lldb/test/API/functionalities/thread/concurrent_events/TestConcurrentWatchpointWithDelayWatchpointThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/main.cpp b/lldb/test/API/functionalities/thread/concurrent_events/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/main.cpp
rename to lldb/test/API/functionalities/thread/concurrent_events/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/crash_during_step/Makefile b/lldb/test/API/functionalities/thread/crash_during_step/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/crash_during_step/Makefile
rename to lldb/test/API/functionalities/thread/crash_during_step/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/crash_during_step/TestCrashDuringStep.py b/lldb/test/API/functionalities/thread/crash_during_step/TestCrashDuringStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/crash_during_step/TestCrashDuringStep.py
rename to lldb/test/API/functionalities/thread/crash_during_step/TestCrashDuringStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/crash_during_step/main.cpp b/lldb/test/API/functionalities/thread/crash_during_step/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/crash_during_step/main.cpp
rename to lldb/test/API/functionalities/thread/crash_during_step/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/create_after_attach/Makefile b/lldb/test/API/functionalities/thread/create_after_attach/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/create_after_attach/Makefile
rename to lldb/test/API/functionalities/thread/create_after_attach/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py b/lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
rename to lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/create_after_attach/main.cpp b/lldb/test/API/functionalities/thread/create_after_attach/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/create_after_attach/main.cpp
rename to lldb/test/API/functionalities/thread/create_after_attach/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/create_during_step/Makefile b/lldb/test/API/functionalities/thread/create_during_step/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/create_during_step/Makefile
rename to lldb/test/API/functionalities/thread/create_during_step/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/create_during_step/TestCreateDuringStep.py b/lldb/test/API/functionalities/thread/create_during_step/TestCreateDuringStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/create_during_step/TestCreateDuringStep.py
rename to lldb/test/API/functionalities/thread/create_during_step/TestCreateDuringStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/create_during_step/main.cpp b/lldb/test/API/functionalities/thread/create_during_step/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/create_during_step/main.cpp
rename to lldb/test/API/functionalities/thread/create_during_step/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_break/Makefile b/lldb/test/API/functionalities/thread/exit_during_break/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_break/Makefile
rename to lldb/test/API/functionalities/thread/exit_during_break/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_break/TestExitDuringBreak.py b/lldb/test/API/functionalities/thread/exit_during_break/TestExitDuringBreak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_break/TestExitDuringBreak.py
rename to lldb/test/API/functionalities/thread/exit_during_break/TestExitDuringBreak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_break/main.cpp b/lldb/test/API/functionalities/thread/exit_during_break/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_break/main.cpp
rename to lldb/test/API/functionalities/thread/exit_during_break/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/Makefile b/lldb/test/API/functionalities/thread/exit_during_step/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/Makefile
rename to lldb/test/API/functionalities/thread/exit_during_step/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/TestExitDuringStep.py b/lldb/test/API/functionalities/thread/exit_during_step/TestExitDuringStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/TestExitDuringStep.py
rename to lldb/test/API/functionalities/thread/exit_during_step/TestExitDuringStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/main.cpp b/lldb/test/API/functionalities/thread/exit_during_step/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/exit_during_step/main.cpp
rename to lldb/test/API/functionalities/thread/exit_during_step/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/Makefile b/lldb/test/API/functionalities/thread/jump/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/Makefile
rename to lldb/test/API/functionalities/thread/jump/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/TestThreadJump.py b/lldb/test/API/functionalities/thread/jump/TestThreadJump.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/TestThreadJump.py
rename to lldb/test/API/functionalities/thread/jump/TestThreadJump.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/main.cpp b/lldb/test/API/functionalities/thread/jump/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/main.cpp
rename to lldb/test/API/functionalities/thread/jump/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/other.cpp b/lldb/test/API/functionalities/thread/jump/other.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/jump/other.cpp
rename to lldb/test/API/functionalities/thread/jump/other.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/multi_break/Makefile b/lldb/test/API/functionalities/thread/multi_break/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/multi_break/Makefile
rename to lldb/test/API/functionalities/thread/multi_break/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/multi_break/TestMultipleBreakpoints.py b/lldb/test/API/functionalities/thread/multi_break/TestMultipleBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/multi_break/TestMultipleBreakpoints.py
rename to lldb/test/API/functionalities/thread/multi_break/TestMultipleBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/multi_break/main.cpp b/lldb/test/API/functionalities/thread/multi_break/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/multi_break/main.cpp
rename to lldb/test/API/functionalities/thread/multi_break/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/num_threads/Makefile b/lldb/test/API/functionalities/thread/num_threads/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/num_threads/Makefile
rename to lldb/test/API/functionalities/thread/num_threads/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/num_threads/TestNumThreads.py b/lldb/test/API/functionalities/thread/num_threads/TestNumThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/num_threads/TestNumThreads.py
rename to lldb/test/API/functionalities/thread/num_threads/TestNumThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/num_threads/main.cpp b/lldb/test/API/functionalities/thread/num_threads/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/num_threads/main.cpp
rename to lldb/test/API/functionalities/thread/num_threads/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/state/Makefile b/lldb/test/API/functionalities/thread/state/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/state/Makefile
rename to lldb/test/API/functionalities/thread/state/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/state/TestThreadStates.py b/lldb/test/API/functionalities/thread/state/TestThreadStates.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/state/TestThreadStates.py
rename to lldb/test/API/functionalities/thread/state/TestThreadStates.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/state/main.cpp b/lldb/test/API/functionalities/thread/state/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/state/main.cpp
rename to lldb/test/API/functionalities/thread/state/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/Makefile b/lldb/test/API/functionalities/thread/step_out/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/Makefile
rename to lldb/test/API/functionalities/thread/step_out/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/TestThreadStepOut.py b/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/TestThreadStepOut.py
rename to lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/main.cpp b/lldb/test/API/functionalities/thread/step_out/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_out/main.cpp
rename to lldb/test/API/functionalities/thread/step_out/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/.categories b/lldb/test/API/functionalities/thread/step_until/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/.categories
rename to lldb/test/API/functionalities/thread/step_until/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/Makefile b/lldb/test/API/functionalities/thread/step_until/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/Makefile
rename to lldb/test/API/functionalities/thread/step_until/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/TestStepUntil.py b/lldb/test/API/functionalities/thread/step_until/TestStepUntil.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/TestStepUntil.py
rename to lldb/test/API/functionalities/thread/step_until/TestStepUntil.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/main.c b/lldb/test/API/functionalities/thread/step_until/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/step_until/main.c
rename to lldb/test/API/functionalities/thread/step_until/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/Makefile b/lldb/test/API/functionalities/thread/thread_exit/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/Makefile
rename to lldb/test/API/functionalities/thread/thread_exit/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/TestThreadExit.py b/lldb/test/API/functionalities/thread/thread_exit/TestThreadExit.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/TestThreadExit.py
rename to lldb/test/API/functionalities/thread/thread_exit/TestThreadExit.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/main.cpp b/lldb/test/API/functionalities/thread/thread_exit/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_exit/main.cpp
rename to lldb/test/API/functionalities/thread/thread_exit/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break/Makefile b/lldb/test/API/functionalities/thread/thread_specific_break/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break/Makefile
rename to lldb/test/API/functionalities/thread/thread_specific_break/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py b/lldb/test/API/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
rename to lldb/test/API/functionalities/thread/thread_specific_break/TestThreadSpecificBreakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break/main.cpp b/lldb/test/API/functionalities/thread/thread_specific_break/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break/main.cpp
rename to lldb/test/API/functionalities/thread/thread_specific_break/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break_plus_condition/Makefile b/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break_plus_condition/Makefile
rename to lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py b/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py
rename to lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break_plus_condition/main.cpp b/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/thread/thread_specific_break_plus_condition/main.cpp
rename to lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/basic/Makefile b/lldb/test/API/functionalities/tsan/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/basic/Makefile
rename to lldb/test/API/functionalities/tsan/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/basic/TestTsanBasic.py b/lldb/test/API/functionalities/tsan/basic/TestTsanBasic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/basic/TestTsanBasic.py
rename to lldb/test/API/functionalities/tsan/basic/TestTsanBasic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/basic/main.c b/lldb/test/API/functionalities/tsan/basic/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/basic/main.c
rename to lldb/test/API/functionalities/tsan/basic/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/cpp_global_location/Makefile b/lldb/test/API/functionalities/tsan/cpp_global_location/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/cpp_global_location/Makefile
rename to lldb/test/API/functionalities/tsan/cpp_global_location/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/cpp_global_location/TestTsanCPPGlobalLocation.py b/lldb/test/API/functionalities/tsan/cpp_global_location/TestTsanCPPGlobalLocation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/cpp_global_location/TestTsanCPPGlobalLocation.py
rename to lldb/test/API/functionalities/tsan/cpp_global_location/TestTsanCPPGlobalLocation.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/cpp_global_location/main.cpp b/lldb/test/API/functionalities/tsan/cpp_global_location/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/cpp_global_location/main.cpp
rename to lldb/test/API/functionalities/tsan/cpp_global_location/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/global_location/Makefile b/lldb/test/API/functionalities/tsan/global_location/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/global_location/Makefile
rename to lldb/test/API/functionalities/tsan/global_location/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/global_location/TestTsanGlobalLocation.py b/lldb/test/API/functionalities/tsan/global_location/TestTsanGlobalLocation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/global_location/TestTsanGlobalLocation.py
rename to lldb/test/API/functionalities/tsan/global_location/TestTsanGlobalLocation.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/global_location/main.c b/lldb/test/API/functionalities/tsan/global_location/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/global_location/main.c
rename to lldb/test/API/functionalities/tsan/global_location/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/multiple/Makefile b/lldb/test/API/functionalities/tsan/multiple/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/multiple/Makefile
rename to lldb/test/API/functionalities/tsan/multiple/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/multiple/TestTsanMultiple.py b/lldb/test/API/functionalities/tsan/multiple/TestTsanMultiple.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/multiple/TestTsanMultiple.py
rename to lldb/test/API/functionalities/tsan/multiple/TestTsanMultiple.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/multiple/main.m b/lldb/test/API/functionalities/tsan/multiple/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/multiple/main.m
rename to lldb/test/API/functionalities/tsan/multiple/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_leak/Makefile b/lldb/test/API/functionalities/tsan/thread_leak/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_leak/Makefile
rename to lldb/test/API/functionalities/tsan/thread_leak/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_leak/TestTsanThreadLeak.py b/lldb/test/API/functionalities/tsan/thread_leak/TestTsanThreadLeak.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_leak/TestTsanThreadLeak.py
rename to lldb/test/API/functionalities/tsan/thread_leak/TestTsanThreadLeak.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_leak/main.c b/lldb/test/API/functionalities/tsan/thread_leak/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_leak/main.c
rename to lldb/test/API/functionalities/tsan/thread_leak/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_numbers/Makefile b/lldb/test/API/functionalities/tsan/thread_numbers/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_numbers/Makefile
rename to lldb/test/API/functionalities/tsan/thread_numbers/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_numbers/TestTsanThreadNumbers.py b/lldb/test/API/functionalities/tsan/thread_numbers/TestTsanThreadNumbers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_numbers/TestTsanThreadNumbers.py
rename to lldb/test/API/functionalities/tsan/thread_numbers/TestTsanThreadNumbers.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_numbers/main.c b/lldb/test/API/functionalities/tsan/thread_numbers/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tsan/thread_numbers/main.c
rename to lldb/test/API/functionalities/tsan/thread_numbers/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tty/Makefile b/lldb/test/API/functionalities/tty/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tty/Makefile
rename to lldb/test/API/functionalities/tty/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tty/TestTerminal.py b/lldb/test/API/functionalities/tty/TestTerminal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tty/TestTerminal.py
rename to lldb/test/API/functionalities/tty/TestTerminal.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tty/main.c b/lldb/test/API/functionalities/tty/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/tty/main.c
rename to lldb/test/API/functionalities/tty/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_completion/Makefile b/lldb/test/API/functionalities/type_completion/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/type_completion/Makefile
rename to lldb/test/API/functionalities/type_completion/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_completion/TestTypeCompletion.py b/lldb/test/API/functionalities/type_completion/TestTypeCompletion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/type_completion/TestTypeCompletion.py
rename to lldb/test/API/functionalities/type_completion/TestTypeCompletion.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_completion/main.cpp b/lldb/test/API/functionalities/type_completion/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/type_completion/main.cpp
rename to lldb/test/API/functionalities/type_completion/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/Makefile b/lldb/test/API/functionalities/type_lookup/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/Makefile
rename to lldb/test/API/functionalities/type_lookup/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/TestTypeLookup.py b/lldb/test/API/functionalities/type_lookup/TestTypeLookup.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/TestTypeLookup.py
rename to lldb/test/API/functionalities/type_lookup/TestTypeLookup.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/main.mm b/lldb/test/API/functionalities/type_lookup/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/main.mm
rename to lldb/test/API/functionalities/type_lookup/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ubsan/basic/Makefile b/lldb/test/API/functionalities/ubsan/basic/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ubsan/basic/Makefile
rename to lldb/test/API/functionalities/ubsan/basic/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ubsan/basic/TestUbsanBasic.py b/lldb/test/API/functionalities/ubsan/basic/TestUbsanBasic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ubsan/basic/TestUbsanBasic.py
rename to lldb/test/API/functionalities/ubsan/basic/TestUbsanBasic.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ubsan/basic/main.c b/lldb/test/API/functionalities/ubsan/basic/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ubsan/basic/main.c
rename to lldb/test/API/functionalities/ubsan/basic/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ubsan/user-expression/Makefile b/lldb/test/API/functionalities/ubsan/user-expression/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ubsan/user-expression/Makefile
rename to lldb/test/API/functionalities/ubsan/user-expression/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ubsan/user-expression/TestUbsanUserExpression.py b/lldb/test/API/functionalities/ubsan/user-expression/TestUbsanUserExpression.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ubsan/user-expression/TestUbsanUserExpression.py
rename to lldb/test/API/functionalities/ubsan/user-expression/TestUbsanUserExpression.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/ubsan/user-expression/main.c b/lldb/test/API/functionalities/ubsan/user-expression/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/ubsan/user-expression/main.c
rename to lldb/test/API/functionalities/ubsan/user-expression/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/ehframe/Makefile b/lldb/test/API/functionalities/unwind/ehframe/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/ehframe/Makefile
rename to lldb/test/API/functionalities/unwind/ehframe/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/ehframe/TestEhFrameUnwind.py b/lldb/test/API/functionalities/unwind/ehframe/TestEhFrameUnwind.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/ehframe/TestEhFrameUnwind.py
rename to lldb/test/API/functionalities/unwind/ehframe/TestEhFrameUnwind.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/ehframe/main.c b/lldb/test/API/functionalities/unwind/ehframe/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/ehframe/main.c
rename to lldb/test/API/functionalities/unwind/ehframe/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/Makefile b/lldb/test/API/functionalities/unwind/noreturn/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/Makefile
rename to lldb/test/API/functionalities/unwind/noreturn/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/TestNoreturnUnwind.py b/lldb/test/API/functionalities/unwind/noreturn/TestNoreturnUnwind.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/TestNoreturnUnwind.py
rename to lldb/test/API/functionalities/unwind/noreturn/TestNoreturnUnwind.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/main.c b/lldb/test/API/functionalities/unwind/noreturn/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/main.c
rename to lldb/test/API/functionalities/unwind/noreturn/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/TestNoReturnModuleEnd.py b/lldb/test/API/functionalities/unwind/noreturn/module-end/TestNoReturnModuleEnd.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/TestNoReturnModuleEnd.py
rename to lldb/test/API/functionalities/unwind/noreturn/module-end/TestNoReturnModuleEnd.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/a.s b/lldb/test/API/functionalities/unwind/noreturn/module-end/a.s
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/a.s
rename to lldb/test/API/functionalities/unwind/noreturn/module-end/a.s
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/test.core b/lldb/test/API/functionalities/unwind/noreturn/module-end/test.core
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/test.core
rename to lldb/test/API/functionalities/unwind/noreturn/module-end/test.core
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/test.out b/lldb/test/API/functionalities/unwind/noreturn/module-end/test.out
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/noreturn/module-end/test.out
rename to lldb/test/API/functionalities/unwind/noreturn/module-end/test.out
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/sigtramp/Makefile b/lldb/test/API/functionalities/unwind/sigtramp/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/sigtramp/Makefile
rename to lldb/test/API/functionalities/unwind/sigtramp/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/sigtramp/TestSigtrampUnwind.py b/lldb/test/API/functionalities/unwind/sigtramp/TestSigtrampUnwind.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/sigtramp/TestSigtrampUnwind.py
rename to lldb/test/API/functionalities/unwind/sigtramp/TestSigtrampUnwind.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/sigtramp/main.c b/lldb/test/API/functionalities/unwind/sigtramp/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/sigtramp/main.c
rename to lldb/test/API/functionalities/unwind/sigtramp/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/Makefile b/lldb/test/API/functionalities/unwind/standard/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/Makefile
rename to lldb/test/API/functionalities/unwind/standard/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/TestStandardUnwind.py b/lldb/test/API/functionalities/unwind/standard/TestStandardUnwind.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/TestStandardUnwind.py
rename to lldb/test/API/functionalities/unwind/standard/TestStandardUnwind.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/hand_written/divmod.cpp b/lldb/test/API/functionalities/unwind/standard/hand_written/divmod.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/hand_written/divmod.cpp
rename to lldb/test/API/functionalities/unwind/standard/hand_written/divmod.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/hand_written/fprintf.cpp b/lldb/test/API/functionalities/unwind/standard/hand_written/fprintf.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/hand_written/fprintf.cpp
rename to lldb/test/API/functionalities/unwind/standard/hand_written/fprintf.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/hand_written/new_delete.cpp b/lldb/test/API/functionalities/unwind/standard/hand_written/new_delete.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/unwind/standard/hand_written/new_delete.cpp
rename to lldb/test/API/functionalities/unwind/standard/hand_written/new_delete.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/value_md5_crash/Makefile b/lldb/test/API/functionalities/value_md5_crash/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/value_md5_crash/Makefile
rename to lldb/test/API/functionalities/value_md5_crash/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/value_md5_crash/TestValueMD5Crash.py b/lldb/test/API/functionalities/value_md5_crash/TestValueMD5Crash.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/value_md5_crash/TestValueMD5Crash.py
rename to lldb/test/API/functionalities/value_md5_crash/TestValueMD5Crash.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/value_md5_crash/main.cpp b/lldb/test/API/functionalities/value_md5_crash/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/value_md5_crash/main.cpp
rename to lldb/test/API/functionalities/value_md5_crash/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/var_path/Makefile b/lldb/test/API/functionalities/var_path/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/var_path/Makefile
rename to lldb/test/API/functionalities/var_path/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/var_path/TestVarPath.py b/lldb/test/API/functionalities/var_path/TestVarPath.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/var_path/TestVarPath.py
rename to lldb/test/API/functionalities/var_path/TestVarPath.py
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/var_path/main.cpp b/lldb/test/API/functionalities/var_path/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/var_path/main.cpp
rename to lldb/test/API/functionalities/var_path/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/wrong_commands/.categories b/lldb/test/API/functionalities/wrong_commands/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/wrong_commands/.categories
rename to lldb/test/API/functionalities/wrong_commands/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/wrong_commands/TestWrongCommands.py b/lldb/test/API/functionalities/wrong_commands/TestWrongCommands.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/functionalities/wrong_commands/TestWrongCommands.py
rename to lldb/test/API/functionalities/wrong_commands/TestWrongCommands.py
diff --git a/lldb/packages/Python/lldbsuite/test/iohandler/completion/TestIOHandlerCompletion.py b/lldb/test/API/iohandler/completion/TestIOHandlerCompletion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/iohandler/completion/TestIOHandlerCompletion.py
rename to lldb/test/API/iohandler/completion/TestIOHandlerCompletion.py
diff --git a/lldb/packages/Python/lldbsuite/test/iohandler/completion/main.c b/lldb/test/API/iohandler/completion/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/iohandler/completion/main.c
rename to lldb/test/API/iohandler/completion/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/iohandler/unicode/TestUnicode.py b/lldb/test/API/iohandler/unicode/TestUnicode.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/iohandler/unicode/TestUnicode.py
rename to lldb/test/API/iohandler/unicode/TestUnicode.py
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/README.txt b/lldb/test/API/issue_verification/README.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/README.txt
rename to lldb/test/API/issue_verification/README.txt
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park b/lldb/test/API/issue_verification/TestExpectedTimeout.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
rename to lldb/test/API/issue_verification/TestExpectedTimeout.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestFail.py.park b/lldb/test/API/issue_verification/TestFail.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestFail.py.park
rename to lldb/test/API/issue_verification/TestFail.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park b/lldb/test/API/issue_verification/TestInvalidDecorator.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park
rename to lldb/test/API/issue_verification/TestInvalidDecorator.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park b/lldb/test/API/issue_verification/TestRerunFail.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFail.py.park
rename to lldb/test/API/issue_verification/TestRerunFail.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFileLevelTimeout.py.park b/lldb/test/API/issue_verification/TestRerunFileLevelTimeout.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunFileLevelTimeout.py.park
rename to lldb/test/API/issue_verification/TestRerunFileLevelTimeout.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunInline.py.park b/lldb/test/API/issue_verification/TestRerunInline.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunInline.py.park
rename to lldb/test/API/issue_verification/TestRerunInline.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park b/lldb/test/API/issue_verification/TestRerunTimeout.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestRerunTimeout.py.park
rename to lldb/test/API/issue_verification/TestRerunTimeout.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestSignal.py.park b/lldb/test/API/issue_verification/TestSignal.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestSignal.py.park
rename to lldb/test/API/issue_verification/TestSignal.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park b/lldb/test/API/issue_verification/TestSignalOutsideTestMethod.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestSignalOutsideTestMethod.py.park
rename to lldb/test/API/issue_verification/TestSignalOutsideTestMethod.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/TestTimeout.py.park b/lldb/test/API/issue_verification/TestTimeout.py.park
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/TestTimeout.py.park
rename to lldb/test/API/issue_verification/TestTimeout.py.park
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/disable.py b/lldb/test/API/issue_verification/disable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/disable.py
rename to lldb/test/API/issue_verification/disable.py
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/enable.py b/lldb/test/API/issue_verification/enable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/enable.py
rename to lldb/test/API/issue_verification/enable.py
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/inline_rerun_inferior.cpp b/lldb/test/API/issue_verification/inline_rerun_inferior.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/inline_rerun_inferior.cpp
rename to lldb/test/API/issue_verification/inline_rerun_inferior.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/issue_verification/rerun_base.py b/lldb/test/API/issue_verification/rerun_base.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/issue_verification/rerun_base.py
rename to lldb/test/API/issue_verification/rerun_base.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/anonymous/Makefile b/lldb/test/API/lang/c/anonymous/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/anonymous/Makefile
rename to lldb/test/API/lang/c/anonymous/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/anonymous/TestAnonymous.py b/lldb/test/API/lang/c/anonymous/TestAnonymous.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/anonymous/TestAnonymous.py
rename to lldb/test/API/lang/c/anonymous/TestAnonymous.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/anonymous/main.c b/lldb/test/API/lang/c/anonymous/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/anonymous/main.c
rename to lldb/test/API/lang/c/anonymous/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/array_types/Makefile b/lldb/test/API/lang/c/array_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/array_types/Makefile
rename to lldb/test/API/lang/c/array_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/array_types/TestArrayTypes.py b/lldb/test/API/lang/c/array_types/TestArrayTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/array_types/TestArrayTypes.py
rename to lldb/test/API/lang/c/array_types/TestArrayTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/array_types/cmds.txt b/lldb/test/API/lang/c/array_types/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/array_types/cmds.txt
rename to lldb/test/API/lang/c/array_types/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/array_types/main.c b/lldb/test/API/lang/c/array_types/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/array_types/main.c
rename to lldb/test/API/lang/c/array_types/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/bitfields/Makefile b/lldb/test/API/lang/c/bitfields/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/bitfields/Makefile
rename to lldb/test/API/lang/c/bitfields/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/bitfields/TestBitfields.py b/lldb/test/API/lang/c/bitfields/TestBitfields.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/bitfields/TestBitfields.py
rename to lldb/test/API/lang/c/bitfields/TestBitfields.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/bitfields/main.c b/lldb/test/API/lang/c/bitfields/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/bitfields/main.c
rename to lldb/test/API/lang/c/bitfields/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/blocks/Makefile b/lldb/test/API/lang/c/blocks/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/blocks/Makefile
rename to lldb/test/API/lang/c/blocks/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/blocks/TestBlocks.py b/lldb/test/API/lang/c/blocks/TestBlocks.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/blocks/TestBlocks.py
rename to lldb/test/API/lang/c/blocks/TestBlocks.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/blocks/main.c b/lldb/test/API/lang/c/blocks/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/blocks/main.c
rename to lldb/test/API/lang/c/blocks/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Makefile b/lldb/test/API/lang/c/conflicting-symbol/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Makefile
rename to lldb/test/API/lang/c/conflicting-symbol/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One.mk b/lldb/test/API/lang/c/conflicting-symbol/One.mk
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One.mk
rename to lldb/test/API/lang/c/conflicting-symbol/One.mk
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One/One.c b/lldb/test/API/lang/c/conflicting-symbol/One/One.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One/One.c
rename to lldb/test/API/lang/c/conflicting-symbol/One/One.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One/One.h b/lldb/test/API/lang/c/conflicting-symbol/One/One.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One/One.h
rename to lldb/test/API/lang/c/conflicting-symbol/One/One.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One/OneConstant.c b/lldb/test/API/lang/c/conflicting-symbol/One/OneConstant.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/One/OneConstant.c
rename to lldb/test/API/lang/c/conflicting-symbol/One/OneConstant.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/TestConflictingSymbol.py b/lldb/test/API/lang/c/conflicting-symbol/TestConflictingSymbol.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/TestConflictingSymbol.py
rename to lldb/test/API/lang/c/conflicting-symbol/TestConflictingSymbol.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two.mk b/lldb/test/API/lang/c/conflicting-symbol/Two.mk
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two.mk
rename to lldb/test/API/lang/c/conflicting-symbol/Two.mk
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two/Two.c b/lldb/test/API/lang/c/conflicting-symbol/Two/Two.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two/Two.c
rename to lldb/test/API/lang/c/conflicting-symbol/Two/Two.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two/Two.h b/lldb/test/API/lang/c/conflicting-symbol/Two/Two.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two/Two.h
rename to lldb/test/API/lang/c/conflicting-symbol/Two/Two.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two/TwoConstant.c b/lldb/test/API/lang/c/conflicting-symbol/Two/TwoConstant.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/Two/TwoConstant.c
rename to lldb/test/API/lang/c/conflicting-symbol/Two/TwoConstant.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/main.c b/lldb/test/API/lang/c/conflicting-symbol/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/conflicting-symbol/main.c
rename to lldb/test/API/lang/c/conflicting-symbol/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/Makefile b/lldb/test/API/lang/c/const_variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/const_variables/Makefile
rename to lldb/test/API/lang/c/const_variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/TestConstVariables.py b/lldb/test/API/lang/c/const_variables/TestConstVariables.py
similarity index 84%
rename from lldb/packages/Python/lldbsuite/test/lang/c/const_variables/TestConstVariables.py
rename to lldb/test/API/lang/c/const_variables/TestConstVariables.py
index 59fa6bcb0a1b6934f31939a5540b3b8f22b7b3e9..2cc94369f38dc5c410d69f90964c699ca97e9791 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/TestConstVariables.py
+++ b/lldb/test/API/lang/c/const_variables/TestConstVariables.py
@@ -12,15 +12,6 @@ class ConstVariableTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    @expectedFailureAll(
-        oslist=["freebsd", "linux"],
-        compiler="clang", compiler_version=["<", "3.5"])
-    @expectedFailureAll(
-        oslist=["freebsd", "linux"],
-        compiler="clang", compiler_version=["=", "3.7"])
-    @expectedFailureAll(
-        oslist=["freebsd", "linux"],
-        compiler="clang", compiler_version=["=", "3.8"])
     @expectedFailureAll(oslist=["freebsd", "linux"], compiler="icc")
     @expectedFailureAll(archs=['mips', 'mipsel', 'mips64', 'mips64el'])
     @expectedFailureAll(
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/functions.c b/lldb/test/API/lang/c/const_variables/functions.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/const_variables/functions.c
rename to lldb/test/API/lang/c/const_variables/functions.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/main.c b/lldb/test/API/lang/c/const_variables/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/const_variables/main.c
rename to lldb/test/API/lang/c/const_variables/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/enum_types/Makefile b/lldb/test/API/lang/c/enum_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/enum_types/Makefile
rename to lldb/test/API/lang/c/enum_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/enum_types/TestEnumTypes.py b/lldb/test/API/lang/c/enum_types/TestEnumTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/enum_types/TestEnumTypes.py
rename to lldb/test/API/lang/c/enum_types/TestEnumTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/enum_types/main.c b/lldb/test/API/lang/c/enum_types/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/enum_types/main.c
rename to lldb/test/API/lang/c/enum_types/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/find_struct_type/Makefile b/lldb/test/API/lang/c/find_struct_type/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/find_struct_type/Makefile
rename to lldb/test/API/lang/c/find_struct_type/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/find_struct_type/TestFindStructTypes.py b/lldb/test/API/lang/c/find_struct_type/TestFindStructTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/find_struct_type/TestFindStructTypes.py
rename to lldb/test/API/lang/c/find_struct_type/TestFindStructTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/find_struct_type/main.c b/lldb/test/API/lang/c/find_struct_type/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/find_struct_type/main.c
rename to lldb/test/API/lang/c/find_struct_type/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/forward/Makefile b/lldb/test/API/lang/c/forward/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/forward/Makefile
rename to lldb/test/API/lang/c/forward/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/forward/README.txt b/lldb/test/API/lang/c/forward/README.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/forward/README.txt
rename to lldb/test/API/lang/c/forward/README.txt
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/forward/TestForwardDeclaration.py b/lldb/test/API/lang/c/forward/TestForwardDeclaration.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/forward/TestForwardDeclaration.py
rename to lldb/test/API/lang/c/forward/TestForwardDeclaration.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/forward/foo.c b/lldb/test/API/lang/c/forward/foo.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/forward/foo.c
rename to lldb/test/API/lang/c/forward/foo.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/forward/foo.h b/lldb/test/API/lang/c/forward/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/forward/foo.h
rename to lldb/test/API/lang/c/forward/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/forward/main.c b/lldb/test/API/lang/c/forward/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/forward/main.c
rename to lldb/test/API/lang/c/forward/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/function_types/Makefile b/lldb/test/API/lang/c/function_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/function_types/Makefile
rename to lldb/test/API/lang/c/function_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/function_types/TestFunctionTypes.py b/lldb/test/API/lang/c/function_types/TestFunctionTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/function_types/TestFunctionTypes.py
rename to lldb/test/API/lang/c/function_types/TestFunctionTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/function_types/main.c b/lldb/test/API/lang/c/function_types/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/function_types/main.c
rename to lldb/test/API/lang/c/function_types/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/global_variables/Makefile b/lldb/test/API/lang/c/global_variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/global_variables/Makefile
rename to lldb/test/API/lang/c/global_variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/global_variables/TestGlobalVariables.py b/lldb/test/API/lang/c/global_variables/TestGlobalVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/global_variables/TestGlobalVariables.py
rename to lldb/test/API/lang/c/global_variables/TestGlobalVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/global_variables/a.c b/lldb/test/API/lang/c/global_variables/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/global_variables/a.c
rename to lldb/test/API/lang/c/global_variables/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/global_variables/cmds.txt b/lldb/test/API/lang/c/global_variables/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/global_variables/cmds.txt
rename to lldb/test/API/lang/c/global_variables/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/global_variables/main.c b/lldb/test/API/lang/c/global_variables/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/global_variables/main.c
rename to lldb/test/API/lang/c/global_variables/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/inlines/TestRedefinitionsInInlines.py b/lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/inlines/TestRedefinitionsInInlines.py
rename to lldb/test/API/lang/c/inlines/TestRedefinitionsInInlines.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/inlines/main.c b/lldb/test/API/lang/c/inlines/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/inlines/main.c
rename to lldb/test/API/lang/c/inlines/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_types/TestUseClosestType.py b/lldb/test/API/lang/c/local_types/TestUseClosestType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/local_types/TestUseClosestType.py
rename to lldb/test/API/lang/c/local_types/TestUseClosestType.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_types/main.c b/lldb/test/API/lang/c/local_types/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/local_types/main.c
rename to lldb/test/API/lang/c/local_types/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_types/other.c b/lldb/test/API/lang/c/local_types/other.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/local_types/other.c
rename to lldb/test/API/lang/c/local_types/other.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/Makefile b/lldb/test/API/lang/c/local_variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/local_variables/Makefile
rename to lldb/test/API/lang/c/local_variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/TestLocalVariables.py b/lldb/test/API/lang/c/local_variables/TestLocalVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/local_variables/TestLocalVariables.py
rename to lldb/test/API/lang/c/local_variables/TestLocalVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/main.c b/lldb/test/API/lang/c/local_variables/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/local_variables/main.c
rename to lldb/test/API/lang/c/local_variables/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/modules/Makefile b/lldb/test/API/lang/c/modules/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/modules/Makefile
rename to lldb/test/API/lang/c/modules/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/modules/TestCModules.py b/lldb/test/API/lang/c/modules/TestCModules.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/modules/TestCModules.py
rename to lldb/test/API/lang/c/modules/TestCModules.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/modules/main.c b/lldb/test/API/lang/c/modules/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/modules/main.c
rename to lldb/test/API/lang/c/modules/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/offsetof/TestOffsetof.py b/lldb/test/API/lang/c/offsetof/TestOffsetof.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/offsetof/TestOffsetof.py
rename to lldb/test/API/lang/c/offsetof/TestOffsetof.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/offsetof/main.c b/lldb/test/API/lang/c/offsetof/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/offsetof/main.c
rename to lldb/test/API/lang/c/offsetof/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/recurse/Makefile b/lldb/test/API/lang/c/recurse/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/recurse/Makefile
rename to lldb/test/API/lang/c/recurse/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/recurse/main.c b/lldb/test/API/lang/c/recurse/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/recurse/main.c
rename to lldb/test/API/lang/c/recurse/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/Makefile b/lldb/test/API/lang/c/register_variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/register_variables/Makefile
rename to lldb/test/API/lang/c/register_variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/TestRegisterVariables.py b/lldb/test/API/lang/c/register_variables/TestRegisterVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/register_variables/TestRegisterVariables.py
rename to lldb/test/API/lang/c/register_variables/TestRegisterVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/test.c b/lldb/test/API/lang/c/register_variables/test.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/register_variables/test.c
rename to lldb/test/API/lang/c/register_variables/test.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/set_values/Makefile b/lldb/test/API/lang/c/set_values/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/set_values/Makefile
rename to lldb/test/API/lang/c/set_values/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/set_values/TestSetValues.py b/lldb/test/API/lang/c/set_values/TestSetValues.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/set_values/TestSetValues.py
rename to lldb/test/API/lang/c/set_values/TestSetValues.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/set_values/main.c b/lldb/test/API/lang/c/set_values/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/set_values/main.c
rename to lldb/test/API/lang/c/set_values/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/Makefile b/lldb/test/API/lang/c/shared_lib/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/Makefile
rename to lldb/test/API/lang/c/shared_lib/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/TestSharedLib.py b/lldb/test/API/lang/c/shared_lib/TestSharedLib.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/TestSharedLib.py
rename to lldb/test/API/lang/c/shared_lib/TestSharedLib.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/foo.c b/lldb/test/API/lang/c/shared_lib/foo.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/foo.c
rename to lldb/test/API/lang/c/shared_lib/foo.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/foo.h b/lldb/test/API/lang/c/shared_lib/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/foo.h
rename to lldb/test/API/lang/c/shared_lib/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/main.c b/lldb/test/API/lang/c/shared_lib/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib/main.c
rename to lldb/test/API/lang/c/shared_lib/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/Makefile b/lldb/test/API/lang/c/shared_lib_stripped_symbols/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/Makefile
rename to lldb/test/API/lang/c/shared_lib_stripped_symbols/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/TestSharedLibStrippedSymbols.py b/lldb/test/API/lang/c/shared_lib_stripped_symbols/TestSharedLibStrippedSymbols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/TestSharedLibStrippedSymbols.py
rename to lldb/test/API/lang/c/shared_lib_stripped_symbols/TestSharedLibStrippedSymbols.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/foo.c b/lldb/test/API/lang/c/shared_lib_stripped_symbols/foo.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/foo.c
rename to lldb/test/API/lang/c/shared_lib_stripped_symbols/foo.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/foo.h b/lldb/test/API/lang/c/shared_lib_stripped_symbols/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/foo.h
rename to lldb/test/API/lang/c/shared_lib_stripped_symbols/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/main.c b/lldb/test/API/lang/c/shared_lib_stripped_symbols/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/shared_lib_stripped_symbols/main.c
rename to lldb/test/API/lang/c/shared_lib_stripped_symbols/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step-target/.categories b/lldb/test/API/lang/c/step-target/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step-target/.categories
rename to lldb/test/API/lang/c/step-target/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step-target/Makefile b/lldb/test/API/lang/c/step-target/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step-target/Makefile
rename to lldb/test/API/lang/c/step-target/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step-target/TestStepTarget.py b/lldb/test/API/lang/c/step-target/TestStepTarget.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step-target/TestStepTarget.py
rename to lldb/test/API/lang/c/step-target/TestStepTarget.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step-target/main.c b/lldb/test/API/lang/c/step-target/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step-target/main.c
rename to lldb/test/API/lang/c/step-target/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step_over_no_deadlock/Makefile b/lldb/test/API/lang/c/step_over_no_deadlock/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step_over_no_deadlock/Makefile
rename to lldb/test/API/lang/c/step_over_no_deadlock/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step_over_no_deadlock/TestStepOverDoesntBlock.py b/lldb/test/API/lang/c/step_over_no_deadlock/TestStepOverDoesntBlock.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step_over_no_deadlock/TestStepOverDoesntBlock.py
rename to lldb/test/API/lang/c/step_over_no_deadlock/TestStepOverDoesntBlock.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/step_over_no_deadlock/locking.cpp b/lldb/test/API/lang/c/step_over_no_deadlock/locking.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/step_over_no_deadlock/locking.cpp
rename to lldb/test/API/lang/c/step_over_no_deadlock/locking.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/stepping/Makefile b/lldb/test/API/lang/c/stepping/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/stepping/Makefile
rename to lldb/test/API/lang/c/stepping/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/stepping/TestStepAndBreakpoints.py b/lldb/test/API/lang/c/stepping/TestStepAndBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/stepping/TestStepAndBreakpoints.py
rename to lldb/test/API/lang/c/stepping/TestStepAndBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/stepping/TestThreadStepping.py b/lldb/test/API/lang/c/stepping/TestThreadStepping.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/stepping/TestThreadStepping.py
rename to lldb/test/API/lang/c/stepping/TestThreadStepping.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/stepping/main.c b/lldb/test/API/lang/c/stepping/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/stepping/main.c
rename to lldb/test/API/lang/c/stepping/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/strings/Makefile b/lldb/test/API/lang/c/strings/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/strings/Makefile
rename to lldb/test/API/lang/c/strings/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/strings/TestCStrings.py b/lldb/test/API/lang/c/strings/TestCStrings.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/strings/TestCStrings.py
rename to lldb/test/API/lang/c/strings/TestCStrings.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/strings/main.c b/lldb/test/API/lang/c/strings/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/strings/main.c
rename to lldb/test/API/lang/c/strings/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/struct_types/TestStructTypes.py b/lldb/test/API/lang/c/struct_types/TestStructTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/struct_types/TestStructTypes.py
rename to lldb/test/API/lang/c/struct_types/TestStructTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/struct_types/main.c b/lldb/test/API/lang/c/struct_types/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/struct_types/main.c
rename to lldb/test/API/lang/c/struct_types/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/Makefile b/lldb/test/API/lang/c/tls_globals/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/Makefile
rename to lldb/test/API/lang/c/tls_globals/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/TestTlsGlobals.py b/lldb/test/API/lang/c/tls_globals/TestTlsGlobals.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/TestTlsGlobals.py
rename to lldb/test/API/lang/c/tls_globals/TestTlsGlobals.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/a.c b/lldb/test/API/lang/c/tls_globals/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/a.c
rename to lldb/test/API/lang/c/tls_globals/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/main.c b/lldb/test/API/lang/c/tls_globals/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/tls_globals/main.c
rename to lldb/test/API/lang/c/tls_globals/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/typedef/Makefile b/lldb/test/API/lang/c/typedef/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/typedef/Makefile
rename to lldb/test/API/lang/c/typedef/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/typedef/Testtypedef.py b/lldb/test/API/lang/c/typedef/Testtypedef.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/typedef/Testtypedef.py
rename to lldb/test/API/lang/c/typedef/Testtypedef.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/typedef/main.c b/lldb/test/API/lang/c/typedef/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/typedef/main.c
rename to lldb/test/API/lang/c/typedef/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unicode/Makefile b/lldb/test/API/lang/c/unicode/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/unicode/Makefile
rename to lldb/test/API/lang/c/unicode/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unicode/TestUnicodeSymbols.py b/lldb/test/API/lang/c/unicode/TestUnicodeSymbols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/unicode/TestUnicodeSymbols.py
rename to lldb/test/API/lang/c/unicode/TestUnicodeSymbols.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unicode/main.c b/lldb/test/API/lang/c/unicode/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/unicode/main.c
rename to lldb/test/API/lang/c/unicode/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unions/Makefile b/lldb/test/API/lang/c/unions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/unions/Makefile
rename to lldb/test/API/lang/c/unions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unions/TestUnionMembers.py b/lldb/test/API/lang/c/unions/TestUnionMembers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/unions/TestUnionMembers.py
rename to lldb/test/API/lang/c/unions/TestUnionMembers.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unions/main.c b/lldb/test/API/lang/c/unions/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/unions/main.c
rename to lldb/test/API/lang/c/unions/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/vla/Makefile b/lldb/test/API/lang/c/vla/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/vla/Makefile
rename to lldb/test/API/lang/c/vla/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/vla/TestVLA.py b/lldb/test/API/lang/c/vla/TestVLA.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/vla/TestVLA.py
rename to lldb/test/API/lang/c/vla/TestVLA.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/vla/main.c b/lldb/test/API/lang/c/vla/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/c/vla/main.c
rename to lldb/test/API/lang/c/vla/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/Makefile b/lldb/test/API/lang/cpp/accelerator-table/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/Makefile
rename to lldb/test/API/lang/cpp/accelerator-table/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/TestCPPAccelerator.py b/lldb/test/API/lang/cpp/accelerator-table/TestCPPAccelerator.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/TestCPPAccelerator.py
rename to lldb/test/API/lang/cpp/accelerator-table/TestCPPAccelerator.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/a.cpp b/lldb/test/API/lang/cpp/accelerator-table/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/a.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/b.cpp b/lldb/test/API/lang/cpp/accelerator-table/b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/b.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/c.cpp b/lldb/test/API/lang/cpp/accelerator-table/c.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/c.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/c.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/d.cpp b/lldb/test/API/lang/cpp/accelerator-table/d.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/d.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/d.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/e.cpp b/lldb/test/API/lang/cpp/accelerator-table/e.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/e.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/e.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/f.cpp b/lldb/test/API/lang/cpp/accelerator-table/f.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/f.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/f.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/g.cpp b/lldb/test/API/lang/cpp/accelerator-table/g.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/g.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/g.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/main.cpp b/lldb/test/API/lang/cpp/accelerator-table/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/main.cpp
rename to lldb/test/API/lang/cpp/accelerator-table/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/source.h b/lldb/test/API/lang/cpp/accelerator-table/source.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/accelerator-table/source.h
rename to lldb/test/API/lang/cpp/accelerator-table/source.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/auto/Makefile b/lldb/test/API/lang/cpp/auto/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/auto/Makefile
rename to lldb/test/API/lang/cpp/auto/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/auto/TestCPPAuto.py b/lldb/test/API/lang/cpp/auto/TestCPPAuto.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/auto/TestCPPAuto.py
rename to lldb/test/API/lang/cpp/auto/TestCPPAuto.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/auto/main.cpp b/lldb/test/API/lang/cpp/auto/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/auto/main.cpp
rename to lldb/test/API/lang/cpp/auto/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/bitfields/Makefile b/lldb/test/API/lang/cpp/bitfields/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/bitfields/Makefile
rename to lldb/test/API/lang/cpp/bitfields/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/bitfields/TestCppBitfields.py b/lldb/test/API/lang/cpp/bitfields/TestCppBitfields.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/bitfields/TestCppBitfields.py
rename to lldb/test/API/lang/cpp/bitfields/TestCppBitfields.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/bitfields/main.cpp b/lldb/test/API/lang/cpp/bitfields/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/bitfields/main.cpp
rename to lldb/test/API/lang/cpp/bitfields/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/bool/Makefile b/lldb/test/API/lang/cpp/bool/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/bool/Makefile
rename to lldb/test/API/lang/cpp/bool/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/bool/TestCPPBool.py b/lldb/test/API/lang/cpp/bool/TestCPPBool.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/bool/TestCPPBool.py
rename to lldb/test/API/lang/cpp/bool/TestCPPBool.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/bool/main.cpp b/lldb/test/API/lang/cpp/bool/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/bool/main.cpp
rename to lldb/test/API/lang/cpp/bool/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint-commands/Makefile b/lldb/test/API/lang/cpp/breakpoint-commands/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint-commands/Makefile
rename to lldb/test/API/lang/cpp/breakpoint-commands/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint-commands/TestCPPBreakpointCommands.py b/lldb/test/API/lang/cpp/breakpoint-commands/TestCPPBreakpointCommands.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint-commands/TestCPPBreakpointCommands.py
rename to lldb/test/API/lang/cpp/breakpoint-commands/TestCPPBreakpointCommands.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint-commands/nested.cpp b/lldb/test/API/lang/cpp/breakpoint-commands/nested.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint-commands/nested.cpp
rename to lldb/test/API/lang/cpp/breakpoint-commands/nested.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/Makefile b/lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/Makefile
rename to lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/TestBreakpointInMemberFuncWNonPrimitiveParams.py b/lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/TestBreakpointInMemberFuncWNonPrimitiveParams.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/TestBreakpointInMemberFuncWNonPrimitiveParams.py
rename to lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/TestBreakpointInMemberFuncWNonPrimitiveParams.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.cpp b/lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.cpp
rename to lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.h b/lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.h
rename to lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/a.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/main.cpp b/lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/main.cpp
rename to lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/module.modulemap b/lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/module.modulemap
rename to lldb/test/API/lang/cpp/breakpoint_in_member_func_w_non_primitive_params/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/call-function/Makefile b/lldb/test/API/lang/cpp/call-function/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/call-function/Makefile
rename to lldb/test/API/lang/cpp/call-function/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/call-function/TestCallCPPFunction.py b/lldb/test/API/lang/cpp/call-function/TestCallCPPFunction.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/call-function/TestCallCPPFunction.py
rename to lldb/test/API/lang/cpp/call-function/TestCallCPPFunction.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/call-function/main.cpp b/lldb/test/API/lang/cpp/call-function/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/call-function/main.cpp
rename to lldb/test/API/lang/cpp/call-function/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/chained-calls/Makefile b/lldb/test/API/lang/cpp/chained-calls/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/chained-calls/Makefile
rename to lldb/test/API/lang/cpp/chained-calls/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/chained-calls/TestCppChainedCalls.py b/lldb/test/API/lang/cpp/chained-calls/TestCppChainedCalls.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/chained-calls/TestCppChainedCalls.py
rename to lldb/test/API/lang/cpp/chained-calls/TestCppChainedCalls.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/chained-calls/main.cpp b/lldb/test/API/lang/cpp/chained-calls/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/chained-calls/main.cpp
rename to lldb/test/API/lang/cpp/chained-calls/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/.categories b/lldb/test/API/lang/cpp/char1632_t/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/.categories
rename to lldb/test/API/lang/cpp/char1632_t/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/Makefile b/lldb/test/API/lang/cpp/char1632_t/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/Makefile
rename to lldb/test/API/lang/cpp/char1632_t/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/TestChar1632T.py b/lldb/test/API/lang/cpp/char1632_t/TestChar1632T.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/TestChar1632T.py
rename to lldb/test/API/lang/cpp/char1632_t/TestChar1632T.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/main.cpp b/lldb/test/API/lang/cpp/char1632_t/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char1632_t/main.cpp
rename to lldb/test/API/lang/cpp/char1632_t/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char8_t/Makefile b/lldb/test/API/lang/cpp/char8_t/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char8_t/Makefile
rename to lldb/test/API/lang/cpp/char8_t/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char8_t/TestCxxChar8_t.py b/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char8_t/TestCxxChar8_t.py
rename to lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/char8_t/main.cpp b/lldb/test/API/lang/cpp/char8_t/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/char8_t/main.cpp
rename to lldb/test/API/lang/cpp/char8_t/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py b/lldb/test/API/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py
rename to lldb/test/API/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class-template-parameter-pack/main.cpp b/lldb/test/API/lang/cpp/class-template-parameter-pack/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class-template-parameter-pack/main.cpp
rename to lldb/test/API/lang/cpp/class-template-parameter-pack/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_static/Makefile b/lldb/test/API/lang/cpp/class_static/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_static/Makefile
rename to lldb/test/API/lang/cpp/class_static/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_static/TestStaticVariables.py b/lldb/test/API/lang/cpp/class_static/TestStaticVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_static/TestStaticVariables.py
rename to lldb/test/API/lang/cpp/class_static/TestStaticVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_static/main.cpp b/lldb/test/API/lang/cpp/class_static/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_static/main.cpp
rename to lldb/test/API/lang/cpp/class_static/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/Makefile b/lldb/test/API/lang/cpp/class_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/Makefile
rename to lldb/test/API/lang/cpp/class_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/TestClassTypes.py b/lldb/test/API/lang/cpp/class_types/TestClassTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/TestClassTypes.py
rename to lldb/test/API/lang/cpp/class_types/TestClassTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/TestClassTypesDisassembly.py b/lldb/test/API/lang/cpp/class_types/TestClassTypesDisassembly.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/TestClassTypesDisassembly.py
rename to lldb/test/API/lang/cpp/class_types/TestClassTypesDisassembly.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/cmds.txt b/lldb/test/API/lang/cpp/class_types/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/cmds.txt
rename to lldb/test/API/lang/cpp/class_types/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/main.cpp b/lldb/test/API/lang/cpp/class_types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/class_types/main.cpp
rename to lldb/test/API/lang/cpp/class_types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/const_this/TestConstThis.py b/lldb/test/API/lang/cpp/const_this/TestConstThis.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/const_this/TestConstThis.py
rename to lldb/test/API/lang/cpp/const_this/TestConstThis.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/const_this/main.cpp b/lldb/test/API/lang/cpp/const_this/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/const_this/main.cpp
rename to lldb/test/API/lang/cpp/const_this/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/constructors/Makefile b/lldb/test/API/lang/cpp/constructors/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/constructors/Makefile
rename to lldb/test/API/lang/cpp/constructors/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/constructors/TestCppConstructors.py b/lldb/test/API/lang/cpp/constructors/TestCppConstructors.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/constructors/TestCppConstructors.py
rename to lldb/test/API/lang/cpp/constructors/TestCppConstructors.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/constructors/main.cpp b/lldb/test/API/lang/cpp/constructors/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/constructors/main.cpp
rename to lldb/test/API/lang/cpp/constructors/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/covariant-return-types/Makefile b/lldb/test/API/lang/cpp/covariant-return-types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/covariant-return-types/Makefile
rename to lldb/test/API/lang/cpp/covariant-return-types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py b/lldb/test/API/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py
rename to lldb/test/API/lang/cpp/covariant-return-types/TestCovariantReturnTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/covariant-return-types/main.cpp b/lldb/test/API/lang/cpp/covariant-return-types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/covariant-return-types/main.cpp
rename to lldb/test/API/lang/cpp/covariant-return-types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/diamond/Makefile b/lldb/test/API/lang/cpp/diamond/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/diamond/Makefile
rename to lldb/test/API/lang/cpp/diamond/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/diamond/TestDiamond.py b/lldb/test/API/lang/cpp/diamond/TestDiamond.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/diamond/TestDiamond.py
rename to lldb/test/API/lang/cpp/diamond/TestDiamond.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/diamond/main.cpp b/lldb/test/API/lang/cpp/diamond/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/diamond/main.cpp
rename to lldb/test/API/lang/cpp/diamond/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value-same-basename/Makefile b/lldb/test/API/lang/cpp/dynamic-value-same-basename/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value-same-basename/Makefile
rename to lldb/test/API/lang/cpp/dynamic-value-same-basename/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value-same-basename/TestDynamicValueSameBase.py b/lldb/test/API/lang/cpp/dynamic-value-same-basename/TestDynamicValueSameBase.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value-same-basename/TestDynamicValueSameBase.py
rename to lldb/test/API/lang/cpp/dynamic-value-same-basename/TestDynamicValueSameBase.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value-same-basename/main.cpp b/lldb/test/API/lang/cpp/dynamic-value-same-basename/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value-same-basename/main.cpp
rename to lldb/test/API/lang/cpp/dynamic-value-same-basename/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/Makefile b/lldb/test/API/lang/cpp/dynamic-value/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/Makefile
rename to lldb/test/API/lang/cpp/dynamic-value/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/TestCppValueCast.py b/lldb/test/API/lang/cpp/dynamic-value/TestCppValueCast.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/TestCppValueCast.py
rename to lldb/test/API/lang/cpp/dynamic-value/TestCppValueCast.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/TestDynamicValue.py b/lldb/test/API/lang/cpp/dynamic-value/TestDynamicValue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/TestDynamicValue.py
rename to lldb/test/API/lang/cpp/dynamic-value/TestDynamicValue.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/pass-to-base.cpp b/lldb/test/API/lang/cpp/dynamic-value/pass-to-base.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/pass-to-base.cpp
rename to lldb/test/API/lang/cpp/dynamic-value/pass-to-base.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/sbvalue-cast.cpp b/lldb/test/API/lang/cpp/dynamic-value/sbvalue-cast.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/dynamic-value/sbvalue-cast.cpp
rename to lldb/test/API/lang/cpp/dynamic-value/sbvalue-cast.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/Makefile b/lldb/test/API/lang/cpp/enum_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/Makefile
rename to lldb/test/API/lang/cpp/enum_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/TestCPP11EnumTypes.py b/lldb/test/API/lang/cpp/enum_types/TestCPP11EnumTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/TestCPP11EnumTypes.py
rename to lldb/test/API/lang/cpp/enum_types/TestCPP11EnumTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/main.cpp b/lldb/test/API/lang/cpp/enum_types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/main.cpp
rename to lldb/test/API/lang/cpp/enum_types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/exceptions/Makefile b/lldb/test/API/lang/cpp/exceptions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/exceptions/Makefile
rename to lldb/test/API/lang/cpp/exceptions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/exceptions/TestCPPExceptionBreakpoints.py b/lldb/test/API/lang/cpp/exceptions/TestCPPExceptionBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/exceptions/TestCPPExceptionBreakpoints.py
rename to lldb/test/API/lang/cpp/exceptions/TestCPPExceptionBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/exceptions/exceptions.cpp b/lldb/test/API/lang/cpp/exceptions/exceptions.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/exceptions/exceptions.cpp
rename to lldb/test/API/lang/cpp/exceptions/exceptions.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/extern_c/TestExternCSymbols.py b/lldb/test/API/lang/cpp/extern_c/TestExternCSymbols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/extern_c/TestExternCSymbols.py
rename to lldb/test/API/lang/cpp/extern_c/TestExternCSymbols.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/extern_c/main.cpp b/lldb/test/API/lang/cpp/extern_c/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/extern_c/main.cpp
rename to lldb/test/API/lang/cpp/extern_c/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/frame-var-anon-unions/Makefile b/lldb/test/API/lang/cpp/frame-var-anon-unions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/frame-var-anon-unions/Makefile
rename to lldb/test/API/lang/cpp/frame-var-anon-unions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/frame-var-anon-unions/TestFrameVariableAnonymousUnions.py b/lldb/test/API/lang/cpp/frame-var-anon-unions/TestFrameVariableAnonymousUnions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/frame-var-anon-unions/TestFrameVariableAnonymousUnions.py
rename to lldb/test/API/lang/cpp/frame-var-anon-unions/TestFrameVariableAnonymousUnions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/frame-var-anon-unions/main.cpp b/lldb/test/API/lang/cpp/frame-var-anon-unions/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/frame-var-anon-unions/main.cpp
rename to lldb/test/API/lang/cpp/frame-var-anon-unions/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function-qualifiers/Makefile b/lldb/test/API/lang/cpp/function-qualifiers/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function-qualifiers/Makefile
rename to lldb/test/API/lang/cpp/function-qualifiers/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function-qualifiers/TestCppFunctionQualifiers.py b/lldb/test/API/lang/cpp/function-qualifiers/TestCppFunctionQualifiers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function-qualifiers/TestCppFunctionQualifiers.py
rename to lldb/test/API/lang/cpp/function-qualifiers/TestCppFunctionQualifiers.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function-qualifiers/main.cpp b/lldb/test/API/lang/cpp/function-qualifiers/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function-qualifiers/main.cpp
rename to lldb/test/API/lang/cpp/function-qualifiers/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function-template-parameter-pack/TestFunctionTemplateParameterPack.py b/lldb/test/API/lang/cpp/function-template-parameter-pack/TestFunctionTemplateParameterPack.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function-template-parameter-pack/TestFunctionTemplateParameterPack.py
rename to lldb/test/API/lang/cpp/function-template-parameter-pack/TestFunctionTemplateParameterPack.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function-template-parameter-pack/main.cpp b/lldb/test/API/lang/cpp/function-template-parameter-pack/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function-template-parameter-pack/main.cpp
rename to lldb/test/API/lang/cpp/function-template-parameter-pack/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function_refs/TestFunctionRefs.py b/lldb/test/API/lang/cpp/function_refs/TestFunctionRefs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function_refs/TestFunctionRefs.py
rename to lldb/test/API/lang/cpp/function_refs/TestFunctionRefs.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/function_refs/main.cpp b/lldb/test/API/lang/cpp/function_refs/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/function_refs/main.cpp
rename to lldb/test/API/lang/cpp/function_refs/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/global_operators/Makefile b/lldb/test/API/lang/cpp/global_operators/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/global_operators/Makefile
rename to lldb/test/API/lang/cpp/global_operators/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/global_operators/TestCppGlobalOperators.py b/lldb/test/API/lang/cpp/global_operators/TestCppGlobalOperators.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/global_operators/TestCppGlobalOperators.py
rename to lldb/test/API/lang/cpp/global_operators/TestCppGlobalOperators.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/global_operators/main.cpp b/lldb/test/API/lang/cpp/global_operators/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/global_operators/main.cpp
rename to lldb/test/API/lang/cpp/global_operators/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/global_variables/Makefile b/lldb/test/API/lang/cpp/global_variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/global_variables/Makefile
rename to lldb/test/API/lang/cpp/global_variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/global_variables/TestCPPGlobalVariables.py b/lldb/test/API/lang/cpp/global_variables/TestCPPGlobalVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/global_variables/TestCPPGlobalVariables.py
rename to lldb/test/API/lang/cpp/global_variables/TestCPPGlobalVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/global_variables/main.cpp b/lldb/test/API/lang/cpp/global_variables/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/global_variables/main.cpp
rename to lldb/test/API/lang/cpp/global_variables/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/TestGModules.py b/lldb/test/API/lang/cpp/gmodules-templates/TestGModules.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/TestGModules.py
rename to lldb/test/API/lang/cpp/gmodules-templates/TestGModules.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/a.h b/lldb/test/API/lang/cpp/gmodules-templates/a.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/a.h
rename to lldb/test/API/lang/cpp/gmodules-templates/a.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/b.h b/lldb/test/API/lang/cpp/gmodules-templates/b.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/b.h
rename to lldb/test/API/lang/cpp/gmodules-templates/b.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/main.cpp b/lldb/test/API/lang/cpp/gmodules-templates/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/main.cpp
rename to lldb/test/API/lang/cpp/gmodules-templates/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/memory.h b/lldb/test/API/lang/cpp/gmodules-templates/memory.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/memory.h
rename to lldb/test/API/lang/cpp/gmodules-templates/memory.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/module.modulemap b/lldb/test/API/lang/cpp/gmodules-templates/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules-templates/module.modulemap
rename to lldb/test/API/lang/cpp/gmodules-templates/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/Makefile b/lldb/test/API/lang/cpp/gmodules/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/Makefile
rename to lldb/test/API/lang/cpp/gmodules/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/TestWithModuleDebugging.py b/lldb/test/API/lang/cpp/gmodules/TestWithModuleDebugging.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/TestWithModuleDebugging.py
rename to lldb/test/API/lang/cpp/gmodules/TestWithModuleDebugging.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/main.cpp b/lldb/test/API/lang/cpp/gmodules/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/main.cpp
rename to lldb/test/API/lang/cpp/gmodules/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/pch.h b/lldb/test/API/lang/cpp/gmodules/pch.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/pch.h
rename to lldb/test/API/lang/cpp/gmodules/pch.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/Makefile b/lldb/test/API/lang/cpp/incomplete-types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/Makefile
rename to lldb/test/API/lang/cpp/incomplete-types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/TestCppIncompleteTypes.py b/lldb/test/API/lang/cpp/incomplete-types/TestCppIncompleteTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/TestCppIncompleteTypes.py
rename to lldb/test/API/lang/cpp/incomplete-types/TestCppIncompleteTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/a.cpp b/lldb/test/API/lang/cpp/incomplete-types/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/a.cpp
rename to lldb/test/API/lang/cpp/incomplete-types/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/a.h b/lldb/test/API/lang/cpp/incomplete-types/a.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/a.h
rename to lldb/test/API/lang/cpp/incomplete-types/a.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/length.cpp b/lldb/test/API/lang/cpp/incomplete-types/length.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/length.cpp
rename to lldb/test/API/lang/cpp/incomplete-types/length.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/length.h b/lldb/test/API/lang/cpp/incomplete-types/length.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/length.h
rename to lldb/test/API/lang/cpp/incomplete-types/length.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/main.cpp b/lldb/test/API/lang/cpp/incomplete-types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/incomplete-types/main.cpp
rename to lldb/test/API/lang/cpp/incomplete-types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/Makefile b/lldb/test/API/lang/cpp/inlines/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/Makefile
rename to lldb/test/API/lang/cpp/inlines/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/TestInlines.py b/lldb/test/API/lang/cpp/inlines/TestInlines.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/TestInlines.py
rename to lldb/test/API/lang/cpp/inlines/TestInlines.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/inlines.cpp b/lldb/test/API/lang/cpp/inlines/inlines.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/inlines.cpp
rename to lldb/test/API/lang/cpp/inlines/inlines.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/inlines.h b/lldb/test/API/lang/cpp/inlines/inlines.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/inlines/inlines.h
rename to lldb/test/API/lang/cpp/inlines/inlines.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/lambdas/TestLambdas.py b/lldb/test/API/lang/cpp/lambdas/TestLambdas.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/lambdas/TestLambdas.py
rename to lldb/test/API/lang/cpp/lambdas/TestLambdas.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/lambdas/main.cpp b/lldb/test/API/lang/cpp/lambdas/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/lambdas/main.cpp
rename to lldb/test/API/lang/cpp/lambdas/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/Makefile b/lldb/test/API/lang/cpp/limit-debug-info/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/Makefile
rename to lldb/test/API/lang/cpp/limit-debug-info/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/TestWithLimitDebugInfo.py b/lldb/test/API/lang/cpp/limit-debug-info/TestWithLimitDebugInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/TestWithLimitDebugInfo.py
rename to lldb/test/API/lang/cpp/limit-debug-info/TestWithLimitDebugInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/base.cpp b/lldb/test/API/lang/cpp/limit-debug-info/base.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/base.cpp
rename to lldb/test/API/lang/cpp/limit-debug-info/base.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/base.h b/lldb/test/API/lang/cpp/limit-debug-info/base.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/base.h
rename to lldb/test/API/lang/cpp/limit-debug-info/base.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/derived.cpp b/lldb/test/API/lang/cpp/limit-debug-info/derived.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/derived.cpp
rename to lldb/test/API/lang/cpp/limit-debug-info/derived.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/derived.h b/lldb/test/API/lang/cpp/limit-debug-info/derived.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/derived.h
rename to lldb/test/API/lang/cpp/limit-debug-info/derived.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/main.cpp b/lldb/test/API/lang/cpp/limit-debug-info/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/main.cpp
rename to lldb/test/API/lang/cpp/limit-debug-info/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/llvm-style/TestLLVMStyle.py b/lldb/test/API/lang/cpp/llvm-style/TestLLVMStyle.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/llvm-style/TestLLVMStyle.py
rename to lldb/test/API/lang/cpp/llvm-style/TestLLVMStyle.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/llvm-style/main.cpp b/lldb/test/API/lang/cpp/llvm-style/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/llvm-style/main.cpp
rename to lldb/test/API/lang/cpp/llvm-style/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/member-and-local-vars-with-same-name/Makefile b/lldb/test/API/lang/cpp/member-and-local-vars-with-same-name/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/member-and-local-vars-with-same-name/Makefile
rename to lldb/test/API/lang/cpp/member-and-local-vars-with-same-name/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/member-and-local-vars-with-same-name/TestMembersAndLocalsWithSameName.py b/lldb/test/API/lang/cpp/member-and-local-vars-with-same-name/TestMembersAndLocalsWithSameName.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/member-and-local-vars-with-same-name/TestMembersAndLocalsWithSameName.py
rename to lldb/test/API/lang/cpp/member-and-local-vars-with-same-name/TestMembersAndLocalsWithSameName.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/member-and-local-vars-with-same-name/main.cpp b/lldb/test/API/lang/cpp/member-and-local-vars-with-same-name/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/member-and-local-vars-with-same-name/main.cpp
rename to lldb/test/API/lang/cpp/member-and-local-vars-with-same-name/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Inputs/Bar.h b/lldb/test/API/lang/cpp/modules-import/Inputs/Bar.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Inputs/Bar.h
rename to lldb/test/API/lang/cpp/modules-import/Inputs/Bar.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Inputs/Foo.h b/lldb/test/API/lang/cpp/modules-import/Inputs/Foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Inputs/Foo.h
rename to lldb/test/API/lang/cpp/modules-import/Inputs/Foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Inputs/module.modulemap b/lldb/test/API/lang/cpp/modules-import/Inputs/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Inputs/module.modulemap
rename to lldb/test/API/lang/cpp/modules-import/Inputs/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Makefile b/lldb/test/API/lang/cpp/modules-import/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Makefile
rename to lldb/test/API/lang/cpp/modules-import/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/TestCXXModulesImport.py b/lldb/test/API/lang/cpp/modules-import/TestCXXModulesImport.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/TestCXXModulesImport.py
rename to lldb/test/API/lang/cpp/modules-import/TestCXXModulesImport.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/main.cpp b/lldb/test/API/lang/cpp/modules-import/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/main.cpp
rename to lldb/test/API/lang/cpp/modules-import/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/multiple-inheritance/Makefile b/lldb/test/API/lang/cpp/multiple-inheritance/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/multiple-inheritance/Makefile
rename to lldb/test/API/lang/cpp/multiple-inheritance/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py b/lldb/test/API/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py
rename to lldb/test/API/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/multiple-inheritance/main.cpp b/lldb/test/API/lang/cpp/multiple-inheritance/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/multiple-inheritance/main.cpp
rename to lldb/test/API/lang/cpp/multiple-inheritance/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/Makefile b/lldb/test/API/lang/cpp/namespace/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/Makefile
rename to lldb/test/API/lang/cpp/namespace/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/TestNamespace.py b/lldb/test/API/lang/cpp/namespace/TestNamespace.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/TestNamespace.py
rename to lldb/test/API/lang/cpp/namespace/TestNamespace.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/TestNamespaceLookup.py b/lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/TestNamespaceLookup.py
rename to lldb/test/API/lang/cpp/namespace/TestNamespaceLookup.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/cmds.txt b/lldb/test/API/lang/cpp/namespace/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/cmds.txt
rename to lldb/test/API/lang/cpp/namespace/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/main.cpp b/lldb/test/API/lang/cpp/namespace/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/main.cpp
rename to lldb/test/API/lang/cpp/namespace/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns.cpp b/lldb/test/API/lang/cpp/namespace/ns.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns.cpp
rename to lldb/test/API/lang/cpp/namespace/ns.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns.h b/lldb/test/API/lang/cpp/namespace/ns.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns.h
rename to lldb/test/API/lang/cpp/namespace/ns.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns2.cpp b/lldb/test/API/lang/cpp/namespace/ns2.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns2.cpp
rename to lldb/test/API/lang/cpp/namespace/ns2.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns3.cpp b/lldb/test/API/lang/cpp/namespace/ns3.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace/ns3.cpp
rename to lldb/test/API/lang/cpp/namespace/ns3.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_conflicts/TestNamespaceConflicts.py b/lldb/test/API/lang/cpp/namespace_conflicts/TestNamespaceConflicts.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_conflicts/TestNamespaceConflicts.py
rename to lldb/test/API/lang/cpp/namespace_conflicts/TestNamespaceConflicts.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_conflicts/main.cpp b/lldb/test/API/lang/cpp/namespace_conflicts/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_conflicts/main.cpp
rename to lldb/test/API/lang/cpp/namespace_conflicts/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/Makefile b/lldb/test/API/lang/cpp/namespace_definitions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/Makefile
rename to lldb/test/API/lang/cpp/namespace_definitions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/TestNamespaceDefinitions.py b/lldb/test/API/lang/cpp/namespace_definitions/TestNamespaceDefinitions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/TestNamespaceDefinitions.py
rename to lldb/test/API/lang/cpp/namespace_definitions/TestNamespaceDefinitions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/a.cpp b/lldb/test/API/lang/cpp/namespace_definitions/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/a.cpp
rename to lldb/test/API/lang/cpp/namespace_definitions/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/b.cpp b/lldb/test/API/lang/cpp/namespace_definitions/b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/b.cpp
rename to lldb/test/API/lang/cpp/namespace_definitions/b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/foo.h b/lldb/test/API/lang/cpp/namespace_definitions/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/foo.h
rename to lldb/test/API/lang/cpp/namespace_definitions/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/main.cpp b/lldb/test/API/lang/cpp/namespace_definitions/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/namespace_definitions/main.cpp
rename to lldb/test/API/lang/cpp/namespace_definitions/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/Makefile b/lldb/test/API/lang/cpp/nested-class-other-compilation-unit/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/Makefile
rename to lldb/test/API/lang/cpp/nested-class-other-compilation-unit/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py b/lldb/test/API/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py
rename to lldb/test/API/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/main.cpp b/lldb/test/API/lang/cpp/nested-class-other-compilation-unit/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/main.cpp
rename to lldb/test/API/lang/cpp/nested-class-other-compilation-unit/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/other.cpp b/lldb/test/API/lang/cpp/nested-class-other-compilation-unit/other.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/other.cpp
rename to lldb/test/API/lang/cpp/nested-class-other-compilation-unit/other.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/shared.h b/lldb/test/API/lang/cpp/nested-class-other-compilation-unit/shared.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/shared.h
rename to lldb/test/API/lang/cpp/nested-class-other-compilation-unit/shared.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nsimport/Makefile b/lldb/test/API/lang/cpp/nsimport/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nsimport/Makefile
rename to lldb/test/API/lang/cpp/nsimport/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nsimport/TestCppNsImport.py b/lldb/test/API/lang/cpp/nsimport/TestCppNsImport.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nsimport/TestCppNsImport.py
rename to lldb/test/API/lang/cpp/nsimport/TestCppNsImport.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nsimport/main.cpp b/lldb/test/API/lang/cpp/nsimport/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/nsimport/main.cpp
rename to lldb/test/API/lang/cpp/nsimport/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/offsetof/TestOffsetofCpp.py b/lldb/test/API/lang/cpp/offsetof/TestOffsetofCpp.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/offsetof/TestOffsetofCpp.py
rename to lldb/test/API/lang/cpp/offsetof/TestOffsetofCpp.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/offsetof/main.cpp b/lldb/test/API/lang/cpp/offsetof/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/offsetof/main.cpp
rename to lldb/test/API/lang/cpp/offsetof/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/Makefile b/lldb/test/API/lang/cpp/operator-overload/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/Makefile
rename to lldb/test/API/lang/cpp/operator-overload/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/TestOperatorOverload.py b/lldb/test/API/lang/cpp/operator-overload/TestOperatorOverload.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/TestOperatorOverload.py
rename to lldb/test/API/lang/cpp/operator-overload/TestOperatorOverload.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/a.cpp b/lldb/test/API/lang/cpp/operator-overload/a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/a.cpp
rename to lldb/test/API/lang/cpp/operator-overload/a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/b.cpp b/lldb/test/API/lang/cpp/operator-overload/b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/operator-overload/b.cpp
rename to lldb/test/API/lang/cpp/operator-overload/b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/operators/TestCppOperators.py b/lldb/test/API/lang/cpp/operators/TestCppOperators.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/operators/TestCppOperators.py
rename to lldb/test/API/lang/cpp/operators/TestCppOperators.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/operators/main.cpp b/lldb/test/API/lang/cpp/operators/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/operators/main.cpp
rename to lldb/test/API/lang/cpp/operators/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/Makefile b/lldb/test/API/lang/cpp/overloaded-functions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/Makefile
rename to lldb/test/API/lang/cpp/overloaded-functions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/TestOverloadedFunctions.py b/lldb/test/API/lang/cpp/overloaded-functions/TestOverloadedFunctions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/TestOverloadedFunctions.py
rename to lldb/test/API/lang/cpp/overloaded-functions/TestOverloadedFunctions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/main.cpp b/lldb/test/API/lang/cpp/overloaded-functions/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/main.cpp
rename to lldb/test/API/lang/cpp/overloaded-functions/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/static-a.cpp b/lldb/test/API/lang/cpp/overloaded-functions/static-a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/static-a.cpp
rename to lldb/test/API/lang/cpp/overloaded-functions/static-a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/static-b.cpp b/lldb/test/API/lang/cpp/overloaded-functions/static-b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/overloaded-functions/static-b.cpp
rename to lldb/test/API/lang/cpp/overloaded-functions/static-b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/printf/TestPrintf.py b/lldb/test/API/lang/cpp/printf/TestPrintf.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/printf/TestPrintf.py
rename to lldb/test/API/lang/cpp/printf/TestPrintf.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/printf/main.cpp b/lldb/test/API/lang/cpp/printf/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/printf/main.cpp
rename to lldb/test/API/lang/cpp/printf/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/Makefile b/lldb/test/API/lang/cpp/rvalue-references/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/Makefile
rename to lldb/test/API/lang/cpp/rvalue-references/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/TestRvalueReferences.py b/lldb/test/API/lang/cpp/rvalue-references/TestRvalueReferences.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/TestRvalueReferences.py
rename to lldb/test/API/lang/cpp/rvalue-references/TestRvalueReferences.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/main.cpp b/lldb/test/API/lang/cpp/rvalue-references/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/main.cpp
rename to lldb/test/API/lang/cpp/rvalue-references/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/scope/Makefile b/lldb/test/API/lang/cpp/scope/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/scope/Makefile
rename to lldb/test/API/lang/cpp/scope/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/scope/TestCppScope.py b/lldb/test/API/lang/cpp/scope/TestCppScope.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/scope/TestCppScope.py
rename to lldb/test/API/lang/cpp/scope/TestCppScope.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/scope/main.cpp b/lldb/test/API/lang/cpp/scope/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/scope/main.cpp
rename to lldb/test/API/lang/cpp/scope/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/signed_types/Makefile b/lldb/test/API/lang/cpp/signed_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/signed_types/Makefile
rename to lldb/test/API/lang/cpp/signed_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/signed_types/TestSignedTypes.py b/lldb/test/API/lang/cpp/signed_types/TestSignedTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/signed_types/TestSignedTypes.py
rename to lldb/test/API/lang/cpp/signed_types/TestSignedTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/signed_types/main.cpp b/lldb/test/API/lang/cpp/signed_types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/signed_types/main.cpp
rename to lldb/test/API/lang/cpp/signed_types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/static_members/Makefile b/lldb/test/API/lang/cpp/static_members/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/static_members/Makefile
rename to lldb/test/API/lang/cpp/static_members/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/static_members/TestCPPStaticMembers.py b/lldb/test/API/lang/cpp/static_members/TestCPPStaticMembers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/static_members/TestCPPStaticMembers.py
rename to lldb/test/API/lang/cpp/static_members/TestCPPStaticMembers.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/static_members/main.cpp b/lldb/test/API/lang/cpp/static_members/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/static_members/main.cpp
rename to lldb/test/API/lang/cpp/static_members/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/static_methods/Makefile b/lldb/test/API/lang/cpp/static_methods/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/static_methods/Makefile
rename to lldb/test/API/lang/cpp/static_methods/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/static_methods/TestCPPStaticMethods.py b/lldb/test/API/lang/cpp/static_methods/TestCPPStaticMethods.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/static_methods/TestCPPStaticMethods.py
rename to lldb/test/API/lang/cpp/static_methods/TestCPPStaticMethods.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/static_methods/main.cpp b/lldb/test/API/lang/cpp/static_methods/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/static_methods/main.cpp
rename to lldb/test/API/lang/cpp/static_methods/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/Makefile b/lldb/test/API/lang/cpp/std-function-step-into-callable/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/Makefile
rename to lldb/test/API/lang/cpp/std-function-step-into-callable/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/TestStdFunctionStepIntoCallable.py b/lldb/test/API/lang/cpp/std-function-step-into-callable/TestStdFunctionStepIntoCallable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/TestStdFunctionStepIntoCallable.py
rename to lldb/test/API/lang/cpp/std-function-step-into-callable/TestStdFunctionStepIntoCallable.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/main.cpp b/lldb/test/API/lang/cpp/std-function-step-into-callable/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/main.cpp
rename to lldb/test/API/lang/cpp/std-function-step-into-callable/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/stl/Makefile b/lldb/test/API/lang/cpp/stl/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/stl/Makefile
rename to lldb/test/API/lang/cpp/stl/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/stl/TestSTL.py b/lldb/test/API/lang/cpp/stl/TestSTL.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/stl/TestSTL.py
rename to lldb/test/API/lang/cpp/stl/TestSTL.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/stl/TestStdCXXDisassembly.py b/lldb/test/API/lang/cpp/stl/TestStdCXXDisassembly.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/stl/TestStdCXXDisassembly.py
rename to lldb/test/API/lang/cpp/stl/TestStdCXXDisassembly.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/stl/cmds.txt b/lldb/test/API/lang/cpp/stl/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/stl/cmds.txt
rename to lldb/test/API/lang/cpp/stl/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/stl/main.cpp b/lldb/test/API/lang/cpp/stl/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/stl/main.cpp
rename to lldb/test/API/lang/cpp/stl/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/symbols/TestSymbols.py b/lldb/test/API/lang/cpp/symbols/TestSymbols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/symbols/TestSymbols.py
rename to lldb/test/API/lang/cpp/symbols/TestSymbols.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/symbols/main.cpp b/lldb/test/API/lang/cpp/symbols/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/symbols/main.cpp
rename to lldb/test/API/lang/cpp/symbols/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/template-function/Makefile b/lldb/test/API/lang/cpp/template-function/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/template-function/Makefile
rename to lldb/test/API/lang/cpp/template-function/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/template-function/TestTemplateFunctions.py b/lldb/test/API/lang/cpp/template-function/TestTemplateFunctions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/template-function/TestTemplateFunctions.py
rename to lldb/test/API/lang/cpp/template-function/TestTemplateFunctions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/template-function/main.cpp b/lldb/test/API/lang/cpp/template-function/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/template-function/main.cpp
rename to lldb/test/API/lang/cpp/template-function/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/template/Makefile b/lldb/test/API/lang/cpp/template/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/template/Makefile
rename to lldb/test/API/lang/cpp/template/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/template/TestTemplateArgs.py b/lldb/test/API/lang/cpp/template/TestTemplateArgs.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/template/TestTemplateArgs.py
rename to lldb/test/API/lang/cpp/template/TestTemplateArgs.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/template/main.cpp b/lldb/test/API/lang/cpp/template/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/template/main.cpp
rename to lldb/test/API/lang/cpp/template/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/this/Makefile b/lldb/test/API/lang/cpp/this/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/this/Makefile
rename to lldb/test/API/lang/cpp/this/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/this/TestCPPThis.py b/lldb/test/API/lang/cpp/this/TestCPPThis.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/this/TestCPPThis.py
rename to lldb/test/API/lang/cpp/this/TestCPPThis.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/this/main.cpp b/lldb/test/API/lang/cpp/this/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/this/main.cpp
rename to lldb/test/API/lang/cpp/this/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/Makefile b/lldb/test/API/lang/cpp/thread_local/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/Makefile
rename to lldb/test/API/lang/cpp/thread_local/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/TestThreadLocal.py b/lldb/test/API/lang/cpp/thread_local/TestThreadLocal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/TestThreadLocal.py
rename to lldb/test/API/lang/cpp/thread_local/TestThreadLocal.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/main.cpp b/lldb/test/API/lang/cpp/thread_local/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/main.cpp
rename to lldb/test/API/lang/cpp/thread_local/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/trivial_abi/Makefile b/lldb/test/API/lang/cpp/trivial_abi/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/trivial_abi/Makefile
rename to lldb/test/API/lang/cpp/trivial_abi/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/trivial_abi/TestTrivialABI.py b/lldb/test/API/lang/cpp/trivial_abi/TestTrivialABI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/trivial_abi/TestTrivialABI.py
rename to lldb/test/API/lang/cpp/trivial_abi/TestTrivialABI.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/trivial_abi/main.cpp b/lldb/test/API/lang/cpp/trivial_abi/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/trivial_abi/main.cpp
rename to lldb/test/API/lang/cpp/trivial_abi/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/type_lookup/Makefile b/lldb/test/API/lang/cpp/type_lookup/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/type_lookup/Makefile
rename to lldb/test/API/lang/cpp/type_lookup/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/type_lookup/TestCppTypeLookup.py b/lldb/test/API/lang/cpp/type_lookup/TestCppTypeLookup.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/type_lookup/TestCppTypeLookup.py
rename to lldb/test/API/lang/cpp/type_lookup/TestCppTypeLookup.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/type_lookup/main.cpp b/lldb/test/API/lang/cpp/type_lookup/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/type_lookup/main.cpp
rename to lldb/test/API/lang/cpp/type_lookup/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unicode-literals/Makefile b/lldb/test/API/lang/cpp/unicode-literals/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unicode-literals/Makefile
rename to lldb/test/API/lang/cpp/unicode-literals/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unicode-literals/TestUnicodeLiterals.py b/lldb/test/API/lang/cpp/unicode-literals/TestUnicodeLiterals.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unicode-literals/TestUnicodeLiterals.py
rename to lldb/test/API/lang/cpp/unicode-literals/TestUnicodeLiterals.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unicode-literals/main.cpp b/lldb/test/API/lang/cpp/unicode-literals/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unicode-literals/main.cpp
rename to lldb/test/API/lang/cpp/unicode-literals/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unique-types/Makefile b/lldb/test/API/lang/cpp/unique-types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unique-types/Makefile
rename to lldb/test/API/lang/cpp/unique-types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unique-types/TestUniqueTypes.py b/lldb/test/API/lang/cpp/unique-types/TestUniqueTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unique-types/TestUniqueTypes.py
rename to lldb/test/API/lang/cpp/unique-types/TestUniqueTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unique-types/main.cpp b/lldb/test/API/lang/cpp/unique-types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unique-types/main.cpp
rename to lldb/test/API/lang/cpp/unique-types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unsigned_types/Makefile b/lldb/test/API/lang/cpp/unsigned_types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unsigned_types/Makefile
rename to lldb/test/API/lang/cpp/unsigned_types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unsigned_types/TestUnsignedTypes.py b/lldb/test/API/lang/cpp/unsigned_types/TestUnsignedTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unsigned_types/TestUnsignedTypes.py
rename to lldb/test/API/lang/cpp/unsigned_types/TestUnsignedTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/unsigned_types/main.cpp b/lldb/test/API/lang/cpp/unsigned_types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/unsigned_types/main.cpp
rename to lldb/test/API/lang/cpp/unsigned_types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-functions/Makefile b/lldb/test/API/lang/cpp/virtual-functions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-functions/Makefile
rename to lldb/test/API/lang/cpp/virtual-functions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-functions/TestCppVirtualFunctions.py b/lldb/test/API/lang/cpp/virtual-functions/TestCppVirtualFunctions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-functions/TestCppVirtualFunctions.py
rename to lldb/test/API/lang/cpp/virtual-functions/TestCppVirtualFunctions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-functions/main.cpp b/lldb/test/API/lang/cpp/virtual-functions/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-functions/main.cpp
rename to lldb/test/API/lang/cpp/virtual-functions/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-overload/TestVirtualOverload.py b/lldb/test/API/lang/cpp/virtual-overload/TestVirtualOverload.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-overload/TestVirtualOverload.py
rename to lldb/test/API/lang/cpp/virtual-overload/TestVirtualOverload.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-overload/main.cpp b/lldb/test/API/lang/cpp/virtual-overload/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/virtual-overload/main.cpp
rename to lldb/test/API/lang/cpp/virtual-overload/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/.categories b/lldb/test/API/lang/cpp/wchar_t/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/.categories
rename to lldb/test/API/lang/cpp/wchar_t/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/Makefile b/lldb/test/API/lang/cpp/wchar_t/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/Makefile
rename to lldb/test/API/lang/cpp/wchar_t/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/TestCxxWCharT.py b/lldb/test/API/lang/cpp/wchar_t/TestCxxWCharT.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/TestCxxWCharT.py
rename to lldb/test/API/lang/cpp/wchar_t/TestCxxWCharT.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/main.cpp b/lldb/test/API/lang/cpp/wchar_t/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/cpp/wchar_t/main.cpp
rename to lldb/test/API/lang/cpp/wchar_t/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/mixed/Makefile b/lldb/test/API/lang/mixed/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/mixed/Makefile
rename to lldb/test/API/lang/mixed/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/mixed/TestMixedLanguages.py b/lldb/test/API/lang/mixed/TestMixedLanguages.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/mixed/TestMixedLanguages.py
rename to lldb/test/API/lang/mixed/TestMixedLanguages.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/mixed/foo.cpp b/lldb/test/API/lang/mixed/foo.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/mixed/foo.cpp
rename to lldb/test/API/lang/mixed/foo.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/mixed/main.c b/lldb/test/API/lang/mixed/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/mixed/main.c
rename to lldb/test/API/lang/mixed/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/.categories b/lldb/test/API/lang/objc/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/.categories
rename to lldb/test/API/lang/objc/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/bitfield_ivars/TestBitfieldIvars.py b/lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/bitfield_ivars/TestBitfieldIvars.py
rename to lldb/test/API/lang/objc/bitfield_ivars/TestBitfieldIvars.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/bitfield_ivars/main.m b/lldb/test/API/lang/objc/bitfield_ivars/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/bitfield_ivars/main.m
rename to lldb/test/API/lang/objc/bitfield_ivars/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/Makefile b/lldb/test/API/lang/objc/blocks/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/blocks/Makefile
rename to lldb/test/API/lang/objc/blocks/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/TestObjCIvarsInBlocks.py b/lldb/test/API/lang/objc/blocks/TestObjCIvarsInBlocks.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/blocks/TestObjCIvarsInBlocks.py
rename to lldb/test/API/lang/objc/blocks/TestObjCIvarsInBlocks.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/ivars-in-blocks.h b/lldb/test/API/lang/objc/blocks/ivars-in-blocks.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/blocks/ivars-in-blocks.h
rename to lldb/test/API/lang/objc/blocks/ivars-in-blocks.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/ivars-in-blocks.m b/lldb/test/API/lang/objc/blocks/ivars-in-blocks.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/blocks/ivars-in-blocks.m
rename to lldb/test/API/lang/objc/blocks/ivars-in-blocks.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/main.m b/lldb/test/API/lang/objc/blocks/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/blocks/main.m
rename to lldb/test/API/lang/objc/blocks/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Makefile b/lldb/test/API/lang/objc/conflicting-definition/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Makefile
rename to lldb/test/API/lang/objc/conflicting-definition/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Test/Foo.h b/lldb/test/API/lang/objc/conflicting-definition/Test/Foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Test/Foo.h
rename to lldb/test/API/lang/objc/conflicting-definition/Test/Foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Test/Test.h b/lldb/test/API/lang/objc/conflicting-definition/Test/Test.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Test/Test.h
rename to lldb/test/API/lang/objc/conflicting-definition/Test/Test.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Test/Test.m b/lldb/test/API/lang/objc/conflicting-definition/Test/Test.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/Test/Test.m
rename to lldb/test/API/lang/objc/conflicting-definition/Test/Test.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestConflictingDefinition.py b/lldb/test/API/lang/objc/conflicting-definition/TestConflictingDefinition.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestConflictingDefinition.py
rename to lldb/test/API/lang/objc/conflicting-definition/TestConflictingDefinition.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestExt/Foo.h b/lldb/test/API/lang/objc/conflicting-definition/TestExt/Foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestExt/Foo.h
rename to lldb/test/API/lang/objc/conflicting-definition/TestExt/Foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestExt/TestExt.h b/lldb/test/API/lang/objc/conflicting-definition/TestExt/TestExt.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestExt/TestExt.h
rename to lldb/test/API/lang/objc/conflicting-definition/TestExt/TestExt.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestExt/TestExt.m b/lldb/test/API/lang/objc/conflicting-definition/TestExt/TestExt.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/TestExt/TestExt.m
rename to lldb/test/API/lang/objc/conflicting-definition/TestExt/TestExt.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/main.m b/lldb/test/API/lang/objc/conflicting-definition/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/conflicting-definition/main.m
rename to lldb/test/API/lang/objc/conflicting-definition/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/direct-dispatch-step/Makefile b/lldb/test/API/lang/objc/direct-dispatch-step/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/direct-dispatch-step/Makefile
rename to lldb/test/API/lang/objc/direct-dispatch-step/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/direct-dispatch-step/TestObjCDirectDispatchStepping.py b/lldb/test/API/lang/objc/direct-dispatch-step/TestObjCDirectDispatchStepping.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/direct-dispatch-step/TestObjCDirectDispatchStepping.py
rename to lldb/test/API/lang/objc/direct-dispatch-step/TestObjCDirectDispatchStepping.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/direct-dispatch-step/stepping-tests.m b/lldb/test/API/lang/objc/direct-dispatch-step/stepping-tests.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/direct-dispatch-step/stepping-tests.m
rename to lldb/test/API/lang/objc/direct-dispatch-step/stepping-tests.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/Makefile b/lldb/test/API/lang/objc/exceptions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/Makefile
rename to lldb/test/API/lang/objc/exceptions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/TestObjCExceptions.py b/lldb/test/API/lang/objc/exceptions/TestObjCExceptions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/TestObjCExceptions.py
rename to lldb/test/API/lang/objc/exceptions/TestObjCExceptions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/main.mm b/lldb/test/API/lang/objc/exceptions/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/main.mm
rename to lldb/test/API/lang/objc/exceptions/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Container.h b/lldb/test/API/lang/objc/forward-decl/Container.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Container.h
rename to lldb/test/API/lang/objc/forward-decl/Container.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Container.m b/lldb/test/API/lang/objc/forward-decl/Container.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Container.m
rename to lldb/test/API/lang/objc/forward-decl/Container.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Makefile b/lldb/test/API/lang/objc/forward-decl/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Makefile
rename to lldb/test/API/lang/objc/forward-decl/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/TestForwardDecl.py b/lldb/test/API/lang/objc/forward-decl/TestForwardDecl.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/TestForwardDecl.py
rename to lldb/test/API/lang/objc/forward-decl/TestForwardDecl.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/main.m b/lldb/test/API/lang/objc/forward-decl/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/main.m
rename to lldb/test/API/lang/objc/forward-decl/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/Makefile b/lldb/test/API/lang/objc/foundation/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/Makefile
rename to lldb/test/API/lang/objc/foundation/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestConstStrings.py b/lldb/test/API/lang/objc/foundation/TestConstStrings.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestConstStrings.py
rename to lldb/test/API/lang/objc/foundation/TestConstStrings.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestFoundationDisassembly.py b/lldb/test/API/lang/objc/foundation/TestFoundationDisassembly.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestFoundationDisassembly.py
rename to lldb/test/API/lang/objc/foundation/TestFoundationDisassembly.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods.py b/lldb/test/API/lang/objc/foundation/TestObjCMethods.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods.py
rename to lldb/test/API/lang/objc/foundation/TestObjCMethods.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods2.py b/lldb/test/API/lang/objc/foundation/TestObjCMethods2.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods2.py
rename to lldb/test/API/lang/objc/foundation/TestObjCMethods2.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethodsNSArray.py b/lldb/test/API/lang/objc/foundation/TestObjCMethodsNSArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethodsNSArray.py
rename to lldb/test/API/lang/objc/foundation/TestObjCMethodsNSArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethodsNSError.py b/lldb/test/API/lang/objc/foundation/TestObjCMethodsNSError.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethodsNSError.py
rename to lldb/test/API/lang/objc/foundation/TestObjCMethodsNSError.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethodsString.py b/lldb/test/API/lang/objc/foundation/TestObjCMethodsString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethodsString.py
rename to lldb/test/API/lang/objc/foundation/TestObjCMethodsString.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjectDescriptionAPI.py b/lldb/test/API/lang/objc/foundation/TestObjectDescriptionAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjectDescriptionAPI.py
rename to lldb/test/API/lang/objc/foundation/TestObjectDescriptionAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestRuntimeTypes.py b/lldb/test/API/lang/objc/foundation/TestRuntimeTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestRuntimeTypes.py
rename to lldb/test/API/lang/objc/foundation/TestRuntimeTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestSymbolTable.py b/lldb/test/API/lang/objc/foundation/TestSymbolTable.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestSymbolTable.py
rename to lldb/test/API/lang/objc/foundation/TestSymbolTable.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/const-strings.m b/lldb/test/API/lang/objc/foundation/const-strings.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/const-strings.m
rename to lldb/test/API/lang/objc/foundation/const-strings.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/main.m b/lldb/test/API/lang/objc/foundation/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/main.m
rename to lldb/test/API/lang/objc/foundation/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/my-base.h b/lldb/test/API/lang/objc/foundation/my-base.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/my-base.h
rename to lldb/test/API/lang/objc/foundation/my-base.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/my-base.m b/lldb/test/API/lang/objc/foundation/my-base.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/foundation/my-base.m
rename to lldb/test/API/lang/objc/foundation/my-base.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/Makefile b/lldb/test/API/lang/objc/global_ptrs/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/Makefile
rename to lldb/test/API/lang/objc/global_ptrs/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/TestGlobalObjects.py b/lldb/test/API/lang/objc/global_ptrs/TestGlobalObjects.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/TestGlobalObjects.py
rename to lldb/test/API/lang/objc/global_ptrs/TestGlobalObjects.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/main.m b/lldb/test/API/lang/objc/global_ptrs/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/main.m
rename to lldb/test/API/lang/objc/global_ptrs/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/InternalDefiner.h b/lldb/test/API/lang/objc/hidden-ivars/InternalDefiner.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/InternalDefiner.h
rename to lldb/test/API/lang/objc/hidden-ivars/InternalDefiner.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/InternalDefiner.m b/lldb/test/API/lang/objc/hidden-ivars/InternalDefiner.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/InternalDefiner.m
rename to lldb/test/API/lang/objc/hidden-ivars/InternalDefiner.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/Makefile b/lldb/test/API/lang/objc/hidden-ivars/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/Makefile
rename to lldb/test/API/lang/objc/hidden-ivars/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/TestHiddenIvars.py b/lldb/test/API/lang/objc/hidden-ivars/TestHiddenIvars.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/TestHiddenIvars.py
rename to lldb/test/API/lang/objc/hidden-ivars/TestHiddenIvars.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/main.m b/lldb/test/API/lang/objc/hidden-ivars/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/hidden-ivars/main.m
rename to lldb/test/API/lang/objc/hidden-ivars/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/Makefile b/lldb/test/API/lang/objc/ivar-IMP/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/Makefile
rename to lldb/test/API/lang/objc/ivar-IMP/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/TestObjCiVarIMP.py b/lldb/test/API/lang/objc/ivar-IMP/TestObjCiVarIMP.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/TestObjCiVarIMP.py
rename to lldb/test/API/lang/objc/ivar-IMP/TestObjCiVarIMP.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/myclass.h b/lldb/test/API/lang/objc/ivar-IMP/myclass.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/myclass.h
rename to lldb/test/API/lang/objc/ivar-IMP/myclass.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/myclass.m b/lldb/test/API/lang/objc/ivar-IMP/myclass.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/myclass.m
rename to lldb/test/API/lang/objc/ivar-IMP/myclass.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/repro.m b/lldb/test/API/lang/objc/ivar-IMP/repro.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/repro.m
rename to lldb/test/API/lang/objc/ivar-IMP/repro.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/Makefile b/lldb/test/API/lang/objc/modules-app-update/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/Makefile
rename to lldb/test/API/lang/objc/modules-app-update/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/TestClangModulesAppUpdate.py b/lldb/test/API/lang/objc/modules-app-update/TestClangModulesAppUpdate.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/TestClangModulesAppUpdate.py
rename to lldb/test/API/lang/objc/modules-app-update/TestClangModulesAppUpdate.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/foo.m b/lldb/test/API/lang/objc/modules-app-update/foo.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/foo.m
rename to lldb/test/API/lang/objc/modules-app-update/foo.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/main.m b/lldb/test/API/lang/objc/modules-app-update/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/main.m
rename to lldb/test/API/lang/objc/modules-app-update/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/module.modulemap b/lldb/test/API/lang/objc/modules-app-update/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/module.modulemap
rename to lldb/test/API/lang/objc/modules-app-update/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/umbrella.h b/lldb/test/API/lang/objc/modules-app-update/umbrella.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-app-update/umbrella.h
rename to lldb/test/API/lang/objc/modules-app-update/umbrella.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/Makefile b/lldb/test/API/lang/objc/modules-auto-import/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/Makefile
rename to lldb/test/API/lang/objc/modules-auto-import/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/TestModulesAutoImport.py b/lldb/test/API/lang/objc/modules-auto-import/TestModulesAutoImport.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/TestModulesAutoImport.py
rename to lldb/test/API/lang/objc/modules-auto-import/TestModulesAutoImport.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/main.m b/lldb/test/API/lang/objc/modules-auto-import/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/main.m
rename to lldb/test/API/lang/objc/modules-auto-import/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/Makefile b/lldb/test/API/lang/objc/modules-cache/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/Makefile
rename to lldb/test/API/lang/objc/modules-cache/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/TestClangModulesCache.py b/lldb/test/API/lang/objc/modules-cache/TestClangModulesCache.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/TestClangModulesCache.py
rename to lldb/test/API/lang/objc/modules-cache/TestClangModulesCache.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/f.h b/lldb/test/API/lang/objc/modules-cache/f.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/f.h
rename to lldb/test/API/lang/objc/modules-cache/f.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/main.m b/lldb/test/API/lang/objc/modules-cache/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/main.m
rename to lldb/test/API/lang/objc/modules-cache/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/module.modulemap b/lldb/test/API/lang/objc/modules-cache/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-cache/module.modulemap
rename to lldb/test/API/lang/objc/modules-cache/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/Makefile b/lldb/test/API/lang/objc/modules-hash-mismatch/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/Makefile
rename to lldb/test/API/lang/objc/modules-hash-mismatch/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/TestClangModulesHashMismatch.py b/lldb/test/API/lang/objc/modules-hash-mismatch/TestClangModulesHashMismatch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/TestClangModulesHashMismatch.py
rename to lldb/test/API/lang/objc/modules-hash-mismatch/TestClangModulesHashMismatch.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/main.m b/lldb/test/API/lang/objc/modules-hash-mismatch/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/main.m
rename to lldb/test/API/lang/objc/modules-hash-mismatch/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/other.m b/lldb/test/API/lang/objc/modules-hash-mismatch/other.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-hash-mismatch/other.m
rename to lldb/test/API/lang/objc/modules-hash-mismatch/other.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/Makefile b/lldb/test/API/lang/objc/modules-incomplete/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/Makefile
rename to lldb/test/API/lang/objc/modules-incomplete/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/TestIncompleteModules.py b/lldb/test/API/lang/objc/modules-incomplete/TestIncompleteModules.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/TestIncompleteModules.py
rename to lldb/test/API/lang/objc/modules-incomplete/TestIncompleteModules.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/main.m b/lldb/test/API/lang/objc/modules-incomplete/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/main.m
rename to lldb/test/API/lang/objc/modules-incomplete/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/minmax.h b/lldb/test/API/lang/objc/modules-incomplete/minmax.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/minmax.h
rename to lldb/test/API/lang/objc/modules-incomplete/minmax.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/module.map b/lldb/test/API/lang/objc/modules-incomplete/module.map
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/module.map
rename to lldb/test/API/lang/objc/modules-incomplete/module.map
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/myModule.h b/lldb/test/API/lang/objc/modules-incomplete/myModule.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/myModule.h
rename to lldb/test/API/lang/objc/modules-incomplete/myModule.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/myModule.m b/lldb/test/API/lang/objc/modules-incomplete/myModule.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/myModule.m
rename to lldb/test/API/lang/objc/modules-incomplete/myModule.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/Makefile b/lldb/test/API/lang/objc/modules-inline-functions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/Makefile
rename to lldb/test/API/lang/objc/modules-inline-functions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py b/lldb/test/API/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py
rename to lldb/test/API/lang/objc/modules-inline-functions/TestModulesInlineFunctions.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/main.m b/lldb/test/API/lang/objc/modules-inline-functions/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/main.m
rename to lldb/test/API/lang/objc/modules-inline-functions/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/module.map b/lldb/test/API/lang/objc/modules-inline-functions/module.map
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/module.map
rename to lldb/test/API/lang/objc/modules-inline-functions/module.map
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/myModule.c b/lldb/test/API/lang/objc/modules-inline-functions/myModule.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/myModule.c
rename to lldb/test/API/lang/objc/modules-inline-functions/myModule.c
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/myModule.h b/lldb/test/API/lang/objc/modules-inline-functions/myModule.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/myModule.h
rename to lldb/test/API/lang/objc/modules-inline-functions/myModule.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/Makefile b/lldb/test/API/lang/objc/modules-update/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/Makefile
rename to lldb/test/API/lang/objc/modules-update/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/TestClangModulesUpdate.py b/lldb/test/API/lang/objc/modules-update/TestClangModulesUpdate.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/TestClangModulesUpdate.py
rename to lldb/test/API/lang/objc/modules-update/TestClangModulesUpdate.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/first.m b/lldb/test/API/lang/objc/modules-update/first.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/first.m
rename to lldb/test/API/lang/objc/modules-update/first.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/module.modulemap b/lldb/test/API/lang/objc/modules-update/module.modulemap
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/module.modulemap
rename to lldb/test/API/lang/objc/modules-update/module.modulemap
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/second.m b/lldb/test/API/lang/objc/modules-update/second.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/second.m
rename to lldb/test/API/lang/objc/modules-update/second.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/umbrella.h b/lldb/test/API/lang/objc/modules-update/umbrella.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules-update/umbrella.h
rename to lldb/test/API/lang/objc/modules-update/umbrella.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules/Makefile b/lldb/test/API/lang/objc/modules/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules/Makefile
rename to lldb/test/API/lang/objc/modules/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules/TestObjCModules.py b/lldb/test/API/lang/objc/modules/TestObjCModules.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules/TestObjCModules.py
rename to lldb/test/API/lang/objc/modules/TestObjCModules.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules/main.m b/lldb/test/API/lang/objc/modules/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/modules/main.m
rename to lldb/test/API/lang/objc/modules/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc++/Makefile b/lldb/test/API/lang/objc/objc++/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc++/Makefile
rename to lldb/test/API/lang/objc/objc++/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc++/TestObjCXX.py b/lldb/test/API/lang/objc/objc++/TestObjCXX.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc++/TestObjCXX.py
rename to lldb/test/API/lang/objc/objc++/TestObjCXX.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc++/main.mm b/lldb/test/API/lang/objc/objc++/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc++/main.mm
rename to lldb/test/API/lang/objc/objc++/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-baseclass-sbtype/Makefile b/lldb/test/API/lang/objc/objc-baseclass-sbtype/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-baseclass-sbtype/Makefile
rename to lldb/test/API/lang/objc/objc-baseclass-sbtype/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-baseclass-sbtype/TestObjCBaseClassSBType.py b/lldb/test/API/lang/objc/objc-baseclass-sbtype/TestObjCBaseClassSBType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-baseclass-sbtype/TestObjCBaseClassSBType.py
rename to lldb/test/API/lang/objc/objc-baseclass-sbtype/TestObjCBaseClassSBType.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-baseclass-sbtype/main.m b/lldb/test/API/lang/objc/objc-baseclass-sbtype/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-baseclass-sbtype/main.m
rename to lldb/test/API/lang/objc/objc-baseclass-sbtype/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-builtin-types/Makefile b/lldb/test/API/lang/objc/objc-builtin-types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-builtin-types/Makefile
rename to lldb/test/API/lang/objc/objc-builtin-types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-builtin-types/TestObjCBuiltinTypes.py b/lldb/test/API/lang/objc/objc-builtin-types/TestObjCBuiltinTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-builtin-types/TestObjCBuiltinTypes.py
rename to lldb/test/API/lang/objc/objc-builtin-types/TestObjCBuiltinTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-builtin-types/main.cpp b/lldb/test/API/lang/objc/objc-builtin-types/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-builtin-types/main.cpp
rename to lldb/test/API/lang/objc/objc-builtin-types/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/Makefile b/lldb/test/API/lang/objc/objc-checker/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/Makefile
rename to lldb/test/API/lang/objc/objc-checker/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/TestObjCCheckers.py b/lldb/test/API/lang/objc/objc-checker/TestObjCCheckers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/TestObjCCheckers.py
rename to lldb/test/API/lang/objc/objc-checker/TestObjCCheckers.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/main.m b/lldb/test/API/lang/objc/objc-checker/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/main.m
rename to lldb/test/API/lang/objc/objc-checker/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/Makefile b/lldb/test/API/lang/objc/objc-class-method/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/Makefile
rename to lldb/test/API/lang/objc/objc-class-method/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/TestObjCClassMethod.py b/lldb/test/API/lang/objc/objc-class-method/TestObjCClassMethod.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/TestObjCClassMethod.py
rename to lldb/test/API/lang/objc/objc-class-method/TestObjCClassMethod.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/class.m b/lldb/test/API/lang/objc/objc-class-method/class.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/class.m
rename to lldb/test/API/lang/objc/objc-class-method/class.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/.categories b/lldb/test/API/lang/objc/objc-dyn-sbtype/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/.categories
rename to lldb/test/API/lang/objc/objc-dyn-sbtype/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/Makefile b/lldb/test/API/lang/objc/objc-dyn-sbtype/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/Makefile
rename to lldb/test/API/lang/objc/objc-dyn-sbtype/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/TestObjCDynamicSBType.py b/lldb/test/API/lang/objc/objc-dyn-sbtype/TestObjCDynamicSBType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/TestObjCDynamicSBType.py
rename to lldb/test/API/lang/objc/objc-dyn-sbtype/TestObjCDynamicSBType.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/main.m b/lldb/test/API/lang/objc/objc-dyn-sbtype/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dyn-sbtype/main.m
rename to lldb/test/API/lang/objc/objc-dyn-sbtype/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/Makefile b/lldb/test/API/lang/objc/objc-dynamic-value/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/Makefile
rename to lldb/test/API/lang/objc/objc-dynamic-value/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/TestObjCDynamicValue.py b/lldb/test/API/lang/objc/objc-dynamic-value/TestObjCDynamicValue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/TestObjCDynamicValue.py
rename to lldb/test/API/lang/objc/objc-dynamic-value/TestObjCDynamicValue.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/dynamic-value.m b/lldb/test/API/lang/objc/objc-dynamic-value/dynamic-value.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/dynamic-value.m
rename to lldb/test/API/lang/objc/objc-dynamic-value/dynamic-value.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-foundation-dictionary-empty/TestNSDictionary0.py b/lldb/test/API/lang/objc/objc-foundation-dictionary-empty/TestNSDictionary0.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-foundation-dictionary-empty/TestNSDictionary0.py
rename to lldb/test/API/lang/objc/objc-foundation-dictionary-empty/TestNSDictionary0.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-foundation-dictionary-empty/main.m b/lldb/test/API/lang/objc/objc-foundation-dictionary-empty/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-foundation-dictionary-empty/main.m
rename to lldb/test/API/lang/objc/objc-foundation-dictionary-empty/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/Makefile b/lldb/test/API/lang/objc/objc-ivar-offsets/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/Makefile
rename to lldb/test/API/lang/objc/objc-ivar-offsets/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/TestObjCIvarOffsets.py b/lldb/test/API/lang/objc/objc-ivar-offsets/TestObjCIvarOffsets.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/TestObjCIvarOffsets.py
rename to lldb/test/API/lang/objc/objc-ivar-offsets/TestObjCIvarOffsets.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/main.m b/lldb/test/API/lang/objc/objc-ivar-offsets/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/main.m
rename to lldb/test/API/lang/objc/objc-ivar-offsets/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/objc-ivar-offsets.h b/lldb/test/API/lang/objc/objc-ivar-offsets/objc-ivar-offsets.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/objc-ivar-offsets.h
rename to lldb/test/API/lang/objc/objc-ivar-offsets/objc-ivar-offsets.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/objc-ivar-offsets.m b/lldb/test/API/lang/objc/objc-ivar-offsets/objc-ivar-offsets.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/objc-ivar-offsets.m
rename to lldb/test/API/lang/objc/objc-ivar-offsets/objc-ivar-offsets.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-protocols/TestIvarProtocols.py b/lldb/test/API/lang/objc/objc-ivar-protocols/TestIvarProtocols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-protocols/TestIvarProtocols.py
rename to lldb/test/API/lang/objc/objc-ivar-protocols/TestIvarProtocols.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-protocols/main.m b/lldb/test/API/lang/objc/objc-ivar-protocols/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-protocols/main.m
rename to lldb/test/API/lang/objc/objc-ivar-protocols/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/Makefile b/lldb/test/API/lang/objc/objc-ivar-stripped/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/Makefile
rename to lldb/test/API/lang/objc/objc-ivar-stripped/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/TestObjCIvarStripped.py b/lldb/test/API/lang/objc/objc-ivar-stripped/TestObjCIvarStripped.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/TestObjCIvarStripped.py
rename to lldb/test/API/lang/objc/objc-ivar-stripped/TestObjCIvarStripped.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/main.m b/lldb/test/API/lang/objc/objc-ivar-stripped/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/main.m
rename to lldb/test/API/lang/objc/objc-ivar-stripped/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/Makefile b/lldb/test/API/lang/objc/objc-new-syntax/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/Makefile
rename to lldb/test/API/lang/objc/objc-new-syntax/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/ObjCNewSyntaxTest.py b/lldb/test/API/lang/objc/objc-new-syntax/ObjCNewSyntaxTest.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/ObjCNewSyntaxTest.py
rename to lldb/test/API/lang/objc/objc-new-syntax/ObjCNewSyntaxTest.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/TestObjCNewSyntaxArray.py b/lldb/test/API/lang/objc/objc-new-syntax/TestObjCNewSyntaxArray.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/TestObjCNewSyntaxArray.py
rename to lldb/test/API/lang/objc/objc-new-syntax/TestObjCNewSyntaxArray.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/TestObjCNewSyntaxDictionary.py b/lldb/test/API/lang/objc/objc-new-syntax/TestObjCNewSyntaxDictionary.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/TestObjCNewSyntaxDictionary.py
rename to lldb/test/API/lang/objc/objc-new-syntax/TestObjCNewSyntaxDictionary.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/TestObjCNewSyntaxLiteral.py b/lldb/test/API/lang/objc/objc-new-syntax/TestObjCNewSyntaxLiteral.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/TestObjCNewSyntaxLiteral.py
rename to lldb/test/API/lang/objc/objc-new-syntax/TestObjCNewSyntaxLiteral.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/main.m b/lldb/test/API/lang/objc/objc-new-syntax/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/main.m
rename to lldb/test/API/lang/objc/objc-new-syntax/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/Makefile b/lldb/test/API/lang/objc/objc-optimized/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/Makefile
rename to lldb/test/API/lang/objc/objc-optimized/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/TestObjcOptimized.py b/lldb/test/API/lang/objc/objc-optimized/TestObjcOptimized.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/TestObjcOptimized.py
rename to lldb/test/API/lang/objc/objc-optimized/TestObjcOptimized.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/main.m b/lldb/test/API/lang/objc/objc-optimized/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/main.m
rename to lldb/test/API/lang/objc/objc-optimized/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/Makefile b/lldb/test/API/lang/objc/objc-property/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/Makefile
rename to lldb/test/API/lang/objc/objc-property/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/TestObjCProperty.py b/lldb/test/API/lang/objc/objc-property/TestObjCProperty.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/TestObjCProperty.py
rename to lldb/test/API/lang/objc/objc-property/TestObjCProperty.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/main.m b/lldb/test/API/lang/objc/objc-property/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/main.m
rename to lldb/test/API/lang/objc/objc-property/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/Makefile b/lldb/test/API/lang/objc/objc-runtime-ivars/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/Makefile
rename to lldb/test/API/lang/objc/objc-runtime-ivars/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/TestRuntimeIvars.py b/lldb/test/API/lang/objc/objc-runtime-ivars/TestRuntimeIvars.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/TestRuntimeIvars.py
rename to lldb/test/API/lang/objc/objc-runtime-ivars/TestRuntimeIvars.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/main.m b/lldb/test/API/lang/objc/objc-runtime-ivars/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/main.m
rename to lldb/test/API/lang/objc/objc-runtime-ivars/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/Makefile b/lldb/test/API/lang/objc/objc-static-method-stripped/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/Makefile
rename to lldb/test/API/lang/objc/objc-static-method-stripped/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/TestObjCStaticMethodStripped.py b/lldb/test/API/lang/objc/objc-static-method-stripped/TestObjCStaticMethodStripped.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/TestObjCStaticMethodStripped.py
rename to lldb/test/API/lang/objc/objc-static-method-stripped/TestObjCStaticMethodStripped.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/static.m b/lldb/test/API/lang/objc/objc-static-method-stripped/static.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/static.m
rename to lldb/test/API/lang/objc/objc-static-method-stripped/static.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/Makefile b/lldb/test/API/lang/objc/objc-static-method/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/Makefile
rename to lldb/test/API/lang/objc/objc-static-method/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/TestObjCStaticMethod.py b/lldb/test/API/lang/objc/objc-static-method/TestObjCStaticMethod.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/TestObjCStaticMethod.py
rename to lldb/test/API/lang/objc/objc-static-method/TestObjCStaticMethod.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/static.m b/lldb/test/API/lang/objc/objc-static-method/static.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/static.m
rename to lldb/test/API/lang/objc/objc-static-method/static.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/Makefile b/lldb/test/API/lang/objc/objc-stepping/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/Makefile
rename to lldb/test/API/lang/objc/objc-stepping/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/TestObjCStepping.py b/lldb/test/API/lang/objc/objc-stepping/TestObjCStepping.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/TestObjCStepping.py
rename to lldb/test/API/lang/objc/objc-stepping/TestObjCStepping.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/stepping-tests.m b/lldb/test/API/lang/objc/objc-stepping/stepping-tests.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/stepping-tests.m
rename to lldb/test/API/lang/objc/objc-stepping/stepping-tests.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/Makefile b/lldb/test/API/lang/objc/objc-struct-argument/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/Makefile
rename to lldb/test/API/lang/objc/objc-struct-argument/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/TestObjCStructArgument.py b/lldb/test/API/lang/objc/objc-struct-argument/TestObjCStructArgument.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/TestObjCStructArgument.py
rename to lldb/test/API/lang/objc/objc-struct-argument/TestObjCStructArgument.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/test.m b/lldb/test/API/lang/objc/objc-struct-argument/test.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/test.m
rename to lldb/test/API/lang/objc/objc-struct-argument/test.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/Makefile b/lldb/test/API/lang/objc/objc-struct-return/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/Makefile
rename to lldb/test/API/lang/objc/objc-struct-return/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/TestObjCStructReturn.py b/lldb/test/API/lang/objc/objc-struct-return/TestObjCStructReturn.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/TestObjCStructReturn.py
rename to lldb/test/API/lang/objc/objc-struct-return/TestObjCStructReturn.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/test.m b/lldb/test/API/lang/objc/objc-struct-return/test.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/test.m
rename to lldb/test/API/lang/objc/objc-struct-return/test.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/Makefile b/lldb/test/API/lang/objc/objc-super/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/Makefile
rename to lldb/test/API/lang/objc/objc-super/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/TestObjCSuper.py b/lldb/test/API/lang/objc/objc-super/TestObjCSuper.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/TestObjCSuper.py
rename to lldb/test/API/lang/objc/objc-super/TestObjCSuper.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/class.m b/lldb/test/API/lang/objc/objc-super/class.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/class.m
rename to lldb/test/API/lang/objc/objc-super/class.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc_direct-methods/Makefile b/lldb/test/API/lang/objc/objc_direct-methods/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc_direct-methods/Makefile
rename to lldb/test/API/lang/objc/objc_direct-methods/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc_direct-methods/TestObjCDirectMethods.py b/lldb/test/API/lang/objc/objc_direct-methods/TestObjCDirectMethods.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc_direct-methods/TestObjCDirectMethods.py
rename to lldb/test/API/lang/objc/objc_direct-methods/TestObjCDirectMethods.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc_direct-methods/main.m b/lldb/test/API/lang/objc/objc_direct-methods/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/objc_direct-methods/main.m
rename to lldb/test/API/lang/objc/objc_direct-methods/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/Makefile b/lldb/test/API/lang/objc/orderedset/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/Makefile
rename to lldb/test/API/lang/objc/orderedset/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/TestOrderedSet.py b/lldb/test/API/lang/objc/orderedset/TestOrderedSet.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/TestOrderedSet.py
rename to lldb/test/API/lang/objc/orderedset/TestOrderedSet.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/main.m b/lldb/test/API/lang/objc/orderedset/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/main.m
rename to lldb/test/API/lang/objc/orderedset/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/Makefile b/lldb/test/API/lang/objc/print-obj/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/Makefile
rename to lldb/test/API/lang/objc/print-obj/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/TestPrintObj.py b/lldb/test/API/lang/objc/print-obj/TestPrintObj.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/TestPrintObj.py
rename to lldb/test/API/lang/objc/print-obj/TestPrintObj.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/blocked.m b/lldb/test/API/lang/objc/print-obj/blocked.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/blocked.m
rename to lldb/test/API/lang/objc/print-obj/blocked.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ptr_refs/Makefile b/lldb/test/API/lang/objc/ptr_refs/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ptr_refs/Makefile
rename to lldb/test/API/lang/objc/ptr_refs/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ptr_refs/TestPtrRefsObjC.py b/lldb/test/API/lang/objc/ptr_refs/TestPtrRefsObjC.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ptr_refs/TestPtrRefsObjC.py
rename to lldb/test/API/lang/objc/ptr_refs/TestPtrRefsObjC.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ptr_refs/main.m b/lldb/test/API/lang/objc/ptr_refs/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/ptr_refs/main.m
rename to lldb/test/API/lang/objc/ptr_refs/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/Makefile b/lldb/test/API/lang/objc/radar-9691614/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/Makefile
rename to lldb/test/API/lang/objc/radar-9691614/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/TestObjCMethodReturningBOOL.py b/lldb/test/API/lang/objc/radar-9691614/TestObjCMethodReturningBOOL.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/TestObjCMethodReturningBOOL.py
rename to lldb/test/API/lang/objc/radar-9691614/TestObjCMethodReturningBOOL.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/main.m b/lldb/test/API/lang/objc/radar-9691614/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/main.m
rename to lldb/test/API/lang/objc/radar-9691614/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/Makefile b/lldb/test/API/lang/objc/rdar-10967107/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/Makefile
rename to lldb/test/API/lang/objc/rdar-10967107/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/TestRdar10967107.py b/lldb/test/API/lang/objc/rdar-10967107/TestRdar10967107.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/TestRdar10967107.py
rename to lldb/test/API/lang/objc/rdar-10967107/TestRdar10967107.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/main.m b/lldb/test/API/lang/objc/rdar-10967107/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/main.m
rename to lldb/test/API/lang/objc/rdar-10967107/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/Makefile b/lldb/test/API/lang/objc/rdar-11355592/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/Makefile
rename to lldb/test/API/lang/objc/rdar-11355592/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/TestRdar11355592.py b/lldb/test/API/lang/objc/rdar-11355592/TestRdar11355592.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/TestRdar11355592.py
rename to lldb/test/API/lang/objc/rdar-11355592/TestRdar11355592.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/main.m b/lldb/test/API/lang/objc/rdar-11355592/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/main.m
rename to lldb/test/API/lang/objc/rdar-11355592/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-12408181/Makefile b/lldb/test/API/lang/objc/rdar-12408181/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-12408181/Makefile
rename to lldb/test/API/lang/objc/rdar-12408181/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-12408181/TestRdar12408181.py b/lldb/test/API/lang/objc/rdar-12408181/TestRdar12408181.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-12408181/TestRdar12408181.py
rename to lldb/test/API/lang/objc/rdar-12408181/TestRdar12408181.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-12408181/main.m b/lldb/test/API/lang/objc/rdar-12408181/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/rdar-12408181/main.m
rename to lldb/test/API/lang/objc/rdar-12408181/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Bar.h b/lldb/test/API/lang/objc/real-definition/Bar.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Bar.h
rename to lldb/test/API/lang/objc/real-definition/Bar.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Bar.m b/lldb/test/API/lang/objc/real-definition/Bar.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Bar.m
rename to lldb/test/API/lang/objc/real-definition/Bar.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Foo.h b/lldb/test/API/lang/objc/real-definition/Foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Foo.h
rename to lldb/test/API/lang/objc/real-definition/Foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Foo.m b/lldb/test/API/lang/objc/real-definition/Foo.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Foo.m
rename to lldb/test/API/lang/objc/real-definition/Foo.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Makefile b/lldb/test/API/lang/objc/real-definition/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Makefile
rename to lldb/test/API/lang/objc/real-definition/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/TestRealDefinition.py b/lldb/test/API/lang/objc/real-definition/TestRealDefinition.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/TestRealDefinition.py
rename to lldb/test/API/lang/objc/real-definition/TestRealDefinition.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/main.m b/lldb/test/API/lang/objc/real-definition/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/main.m
rename to lldb/test/API/lang/objc/real-definition/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/sample/Makefile b/lldb/test/API/lang/objc/sample/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/sample/Makefile
rename to lldb/test/API/lang/objc/sample/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/sample/main.m b/lldb/test/API/lang/objc/sample/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/sample/main.m
rename to lldb/test/API/lang/objc/sample/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/self/Makefile b/lldb/test/API/lang/objc/self/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/self/Makefile
rename to lldb/test/API/lang/objc/self/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/self/TestObjCSelf.py b/lldb/test/API/lang/objc/self/TestObjCSelf.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/self/TestObjCSelf.py
rename to lldb/test/API/lang/objc/self/TestObjCSelf.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/self/main.m b/lldb/test/API/lang/objc/self/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/self/main.m
rename to lldb/test/API/lang/objc/self/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/Makefile b/lldb/test/API/lang/objc/single-entry-dictionary/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/Makefile
rename to lldb/test/API/lang/objc/single-entry-dictionary/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/TestObjCSingleEntryDictionary.py b/lldb/test/API/lang/objc/single-entry-dictionary/TestObjCSingleEntryDictionary.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/TestObjCSingleEntryDictionary.py
rename to lldb/test/API/lang/objc/single-entry-dictionary/TestObjCSingleEntryDictionary.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/main.m b/lldb/test/API/lang/objc/single-entry-dictionary/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/main.m
rename to lldb/test/API/lang/objc/single-entry-dictionary/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/unicode-string/TestUnicodeString.py b/lldb/test/API/lang/objc/unicode-string/TestUnicodeString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/unicode-string/TestUnicodeString.py
rename to lldb/test/API/lang/objc/unicode-string/TestUnicodeString.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/unicode-string/main.m b/lldb/test/API/lang/objc/unicode-string/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/unicode-string/main.m
rename to lldb/test/API/lang/objc/unicode-string/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/variadic_methods/TestVariadicMethods.py b/lldb/test/API/lang/objc/variadic_methods/TestVariadicMethods.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/variadic_methods/TestVariadicMethods.py
rename to lldb/test/API/lang/objc/variadic_methods/TestVariadicMethods.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/variadic_methods/main.m b/lldb/test/API/lang/objc/variadic_methods/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objc/variadic_methods/main.m
rename to lldb/test/API/lang/objc/variadic_methods/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/Makefile b/lldb/test/API/lang/objcxx/class-name-clash/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/Makefile
rename to lldb/test/API/lang/objcxx/class-name-clash/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/TestNameClash.py b/lldb/test/API/lang/objcxx/class-name-clash/TestNameClash.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/TestNameClash.py
rename to lldb/test/API/lang/objcxx/class-name-clash/TestNameClash.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/main.mm b/lldb/test/API/lang/objcxx/class-name-clash/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/main.mm
rename to lldb/test/API/lang/objcxx/class-name-clash/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/myobject.mm b/lldb/test/API/lang/objcxx/class-name-clash/myobject.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/class-name-clash/myobject.mm
rename to lldb/test/API/lang/objcxx/class-name-clash/myobject.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/Makefile b/lldb/test/API/lang/objcxx/cxx-bridged-po/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/Makefile
rename to lldb/test/API/lang/objcxx/cxx-bridged-po/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/TestObjCXXBridgedPO.py b/lldb/test/API/lang/objcxx/cxx-bridged-po/TestObjCXXBridgedPO.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/TestObjCXXBridgedPO.py
rename to lldb/test/API/lang/objcxx/cxx-bridged-po/TestObjCXXBridgedPO.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/main.mm b/lldb/test/API/lang/objcxx/cxx-bridged-po/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/main.mm
rename to lldb/test/API/lang/objcxx/cxx-bridged-po/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/Makefile b/lldb/test/API/lang/objcxx/hide-runtime-values/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/Makefile
rename to lldb/test/API/lang/objcxx/hide-runtime-values/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/TestObjCXXHideRuntimeValues.py b/lldb/test/API/lang/objcxx/hide-runtime-values/TestObjCXXHideRuntimeValues.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/TestObjCXXHideRuntimeValues.py
rename to lldb/test/API/lang/objcxx/hide-runtime-values/TestObjCXXHideRuntimeValues.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/main.mm b/lldb/test/API/lang/objcxx/hide-runtime-values/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/main.mm
rename to lldb/test/API/lang/objcxx/hide-runtime-values/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/objcxx-ivar-vector/TestIvarVector.py b/lldb/test/API/lang/objcxx/objcxx-ivar-vector/TestIvarVector.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/objcxx-ivar-vector/TestIvarVector.py
rename to lldb/test/API/lang/objcxx/objcxx-ivar-vector/TestIvarVector.py
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/objcxx-ivar-vector/main.mm b/lldb/test/API/lang/objcxx/objcxx-ivar-vector/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/objcxx-ivar-vector/main.mm
rename to lldb/test/API/lang/objcxx/objcxx-ivar-vector/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/Makefile b/lldb/test/API/lang/objcxx/sample/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/Makefile
rename to lldb/test/API/lang/objcxx/sample/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/main.mm b/lldb/test/API/lang/objcxx/sample/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/main.mm
rename to lldb/test/API/lang/objcxx/sample/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/linux/add-symbols/Makefile b/lldb/test/API/linux/add-symbols/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/add-symbols/Makefile
rename to lldb/test/API/linux/add-symbols/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/linux/add-symbols/TestTargetSymbolsAddCommand.py b/lldb/test/API/linux/add-symbols/TestTargetSymbolsAddCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/add-symbols/TestTargetSymbolsAddCommand.py
rename to lldb/test/API/linux/add-symbols/TestTargetSymbolsAddCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/linux/add-symbols/main.c b/lldb/test/API/linux/add-symbols/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/add-symbols/main.c
rename to lldb/test/API/linux/add-symbols/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/linux/builtin_trap/Makefile b/lldb/test/API/linux/builtin_trap/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/builtin_trap/Makefile
rename to lldb/test/API/linux/builtin_trap/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/linux/builtin_trap/TestBuiltinTrap.py b/lldb/test/API/linux/builtin_trap/TestBuiltinTrap.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/builtin_trap/TestBuiltinTrap.py
rename to lldb/test/API/linux/builtin_trap/TestBuiltinTrap.py
diff --git a/lldb/packages/Python/lldbsuite/test/linux/builtin_trap/main.cpp b/lldb/test/API/linux/builtin_trap/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/builtin_trap/main.cpp
rename to lldb/test/API/linux/builtin_trap/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/Makefile b/lldb/test/API/linux/mix-dwo-and-regular-objects/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/Makefile
rename to lldb/test/API/linux/mix-dwo-and-regular-objects/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/TestMixedDwarfBinary.py b/lldb/test/API/linux/mix-dwo-and-regular-objects/TestMixedDwarfBinary.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/TestMixedDwarfBinary.py
rename to lldb/test/API/linux/mix-dwo-and-regular-objects/TestMixedDwarfBinary.py
diff --git a/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/a.c b/lldb/test/API/linux/mix-dwo-and-regular-objects/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/a.c
rename to lldb/test/API/linux/mix-dwo-and-regular-objects/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/b.c b/lldb/test/API/linux/mix-dwo-and-regular-objects/b.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/b.c
rename to lldb/test/API/linux/mix-dwo-and-regular-objects/b.c
diff --git a/lldb/packages/Python/lldbsuite/test/linux/sepdebugsymlink/Makefile b/lldb/test/API/linux/sepdebugsymlink/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/sepdebugsymlink/Makefile
rename to lldb/test/API/linux/sepdebugsymlink/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/linux/sepdebugsymlink/TestTargetSymbolsSepDebugSymlink.py b/lldb/test/API/linux/sepdebugsymlink/TestTargetSymbolsSepDebugSymlink.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/sepdebugsymlink/TestTargetSymbolsSepDebugSymlink.py
rename to lldb/test/API/linux/sepdebugsymlink/TestTargetSymbolsSepDebugSymlink.py
diff --git a/lldb/packages/Python/lldbsuite/test/linux/sepdebugsymlink/main.c b/lldb/test/API/linux/sepdebugsymlink/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/sepdebugsymlink/main.c
rename to lldb/test/API/linux/sepdebugsymlink/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/linux/thread/create_during_instruction_step/Makefile b/lldb/test/API/linux/thread/create_during_instruction_step/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/thread/create_during_instruction_step/Makefile
rename to lldb/test/API/linux/thread/create_during_instruction_step/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py b/lldb/test/API/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py
rename to lldb/test/API/linux/thread/create_during_instruction_step/TestCreateDuringInstructionStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/linux/thread/create_during_instruction_step/main.cpp b/lldb/test/API/linux/thread/create_during_instruction_step/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/linux/thread/create_during_instruction_step/main.cpp
rename to lldb/test/API/linux/thread/create_during_instruction_step/main.cpp
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 79401ae47d13502daa08de30f27bfcaf909cc099..10df5600db4cf3d61e31fff98119c8c179f340af 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -17,8 +17,7 @@ config.suffixes = ['.py']
 
 # test_source_root: The root path where tests are located.
 # test_exec_root: The root path where tests should be run.
-config.test_source_root = os.path.join(config.lldb_src_root, 'packages',
-                                       'Python', 'lldbsuite', 'test')
+config.test_source_root = os.path.dirname(__file__)
 config.test_exec_root = config.test_source_root
 
 if 'Address' in config.llvm_use_sanitizer:
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/Inputs/main.c b/lldb/test/API/macosx/DBGSourcePathRemapping/Inputs/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/Inputs/main.c
rename to lldb/test/API/macosx/DBGSourcePathRemapping/Inputs/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/Inputs/relative.c b/lldb/test/API/macosx/DBGSourcePathRemapping/Inputs/relative.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/Inputs/relative.c
rename to lldb/test/API/macosx/DBGSourcePathRemapping/Inputs/relative.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/Makefile b/lldb/test/API/macosx/DBGSourcePathRemapping/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/Makefile
rename to lldb/test/API/macosx/DBGSourcePathRemapping/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/TestDSYMSourcePathRemapping.py b/lldb/test/API/macosx/DBGSourcePathRemapping/TestDSYMSourcePathRemapping.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/DBGSourcePathRemapping/TestDSYMSourcePathRemapping.py
rename to lldb/test/API/macosx/DBGSourcePathRemapping/TestDSYMSourcePathRemapping.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/add-dsym/Makefile b/lldb/test/API/macosx/add-dsym/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/add-dsym/Makefile
rename to lldb/test/API/macosx/add-dsym/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/add-dsym/TestAddDsymMidExecutionCommand.py b/lldb/test/API/macosx/add-dsym/TestAddDsymMidExecutionCommand.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/add-dsym/TestAddDsymMidExecutionCommand.py
rename to lldb/test/API/macosx/add-dsym/TestAddDsymMidExecutionCommand.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/add-dsym/main.c b/lldb/test/API/macosx/add-dsym/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/add-dsym/main.c
rename to lldb/test/API/macosx/add-dsym/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/Makefile b/lldb/test/API/macosx/duplicate-archive-members/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/Makefile
rename to lldb/test/API/macosx/duplicate-archive-members/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/TestDuplicateMembers.py b/lldb/test/API/macosx/duplicate-archive-members/TestDuplicateMembers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/TestDuplicateMembers.py
rename to lldb/test/API/macosx/duplicate-archive-members/TestDuplicateMembers.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/a.c b/lldb/test/API/macosx/duplicate-archive-members/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/a.c
rename to lldb/test/API/macosx/duplicate-archive-members/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/main.c b/lldb/test/API/macosx/duplicate-archive-members/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/main.c
rename to lldb/test/API/macosx/duplicate-archive-members/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/sub1/a.c b/lldb/test/API/macosx/duplicate-archive-members/sub1/a.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/duplicate-archive-members/sub1/a.c
rename to lldb/test/API/macosx/duplicate-archive-members/sub1/a.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/Makefile b/lldb/test/API/macosx/find-app-in-bundle/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/Makefile
rename to lldb/test/API/macosx/find-app-in-bundle/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestApp.app/Contents/Info.plist b/lldb/test/API/macosx/find-app-in-bundle/TestApp.app/Contents/Info.plist
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestApp.app/Contents/Info.plist
rename to lldb/test/API/macosx/find-app-in-bundle/TestApp.app/Contents/Info.plist
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestApp.app/Contents/MacOS/.empty b/lldb/test/API/macosx/find-app-in-bundle/TestApp.app/Contents/MacOS/.empty
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestApp.app/Contents/MacOS/.empty
rename to lldb/test/API/macosx/find-app-in-bundle/TestApp.app/Contents/MacOS/.empty
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestApp.app/Contents/Resources/.empty b/lldb/test/API/macosx/find-app-in-bundle/TestApp.app/Contents/Resources/.empty
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestApp.app/Contents/Resources/.empty
rename to lldb/test/API/macosx/find-app-in-bundle/TestApp.app/Contents/Resources/.empty
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestFindAppInBundle.py b/lldb/test/API/macosx/find-app-in-bundle/TestFindAppInBundle.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/TestFindAppInBundle.py
rename to lldb/test/API/macosx/find-app-in-bundle/TestFindAppInBundle.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/main.c b/lldb/test/API/macosx/find-app-in-bundle/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-app-in-bundle/main.c
rename to lldb/test/API/macosx/find-app-in-bundle/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/Makefile b/lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/Makefile
rename to lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/TestBundleWithDotInFilename.py b/lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/TestBundleWithDotInFilename.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/TestBundleWithDotInFilename.py
rename to lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/TestBundleWithDotInFilename.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/bundle.c b/lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/bundle.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/bundle.c
rename to lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/bundle.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/main.c b/lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/bundle-with-dot-in-filename/main.c
rename to lldb/test/API/macosx/find-dsym/bundle-with-dot-in-filename/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/Info.plist b/lldb/test/API/macosx/find-dsym/deep-bundle/Info.plist
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/Info.plist
rename to lldb/test/API/macosx/find-dsym/deep-bundle/Info.plist
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/Makefile b/lldb/test/API/macosx/find-dsym/deep-bundle/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/Makefile
rename to lldb/test/API/macosx/find-dsym/deep-bundle/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/MyFramework.h b/lldb/test/API/macosx/find-dsym/deep-bundle/MyFramework.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/MyFramework.h
rename to lldb/test/API/macosx/find-dsym/deep-bundle/MyFramework.h
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/TestDeepBundle.py b/lldb/test/API/macosx/find-dsym/deep-bundle/TestDeepBundle.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/TestDeepBundle.py
rename to lldb/test/API/macosx/find-dsym/deep-bundle/TestDeepBundle.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/main.c b/lldb/test/API/macosx/find-dsym/deep-bundle/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/main.c
rename to lldb/test/API/macosx/find-dsym/deep-bundle/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/myframework.c b/lldb/test/API/macosx/find-dsym/deep-bundle/myframework.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/find-dsym/deep-bundle/myframework.c
rename to lldb/test/API/macosx/find-dsym/deep-bundle/myframework.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/function-starts/Makefile b/lldb/test/API/macosx/function-starts/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/function-starts/Makefile
rename to lldb/test/API/macosx/function-starts/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/function-starts/TestFunctionStarts.py b/lldb/test/API/macosx/function-starts/TestFunctionStarts.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/function-starts/TestFunctionStarts.py
rename to lldb/test/API/macosx/function-starts/TestFunctionStarts.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/function-starts/main.cpp b/lldb/test/API/macosx/function-starts/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/function-starts/main.cpp
rename to lldb/test/API/macosx/function-starts/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/Makefile b/lldb/test/API/macosx/indirect_symbol/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/Makefile
rename to lldb/test/API/macosx/indirect_symbol/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/TestIndirectSymbols.py b/lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/TestIndirectSymbols.py
rename to lldb/test/API/macosx/indirect_symbol/TestIndirectSymbols.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/alias.list b/lldb/test/API/macosx/indirect_symbol/alias.list
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/alias.list
rename to lldb/test/API/macosx/indirect_symbol/alias.list
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/indirect.c b/lldb/test/API/macosx/indirect_symbol/indirect.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/indirect.c
rename to lldb/test/API/macosx/indirect_symbol/indirect.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/main.c b/lldb/test/API/macosx/indirect_symbol/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/main.c
rename to lldb/test/API/macosx/indirect_symbol/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/reexport.c b/lldb/test/API/macosx/indirect_symbol/reexport.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/indirect_symbol/reexport.c
rename to lldb/test/API/macosx/indirect_symbol/reexport.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/Makefile b/lldb/test/API/macosx/lc-note/kern-ver-str/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/Makefile
rename to lldb/test/API/macosx/lc-note/kern-ver-str/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/TestKernVerStrLCNOTE.py b/lldb/test/API/macosx/lc-note/kern-ver-str/TestKernVerStrLCNOTE.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/TestKernVerStrLCNOTE.py
rename to lldb/test/API/macosx/lc-note/kern-ver-str/TestKernVerStrLCNOTE.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/create-empty-corefile.cpp b/lldb/test/API/macosx/lc-note/kern-ver-str/create-empty-corefile.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/create-empty-corefile.cpp
rename to lldb/test/API/macosx/lc-note/kern-ver-str/create-empty-corefile.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/main.c b/lldb/test/API/macosx/lc-note/kern-ver-str/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/lc-note/kern-ver-str/main.c
rename to lldb/test/API/macosx/lc-note/kern-ver-str/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/load-kext/TestLoadKext.py b/lldb/test/API/macosx/load-kext/TestLoadKext.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/load-kext/TestLoadKext.py
rename to lldb/test/API/macosx/load-kext/TestLoadKext.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/load-kext/mykext.yaml b/lldb/test/API/macosx/load-kext/mykext.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/load-kext/mykext.yaml
rename to lldb/test/API/macosx/load-kext/mykext.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/macabi/Makefile b/lldb/test/API/macosx/macabi/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/macabi/Makefile
rename to lldb/test/API/macosx/macabi/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/macabi/TestMacABImacOSFramework.py b/lldb/test/API/macosx/macabi/TestMacABImacOSFramework.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/macabi/TestMacABImacOSFramework.py
rename to lldb/test/API/macosx/macabi/TestMacABImacOSFramework.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/macabi/foo.c b/lldb/test/API/macosx/macabi/foo.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/macabi/foo.c
rename to lldb/test/API/macosx/macabi/foo.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/macabi/foo.h b/lldb/test/API/macosx/macabi/foo.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/macabi/foo.h
rename to lldb/test/API/macosx/macabi/foo.h
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/macabi/main.c b/lldb/test/API/macosx/macabi/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/macabi/main.c
rename to lldb/test/API/macosx/macabi/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/nslog/Makefile b/lldb/test/API/macosx/nslog/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/nslog/Makefile
rename to lldb/test/API/macosx/nslog/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/nslog/TestDarwinNSLogOutput.py b/lldb/test/API/macosx/nslog/TestDarwinNSLogOutput.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/nslog/TestDarwinNSLogOutput.py
rename to lldb/test/API/macosx/nslog/TestDarwinNSLogOutput.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/nslog/main.m b/lldb/test/API/macosx/nslog/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/nslog/main.m
rename to lldb/test/API/macosx/nslog/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/order/Makefile b/lldb/test/API/macosx/order/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/order/Makefile
rename to lldb/test/API/macosx/order/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/order/TestOrderFile.py b/lldb/test/API/macosx/order/TestOrderFile.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/order/TestOrderFile.py
rename to lldb/test/API/macosx/order/TestOrderFile.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/order/cmds.txt b/lldb/test/API/macosx/order/cmds.txt
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/order/cmds.txt
rename to lldb/test/API/macosx/order/cmds.txt
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/order/main.c b/lldb/test/API/macosx/order/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/order/main.c
rename to lldb/test/API/macosx/order/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/order/order-file b/lldb/test/API/macosx/order/order-file
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/order/order-file
rename to lldb/test/API/macosx/order/order-file
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/queues/Makefile b/lldb/test/API/macosx/queues/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/queues/Makefile
rename to lldb/test/API/macosx/queues/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/queues/TestQueues.py b/lldb/test/API/macosx/queues/TestQueues.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/queues/TestQueues.py
rename to lldb/test/API/macosx/queues/TestQueues.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/queues/main.c b/lldb/test/API/macosx/queues/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/queues/main.c
rename to lldb/test/API/macosx/queues/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/safe-to-func-call/Makefile b/lldb/test/API/macosx/safe-to-func-call/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/safe-to-func-call/Makefile
rename to lldb/test/API/macosx/safe-to-func-call/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/safe-to-func-call/TestSafeFuncCalls.py b/lldb/test/API/macosx/safe-to-func-call/TestSafeFuncCalls.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/safe-to-func-call/TestSafeFuncCalls.py
rename to lldb/test/API/macosx/safe-to-func-call/TestSafeFuncCalls.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/safe-to-func-call/main.c b/lldb/test/API/macosx/safe-to-func-call/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/safe-to-func-call/main.c
rename to lldb/test/API/macosx/safe-to-func-call/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/thread-names/Makefile b/lldb/test/API/macosx/thread-names/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/thread-names/Makefile
rename to lldb/test/API/macosx/thread-names/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/thread-names/TestInterruptThreadNames.py b/lldb/test/API/macosx/thread-names/TestInterruptThreadNames.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/thread-names/TestInterruptThreadNames.py
rename to lldb/test/API/macosx/thread-names/TestInterruptThreadNames.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/thread-names/main.c b/lldb/test/API/macosx/thread-names/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/thread-names/main.c
rename to lldb/test/API/macosx/thread-names/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/universal/Makefile b/lldb/test/API/macosx/universal/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/universal/Makefile
rename to lldb/test/API/macosx/universal/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/universal/TestUniversal.py b/lldb/test/API/macosx/universal/TestUniversal.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/universal/TestUniversal.py
rename to lldb/test/API/macosx/universal/TestUniversal.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/universal/main.c b/lldb/test/API/macosx/universal/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/universal/main.c
rename to lldb/test/API/macosx/universal/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/version_zero/TestGetVersionZeroVersion.py b/lldb/test/API/macosx/version_zero/TestGetVersionZeroVersion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/version_zero/TestGetVersionZeroVersion.py
rename to lldb/test/API/macosx/version_zero/TestGetVersionZeroVersion.py
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/version_zero/libDylib.dylib.yaml b/lldb/test/API/macosx/version_zero/libDylib.dylib.yaml
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/macosx/version_zero/libDylib.dylib.yaml
rename to lldb/test/API/macosx/version_zero/libDylib.dylib.yaml
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/.categories b/lldb/test/API/python_api/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/.categories
rename to lldb/test/API/python_api/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/breakpoint/Makefile b/lldb/test/API/python_api/breakpoint/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/breakpoint/Makefile
rename to lldb/test/API/python_api/breakpoint/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/breakpoint/TestBreakpointAPI.py b/lldb/test/API/python_api/breakpoint/TestBreakpointAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/breakpoint/TestBreakpointAPI.py
rename to lldb/test/API/python_api/breakpoint/TestBreakpointAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/breakpoint/main.c b/lldb/test/API/python_api/breakpoint/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/breakpoint/main.c
rename to lldb/test/API/python_api/breakpoint/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/class_members/Makefile b/lldb/test/API/python_api/class_members/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/class_members/Makefile
rename to lldb/test/API/python_api/class_members/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/class_members/TestSBTypeClassMembers.py b/lldb/test/API/python_api/class_members/TestSBTypeClassMembers.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/class_members/TestSBTypeClassMembers.py
rename to lldb/test/API/python_api/class_members/TestSBTypeClassMembers.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/class_members/main.mm b/lldb/test/API/python_api/class_members/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/class_members/main.mm
rename to lldb/test/API/python_api/class_members/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/debugger/TestDebuggerAPI.py b/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/debugger/TestDebuggerAPI.py
rename to lldb/test/API/python_api/debugger/TestDebuggerAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/TestDefaultConstructorForAPIObjects.py b/lldb/test/API/python_api/default-constructor/TestDefaultConstructorForAPIObjects.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/TestDefaultConstructorForAPIObjects.py
rename to lldb/test/API/python_api/default-constructor/TestDefaultConstructorForAPIObjects.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_address.py b/lldb/test/API/python_api/default-constructor/sb_address.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_address.py
rename to lldb/test/API/python_api/default-constructor/sb_address.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_block.py b/lldb/test/API/python_api/default-constructor/sb_block.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_block.py
rename to lldb/test/API/python_api/default-constructor/sb_block.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_breakpoint.py b/lldb/test/API/python_api/default-constructor/sb_breakpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_breakpoint.py
rename to lldb/test/API/python_api/default-constructor/sb_breakpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_breakpointlocation.py b/lldb/test/API/python_api/default-constructor/sb_breakpointlocation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_breakpointlocation.py
rename to lldb/test/API/python_api/default-constructor/sb_breakpointlocation.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_breakpointname.py b/lldb/test/API/python_api/default-constructor/sb_breakpointname.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_breakpointname.py
rename to lldb/test/API/python_api/default-constructor/sb_breakpointname.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_broadcaster.py b/lldb/test/API/python_api/default-constructor/sb_broadcaster.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_broadcaster.py
rename to lldb/test/API/python_api/default-constructor/sb_broadcaster.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_communication.py b/lldb/test/API/python_api/default-constructor/sb_communication.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_communication.py
rename to lldb/test/API/python_api/default-constructor/sb_communication.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_compileunit.py b/lldb/test/API/python_api/default-constructor/sb_compileunit.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_compileunit.py
rename to lldb/test/API/python_api/default-constructor/sb_compileunit.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_debugger.py b/lldb/test/API/python_api/default-constructor/sb_debugger.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_debugger.py
rename to lldb/test/API/python_api/default-constructor/sb_debugger.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_error.py b/lldb/test/API/python_api/default-constructor/sb_error.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_error.py
rename to lldb/test/API/python_api/default-constructor/sb_error.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_event.py b/lldb/test/API/python_api/default-constructor/sb_event.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_event.py
rename to lldb/test/API/python_api/default-constructor/sb_event.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_filespec.py b/lldb/test/API/python_api/default-constructor/sb_filespec.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_filespec.py
rename to lldb/test/API/python_api/default-constructor/sb_filespec.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_frame.py b/lldb/test/API/python_api/default-constructor/sb_frame.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_frame.py
rename to lldb/test/API/python_api/default-constructor/sb_frame.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_function.py b/lldb/test/API/python_api/default-constructor/sb_function.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_function.py
rename to lldb/test/API/python_api/default-constructor/sb_function.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_instruction.py b/lldb/test/API/python_api/default-constructor/sb_instruction.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_instruction.py
rename to lldb/test/API/python_api/default-constructor/sb_instruction.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_instructionlist.py b/lldb/test/API/python_api/default-constructor/sb_instructionlist.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_instructionlist.py
rename to lldb/test/API/python_api/default-constructor/sb_instructionlist.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_lineentry.py b/lldb/test/API/python_api/default-constructor/sb_lineentry.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_lineentry.py
rename to lldb/test/API/python_api/default-constructor/sb_lineentry.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_listener.py b/lldb/test/API/python_api/default-constructor/sb_listener.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_listener.py
rename to lldb/test/API/python_api/default-constructor/sb_listener.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_module.py b/lldb/test/API/python_api/default-constructor/sb_module.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_module.py
rename to lldb/test/API/python_api/default-constructor/sb_module.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_process.py b/lldb/test/API/python_api/default-constructor/sb_process.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_process.py
rename to lldb/test/API/python_api/default-constructor/sb_process.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_process_info.py b/lldb/test/API/python_api/default-constructor/sb_process_info.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_process_info.py
rename to lldb/test/API/python_api/default-constructor/sb_process_info.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_section.py b/lldb/test/API/python_api/default-constructor/sb_section.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_section.py
rename to lldb/test/API/python_api/default-constructor/sb_section.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_stringlist.py b/lldb/test/API/python_api/default-constructor/sb_stringlist.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_stringlist.py
rename to lldb/test/API/python_api/default-constructor/sb_stringlist.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_symbol.py b/lldb/test/API/python_api/default-constructor/sb_symbol.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_symbol.py
rename to lldb/test/API/python_api/default-constructor/sb_symbol.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_symbolcontext.py b/lldb/test/API/python_api/default-constructor/sb_symbolcontext.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_symbolcontext.py
rename to lldb/test/API/python_api/default-constructor/sb_symbolcontext.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_target.py b/lldb/test/API/python_api/default-constructor/sb_target.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_target.py
rename to lldb/test/API/python_api/default-constructor/sb_target.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_thread.py b/lldb/test/API/python_api/default-constructor/sb_thread.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_thread.py
rename to lldb/test/API/python_api/default-constructor/sb_thread.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_type.py b/lldb/test/API/python_api/default-constructor/sb_type.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_type.py
rename to lldb/test/API/python_api/default-constructor/sb_type.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_value.py b/lldb/test/API/python_api/default-constructor/sb_value.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_value.py
rename to lldb/test/API/python_api/default-constructor/sb_value.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_valuelist.py b/lldb/test/API/python_api/default-constructor/sb_valuelist.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_valuelist.py
rename to lldb/test/API/python_api/default-constructor/sb_valuelist.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_watchpoint.py b/lldb/test/API/python_api/default-constructor/sb_watchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/default-constructor/sb_watchpoint.py
rename to lldb/test/API/python_api/default-constructor/sb_watchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/disassemble-raw-data/TestDisassembleRawData.py b/lldb/test/API/python_api/disassemble-raw-data/TestDisassembleRawData.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/disassemble-raw-data/TestDisassembleRawData.py
rename to lldb/test/API/python_api/disassemble-raw-data/TestDisassembleRawData.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/disassemble-raw-data/TestDisassemble_VST1_64.py b/lldb/test/API/python_api/disassemble-raw-data/TestDisassemble_VST1_64.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/disassemble-raw-data/TestDisassemble_VST1_64.py
rename to lldb/test/API/python_api/disassemble-raw-data/TestDisassemble_VST1_64.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/event/Makefile b/lldb/test/API/python_api/event/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/event/Makefile
rename to lldb/test/API/python_api/event/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/event/TestEvents.py b/lldb/test/API/python_api/event/TestEvents.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/event/TestEvents.py
rename to lldb/test/API/python_api/event/TestEvents.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/event/main.c b/lldb/test/API/python_api/event/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/event/main.c
rename to lldb/test/API/python_api/event/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/exprpath_synthetic/TestExprPathSynthetic.py b/lldb/test/API/python_api/exprpath_synthetic/TestExprPathSynthetic.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/exprpath_synthetic/TestExprPathSynthetic.py
rename to lldb/test/API/python_api/exprpath_synthetic/TestExprPathSynthetic.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/exprpath_synthetic/main.mm b/lldb/test/API/python_api/exprpath_synthetic/main.mm
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/exprpath_synthetic/main.mm
rename to lldb/test/API/python_api/exprpath_synthetic/main.mm
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py b/lldb/test/API/python_api/file_handle/TestFileHandle.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/file_handle/TestFileHandle.py
rename to lldb/test/API/python_api/file_handle/TestFileHandle.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/Makefile b/lldb/test/API/python_api/findvalue_duplist/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/Makefile
rename to lldb/test/API/python_api/findvalue_duplist/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/TestSBFrameFindValue.py b/lldb/test/API/python_api/findvalue_duplist/TestSBFrameFindValue.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/TestSBFrameFindValue.py
rename to lldb/test/API/python_api/findvalue_duplist/TestSBFrameFindValue.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/main.cpp b/lldb/test/API/python_api/findvalue_duplist/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/main.cpp
rename to lldb/test/API/python_api/findvalue_duplist/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/formatters/Makefile b/lldb/test/API/python_api/formatters/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/formatters/Makefile
rename to lldb/test/API/python_api/formatters/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/formatters/TestFormattersSBAPI.py b/lldb/test/API/python_api/formatters/TestFormattersSBAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/formatters/TestFormattersSBAPI.py
rename to lldb/test/API/python_api/formatters/TestFormattersSBAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/formatters/main.cpp b/lldb/test/API/python_api/formatters/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/formatters/main.cpp
rename to lldb/test/API/python_api/formatters/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/formatters/synth.py b/lldb/test/API/python_api/formatters/synth.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/formatters/synth.py
rename to lldb/test/API/python_api/formatters/synth.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/Makefile b/lldb/test/API/python_api/frame/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/Makefile
rename to lldb/test/API/python_api/frame/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/TestFrames.py b/lldb/test/API/python_api/frame/TestFrames.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/TestFrames.py
rename to lldb/test/API/python_api/frame/TestFrames.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/get-variables/Makefile b/lldb/test/API/python_api/frame/get-variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/get-variables/Makefile
rename to lldb/test/API/python_api/frame/get-variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/get-variables/TestGetVariables.py b/lldb/test/API/python_api/frame/get-variables/TestGetVariables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/get-variables/TestGetVariables.py
rename to lldb/test/API/python_api/frame/get-variables/TestGetVariables.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/get-variables/main.c b/lldb/test/API/python_api/frame/get-variables/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/get-variables/main.c
rename to lldb/test/API/python_api/frame/get-variables/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/Makefile b/lldb/test/API/python_api/frame/inlines/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/Makefile
rename to lldb/test/API/python_api/frame/inlines/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/TestInlinedFrame.py b/lldb/test/API/python_api/frame/inlines/TestInlinedFrame.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/TestInlinedFrame.py
rename to lldb/test/API/python_api/frame/inlines/TestInlinedFrame.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/inlines.c b/lldb/test/API/python_api/frame/inlines/inlines.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/inlines.c
rename to lldb/test/API/python_api/frame/inlines/inlines.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/inlines.h b/lldb/test/API/python_api/frame/inlines/inlines.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/inlines.h
rename to lldb/test/API/python_api/frame/inlines/inlines.h
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/main.c b/lldb/test/API/python_api/frame/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/frame/main.c
rename to lldb/test/API/python_api/frame/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/function_symbol/Makefile b/lldb/test/API/python_api/function_symbol/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/function_symbol/Makefile
rename to lldb/test/API/python_api/function_symbol/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/function_symbol/TestDisasmAPI.py b/lldb/test/API/python_api/function_symbol/TestDisasmAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/function_symbol/TestDisasmAPI.py
rename to lldb/test/API/python_api/function_symbol/TestDisasmAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/function_symbol/TestSymbolAPI.py b/lldb/test/API/python_api/function_symbol/TestSymbolAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/function_symbol/TestSymbolAPI.py
rename to lldb/test/API/python_api/function_symbol/TestSymbolAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/function_symbol/main.c b/lldb/test/API/python_api/function_symbol/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/function_symbol/main.c
rename to lldb/test/API/python_api/function_symbol/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/get-value-32bit-int/Makefile b/lldb/test/API/python_api/get-value-32bit-int/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/get-value-32bit-int/Makefile
rename to lldb/test/API/python_api/get-value-32bit-int/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/get-value-32bit-int/TestGetValue32BitInt.py b/lldb/test/API/python_api/get-value-32bit-int/TestGetValue32BitInt.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/get-value-32bit-int/TestGetValue32BitInt.py
rename to lldb/test/API/python_api/get-value-32bit-int/TestGetValue32BitInt.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/get-value-32bit-int/main.cpp b/lldb/test/API/python_api/get-value-32bit-int/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/get-value-32bit-int/main.cpp
rename to lldb/test/API/python_api/get-value-32bit-int/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/hello_world/Makefile b/lldb/test/API/python_api/hello_world/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/hello_world/Makefile
rename to lldb/test/API/python_api/hello_world/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/hello_world/TestHelloWorld.py b/lldb/test/API/python_api/hello_world/TestHelloWorld.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/hello_world/TestHelloWorld.py
rename to lldb/test/API/python_api/hello_world/TestHelloWorld.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/hello_world/main.c b/lldb/test/API/python_api/hello_world/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/hello_world/main.c
rename to lldb/test/API/python_api/hello_world/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/interpreter/Makefile b/lldb/test/API/python_api/interpreter/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/interpreter/Makefile
rename to lldb/test/API/python_api/interpreter/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/interpreter/TestCommandInterpreterAPI.py b/lldb/test/API/python_api/interpreter/TestCommandInterpreterAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/interpreter/TestCommandInterpreterAPI.py
rename to lldb/test/API/python_api/interpreter/TestCommandInterpreterAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/interpreter/TestRunCommandInterpreterAPI.py b/lldb/test/API/python_api/interpreter/TestRunCommandInterpreterAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/interpreter/TestRunCommandInterpreterAPI.py
rename to lldb/test/API/python_api/interpreter/TestRunCommandInterpreterAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/interpreter/main.c b/lldb/test/API/python_api/interpreter/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/interpreter/main.c
rename to lldb/test/API/python_api/interpreter/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/TestSwigVersion.py b/lldb/test/API/python_api/lldbutil/TestSwigVersion.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/TestSwigVersion.py
rename to lldb/test/API/python_api/lldbutil/TestSwigVersion.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/frame/Makefile b/lldb/test/API/python_api/lldbutil/frame/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/frame/Makefile
rename to lldb/test/API/python_api/lldbutil/frame/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/frame/TestFrameUtils.py b/lldb/test/API/python_api/lldbutil/frame/TestFrameUtils.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/frame/TestFrameUtils.py
rename to lldb/test/API/python_api/lldbutil/frame/TestFrameUtils.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/frame/main.c b/lldb/test/API/python_api/lldbutil/frame/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/frame/main.c
rename to lldb/test/API/python_api/lldbutil/frame/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/Makefile b/lldb/test/API/python_api/lldbutil/iter/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/Makefile
rename to lldb/test/API/python_api/lldbutil/iter/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/TestLLDBIterator.py b/lldb/test/API/python_api/lldbutil/iter/TestLLDBIterator.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/TestLLDBIterator.py
rename to lldb/test/API/python_api/lldbutil/iter/TestLLDBIterator.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/TestRegistersIterator.py b/lldb/test/API/python_api/lldbutil/iter/TestRegistersIterator.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/TestRegistersIterator.py
rename to lldb/test/API/python_api/lldbutil/iter/TestRegistersIterator.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/main.cpp b/lldb/test/API/python_api/lldbutil/iter/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/main.cpp
rename to lldb/test/API/python_api/lldbutil/iter/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/Makefile b/lldb/test/API/python_api/lldbutil/process/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/Makefile
rename to lldb/test/API/python_api/lldbutil/process/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/TestPrintStackTraces.py b/lldb/test/API/python_api/lldbutil/process/TestPrintStackTraces.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/TestPrintStackTraces.py
rename to lldb/test/API/python_api/lldbutil/process/TestPrintStackTraces.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/main.cpp b/lldb/test/API/python_api/lldbutil/process/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/main.cpp
rename to lldb/test/API/python_api/lldbutil/process/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/module_section/Makefile b/lldb/test/API/python_api/module_section/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/module_section/Makefile
rename to lldb/test/API/python_api/module_section/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/module_section/TestModuleAndSection.py b/lldb/test/API/python_api/module_section/TestModuleAndSection.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/module_section/TestModuleAndSection.py
rename to lldb/test/API/python_api/module_section/TestModuleAndSection.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/module_section/b.cpp b/lldb/test/API/python_api/module_section/b.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/module_section/b.cpp
rename to lldb/test/API/python_api/module_section/b.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/module_section/c.cpp b/lldb/test/API/python_api/module_section/c.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/module_section/c.cpp
rename to lldb/test/API/python_api/module_section/c.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/module_section/main.cpp b/lldb/test/API/python_api/module_section/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/module_section/main.cpp
rename to lldb/test/API/python_api/module_section/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/name_lookup/Makefile b/lldb/test/API/python_api/name_lookup/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/name_lookup/Makefile
rename to lldb/test/API/python_api/name_lookup/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/name_lookup/TestNameLookup.py b/lldb/test/API/python_api/name_lookup/TestNameLookup.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/name_lookup/TestNameLookup.py
rename to lldb/test/API/python_api/name_lookup/TestNameLookup.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/name_lookup/main.cpp b/lldb/test/API/python_api/name_lookup/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/name_lookup/main.cpp
rename to lldb/test/API/python_api/name_lookup/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/objc_type/Makefile b/lldb/test/API/python_api/objc_type/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/objc_type/Makefile
rename to lldb/test/API/python_api/objc_type/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/objc_type/TestObjCType.py b/lldb/test/API/python_api/objc_type/TestObjCType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/objc_type/TestObjCType.py
rename to lldb/test/API/python_api/objc_type/TestObjCType.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/objc_type/main.m b/lldb/test/API/python_api/objc_type/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/objc_type/main.m
rename to lldb/test/API/python_api/objc_type/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/Makefile b/lldb/test/API/python_api/process/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/Makefile
rename to lldb/test/API/python_api/process/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/TestProcessAPI.py b/lldb/test/API/python_api/process/TestProcessAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/TestProcessAPI.py
rename to lldb/test/API/python_api/process/TestProcessAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/io/Makefile b/lldb/test/API/python_api/process/io/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/io/Makefile
rename to lldb/test/API/python_api/process/io/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/io/TestProcessIO.py b/lldb/test/API/python_api/process/io/TestProcessIO.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/io/TestProcessIO.py
rename to lldb/test/API/python_api/process/io/TestProcessIO.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/io/main.c b/lldb/test/API/python_api/process/io/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/io/main.c
rename to lldb/test/API/python_api/process/io/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/main.cpp b/lldb/test/API/python_api/process/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/main.cpp
rename to lldb/test/API/python_api/process/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/read-mem-cstring/Makefile b/lldb/test/API/python_api/process/read-mem-cstring/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/read-mem-cstring/Makefile
rename to lldb/test/API/python_api/process/read-mem-cstring/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/read-mem-cstring/TestReadMemCString.py b/lldb/test/API/python_api/process/read-mem-cstring/TestReadMemCString.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/read-mem-cstring/TestReadMemCString.py
rename to lldb/test/API/python_api/process/read-mem-cstring/TestReadMemCString.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/read-mem-cstring/main.c b/lldb/test/API/python_api/process/read-mem-cstring/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/process/read-mem-cstring/main.c
rename to lldb/test/API/python_api/process/read-mem-cstring/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbdata/Makefile b/lldb/test/API/python_api/sbdata/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbdata/Makefile
rename to lldb/test/API/python_api/sbdata/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbdata/TestSBData.py b/lldb/test/API/python_api/sbdata/TestSBData.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbdata/TestSBData.py
rename to lldb/test/API/python_api/sbdata/TestSBData.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbdata/main.cpp b/lldb/test/API/python_api/sbdata/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbdata/main.cpp
rename to lldb/test/API/python_api/sbdata/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sblaunchinfo/TestSBLaunchInfo.py b/lldb/test/API/python_api/sblaunchinfo/TestSBLaunchInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sblaunchinfo/TestSBLaunchInfo.py
rename to lldb/test/API/python_api/sblaunchinfo/TestSBLaunchInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbstructureddata/TestStructuredDataAPI.py b/lldb/test/API/python_api/sbstructureddata/TestStructuredDataAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbstructureddata/TestStructuredDataAPI.py
rename to lldb/test/API/python_api/sbstructureddata/TestStructuredDataAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbtype_typeclass/TestSBTypeTypeClass.py b/lldb/test/API/python_api/sbtype_typeclass/TestSBTypeTypeClass.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbtype_typeclass/TestSBTypeTypeClass.py
rename to lldb/test/API/python_api/sbtype_typeclass/TestSBTypeTypeClass.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbtype_typeclass/main.m b/lldb/test/API/python_api/sbtype_typeclass/main.m
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbtype_typeclass/main.m
rename to lldb/test/API/python_api/sbtype_typeclass/main.m
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/TestSBValueConstAddrOf.py b/lldb/test/API/python_api/sbvalue_const_addrof/TestSBValueConstAddrOf.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/TestSBValueConstAddrOf.py
rename to lldb/test/API/python_api/sbvalue_const_addrof/TestSBValueConstAddrOf.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/main.cpp b/lldb/test/API/python_api/sbvalue_const_addrof/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/main.cpp
rename to lldb/test/API/python_api/sbvalue_const_addrof/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_persist/Makefile b/lldb/test/API/python_api/sbvalue_persist/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbvalue_persist/Makefile
rename to lldb/test/API/python_api/sbvalue_persist/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_persist/TestSBValuePersist.py b/lldb/test/API/python_api/sbvalue_persist/TestSBValuePersist.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbvalue_persist/TestSBValuePersist.py
rename to lldb/test/API/python_api/sbvalue_persist/TestSBValuePersist.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_persist/main.cpp b/lldb/test/API/python_api/sbvalue_persist/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/sbvalue_persist/main.cpp
rename to lldb/test/API/python_api/sbvalue_persist/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/section/Makefile b/lldb/test/API/python_api/section/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/section/Makefile
rename to lldb/test/API/python_api/section/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/section/TestSectionAPI.py b/lldb/test/API/python_api/section/TestSectionAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/section/TestSectionAPI.py
rename to lldb/test/API/python_api/section/TestSectionAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/section/main.c b/lldb/test/API/python_api/section/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/section/main.c
rename to lldb/test/API/python_api/section/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/signals/Makefile b/lldb/test/API/python_api/signals/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/signals/Makefile
rename to lldb/test/API/python_api/signals/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/signals/TestSignalsAPI.py b/lldb/test/API/python_api/signals/TestSignalsAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/signals/TestSignalsAPI.py
rename to lldb/test/API/python_api/signals/TestSignalsAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/signals/main.cpp b/lldb/test/API/python_api/signals/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/signals/main.cpp
rename to lldb/test/API/python_api/signals/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/Makefile b/lldb/test/API/python_api/symbol-context/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/Makefile
rename to lldb/test/API/python_api/symbol-context/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/TestSymbolContext.py b/lldb/test/API/python_api/symbol-context/TestSymbolContext.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/TestSymbolContext.py
rename to lldb/test/API/python_api/symbol-context/TestSymbolContext.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/main.c b/lldb/test/API/python_api/symbol-context/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/main.c
rename to lldb/test/API/python_api/symbol-context/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/Makefile b/lldb/test/API/python_api/symbol-context/two-files/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/Makefile
rename to lldb/test/API/python_api/symbol-context/two-files/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/TestSymbolContextTwoFiles.py b/lldb/test/API/python_api/symbol-context/two-files/TestSymbolContextTwoFiles.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/TestSymbolContextTwoFiles.py
rename to lldb/test/API/python_api/symbol-context/two-files/TestSymbolContextTwoFiles.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/decls.h b/lldb/test/API/python_api/symbol-context/two-files/decls.h
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/decls.h
rename to lldb/test/API/python_api/symbol-context/two-files/decls.h
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/file1.cpp b/lldb/test/API/python_api/symbol-context/two-files/file1.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/file1.cpp
rename to lldb/test/API/python_api/symbol-context/two-files/file1.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/file2.cpp b/lldb/test/API/python_api/symbol-context/two-files/file2.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/symbol-context/two-files/file2.cpp
rename to lldb/test/API/python_api/symbol-context/two-files/file2.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/target/Makefile b/lldb/test/API/python_api/target/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/target/Makefile
rename to lldb/test/API/python_api/target/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/target/TestTargetAPI.py b/lldb/test/API/python_api/target/TestTargetAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/target/TestTargetAPI.py
rename to lldb/test/API/python_api/target/TestTargetAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/target/main.c b/lldb/test/API/python_api/target/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/target/main.c
rename to lldb/test/API/python_api/target/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/thread/Makefile b/lldb/test/API/python_api/thread/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/thread/Makefile
rename to lldb/test/API/python_api/thread/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/thread/TestThreadAPI.py b/lldb/test/API/python_api/thread/TestThreadAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/thread/TestThreadAPI.py
rename to lldb/test/API/python_api/thread/TestThreadAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/thread/main.cpp b/lldb/test/API/python_api/thread/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/thread/main.cpp
rename to lldb/test/API/python_api/thread/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/thread/main2.cpp b/lldb/test/API/python_api/thread/main2.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/thread/main2.cpp
rename to lldb/test/API/python_api/thread/main2.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/type/Makefile b/lldb/test/API/python_api/type/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/type/Makefile
rename to lldb/test/API/python_api/type/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/type/TestTypeList.py b/lldb/test/API/python_api/type/TestTypeList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/type/TestTypeList.py
rename to lldb/test/API/python_api/type/TestTypeList.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/type/main.cpp b/lldb/test/API/python_api/type/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/type/main.cpp
rename to lldb/test/API/python_api/type/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/Makefile b/lldb/test/API/python_api/value/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/Makefile
rename to lldb/test/API/python_api/value/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/TestValueAPI.py b/lldb/test/API/python_api/value/TestValueAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/TestValueAPI.py
rename to lldb/test/API/python_api/value/TestValueAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/change_values/Makefile b/lldb/test/API/python_api/value/change_values/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/change_values/Makefile
rename to lldb/test/API/python_api/value/change_values/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/change_values/TestChangeValueAPI.py b/lldb/test/API/python_api/value/change_values/TestChangeValueAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/change_values/TestChangeValueAPI.py
rename to lldb/test/API/python_api/value/change_values/TestChangeValueAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/change_values/main.c b/lldb/test/API/python_api/value/change_values/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/change_values/main.c
rename to lldb/test/API/python_api/value/change_values/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/empty_class/Makefile b/lldb/test/API/python_api/value/empty_class/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/empty_class/Makefile
rename to lldb/test/API/python_api/value/empty_class/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/empty_class/TestValueAPIEmptyClass.py b/lldb/test/API/python_api/value/empty_class/TestValueAPIEmptyClass.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/empty_class/TestValueAPIEmptyClass.py
rename to lldb/test/API/python_api/value/empty_class/TestValueAPIEmptyClass.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/empty_class/main.cpp b/lldb/test/API/python_api/value/empty_class/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/empty_class/main.cpp
rename to lldb/test/API/python_api/value/empty_class/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/linked_list/Makefile b/lldb/test/API/python_api/value/linked_list/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/linked_list/Makefile
rename to lldb/test/API/python_api/value/linked_list/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/linked_list/TestValueAPILinkedList.py b/lldb/test/API/python_api/value/linked_list/TestValueAPILinkedList.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/linked_list/TestValueAPILinkedList.py
rename to lldb/test/API/python_api/value/linked_list/TestValueAPILinkedList.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/linked_list/main.cpp b/lldb/test/API/python_api/value/linked_list/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/linked_list/main.cpp
rename to lldb/test/API/python_api/value/linked_list/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value/main.c b/lldb/test/API/python_api/value/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value/main.c
rename to lldb/test/API/python_api/value/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/Makefile b/lldb/test/API/python_api/value_var_update/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value_var_update/Makefile
rename to lldb/test/API/python_api/value_var_update/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/TestValueVarUpdate.py b/lldb/test/API/python_api/value_var_update/TestValueVarUpdate.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value_var_update/TestValueVarUpdate.py
rename to lldb/test/API/python_api/value_var_update/TestValueVarUpdate.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/main.c b/lldb/test/API/python_api/value_var_update/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/value_var_update/main.c
rename to lldb/test/API/python_api/value_var_update/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/.categories b/lldb/test/API/python_api/watchpoint/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/.categories
rename to lldb/test/API/python_api/watchpoint/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/Makefile b/lldb/test/API/python_api/watchpoint/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/Makefile
rename to lldb/test/API/python_api/watchpoint/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/TestSetWatchpoint.py b/lldb/test/API/python_api/watchpoint/TestSetWatchpoint.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/TestSetWatchpoint.py
rename to lldb/test/API/python_api/watchpoint/TestSetWatchpoint.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/TestWatchpointIgnoreCount.py b/lldb/test/API/python_api/watchpoint/TestWatchpointIgnoreCount.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/TestWatchpointIgnoreCount.py
rename to lldb/test/API/python_api/watchpoint/TestWatchpointIgnoreCount.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/TestWatchpointIter.py b/lldb/test/API/python_api/watchpoint/TestWatchpointIter.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/TestWatchpointIter.py
rename to lldb/test/API/python_api/watchpoint/TestWatchpointIter.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/condition/Makefile b/lldb/test/API/python_api/watchpoint/condition/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/condition/Makefile
rename to lldb/test/API/python_api/watchpoint/condition/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/condition/TestWatchpointConditionAPI.py b/lldb/test/API/python_api/watchpoint/condition/TestWatchpointConditionAPI.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/condition/TestWatchpointConditionAPI.py
rename to lldb/test/API/python_api/watchpoint/condition/TestWatchpointConditionAPI.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/condition/main.cpp b/lldb/test/API/python_api/watchpoint/condition/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/condition/main.cpp
rename to lldb/test/API/python_api/watchpoint/condition/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/main.c b/lldb/test/API/python_api/watchpoint/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/main.c
rename to lldb/test/API/python_api/watchpoint/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/Makefile b/lldb/test/API/python_api/watchpoint/watchlocation/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/Makefile
rename to lldb/test/API/python_api/watchpoint/watchlocation/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/TestSetWatchlocation.py b/lldb/test/API/python_api/watchpoint/watchlocation/TestSetWatchlocation.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/TestSetWatchlocation.py
rename to lldb/test/API/python_api/watchpoint/watchlocation/TestSetWatchlocation.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
rename to lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/main.cpp b/lldb/test/API/python_api/watchpoint/watchlocation/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/python_api/watchpoint/watchlocation/main.cpp
rename to lldb/test/API/python_api/watchpoint/watchlocation/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/sample_test/Makefile b/lldb/test/API/sample_test/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/sample_test/Makefile
rename to lldb/test/API/sample_test/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/sample_test/TestSampleInlineTest.py b/lldb/test/API/sample_test/TestSampleInlineTest.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/sample_test/TestSampleInlineTest.py
rename to lldb/test/API/sample_test/TestSampleInlineTest.py
diff --git a/lldb/packages/Python/lldbsuite/test/sample_test/TestSampleTest.py b/lldb/test/API/sample_test/TestSampleTest.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/sample_test/TestSampleTest.py
rename to lldb/test/API/sample_test/TestSampleTest.py
diff --git a/lldb/packages/Python/lldbsuite/test/sample_test/main.c b/lldb/test/API/sample_test/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/sample_test/main.c
rename to lldb/test/API/sample_test/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/sanity/TestModuleCacheSanity.py b/lldb/test/API/sanity/TestModuleCacheSanity.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/sanity/TestModuleCacheSanity.py
rename to lldb/test/API/sanity/TestModuleCacheSanity.py
diff --git a/lldb/packages/Python/lldbsuite/test/source-manager/Makefile b/lldb/test/API/source-manager/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/source-manager/Makefile
rename to lldb/test/API/source-manager/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/source-manager/TestSourceManager.py b/lldb/test/API/source-manager/TestSourceManager.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/source-manager/TestSourceManager.py
rename to lldb/test/API/source-manager/TestSourceManager.py
diff --git a/lldb/packages/Python/lldbsuite/test/source-manager/hidden/.keep b/lldb/test/API/source-manager/hidden/.keep
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/source-manager/hidden/.keep
rename to lldb/test/API/source-manager/hidden/.keep
diff --git a/lldb/packages/Python/lldbsuite/test/source-manager/main.c b/lldb/test/API/source-manager/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/source-manager/main.c
rename to lldb/test/API/source-manager/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/terminal/TestEditline.py b/lldb/test/API/terminal/TestEditline.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/terminal/TestEditline.py
rename to lldb/test/API/terminal/TestEditline.py
diff --git a/lldb/packages/Python/lldbsuite/test/terminal/TestSTTYBeforeAndAfter.py b/lldb/test/API/terminal/TestSTTYBeforeAndAfter.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/terminal/TestSTTYBeforeAndAfter.py
rename to lldb/test/API/terminal/TestSTTYBeforeAndAfter.py
diff --git a/lldb/packages/Python/lldbsuite/test/test_runner/test/__init__.py b/lldb/test/API/test_runner/test/__init__.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/test_runner/test/__init__.py
rename to lldb/test/API/test_runner/test/__init__.py
diff --git a/lldb/packages/Python/lldbsuite/test/test_runner/test/inferior.py b/lldb/test/API/test_runner/test/inferior.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/test_runner/test/inferior.py
rename to lldb/test/API/test_runner/test/inferior.py
diff --git a/lldb/packages/Python/lldbsuite/test/test_runner/test/test_process_control.py b/lldb/test/API/test_runner/test/test_process_control.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/test_runner/test/test_process_control.py
rename to lldb/test/API/test_runner/test/test_process_control.py
diff --git a/lldb/test/API/testcases b/lldb/test/API/testcases
deleted file mode 120000
index 370755973a7300188d18c6ee0123b1de81715d93..0000000000000000000000000000000000000000
--- a/lldb/test/API/testcases
+++ /dev/null
@@ -1 +0,0 @@
-../../packages/Python/lldbsuite/test
\ No newline at end of file
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/.clang-format b/lldb/test/API/tools/lldb-server/.clang-format
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/.clang-format
rename to lldb/test/API/tools/lldb-server/.clang-format
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/Makefile b/lldb/test/API/tools/lldb-server/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/Makefile
rename to lldb/test/API/tools/lldb-server/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestAppleSimulatorOSType.py b/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestAppleSimulatorOSType.py
rename to lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteAttach.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteAttach.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteAttach.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteAttach.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteAuxvSupport.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteAuxvSupport.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteAuxvSupport.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteAuxvSupport.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteExitCode.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteExitCode.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteExitCode.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteExitCode.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteHostInfo.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteHostInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteHostInfo.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteHostInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteKill.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteKill.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteKill.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteKill.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteModuleInfo.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteModuleInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteModuleInfo.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteModuleInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteProcessInfo.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteProcessInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteProcessInfo.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteProcessInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteRegisterState.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteRegisterState.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteRegisterState.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteRegisterState.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteSingleStep.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteSingleStep.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteSingleStep.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteSingleStep.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteThreadsInStopReply.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteThreadsInStopReply.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemoteThreadsInStopReply.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemoteThreadsInStopReply.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py b/lldb/test/API/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemote_vCont.py b/lldb/test/API/tools/lldb-server/TestGdbRemote_vCont.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemote_vCont.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemote_vCont.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemote_vContThreads.py b/lldb/test/API/tools/lldb-server/TestGdbRemote_vContThreads.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestGdbRemote_vContThreads.py
rename to lldb/test/API/tools/lldb-server/TestGdbRemote_vContThreads.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestLldbGdbServer.py b/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/TestLldbGdbServer.py
rename to lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/commandline/TestStubReverseConnect.py b/lldb/test/API/tools/lldb-server/commandline/TestStubReverseConnect.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/commandline/TestStubReverseConnect.py
rename to lldb/test/API/tools/lldb-server/commandline/TestStubReverseConnect.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/commandline/TestStubSetSID.py b/lldb/test/API/tools/lldb-server/commandline/TestStubSetSID.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/commandline/TestStubSetSID.py
rename to lldb/test/API/tools/lldb-server/commandline/TestStubSetSID.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/Makefile b/lldb/test/API/tools/lldb-server/inferior-crash/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/Makefile
rename to lldb/test/API/tools/lldb-server/inferior-crash/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/TestGdbRemoteAbort.py b/lldb/test/API/tools/lldb-server/inferior-crash/TestGdbRemoteAbort.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/TestGdbRemoteAbort.py
rename to lldb/test/API/tools/lldb-server/inferior-crash/TestGdbRemoteAbort.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/TestGdbRemoteSegFault.py b/lldb/test/API/tools/lldb-server/inferior-crash/TestGdbRemoteSegFault.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/TestGdbRemoteSegFault.py
rename to lldb/test/API/tools/lldb-server/inferior-crash/TestGdbRemoteSegFault.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/main.cpp b/lldb/test/API/tools/lldb-server/inferior-crash/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/main.cpp
rename to lldb/test/API/tools/lldb-server/inferior-crash/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/Makefile b/lldb/test/API/tools/lldb-server/libraries-svr4/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/Makefile
rename to lldb/test/API/tools/lldb-server/libraries-svr4/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py b/lldb/test/API/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py
rename to lldb/test/API/tools/lldb-server/libraries-svr4/TestGdbRemoteLibrariesSvr4Support.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/main.cpp b/lldb/test/API/tools/lldb-server/libraries-svr4/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/main.cpp
rename to lldb/test/API/tools/lldb-server/libraries-svr4/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/svr4lib_a.cpp b/lldb/test/API/tools/lldb-server/libraries-svr4/svr4lib_a.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/svr4lib_a.cpp
rename to lldb/test/API/tools/lldb-server/libraries-svr4/svr4lib_a.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/svr4lib_b_quote.cpp b/lldb/test/API/tools/lldb-server/libraries-svr4/svr4lib_b_quote.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/libraries-svr4/svr4lib_b_quote.cpp
rename to lldb/test/API/tools/lldb-server/libraries-svr4/svr4lib_b_quote.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/main.cpp b/lldb/test/API/tools/lldb-server/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/main.cpp
rename to lldb/test/API/tools/lldb-server/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/platform-process-connect/Makefile b/lldb/test/API/tools/lldb-server/platform-process-connect/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/platform-process-connect/Makefile
rename to lldb/test/API/tools/lldb-server/platform-process-connect/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py b/lldb/test/API/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py
rename to lldb/test/API/tools/lldb-server/platform-process-connect/TestPlatformProcessConnect.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/platform-process-connect/main.cpp b/lldb/test/API/tools/lldb-server/platform-process-connect/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/platform-process-connect/main.cpp
rename to lldb/test/API/tools/lldb-server/platform-process-connect/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/Makefile b/lldb/test/API/tools/lldb-server/register-reading/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/Makefile
rename to lldb/test/API/tools/lldb-server/register-reading/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py b/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
rename to lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/main.cpp b/lldb/test/API/tools/lldb-server/register-reading/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/register-reading/main.cpp
rename to lldb/test/API/tools/lldb-server/register-reading/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/signal-filtering/Makefile b/lldb/test/API/tools/lldb-server/signal-filtering/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/signal-filtering/Makefile
rename to lldb/test/API/tools/lldb-server/signal-filtering/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/signal-filtering/TestGdbRemote_QPassSignals.py b/lldb/test/API/tools/lldb-server/signal-filtering/TestGdbRemote_QPassSignals.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/signal-filtering/TestGdbRemote_QPassSignals.py
rename to lldb/test/API/tools/lldb-server/signal-filtering/TestGdbRemote_QPassSignals.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/signal-filtering/main.cpp b/lldb/test/API/tools/lldb-server/signal-filtering/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/signal-filtering/main.cpp
rename to lldb/test/API/tools/lldb-server/signal-filtering/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/test/test_lldbgdbserverutils.py b/lldb/test/API/tools/lldb-server/test/test_lldbgdbserverutils.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/test/test_lldbgdbserverutils.py
rename to lldb/test/API/tools/lldb-server/test/test_lldbgdbserverutils.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/thread-name/Makefile b/lldb/test/API/tools/lldb-server/thread-name/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/thread-name/Makefile
rename to lldb/test/API/tools/lldb-server/thread-name/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/thread-name/TestGdbRemoteThreadName.py b/lldb/test/API/tools/lldb-server/thread-name/TestGdbRemoteThreadName.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/thread-name/TestGdbRemoteThreadName.py
rename to lldb/test/API/tools/lldb-server/thread-name/TestGdbRemoteThreadName.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/thread-name/main.cpp b/lldb/test/API/tools/lldb-server/thread-name/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-server/thread-name/main.cpp
rename to lldb/test/API/tools/lldb-server/thread-name/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/.categories b/lldb/test/API/tools/lldb-vscode/.categories
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/.categories
rename to lldb/test/API/tools/lldb-vscode/.categories
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/attach/Makefile b/lldb/test/API/tools/lldb-vscode/attach/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/attach/Makefile
rename to lldb/test/API/tools/lldb-vscode/attach/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/attach/TestVSCode_attach.py b/lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/attach/TestVSCode_attach.py
rename to lldb/test/API/tools/lldb-vscode/attach/TestVSCode_attach.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/attach/main.c b/lldb/test/API/tools/lldb-vscode/attach/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/attach/main.c
rename to lldb/test/API/tools/lldb-vscode/attach/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/Makefile b/lldb/test/API/tools/lldb-vscode/breakpoint/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/Makefile
rename to lldb/test/API/tools/lldb-vscode/breakpoint/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
rename to lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py
rename to lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setExceptionBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py
rename to lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setFunctionBreakpoints.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/main.cpp b/lldb/test/API/tools/lldb-vscode/breakpoint/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/breakpoint/main.cpp
rename to lldb/test/API/tools/lldb-vscode/breakpoint/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/completions/Makefile b/lldb/test/API/tools/lldb-vscode/completions/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/completions/Makefile
rename to lldb/test/API/tools/lldb-vscode/completions/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/completions/TestVSCode_completions.py b/lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/completions/TestVSCode_completions.py
rename to lldb/test/API/tools/lldb-vscode/completions/TestVSCode_completions.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/completions/main.cpp b/lldb/test/API/tools/lldb-vscode/completions/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/completions/main.cpp
rename to lldb/test/API/tools/lldb-vscode/completions/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/launch/Makefile b/lldb/test/API/tools/lldb-vscode/launch/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/launch/Makefile
rename to lldb/test/API/tools/lldb-vscode/launch/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/launch/TestVSCode_launch.py b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/launch/TestVSCode_launch.py
rename to lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/launch/main.c b/lldb/test/API/tools/lldb-vscode/launch/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/launch/main.c
rename to lldb/test/API/tools/lldb-vscode/launch/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/stackTrace/Makefile b/lldb/test/API/tools/lldb-vscode/stackTrace/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/stackTrace/Makefile
rename to lldb/test/API/tools/lldb-vscode/stackTrace/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py b/lldb/test/API/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py
rename to lldb/test/API/tools/lldb-vscode/stackTrace/TestVSCode_stackTrace.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/stackTrace/main.c b/lldb/test/API/tools/lldb-vscode/stackTrace/main.c
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/stackTrace/main.c
rename to lldb/test/API/tools/lldb-vscode/stackTrace/main.c
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/step/Makefile b/lldb/test/API/tools/lldb-vscode/step/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/step/Makefile
rename to lldb/test/API/tools/lldb-vscode/step/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/step/TestVSCode_step.py b/lldb/test/API/tools/lldb-vscode/step/TestVSCode_step.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/step/TestVSCode_step.py
rename to lldb/test/API/tools/lldb-vscode/step/TestVSCode_step.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/step/main.cpp b/lldb/test/API/tools/lldb-vscode/step/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/step/main.cpp
rename to lldb/test/API/tools/lldb-vscode/step/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/variables/Makefile b/lldb/test/API/tools/lldb-vscode/variables/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/variables/Makefile
rename to lldb/test/API/tools/lldb-vscode/variables/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/variables/TestVSCode_variables.py b/lldb/test/API/tools/lldb-vscode/variables/TestVSCode_variables.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/variables/TestVSCode_variables.py
rename to lldb/test/API/tools/lldb-vscode/variables/TestVSCode_variables.py
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/variables/main.cpp b/lldb/test/API/tools/lldb-vscode/variables/main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/tools/lldb-vscode/variables/main.cpp
rename to lldb/test/API/tools/lldb-vscode/variables/main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/AbstractBase.py b/lldb/test/API/types/AbstractBase.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/AbstractBase.py
rename to lldb/test/API/types/AbstractBase.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/HideTestFailures.py b/lldb/test/API/types/HideTestFailures.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/HideTestFailures.py
rename to lldb/test/API/types/HideTestFailures.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/Makefile b/lldb/test/API/types/Makefile
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/Makefile
rename to lldb/test/API/types/Makefile
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestCharType.py b/lldb/test/API/types/TestCharType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestCharType.py
rename to lldb/test/API/types/TestCharType.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestCharTypeExpr.py b/lldb/test/API/types/TestCharTypeExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestCharTypeExpr.py
rename to lldb/test/API/types/TestCharTypeExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestDoubleTypes.py b/lldb/test/API/types/TestDoubleTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestDoubleTypes.py
rename to lldb/test/API/types/TestDoubleTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestDoubleTypesExpr.py b/lldb/test/API/types/TestDoubleTypesExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestDoubleTypesExpr.py
rename to lldb/test/API/types/TestDoubleTypesExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestFloatTypes.py b/lldb/test/API/types/TestFloatTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestFloatTypes.py
rename to lldb/test/API/types/TestFloatTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestFloatTypesExpr.py b/lldb/test/API/types/TestFloatTypesExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestFloatTypesExpr.py
rename to lldb/test/API/types/TestFloatTypesExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestIntegerType.py b/lldb/test/API/types/TestIntegerType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestIntegerType.py
rename to lldb/test/API/types/TestIntegerType.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestIntegerTypeExpr.py b/lldb/test/API/types/TestIntegerTypeExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestIntegerTypeExpr.py
rename to lldb/test/API/types/TestIntegerTypeExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestLongTypes.py b/lldb/test/API/types/TestLongTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestLongTypes.py
rename to lldb/test/API/types/TestLongTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestLongTypesExpr.py b/lldb/test/API/types/TestLongTypesExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestLongTypesExpr.py
rename to lldb/test/API/types/TestLongTypesExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestRecursiveTypes.py b/lldb/test/API/types/TestRecursiveTypes.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestRecursiveTypes.py
rename to lldb/test/API/types/TestRecursiveTypes.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestShortType.py b/lldb/test/API/types/TestShortType.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestShortType.py
rename to lldb/test/API/types/TestShortType.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/TestShortTypeExpr.py b/lldb/test/API/types/TestShortTypeExpr.py
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/TestShortTypeExpr.py
rename to lldb/test/API/types/TestShortTypeExpr.py
diff --git a/lldb/packages/Python/lldbsuite/test/types/basic_type.cpp b/lldb/test/API/types/basic_type.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/basic_type.cpp
rename to lldb/test/API/types/basic_type.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/char.cpp b/lldb/test/API/types/char.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/char.cpp
rename to lldb/test/API/types/char.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/double.cpp b/lldb/test/API/types/double.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/double.cpp
rename to lldb/test/API/types/double.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/float.cpp b/lldb/test/API/types/float.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/float.cpp
rename to lldb/test/API/types/float.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/int.cpp b/lldb/test/API/types/int.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/int.cpp
rename to lldb/test/API/types/int.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/long.cpp b/lldb/test/API/types/long.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/long.cpp
rename to lldb/test/API/types/long.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/long_long.cpp b/lldb/test/API/types/long_long.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/long_long.cpp
rename to lldb/test/API/types/long_long.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/recursive_type_1.cpp b/lldb/test/API/types/recursive_type_1.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/recursive_type_1.cpp
rename to lldb/test/API/types/recursive_type_1.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/recursive_type_2.cpp b/lldb/test/API/types/recursive_type_2.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/recursive_type_2.cpp
rename to lldb/test/API/types/recursive_type_2.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/recursive_type_main.cpp b/lldb/test/API/types/recursive_type_main.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/recursive_type_main.cpp
rename to lldb/test/API/types/recursive_type_main.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/short.cpp b/lldb/test/API/types/short.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/short.cpp
rename to lldb/test/API/types/short.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/unsigned_char.cpp b/lldb/test/API/types/unsigned_char.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/unsigned_char.cpp
rename to lldb/test/API/types/unsigned_char.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/unsigned_int.cpp b/lldb/test/API/types/unsigned_int.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/unsigned_int.cpp
rename to lldb/test/API/types/unsigned_int.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/unsigned_long.cpp b/lldb/test/API/types/unsigned_long.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/unsigned_long.cpp
rename to lldb/test/API/types/unsigned_long.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/unsigned_long_long.cpp b/lldb/test/API/types/unsigned_long_long.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/unsigned_long_long.cpp
rename to lldb/test/API/types/unsigned_long_long.cpp
diff --git a/lldb/packages/Python/lldbsuite/test/types/unsigned_short.cpp b/lldb/test/API/types/unsigned_short.cpp
similarity index 100%
rename from lldb/packages/Python/lldbsuite/test/types/unsigned_short.cpp
rename to lldb/test/API/types/unsigned_short.cpp
diff --git a/lldb/tools/debugserver/source/RNBRemote.cpp b/lldb/tools/debugserver/source/RNBRemote.cpp
index 87aca91c00c4dda07601513c835e038831646170..8eed06381d3a0c7f11414c5edc14fb7e8e6c82bb 100644
--- a/lldb/tools/debugserver/source/RNBRemote.cpp
+++ b/lldb/tools/debugserver/source/RNBRemote.cpp
@@ -4076,6 +4076,16 @@ rnb_err_t RNBRemote::HandlePacket_v(const char *p) {
 
       std::string error_explainer = "attach failed";
       if (err_str[0] != '\0') {
+        // This is not a super helpful message for end users
+        if (strcmp (err_str, "unable to start the exception thread") == 0) {
+          snprintf (err_str, sizeof (err_str) - 1,
+                    "Not allowed to attach to process.  Look in the console "
+                    "messages (Console.app), near the debugserver entries "
+                    "when the attached failed.  The subsystem that denied "
+                    "the attach permission will likely have logged an "
+                    "informative message about why it was denied.");
+          err_str[sizeof (err_str) - 1] = '\0';
+        }
         error_explainer += " (";
         error_explainer += err_str;
         error_explainer += ")";
diff --git a/lldb/tools/lldb-test/SystemInitializerTest.cpp b/lldb/tools/lldb-test/SystemInitializerTest.cpp
index 3d8c80db55002bf03f67f9bae99c3d77396d8925..a9b4ba67c2caeeae1ee6f177dc6826b6d0865891 100644
--- a/lldb/tools/lldb-test/SystemInitializerTest.cpp
+++ b/lldb/tools/lldb-test/SystemInitializerTest.cpp
@@ -46,7 +46,6 @@ LLDB_PLUGIN_DECLARE(PlatformNetBSD);
 LLDB_PLUGIN_DECLARE(PlatformOpenBSD);
 LLDB_PLUGIN_DECLARE(PlatformWindows);
 LLDB_PLUGIN_DECLARE(PlatformAndroid);
-LLDB_PLUGIN_DECLARE(PlatformRemoteiOS);
 LLDB_PLUGIN_DECLARE(PlatformMacOSX);
 LLDB_PLUGIN_DECLARE(TypeSystemClang);
 LLDB_PLUGIN_DECLARE(ArchitectureArm);
@@ -162,7 +161,6 @@ llvm::Error SystemInitializerTest::Initialize() {
   LLDB_PLUGIN_INITIALIZE(PlatformOpenBSD);
   LLDB_PLUGIN_INITIALIZE(PlatformWindows);
   LLDB_PLUGIN_INITIALIZE(PlatformAndroid);
-  LLDB_PLUGIN_INITIALIZE(PlatformRemoteiOS);
   LLDB_PLUGIN_INITIALIZE(PlatformMacOSX);
 
   // Initialize LLVM and Clang
@@ -338,7 +336,6 @@ void SystemInitializerTest::Terminate() {
   LLDB_PLUGIN_TERMINATE(PlatformOpenBSD);
   LLDB_PLUGIN_TERMINATE(PlatformWindows);
   LLDB_PLUGIN_TERMINATE(PlatformAndroid);
-  LLDB_PLUGIN_TERMINATE(PlatformRemoteiOS);
   LLDB_PLUGIN_TERMINATE(PlatformMacOSX);
 
   LLDB_PLUGIN_TERMINATE(ObjectFileBreakpad);
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 740523d6d70e40f893f13786d7152c3d79991638..2693f041811d9582e3a69cf077e904d16a09a770 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -1327,36 +1327,6 @@ function(add_benchmark benchmark_name)
   target_link_libraries(${benchmark_name} PRIVATE benchmark)
 endfunction()
 
-function(llvm_add_go_executable binary pkgpath)
-  cmake_parse_arguments(ARG "ALL" "" "DEPENDS;GOFLAGS" ${ARGN})
-
-  if(LLVM_BINDINGS MATCHES "go")
-    # FIXME: This should depend only on the libraries Go needs.
-    get_property(llvmlibs GLOBAL PROPERTY LLVM_LIBS)
-    set(binpath ${CMAKE_BINARY_DIR}/bin/${binary}${CMAKE_EXECUTABLE_SUFFIX})
-    set(cc "${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}")
-    set(cxx "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
-    set(cppflags "")
-    get_property(include_dirs DIRECTORY PROPERTY INCLUDE_DIRECTORIES)
-    foreach(d ${include_dirs})
-      set(cppflags "${cppflags} -I${d}")
-    endforeach(d)
-    set(ldflags "${CMAKE_EXE_LINKER_FLAGS}")
-    add_custom_command(OUTPUT ${binpath}
-      COMMAND ${CMAKE_BINARY_DIR}/bin/llvm-go "go=${GO_EXECUTABLE}" "cc=${cc}" "cxx=${cxx}" "cppflags=${cppflags}" "ldflags=${ldflags}" "packages=${LLVM_GO_PACKAGES}"
-              ${ARG_GOFLAGS} build -o ${binpath} ${pkgpath}
-      DEPENDS llvm-config ${CMAKE_BINARY_DIR}/bin/llvm-go${CMAKE_EXECUTABLE_SUFFIX}
-              ${llvmlibs} ${ARG_DEPENDS}
-      COMMENT "Building Go executable ${binary}"
-      VERBATIM)
-    if (ARG_ALL)
-      add_custom_target(${binary} ALL DEPENDS ${binpath})
-    else()
-      add_custom_target(${binary} DEPENDS ${binpath})
-    endif()
-  endif()
-endfunction()
-
 # This function canonicalize the CMake variables passed by names
 # from CMake boolean to 0/1 suitable for passing into Python or C++,
 # in place.
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 4cf6c726c53549f5706e166f8cb1d00ce686553b..2ade20b01c1d052309fa5f4fda1066c3586c7456 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -92,6 +92,10 @@ We explicitly avoid some standard facilities, like the I/O streams, and instead
 use LLVM's streams library (raw_ostream_). More detailed information on these
 subjects is available in the :doc:`ProgrammersManual`.
 
+For more information about LLVM's data structures and the tradeoffs they make,
+please consult [that section of the programmer's
+manual](https://llvm.org/docs/ProgrammersManual.html#picking-the-right-data-structure-for-a-task).
+
 Guidelines for Go code
 ----------------------
 
diff --git a/llvm/examples/Bye/CMakeLists.txt b/llvm/examples/Bye/CMakeLists.txt
index 60eadede8a8de3156ab094d6912013a8f2446071..3206f90d09166d36df7c9e72afe486e8af1a853b 100644
--- a/llvm/examples/Bye/CMakeLists.txt
+++ b/llvm/examples/Bye/CMakeLists.txt
@@ -7,10 +7,6 @@ add_llvm_pass_plugin(Bye
   DEPENDS
   intrinsics_gen
   BUILDTREE_ONLY
-  LINK_COMPONENTS
-    ipo
-    Core
-    Support
  )
 
 install(TARGETS ${name} RUNTIME DESTINATION examples)
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index ed25b2cd89f13188269f298f2ea91b064327f113..ce8fd6536b4b6fb4eac5aacaebed29d9bccee488 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -511,6 +511,7 @@ private:
   opStatus divideSpecials(const IEEEFloat &);
   opStatus multiplySpecials(const IEEEFloat &);
   opStatus modSpecials(const IEEEFloat &);
+  opStatus remainderSpecials(const IEEEFloat&);
 
   /// @}
 
@@ -853,8 +854,8 @@ public:
   APFloat(const fltSemantics &Semantics) : U(Semantics) {}
   APFloat(const fltSemantics &Semantics, StringRef S);
   APFloat(const fltSemantics &Semantics, integerPart I) : U(Semantics, I) {}
-  template <typename T, typename = typename std::enable_if<
-                            std::is_floating_point<T>::value>::type>
+  template <typename T,
+            typename = std::enable_if_t<std::is_floating_point<T>::value>>
   APFloat(const fltSemantics &Semantics, T V) = delete;
   // TODO: Remove this constructor. This isn't faster than the first one.
   APFloat(const fltSemantics &Semantics, uninitializedTag)
diff --git a/llvm/include/llvm/ADT/AllocatorList.h b/llvm/include/llvm/ADT/AllocatorList.h
index 405a2e4264dfbd2d6c477505ab175e6c5fef41b8..447d7a7538db654c86b719d7b13b638ea09e2cec 100644
--- a/llvm/include/llvm/ADT/AllocatorList.h
+++ b/llvm/include/llvm/ADT/AllocatorList.h
@@ -110,8 +110,8 @@ private:
 
     template <class OtherValueT, class OtherIteratorBase>
     IteratorImpl(const IteratorImpl<OtherValueT, OtherIteratorBase> &X,
-                 typename std::enable_if<std::is_convertible<
-                     OtherIteratorBase, IteratorBase>::value>::type * = nullptr)
+                 std::enable_if_t<std::is_convertible<
+                     OtherIteratorBase, IteratorBase>::value> * = nullptr)
         : base_type(X.wrapped()) {}
 
     ~IteratorImpl() = default;
diff --git a/llvm/include/llvm/ADT/Any.h b/llvm/include/llvm/ADT/Any.h
index 49657e02a9911059e8ac3e964fa9c5b7d7ab2d3b..9d819841e3fc10782a848a6cdf5de5dd443b95c0 100644
--- a/llvm/include/llvm/ADT/Any.h
+++ b/llvm/include/llvm/ADT/Any.h
@@ -59,26 +59,26 @@ public:
   // When T is Any or T is not copy-constructible we need to explicitly disable
   // the forwarding constructor so that the copy constructor gets selected
   // instead.
-  template <
-      typename T,
-      typename std::enable_if<
-          llvm::conjunction<
-              llvm::negation<std::is_same<typename std::decay<T>::type, Any>>,
-              // We also disable this overload when an `Any` object can be
-              // converted to the parameter type because in that case, this
-              // constructor may combine with that conversion during overload
-              // resolution for determining copy constructibility, and then
-              // when we try to determine copy constructibility below we may
-              // infinitely recurse. This is being evaluated by the standards
-              // committee as a potential DR in `std::any` as well, but we're
-              // going ahead and adopting it to work-around usage of `Any` with
-              // types that need to be implicitly convertible from an `Any`.
-              llvm::negation<std::is_convertible<Any, typename std::decay<T>::type>>,
-              std::is_copy_constructible<typename std::decay<T>::type>>::value,
-          int>::type = 0>
+  template <typename T,
+            std::enable_if_t<
+                llvm::conjunction<
+                    llvm::negation<std::is_same<std::decay_t<T>, Any>>,
+                    // We also disable this overload when an `Any` object can be
+                    // converted to the parameter type because in that case,
+                    // this constructor may combine with that conversion during
+                    // overload resolution for determining copy
+                    // constructibility, and then when we try to determine copy
+                    // constructibility below we may infinitely recurse. This is
+                    // being evaluated by the standards committee as a potential
+                    // DR in `std::any` as well, but we're going ahead and
+                    // adopting it to work-around usage of `Any` with types that
+                    // need to be implicitly convertible from an `Any`.
+                    llvm::negation<std::is_convertible<Any, std::decay_t<T>>>,
+                    std::is_copy_constructible<std::decay<T>>>::value,
+                int> = 0>
   Any(T &&Value) {
-    using U = typename std::decay<T>::type;
-    Storage = std::make_unique<StorageImpl<U>>(std::forward<T>(Value));
+    Storage =
+        std::make_unique<StorageImpl<std::decay_t<T>>>(std::forward<T>(Value));
   }
 
   Any(Any &&Other) : Storage(std::move(Other.Storage)) {}
@@ -114,32 +114,27 @@ template <typename T> const char Any::TypeId<T>::Id = 0;
 template <typename T> bool any_isa(const Any &Value) {
   if (!Value.Storage)
     return false;
-  using U =
-      typename std::remove_cv<typename std::remove_reference<T>::type>::type;
-  return Value.Storage->id() == &Any::TypeId<U>::Id;
+  return Value.Storage->id() ==
+         &Any::TypeId<std::remove_cv_t<std::remove_reference_t<T>>>::Id;
 }
 
 template <class T> T any_cast(const Any &Value) {
-  using U =
-      typename std::remove_cv<typename std::remove_reference<T>::type>::type;
-  return static_cast<T>(*any_cast<U>(&Value));
+  return static_cast<T>(
+      *any_cast<std::remove_cv_t<std::remove_reference_t<T>>>(&Value));
 }
 
 template <class T> T any_cast(Any &Value) {
-  using U =
-      typename std::remove_cv<typename std::remove_reference<T>::type>::type;
-  return static_cast<T>(*any_cast<U>(&Value));
+  return static_cast<T>(
+      *any_cast<std::remove_cv_t<std::remove_reference_t<T>>>(&Value));
 }
 
 template <class T> T any_cast(Any &&Value) {
-  using U =
-      typename std::remove_cv<typename std::remove_reference<T>::type>::type;
-  return static_cast<T>(std::move(*any_cast<U>(&Value)));
+  return static_cast<T>(std::move(
+      *any_cast<std::remove_cv_t<std::remove_reference_t<T>>>(&Value)));
 }
 
 template <class T> const T *any_cast(const Any *Value) {
-  using U =
-      typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+  using U = std::remove_cv_t<std::remove_reference_t<T>>;
   assert(Value && any_isa<T>(*Value) && "Bad any cast!");
   if (!Value || !any_isa<U>(*Value))
     return nullptr;
@@ -147,7 +142,7 @@ template <class T> const T *any_cast(const Any *Value) {
 }
 
 template <class T> T *any_cast(Any *Value) {
-  using U = typename std::decay<T>::type;
+  using U = std::decay_t<T>;
   assert(Value && any_isa<U>(*Value) && "Bad any cast!");
   if (!Value || !any_isa<U>(*Value))
     return nullptr;
diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index 898a7c707a8d998f48ad2ffbcfa8db732f1c6bc8..5cba73e67f0423aa4af9a72f8e0fa11fe9c33cc1 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -114,30 +114,28 @@ namespace llvm {
     /// Construct an ArrayRef<const T*> from ArrayRef<T*>. This uses SFINAE to
     /// ensure that only ArrayRefs of pointers can be converted.
     template <typename U>
-    ArrayRef(
-        const ArrayRef<U *> &A,
-        typename std::enable_if<
-           std::is_convertible<U *const *, T const *>::value>::type * = nullptr)
-      : Data(A.data()), Length(A.size()) {}
+    ArrayRef(const ArrayRef<U *> &A,
+             std::enable_if_t<std::is_convertible<U *const *, T const *>::value>
+                 * = nullptr)
+        : Data(A.data()), Length(A.size()) {}
 
     /// Construct an ArrayRef<const T*> from a SmallVector<T*>. This is
     /// templated in order to avoid instantiating SmallVectorTemplateCommon<T>
     /// whenever we copy-construct an ArrayRef.
-    template<typename U, typename DummyT>
+    template <typename U, typename DummyT>
     /*implicit*/ ArrayRef(
-      const SmallVectorTemplateCommon<U *, DummyT> &Vec,
-      typename std::enable_if<
-          std::is_convertible<U *const *, T const *>::value>::type * = nullptr)
-      : Data(Vec.data()), Length(Vec.size()) {
-    }
+        const SmallVectorTemplateCommon<U *, DummyT> &Vec,
+        std::enable_if_t<std::is_convertible<U *const *, T const *>::value> * =
+            nullptr)
+        : Data(Vec.data()), Length(Vec.size()) {}
 
     /// Construct an ArrayRef<const T*> from std::vector<T*>. This uses SFINAE
     /// to ensure that only vectors of pointers can be converted.
-    template<typename U, typename A>
+    template <typename U, typename A>
     ArrayRef(const std::vector<U *, A> &Vec,
-             typename std::enable_if<
-                 std::is_convertible<U *const *, T const *>::value>::type* = 0)
-      : Data(Vec.data()), Length(Vec.size()) {}
+             std::enable_if_t<std::is_convertible<U *const *, T const *>::value>
+                 * = 0)
+        : Data(Vec.data()), Length(Vec.size()) {}
 
     /// @}
     /// @name Simple Operations
@@ -256,7 +254,7 @@ namespace llvm {
     /// The declaration here is extra complicated so that "arrayRef = {}"
     /// continues to select the move assignment operator.
     template <typename U>
-    typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type &
+    std::enable_if_t<std::is_same<U, T>::value, ArrayRef<T>> &
     operator=(U &&Temporary) = delete;
 
     /// Disallow accidental assignment from a temporary.
@@ -264,7 +262,7 @@ namespace llvm {
     /// The declaration here is extra complicated so that "arrayRef = {}"
     /// continues to select the move assignment operator.
     template <typename U>
-    typename std::enable_if<std::is_same<U, T>::value, ArrayRef<T>>::type &
+    std::enable_if_t<std::is_same<U, T>::value, ArrayRef<T>> &
     operator=(std::initializer_list<U>) = delete;
 
     /// @}
diff --git a/llvm/include/llvm/ADT/BitmaskEnum.h b/llvm/include/llvm/ADT/BitmaskEnum.h
index 1a18bc721b214edcda64a3fc9d6b287a8df617da..d5854d0de2006caf6275c70aa9946a60a62cdc80 100644
--- a/llvm/include/llvm/ADT/BitmaskEnum.h
+++ b/llvm/include/llvm/ADT/BitmaskEnum.h
@@ -71,49 +71,45 @@ struct is_bitmask_enum : std::false_type {};
 
 template <typename E>
 struct is_bitmask_enum<
-    E, typename std::enable_if<sizeof(E::LLVM_BITMASK_LARGEST_ENUMERATOR) >=
-                               0>::type> : std::true_type {};
+    E, std::enable_if_t<sizeof(E::LLVM_BITMASK_LARGEST_ENUMERATOR) >= 0>>
+    : std::true_type {};
 namespace BitmaskEnumDetail {
 
 /// Get a bitmask with 1s in all places up to the high-order bit of E's largest
 /// value.
-template <typename E> typename std::underlying_type<E>::type Mask() {
+template <typename E> std::underlying_type_t<E> Mask() {
   // On overflow, NextPowerOf2 returns zero with the type uint64_t, so
   // subtracting 1 gives us the mask with all bits set, like we want.
-  return NextPowerOf2(static_cast<typename std::underlying_type<E>::type>(
+  return NextPowerOf2(static_cast<std::underlying_type_t<E>>(
              E::LLVM_BITMASK_LARGEST_ENUMERATOR)) -
          1;
 }
 
 /// Check that Val is in range for E, and return Val cast to E's underlying
 /// type.
-template <typename E> typename std::underlying_type<E>::type Underlying(E Val) {
-  auto U = static_cast<typename std::underlying_type<E>::type>(Val);
+template <typename E> std::underlying_type_t<E> Underlying(E Val) {
+  auto U = static_cast<std::underlying_type_t<E>>(Val);
   assert(U >= 0 && "Negative enum values are not allowed.");
   assert(U <= Mask<E>() && "Enum value too large (or largest val too small?)");
   return U;
 }
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E operator~(E Val) {
   return static_cast<E>(~Underlying(Val) & Mask<E>());
 }
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E operator|(E LHS, E RHS) {
   return static_cast<E>(Underlying(LHS) | Underlying(RHS));
 }
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E operator&(E LHS, E RHS) {
   return static_cast<E>(Underlying(LHS) & Underlying(RHS));
 }
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E operator^(E LHS, E RHS) {
   return static_cast<E>(Underlying(LHS) ^ Underlying(RHS));
 }
@@ -121,22 +117,19 @@ E operator^(E LHS, E RHS) {
 // |=, &=, and ^= return a reference to LHS, to match the behavior of the
 // operators on builtin types.
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E &operator|=(E &LHS, E RHS) {
   LHS = LHS | RHS;
   return LHS;
 }
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E &operator&=(E &LHS, E RHS) {
   LHS = LHS & RHS;
   return LHS;
 }
 
-template <typename E,
-          typename = typename std::enable_if<is_bitmask_enum<E>::value>::type>
+template <typename E, typename = std::enable_if_t<is_bitmask_enum<E>::value>>
 E &operator^=(E &LHS, E RHS) {
   LHS = LHS ^ RHS;
   return LHS;
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index 148d319c86036b2603107af75f969d614fea5b3a..b73afae4bce6f536db54d547a8b9a0736bda17c5 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -1194,7 +1194,7 @@ public:
   // for const iterator destinations so it doesn't end up as a user defined copy
   // constructor.
   template <bool IsConstSrc,
-            typename = typename std::enable_if<!IsConstSrc && IsConst>::type>
+            typename = std::enable_if_t<!IsConstSrc && IsConst>>
   DenseMapIterator(
       const DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, IsConstSrc> &I)
       : DebugEpochBase::HandleBase(I), Ptr(I.Ptr), End(I.End) {}
diff --git a/llvm/include/llvm/ADT/Hashing.h b/llvm/include/llvm/ADT/Hashing.h
index adcc5cf54da920c8690862abffc3e2b459cee445..dac0e0556ff39c3bd41fec381839c1c52f3412e8 100644
--- a/llvm/include/llvm/ADT/Hashing.h
+++ b/llvm/include/llvm/ADT/Hashing.h
@@ -101,8 +101,7 @@ public:
 /// differing argument types even if they would implicit promote to a common
 /// type without changing the value.
 template <typename T>
-typename std::enable_if<is_integral_or_enum<T>::value, hash_code>::type
-hash_value(T value);
+std::enable_if_t<is_integral_or_enum<T>::value, hash_code> hash_value(T value);
 
 /// Compute a hash_code for a pointer's address.
 ///
@@ -360,7 +359,7 @@ template <typename T, typename U> struct is_hashable_data<std::pair<T, U> >
 /// Helper to get the hashable data representation for a type.
 /// This variant is enabled when the type itself can be used.
 template <typename T>
-typename std::enable_if<is_hashable_data<T>::value, T>::type
+std::enable_if_t<is_hashable_data<T>::value, T>
 get_hashable_data(const T &value) {
   return value;
 }
@@ -368,7 +367,7 @@ get_hashable_data(const T &value) {
 /// This variant is enabled when we must first call hash_value and use the
 /// result as our data.
 template <typename T>
-typename std::enable_if<!is_hashable_data<T>::value, size_t>::type
+std::enable_if_t<!is_hashable_data<T>::value, size_t>
 get_hashable_data(const T &value) {
   using ::llvm::hash_value;
   return hash_value(value);
@@ -442,7 +441,7 @@ hash_code hash_combine_range_impl(InputIteratorT first, InputIteratorT last) {
 /// are stored in contiguous memory, this routine avoids copying each value
 /// and directly reads from the underlying memory.
 template <typename ValueT>
-typename std::enable_if<is_hashable_data<ValueT>::value, hash_code>::type
+std::enable_if_t<is_hashable_data<ValueT>::value, hash_code>
 hash_combine_range_impl(ValueT *first, ValueT *last) {
   const uint64_t seed = get_execution_seed();
   const char *s_begin = reinterpret_cast<const char *>(first);
@@ -627,8 +626,7 @@ inline hash_code hash_integer_value(uint64_t value) {
 // Declared and documented above, but defined here so that any of the hashing
 // infrastructure is available.
 template <typename T>
-typename std::enable_if<is_integral_or_enum<T>::value, hash_code>::type
-hash_value(T value) {
+std::enable_if_t<is_integral_or_enum<T>::value, hash_code> hash_value(T value) {
   return ::llvm::hashing::detail::hash_integer_value(
       static_cast<uint64_t>(value));
 }
diff --git a/llvm/include/llvm/ADT/PriorityWorklist.h b/llvm/include/llvm/ADT/PriorityWorklist.h
index 96d22c87557e6a0bce8fe3759a106565485d3d70..01dd59a2e71a31f672482693d8c2f4dd5929212b 100644
--- a/llvm/include/llvm/ADT/PriorityWorklist.h
+++ b/llvm/include/llvm/ADT/PriorityWorklist.h
@@ -110,7 +110,7 @@ public:
 
   /// Insert a sequence of new elements into the PriorityWorklist.
   template <typename SequenceT>
-  typename std::enable_if<!std::is_convertible<SequenceT, T>::value>::type
+  std::enable_if_t<!std::is_convertible<SequenceT, T>::value>
   insert(SequenceT &&Input) {
     if (std::begin(Input) == std::end(Input))
       // Nothing to do for an empty input sequence.
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index b61dab2459d18e696314a729a69a75fec5882cf1..8888e8d489784eaf7ecc187768c4be267993cffc 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -115,9 +115,8 @@ public:
 
   template <typename Callable>
   function_ref(Callable &&callable,
-               typename std::enable_if<
-                   !std::is_same<typename std::remove_reference<Callable>::type,
-                                 function_ref>::value>::type * = nullptr)
+               std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
+                                              function_ref>::value> * = nullptr)
       : callback(callback_fn<typename std::remove_reference<Callable>::type>),
         callable(reinterpret_cast<intptr_t>(&callable)) {}
 
@@ -146,16 +145,14 @@ namespace adl_detail {
 using std::begin;
 
 template <typename ContainerTy>
-auto adl_begin(ContainerTy &&container)
-    -> decltype(begin(std::forward<ContainerTy>(container))) {
+decltype(auto) adl_begin(ContainerTy &&container) {
   return begin(std::forward<ContainerTy>(container));
 }
 
 using std::end;
 
 template <typename ContainerTy>
-auto adl_end(ContainerTy &&container)
-    -> decltype(end(std::forward<ContainerTy>(container))) {
+decltype(auto) adl_end(ContainerTy &&container) {
   return end(std::forward<ContainerTy>(container));
 }
 
@@ -170,14 +167,12 @@ void adl_swap(T &&lhs, T &&rhs) noexcept(noexcept(swap(std::declval<T>(),
 } // end namespace adl_detail
 
 template <typename ContainerTy>
-auto adl_begin(ContainerTy &&container)
-    -> decltype(adl_detail::adl_begin(std::forward<ContainerTy>(container))) {
+decltype(auto) adl_begin(ContainerTy &&container) {
   return adl_detail::adl_begin(std::forward<ContainerTy>(container));
 }
 
 template <typename ContainerTy>
-auto adl_end(ContainerTy &&container)
-    -> decltype(adl_detail::adl_end(std::forward<ContainerTy>(container))) {
+decltype(auto) adl_end(ContainerTy &&container) {
   return adl_detail::adl_end(std::forward<ContainerTy>(container));
 }
 
@@ -195,9 +190,7 @@ constexpr bool empty(const T &RangeOrContainer) {
 
 /// Return a range covering \p RangeOrContainer with the first N elements
 /// excluded.
-template <typename T>
-auto drop_begin(T &&RangeOrContainer, size_t N) ->
-    iterator_range<decltype(adl_begin(RangeOrContainer))> {
+template <typename T> auto drop_begin(T &&RangeOrContainer, size_t N) {
   return make_range(std::next(adl_begin(RangeOrContainer), N),
                     adl_end(RangeOrContainer));
 }
@@ -233,9 +226,7 @@ inline mapped_iterator<ItTy, FuncTy> map_iterator(ItTy I, FuncTy F) {
 }
 
 template <class ContainerTy, class FuncTy>
-auto map_range(ContainerTy &&C, FuncTy F)
-    -> decltype(make_range(map_iterator(C.begin(), F),
-                           map_iterator(C.end(), F))) {
+auto map_range(ContainerTy &&C, FuncTy F) {
   return make_range(map_iterator(C.begin(), F), map_iterator(C.end(), F));
 }
 
@@ -263,8 +254,7 @@ struct has_rbegin : has_rbegin_impl<typename std::remove_reference<Ty>::type> {
 // Note that the container must have rbegin()/rend() methods for this to work.
 template <typename ContainerTy>
 auto reverse(ContainerTy &&C,
-             typename std::enable_if<has_rbegin<ContainerTy>::value>::type * =
-                 nullptr) -> decltype(make_range(C.rbegin(), C.rend())) {
+             std::enable_if_t<has_rbegin<ContainerTy>::value> * = nullptr) {
   return make_range(C.rbegin(), C.rend());
 }
 
@@ -278,11 +268,8 @@ std::reverse_iterator<IteratorTy> make_reverse_iterator(IteratorTy It) {
 // Note that the container must have begin()/end() methods which return
 // bidirectional iterators for this to work.
 template <typename ContainerTy>
-auto reverse(
-    ContainerTy &&C,
-    typename std::enable_if<!has_rbegin<ContainerTy>::value>::type * = nullptr)
-    -> decltype(make_range(llvm::make_reverse_iterator(std::end(C)),
-                           llvm::make_reverse_iterator(std::begin(C)))) {
+auto reverse(ContainerTy &&C,
+             std::enable_if_t<!has_rbegin<ContainerTy>::value> * = nullptr) {
   return make_range(llvm::make_reverse_iterator(std::end(C)),
                     llvm::make_reverse_iterator(std::begin(C)));
 }
@@ -680,9 +667,8 @@ static Iter next_or_end(const Iter &I, const Iter &End) {
 }
 
 template <typename Iter>
-static auto deref_or_none(const Iter &I, const Iter &End)
-    -> llvm::Optional<typename std::remove_const<
-        typename std::remove_reference<decltype(*I)>::type>::type> {
+static auto deref_or_none(const Iter &I, const Iter &End) -> llvm::Optional<
+    std::remove_const_t<std::remove_reference_t<decltype(*I)>>> {
   if (I == End)
     return None;
   return *I;
@@ -983,8 +969,7 @@ struct on_first {
   FuncTy func;
 
   template <typename T>
-  auto operator()(const T &lhs, const T &rhs) const
-      -> decltype(func(lhs.first, rhs.first)) {
+  decltype(auto) operator()(const T &lhs, const T &rhs) const {
     return func(lhs.first, rhs.first);
   }
 };
@@ -1160,12 +1145,11 @@ void DeleteContainerSeconds(Container &C) {
 /// Get the size of a range. This is a wrapper function around std::distance
 /// which is only enabled when the operation is O(1).
 template <typename R>
-auto size(R &&Range, typename std::enable_if<
-                         std::is_same<typename std::iterator_traits<decltype(
-                                          Range.begin())>::iterator_category,
-                                      std::random_access_iterator_tag>::value,
-                         void>::type * = nullptr)
-    -> decltype(std::distance(Range.begin(), Range.end())) {
+auto size(R &&Range,
+          std::enable_if_t<std::is_same<typename std::iterator_traits<decltype(
+                                            Range.begin())>::iterator_category,
+                                        std::random_access_iterator_tag>::value,
+                           void> * = nullptr) {
   return std::distance(Range.begin(), Range.end());
 }
 
@@ -1199,27 +1183,26 @@ bool none_of(R &&Range, UnaryPredicate P) {
 
 /// Provide wrappers to std::find which take ranges instead of having to pass
 /// begin/end explicitly.
-template <typename R, typename T>
-auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range)) {
+template <typename R, typename T> auto find(R &&Range, const T &Val) {
   return std::find(adl_begin(Range), adl_end(Range), Val);
 }
 
 /// Provide wrappers to std::find_if which take ranges instead of having to pass
 /// begin/end explicitly.
 template <typename R, typename UnaryPredicate>
-auto find_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) {
+auto find_if(R &&Range, UnaryPredicate P) {
   return std::find_if(adl_begin(Range), adl_end(Range), P);
 }
 
 template <typename R, typename UnaryPredicate>
-auto find_if_not(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) {
+auto find_if_not(R &&Range, UnaryPredicate P) {
   return std::find_if_not(adl_begin(Range), adl_end(Range), P);
 }
 
 /// Provide wrappers to std::remove_if which take ranges instead of having to
 /// pass begin/end explicitly.
 template <typename R, typename UnaryPredicate>
-auto remove_if(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) {
+auto remove_if(R &&Range, UnaryPredicate P) {
   return std::remove_if(adl_begin(Range), adl_end(Range), P);
 }
 
@@ -1244,17 +1227,14 @@ bool is_contained(R &&Range, const E &Element) {
 
 /// Wrapper function around std::count to count the number of times an element
 /// \p Element occurs in the given range \p Range.
-template <typename R, typename E>
-auto count(R &&Range, const E &Element) ->
-    typename std::iterator_traits<decltype(adl_begin(Range))>::difference_type {
+template <typename R, typename E> auto count(R &&Range, const E &Element) {
   return std::count(adl_begin(Range), adl_end(Range), Element);
 }
 
 /// Wrapper function around std::count_if to count the number of times an
 /// element satisfying a given predicate occurs in a range.
 template <typename R, typename UnaryPredicate>
-auto count_if(R &&Range, UnaryPredicate P) ->
-    typename std::iterator_traits<decltype(adl_begin(Range))>::difference_type {
+auto count_if(R &&Range, UnaryPredicate P) {
   return std::count_if(adl_begin(Range), adl_end(Range), P);
 }
 
@@ -1268,36 +1248,32 @@ OutputIt transform(R &&Range, OutputIt d_first, UnaryPredicate P) {
 /// Provide wrappers to std::partition which take ranges instead of having to
 /// pass begin/end explicitly.
 template <typename R, typename UnaryPredicate>
-auto partition(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) {
+auto partition(R &&Range, UnaryPredicate P) {
   return std::partition(adl_begin(Range), adl_end(Range), P);
 }
 
 /// Provide wrappers to std::lower_bound which take ranges instead of having to
 /// pass begin/end explicitly.
-template <typename R, typename T>
-auto lower_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range)) {
+template <typename R, typename T> auto lower_bound(R &&Range, T &&Value) {
   return std::lower_bound(adl_begin(Range), adl_end(Range),
                           std::forward<T>(Value));
 }
 
 template <typename R, typename T, typename Compare>
-auto lower_bound(R &&Range, T &&Value, Compare C)
-    -> decltype(adl_begin(Range)) {
+auto lower_bound(R &&Range, T &&Value, Compare C) {
   return std::lower_bound(adl_begin(Range), adl_end(Range),
                           std::forward<T>(Value), C);
 }
 
 /// Provide wrappers to std::upper_bound which take ranges instead of having to
 /// pass begin/end explicitly.
-template <typename R, typename T>
-auto upper_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range)) {
+template <typename R, typename T> auto upper_bound(R &&Range, T &&Value) {
   return std::upper_bound(adl_begin(Range), adl_end(Range),
                           std::forward<T>(Value));
 }
 
 template <typename R, typename T, typename Compare>
-auto upper_bound(R &&Range, T &&Value, Compare C)
-    -> decltype(adl_begin(Range)) {
+auto upper_bound(R &&Range, T &&Value, Compare C) {
   return std::upper_bound(adl_begin(Range), adl_end(Range),
                           std::forward<T>(Value), C);
 }
@@ -1316,7 +1292,7 @@ void stable_sort(R &&Range, Compare C) {
 /// Requires that C is always true below some limit, and always false above it.
 template <typename R, typename Predicate,
           typename Val = decltype(*adl_begin(std::declval<R>()))>
-auto partition_point(R &&Range, Predicate P) -> decltype(adl_begin(Range)) {
+auto partition_point(R &&Range, Predicate P) {
   return std::partition_point(adl_begin(Range), adl_end(Range), P);
 }
 
@@ -1393,8 +1369,7 @@ template <typename T> struct deref {
   // Could be further improved to cope with non-derivable functors and
   // non-binary functors (should be a variadic template member function
   // operator()).
-  template <typename A, typename B>
-  auto operator()(A &lhs, B &rhs) const -> decltype(func(*lhs, *rhs)) {
+  template <typename A, typename B> auto operator()(A &lhs, B &rhs) const {
     assert(lhs);
     assert(rhs);
     return func(*lhs, *rhs);
@@ -1515,8 +1490,7 @@ template <typename R> detail::enumerator<R> enumerate(R &&TheRange) {
 namespace detail {
 
 template <typename F, typename Tuple, std::size_t... I>
-auto apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>)
-    -> decltype(std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...)) {
+decltype(auto) apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>) {
   return std::forward<F>(f)(std::get<I>(std::forward<Tuple>(t))...);
 }
 
@@ -1526,10 +1500,7 @@ auto apply_tuple_impl(F &&f, Tuple &&t, std::index_sequence<I...>)
 /// tuple variadically to f as if by calling f(a1, a2, ..., an) and
 /// return the result.
 template <typename F, typename Tuple>
-auto apply_tuple(F &&f, Tuple &&t) -> decltype(detail::apply_tuple_impl(
-    std::forward<F>(f), std::forward<Tuple>(t),
-    std::make_index_sequence<
-        std::tuple_size<typename std::decay<Tuple>::type>::value>{})) {
+decltype(auto) apply_tuple(F &&f, Tuple &&t) {
   using Indices = std::make_index_sequence<
       std::tuple_size<typename std::decay<Tuple>::type>::value>;
 
@@ -1542,12 +1513,11 @@ auto apply_tuple(F &&f, Tuple &&t) -> decltype(detail::apply_tuple_impl(
 template <typename IterTy>
 bool hasNItems(
     IterTy &&Begin, IterTy &&End, unsigned N,
-    typename std::enable_if<
-        !std::is_same<
-            typename std::iterator_traits<typename std::remove_reference<
-                decltype(Begin)>::type>::iterator_category,
-            std::random_access_iterator_tag>::value,
-        void>::type * = nullptr) {
+    std::enable_if_t<
+        !std::is_same<typename std::iterator_traits<std::remove_reference_t<
+                          decltype(Begin)>>::iterator_category,
+                      std::random_access_iterator_tag>::value,
+        void> * = nullptr) {
   for (; N; --N, ++Begin)
     if (Begin == End)
       return false; // Too few.
@@ -1559,12 +1529,11 @@ bool hasNItems(
 template <typename IterTy>
 bool hasNItemsOrMore(
     IterTy &&Begin, IterTy &&End, unsigned N,
-    typename std::enable_if<
-        !std::is_same<
-            typename std::iterator_traits<typename std::remove_reference<
-                decltype(Begin)>::type>::iterator_category,
-            std::random_access_iterator_tag>::value,
-        void>::type * = nullptr) {
+    std::enable_if_t<
+        !std::is_same<typename std::iterator_traits<std::remove_reference_t<
+                          decltype(Begin)>>::iterator_category,
+                      std::random_access_iterator_tag>::value,
+        void> * = nullptr) {
   for (; N; --N, ++Begin)
     if (Begin == End)
       return false; // Too few.
@@ -1573,15 +1542,12 @@ bool hasNItemsOrMore(
 
 /// Returns a raw pointer that represents the same address as the argument.
 ///
-/// The late bound return should be removed once we move to C++14 to better
-/// align with the C++20 declaration. Also, this implementation can be removed
-/// once we move to C++20 where it's defined as std::to_addres()
+/// This implementation can be removed once we move to C++20 where it's defined
+/// as std::to_addres().
 ///
 /// The std::pointer_traits<>::to_address(p) variations of these overloads has
 /// not been implemented.
-template <class Ptr> auto to_address(const Ptr &P) -> decltype(P.operator->()) {
-  return P.operator->();
-}
+template <class Ptr> auto to_address(const Ptr &P) { return P.operator->(); }
 template <class T> constexpr T *to_address(T *P) { return P; }
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h
index 8c46aa9069058fc6db33ae126006285bf8264103..28b514d530dc93e594e7434553c4964627e0f810 100644
--- a/llvm/include/llvm/ADT/SmallVector.h
+++ b/llvm/include/llvm/ADT/SmallVector.h
@@ -284,8 +284,8 @@ protected:
   template <typename T1, typename T2>
   static void uninitialized_copy(
       T1 *I, T1 *E, T2 *Dest,
-      typename std::enable_if<std::is_same<typename std::remove_const<T1>::type,
-                                           T2>::value>::type * = nullptr) {
+      std::enable_if_t<std::is_same<typename std::remove_const<T1>::type,
+                                    T2>::value> * = nullptr) {
     // Use memcpy for PODs iterated by pointers (which includes SmallVector
     // iterators): std::uninitialized_copy optimizes to memmove, but we can
     // use memcpy here. Note that I and E are iterators and thus might be
@@ -381,9 +381,9 @@ public:
 
   /// Add the specified range to the end of the SmallVector.
   template <typename in_iter,
-            typename = typename std::enable_if<std::is_convertible<
+            typename = std::enable_if_t<std::is_convertible<
                 typename std::iterator_traits<in_iter>::iterator_category,
-                std::input_iterator_tag>::value>::type>
+                std::input_iterator_tag>::value>>
   void append(in_iter in_start, in_iter in_end) {
     size_type NumInputs = std::distance(in_start, in_end);
     if (NumInputs > this->capacity() - this->size())
@@ -418,9 +418,9 @@ public:
   }
 
   template <typename in_iter,
-            typename = typename std::enable_if<std::is_convertible<
+            typename = std::enable_if_t<std::is_convertible<
                 typename std::iterator_traits<in_iter>::iterator_category,
-                std::input_iterator_tag>::value>::type>
+                std::input_iterator_tag>::value>>
   void assign(in_iter in_start, in_iter in_end) {
     clear();
     append(in_start, in_end);
@@ -575,9 +575,9 @@ public:
   }
 
   template <typename ItTy,
-            typename = typename std::enable_if<std::is_convertible<
+            typename = std::enable_if_t<std::is_convertible<
                 typename std::iterator_traits<ItTy>::iterator_category,
-                std::input_iterator_tag>::value>::type>
+                std::input_iterator_tag>::value>>
   iterator insert(iterator I, ItTy From, ItTy To) {
     // Convert iterator to elt# to avoid invalidating iterator when we reserve()
     size_t InsertElt = I - this->begin();
@@ -849,9 +849,9 @@ public:
   }
 
   template <typename ItTy,
-            typename = typename std::enable_if<std::is_convertible<
+            typename = std::enable_if_t<std::is_convertible<
                 typename std::iterator_traits<ItTy>::iterator_category,
-                std::input_iterator_tag>::value>::type>
+                std::input_iterator_tag>::value>>
   SmallVector(ItTy S, ItTy E) : SmallVectorImpl<T>(N) {
     this->append(S, E);
   }
diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 0c36e33cca91aadff1efb2cf2eaadd1f7386a9a6..ad31517a1ea7a04784cc665c36e2288fc2595aed 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -265,8 +265,7 @@ namespace llvm {
     /// The declaration here is extra complicated so that `stringRef = {}`
     /// and `stringRef = "abc"` continue to select the move assignment operator.
     template <typename T>
-    typename std::enable_if<std::is_same<T, std::string>::value,
-                            StringRef>::type &
+    std::enable_if_t<std::is_same<T, std::string>::value, StringRef> &
     operator=(T &&Str) = delete;
 
     /// @}
@@ -508,7 +507,7 @@ namespace llvm {
     /// this returns true to signify the error.  The string is considered
     /// erroneous if empty or if it overflows T.
     template <typename T>
-    typename std::enable_if<std::numeric_limits<T>::is_signed, bool>::type
+    std::enable_if_t<std::numeric_limits<T>::is_signed, bool>
     getAsInteger(unsigned Radix, T &Result) const {
       long long LLVal;
       if (getAsSignedInteger(*this, Radix, LLVal) ||
@@ -519,7 +518,7 @@ namespace llvm {
     }
 
     template <typename T>
-    typename std::enable_if<!std::numeric_limits<T>::is_signed, bool>::type
+    std::enable_if_t<!std::numeric_limits<T>::is_signed, bool>
     getAsInteger(unsigned Radix, T &Result) const {
       unsigned long long ULLVal;
       // The additional cast to unsigned long long is required to avoid the
@@ -542,7 +541,7 @@ namespace llvm {
     /// The portion of the string representing the discovered numeric value
     /// is removed from the beginning of the string.
     template <typename T>
-    typename std::enable_if<std::numeric_limits<T>::is_signed, bool>::type
+    std::enable_if_t<std::numeric_limits<T>::is_signed, bool>
     consumeInteger(unsigned Radix, T &Result) {
       long long LLVal;
       if (consumeSignedInteger(*this, Radix, LLVal) ||
@@ -553,7 +552,7 @@ namespace llvm {
     }
 
     template <typename T>
-    typename std::enable_if<!std::numeric_limits<T>::is_signed, bool>::type
+    std::enable_if_t<!std::numeric_limits<T>::is_signed, bool>
     consumeInteger(unsigned Radix, T &Result) {
       unsigned long long ULLVal;
       if (consumeUnsignedInteger(*this, Radix, ULLVal) ||
diff --git a/llvm/include/llvm/ADT/TinyPtrVector.h b/llvm/include/llvm/ADT/TinyPtrVector.h
index 6b76d35d4e92d3ea4ee1f1f2b9c9f900adb3ecbe..ed20a762f30715f80876709cdbcab650e1143204 100644
--- a/llvm/include/llvm/ADT/TinyPtrVector.h
+++ b/llvm/include/llvm/ADT/TinyPtrVector.h
@@ -152,10 +152,10 @@ public:
   }
 
   // Implicit conversion to ArrayRef<U> if EltTy* implicitly converts to U*.
-  template<typename U,
-           typename std::enable_if<
-               std::is_convertible<ArrayRef<EltTy>, ArrayRef<U>>::value,
-               bool>::type = false>
+  template <
+      typename U,
+      std::enable_if_t<std::is_convertible<ArrayRef<EltTy>, ArrayRef<U>>::value,
+                       bool> = false>
   operator ArrayRef<U>() const {
     return operator ArrayRef<EltTy>();
   }
diff --git a/llvm/include/llvm/ADT/fallible_iterator.h b/llvm/include/llvm/ADT/fallible_iterator.h
index 6501ad2233cdb95136de1114b6239929c00b2bcb..62396ddfe8adbbf0e5b3f917f2d55466527f94d7 100644
--- a/llvm/include/llvm/ADT/fallible_iterator.h
+++ b/llvm/include/llvm/ADT/fallible_iterator.h
@@ -96,12 +96,10 @@ public:
   }
 
   /// Forward dereference to the underlying iterator.
-  auto operator*() -> decltype(*std::declval<Underlying>()) { return *I; }
+  decltype(auto) operator*() { return *I; }
 
   /// Forward const dereference to the underlying iterator.
-  auto operator*() const -> decltype(*std::declval<const Underlying>()) {
-    return *I;
-  }
+  decltype(auto) operator*() const { return *I; }
 
   /// Forward structure dereference to the underlying iterator (if the
   /// underlying iterator supports it).
diff --git a/llvm/include/llvm/ADT/ilist_iterator.h b/llvm/include/llvm/ADT/ilist_iterator.h
index cbe5cefa96d16085a01c56b41840c77667044db8..be876347907bb2b5f86b0d2440ceb3075ed745bf 100644
--- a/llvm/include/llvm/ADT/ilist_iterator.h
+++ b/llvm/include/llvm/ADT/ilist_iterator.h
@@ -88,15 +88,14 @@ public:
   // This is templated so that we can allow constructing a const iterator from
   // a nonconst iterator...
   template <bool RHSIsConst>
-  ilist_iterator(
-      const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS,
-      typename std::enable_if<IsConst || !RHSIsConst, void *>::type = nullptr)
+  ilist_iterator(const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS,
+                 std::enable_if_t<IsConst || !RHSIsConst, void *> = nullptr)
       : NodePtr(RHS.NodePtr) {}
 
   // This is templated so that we can allow assigning to a const iterator from
   // a nonconst iterator...
   template <bool RHSIsConst>
-  typename std::enable_if<IsConst || !RHSIsConst, ilist_iterator &>::type
+  std::enable_if_t<IsConst || !RHSIsConst, ilist_iterator &>
   operator=(const ilist_iterator<OptionsT, IsReverse, RHSIsConst> &RHS) {
     NodePtr = RHS.NodePtr;
     return *this;
diff --git a/llvm/include/llvm/ADT/iterator.h b/llvm/include/llvm/ADT/iterator.h
index 8fd5c11a2dcb96ac97dea70b39c2d9cdd8261398..9a1f6e1511e7f7a7a1d5dec6991fe5469e72385f 100644
--- a/llvm/include/llvm/ADT/iterator.h
+++ b/llvm/include/llvm/ADT/iterator.h
@@ -194,14 +194,14 @@ template <
     typename T = typename std::iterator_traits<WrappedIteratorT>::value_type,
     typename DifferenceTypeT =
         typename std::iterator_traits<WrappedIteratorT>::difference_type,
-    typename PointerT = typename std::conditional<
+    typename PointerT = std::conditional_t<
         std::is_same<T, typename std::iterator_traits<
                             WrappedIteratorT>::value_type>::value,
-        typename std::iterator_traits<WrappedIteratorT>::pointer, T *>::type,
-    typename ReferenceT = typename std::conditional<
+        typename std::iterator_traits<WrappedIteratorT>::pointer, T *>,
+    typename ReferenceT = std::conditional_t<
         std::is_same<T, typename std::iterator_traits<
                             WrappedIteratorT>::value_type>::value,
-        typename std::iterator_traits<WrappedIteratorT>::reference, T &>::type>
+        typename std::iterator_traits<WrappedIteratorT>::reference, T &>>
 class iterator_adaptor_base
     : public iterator_facade_base<DerivedT, IteratorCategoryT, T,
                                   DifferenceTypeT, PointerT, ReferenceT> {
@@ -281,8 +281,8 @@ public:
 ///   using iterator = pointee_iterator<SmallVectorImpl<T *>::iterator>;
 /// \endcode
 template <typename WrappedIteratorT,
-          typename T = typename std::remove_reference<
-              decltype(**std::declval<WrappedIteratorT>())>::type>
+          typename T = std::remove_reference_t<decltype(
+              **std::declval<WrappedIteratorT>())>>
 struct pointee_iterator
     : iterator_adaptor_base<
           pointee_iterator<WrappedIteratorT, T>, WrappedIteratorT,
@@ -334,9 +334,11 @@ make_pointer_range(RangeT &&Range) {
 }
 
 template <typename WrappedIteratorT,
-          typename T1 = typename std::remove_reference<decltype(**std::declval<WrappedIteratorT>())>::type,
-          typename T2 = typename std::add_pointer<T1>::type>
-using raw_pointer_iterator = pointer_iterator<pointee_iterator<WrappedIteratorT, T1>, T2>;
+          typename T1 = std::remove_reference_t<decltype(
+              **std::declval<WrappedIteratorT>())>,
+          typename T2 = std::add_pointer_t<T1>>
+using raw_pointer_iterator =
+    pointer_iterator<pointee_iterator<WrappedIteratorT, T1>, T2>;
 
 // Wrapper iterator over iterator ItType, adding DataRef to the type of ItType,
 // to create NodeRef = std::pair<InnerTypeOfItType, DataRef>.
diff --git a/llvm/include/llvm/Analysis/RegionInfo.h b/llvm/include/llvm/Analysis/RegionInfo.h
index 8bcc3e8512003467beed17e9307520e91bdb929f..b0336c5597741bdffe9075efd05b2adbdf147305 100644
--- a/llvm/include/llvm/Analysis/RegionInfo.h
+++ b/llvm/include/llvm/Analysis/RegionInfo.h
@@ -574,10 +574,9 @@ public:
   template <bool IsConst>
   class block_iterator_wrapper
       : public df_iterator<
-            typename std::conditional<IsConst, const BlockT, BlockT>::type *> {
+            std::conditional_t<IsConst, const BlockT, BlockT> *> {
     using super =
-        df_iterator<
-            typename std::conditional<IsConst, const BlockT, BlockT>::type *>;
+        df_iterator<std::conditional_t<IsConst, const BlockT, BlockT> *>;
 
   public:
     using Self = block_iterator_wrapper<IsConst>;
diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h
index 539f15d13f2ffe2d70d14b89c74681bbe733d410..8d9ec646f519598fd46ce93477d9591d28e38926 100644
--- a/llvm/include/llvm/Analysis/RegionInfoImpl.h
+++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h
@@ -724,7 +724,7 @@ void RegionInfoBase<Tr>::findRegionsWithEntry(BlockT *entry,
 
 template <class Tr>
 void RegionInfoBase<Tr>::scanForRegions(FuncT &F, BBtoBBMap *ShortCut) {
-  using FuncPtrT = typename std::add_pointer<FuncT>::type;
+  using FuncPtrT = std::add_pointer_t<FuncT>;
 
   BlockT *entry = GraphTraits<FuncPtrT>::getEntryNode(&F);
   DomTreeNodeT *N = DT->getNode(entry);
@@ -912,7 +912,7 @@ RegionInfoBase<Tr>::getCommonRegion(SmallVectorImpl<BlockT *> &BBs) const {
 
 template <class Tr>
 void RegionInfoBase<Tr>::calculate(FuncT &F) {
-  using FuncPtrT = typename std::add_pointer<FuncT>::type;
+  using FuncPtrT = std::add_pointer_t<FuncT>;
 
   // ShortCut a function where for every BB the exit of the largest region
   // starting with BB is stored. These regions can be threated as single BBS.
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h
index 6195079c7ff5e83c5ade505867ada83cd7d4cb44..27747300f2296942583078ca7c68eb81de939556 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -672,8 +672,7 @@ template <> struct EnumTraits<LineNumberOps> : public std::true_type {
 /// dumping functions above, these format unknown enumerator values as
 /// DW_TYPE_unknown_1234 (e.g. DW_TAG_unknown_ffff).
 template <typename Enum>
-struct format_provider<
-    Enum, typename std::enable_if<dwarf::EnumTraits<Enum>::value>::type> {
+struct format_provider<Enum, std::enable_if_t<dwarf::EnumTraits<Enum>::value>> {
   static void format(const Enum &E, raw_ostream &OS, StringRef Style) {
     StringRef Str = dwarf::EnumTraits<Enum>::StringFn(E);
     if (Str.empty()) {
diff --git a/llvm/include/llvm/CodeGen/CommandFlags.inc b/llvm/include/llvm/CodeGen/CommandFlags.inc
index e122934617594b7d931ff85393e3941647c6556e..8b676655d2d3afcf66b2cfdfcb088828331c0528 100644
--- a/llvm/include/llvm/CodeGen/CommandFlags.inc
+++ b/llvm/include/llvm/CodeGen/CommandFlags.inc
@@ -277,7 +277,7 @@ static cl::opt<bool>
 
 static cl::opt<bool>
     EnableDebugEntryValues("debug-entry-values",
-                           cl::desc("Emit debug info about parameter's entry values"),
+                           cl::desc("Enable debug info for the debug entry values"),
                            cl::init(false));
 
 static cl::opt<bool>
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index b5a434bc3817124c574b3f579a887011c49587b1..afeaf5e2c26fad2827d10a6d8adde80985965527 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -633,6 +633,7 @@ namespace ISD {
     /// form a semi-softened interface for dealing with f16 (as an i16), which
     /// is often a storage-only type but has native conversions.
     FP16_TO_FP, FP_TO_FP16,
+    STRICT_FP16_TO_FP, STRICT_FP_TO_FP16,
 
     /// Perform various unary floating-point operations inspired by libm. For
     /// FPOWI, the result is undefined if if the integer operand doesn't fit
diff --git a/llvm/include/llvm/CodeGen/LiveInterval.h b/llvm/include/llvm/CodeGen/LiveInterval.h
index fe5adb59dac2f9d690a3e9c59f52b519db4e2e7e..dfb104a7df7b5d224c1ab2a78cdfb2c078fe7662 100644
--- a/llvm/include/llvm/CodeGen/LiveInterval.h
+++ b/llvm/include/llvm/CodeGen/LiveInterval.h
@@ -625,11 +625,12 @@ namespace llvm {
         // if the Seg is lower find first segment that is above Idx using binary
         // search
         if (Seg->end <= *Idx) {
-          Seg = std::upper_bound(++Seg, EndSeg, *Idx,
-            [=](typename std::remove_reference<decltype(*Idx)>::type V,
-                const typename std::remove_reference<decltype(*Seg)>::type &S) {
-              return V < S.end;
-            });
+          Seg = std::upper_bound(
+              ++Seg, EndSeg, *Idx,
+              [=](std::remove_reference_t<decltype(*Idx)> V,
+                  const std::remove_reference_t<decltype(*Seg)> &S) {
+                return V < S.end;
+              });
           if (Seg == EndSeg)
             break;
         }
diff --git a/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h b/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
index 0f59563e7e1bec7239304ed2e0c93d715c8264a4..250cb0d78a68f33052c7c52a7f703276b415d433 100644
--- a/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
+++ b/llvm/include/llvm/CodeGen/MachineInstrBundleIterator.h
@@ -152,8 +152,8 @@ public:
   template <class OtherTy>
   MachineInstrBundleIterator(
       const MachineInstrBundleIterator<OtherTy, IsReverse> &I,
-      typename std::enable_if<std::is_convertible<OtherTy *, Ty *>::value,
-                              void *>::type = nullptr)
+      std::enable_if_t<std::is_convertible<OtherTy *, Ty *>::value, void *> =
+          nullptr)
       : MII(I.getInstrIterator()) {}
 
   MachineInstrBundleIterator() : MII(nullptr) {}
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 28a4653b34fe07c8c2a186717cad2af6ca649eba..4ef7b177e8c16aea78629fe3bd534924e404f525 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -701,6 +701,8 @@ public:
     switch (NodeType) {
       default:
         return false;
+      case ISD::STRICT_FP16_TO_FP:
+      case ISD::STRICT_FP_TO_FP16:
 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
       case ISD::STRICT_##DAGN:
 #include "llvm/IR/ConstrainedOps.def"
diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
index 60829a51dc25b7527289f0550879d3b94b846f03..c5caddab15bf4a46f42a42977d578db99a1a1a8f 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
@@ -114,7 +114,7 @@ public:
     if (!isStreaming() && sizeof(Value) > maxFieldLength())
       return make_error<CodeViewError>(cv_error_code::insufficient_buffer);
 
-    using U = typename std::underlying_type<T>::type;
+    using U = std::underlying_type_t<T>;
     U X;
 
     if (isWriting() || isStreaming())
diff --git a/llvm/include/llvm/DebugInfo/DIContext.h b/llvm/include/llvm/DebugInfo/DIContext.h
index fbebfe634b63e6f7ba055d71830074b69c6b149c..b32bb17a1610a87c24a2587e55a4af8e1cd59cfe 100644
--- a/llvm/include/llvm/DebugInfo/DIContext.h
+++ b/llvm/include/llvm/DebugInfo/DIContext.h
@@ -133,7 +133,12 @@ enum class DINameKind { None, ShortName, LinkageName };
 /// Controls which fields of DILineInfo container should be filled
 /// with data.
 struct DILineInfoSpecifier {
-  enum class FileLineInfoKind { None, Default, AbsoluteFilePath };
+  enum class FileLineInfoKind {
+    None,
+    Default,
+    RelativeFilePath,
+    AbsoluteFilePath
+  };
   using FunctionNameKind = DINameKind;
 
   FileLineInfoKind FLIKind;
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h
index 4539b9c9d5818a0a1bc54584ae065e5f19f1935c..32844ffd570ffd9a4c0994d3329a0865a2c8bba0 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h
@@ -27,69 +27,53 @@ class raw_ostream;
 /// The table consists of a header followed by an array of address values from
 /// .debug_addr section.
 class DWARFDebugAddrTable {
-public:
-  struct Header {
-    /// The total length of the entries for this table, not including the length
-    /// field itself.
-    uint32_t Length = 0;
-    /// The DWARF version number.
-    uint16_t Version = 5;
-    /// The size in bytes of an address on the target architecture. For
-    /// segmented addressing, this is the size of the offset portion of the
-    /// address.
-    uint8_t AddrSize;
-    /// The size in bytes of a segment selector on the target architecture.
-    /// If the target system uses a flat address space, this value is 0.
-    uint8_t SegSize = 0;
-  };
-
-private:
   dwarf::DwarfFormat Format;
-  uint64_t HeaderOffset;
-  Header HeaderData;
-  uint32_t DataSize = 0;
+  uint64_t Offset;
+  /// The total length of the entries for this table, not including the length
+  /// field itself.
+  uint64_t Length = 0;
+  /// The DWARF version number.
+  uint16_t Version;
+  /// The size in bytes of an address on the target architecture. For
+  /// segmented addressing, this is the size of the offset portion of the
+  /// address.
+  uint8_t AddrSize;
+  /// The size in bytes of a segment selector on the target architecture.
+  /// If the target system uses a flat address space, this value is 0.
+  uint8_t SegSize;
   std::vector<uint64_t> Addrs;
 
+  /// Invalidate Length field to stop further processing.
+  void invalidateLength() { Length = 0; }
+
+  Error extractAddresses(const DWARFDataExtractor &Data, uint64_t *OffsetPtr,
+                         uint64_t EndOffset);
+
 public:
-  void clear();
 
-  /// Extract an entire table, including all addresses.
-  Error extract(DWARFDataExtractor Data, uint64_t *OffsetPtr,
-                uint16_t Version, uint8_t AddrSize,
+  /// Extract the entire table, including all addresses.
+  Error extract(const DWARFDataExtractor &Data, uint64_t *OffsetPtr,
+                uint16_t CUVersion, uint8_t CUAddrSize,
                 std::function<void(Error)> WarnCallback);
 
-  uint64_t getHeaderOffset() const { return HeaderOffset; }
-  uint8_t getAddrSize() const { return HeaderData.AddrSize; }
+  /// Extract a DWARFv5 address table.
+  Error extractV5(const DWARFDataExtractor &Data, uint64_t *OffsetPtr,
+                  uint8_t CUAddrSize, std::function<void(Error)> WarnCallback);
+
+  /// Extract a pre-DWARFv5 address table. Such tables do not have a header
+  /// and consist only of a series of addresses.
+  /// See https://gcc.gnu.org/wiki/DebugFission for details.
+  Error extractPreStandard(const DWARFDataExtractor &Data, uint64_t *OffsetPtr,
+                           uint16_t CUVersion, uint8_t CUAddrSize);
+
   void dump(raw_ostream &OS, DIDumpOptions DumpOpts = {}) const;
 
   /// Return the address based on a given index.
   Expected<uint64_t> getAddrEntry(uint32_t Index) const;
 
-  /// Return the size of the table header including the length
-  /// but not including the addresses.
-  uint8_t getHeaderSize() const {
-    switch (Format) {
-    case dwarf::DwarfFormat::DWARF32:
-      return 8; // 4 + 2 + 1 + 1
-    case dwarf::DwarfFormat::DWARF64:
-      return 16; // 12 + 2 + 1 + 1
-    }
-    llvm_unreachable("Invalid DWARF format (expected DWARF32 or DWARF64)");
-  }
-
-  /// Returns the length of this table, including the length field, or 0 if the
-  /// length has not been determined (e.g. because the table has not yet been
-  /// parsed, or there was a problem in parsing).
-  uint32_t getLength() const;
-
-  /// Verify that the given length is valid for this table.
-  bool hasValidLength() const { return getLength() != 0; }
-
-  /// Invalidate Length field to stop further processing.
-  void invalidateLength() { HeaderData.Length = 0; }
-
-  /// Returns the length of the array of addresses.
-  uint32_t getDataSize() const;
+  /// Return the full length of this table, including the length field.
+  /// Return None if the length cannot be identified reliably.
+  Optional<uint64_t> getFullLength() const;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h b/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h
index 0d95a246755647d3c51feaa3f700a40a24f88de0..2982146f960c918ff1e881fab0e1fddaa3ae1add 100644
--- a/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h
+++ b/llvm/include/llvm/DebugInfo/PDB/PDBSymbol.h
@@ -17,13 +17,11 @@
 #include "llvm/Support/Casting.h"
 
 #define FORWARD_SYMBOL_METHOD(MethodName)                                      \
-  auto MethodName() const->decltype(RawSymbol->MethodName()) {                 \
-    return RawSymbol->MethodName();                                            \
-  }
+  decltype(auto) MethodName() const { return RawSymbol->MethodName(); }
 
 #define FORWARD_CONCRETE_SYMBOL_ID_METHOD_WITH_NAME(ConcreteType, PrivateName, \
                                                     PublicName)                \
-  auto PublicName##Id() const->decltype(RawSymbol->PrivateName##Id()) {        \
+  decltype(auto) PublicName##Id() const {                                      \
     return RawSymbol->PrivateName##Id();                                       \
   }                                                                            \
   std::unique_ptr<ConcreteType> PublicName() const {                           \
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index fa04653fa7bd2316f8279d1eaa73a37238852e0c..1a8f5cacbcd4ac9fbaa578fb77a0f039b11185c5 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/Support/Allocator.h"
@@ -990,6 +991,11 @@ public:
 
   /// Remove a block.
   void removeBlock(Block &B) {
+    assert(llvm::none_of(B.getSection().symbols(),
+                         [&](const Symbol *Sym) {
+                           return &Sym->getBlock() == &B;
+                         }) &&
+           "Block still has symbols attached");
     B.getSection().removeBlock(B);
     destroyBlock(B);
   }
diff --git a/llvm/include/llvm/ExecutionEngine/JITSymbol.h b/llvm/include/llvm/ExecutionEngine/JITSymbol.h
index 7a2a6cfa520377236295b091e14d56940a955164..67e531afb9b92dafa5ba5acc0eb9b90ab52df164 100644
--- a/llvm/include/llvm/ExecutionEngine/JITSymbol.h
+++ b/llvm/include/llvm/ExecutionEngine/JITSymbol.h
@@ -58,10 +58,9 @@ template <typename T> T jitTargetAddressToPointer(JITTargetAddress Addr) {
 /// Casts the given address to a callable function pointer. This operation
 /// will perform pointer signing for platforms that require it (e.g. arm64e).
 template <typename T> T jitTargetAddressToFunction(JITTargetAddress Addr) {
-  static_assert(
-      std::is_pointer<T>::value &&
-          std::is_function<typename std::remove_pointer<T>::type>::value,
-      "T must be a function pointer type");
+  static_assert(std::is_pointer<T>::value &&
+                    std::is_function<std::remove_pointer_t<T>>::value,
+                "T must be a function pointer type");
   return jitTargetAddressToPointer<T>(Addr);
 }
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index ecba454887b36f3ddc1dcf49d0a1b2dc2ab135cb..1a55801b9d87f97b224d0983a35b358b88d51396 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -202,10 +202,10 @@ public:
   /// If Body returns true then the element just passed in is removed from the
   /// set. If Body returns false then the element is retained.
   template <typename BodyFn>
-  auto forEachWithRemoval(BodyFn &&Body) -> typename std::enable_if<
+  auto forEachWithRemoval(BodyFn &&Body) -> std::enable_if_t<
       std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(),
                                  std::declval<SymbolLookupFlags>())),
-                   bool>::value>::type {
+                   bool>::value> {
     UnderlyingVector::size_type I = 0;
     while (I != Symbols.size()) {
       const auto &Name = Symbols[I].first;
@@ -224,11 +224,11 @@ public:
   /// returns true then the element just passed in is removed from the set. If
   /// Body returns false then the element is retained.
   template <typename BodyFn>
-  auto forEachWithRemoval(BodyFn &&Body) -> typename std::enable_if<
+  auto forEachWithRemoval(BodyFn &&Body) -> std::enable_if_t<
       std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(),
                                  std::declval<SymbolLookupFlags>())),
                    Expected<bool>>::value,
-      Error>::type {
+      Error> {
     UnderlyingVector::size_type I = 0;
     while (I != Symbols.size()) {
       const auto &Name = Symbols[I].first;
@@ -1079,7 +1079,7 @@ public:
   std::shared_ptr<SymbolStringPool> getSymbolStringPool() const { return SSP; }
 
   /// Run the given lambda with the session mutex locked.
-  template <typename Func> auto runSessionLocked(Func &&F) -> decltype(F()) {
+  template <typename Func> decltype(auto) runSessionLocked(Func &&F) {
     std::lock_guard<std::recursive_mutex> Lock(SessionMutex);
     return F();
   }
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h b/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h
index 148e260c95693dbca9553b868d7cd346ea922a82..b20202a49ef658a3380e4fab7cf29cb1de520ec4 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Legacy.h
@@ -73,17 +73,13 @@ private:
 ///        function objects.
 template <typename GetResponsibilitySetFn, typename LookupFn>
 std::unique_ptr<LambdaSymbolResolver<
-    typename std::remove_cv<
-        typename std::remove_reference<GetResponsibilitySetFn>::type>::type,
-    typename std::remove_cv<
-        typename std::remove_reference<LookupFn>::type>::type>>
+    std::remove_cv_t<std::remove_reference_t<GetResponsibilitySetFn>>,
+    std::remove_cv_t<std::remove_reference_t<LookupFn>>>>
 createSymbolResolver(GetResponsibilitySetFn &&GetResponsibilitySet,
                      LookupFn &&Lookup) {
   using LambdaSymbolResolverImpl = LambdaSymbolResolver<
-      typename std::remove_cv<
-          typename std::remove_reference<GetResponsibilitySetFn>::type>::type,
-      typename std::remove_cv<
-          typename std::remove_reference<LookupFn>::type>::type>;
+      std::remove_cv_t<std::remove_reference_t<GetResponsibilitySetFn>>,
+      std::remove_cv_t<std::remove_reference_t<LookupFn>>>;
   return std::make_unique<LambdaSymbolResolverImpl>(
       std::forward<GetResponsibilitySetFn>(GetResponsibilitySet),
       std::forward<LookupFn>(Lookup));
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h
index 3ff5a5f6e90e5e8dbdb19389e67cf8b3dbed96ec..52a328165240c794779d53e16be8fb0d61317009 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h
@@ -108,8 +108,7 @@ public:
 template <typename ChannelT>
 class SerializationTraits<
     ChannelT, remote::DirectBufferWriter, remote::DirectBufferWriter,
-    typename std::enable_if<
-        std::is_base_of<RawByteChannel, ChannelT>::value>::type> {
+    std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> {
 public:
   static Error serialize(ChannelT &C, const remote::DirectBufferWriter &DBW) {
     if (auto EC = serializeSeq(C, DBW.getDst()))
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h
index 4c8e2ea1a7be6bd8064ab34afa794cd9fc594b44..ac1df847cf7e2cbf49046c18848c1fb7b0351b63 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h
@@ -60,7 +60,7 @@ public:
         SymbolLookup(std::move(SymbolLookup)),
         EHFramesRegister(std::move(EHFramesRegister)),
         EHFramesDeregister(std::move(EHFramesDeregister)) {
-    using ThisT = typename std::remove_reference<decltype(*this)>::type;
+    using ThisT = std::remove_reference_t<decltype(*this)>;
     addHandler<exec::CallIntVoid>(*this, &ThisT::handleCallIntVoid);
     addHandler<exec::CallMain>(*this, &ThisT::handleCallMain);
     addHandler<exec::CallVoidVoid>(*this, &ThisT::handleCallVoidVoid);
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h
index 9c69a84f4c677e435fc44d050459d3bfad1d9f23..2f37ab40c7f841ae0af3bad1b77066963ca0760e 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCSerialization.h
@@ -230,9 +230,9 @@ public:
 ///
 ///   template <DerivedChannelT>
 ///   class SerializationTraits<DerivedChannelT, bool,
-///         typename std::enable_if<
+///         std::enable_if_t<
 ///           std::is_base_of<VirtChannel, DerivedChannel>::value
-///         >::type> {
+///         >> {
 ///   public:
 ///     static const char* getName() { ... };
 ///   }
@@ -274,9 +274,8 @@ public:
 
   template <typename CArgT>
   static Error serialize(ChannelT &C, CArgT &&CArg) {
-    return SerializationTraits<ChannelT, ArgT,
-                               typename std::decay<CArgT>::type>::
-             serialize(C, std::forward<CArgT>(CArg));
+    return SerializationTraits<ChannelT, ArgT, std::decay_t<CArgT>>::serialize(
+        C, std::forward<CArgT>(CArg));
   }
 
   template <typename CArgT>
@@ -293,8 +292,8 @@ public:
   static Error serialize(ChannelT &C, CArgT &&CArg,
                          CArgTs &&... CArgs) {
     if (auto Err =
-        SerializationTraits<ChannelT, ArgT, typename std::decay<CArgT>::type>::
-          serialize(C, std::forward<CArgT>(CArg)))
+            SerializationTraits<ChannelT, ArgT, std::decay_t<CArgT>>::serialize(
+                C, std::forward<CArgT>(CArg)))
       return Err;
     if (auto Err = SequenceTraits<ChannelT>::emitSeparator(C))
       return Err;
@@ -316,8 +315,8 @@ public:
 
 template <typename ChannelT, typename... ArgTs>
 Error serializeSeq(ChannelT &C, ArgTs &&... Args) {
-  return SequenceSerialization<ChannelT, typename std::decay<ArgTs>::type...>::
-           serialize(C, std::forward<ArgTs>(Args)...);
+  return SequenceSerialization<ChannelT, std::decay_t<ArgTs>...>::serialize(
+      C, std::forward<ArgTs>(Args)...);
 }
 
 template <typename ChannelT, typename... ArgTs>
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h
index ed09363dcecc38e42a95d372635f3f9ab8a5428a..f348844f39ce3cb6436b0c3d5807c03e5d513a95 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RPCUtils.h
@@ -184,8 +184,7 @@ template <typename T, typename = void> class RPCFunctionIdAllocator;
 /// This specialization of RPCFunctionIdAllocator provides a default
 /// implementation for integral types.
 template <typename T>
-class RPCFunctionIdAllocator<
-    T, typename std::enable_if<std::is_integral<T>::value>::type> {
+class RPCFunctionIdAllocator<T, std::enable_if_t<std::is_integral<T>::value>> {
 public:
   static T getInvalidId() { return T(0); }
   static T getResponseId() { return T(1); }
@@ -205,8 +204,7 @@ template <typename T> class FunctionArgsTuple;
 template <typename RetT, typename... ArgTs>
 class FunctionArgsTuple<RetT(ArgTs...)> {
 public:
-  using Type = std::tuple<typename std::decay<
-      typename std::remove_reference<ArgTs>::type>::type...>;
+  using Type = std::tuple<std::decay_t<std::remove_reference_t<ArgTs>>...>;
 };
 
 // ResultTraits provides typedefs and utilities specific to the return type
@@ -483,9 +481,9 @@ public:
 };
 
 template <typename ResponseHandlerT, typename... ArgTs>
-class AsyncHandlerTraits<Error(ResponseHandlerT, ArgTs...)> :
-    public AsyncHandlerTraits<Error(typename std::decay<ResponseHandlerT>::type,
-                                    ArgTs...)> {};
+class AsyncHandlerTraits<Error(ResponseHandlerT, ArgTs...)>
+    : public AsyncHandlerTraits<Error(std::decay_t<ResponseHandlerT>,
+                                      ArgTs...)> {};
 
 // This template class provides utilities related to RPC function handlers.
 // The base case applies to non-function types (the template class is
@@ -524,18 +522,17 @@ public:
 
   // Call the given handler with the given arguments.
   template <typename HandlerT>
-  static typename std::enable_if<
-      std::is_void<typename HandlerTraits<HandlerT>::ReturnType>::value,
-      Error>::type
+  static std::enable_if_t<
+      std::is_void<typename HandlerTraits<HandlerT>::ReturnType>::value, Error>
   run(HandlerT &Handler, ArgTs &&... Args) {
     Handler(std::move(Args)...);
     return Error::success();
   }
 
   template <typename HandlerT, typename... TArgTs>
-  static typename std::enable_if<
+  static std::enable_if_t<
       !std::is_void<typename HandlerTraits<HandlerT>::ReturnType>::value,
-      typename HandlerTraits<HandlerT>::ReturnType>::type
+      typename HandlerTraits<HandlerT>::ReturnType>
   run(HandlerT &Handler, TArgTs... Args) {
     return Handler(std::move(Args)...);
   }
@@ -894,12 +891,12 @@ private:
   using S = SerializationTraits<ChannelT, WireT, ConcreteT>;
 
   template <typename T>
-  static std::true_type
-  check(typename std::enable_if<
-        std::is_same<decltype(T::serialize(std::declval<ChannelT &>(),
-                                           std::declval<const ConcreteT &>())),
-                     Error>::value,
-        void *>::type);
+  static std::true_type check(
+      std::enable_if_t<std::is_same<decltype(T::serialize(
+                                        std::declval<ChannelT &>(),
+                                        std::declval<const ConcreteT &>())),
+                                    Error>::value,
+                       void *>);
 
   template <typename> static std::false_type check(...);
 
@@ -914,11 +911,11 @@ private:
 
   template <typename T>
   static std::true_type
-  check(typename std::enable_if<
-        std::is_same<decltype(T::deserialize(std::declval<ChannelT &>(),
-                                             std::declval<ConcreteT &>())),
-                     Error>::value,
-        void *>::type);
+      check(std::enable_if_t<
+            std::is_same<decltype(T::deserialize(std::declval<ChannelT &>(),
+                                                 std::declval<ConcreteT &>())),
+                         Error>::value,
+            void *>);
 
   template <typename> static std::false_type check(...);
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h
index 50e26f8449df103c8af24b268a9d0a117109f54b..35745993248cd9663f30ff8bd1391fdfb3b86158 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RPC/RawByteChannel.h
@@ -87,13 +87,13 @@ private:
 template <typename ChannelT, typename T>
 class SerializationTraits<
     ChannelT, T, T,
-    typename std::enable_if<
+    std::enable_if_t<
         std::is_base_of<RawByteChannel, ChannelT>::value &&
         (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value ||
          std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value ||
          std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value ||
          std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value ||
-         std::is_same<T, char>::value)>::type> {
+         std::is_same<T, char>::value)>> {
 public:
   static Error serialize(ChannelT &C, T V) {
     support::endian::byte_swap<T, support::big>(V);
@@ -109,9 +109,9 @@ public:
 };
 
 template <typename ChannelT>
-class SerializationTraits<ChannelT, bool, bool,
-                          typename std::enable_if<std::is_base_of<
-                              RawByteChannel, ChannelT>::value>::type> {
+class SerializationTraits<
+    ChannelT, bool, bool,
+    std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> {
 public:
   static Error serialize(ChannelT &C, bool V) {
     uint8_t Tmp = V ? 1 : 0;
@@ -131,9 +131,9 @@ public:
 };
 
 template <typename ChannelT>
-class SerializationTraits<ChannelT, std::string, StringRef,
-                          typename std::enable_if<std::is_base_of<
-                              RawByteChannel, ChannelT>::value>::type> {
+class SerializationTraits<
+    ChannelT, std::string, StringRef,
+    std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> {
 public:
   /// RPC channel serialization for std::strings.
   static Error serialize(RawByteChannel &C, StringRef S) {
@@ -144,11 +144,11 @@ public:
 };
 
 template <typename ChannelT, typename T>
-class SerializationTraits<ChannelT, std::string, T,
-                          typename std::enable_if<
-                            std::is_base_of<RawByteChannel, ChannelT>::value &&
-                            (std::is_same<T, const char*>::value ||
-                             std::is_same<T, char*>::value)>::type> {
+class SerializationTraits<
+    ChannelT, std::string, T,
+    std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value &&
+                     (std::is_same<T, const char *>::value ||
+                      std::is_same<T, char *>::value)>> {
 public:
   static Error serialize(RawByteChannel &C, const char *S) {
     return SerializationTraits<ChannelT, std::string, StringRef>::serialize(C,
@@ -157,9 +157,9 @@ public:
 };
 
 template <typename ChannelT>
-class SerializationTraits<ChannelT, std::string, std::string,
-                          typename std::enable_if<std::is_base_of<
-                              RawByteChannel, ChannelT>::value>::type> {
+class SerializationTraits<
+    ChannelT, std::string, std::string,
+    std::enable_if_t<std::is_base_of<RawByteChannel, ChannelT>::value>> {
 public:
   /// RPC channel serialization for std::strings.
   static Error serialize(RawByteChannel &C, const std::string &S) {
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
index 2347faed37a2e63cedbc52012a508ee65cfbdd17..58c96737e58076a39296f16a9a0e8c1fc78bf878 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
@@ -130,8 +130,7 @@ public:
 
   /// Locks the associated ThreadSafeContext and calls the given function
   /// on the contained Module.
-  template <typename Func>
-  auto withModuleDo(Func &&F) -> decltype(F(std::declval<Module &>())) {
+  template <typename Func> decltype(auto) withModuleDo(Func &&F) {
     assert(M && "Can not call on null module");
     auto Lock = TSCtx.getLock();
     return F(*M);
@@ -139,9 +138,7 @@ public:
 
   /// Locks the associated ThreadSafeContext and calls the given function
   /// on the contained Module.
-  template <typename Func>
-  auto withModuleDo(Func &&F) const
-      -> decltype(F(std::declval<const Module &>())) {
+  template <typename Func> decltype(auto) withModuleDo(Func &&F) const {
     auto Lock = TSCtx.getLock();
     return F(*M);
   }
diff --git a/llvm/include/llvm/FuzzMutate/Random.h b/llvm/include/llvm/FuzzMutate/Random.h
index 615b15f04ceb7b087106952b5a544cdcb135f68c..9d3af3accb151b0695e574f1d9d255d0cbe51340 100644
--- a/llvm/include/llvm/FuzzMutate/Random.h
+++ b/llvm/include/llvm/FuzzMutate/Random.h
@@ -32,7 +32,7 @@ template <typename T, typename GenT> T uniform(GenT &Gen) {
 /// elements, which may each be weighted to be more likely choices.
 template <typename T, typename GenT> class ReservoirSampler {
   GenT &RandGen;
-  typename std::remove_const<T>::type Selection = {};
+  std::remove_const_t<T> Selection = {};
   uint64_t TotalWeight = 0;
 
 public:
@@ -70,8 +70,8 @@ public:
 };
 
 template <typename GenT, typename RangeT,
-          typename ElT = typename std::remove_reference<
-              decltype(*std::begin(std::declval<RangeT>()))>::type>
+          typename ElT = std::remove_reference_t<
+              decltype(*std::begin(std::declval<RangeT>()))>>
 ReservoirSampler<ElT, GenT> makeSampler(GenT &RandGen, RangeT &&Items) {
   ReservoirSampler<ElT, GenT> RS(RandGen);
   RS.sample(Items);
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index 9b3c1e723a1083a3014247983afee73ae918b236..90bf22bd4344879a911dd150bffb1e44ed09e190 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -460,8 +460,7 @@ public:
   static Constant *get(StructType *T, ArrayRef<Constant*> V);
 
   template <typename... Csts>
-  static typename std::enable_if<are_base_of<Constant, Csts...>::value,
-                                 Constant *>::type
+  static std::enable_if_t<are_base_of<Constant, Csts...>::value, Constant *>
   get(StructType *T, Csts *... Vs) {
     SmallVector<Constant *, 8> Values({Vs...});
     return get(T, Values);
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index 20097ef3f31a559abd3d65085138cd8370956e34..3b56da680c6e2cb79de95a315aaaec291c8c582a 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -267,8 +267,7 @@ public:
                             StringRef Name, bool isPacked = false);
   static StructType *create(LLVMContext &Context, ArrayRef<Type *> Elements);
   template <class... Tys>
-  static typename std::enable_if<are_base_of<Type, Tys...>::value,
-                                 StructType *>::type
+  static std::enable_if_t<are_base_of<Type, Tys...>::value, StructType *>
   create(StringRef Name, Type *elt1, Tys *... elts) {
     assert(elt1 && "Cannot create a struct type with no elements with this");
     SmallVector<llvm::Type *, 8> StructFields({elt1, elts...});
@@ -286,8 +285,7 @@ public:
   /// specifying the elements as arguments. Note that this method always returns
   /// a non-packed struct, and requires at least one element type.
   template <class... Tys>
-  static typename std::enable_if<are_base_of<Type, Tys...>::value,
-                                 StructType *>::type
+  static std::enable_if_t<are_base_of<Type, Tys...>::value, StructType *>
   get(Type *elt1, Tys *... elts) {
     assert(elt1 && "Cannot create a struct type with no elements with this");
     LLVMContext &Ctx = elt1->getContext();
@@ -324,7 +322,7 @@ public:
   void setBody(ArrayRef<Type*> Elements, bool isPacked = false);
 
   template <typename... Tys>
-  typename std::enable_if<are_base_of<Type, Tys...>::value, void>::type
+  std::enable_if_t<are_base_of<Type, Tys...>::value, void>
   setBody(Type *elt1, Tys *... elts) {
     assert(elt1 && "Cannot create a struct type with no elements with this");
     SmallVector<llvm::Type *, 8> StructFields({elt1, elts...});
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index ec469982d378ba19bc5ad192156d4770ccf567ef..44db0eacca1a336811e25a8ed5ec039ddeb2dae9 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -531,9 +531,10 @@ protected:
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               StringRef>::type S) {
+               StringRef>
+               S) {
   R.insert(S);
   return R;
 }
@@ -543,9 +544,10 @@ operator<<(RemarkT &R,
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &&R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               StringRef>::type S) {
+               StringRef>
+               S) {
   R.insert(S);
   return R;
 }
@@ -553,9 +555,10 @@ operator<<(RemarkT &&R,
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               DiagnosticInfoOptimizationBase::Argument>::type A) {
+               DiagnosticInfoOptimizationBase::Argument>
+               A) {
   R.insert(A);
   return R;
 }
@@ -563,9 +566,10 @@ operator<<(RemarkT &R,
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &&R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               DiagnosticInfoOptimizationBase::Argument>::type A) {
+               DiagnosticInfoOptimizationBase::Argument>
+               A) {
   R.insert(A);
   return R;
 }
@@ -573,9 +577,10 @@ operator<<(RemarkT &&R,
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               DiagnosticInfoOptimizationBase::setIsVerbose>::type V) {
+               DiagnosticInfoOptimizationBase::setIsVerbose>
+               V) {
   R.insert(V);
   return R;
 }
@@ -583,9 +588,10 @@ operator<<(RemarkT &R,
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &&R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               DiagnosticInfoOptimizationBase::setIsVerbose>::type V) {
+               DiagnosticInfoOptimizationBase::setIsVerbose>
+               V) {
   R.insert(V);
   return R;
 }
@@ -593,9 +599,10 @@ operator<<(RemarkT &&R,
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &R,
-           typename std::enable_if<
+           std::enable_if_t<
                std::is_base_of<DiagnosticInfoOptimizationBase, RemarkT>::value,
-               DiagnosticInfoOptimizationBase::setExtraArgs>::type EA) {
+               DiagnosticInfoOptimizationBase::setExtraArgs>
+               EA) {
   R.insert(EA);
   return R;
 }
diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h
index dda939b975759753b7950628fcfa022775209284..46526c70ea3b85583e552d9a45d0dc4a0977198c 100644
--- a/llvm/include/llvm/IR/Metadata.h
+++ b/llvm/include/llvm/IR/Metadata.h
@@ -527,7 +527,7 @@ template <class V, class M> struct IsValidReference {
 /// As an analogue to \a isa(), check whether \c MD has an \a Value inside of
 /// type \c X.
 template <class X, class Y>
-inline typename std::enable_if<detail::IsValidPointer<X, Y>::value, bool>::type
+inline std::enable_if_t<detail::IsValidPointer<X, Y>::value, bool>
 hasa(Y &&MD) {
   assert(MD && "Null pointer sent into hasa");
   if (auto *V = dyn_cast<ConstantAsMetadata>(MD))
@@ -535,9 +535,8 @@ hasa(Y &&MD) {
   return false;
 }
 template <class X, class Y>
-inline
-    typename std::enable_if<detail::IsValidReference<X, Y &>::value, bool>::type
-    hasa(Y &MD) {
+inline std::enable_if_t<detail::IsValidReference<X, Y &>::value, bool>
+hasa(Y &MD) {
   return hasa(&MD);
 }
 
@@ -545,14 +544,13 @@ inline
 ///
 /// As an analogue to \a cast(), extract the \a Value subclass \c X from \c MD.
 template <class X, class Y>
-inline typename std::enable_if<detail::IsValidPointer<X, Y>::value, X *>::type
+inline std::enable_if_t<detail::IsValidPointer<X, Y>::value, X *>
 extract(Y &&MD) {
   return cast<X>(cast<ConstantAsMetadata>(MD)->getValue());
 }
 template <class X, class Y>
-inline
-    typename std::enable_if<detail::IsValidReference<X, Y &>::value, X *>::type
-    extract(Y &MD) {
+inline std::enable_if_t<detail::IsValidReference<X, Y &>::value, X *>
+extract(Y &MD) {
   return extract(&MD);
 }
 
@@ -561,7 +559,7 @@ inline
 /// As an analogue to \a cast_or_null(), extract the \a Value subclass \c X
 /// from \c MD, allowing \c MD to be null.
 template <class X, class Y>
-inline typename std::enable_if<detail::IsValidPointer<X, Y>::value, X *>::type
+inline std::enable_if_t<detail::IsValidPointer<X, Y>::value, X *>
 extract_or_null(Y &&MD) {
   if (auto *V = cast_or_null<ConstantAsMetadata>(MD))
     return cast<X>(V->getValue());
@@ -574,7 +572,7 @@ extract_or_null(Y &&MD) {
 /// from \c MD, return null if \c MD doesn't contain a \a Value or if the \a
 /// Value it does contain is of the wrong subclass.
 template <class X, class Y>
-inline typename std::enable_if<detail::IsValidPointer<X, Y>::value, X *>::type
+inline std::enable_if_t<detail::IsValidPointer<X, Y>::value, X *>
 dyn_extract(Y &&MD) {
   if (auto *V = dyn_cast<ConstantAsMetadata>(MD))
     return dyn_cast<X>(V->getValue());
@@ -587,7 +585,7 @@ dyn_extract(Y &&MD) {
 /// from \c MD, return null if \c MD doesn't contain a \a Value or if the \a
 /// Value it does contain is of the wrong subclass, allowing \c MD to be null.
 template <class X, class Y>
-inline typename std::enable_if<detail::IsValidPointer<X, Y>::value, X *>::type
+inline std::enable_if_t<detail::IsValidPointer<X, Y>::value, X *>
 dyn_extract_or_null(Y &&MD) {
   if (auto *V = dyn_cast_or_null<ConstantAsMetadata>(MD))
     return dyn_cast<X>(V->getValue());
@@ -976,7 +974,7 @@ public:
   /// Try to create a uniqued version of \c N -- in place, if possible -- and
   /// return it.  If \c N cannot be uniqued, return a distinct node instead.
   template <class T>
-  static typename std::enable_if<std::is_base_of<MDNode, T>::value, T *>::type
+  static std::enable_if_t<std::is_base_of<MDNode, T>::value, T *>
   replaceWithPermanent(std::unique_ptr<T, TempMDNodeDeleter> N) {
     return cast<T>(N.release()->replaceWithPermanentImpl());
   }
@@ -988,7 +986,7 @@ public:
   ///
   /// \pre N does not self-reference.
   template <class T>
-  static typename std::enable_if<std::is_base_of<MDNode, T>::value, T *>::type
+  static std::enable_if_t<std::is_base_of<MDNode, T>::value, T *>
   replaceWithUniqued(std::unique_ptr<T, TempMDNodeDeleter> N) {
     return cast<T>(N.release()->replaceWithUniquedImpl());
   }
@@ -998,7 +996,7 @@ public:
   /// Create a distinct version of \c N -- in place, if possible -- and return
   /// it.  Takes ownership of the temporary node.
   template <class T>
-  static typename std::enable_if<std::is_base_of<MDNode, T>::value, T *>::type
+  static std::enable_if_t<std::is_base_of<MDNode, T>::value, T *>
   replaceWithDistinct(std::unique_ptr<T, TempMDNodeDeleter> N) {
     return cast<T>(N.release()->replaceWithDistinctImpl());
   }
@@ -1237,15 +1235,13 @@ public:
   template <class U>
   MDTupleTypedArrayWrapper(
       const MDTupleTypedArrayWrapper<U> &Other,
-      typename std::enable_if<std::is_convertible<U *, T *>::value>::type * =
-          nullptr)
+      std::enable_if_t<std::is_convertible<U *, T *>::value> * = nullptr)
       : N(Other.get()) {}
 
   template <class U>
   explicit MDTupleTypedArrayWrapper(
       const MDTupleTypedArrayWrapper<U> &Other,
-      typename std::enable_if<!std::is_convertible<U *, T *>::value>::type * =
-          nullptr)
+      std::enable_if_t<!std::is_convertible<U *, T *>::value> * = nullptr)
       : N(Other.get()) {}
 
   explicit operator bool() const { return get(); }
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h
index dab0ad9fe055c4671d5070c390a8ec6f5c6944b0..69f4bd6313d4978e1cb4f1fd03ff43a958c3b4eb 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -833,7 +833,8 @@ struct TypeTestResolution {
     Single,    ///< Single element (last example in "Short Inline Bit Vectors")
     AllOnes,   ///< All-ones bit vector ("Eliminating Bit Vector Checks for
                ///  All-Ones Bit Vectors")
-  } TheKind = Unsat;
+    Unknown,   ///< Unknown (analysis not performed, don't lower)
+  } TheKind = Unknown;
 
   /// Range of size-1 expressed as a bit width. For example, if the size is in
   /// range [1,256], this number will be 8. This helps generate the most compact
@@ -1027,7 +1028,7 @@ public:
   // in the way some record are interpreted, like flags for instance.
   // Note that incrementing this may require changes in both BitcodeReader.cpp
   // and BitcodeWriter.cpp.
-  static constexpr uint64_t BitcodeSummaryVersion = 8;
+  static constexpr uint64_t BitcodeSummaryVersion = 9;
 
   // Regular LTO module name for ASM writer
   static constexpr const char *getRegularLTOModuleName() {
diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
index 7dcb455274f8916db9828293f92921d46775150d..e51ec67b4c914f7bcdd74725e50100d4ae9e7b65 100644
--- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
+++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h
@@ -17,6 +17,7 @@ namespace yaml {
 
 template <> struct ScalarEnumerationTraits<TypeTestResolution::Kind> {
   static void enumeration(IO &io, TypeTestResolution::Kind &value) {
+    io.enumCase(value, "Unknown", TypeTestResolution::Unknown);
     io.enumCase(value, "Unsat", TypeTestResolution::Unsat);
     io.enumCase(value, "ByteArray", TypeTestResolution::ByteArray);
     io.enumCase(value, "Inline", TypeTestResolution::Inline);
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index 9073878476b6af532f5a79c339c6c390fd046bc0..c03cdee1eb8313d44e83d5561f0622d1dd8208b7 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -81,10 +81,6 @@ public:
     return BinaryOperator::CreateExactUDiv(LHS, RHS);
   }
 
-  Instruction *CreateExactUDiv(Constant *LHS, Constant *RHS) const {
-    return BinaryOperator::CreateExactUDiv(LHS, RHS);
-  }
-
   Instruction *CreateSDiv(Constant *LHS, Constant *RHS,
                           bool isExact = false) const {
     if (!isExact)
@@ -92,10 +88,6 @@ public:
     return BinaryOperator::CreateExactSDiv(LHS, RHS);
   }
 
-  Instruction *CreateExactSDiv(Constant *LHS, Constant *RHS) const {
-    return BinaryOperator::CreateExactSDiv(LHS, RHS);
-  }
-
   Instruction *CreateFDiv(Constant *LHS, Constant *RHS) const {
     return BinaryOperator::CreateFDiv(LHS, RHS);
   }
@@ -163,14 +155,6 @@ public:
     return BO;
   }
 
-  Instruction *CreateNSWNeg(Constant *C) const {
-    return BinaryOperator::CreateNSWNeg(C);
-  }
-
-  Instruction *CreateNUWNeg(Constant *C) const {
-    return BinaryOperator::CreateNUWNeg(C);
-  }
-
   Instruction *CreateFNeg(Constant *C) const {
     return UnaryOperator::CreateFNeg(C);
   }
diff --git a/llvm/include/llvm/IR/ValueMap.h b/llvm/include/llvm/IR/ValueMap.h
index fb5440d5efe8cdac2de79e0668e70f77ac0a192c..a5a06b76dbf6d6ba23f1d6c18cb96b8d7a1117bc 100644
--- a/llvm/include/llvm/IR/ValueMap.h
+++ b/llvm/include/llvm/IR/ValueMap.h
@@ -243,7 +243,7 @@ class ValueMapCallbackVH final : public CallbackVH {
   friend struct DenseMapInfo<ValueMapCallbackVH>;
 
   using ValueMapT = ValueMap<KeyT, ValueT, Config>;
-  using KeySansPointerT = typename std::remove_pointer<KeyT>::type;
+  using KeySansPointerT = std::remove_pointer_t<KeyT>;
 
   ValueMapT *Map;
 
diff --git a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h
index 05e47deadfc69d747710e389944d097bb63d3cec..737edb3ae21ae39f11e274efd68b0da1f1b2f3ff 100644
--- a/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h
+++ b/llvm/include/llvm/MC/MCDisassembler/MCDisassembler.h
@@ -22,15 +22,19 @@ struct SymbolInfoTy {
 	StringRef Name;
 	uint8_t   Type;
 
-       SymbolInfoTy(uint64_t Addr, StringRef Name, uint8_t Type):
-        Addr(Addr),Name(Name),Type(Type) {};
+        SymbolInfoTy(uint64_t Addr, StringRef Name, uint8_t Type)
+            : Addr(Addr), Name(Name), Type(Type){};
+
+        friend bool operator<(const SymbolInfoTy &P1, const SymbolInfoTy &P2) {
+          return std::tie(P1.Addr, P1.Name, P1.Type) <
+                 std::tie(P2.Addr, P2.Name, P2.Type);
+        }
 };
 
 using SectionSymbolsTy = std::vector<SymbolInfoTy>;
 
 
 template <typename T> class ArrayRef;
-class StringRef;
 class MCContext;
 class MCInst;
 class MCSubtargetInfo;
diff --git a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
index 6e4821cbc7b9cbcbaa86cdfc33b06ed0e5d0e3a1..9ce1890916e9f84d919a5bdd78fdacac130c13a7 100644
--- a/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCTargetAsmParser.h
@@ -376,6 +376,14 @@ public:
   virtual bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                              SMLoc &EndLoc) = 0;
 
+  /// tryParseRegister - parse one register if possible
+  ///
+  /// Check whether a register specification can be parsed at the current
+  /// location, without failing the entire parse if it can't. Must not consume
+  /// tokens if the parse fails.
+  virtual OperandMatchResultTy
+  tryParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) = 0;
+
   /// ParseInstruction - Parse one assembly instruction.
   ///
   /// The parser is positioned following the instruction name. The target
diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h
index 7d1ade4d5437db87ea8687561497c55c98c719fc..32ad1e4ea40bee6c6d96d407c5c6cb03ac1a8c63 100644
--- a/llvm/include/llvm/Object/ELFTypes.h
+++ b/llvm/include/llvm/Object/ELFTypes.h
@@ -53,7 +53,7 @@ public:
   static const endianness TargetEndianness = E;
   static const bool Is64Bits = Is64;
 
-  using uint = typename std::conditional<Is64, uint64_t, uint32_t>::type;
+  using uint = std::conditional_t<Is64, uint64_t, uint32_t>;
   using Ehdr = Elf_Ehdr_Impl<ELFType<E, Is64>>;
   using Shdr = Elf_Shdr_Impl<ELFType<E, Is64>>;
   using Sym = Elf_Sym_Impl<ELFType<E, Is64>>;
@@ -346,10 +346,8 @@ template <class ELFT>
 struct Elf_Dyn_Impl : Elf_Dyn_Base<ELFT> {
   using Elf_Dyn_Base<ELFT>::d_tag;
   using Elf_Dyn_Base<ELFT>::d_un;
-  using intX_t = typename std::conditional<ELFT::Is64Bits,
-                                           int64_t, int32_t>::type;
-  using uintX_t = typename std::conditional<ELFT::Is64Bits,
-                                            uint64_t, uint32_t>::type;
+  using intX_t = std::conditional_t<ELFT::Is64Bits, int64_t, int32_t>;
+  using uintX_t = std::conditional_t<ELFT::Is64Bits, uint64_t, uint32_t>;
   intX_t getTag() const { return d_tag; }
   uintX_t getVal() const { return d_un.d_val; }
   uintX_t getPtr() const { return d_un.d_ptr; }
diff --git a/llvm/include/llvm/Support/AllocatorBase.h b/llvm/include/llvm/Support/AllocatorBase.h
index c5a2f03e6bf8ccdba57190eb715ab434040605be..7f430040ddfbb9e1b0760d5f8517723dd856ddd7 100644
--- a/llvm/include/llvm/Support/AllocatorBase.h
+++ b/llvm/include/llvm/Support/AllocatorBase.h
@@ -70,8 +70,7 @@ public:
 
   /// Deallocate space for a sequence of objects without constructing them.
   template <typename T>
-  typename std::enable_if<
-      !std::is_same<typename std::remove_cv<T>::type, void>::value, void>::type
+  std::enable_if_t<!std::is_same<std::remove_cv_t<T>, void>::value, void>
   Deallocate(T *Ptr, size_t Num = 1) {
     Deallocate(static_cast<const void *>(Ptr), Num * sizeof(T));
   }
diff --git a/llvm/include/llvm/Support/BinaryStreamReader.h b/llvm/include/llvm/Support/BinaryStreamReader.h
index c586f865fdc54341a087b1854ac27ab5d05852fa..b611707807c0a58e8e63536c00410fc72ccfafcb 100644
--- a/llvm/include/llvm/Support/BinaryStreamReader.h
+++ b/llvm/include/llvm/Support/BinaryStreamReader.h
@@ -90,7 +90,7 @@ public:
   template <typename T> Error readEnum(T &Dest) {
     static_assert(std::is_enum<T>::value,
                   "Cannot call readEnum with non-enum value!");
-    typename std::underlying_type<T>::type N;
+    std::underlying_type_t<T> N;
     if (auto EC = readInteger(N))
       return EC;
     Dest = static_cast<T>(N);
diff --git a/llvm/include/llvm/Support/BinaryStreamWriter.h b/llvm/include/llvm/Support/BinaryStreamWriter.h
index 86d2389d91820514bc98e7ba8130f4ddde1bdcdb..ceba792e6b26633f60227988c12f14a4d916ec7a 100644
--- a/llvm/include/llvm/Support/BinaryStreamWriter.h
+++ b/llvm/include/llvm/Support/BinaryStreamWriter.h
@@ -75,7 +75,7 @@ public:
     static_assert(std::is_enum<T>::value,
                   "Cannot call writeEnum with non-Enum type");
 
-    using U = typename std::underlying_type<T>::type;
+    using U = std::underlying_type_t<T>;
     return writeInteger<U>(static_cast<U>(Num));
   }
 
diff --git a/llvm/include/llvm/Support/Casting.h b/llvm/include/llvm/Support/Casting.h
index 46bdedb04cfe63b5eca512381ceafc0c3b8ce721..41f1b6740c6ff876142079172a69b4d93fbdda5d 100644
--- a/llvm/include/llvm/Support/Casting.h
+++ b/llvm/include/llvm/Support/Casting.h
@@ -61,8 +61,7 @@ struct isa_impl {
 
 /// Always allow upcasts, and perform no dynamic check for them.
 template <typename To, typename From>
-struct isa_impl<
-    To, From, typename std::enable_if<std::is_base_of<To, From>::value>::type> {
+struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
   static inline bool doit(const From &) { return true; }
 };
 
@@ -184,7 +183,7 @@ template <class To, class From>
 struct cast_retty_impl<To, std::unique_ptr<From>> {
 private:
   using PointerType = typename cast_retty_impl<To, From *>::ret_type;
-  using ResultType = typename std::remove_pointer<PointerType>::type;
+  using ResultType = std::remove_pointer_t<PointerType>;
 
 public:
   using ret_type = std::unique_ptr<ResultType>;
@@ -244,8 +243,8 @@ template <class X> struct is_simple_type {
 //  cast<Instruction>(myVal)->getParent()
 //
 template <class X, class Y>
-inline typename std::enable_if<!is_simple_type<Y>::value,
-                               typename cast_retty<X, const Y>::ret_type>::type
+inline std::enable_if_t<!is_simple_type<Y>::value,
+                        typename cast_retty<X, const Y>::ret_type>
 cast(const Y &Val) {
   assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!");
   return cast_convert_val<
@@ -280,10 +279,9 @@ cast(std::unique_ptr<Y> &&Val) {
 // accepted.
 //
 template <class X, class Y>
-LLVM_NODISCARD inline
-    typename std::enable_if<!is_simple_type<Y>::value,
-                            typename cast_retty<X, const Y>::ret_type>::type
-    cast_or_null(const Y &Val) {
+LLVM_NODISCARD inline std::enable_if_t<
+    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
+cast_or_null(const Y &Val) {
   if (!Val)
     return nullptr;
   assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!");
@@ -291,10 +289,9 @@ LLVM_NODISCARD inline
 }
 
 template <class X, class Y>
-LLVM_NODISCARD inline
-    typename std::enable_if<!is_simple_type<Y>::value,
-                            typename cast_retty<X, Y>::ret_type>::type
-    cast_or_null(Y &Val) {
+LLVM_NODISCARD inline std::enable_if_t<!is_simple_type<Y>::value,
+                                       typename cast_retty<X, Y>::ret_type>
+cast_or_null(Y &Val) {
   if (!Val)
     return nullptr;
   assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!");
@@ -326,10 +323,9 @@ cast_or_null(std::unique_ptr<Y> &&Val) {
 //
 
 template <class X, class Y>
-LLVM_NODISCARD inline
-    typename std::enable_if<!is_simple_type<Y>::value,
-                            typename cast_retty<X, const Y>::ret_type>::type
-    dyn_cast(const Y &Val) {
+LLVM_NODISCARD inline std::enable_if_t<
+    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
+dyn_cast(const Y &Val) {
   return isa<X>(Val) ? cast<X>(Val) : nullptr;
 }
 
@@ -347,18 +343,16 @@ LLVM_NODISCARD inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
 // value is accepted.
 //
 template <class X, class Y>
-LLVM_NODISCARD inline
-    typename std::enable_if<!is_simple_type<Y>::value,
-                            typename cast_retty<X, const Y>::ret_type>::type
-    dyn_cast_or_null(const Y &Val) {
+LLVM_NODISCARD inline std::enable_if_t<
+    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
+dyn_cast_or_null(const Y &Val) {
   return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
 }
 
 template <class X, class Y>
-LLVM_NODISCARD inline
-    typename std::enable_if<!is_simple_type<Y>::value,
-                            typename cast_retty<X, Y>::ret_type>::type
-    dyn_cast_or_null(Y &Val) {
+LLVM_NODISCARD inline std::enable_if_t<!is_simple_type<Y>::value,
+                                       typename cast_retty<X, Y>::ret_type>
+dyn_cast_or_null(Y &Val) {
   return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
 }
 
@@ -382,8 +376,7 @@ LLVM_NODISCARD inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
 }
 
 template <class X, class Y>
-LLVM_NODISCARD inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val)
-    -> decltype(cast<X>(Val)) {
+LLVM_NODISCARD inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
   return unique_dyn_cast<X, Y>(Val);
 }
 
@@ -398,8 +391,7 @@ LLVM_NODISCARD inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
 }
 
 template <class X, class Y>
-LLVM_NODISCARD inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val)
-    -> decltype(cast<X>(Val)) {
+LLVM_NODISCARD inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
   return unique_dyn_cast_or_null<X, Y>(Val);
 }
 
diff --git a/llvm/include/llvm/Support/CheckedArithmetic.h b/llvm/include/llvm/Support/CheckedArithmetic.h
index 8a50e3d5ddf6f5ba4fe329e95d7d1829e55312a1..035e4533322cbcccd253963423dd6c75d1e109cb 100644
--- a/llvm/include/llvm/Support/CheckedArithmetic.h
+++ b/llvm/include/llvm/Support/CheckedArithmetic.h
@@ -25,8 +25,8 @@ namespace {
 /// \p RHS.
 /// \return Empty optional if the operation overflows, or result otherwise.
 template <typename T, typename F>
-typename std::enable_if<std::is_integral<T>::value && sizeof(T) * 8 <= 64,
-                        llvm::Optional<T>>::type
+std::enable_if_t<std::is_integral<T>::value && sizeof(T) * 8 <= 64,
+                 llvm::Optional<T>>
 checkedOp(T LHS, T RHS, F Op, bool Signed = true) {
   llvm::APInt ALHS(/*BitSize=*/sizeof(T) * 8, LHS, Signed);
   llvm::APInt ARHS(/*BitSize=*/sizeof(T) * 8, RHS, Signed);
@@ -44,7 +44,7 @@ namespace llvm {
 /// \return Optional of sum if no signed overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_signed<T>::value, llvm::Optional<T>>
 checkedAdd(T LHS, T RHS) {
   return checkedOp(LHS, RHS, &llvm::APInt::sadd_ov);
 }
@@ -53,7 +53,7 @@ checkedAdd(T LHS, T RHS) {
 /// \return Optional of sum if no signed overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_signed<T>::value, llvm::Optional<T>>
 checkedSub(T LHS, T RHS) {
   return checkedOp(LHS, RHS, &llvm::APInt::ssub_ov);
 }
@@ -62,7 +62,7 @@ checkedSub(T LHS, T RHS) {
 /// \return Optional of product if no signed overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_signed<T>::value, llvm::Optional<T>>
 checkedMul(T LHS, T RHS) {
   return checkedOp(LHS, RHS, &llvm::APInt::smul_ov);
 }
@@ -71,7 +71,7 @@ checkedMul(T LHS, T RHS) {
 /// \return Optional of result if no signed overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_signed<T>::value, llvm::Optional<T>>
 checkedMulAdd(T A, T B, T C) {
   if (auto Product = checkedMul(A, B))
     return checkedAdd(*Product, C);
@@ -82,7 +82,7 @@ checkedMulAdd(T A, T B, T C) {
 /// \return Optional of sum if no unsigned overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_unsigned<T>::value, llvm::Optional<T>>
 checkedAddUnsigned(T LHS, T RHS) {
   return checkedOp(LHS, RHS, &llvm::APInt::uadd_ov, /*Signed=*/false);
 }
@@ -91,7 +91,7 @@ checkedAddUnsigned(T LHS, T RHS) {
 /// \return Optional of product if no unsigned overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_unsigned<T>::value, llvm::Optional<T>>
 checkedMulUnsigned(T LHS, T RHS) {
   return checkedOp(LHS, RHS, &llvm::APInt::umul_ov, /*Signed=*/false);
 }
@@ -100,7 +100,7 @@ checkedMulUnsigned(T LHS, T RHS) {
 /// \return Optional of result if no unsigned overflow occurred,
 /// \c None otherwise.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, llvm::Optional<T>>::type
+std::enable_if_t<std::is_unsigned<T>::value, llvm::Optional<T>>
 checkedMulAddUnsigned(T A, T B, T C) {
   if (auto Product = checkedMulUnsigned(A, B))
     return checkedAddUnsigned(*Product, C);
diff --git a/llvm/include/llvm/Support/Chrono.h b/llvm/include/llvm/Support/Chrono.h
index 334ab60835a44a95d44bd6b9e9e27fe35fe66d6c..098512dce783b157a30ab361eb9e3dbb2294e9fd 100644
--- a/llvm/include/llvm/Support/Chrono.h
+++ b/llvm/include/llvm/Support/Chrono.h
@@ -112,8 +112,8 @@ template <typename Rep, typename Period>
 struct format_provider<std::chrono::duration<Rep, Period>> {
 private:
   typedef std::chrono::duration<Rep, Period> Dur;
-  typedef typename std::conditional<
-      std::chrono::treat_as_floating_point<Rep>::value, double, intmax_t>::type
+  typedef std::conditional_t<std::chrono::treat_as_floating_point<Rep>::value,
+                             double, intmax_t>
       InternalRep;
 
   template <typename AsPeriod> static InternalRep getAs(const Dur &D) {
diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h
index 05374e34aa7d4d1f1ce6a7d7f40d3d72ea469214..0242e35c05bd520fdc4728c3952befe9ae000004 100644
--- a/llvm/include/llvm/Support/CommandLine.h
+++ b/llvm/include/llvm/Support/CommandLine.h
@@ -488,14 +488,13 @@ struct callback_traits : public callback_traits<decltype(&F::operator())> {};
 template <typename R, typename C, typename... Args>
 struct callback_traits<R (C::*)(Args...) const> {
   using result_type = R;
-  using arg_type = typename std::tuple_element<0, std::tuple<Args...>>::type;
+  using arg_type = std::tuple_element_t<0, std::tuple<Args...>>;
   static_assert(sizeof...(Args) == 1, "callback function must have one and only one parameter");
   static_assert(std::is_same<result_type, void>::value,
                 "callback return type must be void");
-  static_assert(
-      std::is_lvalue_reference<arg_type>::value &&
-          std::is_const<typename std::remove_reference<arg_type>::type>::value,
-      "callback arg_type must be a const lvalue reference");
+  static_assert(std::is_lvalue_reference<arg_type>::value &&
+                    std::is_const<std::remove_reference_t<arg_type>>::value,
+                "callback arg_type must be a const lvalue reference");
 };
 } // namespace detail
 
@@ -1453,16 +1452,16 @@ class opt : public Option,
     }
   }
 
-  template <class T, class = typename std::enable_if<
-            std::is_assignable<T&, T>::value>::type>
+  template <class T,
+            class = std::enable_if_t<std::is_assignable<T &, T>::value>>
   void setDefaultImpl() {
     const OptionValue<DataType> &V = this->getDefault();
     if (V.hasValue())
       this->setValue(V.getValue());
   }
 
-  template <class T, class = typename std::enable_if<
-            !std::is_assignable<T&, T>::value>::type>
+  template <class T,
+            class = std::enable_if_t<!std::is_assignable<T &, T>::value>>
   void setDefaultImpl(...) {}
 
   void setDefault() override { setDefaultImpl<DataType>(); }
diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h
index beeb855c7c58c00b19466c323e2df3e6593c217e..61a1bd405a4d4da0a267ec4dca2132c329c2d366 100644
--- a/llvm/include/llvm/Support/CrashRecoveryContext.h
+++ b/llvm/include/llvm/Support/CrashRecoveryContext.h
@@ -97,6 +97,11 @@ public:
     return RunSafelyOnThread([&]() { Fn(UserData); }, RequestedStackSize);
   }
 
+  /// Explicitly trigger a crash recovery in the current process, and
+  /// return failure from RunSafely(). This function does not return.
+  LLVM_ATTRIBUTE_NORETURN
+  void HandleExit(int RetCode);
+
   /// In case of a crash, this is the crash identifier.
   int RetCode = 0;
 
diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h
index 87aecedd3a4b4afaa3d6223e2f9e062fcc71d1da..0b9f866cd976e457c1c4539e719c7d5f525edf58 100644
--- a/llvm/include/llvm/Support/Endian.h
+++ b/llvm/include/llvm/Support/Endian.h
@@ -111,7 +111,7 @@ inline void write(void *memory, value_type value) {
 }
 
 template <typename value_type>
-using make_unsigned_t = typename std::make_unsigned<value_type>::type;
+using make_unsigned_t = std::make_unsigned_t<value_type>;
 
 /// Read a value of a particular endianness from memory, for a location
 /// that starts at the given bit offset within the first byte.
diff --git a/llvm/include/llvm/Support/Errno.h b/llvm/include/llvm/Support/Errno.h
index aedb5fb292b84d81d0e4b490070a33ed93325dca..dc3b3322ed982d9cb687d0462013676e56c3e8b2 100644
--- a/llvm/include/llvm/Support/Errno.h
+++ b/llvm/include/llvm/Support/Errno.h
@@ -30,8 +30,8 @@ std::string StrError();
 std::string StrError(int errnum);
 
 template <typename FailT, typename Fun, typename... Args>
-inline auto RetryAfterSignal(const FailT &Fail, const Fun &F,
-                             const Args &... As) -> decltype(F(As...)) {
+inline decltype(auto) RetryAfterSignal(const FailT &Fail, const Fun &F,
+                                       const Args &... As) {
   decltype(F(As...)) Res;
   do {
     errno = 0;
diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h
index 44676338808b4c66adfa879926066b150b43947e..f1e3059368d5d296f9e61238021d6e8cf9dc4874 100644
--- a/llvm/include/llvm/Support/Error.h
+++ b/llvm/include/llvm/Support/Error.h
@@ -436,19 +436,19 @@ template <class T> class LLVM_NODISCARD Expected {
 
   static const bool isRef = std::is_reference<T>::value;
 
-  using wrap = std::reference_wrapper<typename std::remove_reference<T>::type>;
+  using wrap = std::reference_wrapper<std::remove_reference_t<T>>;
 
   using error_type = std::unique_ptr<ErrorInfoBase>;
 
 public:
-  using storage_type = typename std::conditional<isRef, wrap, T>::type;
+  using storage_type = std::conditional_t<isRef, wrap, T>;
   using value_type = T;
 
 private:
-  using reference = typename std::remove_reference<T>::type &;
-  using const_reference = const typename std::remove_reference<T>::type &;
-  using pointer = typename std::remove_reference<T>::type *;
-  using const_pointer = const typename std::remove_reference<T>::type *;
+  using reference = std::remove_reference_t<T> &;
+  using const_reference = const std::remove_reference_t<T> &;
+  using pointer = std::remove_reference_t<T> *;
+  using const_pointer = const std::remove_reference_t<T> *;
 
 public:
   /// Create an Expected<T> error value from the given Error.
@@ -472,12 +472,12 @@ public:
   /// must be convertible to T.
   template <typename OtherT>
   Expected(OtherT &&Val,
-           typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
-               * = nullptr)
+           std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
       : HasError(false)
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
         // Expected is unchecked upon construction in Debug builds.
-        , Unchecked(true)
+        ,
+        Unchecked(true)
 #endif
   {
     new (getStorage()) storage_type(std::forward<OtherT>(Val));
@@ -489,9 +489,9 @@ public:
   /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
   /// must be convertible to T.
   template <class OtherT>
-  Expected(Expected<OtherT> &&Other,
-           typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
-               * = nullptr) {
+  Expected(
+      Expected<OtherT> &&Other,
+      std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
     moveConstruct(std::move(Other));
   }
 
@@ -500,8 +500,7 @@ public:
   template <class OtherT>
   explicit Expected(
       Expected<OtherT> &&Other,
-      typename std::enable_if<!std::is_convertible<OtherT, T>::value>::type * =
-          nullptr) {
+      std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) {
     moveConstruct(std::move(Other));
   }
 
diff --git a/llvm/include/llvm/Support/ErrorOr.h b/llvm/include/llvm/Support/ErrorOr.h
index 8211f4d8a098b458802dfba1d58ed2656da5b760..4750cd832ef73d1e964f1228b9023d624114b5d6 100644
--- a/llvm/include/llvm/Support/ErrorOr.h
+++ b/llvm/include/llvm/Support/ErrorOr.h
@@ -58,23 +58,23 @@ class ErrorOr {
 
   static const bool isRef = std::is_reference<T>::value;
 
-  using wrap = std::reference_wrapper<typename std::remove_reference<T>::type>;
+  using wrap = std::reference_wrapper<std::remove_reference_t<T>>;
 
 public:
-  using storage_type = typename std::conditional<isRef, wrap, T>::type;
+  using storage_type = std::conditional_t<isRef, wrap, T>;
 
 private:
-  using reference = typename std::remove_reference<T>::type &;
-  using const_reference = const typename std::remove_reference<T>::type &;
-  using pointer = typename std::remove_reference<T>::type *;
-  using const_pointer = const typename std::remove_reference<T>::type *;
+  using reference = std::remove_reference_t<T> &;
+  using const_reference = const std::remove_reference_t<T> &;
+  using pointer = std::remove_reference_t<T> *;
+  using const_pointer = const std::remove_reference_t<T> *;
 
 public:
   template <class E>
   ErrorOr(E ErrorCode,
-          typename std::enable_if<std::is_error_code_enum<E>::value ||
-                                      std::is_error_condition_enum<E>::value,
-                                  void *>::type = nullptr)
+          std::enable_if_t<std::is_error_code_enum<E>::value ||
+                               std::is_error_condition_enum<E>::value,
+                           void *> = nullptr)
       : HasError(true) {
     new (getErrorStorage()) std::error_code(make_error_code(ErrorCode));
   }
@@ -85,8 +85,7 @@ public:
 
   template <class OtherT>
   ErrorOr(OtherT &&Val,
-          typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
-              * = nullptr)
+          std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
       : HasError(false) {
     new (getStorage()) storage_type(std::forward<OtherT>(Val));
   }
@@ -96,18 +95,16 @@ public:
   }
 
   template <class OtherT>
-  ErrorOr(
-      const ErrorOr<OtherT> &Other,
-      typename std::enable_if<std::is_convertible<OtherT, T>::value>::type * =
-          nullptr) {
+  ErrorOr(const ErrorOr<OtherT> &Other,
+          std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
     copyConstruct(Other);
   }
 
   template <class OtherT>
   explicit ErrorOr(
       const ErrorOr<OtherT> &Other,
-      typename std::enable_if<
-          !std::is_convertible<OtherT, const T &>::value>::type * = nullptr) {
+      std::enable_if_t<!std::is_convertible<OtherT, const T &>::value> * =
+          nullptr) {
     copyConstruct(Other);
   }
 
@@ -116,10 +113,8 @@ public:
   }
 
   template <class OtherT>
-  ErrorOr(
-      ErrorOr<OtherT> &&Other,
-      typename std::enable_if<std::is_convertible<OtherT, T>::value>::type * =
-          nullptr) {
+  ErrorOr(ErrorOr<OtherT> &&Other,
+          std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
     moveConstruct(std::move(Other));
   }
 
@@ -128,8 +123,7 @@ public:
   template <class OtherT>
   explicit ErrorOr(
       ErrorOr<OtherT> &&Other,
-      typename std::enable_if<!std::is_convertible<OtherT, T>::value>::type * =
-          nullptr) {
+      std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) {
     moveConstruct(std::move(Other));
   }
 
@@ -266,9 +260,9 @@ private:
 };
 
 template <class T, class E>
-typename std::enable_if<std::is_error_code_enum<E>::value ||
-                            std::is_error_condition_enum<E>::value,
-                        bool>::type
+std::enable_if_t<std::is_error_code_enum<E>::value ||
+                     std::is_error_condition_enum<E>::value,
+                 bool>
 operator==(const ErrorOr<T> &Err, E Code) {
   return Err.getError() == Code;
 }
diff --git a/llvm/include/llvm/Support/FormatProviders.h b/llvm/include/llvm/Support/FormatProviders.h
index 629a4845716a42b9405e5255b01e88c255f46c4d..c31481a292595e020411b52c6573ec31630ed458 100644
--- a/llvm/include/llvm/Support/FormatProviders.h
+++ b/llvm/include/llvm/Support/FormatProviders.h
@@ -124,7 +124,7 @@ protected:
 
 template <typename T>
 struct format_provider<
-    T, typename std::enable_if<detail::use_integral_formatter<T>::value>::type>
+    T, std::enable_if_t<detail::use_integral_formatter<T>::value>>
     : public detail::HelperFunctions {
 private:
 public:
@@ -173,7 +173,7 @@ public:
 /// cases indicates the minimum number of nibbles to print.
 template <typename T>
 struct format_provider<
-    T, typename std::enable_if<detail::use_pointer_formatter<T>::value>::type>
+    T, std::enable_if_t<detail::use_pointer_formatter<T>::value>>
     : public detail::HelperFunctions {
 private:
 public:
@@ -198,7 +198,7 @@ public:
 
 template <typename T>
 struct format_provider<
-    T, typename std::enable_if<detail::use_string_formatter<T>::value>::type> {
+    T, std::enable_if_t<detail::use_string_formatter<T>::value>> {
   static void format(const T &V, llvm::raw_ostream &Stream, StringRef Style) {
     size_t N = StringRef::npos;
     if (!Style.empty() && Style.getAsInteger(10, N)) {
@@ -230,8 +230,8 @@ template <> struct format_provider<Twine> {
 /// character.  Otherwise, it is treated as an integer options string.
 ///
 template <typename T>
-struct format_provider<
-    T, typename std::enable_if<detail::use_char_formatter<T>::value>::type> {
+struct format_provider<T,
+                       std::enable_if_t<detail::use_char_formatter<T>::value>> {
   static void format(const char &V, llvm::raw_ostream &Stream,
                      StringRef Style) {
     if (Style.empty())
@@ -296,8 +296,8 @@ template <> struct format_provider<bool> {
 /// else.
 
 template <typename T>
-struct format_provider<
-    T, typename std::enable_if<detail::use_double_formatter<T>::value>::type>
+struct format_provider<T,
+                       std::enable_if_t<detail::use_double_formatter<T>::value>>
     : public detail::HelperFunctions {
   static void format(const T &V, llvm::raw_ostream &Stream, StringRef Style) {
     FloatStyle S;
diff --git a/llvm/include/llvm/Support/FormatVariadicDetails.h b/llvm/include/llvm/Support/FormatVariadicDetails.h
index e3c185134daa4d4c1bf2612174f753053a4bf2e4..d5e67b756a472f1b45965a0b13ca284362a2d962 100644
--- a/llvm/include/llvm/Support/FormatVariadicDetails.h
+++ b/llvm/include/llvm/Support/FormatVariadicDetails.h
@@ -36,7 +36,7 @@ public:
   explicit provider_format_adapter(T &&Item) : Item(std::forward<T>(Item)) {}
 
   void format(llvm::raw_ostream &S, StringRef Options) override {
-    format_provider<typename std::decay<T>::type>::format(Item, S, Options);
+    format_provider<std::decay_t<T>>::format(Item, S, Options);
   }
 };
 
@@ -59,7 +59,7 @@ template <typename T> class missing_format_adapter;
 //
 template <class T> class has_FormatProvider {
 public:
-  using Decayed = typename std::decay<T>::type;
+  using Decayed = std::decay_t<T>;
   typedef void (*Signature_format)(const Decayed &, llvm::raw_ostream &,
                                    StringRef);
 
@@ -75,14 +75,14 @@ public:
 // Test if raw_ostream& << T -> raw_ostream& is findable via ADL.
 template <class T> class has_StreamOperator {
 public:
-  using ConstRefT = const typename std::decay<T>::type &;
+  using ConstRefT = const std::decay_t<T> &;
 
   template <typename U>
-  static char test(typename std::enable_if<
-                   std::is_same<decltype(std::declval<llvm::raw_ostream &>()
-                                         << std::declval<U>()),
-                                llvm::raw_ostream &>::value,
-                   int *>::type);
+  static char test(
+      std::enable_if_t<std::is_same<decltype(std::declval<llvm::raw_ostream &>()
+                                             << std::declval<U>()),
+                                    llvm::raw_ostream &>::value,
+                       int *>);
 
   template <typename U> static double test(...);
 
@@ -95,8 +95,8 @@ template <typename T>
 struct uses_format_member
     : public std::integral_constant<
           bool,
-          std::is_base_of<format_adapter,
-                          typename std::remove_reference<T>::type>::value> {};
+          std::is_base_of<format_adapter, std::remove_reference_t<T>>::value> {
+};
 
 // Simple template that decides whether a type T should use the format_provider
 // based format() invocation.  The member function takes priority, so this test
@@ -127,34 +127,32 @@ struct uses_missing_provider
 };
 
 template <typename T>
-typename std::enable_if<uses_format_member<T>::value, T>::type
+std::enable_if_t<uses_format_member<T>::value, T>
 build_format_adapter(T &&Item) {
   return std::forward<T>(Item);
 }
 
 template <typename T>
-typename std::enable_if<uses_format_provider<T>::value,
-                        provider_format_adapter<T>>::type
+std::enable_if_t<uses_format_provider<T>::value, provider_format_adapter<T>>
 build_format_adapter(T &&Item) {
   return provider_format_adapter<T>(std::forward<T>(Item));
 }
 
 template <typename T>
-typename std::enable_if<uses_stream_operator<T>::value,
-                        stream_operator_format_adapter<T>>::type
+std::enable_if_t<uses_stream_operator<T>::value,
+                 stream_operator_format_adapter<T>>
 build_format_adapter(T &&Item) {
   // If the caller passed an Error by value, then stream_operator_format_adapter
   // would be responsible for consuming it.
   // Make the caller opt into this by calling fmt_consume().
   static_assert(
-      !std::is_same<llvm::Error, typename std::remove_cv<T>::type>::value,
+      !std::is_same<llvm::Error, std::remove_cv_t<T>>::value,
       "llvm::Error-by-value must be wrapped in fmt_consume() for formatv");
   return stream_operator_format_adapter<T>(std::forward<T>(Item));
 }
 
 template <typename T>
-typename std::enable_if<uses_missing_provider<T>::value,
-                        missing_format_adapter<T>>::type
+std::enable_if_t<uses_missing_provider<T>::value, missing_format_adapter<T>>
 build_format_adapter(T &&Item) {
   return missing_format_adapter<T>();
 }
diff --git a/llvm/include/llvm/Support/GenericDomTree.h b/llvm/include/llvm/Support/GenericDomTree.h
index 2545a075062a1ee1e3fbba34b3600775c71a7a34..1e1fc1fa2486ef77f83f8cc89b2c6dd973fdf095 100644
--- a/llvm/include/llvm/Support/GenericDomTree.h
+++ b/llvm/include/llvm/Support/GenericDomTree.h
@@ -225,7 +225,7 @@ class DominatorTreeBase {
   using ParentPtr = decltype(std::declval<NodeT *>()->getParent());
   static_assert(std::is_pointer<ParentPtr>::value,
                 "Currently NodeT's parent must be a pointer type");
-  using ParentType = typename std::remove_pointer<ParentPtr>::type;
+  using ParentType = std::remove_pointer_t<ParentPtr>;
   static constexpr bool IsPostDominator = IsPostDom;
 
   using UpdateType = cfg::Update<NodePtr>;
diff --git a/llvm/include/llvm/Support/GenericIteratedDominanceFrontier.h b/llvm/include/llvm/Support/GenericIteratedDominanceFrontier.h
index 25eb7cd7b6d5741175b48a273448c82be4c6b193..ddc0f1c246065f579f277d92d7ac9bd96aff759d 100644
--- a/llvm/include/llvm/Support/GenericIteratedDominanceFrontier.h
+++ b/llvm/include/llvm/Support/GenericIteratedDominanceFrontier.h
@@ -57,7 +57,7 @@ template <class NodeTy, bool IsPostDom> struct ChildrenGetterTy {
 template <class NodeTy, bool IsPostDom> class IDFCalculatorBase {
 public:
   using OrderedNodeTy =
-      typename std::conditional<IsPostDom, Inverse<NodeTy *>, NodeTy *>::type;
+      std::conditional_t<IsPostDom, Inverse<NodeTy *>, NodeTy *>;
   using ChildrenGetterTy =
       IDFCalculatorDetail::ChildrenGetterTy<NodeTy, IsPostDom>;
 
diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h
index f0bab51ad4d9d15f86e30c92a50c672f56454571..8b1c66234fe8746c1fa53e019a495fcfa82a64c5 100644
--- a/llvm/include/llvm/Support/JSON.h
+++ b/llvm/include/llvm/Support/JSON.h
@@ -329,32 +329,28 @@ public:
   Value(std::nullptr_t) : Type(T_Null) {}
   // Boolean (disallow implicit conversions).
   // (The last template parameter is a dummy to keep templates distinct.)
-  template <
-      typename T,
-      typename = typename std::enable_if<std::is_same<T, bool>::value>::type,
-      bool = false>
+  template <typename T,
+            typename = std::enable_if_t<std::is_same<T, bool>::value>,
+            bool = false>
   Value(T B) : Type(T_Boolean) {
     create<bool>(B);
   }
   // Integers (except boolean). Must be non-narrowing convertible to int64_t.
-  template <
-      typename T,
-      typename = typename std::enable_if<std::is_integral<T>::value>::type,
-      typename = typename std::enable_if<!std::is_same<T, bool>::value>::type>
+  template <typename T, typename = std::enable_if_t<std::is_integral<T>::value>,
+            typename = std::enable_if_t<!std::is_same<T, bool>::value>>
   Value(T I) : Type(T_Integer) {
     create<int64_t>(int64_t{I});
   }
   // Floating point. Must be non-narrowing convertible to double.
   template <typename T,
-            typename =
-                typename std::enable_if<std::is_floating_point<T>::value>::type,
+            typename = std::enable_if_t<std::is_floating_point<T>::value>,
             double * = nullptr>
   Value(T D) : Type(T_Double) {
     create<double>(double{D});
   }
   // Serializable types: with a toJSON(const T&)->Value function, found by ADL.
   template <typename T,
-            typename = typename std::enable_if<std::is_same<
+            typename = std::enable_if_t<std::is_same<
                 Value, decltype(toJSON(*(const T *)nullptr))>::value>,
             Value * = nullptr>
   Value(const T &V) : Value(toJSON(V)) {}
diff --git a/llvm/include/llvm/Support/MSVCErrorWorkarounds.h b/llvm/include/llvm/Support/MSVCErrorWorkarounds.h
index 30e8febae20b9a91db0a02678feea245b9c166d2..bf983dc1e406b9c0e30d8c7bb94cd0f10fc09ff8 100644
--- a/llvm/include/llvm/Support/MSVCErrorWorkarounds.h
+++ b/llvm/include/llvm/Support/MSVCErrorWorkarounds.h
@@ -59,22 +59,19 @@ public:
   template <typename OtherT>
   MSVCPExpected(
       OtherT &&Val,
-      typename std::enable_if<std::is_convertible<OtherT, T>::value>::type * =
-          nullptr)
+      std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
       : Expected<T>(std::move(Val)) {}
 
   template <class OtherT>
   MSVCPExpected(
       Expected<OtherT> &&Other,
-      typename std::enable_if<std::is_convertible<OtherT, T>::value>::type * =
-          nullptr)
+      std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
       : Expected<T>(std::move(Other)) {}
 
   template <class OtherT>
   explicit MSVCPExpected(
       Expected<OtherT> &&Other,
-      typename std::enable_if<!std::is_convertible<OtherT, T>::value>::type * =
-          nullptr)
+      std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr)
       : Expected<T>(std::move(Other)) {}
 };
 
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index 37b9669cbeed989ce982d56499c959b32ba69f51..5ad055a4f4780eec48aae6fad08d3832480f2f2c 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -364,14 +364,12 @@ constexpr inline bool isShiftedInt(int64_t x) {
 /// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
 /// left too many places.
 template <unsigned N>
-constexpr inline typename std::enable_if<(N < 64), bool>::type
-isUInt(uint64_t X) {
+constexpr inline std::enable_if_t<(N < 64), bool> isUInt(uint64_t X) {
   static_assert(N > 0, "isUInt<0> doesn't make sense");
   return X < (UINT64_C(1) << (N));
 }
 template <unsigned N>
-constexpr inline typename std::enable_if<N >= 64, bool>::type
-isUInt(uint64_t X) {
+constexpr inline std::enable_if_t<N >= 64, bool> isUInt(uint64_t X) {
   return true;
 }
 
@@ -780,8 +778,7 @@ inline int64_t SignExtend64(uint64_t X, unsigned B) {
 /// Subtract two unsigned integers, X and Y, of type T and return the absolute
 /// value of the result.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, T>::type
-AbsoluteDifference(T X, T Y) {
+std::enable_if_t<std::is_unsigned<T>::value, T> AbsoluteDifference(T X, T Y) {
   return std::max(X, Y) - std::min(X, Y);
 }
 
@@ -789,7 +786,7 @@ AbsoluteDifference(T X, T Y) {
 /// maximum representable value of T on overflow.  ResultOverflowed indicates if
 /// the result is larger than the maximum representable value of type T.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, T>::type
+std::enable_if_t<std::is_unsigned<T>::value, T>
 SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
   bool Dummy;
   bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
@@ -806,7 +803,7 @@ SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
 /// maximum representable value of T on overflow.  ResultOverflowed indicates if
 /// the result is larger than the maximum representable value of type T.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, T>::type
+std::enable_if_t<std::is_unsigned<T>::value, T>
 SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
   bool Dummy;
   bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
@@ -852,7 +849,7 @@ SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
 /// overflow. ResultOverflowed indicates if the result is larger than the
 /// maximum representable value of type T.
 template <typename T>
-typename std::enable_if<std::is_unsigned<T>::value, T>::type
+std::enable_if_t<std::is_unsigned<T>::value, T>
 SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
   bool Dummy;
   bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
@@ -871,13 +868,12 @@ extern const float huge_valf;
 /// Add two signed integers, computing the two's complement truncated result,
 /// returning true if overflow occured.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, T>::type
-AddOverflow(T X, T Y, T &Result) {
+std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) {
 #if __has_builtin(__builtin_add_overflow)
   return __builtin_add_overflow(X, Y, &Result);
 #else
   // Perform the unsigned addition.
-  using U = typename std::make_unsigned<T>::type;
+  using U = std::make_unsigned_t<T>;
   const U UX = static_cast<U>(X);
   const U UY = static_cast<U>(Y);
   const U UResult = UX + UY;
@@ -898,13 +894,12 @@ AddOverflow(T X, T Y, T &Result) {
 /// Subtract two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow ocurred.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, T>::type
-SubOverflow(T X, T Y, T &Result) {
+std::enable_if_t<std::is_signed<T>::value, T> SubOverflow(T X, T Y, T &Result) {
 #if __has_builtin(__builtin_sub_overflow)
   return __builtin_sub_overflow(X, Y, &Result);
 #else
   // Perform the unsigned addition.
-  using U = typename std::make_unsigned<T>::type;
+  using U = std::make_unsigned_t<T>;
   const U UX = static_cast<U>(X);
   const U UY = static_cast<U>(Y);
   const U UResult = UX - UY;
@@ -922,14 +917,12 @@ SubOverflow(T X, T Y, T &Result) {
 #endif
 }
 
-
 /// Multiply two signed integers, computing the two's complement truncated
 /// result, returning true if an overflow ocurred.
 template <typename T>
-typename std::enable_if<std::is_signed<T>::value, T>::type
-MulOverflow(T X, T Y, T &Result) {
+std::enable_if_t<std::is_signed<T>::value, T> MulOverflow(T X, T Y, T &Result) {
   // Perform the unsigned multiplication on absolute values.
-  using U = typename std::make_unsigned<T>::type;
+  using U = std::make_unsigned_t<T>;
   const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
   const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
   const U UResult = UX * UY;
diff --git a/llvm/include/llvm/Support/Process.h b/llvm/include/llvm/Support/Process.h
index 9444e165e142a6196425334188e58dec9d6ef995..bb5c33dfb38d5afe49323a8a6d7cea83c3d1ff8b 100644
--- a/llvm/include/llvm/Support/Process.h
+++ b/llvm/include/llvm/Support/Process.h
@@ -201,6 +201,12 @@ public:
   /// Get the result of a process wide random number generator. The
   /// generator will be automatically seeded in non-deterministic fashion.
   static unsigned GetRandomNumber();
+
+  /// Equivalent to ::exit(), except when running inside a CrashRecoveryContext.
+  /// In that case, the control flow will resume after RunSafely(), like for a
+  /// crash, rather than exiting the current process.
+  LLVM_ATTRIBUTE_NORETURN
+  static void Exit(int RetCode);
 };
 
 }
diff --git a/llvm/include/llvm/Support/SwapByteOrder.h b/llvm/include/llvm/Support/SwapByteOrder.h
index 6cec87006c02cf1b267f7eede5720506dc9bd0ab..700ede7eb589c4d371082f9a95f41ac54ce4765f 100644
--- a/llvm/include/llvm/Support/SwapByteOrder.h
+++ b/llvm/include/llvm/Support/SwapByteOrder.h
@@ -143,10 +143,9 @@ inline double getSwappedBytes(double C) {
 }
 
 template <typename T>
-inline typename std::enable_if<std::is_enum<T>::value, T>::type
-getSwappedBytes(T C) {
+inline std::enable_if_t<std::is_enum<T>::value, T> getSwappedBytes(T C) {
   return static_cast<T>(
-      getSwappedBytes(static_cast<typename std::underlying_type<T>::type>(C)));
+      getSwappedBytes(static_cast<std::underlying_type_t<T>>(C)));
 }
 
 template<typename T>
diff --git a/llvm/include/llvm/Support/TaskQueue.h b/llvm/include/llvm/Support/TaskQueue.h
index df2ffdee2cc24ad6f417e8181b2883de5cf8a0f5..4ceb056391af07f7217fc7a5cca8812c4c93ca08 100644
--- a/llvm/include/llvm/Support/TaskQueue.h
+++ b/llvm/include/llvm/Support/TaskQueue.h
@@ -38,7 +38,7 @@ class TaskQueue {
   // type-specialized domain (before type erasure) and then erase this into a
   // std::function.
   template <typename Callable> struct Task {
-    using ResultTy = typename std::result_of<Callable()>::type;
+    using ResultTy = std::result_of_t<Callable()>;
     explicit Task(Callable C, TaskQueue &Parent)
         : C(std::move(C)), P(std::make_shared<std::promise<ResultTy>>()),
           Parent(&Parent) {}
@@ -78,13 +78,13 @@ public:
   /// used to wait for the task (and all previous tasks that have not yet
   /// completed) to finish.
   template <typename Callable>
-  std::future<typename std::result_of<Callable()>::type> async(Callable &&C) {
+  std::future<std::result_of_t<Callable()>> async(Callable &&C) {
 #if !LLVM_ENABLE_THREADS
     static_assert(false,
                   "TaskQueue requires building with LLVM_ENABLE_THREADS!");
 #endif
     Task<Callable> T{std::move(C), *this};
-    using ResultTy = typename std::result_of<Callable()>::type;
+    using ResultTy = std::result_of_t<Callable()>;
     std::future<ResultTy> F = T.P->get_future();
     {
       std::lock_guard<std::mutex> Lock(QueueLock);
diff --git a/llvm/include/llvm/Support/TrailingObjects.h b/llvm/include/llvm/Support/TrailingObjects.h
index 49be89613c43498609d9f94ab5a150c2b2575b08..0d9c4503aa9be454b898ddbdacd9d9ed68411d73 100644
--- a/llvm/include/llvm/Support/TrailingObjects.h
+++ b/llvm/include/llvm/Support/TrailingObjects.h
@@ -326,8 +326,8 @@ public:
   /// used in the class; they are supplied here redundantly only so
   /// that it's clear what the counts are counting in callers.
   template <typename... Tys>
-  static constexpr typename std::enable_if<
-      std::is_same<Foo<TrailingTys...>, Foo<Tys...>>::value, size_t>::type
+  static constexpr std::enable_if_t<
+      std::is_same<Foo<TrailingTys...>, Foo<Tys...>>::value, size_t>
   additionalSizeToAlloc(typename trailing_objects_internal::ExtractSecondType<
                         TrailingTys, size_t>::type... Counts) {
     return ParentType::additionalSizeToAllocImpl(0, Counts...);
@@ -338,8 +338,8 @@ public:
   /// additionalSizeToAlloc, except it *does* include the size of the base
   /// object.
   template <typename... Tys>
-  static constexpr typename std::enable_if<
-      std::is_same<Foo<TrailingTys...>, Foo<Tys...>>::value, size_t>::type
+  static constexpr std::enable_if_t<
+      std::is_same<Foo<TrailingTys...>, Foo<Tys...>>::value, size_t>
   totalSizeToAlloc(typename trailing_objects_internal::ExtractSecondType<
                    TrailingTys, size_t>::type... Counts) {
     return sizeof(BaseTy) + ParentType::additionalSizeToAllocImpl(0, Counts...);
diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h
index 757dc8ea191a65513a7f495d89fe7cccdc4a1802..316a6ad5d0944516bd7d05895726d6a7dd16f8f3 100644
--- a/llvm/include/llvm/Support/YAMLTraits.h
+++ b/llvm/include/llvm/Support/YAMLTraits.h
@@ -868,7 +868,7 @@ public:
   }
 
   template <typename T, typename Context>
-  typename std::enable_if<has_SequenceTraits<T>::value, void>::type
+  std::enable_if_t<has_SequenceTraits<T>::value, void>
   mapOptionalWithContext(const char *Key, T &Val, Context &Ctx) {
     // omit key/value instead of outputting empty sequence
     if (this->canElideEmptySequence() && !(Val.begin() != Val.end()))
@@ -883,7 +883,7 @@ public:
   }
 
   template <typename T, typename Context>
-  typename std::enable_if<!has_SequenceTraits<T>::value, void>::type
+  std::enable_if_t<!has_SequenceTraits<T>::value, void>
   mapOptionalWithContext(const char *Key, T &Val, Context &Ctx) {
     this->processKey(Key, Val, false, Ctx);
   }
@@ -965,7 +965,7 @@ template <typename T> void doMapping(IO &io, T &Val, EmptyContext &Ctx) {
 } // end namespace detail
 
 template <typename T>
-typename std::enable_if<has_ScalarEnumerationTraits<T>::value, void>::type
+std::enable_if_t<has_ScalarEnumerationTraits<T>::value, void>
 yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
   io.beginEnumScalar();
   ScalarEnumerationTraits<T>::enumeration(io, Val);
@@ -973,7 +973,7 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<has_ScalarBitSetTraits<T>::value, void>::type
+std::enable_if_t<has_ScalarBitSetTraits<T>::value, void>
 yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
   bool DoClear;
   if ( io.beginBitSetScalar(DoClear) ) {
@@ -985,8 +985,8 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<has_ScalarTraits<T>::value, void>::type
-yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
+std::enable_if_t<has_ScalarTraits<T>::value, void> yamlize(IO &io, T &Val, bool,
+                                                           EmptyContext &Ctx) {
   if ( io.outputting() ) {
     std::string Storage;
     raw_string_ostream Buffer(Storage);
@@ -1005,7 +1005,7 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<has_BlockScalarTraits<T>::value, void>::type
+std::enable_if_t<has_BlockScalarTraits<T>::value, void>
 yamlize(IO &YamlIO, T &Val, bool, EmptyContext &Ctx) {
   if (YamlIO.outputting()) {
     std::string Storage;
@@ -1024,7 +1024,7 @@ yamlize(IO &YamlIO, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<has_TaggedScalarTraits<T>::value, void>::type
+std::enable_if_t<has_TaggedScalarTraits<T>::value, void>
 yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
   if (io.outputting()) {
     std::string ScalarStorage, TagStorage;
@@ -1049,7 +1049,7 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T, typename Context>
-typename std::enable_if<validatedMappingTraits<T, Context>::value, void>::type
+std::enable_if_t<validatedMappingTraits<T, Context>::value, void>
 yamlize(IO &io, T &Val, bool, Context &Ctx) {
   if (has_FlowTraits<MappingTraits<T>>::value)
     io.beginFlowMapping();
@@ -1075,7 +1075,7 @@ yamlize(IO &io, T &Val, bool, Context &Ctx) {
 }
 
 template <typename T, typename Context>
-typename std::enable_if<unvalidatedMappingTraits<T, Context>::value, void>::type
+std::enable_if_t<unvalidatedMappingTraits<T, Context>::value, void>
 yamlize(IO &io, T &Val, bool, Context &Ctx) {
   if (has_FlowTraits<MappingTraits<T>>::value) {
     io.beginFlowMapping();
@@ -1089,7 +1089,7 @@ yamlize(IO &io, T &Val, bool, Context &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<has_CustomMappingTraits<T>::value, void>::type
+std::enable_if_t<has_CustomMappingTraits<T>::value, void>
 yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
   if ( io.outputting() ) {
     io.beginMapping();
@@ -1104,7 +1104,7 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<has_PolymorphicTraits<T>::value, void>::type
+std::enable_if_t<has_PolymorphicTraits<T>::value, void>
 yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
   switch (io.outputting() ? PolymorphicTraits<T>::getKind(Val)
                           : io.getNodeKind()) {
@@ -1118,13 +1118,13 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
 }
 
 template <typename T>
-typename std::enable_if<missingTraits<T, EmptyContext>::value, void>::type
+std::enable_if_t<missingTraits<T, EmptyContext>::value, void>
 yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) {
   char missing_yaml_trait_for_type[sizeof(MissingTrait<T>)];
 }
 
 template <typename T, typename Context>
-typename std::enable_if<has_SequenceTraits<T>::value, void>::type
+std::enable_if_t<has_SequenceTraits<T>::value, void>
 yamlize(IO &io, T &Seq, bool, Context &Ctx) {
   if ( has_FlowTraits< SequenceTraits<T>>::value ) {
     unsigned incnt = io.beginFlowSequence();
@@ -1247,10 +1247,9 @@ struct ScalarTraits<double> {
 // type.  This way endian aware types are supported whenever the traits are
 // defined for the underlying type.
 template <typename value_type, support::endianness endian, size_t alignment>
-struct ScalarTraits<
-    support::detail::packed_endian_specific_integral<value_type, endian,
-                                                     alignment>,
-    typename std::enable_if<has_ScalarTraits<value_type>::value>::type> {
+struct ScalarTraits<support::detail::packed_endian_specific_integral<
+                        value_type, endian, alignment>,
+                    std::enable_if_t<has_ScalarTraits<value_type>::value>> {
   using endian_type =
       support::detail::packed_endian_specific_integral<value_type, endian,
                                                        alignment>;
@@ -1275,8 +1274,7 @@ template <typename value_type, support::endianness endian, size_t alignment>
 struct ScalarEnumerationTraits<
     support::detail::packed_endian_specific_integral<value_type, endian,
                                                      alignment>,
-    typename std::enable_if<
-        has_ScalarEnumerationTraits<value_type>::value>::type> {
+    std::enable_if_t<has_ScalarEnumerationTraits<value_type>::value>> {
   using endian_type =
       support::detail::packed_endian_specific_integral<value_type, endian,
                                                        alignment>;
@@ -1292,7 +1290,7 @@ template <typename value_type, support::endianness endian, size_t alignment>
 struct ScalarBitSetTraits<
     support::detail::packed_endian_specific_integral<value_type, endian,
                                                      alignment>,
-    typename std::enable_if<has_ScalarBitSetTraits<value_type>::value>::type> {
+    std::enable_if_t<has_ScalarBitSetTraits<value_type>::value>> {
   using endian_type =
       support::detail::packed_endian_specific_integral<value_type, endian,
                                                        alignment>;
@@ -1688,8 +1686,7 @@ struct ScalarTraits<Hex64> {
 
 // Define non-member operator>> so that Input can stream in a document list.
 template <typename T>
-inline
-typename std::enable_if<has_DocumentListTraits<T>::value, Input &>::type
+inline std::enable_if_t<has_DocumentListTraits<T>::value, Input &>
 operator>>(Input &yin, T &docList) {
   int i = 0;
   EmptyContext Ctx;
@@ -1705,8 +1702,7 @@ operator>>(Input &yin, T &docList) {
 
 // Define non-member operator>> so that Input can stream in a map as a document.
 template <typename T>
-inline typename std::enable_if<has_MappingTraits<T, EmptyContext>::value,
-                               Input &>::type
+inline std::enable_if_t<has_MappingTraits<T, EmptyContext>::value, Input &>
 operator>>(Input &yin, T &docMap) {
   EmptyContext Ctx;
   yin.setCurrentDocument();
@@ -1717,8 +1713,7 @@ operator>>(Input &yin, T &docMap) {
 // Define non-member operator>> so that Input can stream in a sequence as
 // a document.
 template <typename T>
-inline
-typename std::enable_if<has_SequenceTraits<T>::value, Input &>::type
+inline std::enable_if_t<has_SequenceTraits<T>::value, Input &>
 operator>>(Input &yin, T &docSeq) {
   EmptyContext Ctx;
   if (yin.setCurrentDocument())
@@ -1728,8 +1723,7 @@ operator>>(Input &yin, T &docSeq) {
 
 // Define non-member operator>> so that Input can stream in a block scalar.
 template <typename T>
-inline
-typename std::enable_if<has_BlockScalarTraits<T>::value, Input &>::type
+inline std::enable_if_t<has_BlockScalarTraits<T>::value, Input &>
 operator>>(Input &In, T &Val) {
   EmptyContext Ctx;
   if (In.setCurrentDocument())
@@ -1739,8 +1733,7 @@ operator>>(Input &In, T &Val) {
 
 // Define non-member operator>> so that Input can stream in a string map.
 template <typename T>
-inline
-typename std::enable_if<has_CustomMappingTraits<T>::value, Input &>::type
+inline std::enable_if_t<has_CustomMappingTraits<T>::value, Input &>
 operator>>(Input &In, T &Val) {
   EmptyContext Ctx;
   if (In.setCurrentDocument())
@@ -1750,7 +1743,7 @@ operator>>(Input &In, T &Val) {
 
 // Define non-member operator>> so that Input can stream in a polymorphic type.
 template <typename T>
-inline typename std::enable_if<has_PolymorphicTraits<T>::value, Input &>::type
+inline std::enable_if_t<has_PolymorphicTraits<T>::value, Input &>
 operator>>(Input &In, T &Val) {
   EmptyContext Ctx;
   if (In.setCurrentDocument())
@@ -1760,8 +1753,7 @@ operator>>(Input &In, T &Val) {
 
 // Provide better error message about types missing a trait specialization
 template <typename T>
-inline typename std::enable_if<missingTraits<T, EmptyContext>::value,
-                               Input &>::type
+inline std::enable_if_t<missingTraits<T, EmptyContext>::value, Input &>
 operator>>(Input &yin, T &docSeq) {
   char missing_yaml_trait_for_type[sizeof(MissingTrait<T>)];
   return yin;
@@ -1769,8 +1761,7 @@ operator>>(Input &yin, T &docSeq) {
 
 // Define non-member operator<< so that Output can stream out document list.
 template <typename T>
-inline
-typename std::enable_if<has_DocumentListTraits<T>::value, Output &>::type
+inline std::enable_if_t<has_DocumentListTraits<T>::value, Output &>
 operator<<(Output &yout, T &docList) {
   EmptyContext Ctx;
   yout.beginDocuments();
@@ -1788,8 +1779,7 @@ operator<<(Output &yout, T &docList) {
 
 // Define non-member operator<< so that Output can stream out a map.
 template <typename T>
-inline typename std::enable_if<has_MappingTraits<T, EmptyContext>::value,
-                               Output &>::type
+inline std::enable_if_t<has_MappingTraits<T, EmptyContext>::value, Output &>
 operator<<(Output &yout, T &map) {
   EmptyContext Ctx;
   yout.beginDocuments();
@@ -1803,8 +1793,7 @@ operator<<(Output &yout, T &map) {
 
 // Define non-member operator<< so that Output can stream out a sequence.
 template <typename T>
-inline
-typename std::enable_if<has_SequenceTraits<T>::value, Output &>::type
+inline std::enable_if_t<has_SequenceTraits<T>::value, Output &>
 operator<<(Output &yout, T &seq) {
   EmptyContext Ctx;
   yout.beginDocuments();
@@ -1818,8 +1807,7 @@ operator<<(Output &yout, T &seq) {
 
 // Define non-member operator<< so that Output can stream out a block scalar.
 template <typename T>
-inline
-typename std::enable_if<has_BlockScalarTraits<T>::value, Output &>::type
+inline std::enable_if_t<has_BlockScalarTraits<T>::value, Output &>
 operator<<(Output &Out, T &Val) {
   EmptyContext Ctx;
   Out.beginDocuments();
@@ -1833,8 +1821,7 @@ operator<<(Output &Out, T &Val) {
 
 // Define non-member operator<< so that Output can stream out a string map.
 template <typename T>
-inline
-typename std::enable_if<has_CustomMappingTraits<T>::value, Output &>::type
+inline std::enable_if_t<has_CustomMappingTraits<T>::value, Output &>
 operator<<(Output &Out, T &Val) {
   EmptyContext Ctx;
   Out.beginDocuments();
@@ -1849,7 +1836,7 @@ operator<<(Output &Out, T &Val) {
 // Define non-member operator<< so that Output can stream out a polymorphic
 // type.
 template <typename T>
-inline typename std::enable_if<has_PolymorphicTraits<T>::value, Output &>::type
+inline std::enable_if_t<has_PolymorphicTraits<T>::value, Output &>
 operator<<(Output &Out, T &Val) {
   EmptyContext Ctx;
   Out.beginDocuments();
@@ -1866,8 +1853,7 @@ operator<<(Output &Out, T &Val) {
 
 // Provide better error message about types missing a trait specialization
 template <typename T>
-inline typename std::enable_if<missingTraits<T, EmptyContext>::value,
-                               Output &>::type
+inline std::enable_if_t<missingTraits<T, EmptyContext>::value, Output &>
 operator<<(Output &yout, T &seq) {
   char missing_yaml_trait_for_type[sizeof(MissingTrait<T>)];
   return yout;
@@ -1898,25 +1884,25 @@ template <bool> struct CheckIsBool { static const bool value = true; };
 // If T has SequenceElementTraits, then vector<T> and SmallVector<T, N> have
 // SequenceTraits that do the obvious thing.
 template <typename T>
-struct SequenceTraits<std::vector<T>,
-                      typename std::enable_if<CheckIsBool<
-                          SequenceElementTraits<T>::flow>::value>::type>
+struct SequenceTraits<
+    std::vector<T>,
+    std::enable_if_t<CheckIsBool<SequenceElementTraits<T>::flow>::value>>
     : SequenceTraitsImpl<std::vector<T>, SequenceElementTraits<T>::flow> {};
 template <typename T, unsigned N>
-struct SequenceTraits<SmallVector<T, N>,
-                      typename std::enable_if<CheckIsBool<
-                          SequenceElementTraits<T>::flow>::value>::type>
+struct SequenceTraits<
+    SmallVector<T, N>,
+    std::enable_if_t<CheckIsBool<SequenceElementTraits<T>::flow>::value>>
     : SequenceTraitsImpl<SmallVector<T, N>, SequenceElementTraits<T>::flow> {};
 template <typename T>
-struct SequenceTraits<SmallVectorImpl<T>,
-                      typename std::enable_if<CheckIsBool<
-                          SequenceElementTraits<T>::flow>::value>::type>
+struct SequenceTraits<
+    SmallVectorImpl<T>,
+    std::enable_if_t<CheckIsBool<SequenceElementTraits<T>::flow>::value>>
     : SequenceTraitsImpl<SmallVectorImpl<T>, SequenceElementTraits<T>::flow> {};
 
 // Sequences of fundamental types use flow formatting.
 template <typename T>
-struct SequenceElementTraits<
-    T, typename std::enable_if<std::is_fundamental<T>::value>::type> {
+struct SequenceElementTraits<T,
+                             std::enable_if_t<std::is_fundamental<T>::value>> {
   static const bool flow = true;
 };
 
diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h
index 12a4ff2540ee07872d8594928455aa6ed5da6e2c..30a1e62330dfea7ee34a569e010cd8ed5247a518 100644
--- a/llvm/include/llvm/Support/raw_ostream.h
+++ b/llvm/include/llvm/Support/raw_ostream.h
@@ -358,9 +358,9 @@ private:
 /// Call the appropriate insertion operator, given an rvalue reference to a
 /// raw_ostream object and return a stream of the same type as the argument.
 template <typename OStream, typename T>
-typename std::enable_if<!std::is_reference<OStream>::value &&
-                            std::is_base_of<raw_ostream, OStream>::value,
-                        OStream &&>::type
+std::enable_if_t<!std::is_reference<OStream>::value &&
+                     std::is_base_of<raw_ostream, OStream>::value,
+                 OStream &&>
 operator<<(OStream &&OS, const T &Value) {
   OS << Value;
   return std::move(OS);
diff --git a/llvm/include/llvm/Support/type_traits.h b/llvm/include/llvm/Support/type_traits.h
index b7d48e8e1ade5a8db99916cbb9f3e4bf259cffa4..7b7d5d991f3f51862fad3267798737bc6ef8c845 100644
--- a/llvm/include/llvm/Support/type_traits.h
+++ b/llvm/include/llvm/Support/type_traits.h
@@ -28,7 +28,7 @@ namespace llvm {
 /// Also note that enum classes aren't implicitly convertible to integral types,
 /// the value may therefore need to be explicitly converted before being used.
 template <typename T> class is_integral_or_enum {
-  using UnderlyingT = typename std::remove_reference<T>::type;
+  using UnderlyingT = std::remove_reference_t<T>;
 
 public:
   static const bool value =
@@ -45,7 +45,7 @@ struct add_lvalue_reference_if_not_pointer { using type = T &; };
 
 template <typename T>
 struct add_lvalue_reference_if_not_pointer<
-    T, typename std::enable_if<std::is_pointer<T>::value>::type> {
+    T, std::enable_if_t<std::is_pointer<T>::value>> {
   using type = T;
 };
 
@@ -55,9 +55,8 @@ template<typename T, typename Enable = void>
 struct add_const_past_pointer { using type = const T; };
 
 template <typename T>
-struct add_const_past_pointer<
-    T, typename std::enable_if<std::is_pointer<T>::value>::type> {
-  using type = const typename std::remove_pointer<T>::type *;
+struct add_const_past_pointer<T, std::enable_if_t<std::is_pointer<T>::value>> {
+  using type = const std::remove_pointer_t<T> *;
 };
 
 template <typename T, typename Enable = void>
@@ -65,8 +64,8 @@ struct const_pointer_or_const_ref {
   using type = const T &;
 };
 template <typename T>
-struct const_pointer_or_const_ref<
-    T, typename std::enable_if<std::is_pointer<T>::value>::type> {
+struct const_pointer_or_const_ref<T,
+                                  std::enable_if_t<std::is_pointer<T>::value>> {
   using type = typename add_const_past_pointer<T>::type;
 };
 
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 176ae39b17a7cd01d39859165dc552862aebb76f..3c2e2fe1a37f26e36534689c2b4f8fe4e37cb3bf 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -237,6 +237,9 @@ public:
   void setSupportsDefaultOutlining(bool Enable) {
     Options.SupportsDefaultOutlining = Enable;
   }
+  void setSupportsDebugEntryValues(bool Enable) {
+    Options.SupportsDebugEntryValues = Enable;
+  }
 
   bool shouldPrintMachineCode() const { return Options.PrintMachineCode; }
 
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 84c6ee2a63878a60433e6e0a0a1f916a9a644517..535909cc77c3137ccb61e2c072a370a6de430828 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -119,7 +119,8 @@ namespace llvm {
           ExplicitEmulatedTLS(false), EnableIPRA(false),
           EmitStackSizeSection(false), EnableMachineOutliner(false),
           SupportsDefaultOutlining(false), EmitAddrsig(false),
-          EnableDebugEntryValues(false), ForceDwarfFrameSection(false) {}
+          SupportsDebugEntryValues(false), EnableDebugEntryValues(false),
+          ForceDwarfFrameSection(false) {}
 
     /// PrintMachineCode - This flag is enabled when the -print-machineinstrs
     /// option is specified on the command line, and should enable debugging
@@ -256,8 +257,18 @@ namespace llvm {
     /// Emit address-significance table.
     unsigned EmitAddrsig : 1;
 
-    /// Emit debug info about parameter's entry values.
+    /// Set if the target supports the debug entry values by default.
+    unsigned SupportsDebugEntryValues : 1;
+    /// When set to true, the EnableDebugEntryValues option forces production
+    /// of debug entry values even if the target does not officially support
+    /// it. Useful for testing purposes only. This flag should never be checked
+    /// directly, always use \ref ShouldEmitDebugEntryValues instead.
     unsigned EnableDebugEntryValues : 1;
+    /// NOTE: There are targets that still do not support the call site info
+    /// production (the info about the arguments passed to the call, necessary
+    /// for the debug entry values), so we keep using the experimental option
+    /// (-debug-entry-values) to test them as well.
+    bool ShouldEmitDebugEntryValues() const;
 
     /// Emit DWARF debug frame section.
     unsigned ForceDwarfFrameSection : 1;
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 2228b62c0ca6910de996064ce7eb2b1f6a5b97ad..8437ed260f6c81f02843f41814db604e949d02dc 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -1023,17 +1023,6 @@ struct Attributor {
   /// Return the data layout associated with the anchor scope.
   const DataLayout &getDataLayout() const { return InfoCache.DL; }
 
-  /// Replace all uses of \p Old with \p New and, for calls (and invokes),
-  /// update the call graph.
-  void replaceAllUsesWith(Value &Old, Value &New) {
-    if (CallBase *OldCB = dyn_cast<CallBase>(&Old)) {
-      // We do not modify the call graph here but simply reanalyze the old
-      // function. This should be revisited once the old PM is gone.
-      CGModifiedFunctions.insert(OldCB->getFunction());
-    }
-    Old.replaceAllUsesWith(&New);
-  }
-
 private:
   /// Check \p Pred on all call sites of \p Fn.
   ///
diff --git a/llvm/include/llvm/XRay/Graph.h b/llvm/include/llvm/XRay/Graph.h
index 004681512800f9b80c08ac8791f1b36acc12333f..d368f7e724d88595487a26fcca957a22faf1582f 100644
--- a/llvm/include/llvm/XRay/Graph.h
+++ b/llvm/include/llvm/XRay/Graph.h
@@ -126,14 +126,14 @@ private:
   /// set.
   template <bool IsConst, bool IsOut,
             typename BaseIt = typename NeighborSetT::const_iterator,
-            typename T = typename std::conditional<IsConst, const EdgeValueType,
-                                                   EdgeValueType>::type>
+            typename T =
+                std::conditional_t<IsConst, const EdgeValueType, EdgeValueType>>
   class NeighborEdgeIteratorT
       : public iterator_adaptor_base<
             NeighborEdgeIteratorT<IsConst, IsOut>, BaseIt,
             typename std::iterator_traits<BaseIt>::iterator_category, T> {
     using InternalEdgeMapT =
-        typename std::conditional<IsConst, const EdgeMapT, EdgeMapT>::type;
+        std::conditional_t<IsConst, const EdgeMapT, EdgeMapT>;
 
     friend class NeighborEdgeIteratorT<false, IsOut, BaseIt, EdgeValueType>;
     friend class NeighborEdgeIteratorT<true, IsOut, BaseIt,
@@ -144,7 +144,7 @@ private:
 
   public:
     template <bool IsConstDest,
-              typename = typename std::enable_if<IsConstDest && !IsConst>::type>
+              typename = std::enable_if<IsConstDest && !IsConst>>
     operator NeighborEdgeIteratorT<IsConstDest, IsOut, BaseIt,
                                    const EdgeValueType>() const {
       return NeighborEdgeIteratorT<IsConstDest, IsOut, BaseIt,
@@ -199,9 +199,9 @@ public:
   public:
     using iterator = NeighborEdgeIteratorT<isConst, isOut>;
     using const_iterator = NeighborEdgeIteratorT<true, isOut>;
-    using GraphT = typename std::conditional<isConst, const Graph, Graph>::type;
+    using GraphT = std::conditional_t<isConst, const Graph, Graph>;
     using InternalEdgeMapT =
-        typename std::conditional<isConst, const EdgeMapT, EdgeMapT>::type;
+        std::conditional_t<isConst, const EdgeMapT, EdgeMapT>;
 
   private:
     InternalEdgeMapT &M;
@@ -272,10 +272,10 @@ public:
   /// the number of elements in the range and whether the range is empty.
   template <bool isConst> class VertexView {
   public:
-    using iterator = typename std::conditional<isConst, ConstVertexIterator,
-                                               VertexIterator>::type;
+    using iterator =
+        std::conditional_t<isConst, ConstVertexIterator, VertexIterator>;
     using const_iterator = ConstVertexIterator;
-    using GraphT = typename std::conditional<isConst, const Graph, Graph>::type;
+    using GraphT = std::conditional_t<isConst, const Graph, Graph>;
 
   private:
     GraphT &G;
@@ -309,10 +309,10 @@ public:
   /// the number of elements in the range and whether the range is empty.
   template <bool isConst> class EdgeView {
   public:
-    using iterator = typename std::conditional<isConst, ConstEdgeIterator,
-                                               EdgeIterator>::type;
+    using iterator =
+        std::conditional_t<isConst, ConstEdgeIterator, EdgeIterator>;
     using const_iterator = ConstEdgeIterator;
-    using GraphT = typename std::conditional<isConst, const Graph, Graph>::type;
+    using GraphT = std::conditional_t<isConst, const Graph, Graph>;
 
   private:
     GraphT &G;
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 894afc643ca03656f30484163a6a7e243f766445..0cc28efd44898127ed715e0d8d595d2e095f92bf 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -2059,12 +2059,13 @@ char BasicAAWrapperPass::ID = 0;
 void BasicAAWrapperPass::anchor() {}
 
 INITIALIZE_PASS_BEGIN(BasicAAWrapperPass, "basicaa",
-                      "Basic Alias Analysis (stateless AA impl)", false, true)
+                      "Basic Alias Analysis (stateless AA impl)", true, true)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(PhiValuesWrapperPass)
 INITIALIZE_PASS_END(BasicAAWrapperPass, "basicaa",
-                    "Basic Alias Analysis (stateless AA impl)", false, true)
+                    "Basic Alias Analysis (stateless AA impl)", true, true)
 
 FunctionPass *llvm::createBasicAAWrapperPass() {
   return new BasicAAWrapperPass();
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index d61edc07116c3c27ef2b630b353a0c24182ca86c..730ae2ffbd413474b9988ff242e863419661b43e 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2400,6 +2400,11 @@ static Constant *ConstantFoldVectorCall(StringRef Name,
   SmallVector<Constant *, 4> Lane(Operands.size());
   Type *Ty = VTy->getElementType();
 
+  // Do not iterate on scalable vector. The number of elements is unknown at
+  // compile-time.
+  if (VTy->getVectorIsScalable())
+    return nullptr;
+
   if (IntrinsicID == Intrinsic::masked_load) {
     auto *SrcPtr = Operands[0];
     auto *Mask = Operands[2];
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 453c5cdd5410a12a924ad8a225f28e5394a57cdc..b22e7cb4f581bb6b8e27780da63e2450e17804c2 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -7660,6 +7660,9 @@ bool LLParser::ParseTypeTestResolution(TypeTestResolution &TTRes) {
     return true;
 
   switch (Lex.getKind()) {
+  case lltok::kw_unknown:
+    TTRes.TheKind = TypeTestResolution::Unknown;
+    break;
   case lltok::kw_unsat:
     TTRes.TheKind = TypeTestResolution::Unsat;
     break;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index fd27f241d808ae8521c5c7ac9cfe239e19bbfe30..cbfbf62a48660f9e5b144a1df79e54074174501d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -95,6 +95,10 @@ static cl::opt<bool> UseDwarfRangesBaseAddressSpecifier(
     "use-dwarf-ranges-base-address-specifier", cl::Hidden,
     cl::desc("Use base address specifiers in debug_ranges"), cl::init(false));
 
+static cl::opt<bool> EmitDwarfDebugEntryValues(
+    "emit-debug-entry-values", cl::Hidden,
+    cl::desc("Emit the debug entry values"), cl::init(false));
+
 static cl::opt<bool> GenerateARangeSection("generate-arange-section",
                                            cl::Hidden,
                                            cl::desc("Generate dwarf aranges"),
@@ -419,6 +423,12 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
   // a monolithic string offsets table without any header.
   UseSegmentedStringOffsetsTable = DwarfVersion >= 5;
 
+  // Emit call-site-param debug info for GDB and LLDB, if the target supports
+  // the debug entry values feature. It can also be enabled explicitly.
+  EmitDebugEntryValues = (Asm->TM.Options.ShouldEmitDebugEntryValues() &&
+                          (tuneForGDB() || tuneForLLDB())) ||
+                         EmitDwarfDebugEntryValues;
+
   Asm->OutStreamer->getContext().setDwarfVersion(DwarfVersion);
 }
 
@@ -767,6 +777,11 @@ void DwarfDebug::constructCallSiteEntryDIEs(const DISubprogram &SP,
       if (!MI.isCandidateForCallSiteEntry())
         continue;
 
+      // Skip instructions marked as frame setup, as they are not interesting to
+      // the user.
+      if (MI.getFlag(MachineInstr::FrameSetup))
+        continue;
+
       // TODO: Add support for targets with delay slots (see: beginInstruction).
       if (MI.hasDelaySlot())
         return;
@@ -835,9 +850,8 @@ void DwarfDebug::constructCallSiteEntryDIEs(const DISubprogram &SP,
       DIE &CallSiteDIE = CU.constructCallSiteEntryDIE(ScopeDIE, CalleeDIE,
                                                       IsTail, PCAddr, CallReg);
 
-      // GDB and LLDB support call site parameter debug info.
-      if (Asm->TM.Options.EnableDebugEntryValues &&
-          (tuneForGDB() || tuneForLLDB())) {
+      // Optionally emit call-site-param debug info.
+      if (emitDebugEntryValues()) {
         ParamSet Params;
         // Try to interpret values of call site parameters.
         collectCallSiteParameters(&MI, Params);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index a44960589d89fd47a682607fcbb4470718c5e041..882fc739d792ac3b4e96cdb2f5ddfa6dac3e5738 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -386,6 +386,11 @@ class DwarfDebug : public DebugHandlerBase {
   /// a monolithic sequence of string offsets.
   bool UseSegmentedStringOffsetsTable;
 
+  /// Enable production of call site parameters needed to print the debug entry
+  /// values. Useful for testing purposes when a debugger does not support the
+  /// feature yet.
+  bool EmitDebugEntryValues;
+
   /// Separated Dwarf Variables
   /// In general these will all be for bits that are left in the
   /// original object file, rather than things that are meant
@@ -708,6 +713,10 @@ public:
     return UseSegmentedStringOffsetsTable;
   }
 
+  bool emitDebugEntryValues() const {
+    return EmitDebugEntryValues;
+  }
+
   bool shareAcrossDWOCUs() const;
 
   /// Returns the Dwarf Version.
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 9a1ec1232dd11d8f9235bdcbd1927294a651b98d..ca5d4c04e58d958e9409b8a8afdb69ac676edbbb 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6865,12 +6865,19 @@ static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL,
     Value *Addr = Builder.CreateBitCast(
         SI.getOperand(1),
         SplitStoreType->getPointerTo(SI.getPointerAddressSpace()));
-    if ((IsLE && Upper) || (!IsLE && !Upper))
+    const bool IsOffsetStore = (IsLE && Upper) || (!IsLE && !Upper);
+    if (IsOffsetStore)
       Addr = Builder.CreateGEP(
           SplitStoreType, Addr,
           ConstantInt::get(Type::getInt32Ty(SI.getContext()), 1));
-    Builder.CreateAlignedStore(V, Addr,
-                               Upper ? SI.getAlign() / 2 : SI.getAlign());
+    MaybeAlign Alignment = SI.getAlign();
+    if (IsOffsetStore && Alignment) {
+      // When splitting the store in half, naturally one half will retain the
+      // alignment of the original wider store, regardless of whether it was
+      // over-aligned or not, while the other will require adjustment.
+      Alignment = commonAlignment(Alignment, HalfValBitSize / 8);
+    }
+    Builder.CreateAlignedStore(V, Addr, Alignment);
   };
 
   CreateSplitStore(LValue, false);
diff --git a/llvm/lib/CodeGen/LiveDebugValues.cpp b/llvm/lib/CodeGen/LiveDebugValues.cpp
index 6acaf4099c3e70792be564cb728702e3d66b0856..a07fd4028123a65791c3b5d9f84f64ea9fd150d3 100644
--- a/llvm/lib/CodeGen/LiveDebugValues.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues.cpp
@@ -960,7 +960,7 @@ void LiveDebugValues::transferRegisterDef(
 
   if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>()) {
     auto &TM = TPC->getTM<TargetMachine>();
-    if (TM.Options.EnableDebugEntryValues)
+    if (TM.Options.ShouldEmitDebugEntryValues())
       emitEntryValues(MI, OpenRanges, VarLocIDs, Transfers, KillSet);
   }
 }
@@ -1460,7 +1460,7 @@ void LiveDebugValues::recordEntryValue(const MachineInstr &MI,
                                        VarLocMap &VarLocIDs) {
   if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>()) {
     auto &TM = TPC->getTM<TargetMachine>();
-    if (!TM.Options.EnableDebugEntryValues)
+    if (!TM.Options.ShouldEmitDebugEntryValues())
       return;
   }
 
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 10157c746b462c4e8118d79919f5a0f6698a5b74..69d14cf68eca3852d21b9fef80c1a98b9b6591fd 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -381,11 +381,11 @@ bool MIRParserImpl::initializeCallSiteInfo(
       CSInfo.emplace_back(Reg, ArgRegPair.ArgNo);
     }
 
-    if (TM.Options.EnableDebugEntryValues)
+    if (TM.Options.ShouldEmitDebugEntryValues())
       MF.addCallArgsForwardingRegs(&*CallI, std::move(CSInfo));
   }
 
-  if (YamlMF.CallSitesInfo.size() && !TM.Options.EnableDebugEntryValues)
+  if (YamlMF.CallSitesInfo.size() && !TM.Options.ShouldEmitDebugEntryValues())
     return error(Twine("Call site info provided but not used"));
   return false;
 }
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 06b5ab5b7f4ca9ce09b2d9ba3dc7976aeb831bd7..2eb8d9460e053c930bcb564f63aaa39026a706f4 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -865,8 +865,7 @@ MachineFunction::CallSiteInfoMap::iterator
 MachineFunction::getCallSiteInfo(const MachineInstr *MI) {
   assert(MI->isCandidateForCallSiteEntry() &&
          "Call site info refers only to call (MI) candidates");
-
-  if (!Target.Options.EnableDebugEntryValues)
+  if (!Target.Options.ShouldEmitDebugEntryValues())
     return CallSitesInfo.end();
   return CallSitesInfo.find(MI);
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index d03f765a5779fbf5887fdefd75ccd34d1892bf37..6fd048a911b4f74464d121903b10b05be3eb4fee 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1009,6 +1009,7 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     Action = TLI.getOperationAction(Node->getOpcode(),
                                     Node->getOperand(0).getValueType());
     break;
+  case ISD::STRICT_FP_TO_FP16:
   case ISD::STRICT_SINT_TO_FP:
   case ISD::STRICT_UINT_TO_FP:
   case ISD::STRICT_LRINT:
@@ -3272,6 +3273,21 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
           DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
     }
     break;
+  case ISD::STRICT_FP16_TO_FP:
+    if (Node->getValueType(0) != MVT::f32) {
+      // We can extend to types bigger than f32 in two steps without changing
+      // the result. Since "f16 -> f32" is much more commonly available, give
+      // CodeGen the option of emitting that before resorting to a libcall.
+      SDValue Res =
+          DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
+                      {Node->getOperand(0), Node->getOperand(1)});
+      Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
+                        {Node->getValueType(0), MVT::Other},
+                        {Res.getValue(1), Res});
+      Results.push_back(Res);
+      Results.push_back(Res.getValue(1));
+    }
+    break;
   case ISD::FP_TO_FP16:
     LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
     if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
@@ -4234,6 +4250,17 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
       Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
     }
     break;
+  case ISD::STRICT_FP16_TO_FP: {
+    if (Node->getValueType(0) == MVT::f32) {
+      TargetLowering::MakeLibCallOptions CallOptions;
+      std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
+          DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
+          SDLoc(Node), Node->getOperand(0));
+      Results.push_back(Tmp.first);
+      Results.push_back(Tmp.second);
+    }
+    break;
+  }
   case ISD::FP_TO_FP16: {
     RTLIB::Libcall LC =
         RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
@@ -4241,6 +4268,19 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
     Results.push_back(ExpandLibCall(LC, Node, false));
     break;
   }
+  case ISD::STRICT_FP_TO_FP16: {
+    RTLIB::Libcall LC =
+        RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16);
+    assert(LC != RTLIB::UNKNOWN_LIBCALL &&
+           "Unable to expand strict_fp_to_fp16");
+    TargetLowering::MakeLibCallOptions CallOptions;
+    std::pair<SDValue, SDValue> Tmp =
+        TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
+                        CallOptions, SDLoc(Node), Node->getOperand(0));
+    Results.push_back(Tmp.first);
+    Results.push_back(Tmp.second);
+    break;
+  }
   case ISD::FSUB:
   case ISD::STRICT_FSUB:
     ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 12fef02267ec3653c16498e84985c72cfb2de2d1..428dc83e2fd3d466aad72de221bc5d80f8b8f8db 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -2440,6 +2440,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
   case ISD::EXTRACT_VECTOR_ELT:
     R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
   case ISD::FCOPYSIGN:  R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
+  case ISD::STRICT_FP_ROUND:
   case ISD::FP_ROUND:   R = SoftPromoteHalfRes_FP_ROUND(N); break;
 
   // Unary FP Operations
@@ -2592,6 +2593,14 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
+  if (N->isStrictFPOpcode()) {
+    SDValue Res =
+        DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
+                    {N->getOperand(0), N->getOperand(1)});
+    ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+    return Res;
+  }
+
   return DAG.getNode(ISD::FP_TO_FP16, SDLoc(N), MVT::i16, N->getOperand(0));
 }
 
@@ -2701,6 +2710,7 @@ bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
   case ISD::FCOPYSIGN:  Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
+  case ISD::STRICT_FP_EXTEND:
   case ISD::FP_EXTEND:  Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
   case ISD::SELECT_CC:  Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
   case ISD::SETCC:      Res = SoftPromoteHalfOp_SETCC(N); break;
@@ -2741,7 +2751,18 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
 }
 
 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
-  SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
+  bool IsStrict = N->isStrictFPOpcode();
+  SDValue Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
+
+  if (IsStrict) {
+    SDValue Res =
+        DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
+                    {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
+    ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+    ReplaceValueWith(SDValue(N, 0), Res);
+    return SDValue();
+  }
+
   return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), Op);
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index a297db17883583850d637dbff5c5a0efccac0647..ea0bcd1fd8880746092996e7fee71b1c255b21ec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -198,10 +198,10 @@ static void RemoveUnusedGlue(SDNode *N, SelectionDAG *DAG) {
 /// outputs to ensure they are scheduled together and in order. This
 /// optimization may benefit some targets by improving cache locality.
 void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) {
-  SDNode *Chain = nullptr;
+  SDValue Chain;
   unsigned NumOps = Node->getNumOperands();
   if (Node->getOperand(NumOps-1).getValueType() == MVT::Other)
-    Chain = Node->getOperand(NumOps-1).getNode();
+    Chain = Node->getOperand(NumOps-1);
   if (!Chain)
     return;
 
@@ -234,6 +234,9 @@ void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) {
   unsigned UseCount = 0;
   for (SDNode::use_iterator I = Chain->use_begin(), E = Chain->use_end();
        I != E && UseCount < 100; ++I, ++UseCount) {
+    if (I.getUse().getResNo() != Chain.getResNo())
+      continue;
+
     SDNode *User = *I;
     if (User == Node || !Visited.insert(User).second)
       continue;
@@ -864,7 +867,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
     }
 
     if (MI->isCandidateForCallSiteEntry() &&
-        DAG->getTarget().Options.EnableDebugEntryValues)
+        DAG->getTarget().Options.SupportsDebugEntryValues)
       MF.addCallArgsForwardingRegs(MI, DAG->getSDCallSiteInfo(Node));
 
     return MI;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 8736ee23b8ea1bfedcc14f54cb59af1aefc0eb3e..f57852ced66072cb5c060f698d79bf407f06fa02 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -342,7 +342,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::BITCAST:                    return "bitcast";
   case ISD::ADDRSPACECAST:              return "addrspacecast";
   case ISD::FP16_TO_FP:                 return "fp16_to_fp";
+  case ISD::STRICT_FP16_TO_FP:          return "strict_fp16_to_fp";
   case ISD::FP_TO_FP16:                 return "fp_to_fp16";
+  case ISD::STRICT_FP_TO_FP16:          return "strict_fp_to_fp16";
   case ISD::LROUND:                     return "lround";
   case ISD::STRICT_LROUND:              return "strict_lround";
   case ISD::LLROUND:                    return "llround";
diff --git a/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
index d794a261ecb2f81d22a59a2e7b0ef9b1d6682685..3db0f2f2829a43fffbcfcf1d62a2f4dbeb6cbf79 100644
--- a/llvm/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
@@ -45,3 +45,11 @@ bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
 bool TargetOptions::HonorSignDependentRoundingFPMath() const {
   return !UnsafeFPMath && HonorSignDependentRoundingFPMathOption;
 }
+
+/// NOTE: There are targets that still do not support the call site info
+/// production (the info about the arguments passed to the call, necessary
+/// for the debug entry values), so we keep using the experimental option
+/// (-debug-entry-values) to test them as well.
+bool TargetOptions::ShouldEmitDebugEntryValues() const {
+  return SupportsDebugEntryValues || EnableDebugEntryValues;
+}
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
index eb736b39d7d2fbeed0314af388fc83fb5c80d1a9..62bc5138e32ac5eb2cf3a1450568cf90ac8c2aa5 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -252,12 +252,13 @@ static void dumpAddrSection(raw_ostream &OS, DWARFDataExtractor &AddrData,
       WithColor::error() << toString(std::move(Err)) << '\n';
       // Keep going after an error, if we can, assuming that the length field
       // could be read. If it couldn't, stop reading the section.
-      if (!AddrTable.hasValidLength())
-        break;
-      Offset = TableOffset + AddrTable.getLength();
-    } else {
-      AddrTable.dump(OS, DumpOpts);
+      if (auto TableLength = AddrTable.getFullLength()) {
+        Offset = TableOffset + *TableLength;
+        continue;
+      }
+      break;
     }
+    AddrTable.dump(OS, DumpOpts);
   }
 }
 
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp
index 02b8b8ee4bfe225ccc4e5f2ca9102324ce38637b..273d316e0912acb34fe9a2b83be4165783de9a6b 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp
@@ -12,144 +12,162 @@
 
 using namespace llvm;
 
-void DWARFDebugAddrTable::clear() {
-  HeaderData = {};
+Error DWARFDebugAddrTable::extractAddresses(const DWARFDataExtractor &Data,
+                                            uint64_t *OffsetPtr,
+                                            uint64_t EndOffset) {
+  assert(EndOffset >= *OffsetPtr);
+  uint64_t DataSize = EndOffset - *OffsetPtr;
+  assert(Data.isValidOffsetForDataOfSize(*OffsetPtr, DataSize));
+  if (AddrSize != 4 && AddrSize != 8)
+    return createStringError(errc::not_supported,
+                             "address table at offset 0x%" PRIx64
+                             " has unsupported address size %" PRIu8
+                             " (4 and 8 are supported)",
+                             Offset, AddrSize);
+  if (DataSize % AddrSize != 0) {
+    invalidateLength();
+    return createStringError(errc::invalid_argument,
+                             "address table at offset 0x%" PRIx64
+                             " contains data of size 0x%" PRIx64
+                             " which is not a multiple of addr size %" PRIu8,
+                             Offset, DataSize, AddrSize);
+  }
   Addrs.clear();
-  invalidateLength();
+  size_t Count = DataSize / AddrSize;
+  Addrs.reserve(Count);
+  while (Count--)
+    Addrs.push_back(Data.getRelocatedValue(AddrSize, OffsetPtr));
+  return Error::success();
 }
 
-Error DWARFDebugAddrTable::extract(DWARFDataExtractor Data,
-                                   uint64_t *OffsetPtr,
-                                   uint16_t Version,
-                                   uint8_t AddrSize,
-                                   std::function<void(Error)> WarnCallback) {
-  clear();
-  HeaderOffset = *OffsetPtr;
-  // Read and verify the length field.
-  if (!Data.isValidOffsetForDataOfSize(*OffsetPtr, sizeof(uint32_t)))
+Error DWARFDebugAddrTable::extractV5(const DWARFDataExtractor &Data,
+                                     uint64_t *OffsetPtr, uint8_t CUAddrSize,
+                                     std::function<void(Error)> WarnCallback) {
+  Offset = *OffsetPtr;
+  // Check that we can read the unit length field.
+  if (!Data.isValidOffsetForDataOfSize(Offset, 4))
     return createStringError(errc::invalid_argument,
-                       "section is not large enough to contain a "
-                       ".debug_addr table length at offset 0x%"
-                       PRIx64, *OffsetPtr);
-  uint16_t UnitVersion;
-  if (Version == 0) {
-    WarnCallback(createStringError(errc::invalid_argument,
-                       "DWARF version is not defined in CU,"
-                       " assuming version 5"));
-    UnitVersion = 5;
-  } else {
-    UnitVersion = Version;
-  }
-  // TODO: Add support for DWARF64.
+                             "section is not large enough to contain an "
+                             "address table length at offset 0x%" PRIx64,
+                             Offset);
   Format = dwarf::DwarfFormat::DWARF32;
-  if (UnitVersion >= 5) {
-    HeaderData.Length = Data.getU32(OffsetPtr);
-    if (HeaderData.Length == dwarf::DW_LENGTH_DWARF64) {
-      invalidateLength();
-      return createStringError(errc::not_supported,
-          "DWARF64 is not supported in .debug_addr at offset 0x%" PRIx64,
-          HeaderOffset);
-    }
-    if (HeaderData.Length + sizeof(uint32_t) < sizeof(Header)) {
-      uint32_t TmpLength = getLength();
+  Length = Data.getU32(OffsetPtr);
+  if (Length == dwarf::DW_LENGTH_DWARF64) {
+    // Check that we can read the extended unit length field.
+    if (!Data.isValidOffsetForDataOfSize(*OffsetPtr, 8)) {
       invalidateLength();
-      return createStringError(errc::invalid_argument,
-                         ".debug_addr table at offset 0x%" PRIx64
-                         " has too small length (0x%" PRIx32
-                         ") to contain a complete header",
-                         HeaderOffset, TmpLength);
-    }
-    uint64_t End = HeaderOffset + getLength();
-    if (!Data.isValidOffsetForDataOfSize(HeaderOffset, End - HeaderOffset)) {
-      uint32_t TmpLength = getLength();
-      invalidateLength();
-      return createStringError(errc::invalid_argument,
-          "section is not large enough to contain a .debug_addr table "
-          "of length 0x%" PRIx32 " at offset 0x%" PRIx64,
-          TmpLength, HeaderOffset);
+      return createStringError(
+          errc::invalid_argument,
+          "section is not large enough to contain an extended length field "
+          "of the address table at offset 0x%" PRIx64,
+          Offset);
     }
+    Format = dwarf::DwarfFormat::DWARF64;
+    Length = Data.getU64(OffsetPtr);
+  } else if (Length >= dwarf::DW_LENGTH_lo_reserved) {
+    uint64_t DiagnosticLength = Length;
+    invalidateLength();
+    return createStringError(
+        errc::not_supported,
+        "address table at offset 0x%" PRIx64
+        " has unsupported reserved unit length of value 0x%" PRIx64,
+        Offset, DiagnosticLength);
+  }
 
-    HeaderData.Version = Data.getU16(OffsetPtr);
-    HeaderData.AddrSize = Data.getU8(OffsetPtr);
-    HeaderData.SegSize = Data.getU8(OffsetPtr);
-    DataSize = getDataSize();
-  } else {
-    HeaderData.Version = UnitVersion;
-    HeaderData.AddrSize = AddrSize;
-    // TODO: Support for non-zero SegSize.
-    HeaderData.SegSize = 0;
-    DataSize = Data.size();
+  if (!Data.isValidOffsetForDataOfSize(*OffsetPtr, Length)) {
+    uint64_t DiagnosticLength = Length;
+    invalidateLength();
+    return createStringError(
+        errc::invalid_argument,
+        "section is not large enough to contain an address table "
+        "at offset 0x%" PRIx64 " with a unit_length value of 0x%" PRIx64,
+        Offset, DiagnosticLength);
+  }
+  uint64_t EndOffset = *OffsetPtr + Length;
+  // Ensure that we can read the remaining header fields.
+  if (Length < 4) {
+    uint64_t DiagnosticLength = Length;
+    invalidateLength();
+    return createStringError(
+        errc::invalid_argument,
+        "address table at offset 0x%" PRIx64
+        " has a unit_length value of 0x%" PRIx64
+        ", which is too small to contain a complete header",
+        Offset, DiagnosticLength);
   }
 
-  // Perform basic validation of the remaining header fields.
+  Version = Data.getU16(OffsetPtr);
+  AddrSize = Data.getU8(OffsetPtr);
+  SegSize = Data.getU8(OffsetPtr);
 
-  // We support DWARF version 5 for now as well as pre-DWARF5
-  // implementations of .debug_addr table, which doesn't contain a header
-  // and consists only of a series of addresses.
-  if (HeaderData.Version > 5) {
-    return createStringError(errc::not_supported, "version %" PRIu16
-        " of .debug_addr section at offset 0x%" PRIx64 " is not supported",
-        HeaderData.Version, HeaderOffset);
-  }
-  // FIXME: For now we just treat version mismatch as an error,
-  // however the correct way to associate a .debug_addr table
-  // with a .debug_info table is to look at the DW_AT_addr_base
-  // attribute in the info table.
-  if (HeaderData.Version != UnitVersion)
-    return createStringError(errc::invalid_argument,
-                       ".debug_addr table at offset 0x%" PRIx64
-                       " has version %" PRIu16
-                       " which is different from the version suggested"
-                       " by the DWARF unit header: %" PRIu16,
-                       HeaderOffset, HeaderData.Version, UnitVersion);
-  if (HeaderData.AddrSize != 4 && HeaderData.AddrSize != 8)
+  // Perform a basic validation of the header fields.
+  if (Version != 5)
+    return createStringError(errc::not_supported,
+                             "address table at offset 0x%" PRIx64
+                             " has unsupported version %" PRIu16,
+                             Offset, Version);
+  // TODO: add support for non-zero segment selector size.
+  if (SegSize != 0)
     return createStringError(errc::not_supported,
-                       ".debug_addr table at offset 0x%" PRIx64
-                       " has unsupported address size %" PRIu8,
-                       HeaderOffset, HeaderData.AddrSize);
-  if (HeaderData.AddrSize != AddrSize && AddrSize != 0)
+                             "address table at offset 0x%" PRIx64
+                             " has unsupported segment selector size %" PRIu8,
+                             Offset, SegSize);
+
+  if (Error Err = extractAddresses(Data, OffsetPtr, EndOffset))
+    return Err;
+  if (CUAddrSize && AddrSize != CUAddrSize) {
     WarnCallback(createStringError(
         errc::invalid_argument,
-        ".debug_addr table at offset 0x%" PRIx64 " has address size %" PRIu8
+        "address table at offset 0x%" PRIx64 " has address size %" PRIu8
         " which is different from CU address size %" PRIu8,
-        HeaderOffset, HeaderData.AddrSize, AddrSize));
-
-  // TODO: add support for non-zero segment selector size.
-  if (HeaderData.SegSize != 0)
-    return createStringError(errc::not_supported,
-                       ".debug_addr table at offset 0x%" PRIx64
-                       " has unsupported segment selector size %" PRIu8,
-                       HeaderOffset, HeaderData.SegSize);
-  if (DataSize % HeaderData.AddrSize != 0) {
-    invalidateLength();
-    return createStringError(errc::invalid_argument,
-                       ".debug_addr table at offset 0x%" PRIx64
-                       " contains data of size %" PRIu32
-                       " which is not a multiple of addr size %" PRIu8,
-                       HeaderOffset, DataSize, HeaderData.AddrSize);
+        Offset, AddrSize, CUAddrSize));
   }
-  Data.setAddressSize(HeaderData.AddrSize);
-  uint32_t AddrCount = DataSize / HeaderData.AddrSize;
-  for (uint32_t I = 0; I < AddrCount; ++I)
-    if (HeaderData.AddrSize == 4)
-      Addrs.push_back(Data.getU32(OffsetPtr));
-    else
-      Addrs.push_back(Data.getU64(OffsetPtr));
   return Error::success();
 }
 
+Error DWARFDebugAddrTable::extractPreStandard(const DWARFDataExtractor &Data,
+                                              uint64_t *OffsetPtr,
+                                              uint16_t CUVersion,
+                                              uint8_t CUAddrSize) {
+  assert(CUVersion > 0 && CUVersion < 5);
+
+  Offset = *OffsetPtr;
+  Length = 0;
+  Version = CUVersion;
+  AddrSize = CUAddrSize;
+  SegSize = 0;
+
+  return extractAddresses(Data, OffsetPtr, Data.size());
+}
+
+Error DWARFDebugAddrTable::extract(const DWARFDataExtractor &Data,
+                                   uint64_t *OffsetPtr,
+                                   uint16_t CUVersion,
+                                   uint8_t CUAddrSize,
+                                   std::function<void(Error)> WarnCallback) {
+  if (CUVersion > 0 && CUVersion < 5)
+    return extractPreStandard(Data, OffsetPtr, CUVersion, CUAddrSize);
+  if (CUVersion == 0)
+    WarnCallback(createStringError(errc::invalid_argument,
+                                   "DWARF version is not defined in CU,"
+                                   " assuming version 5"));
+  return extractV5(Data, OffsetPtr, CUAddrSize, WarnCallback);
+}
+
 void DWARFDebugAddrTable::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
   if (DumpOpts.Verbose)
-    OS << format("0x%8.8" PRIx32 ": ", HeaderOffset);
-  OS << format("Addr Section: length = 0x%8.8" PRIx32
-               ", version = 0x%4.4" PRIx16 ", "
-               "addr_size = 0x%2.2" PRIx8 ", seg_size = 0x%2.2" PRIx8 "\n",
-               HeaderData.Length, HeaderData.Version, HeaderData.AddrSize,
-               HeaderData.SegSize);
+    OS << format("0x%8.8" PRIx64 ": ", Offset);
+  if (Length) {
+    int LengthFieldWidth = (Format == dwarf::DwarfFormat::DWARF64) ? 16 : 8;
+    OS << format("Address table header: length = 0x%0*" PRIx64
+                 ", version = 0x%4.4" PRIx16 ", addr_size = 0x%2.2" PRIx8
+                 ", seg_size = 0x%2.2" PRIx8 "\n",
+                 LengthFieldWidth, Length, Version, AddrSize, SegSize);
+  }
 
   if (Addrs.size() > 0) {
-    const char *AddrFmt = (HeaderData.AddrSize == 4) ? "0x%8.8" PRIx64 "\n"
-                                                     : "0x%16.16" PRIx64 "\n";
+    const char *AddrFmt =
+        (AddrSize == 4) ? "0x%8.8" PRIx64 "\n" : "0x%16.16" PRIx64 "\n";
     OS << "Addrs: [\n";
     for (uint64_t Addr : Addrs)
       OS << format(AddrFmt, Addr);
@@ -162,21 +180,13 @@ Expected<uint64_t> DWARFDebugAddrTable::getAddrEntry(uint32_t Index) const {
     return Addrs[Index];
   return createStringError(errc::invalid_argument,
                            "Index %" PRIu32 " is out of range of the "
-                           ".debug_addr table at offset 0x%" PRIx64,
-                           Index, HeaderOffset);
+                           "address table at offset 0x%" PRIx64,
+                           Index, Offset);
 }
 
-uint32_t DWARFDebugAddrTable::getLength() const {
-  if (HeaderData.Length == 0)
-    return 0;
-  // TODO: DWARF64 support.
-  return HeaderData.Length + sizeof(uint32_t);
+Optional<uint64_t> DWARFDebugAddrTable::getFullLength() const {
+  if (Length == 0)
+    return None;
+  return Length + dwarf::getUnitLengthFieldByteSize(Format);
 }
 
-uint32_t DWARFDebugAddrTable::getDataSize() const {
-  if (DataSize != 0)
-    return DataSize;
-  if (getLength() == 0)
-    return 0;
-  return getLength() - getHeaderSize();
-}
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
index 4ede9ffc2cfa92a463a35cd8fa3f57de08beafd7..d3d2c58d3395e21f048479e0880e3c2565e844b4 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
@@ -1082,7 +1082,7 @@ bool DWARFDebugLine::Prologue::getFileNameByIndex(
   if (!Name)
     return false;
   StringRef FileName = *Name;
-  if (Kind != FileLineInfoKind::AbsoluteFilePath ||
+  if (Kind == FileLineInfoKind::Default ||
       isPathAbsoluteOnWindowsOrPosix(FileName)) {
     Result = std::string(FileName);
     return true;
@@ -1099,12 +1099,17 @@ bool DWARFDebugLine::Prologue::getFileNameByIndex(
       IncludeDir =
           IncludeDirectories[Entry.DirIdx - 1].getAsCString().getValue();
   }
-  // We may still need to append compilation directory of compile unit.
-  // We know that FileName is not absolute, the only way to have an
-  // absolute path at this point would be if IncludeDir is absolute.
-  if (!CompDir.empty() && !isPathAbsoluteOnWindowsOrPosix(IncludeDir))
+  // For absolute paths only, include the compilation directory of compile unit.
+  // We know that FileName is not absolute, the only way to have an absolute
+  // path at this point would be if IncludeDir is absolute.
+  if (Kind == FileLineInfoKind::AbsoluteFilePath && !CompDir.empty() &&
+      !isPathAbsoluteOnWindowsOrPosix(IncludeDir))
     sys::path::append(FilePath, Style, CompDir);
 
+  assert((Kind == FileLineInfoKind::AbsoluteFilePath ||
+          Kind == FileLineInfoKind::RelativeFilePath) &&
+         "invalid FileLineInfo Kind");
+
   // sys::path::append skips empty strings.
   sys::path::append(FilePath, Style, IncludeDir, FileName);
   Result = std::string(FilePath.str());
diff --git a/llvm/lib/Demangle/ItaniumDemangle.cpp b/llvm/lib/Demangle/ItaniumDemangle.cpp
index e112d5c5ec77b1dc723a584c7893d6c57ed32531..fad9b6b7b63bd3e19aec0d4bf9a75fc82c576e22 100644
--- a/llvm/lib/Demangle/ItaniumDemangle.cpp
+++ b/llvm/lib/Demangle/ItaniumDemangle.cpp
@@ -107,13 +107,11 @@ struct DumpVisitor {
   // Overload used when T is exactly 'bool', not merely convertible to 'bool'.
   void print(bool B) { printStr(B ? "true" : "false"); }
 
-  template <class T>
-  typename std::enable_if<std::is_unsigned<T>::value>::type print(T N) {
+  template <class T> std::enable_if_t<std::is_unsigned<T>::value> print(T N) {
     fprintf(stderr, "%llu", (unsigned long long)N);
   }
 
-  template <class T>
-  typename std::enable_if<std::is_signed<T>::value>::type print(T N) {
+  template <class T> std::enable_if_t<std::is_signed<T>::value> print(T N) {
     fprintf(stderr, "%lld", (long long)N);
   }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index f2b161cc9fc16163c05ec98d9b68470ee7c5d507..a7af34a2192666b20536647344b4ab0e88b3c41e 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -205,7 +205,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) {
 
 raw_ostream &operator<<(raw_ostream &OS,
                         const SymbolDependenceMap::value_type &KV) {
-  return OS << "(" << KV.first << ", " << KV.second << ")";
+  return OS << "(" << KV.first->getName() << ", " << KV.second << ")";
 }
 
 raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) {
@@ -941,6 +941,11 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name,
   assert(Symbols[Name].isInMaterializationPhase() &&
          "Can not add dependencies for a symbol that is not materializing");
 
+  LLVM_DEBUG({
+      dbgs() << "In " << getName() << " adding dependencies for "
+             << *Name << ": " << Dependencies << "\n";
+    });
+
   // If Name is already in an error state then just bail out.
   if (Symbols[Name].getFlags().hasError())
     return;
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 2572b7f4878d98a108238a41d945f9716fc71134..6575b0a6ccb282152b2de308e02f26bb6f150e10 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -84,6 +84,12 @@ public:
       }
     };
 
+    for (auto &KV : InternalNamedSymbolDeps) {
+      SymbolDependenceMap InternalDeps;
+      InternalDeps[&MR.getTargetJITDylib()] = std::move(KV.second);
+      MR.addDependencies(KV.first, InternalDeps);
+    }
+
     ES.lookup(LookupKind::Static, SearchOrder, std::move(LookupSet),
               SymbolState::Resolved, std::move(OnResolve),
               [this](const SymbolDependenceMap &Deps) {
@@ -168,16 +174,18 @@ public:
     // link graph to build the symbol dependence graph.
     Config.PrePrunePasses.push_back(
         [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); });
-    Config.PostPrunePasses.push_back(
-        [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); });
 
     Layer.modifyPassConfig(MR, TT, Config);
 
+    Config.PostPrunePasses.push_back(
+        [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); });
+
     return Error::success();
   }
 
 private:
-  using AnonToNamedDependenciesMap = DenseMap<const Symbol *, SymbolNameSet>;
+  using JITLinkSymbolSet = DenseSet<const Symbol *>;
+  using LocalToNamedDependenciesMap = DenseMap<const Symbol *, JITLinkSymbolSet>;
 
   Error externalizeWeakAndCommonSymbols(LinkGraph &G) {
     auto &ES = Layer.getExecutionSession();
@@ -206,90 +214,107 @@ private:
 
   Error computeNamedSymbolDependencies(LinkGraph &G) {
     auto &ES = MR.getTargetJITDylib().getExecutionSession();
-    auto AnonDeps = computeAnonDeps(G);
+    auto LocalDeps = computeLocalDeps(G);
 
     for (auto *Sym : G.defined_symbols()) {
 
-      // Skip anonymous and non-global atoms: we do not need dependencies for
-      // these.
+      // Skip local symbols: we do not track dependencies for these.
       if (Sym->getScope() == Scope::Local)
         continue;
+      assert(Sym->hasName() &&
+             "Defined non-local jitlink::Symbol should have a name");
 
-      auto SymName = ES.intern(Sym->getName());
-      SymbolNameSet &SymDeps = NamedSymbolDeps[SymName];
+      SymbolNameSet ExternalSymDeps, InternalSymDeps;
 
+      // Find internal and external named symbol dependencies.
       for (auto &E : Sym->getBlock().edges()) {
         auto &TargetSym = E.getTarget();
 
-        if (TargetSym.getScope() != Scope::Local)
-          SymDeps.insert(ES.intern(TargetSym.getName()));
-        else {
+        if (TargetSym.getScope() != Scope::Local) {
+          if (TargetSym.isExternal())
+            ExternalSymDeps.insert(ES.intern(TargetSym.getName()));
+          else if (&TargetSym != Sym)
+            InternalSymDeps.insert(ES.intern(TargetSym.getName()));
+        } else {
           assert(TargetSym.isDefined() &&
-                 "Anonymous/local symbols must be defined");
-          auto I = AnonDeps.find(&TargetSym);
-          if (I != AnonDeps.end())
-            for (auto &S : I->second)
-              SymDeps.insert(S);
+                 "local symbols must be defined");
+          auto I = LocalDeps.find(&TargetSym);
+          if (I != LocalDeps.end())
+            for (auto &S : I->second) {
+              assert(S->hasName() &&
+                     "LocalDeps should only contain named values");
+              if (S->isExternal())
+                ExternalSymDeps.insert(ES.intern(S->getName()));
+              else if (S != Sym)
+                InternalSymDeps.insert(ES.intern(S->getName()));
+            }
         }
       }
+
+      if (ExternalSymDeps.empty() && InternalSymDeps.empty())
+        continue;
+
+      auto SymName = ES.intern(Sym->getName());
+      if (!ExternalSymDeps.empty())
+        ExternalNamedSymbolDeps[SymName] = std::move(ExternalSymDeps);
+      if (!InternalSymDeps.empty())
+        InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps);
     }
 
     return Error::success();
   }
 
-  AnonToNamedDependenciesMap computeAnonDeps(LinkGraph &G) {
+  LocalToNamedDependenciesMap computeLocalDeps(LinkGraph &G) {
+    LocalToNamedDependenciesMap DepMap;
 
-    auto &ES = MR.getTargetJITDylib().getExecutionSession();
-    AnonToNamedDependenciesMap DepMap;
-
-    // For all anonymous symbols:
+    // For all local symbols:
     // (1) Add their named dependencies.
     // (2) Add them to the worklist for further iteration if they have any
-    //     depend on any other anonymous symbols.
+    //     depend on any other local symbols.
     struct WorklistEntry {
-      WorklistEntry(Symbol *Sym, DenseSet<Symbol *> SymAnonDeps)
-          : Sym(Sym), SymAnonDeps(std::move(SymAnonDeps)) {}
+      WorklistEntry(Symbol *Sym, DenseSet<Symbol *> LocalDeps)
+          : Sym(Sym), LocalDeps(std::move(LocalDeps)) {}
 
       Symbol *Sym = nullptr;
-      DenseSet<Symbol *> SymAnonDeps;
+      DenseSet<Symbol *> LocalDeps;
     };
     std::vector<WorklistEntry> Worklist;
     for (auto *Sym : G.defined_symbols())
-      if (!Sym->hasName()) {
+      if (Sym->getScope() == Scope::Local) {
         auto &SymNamedDeps = DepMap[Sym];
-        DenseSet<Symbol *> SymAnonDeps;
+        DenseSet<Symbol *> LocalDeps;
 
         for (auto &E : Sym->getBlock().edges()) {
           auto &TargetSym = E.getTarget();
-          if (TargetSym.hasName())
-            SymNamedDeps.insert(ES.intern(TargetSym.getName()));
+          if (TargetSym.getScope() != Scope::Local)
+            SymNamedDeps.insert(&TargetSym);
           else {
             assert(TargetSym.isDefined() &&
-                   "Anonymous symbols must be defined");
-            SymAnonDeps.insert(&TargetSym);
+                   "local symbols must be defined");
+            LocalDeps.insert(&TargetSym);
           }
         }
 
-        if (!SymAnonDeps.empty())
-          Worklist.push_back(WorklistEntry(Sym, std::move(SymAnonDeps)));
+        if (!LocalDeps.empty())
+          Worklist.push_back(WorklistEntry(Sym, std::move(LocalDeps)));
       }
 
-    // Loop over all anonymous symbols with anonymous dependencies, propagating
-    // their respective *named* dependencies. Iterate until we hit a stable
+    // Loop over all local symbols with local dependencies, propagating
+    // their respective non-local dependencies. Iterate until we hit a stable
     // state.
     bool Changed;
     do {
       Changed = false;
       for (auto &WLEntry : Worklist) {
         auto *Sym = WLEntry.Sym;
-        auto &SymNamedDeps = DepMap[Sym];
-        auto &SymAnonDeps = WLEntry.SymAnonDeps;
+        auto &NamedDeps = DepMap[Sym];
+        auto &LocalDeps = WLEntry.LocalDeps;
 
-        for (auto *TargetSym : SymAnonDeps) {
+        for (auto *TargetSym : LocalDeps) {
           auto I = DepMap.find(TargetSym);
           if (I != DepMap.end())
             for (const auto &S : I->second)
-              Changed |= SymNamedDeps.insert(S).second;
+              Changed |= NamedDeps.insert(S).second;
         }
       }
     } while (Changed);
@@ -298,7 +323,7 @@ private:
   }
 
   void registerDependencies(const SymbolDependenceMap &QueryDeps) {
-    for (auto &NamedDepsEntry : NamedSymbolDeps) {
+    for (auto &NamedDepsEntry : ExternalNamedSymbolDeps) {
       auto &Name = NamedDepsEntry.first;
       auto &NameDeps = NamedDepsEntry.second;
       SymbolDependenceMap SymbolDeps;
@@ -323,7 +348,8 @@ private:
   ObjectLinkingLayer &Layer;
   MaterializationResponsibility MR;
   std::unique_ptr<MemoryBuffer> ObjBuffer;
-  DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps;
+  DenseMap<SymbolStringPtr, SymbolNameSet> ExternalNamedSymbolDeps;
+  DenseMap<SymbolStringPtr, SymbolNameSet> InternalNamedSymbolDeps;
 };
 
 ObjectLinkingLayer::Plugin::~Plugin() {}
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 953d7ae49ffcb9cde59d5a731a33be8bbe5f0fee..1a3043b9da0852839b7920ebf35697161289f4db 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -2772,6 +2772,8 @@ static const char *getWholeProgDevirtResByArgKindName(
 
 static const char *getTTResKindName(TypeTestResolution::Kind K) {
   switch (K) {
+  case TypeTestResolution::Unknown:
+    return "unknown";
   case TypeTestResolution::Unsat:
     return "unsat";
   case TypeTestResolution::ByteArray:
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 9f6af981aca108583b3bb83babc17b24ac52d586..65a282002ccf1811d0ec47bd43f15f014ebcf80d 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -831,11 +831,11 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm,
     SectionDataFileSize = std::max(SectionDataFileSize, Address + FileSize);
   }
 
-  // The section data is padded to 4 bytes.
+  // The section data is padded to pointer size bytes.
   //
   // FIXME: Is this machine dependent?
   unsigned SectionDataPadding =
-      offsetToAlignment(SectionDataFileSize, Align(4));
+      offsetToAlignment(SectionDataFileSize, is64Bit() ? Align(8) : Align(4));
   SectionDataFileSize += SectionDataPadding;
 
   // Write the prolog, starting with the header and load command...
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index d47b56f628d9524e18a4b258f097a55fe7436e98..72719c77af7031c3f8c2682055736abc61cfbcc4 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -751,6 +751,12 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   }
   MPM.addPass(AttributorPass());
 
+  // Lower type metadata and the type.test intrinsic in the ThinLTO
+  // post link pipeline after ICP. This is to enable usage of the type
+  // tests in ICP sequences.
+  if (Phase == ThinLTOPhase::PostLink)
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+
   // Interprocedural constant propagation now that basic cleanup has occurred
   // and prior to optimizing globals.
   // FIXME: This position in the pipeline hasn't been carefully considered in
@@ -1183,6 +1189,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
     // metadata and intrinsics.
     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
+    // Run a second time to clean up any type tests left behind by WPD for use
+    // in ICP.
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
     return MPM;
   }
 
@@ -1249,6 +1258,10 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
     // The LowerTypeTestsPass needs to run to lower type metadata and the
     // type.test intrinsics. The pass does nothing if CFI is disabled.
     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
+    // Run a second time to clean up any type tests left behind by WPD for use
+    // in ICP (which is performed earlier than this in the regular LTO
+    // pipeline).
+    MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
     return MPM;
   }
 
@@ -1376,6 +1389,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
   // to be run at link time if CFI is enabled. This pass does nothing if
   // CFI is disabled.
   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
+  // Run a second time to clean up any type tests left behind by WPD for use
+  // in ICP (which is performed earlier than this in the regular LTO pipeline).
+  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
 
   // Enable splitting late in the FullLTO post-link pipeline. This is done in
   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
diff --git a/llvm/lib/Support/APFloat.cpp b/llvm/lib/Support/APFloat.cpp
index 03f0bb1f705eb9dda95421218b7311f257bc1f4d..54f8f8e4bfdb7daee01417dc7109bd6de67d8c0b 100644
--- a/llvm/lib/Support/APFloat.cpp
+++ b/llvm/lib/Support/APFloat.cpp
@@ -1686,6 +1686,44 @@ IEEEFloat::opStatus IEEEFloat::modSpecials(const IEEEFloat &rhs) {
   }
 }
 
+IEEEFloat::opStatus IEEEFloat::remainderSpecials(const IEEEFloat &rhs) {
+  switch (PackCategoriesIntoKey(category, rhs.category)) {
+  default:
+    llvm_unreachable(nullptr);
+
+  case PackCategoriesIntoKey(fcZero, fcNaN):
+  case PackCategoriesIntoKey(fcNormal, fcNaN):
+  case PackCategoriesIntoKey(fcInfinity, fcNaN):
+    assign(rhs);
+    LLVM_FALLTHROUGH;
+  case PackCategoriesIntoKey(fcNaN, fcZero):
+  case PackCategoriesIntoKey(fcNaN, fcNormal):
+  case PackCategoriesIntoKey(fcNaN, fcInfinity):
+  case PackCategoriesIntoKey(fcNaN, fcNaN):
+    if (isSignaling()) {
+      makeQuiet();
+      return opInvalidOp;
+    }
+    return rhs.isSignaling() ? opInvalidOp : opOK;
+
+  case PackCategoriesIntoKey(fcZero, fcInfinity):
+  case PackCategoriesIntoKey(fcZero, fcNormal):
+  case PackCategoriesIntoKey(fcNormal, fcInfinity):
+    return opOK;
+
+  case PackCategoriesIntoKey(fcNormal, fcZero):
+  case PackCategoriesIntoKey(fcInfinity, fcZero):
+  case PackCategoriesIntoKey(fcInfinity, fcNormal):
+  case PackCategoriesIntoKey(fcInfinity, fcInfinity):
+  case PackCategoriesIntoKey(fcZero, fcZero):
+    makeNaN();
+    return opInvalidOp;
+
+  case PackCategoriesIntoKey(fcNormal, fcNormal):
+    return opDivByZero; // fake status, indicating this is not a special case
+  }
+}
+
 /* Change sign.  */
 void IEEEFloat::changeSign() {
   /* Look mummy, this one's easy.  */
@@ -1770,40 +1808,108 @@ IEEEFloat::opStatus IEEEFloat::divide(const IEEEFloat &rhs,
   return fs;
 }
 
-/* Normalized remainder.  This is not currently correct in all cases.  */
+/* Normalized remainder.  */
 IEEEFloat::opStatus IEEEFloat::remainder(const IEEEFloat &rhs) {
   opStatus fs;
-  IEEEFloat V = *this;
   unsigned int origSign = sign;
 
-  fs = V.divide(rhs, rmNearestTiesToEven);
-  if (fs == opDivByZero)
+  // First handle the special cases.
+  fs = remainderSpecials(rhs);
+  if (fs != opDivByZero)
     return fs;
 
-  int parts = partCount();
-  integerPart *x = new integerPart[parts];
-  bool ignored;
-  fs = V.convertToInteger(makeMutableArrayRef(x, parts),
-                          parts * integerPartWidth, true, rmNearestTiesToEven,
-                          &ignored);
-  if (fs == opInvalidOp) {
-    delete[] x;
-    return fs;
+  fs = opOK;
+
+  // Make sure the current value is less than twice the denom. If the addition
+  // did not succeed (an overflow has happened), which means that the finite
+  // value we currently posses must be less than twice the denom (as we are
+  // using the same semantics).
+  IEEEFloat P2 = rhs;
+  if (P2.add(rhs, rmNearestTiesToEven) == opOK) {
+    fs = mod(P2);
+    assert(fs == opOK);
   }
 
-  fs = V.convertFromZeroExtendedInteger(x, parts * integerPartWidth, true,
-                                        rmNearestTiesToEven);
-  assert(fs==opOK);   // should always work
+  // Lets work with absolute numbers.
+  IEEEFloat P = rhs;
+  P.sign = false;
+  sign = false;
 
-  fs = V.multiply(rhs, rmNearestTiesToEven);
-  assert(fs==opOK || fs==opInexact);   // should not overflow or underflow
+  //
+  // To calculate the remainder we use the following scheme.
+  //
+  // The remainder is defained as follows:
+  //
+  // remainder = numer - rquot * denom = x - r * p
+  //
+  // Where r is the result of: x/p, rounded toward the nearest integral value
+  // (with halfway cases rounded toward the even number).
+  //
+  // Currently, (after x mod 2p):
+  // r is the number of 2p's present inside x, which is inherently, an even
+  // number of p's.
+  //
+  // We may split the remaining calculation into 4 options:
+  // - if x < 0.5p then we round to the nearest number with is 0, and are done.
+  // - if x == 0.5p then we round to the nearest even number which is 0, and we
+  //   are done as well.
+  // - if 0.5p < x < p then we round to nearest number which is 1, and we have
+  //   to subtract 1p at least once.
+  // - if x >= p then we must subtract p at least once, as x must be a
+  //   remainder.
+  //
+  // By now, we were done, or we added 1 to r, which in turn, now an odd number.
+  //
+  // We can now split the remaining calculation to the following 3 options:
+  // - if x < 0.5p then we round to the nearest number with is 0, and are done.
+  // - if x == 0.5p then we round to the nearest even number. As r is odd, we
+  //   must round up to the next even number. so we must subtract p once more.
+  // - if x > 0.5p (and inherently x < p) then we must round r up to the next
+  //   integral, and subtract p once more.
+  //
+
+  // Extend the semantics to prevent an overflow/underflow or inexact result.
+  bool losesInfo;
+  fltSemantics extendedSemantics = *semantics;
+  extendedSemantics.maxExponent++;
+  extendedSemantics.minExponent--;
+  extendedSemantics.precision += 2;
 
-  fs = subtract(V, rmNearestTiesToEven);
-  assert(fs==opOK || fs==opInexact);   // likewise
+  IEEEFloat VEx = *this;
+  fs = VEx.convert(extendedSemantics, rmNearestTiesToEven, &losesInfo);
+  assert(fs == opOK && !losesInfo);
+  IEEEFloat PEx = P;
+  fs = PEx.convert(extendedSemantics, rmNearestTiesToEven, &losesInfo);
+  assert(fs == opOK && !losesInfo);
+
+  // It is simpler to work with 2x instead of 0.5p, and we do not need to lose
+  // any fraction.
+  fs = VEx.add(VEx, rmNearestTiesToEven);
+  assert(fs == opOK);
+
+  if (VEx.compare(PEx) == cmpGreaterThan) {
+    fs = subtract(P, rmNearestTiesToEven);
+    assert(fs == opOK);
+
+    // Make VEx = this.add(this), but because we have different semantics, we do
+    // not want to `convert` again, so we just subtract PEx twice (which equals
+    // to the desired value).
+    fs = VEx.subtract(PEx, rmNearestTiesToEven);
+    assert(fs == opOK);
+    fs = VEx.subtract(PEx, rmNearestTiesToEven);
+    assert(fs == opOK);
+
+    cmpResult result = VEx.compare(PEx);
+    if (result == cmpGreaterThan || result == cmpEqual) {
+      fs = subtract(P, rmNearestTiesToEven);
+      assert(fs == opOK);
+    }
+  }
 
   if (isZero())
     sign = origSign;    // IEEE754 requires this
-  delete[] x;
+  else
+    sign ^= origSign;
   return fs;
 }
 
diff --git a/llvm/lib/Support/CrashRecoveryContext.cpp b/llvm/lib/Support/CrashRecoveryContext.cpp
index 0cc2d4059fe95286b2a764f516885cdbf950f6e9..356835609830a808155950474fb56fa8486aa091 100644
--- a/llvm/lib/Support/CrashRecoveryContext.cpp
+++ b/llvm/lib/Support/CrashRecoveryContext.cpp
@@ -14,9 +14,6 @@
 #include "llvm/Support/ThreadLocal.h"
 #include <mutex>
 #include <setjmp.h>
-#ifdef _WIN32
-#include <windows.h> // for GetExceptionInformation
-#endif
 #if LLVM_ON_UNIX
 #include <sysexits.h> // EX_IOERR
 #endif
@@ -44,7 +41,7 @@ struct CrashRecoveryContextImpl {
   unsigned ValidJumpBuffer : 1;
 
 public:
-  CrashRecoveryContextImpl(CrashRecoveryContext *CRC)
+  CrashRecoveryContextImpl(CrashRecoveryContext *CRC) noexcept
       : CRC(CRC), Failed(false), SwitchedThread(false), ValidJumpBuffer(false) {
     Next = CurrentContext->get();
     CurrentContext->set(this);
@@ -178,6 +175,9 @@ CrashRecoveryContext::unregisterCleanup(CrashRecoveryContextCleanup *cleanup) {
 }
 
 #if defined(_MSC_VER)
+
+#include <windows.h> // for GetExceptionInformation
+
 // If _MSC_VER is defined, we must have SEH. Use it if it's available. It's way
 // better than VEH. Vectored exception handling catches all exceptions happening
 // on the thread with installed exception handlers, so it can interfere with
@@ -203,6 +203,8 @@ static int ExceptionFilter(_EXCEPTION_POINTERS *Except) {
   }
 
   int RetCode = (int)Except->ExceptionRecord->ExceptionCode;
+  if ((RetCode & 0xF0000000) == 0xE0000000)
+    RetCode &= ~0xF0000000; // this crash was generated by sys::Process::Exit
 
   // Handle the crash
   const_cast<CrashRecoveryContextImpl *>(CRCI)->HandleCrash(
@@ -280,10 +282,13 @@ static LONG CALLBACK ExceptionHandler(PEXCEPTION_POINTERS ExceptionInfo)
   // TODO: We can capture the stack backtrace here and store it on the
   // implementation if we so choose.
 
+  int RetCode = (int)ExceptionInfo->ExceptionRecord->ExceptionCode;
+  if ((RetCode & 0xF0000000) == 0xE0000000)
+    RetCode &= ~0xF0000000; // this crash was generated by sys::Process::Exit
+
   // Handle the crash
   const_cast<CrashRecoveryContextImpl *>(CRCI)->HandleCrash(
-      (int)ExceptionInfo->ExceptionRecord->ExceptionCode,
-      reinterpret_cast<uintptr_t>(ExceptionInfo));
+      RetCode, reinterpret_cast<uintptr_t>(ExceptionInfo));
 
   // Note that we don't actually get here because HandleCrash calls
   // longjmp, which means the HandleCrash function never returns.
@@ -416,6 +421,21 @@ bool CrashRecoveryContext::RunSafely(function_ref<void()> Fn) {
 
 #endif // !_MSC_VER
 
+LLVM_ATTRIBUTE_NORETURN
+void CrashRecoveryContext::HandleExit(int RetCode) {
+#if defined(_WIN32)
+  // SEH and VEH
+  ::RaiseException(0xE0000000 | RetCode, 0, 0, NULL);
+#else
+  // On Unix we don't need to raise an exception, we go directly to
+  // HandleCrash(), then longjmp will unwind the stack for us.
+  CrashRecoveryContextImpl *CRCI = (CrashRecoveryContextImpl *)Impl;
+  assert(CRCI && "Crash recovery context never initialized!");
+  CRCI->HandleCrash(RetCode, 0 /*no sig num*/);
+#endif
+  llvm_unreachable("Most likely setjmp wasn't called!");
+}
+
 // FIXME: Portability.
 static void setThreadBackgroundPriority() {
 #ifdef __APPLE__
diff --git a/llvm/lib/Support/ErrorHandling.cpp b/llvm/lib/Support/ErrorHandling.cpp
index 0f13f7a536f1d11cdc7618a5af3b5c6dbbfd6e13..a9463024c420f740a68dad52564606c4490a75c9 100644
--- a/llvm/lib/Support/ErrorHandling.cpp
+++ b/llvm/lib/Support/ErrorHandling.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/WindowsError.h"
@@ -122,7 +123,7 @@ void llvm::report_fatal_error(const Twine &Reason, bool GenCrashDiag) {
   // files registered with RemoveFileOnSignal.
   sys::RunInterruptHandlers();
 
-  exit(1);
+  sys::Process::Exit(1);
 }
 
 void llvm::install_bad_alloc_error_handler(fatal_error_handler_t handler,
diff --git a/llvm/lib/Support/ItaniumManglingCanonicalizer.cpp b/llvm/lib/Support/ItaniumManglingCanonicalizer.cpp
index bbc06d186fba47324fddf5f95fa0e89dfb76c07e..7594c74b83b2e50cd7fd80c379cf054244ad0d1a 100644
--- a/llvm/lib/Support/ItaniumManglingCanonicalizer.cpp
+++ b/llvm/lib/Support/ItaniumManglingCanonicalizer.cpp
@@ -30,9 +30,8 @@ struct FoldingSetNodeIDBuilder {
   void operator()(StringView Str) {
     ID.AddString(llvm::StringRef(Str.begin(), Str.size()));
   }
-  template<typename T>
-  typename std::enable_if<std::is_integral<T>::value ||
-                          std::is_enum<T>::value>::type
+  template <typename T>
+  std::enable_if_t<std::is_integral<T>::value || std::is_enum<T>::value>
   operator()(T V) {
     ID.AddInteger((unsigned long long)V);
   }
diff --git a/llvm/lib/Support/NativeFormatting.cpp b/llvm/lib/Support/NativeFormatting.cpp
index 3731e0c5635991d2ac373118f4d56a8bec3b1d49..9cd494908016f86faeced5519bc6b5a7f4bdbf6e 100644
--- a/llvm/lib/Support/NativeFormatting.cpp
+++ b/llvm/lib/Support/NativeFormatting.cpp
@@ -89,7 +89,7 @@ static void write_signed(raw_ostream &S, T N, size_t MinDigits,
                          IntegerStyle Style) {
   static_assert(std::is_signed<T>::value, "Value is not signed!");
 
-  using UnsignedT = typename std::make_unsigned<T>::type;
+  using UnsignedT = std::make_unsigned_t<T>;
 
   if (N >= 0) {
     write_unsigned(S, static_cast<UnsignedT>(N), MinDigits, Style);
diff --git a/llvm/lib/Support/Process.cpp b/llvm/lib/Support/Process.cpp
index 0b2d41bcdeca86c00ec22c084163b4fd7cd7f42d..9e6e233b26ac24cc69c92f598be0fda877d1c0d5 100644
--- a/llvm/lib/Support/Process.cpp
+++ b/llvm/lib/Support/Process.cpp
@@ -13,8 +13,9 @@
 #include "llvm/Support/Process.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/Config/config.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
@@ -88,6 +89,13 @@ static bool coreFilesPrevented = !LLVM_ENABLE_CRASH_DUMPS;
 
 bool Process::AreCoreFilesPrevented() { return coreFilesPrevented; }
 
+LLVM_ATTRIBUTE_NORETURN
+void Process::Exit(int RetCode) {
+  if (CrashRecoveryContext *CRC = CrashRecoveryContext::GetCurrent())
+    CRC->HandleExit(RetCode);
+  ::exit(RetCode);
+}
+
 // Include the platform-specific parts of this class.
 #ifdef LLVM_ON_UNIX
 #include "Unix/Process.inc"
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e01762e2ca9bca23f05e337b31d9430e0d87598a..57dc51d068fd4514caf0efa839197706844f37ec 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -4169,7 +4169,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
         RegsToPass.emplace_back(VA.getLocReg(), Arg);
         RegsUsed.insert(VA.getLocReg());
         const TargetOptions &Options = DAG.getTarget().Options;
-        if (Options.EnableDebugEntryValues)
+        if (Options.SupportsDebugEntryValues)
           CSInfo.emplace_back(VA.getLocReg(), i);
       }
     } else {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index bc7b4384662a600024d2fa587efcb8438c71325f..d50bd090aeeeca617887ebae0a27a1b18e28a8f9 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -309,6 +309,9 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
 
   // AArch64 supports default outlining behaviour.
   setSupportsDefaultOutlining(true);
+
+  // AArch64 supports the debug entry values.
+  setSupportsDebugEntryValues(true);
 }
 
 AArch64TargetMachine::~AArch64TargetMachine() = default;
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index fb09cc247a1ace9c34feb7ba066b8000f5a5e473..fa3d64b4666d432dafa2251cb673fc7c4da6bbe4 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -260,6 +260,8 @@ public:
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
   bool ParseDirective(AsmToken DirectiveID) override;
   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                       unsigned Kind) override;
@@ -755,8 +757,8 @@ public:
       return false;
 
     int64_t Val = MCE->getValue();
-    int64_t SVal = typename std::make_signed<T>::type(Val);
-    int64_t UVal = typename std::make_unsigned<T>::type(Val);
+    int64_t SVal = std::make_signed_t<T>(Val);
+    int64_t UVal = std::make_unsigned_t<T>(Val);
     if (Val != SVal && Val != UVal)
       return false;
 
@@ -852,8 +854,7 @@ public:
     if (!isShiftedImm() && (!isImm() || !isa<MCConstantExpr>(getImm())))
       return DiagnosticPredicateTy::NoMatch;
 
-    bool IsByte =
-        std::is_same<int8_t, typename std::make_signed<T>::type>::value;
+    bool IsByte = std::is_same<int8_t, std::make_signed_t<T>>::value;
     if (auto ShiftedImm = getShiftedVal<8>())
       if (!(IsByte && ShiftedImm->second) &&
           AArch64_AM::isSVECpyImm<T>(uint64_t(ShiftedImm->first)
@@ -870,8 +871,7 @@ public:
     if (!isShiftedImm() && (!isImm() || !isa<MCConstantExpr>(getImm())))
       return DiagnosticPredicateTy::NoMatch;
 
-    bool IsByte =
-        std::is_same<int8_t, typename std::make_signed<T>::type>::value;
+    bool IsByte = std::is_same<int8_t, std::make_signed_t<T>>::value;
     if (auto ShiftedImm = getShiftedVal<8>())
       if (!(IsByte && ShiftedImm->second) &&
           AArch64_AM::isSVEAddSubImm<T>(ShiftedImm->first
@@ -1608,7 +1608,7 @@ public:
   void addLogicalImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    typename std::make_unsigned<T>::type Val = MCE->getValue();
+    std::make_unsigned_t<T> Val = MCE->getValue();
     uint64_t encoding = AArch64_AM::encodeLogicalImmediate(Val, sizeof(T) * 8);
     Inst.addOperand(MCOperand::createImm(encoding));
   }
@@ -1617,7 +1617,7 @@ public:
   void addLogicalImmNotOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
-    typename std::make_unsigned<T>::type Val = ~MCE->getValue();
+    std::make_unsigned_t<T> Val = ~MCE->getValue();
     uint64_t encoding = AArch64_AM::encodeLogicalImmediate(Val, sizeof(T) * 8);
     Inst.addOperand(MCOperand::createImm(encoding));
   }
@@ -2245,10 +2245,16 @@ static unsigned matchSVEPredicateVectorRegName(StringRef Name) {
 
 bool AArch64AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                      SMLoc &EndLoc) {
+  return tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success;
+}
+
+OperandMatchResultTy AArch64AsmParser::tryParseRegister(unsigned &RegNo,
+                                                        SMLoc &StartLoc,
+                                                        SMLoc &EndLoc) {
   StartLoc = getLoc();
   auto Res = tryParseScalarRegister(RegNo);
   EndLoc = SMLoc::getFromPointer(getLoc().getPointer() - 1);
-  return Res != MatchOperand_Success;
+  return Res;
 }
 
 // Matches a register name or register alias previously defined by '.req'
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
index 05a909f1780a0e0db469fe000a7cc8cece8f0b3f..9814f762585385a43596a067989098f487b54af1 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
@@ -763,10 +763,10 @@ static inline bool isSVECpyImm(int64_t Imm) {
   bool IsImm8 = int8_t(Imm) == Imm;
   bool IsImm16 = int16_t(Imm & ~0xff) == Imm;
 
-  if (std::is_same<int8_t, typename std::make_signed<T>::type>::value)
+  if (std::is_same<int8_t, std::make_signed_t<T>>::value)
     return IsImm8 || uint8_t(Imm) == Imm;
 
-  if (std::is_same<int16_t, typename std::make_signed<T>::type>::value)
+  if (std::is_same<int16_t, std::make_signed_t<T>>::value)
     return IsImm8 || IsImm16 || uint16_t(Imm & ~0xff) == Imm;
 
   return IsImm8 || IsImm16;
@@ -775,8 +775,7 @@ static inline bool isSVECpyImm(int64_t Imm) {
 /// Returns true if Imm is valid for ADD/SUB.
 template <typename T>
 static inline bool isSVEAddSubImm(int64_t Imm) {
-  bool IsInt8t =
-      std::is_same<int8_t, typename std::make_signed<T>::type>::value;
+  bool IsInt8t = std::is_same<int8_t, std::make_signed_t<T>>::value;
   return uint8_t(Imm) == Imm || (!IsInt8t && uint16_t(Imm & ~0xff) == Imm);
 }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index f0a2dd76710aa4a27ecc1e07d1b8dc52baa047f9..6a23cdae2083eae9a5ec6a4639f7db69affe02ff 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -1511,7 +1511,7 @@ void AArch64InstPrinter::printSVERegOp(const MCInst *MI, unsigned OpNum,
 
 template <typename T>
 void AArch64InstPrinter::printImmSVE(T Value, raw_ostream &O) {
-  typename std::make_unsigned<T>::type HexValue = Value;
+  std::make_unsigned_t<T> HexValue = Value;
 
   if (getPrintImmHex())
     O << '#' << formatHex((uint64_t)HexValue);
@@ -1556,8 +1556,8 @@ template <typename T>
 void AArch64InstPrinter::printSVELogicalImm(const MCInst *MI, unsigned OpNum,
                                             const MCSubtargetInfo &STI,
                                             raw_ostream &O) {
-  typedef typename std::make_signed<T>::type SignedT;
-  typedef typename std::make_unsigned<T>::type UnsignedT;
+  typedef std::make_signed_t<T> SignedT;
+  typedef std::make_unsigned_t<T> UnsignedT;
 
   uint64_t Val = MI->getOperand(OpNum).getImm();
   UnsignedT PrintVal = AArch64_AM::decodeLogicalImmediate(Val, 64);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 106480311b5d124f30e5b46036220c2f1c512966..9623e1d0f165d6aca5ae21e277303adb8b01b137 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1175,7 +1175,7 @@ def EnableLateCFGStructurize : Predicate<
 include "SISchedule.td"
 include "GCNProcessors.td"
 include "AMDGPUInstrInfo.td"
-include "AMDGPURegisterInfo.td"
+include "SIRegisterInfo.td"
 include "AMDGPURegisterBanks.td"
 include "AMDGPUInstructions.td"
 include "SIInstrInfo.td"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index e3cc2a4abdee0932ee2c27aad83f02a0ff046291..8840b0a180c09ac7f1abc630455ade824ab59a3c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -67,6 +67,7 @@ class AMDGPUCodeGenPrepare : public FunctionPass,
                              public InstVisitor<AMDGPUCodeGenPrepare, bool> {
   const GCNSubtarget *ST = nullptr;
   AssumptionCache *AC = nullptr;
+  DominatorTree *DT = nullptr;
   LegacyDivergenceAnalysis *DA = nullptr;
   Module *Mod = nullptr;
   const DataLayout *DL = nullptr;
@@ -157,6 +158,9 @@ class AMDGPUCodeGenPrepare : public FunctionPass,
   /// we expand some divisions here, we need to perform this before obscuring.
   bool foldBinOpIntoSelect(BinaryOperator &I) const;
 
+  bool divHasSpecialOptimization(BinaryOperator &I,
+                                 Value *Num, Value *Den) const;
+
   /// Expands 24 bit div or rem.
   Value* expandDivRem24(IRBuilder<> &Builder, BinaryOperator &I,
                         Value *Num, Value *Den,
@@ -620,10 +624,12 @@ static Value *optimizeWithRcp(Value *Num, Value *Den, bool AllowInaccurateRcp,
     return nullptr;
 
   Type *Ty = Den->getType();
-  Function *Decl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp, Ty);
   if (const ConstantFP *CLHS = dyn_cast<ConstantFP>(Num)) {
     if (AllowInaccurateRcp || RcpIsAccurate) {
       if (CLHS->isExactlyValue(1.0)) {
+        Function *Decl = Intrinsic::getDeclaration(
+          Mod, Intrinsic::amdgcn_rcp, Ty);
+
         // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
         // the CI documentation has a worst case error of 1 ulp.
         // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
@@ -636,10 +642,13 @@ static Value *optimizeWithRcp(Value *Num, Value *Den, bool AllowInaccurateRcp,
 
         // 1.0 / x -> rcp(x)
         return Builder.CreateCall(Decl, { Den });
-       }
+      }
 
        // Same as for 1.0, but expand the sign out of the constant.
-       if (CLHS->isExactlyValue(-1.0)) {
+      if (CLHS->isExactlyValue(-1.0)) {
+        Function *Decl = Intrinsic::getDeclaration(
+          Mod, Intrinsic::amdgcn_rcp, Ty);
+
          // -1.0 / x -> rcp (fneg x)
          Value *FNeg = Builder.CreateFNeg(Den);
          return Builder.CreateCall(Decl, { FNeg });
@@ -648,6 +657,9 @@ static Value *optimizeWithRcp(Value *Num, Value *Den, bool AllowInaccurateRcp,
   }
 
   if (AllowInaccurateRcp) {
+    Function *Decl = Intrinsic::getDeclaration(
+      Mod, Intrinsic::amdgcn_rcp, Ty);
+
     // Turn into multiply by the reciprocal.
     // x / y -> x * (1.0 / y)
     Value *Recip = Builder.CreateCall(Decl, { Den });
@@ -847,7 +859,9 @@ Value* AMDGPUCodeGenPrepare::expandDivRem24(IRBuilder<> &Builder,
   Value *FB = IsSigned ? Builder.CreateSIToFP(IB,F32Ty)
                        : Builder.CreateUIToFP(IB,F32Ty);
 
-  Value *RCP = Builder.CreateFDiv(ConstantFP::get(F32Ty, 1.0), FB);
+  Function *RcpDecl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp,
+                                                Builder.getFloatTy());
+  Value *RCP = Builder.CreateCall(RcpDecl, { FB });
   Value *FQM = Builder.CreateFMul(FA, RCP);
 
   // fq = trunc(fqm);
@@ -899,6 +913,42 @@ Value* AMDGPUCodeGenPrepare::expandDivRem24(IRBuilder<> &Builder,
   return Res;
 }
 
+// Try to recognize special cases the DAG will emit special, better expansions
+// than the general expansion we do here.
+
+// TODO: It would be better to just directly handle those optimizations here.
+bool AMDGPUCodeGenPrepare::divHasSpecialOptimization(
+  BinaryOperator &I, Value *Num, Value *Den) const {
+  if (Constant *C = dyn_cast<Constant>(Den)) {
+    // Arbitrary constants get a better expansion as long as a wider mulhi is
+    // legal.
+    if (C->getType()->getScalarSizeInBits() <= 32)
+      return true;
+
+    // TODO: Sdiv check for not exact for some reason.
+
+    // If there's no wider mulhi, there's only a better expansion for powers of
+    // two.
+    // TODO: Should really know for each vector element.
+    if (isKnownToBeAPowerOfTwo(C, *DL, true, 0, AC, &I, DT))
+      return true;
+
+    return false;
+  }
+
+  if (BinaryOperator *BinOpDen = dyn_cast<BinaryOperator>(Den)) {
+    // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
+    if (BinOpDen->getOpcode() == Instruction::Shl &&
+        isa<Constant>(BinOpDen->getOperand(0)) &&
+        isKnownToBeAPowerOfTwo(BinOpDen->getOperand(0), *DL, true,
+                               0, AC, &I, DT)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
                                             BinaryOperator &I,
                                             Value *Num, Value *Den) const {
@@ -910,8 +960,8 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
   FMF.setFast();
   Builder.setFastMathFlags(FMF);
 
-  if (isa<Constant>(Den))
-    return nullptr; // Keep it for optimization
+  if (divHasSpecialOptimization(I, Num, Den))
+    return nullptr;  // Keep it for later optimization.
 
   bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
   bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
@@ -937,7 +987,6 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
 
   ConstantInt *Zero = Builder.getInt32(0);
   ConstantInt *One = Builder.getInt32(1);
-  ConstantInt *MinusOne = Builder.getInt32(~0);
 
   Value *Sign = nullptr;
   if (IsSigned) {
@@ -957,7 +1006,10 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
   // RCP =  URECIP(Den) = 2^32 / Den + e
   // e is rounding error.
   Value *DEN_F32 = Builder.CreateUIToFP(Den, F32Ty);
-  Value *RCP_F32 = Builder.CreateFDiv(ConstantFP::get(F32Ty, 1.0), DEN_F32);
+
+  Function *RcpDecl = Intrinsic::getDeclaration(Mod, Intrinsic::amdgcn_rcp,
+                                                Builder.getFloatTy());
+  Value *RCP_F32 = Builder.CreateCall(RcpDecl, { DEN_F32 });
   Constant *UINT_MAX_PLUS_1 = ConstantFP::get(F32Ty, BitsToFloat(0x4f800000));
   Value *RCP_SCALE = Builder.CreateFMul(RCP_F32, UINT_MAX_PLUS_1);
   Value *RCP = Builder.CreateFPToUI(RCP_SCALE, I32Ty);
@@ -995,18 +1047,14 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
   // Remainder = Num - Num_S_Remainder
   Value *Remainder = Builder.CreateSub(Num, Num_S_Remainder);
 
-  // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
-  Value *Rem_GE_Den_CC = Builder.CreateICmpUGE(Remainder, Den);
-  Value *Remainder_GE_Den = Builder.CreateSelect(Rem_GE_Den_CC, MinusOne, Zero);
+  // Remainder_GE_Den = Remainder >= Den;
+  Value *Remainder_GE_Den = Builder.CreateICmpUGE(Remainder, Den);
 
-  // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
-  Value *Num_GE_Num_S_Rem_CC = Builder.CreateICmpUGE(Num, Num_S_Remainder);
-  Value *Remainder_GE_Zero = Builder.CreateSelect(Num_GE_Num_S_Rem_CC,
-                                                  MinusOne, Zero);
+  // Remainder_GE_Zero = Num >= Num_S_Remainder
+  Value *Remainder_GE_Zero = Builder.CreateICmpUGE(Num, Num_S_Remainder);
 
   // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
   Value *Tmp1 = Builder.CreateAnd(Remainder_GE_Den, Remainder_GE_Zero);
-  Value *Tmp1_0_CC = Builder.CreateICmpEQ(Tmp1, Zero);
 
   Value *Res;
   if (IsDiv) {
@@ -1016,11 +1064,11 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
     // Quotient_S_One = Quotient - 1
     Value *Quotient_S_One = Builder.CreateSub(Quotient, One);
 
-    // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
-    Value *Div = Builder.CreateSelect(Tmp1_0_CC, Quotient, Quotient_A_One);
+    // Div = (Tmp1 ? Quotient_A_One : Quotient)
+    Value *Div = Builder.CreateSelect(Tmp1, Quotient_A_One, Quotient);
 
-    // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
-    Res = Builder.CreateSelect(Num_GE_Num_S_Rem_CC, Div, Quotient_S_One);
+    // Div = (Remainder_GE_Zero ? Div : Quotient_S_One)
+    Res = Builder.CreateSelect(Remainder_GE_Zero, Div, Quotient_S_One);
   } else {
     // Remainder_S_Den = Remainder - Den
     Value *Remainder_S_Den = Builder.CreateSub(Remainder, Den);
@@ -1028,11 +1076,11 @@ Value* AMDGPUCodeGenPrepare::expandDivRem32(IRBuilder<> &Builder,
     // Remainder_A_Den = Remainder + Den
     Value *Remainder_A_Den = Builder.CreateAdd(Remainder, Den);
 
-    // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
-    Value *Rem = Builder.CreateSelect(Tmp1_0_CC, Remainder, Remainder_S_Den);
+    // Rem = (Tmp1 ?  Remainder_S_Den : Remainder)
+    Value *Rem = Builder.CreateSelect(Tmp1, Remainder_S_Den, Remainder);
 
-    // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
-    Res = Builder.CreateSelect(Num_GE_Num_S_Rem_CC, Rem, Remainder_A_Den);
+    // Rem = (Remainder_GE_Zero ? Rem : Remainder_A_Den)
+    Res = Builder.CreateSelect(Remainder_GE_Zero, Rem, Remainder_A_Den);
   }
 
   if (IsSigned) {
@@ -1198,6 +1246,10 @@ bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
   ST = &TM.getSubtarget<GCNSubtarget>(F);
   AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   DA = &getAnalysis<LegacyDivergenceAnalysis>();
+
+  auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
+  DT = DTWP ? &DTWP->getDomTree() : nullptr;
+
   HasUnsafeFPMath = hasUnsafeFPMath(F);
   HasFP32Denormals = ST->hasFP32Denormals(F);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 2abba0cfb727725203969d291ce7fd7c14747d9d..16a41be23eed3bed12b2eec003290fafe0b4dbf7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -16,7 +16,6 @@
 #include "AMDGPUISelLowering.h" // For AMDGPUISD
 #include "AMDGPUInstrInfo.h"
 #include "AMDGPUPerfHintAnalysis.h"
-#include "AMDGPURegisterInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index e0cd2c6b29c6e4d4fbbafdecdb3edb7369385945..24e0b9a6d02e6170a9aad6e824ccd1444b07ab8e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -16,7 +16,6 @@
 #include "AMDGPU.h"
 #include "AMDGPUCallLowering.h"
 #include "AMDGPUFrameLowering.h"
-#include "AMDGPURegisterInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
 #include "Utils/AMDGPUBaseInfo.h"
@@ -1020,6 +1019,8 @@ void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
         assert(MemVT.getVectorNumElements() == 3 ||
                MemVT.getVectorNumElements() == 5);
         MemVT = MemVT.getPow2VectorType(State.getContext());
+      } else if (!MemVT.isSimple() && !MemVT.isVector()) {
+        MemVT = MemVT.getRoundIntegerType(State.getContext());
       }
 
       unsigned PartOffset = 0;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
index 9951cbf2326e35ee94d8eabc19d5dc7a45013161..6c13bc8599dbbd9395e6115f1f588568b522b211 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
@@ -13,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUInstrInfo.h"
-#include "AMDGPURegisterInfo.h"
 #include "AMDGPUTargetMachine.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index ff7cb9dc2c1a9ed9bbb1d1aa8dd0d9420f18e7e6..935db23da63558e8e907c7f6da4872400f626e63 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -15,7 +15,6 @@
 #include "AMDGPUInstrInfo.h"
 #include "AMDGPUGlobalISelUtils.h"
 #include "AMDGPURegisterBankInfo.h"
-#include "AMDGPURegisterInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -2375,18 +2374,16 @@ AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const
     return std::make_pair(Root.getReg(), 0);
 
   int64_t ConstAddr = 0;
-  if (isBaseWithConstantOffset(Root, *MRI)) {
-    const MachineOperand &LHS = RootDef->getOperand(1);
-    const MachineOperand &RHS = RootDef->getOperand(2);
-    const MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
-    const MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
-    if (LHSDef && RHSDef) {
-      int64_t PossibleOffset =
-        RHSDef->getOperand(1).getCImm()->getSExtValue();
-      if (isDSOffsetLegal(LHS.getReg(), PossibleOffset, 16)) {
-        // (add n0, c0)
-        return std::make_pair(LHS.getReg(), PossibleOffset);
-      }
+
+  Register PtrBase;
+  int64_t Offset;
+  std::tie(PtrBase, Offset) =
+    getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
+
+  if (Offset) {
+    if (isDSOffsetLegal(PtrBase, Offset, 16)) {
+      // (add n0, c0)
+      return std::make_pair(PtrBase, Offset);
     }
   } else if (RootDef->getOpcode() == AMDGPU::G_SUB) {
     // TODO
@@ -2402,7 +2399,6 @@ AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
-
   Register Reg;
   unsigned Offset;
   std::tie(Reg, Offset) = selectDS1Addr1OffsetImpl(Root);
@@ -2414,19 +2410,26 @@ AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
+  Register Reg;
+  unsigned Offset;
+  std::tie(Reg, Offset) = selectDS64Bit4ByteAlignedImpl(Root);
+  return {{
+      [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); },
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); },
+      [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset+1); }
+    }};
+}
+
+std::pair<Register, unsigned>
+AMDGPUInstructionSelector::selectDS64Bit4ByteAlignedImpl(MachineOperand &Root) const {
   const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
-  if (!RootDef) {
-    return {{
-        [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
-        [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
-        [=](MachineInstrBuilder &MIB) { MIB.addImm(1); }
-      }};
-  }
+  if (!RootDef)
+    return std::make_pair(Root.getReg(), 0);
 
   int64_t ConstAddr = 0;
+
   Register PtrBase;
   int64_t Offset;
-
   std::tie(PtrBase, Offset) =
     getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
 
@@ -2435,11 +2438,7 @@ AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const
     int64_t DWordOffset1 = DWordOffset0 + 1;
     if (isDSOffsetLegal(PtrBase, DWordOffset1, 8)) {
       // (add n0, c0)
-      return {{
-          [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrBase); },
-          [=](MachineInstrBuilder &MIB) { MIB.addImm(DWordOffset0); },
-          [=](MachineInstrBuilder &MIB) { MIB.addImm(DWordOffset1); }
-        }};
+      return std::make_pair(PtrBase, DWordOffset0);
     }
   } else if (RootDef->getOpcode() == AMDGPU::G_SUB) {
     // TODO
@@ -2449,11 +2448,7 @@ AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const
 
   }
 
-  return {{
-      [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
-      [=](MachineInstrBuilder &MIB) { MIB.addImm(1); }
-    }};
+  return std::make_pair(Root.getReg(), 0);
 }
 
 /// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 38b76a76299ff63f84b5ed9184d6fed56ad3b70a..d440932c72dd80013266cd925a91265844417daf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -175,10 +175,12 @@ private:
                        unsigned OffsetBits) const;
 
   std::pair<Register, unsigned>
-  selectDS1Addr1OffsetImpl(MachineOperand &Src) const;
-
+  selectDS1Addr1OffsetImpl(MachineOperand &Root) const;
   InstructionSelector::ComplexRendererFns
   selectDS1Addr1Offset(MachineOperand &Root) const;
+
+  std::pair<Register, unsigned>
+  selectDS64Bit4ByteAlignedImpl(MachineOperand &Root) const;
   InstructionSelector::ComplexRendererFns
   selectDS64Bit4ByteAligned(MachineOperand &Root) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
deleted file mode 100644
index 148e761125a3cecc45c8a6316a4a6b22eeee5480..0000000000000000000000000000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-//===-- AMDGPURegisterInfo.cpp - AMDGPU Register Information -------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// Parent TargetRegisterInfo class common to all hw codegen targets.
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPURegisterInfo.h"
-#include "AMDGPUTargetMachine.h"
-#include "SIMachineFunctionInfo.h"
-#include "SIRegisterInfo.h"
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-
-using namespace llvm;
-
-AMDGPURegisterInfo::AMDGPURegisterInfo() : AMDGPUGenRegisterInfo(0) {}
-
-void AMDGPURegisterInfo::reserveRegisterTuples(BitVector &Reserved, unsigned Reg) const {
-  MCRegAliasIterator R(Reg, this, true);
-
-  for (; R.isValid(); ++R)
-    Reserved.set(*R);
-}
-
-#define GET_REGINFO_TARGET_DESC
-#include "AMDGPUGenRegisterInfo.inc"
-
-// Forced to be here by one .inc
-const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs(
-  const MachineFunction *MF) const {
-  CallingConv::ID CC = MF->getFunction().getCallingConv();
-  switch (CC) {
-  case CallingConv::C:
-  case CallingConv::Fast:
-  case CallingConv::Cold:
-    return CSR_AMDGPU_HighRegs_SaveList;
-  default: {
-    // Dummy to not crash RegisterClassInfo.
-    static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister;
-    return &NoCalleeSavedReg;
-  }
-  }
-}
-
-const MCPhysReg *
-SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const {
-  return nullptr;
-}
-
-const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
-                                                     CallingConv::ID CC) const {
-  switch (CC) {
-  case CallingConv::C:
-  case CallingConv::Fast:
-  case CallingConv::Cold:
-    return CSR_AMDGPU_HighRegs_RegMask;
-  default:
-    return nullptr;
-  }
-}
-
-Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  const SIFrameLowering *TFI =
-      MF.getSubtarget<GCNSubtarget>().getFrameLowering();
-  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
-  return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg()
-                        : FuncInfo->getStackPtrOffsetReg();
-}
-
-const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const {
-  return CSR_AMDGPU_AllVGPRs_RegMask;
-}
-
-const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const {
-  return CSR_AMDGPU_AllAllocatableSRegs_RegMask;
-}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.h
deleted file mode 100644
index e8dc8ba66e1d6229758d52dbadfbff61b72bdb2c..0000000000000000000000000000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.h
+++ /dev/null
@@ -1,34 +0,0 @@
-//===-- AMDGPURegisterInfo.h - AMDGPURegisterInfo Interface -*- C++ -*-----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// TargetRegisterInfo interface that is implemented by all hw codegen
-/// targets.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUREGISTERINFO_H
-#define LLVM_LIB_TARGET_AMDGPU_AMDGPUREGISTERINFO_H
-
-#define GET_REGINFO_HEADER
-#include "AMDGPUGenRegisterInfo.inc"
-
-namespace llvm {
-
-class GCNSubtarget;
-class TargetInstrInfo;
-
-struct AMDGPURegisterInfo : public AMDGPUGenRegisterInfo {
-  AMDGPURegisterInfo();
-
-  void reserveRegisterTuples(BitVector &, unsigned Reg) const;
-};
-
-} // End namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.td b/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.td
deleted file mode 100644
index ab71b7aa8a57223fbbf7425406cbbaa69c867658..0000000000000000000000000000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterInfo.td
+++ /dev/null
@@ -1,21 +0,0 @@
-//===-- AMDGPURegisterInfo.td - AMDGPU register info -------*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Tablegen register definitions common to all hw codegen targets.
-//
-//===----------------------------------------------------------------------===//
-
-let Namespace = "AMDGPU" in {
-
-foreach Index = 0-31 in {
-  def sub#Index : SubRegIndex<32, !shl(Index, 5)>;
-}
-
-}
-
-include "SIRegisterInfo.td"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index a7eb081d1a25dc15905d981d1a75ec96cef71187..103b2bb8a6cc944765315246c9f27fc913a79503 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -247,6 +247,7 @@ def : SourceOfDivergence<int_amdgcn_permlanex16>;
 def : SourceOfDivergence<int_amdgcn_mov_dpp>;
 def : SourceOfDivergence<int_amdgcn_mov_dpp8>;
 def : SourceOfDivergence<int_amdgcn_update_dpp>;
+def : SourceOfDivergence<int_amdgcn_writelane>;
 
 def : SourceOfDivergence<int_amdgcn_mfma_f32_4x4x1f32>;
 def : SourceOfDivergence<int_amdgcn_mfma_f32_4x4x1f32>;
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 8c1f63e4ca864ad7433888f88aab0bf4000dbd50..3ef493bcb1565907462e2d98ee34937ebba80b63 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1065,17 +1065,20 @@ private:
 
   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
                              RegisterKind RegKind, unsigned Reg1);
-  bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
-                           unsigned& RegNum, unsigned& RegWidth);
-  unsigned ParseRegularReg(RegisterKind &RegKind,
-                           unsigned &RegNum,
-                           unsigned &RegWidth);
-  unsigned ParseSpecialReg(RegisterKind &RegKind,
-                           unsigned &RegNum,
-                           unsigned &RegWidth);
-  unsigned ParseRegList(RegisterKind &RegKind,
-                        unsigned &RegNum,
-                        unsigned &RegWidth);
+  bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
+                           unsigned &RegNum, unsigned &RegWidth,
+                           bool RestoreOnFailure = false);
+  bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
+                           unsigned &RegNum, unsigned &RegWidth,
+                           SmallVectorImpl<AsmToken> &Tokens);
+  unsigned ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum,
+                           unsigned &RegWidth,
+                           SmallVectorImpl<AsmToken> &Tokens);
+  unsigned ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum,
+                           unsigned &RegWidth,
+                           SmallVectorImpl<AsmToken> &Tokens);
+  unsigned ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
+                        unsigned &RegWidth, SmallVectorImpl<AsmToken> &Tokens);
   bool ParseRegRange(unsigned& Num, unsigned& Width);
   unsigned getRegularReg(RegisterKind RegKind,
                          unsigned RegNum,
@@ -1233,8 +1236,12 @@ public:
   bool isForcedSDWA() const { return ForcedSDWA; }
   ArrayRef<unsigned> getMatchedVariants() const;
 
-  std::unique_ptr<AMDGPUOperand> parseRegister();
+  std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false);
+  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
+                     bool RestoreOnFailure);
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
   unsigned checkTargetMatchPredicate(MCInst &Inst) override;
   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                       unsigned Kind) override;
@@ -1987,7 +1994,7 @@ static unsigned getSpecialRegForName(StringRef RegName) {
 }
 
 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
-                                    SMLoc &EndLoc) {
+                                    SMLoc &EndLoc, bool RestoreOnFailure) {
   auto R = parseRegister();
   if (!R) return true;
   assert(R->isReg());
@@ -1997,6 +2004,25 @@ bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
   return false;
 }
 
+bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                    SMLoc &EndLoc) {
+  return ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
+}
+
+OperandMatchResultTy AMDGPUAsmParser::tryParseRegister(unsigned &RegNo,
+                                                       SMLoc &StartLoc,
+                                                       SMLoc &EndLoc) {
+  bool Result =
+      ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
+  bool PendingErrors = getParser().hasPendingError();
+  getParser().clearPendingErrors();
+  if (PendingErrors)
+    return MatchOperand_ParseFail;
+  if (Result)
+    return MatchOperand_NoMatch;
+  return MatchOperand_Success;
+}
+
 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
                                             RegisterKind RegKind, unsigned Reg1) {
   switch (RegKind) {
@@ -2173,31 +2199,31 @@ AMDGPUAsmParser::ParseRegRange(unsigned& Num, unsigned& Width) {
   return true;
 }
 
-unsigned
-AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
-                                 unsigned &RegNum,
-                                 unsigned &RegWidth) {
+unsigned AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
+                                          unsigned &RegNum, unsigned &RegWidth,
+                                          SmallVectorImpl<AsmToken> &Tokens) {
   assert(isToken(AsmToken::Identifier));
   unsigned Reg = getSpecialRegForName(getTokenStr());
   if (Reg) {
     RegNum = 0;
     RegWidth = 1;
     RegKind = IS_SPECIAL;
+    Tokens.push_back(getToken());
     lex(); // skip register name
   }
   return Reg;
 }
 
-unsigned
-AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
-                                 unsigned &RegNum,
-                                 unsigned &RegWidth) {
+unsigned AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
+                                          unsigned &RegNum, unsigned &RegWidth,
+                                          SmallVectorImpl<AsmToken> &Tokens) {
   assert(isToken(AsmToken::Identifier));
   StringRef RegName = getTokenStr();
 
   const RegInfo *RI = getRegularRegInfo(RegName);
   if (!RI)
     return AMDGPU::NoRegister;
+  Tokens.push_back(getToken());
   lex(); // skip register name
 
   RegKind = RI->Kind;
@@ -2216,10 +2242,9 @@ AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
   return getRegularReg(RegKind, RegNum, RegWidth);
 }
 
-unsigned
-AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
-                              unsigned &RegNum,
-                              unsigned &RegWidth) {
+unsigned AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
+                                       unsigned &RegWidth,
+                                       SmallVectorImpl<AsmToken> &Tokens) {
   unsigned Reg = AMDGPU::NoRegister;
 
   if (!trySkipToken(AsmToken::LBrac))
@@ -2236,7 +2261,8 @@ AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
     RegisterKind NextRegKind;
     unsigned NextReg, NextRegNum, NextRegWidth;
 
-    if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth))
+    if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
+                             Tokens))
       return AMDGPU::NoRegister;
     if (NextRegWidth != 1)
       return AMDGPU::NoRegister;
@@ -2255,24 +2281,40 @@ AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
   return Reg;
 }
 
-bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
-                                          unsigned &Reg,
-                                          unsigned &RegNum,
-                                          unsigned &RegWidth) {
+bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
+                                          unsigned &RegNum, unsigned &RegWidth,
+                                          SmallVectorImpl<AsmToken> &Tokens) {
   Reg = AMDGPU::NoRegister;
 
   if (isToken(AsmToken::Identifier)) {
-    Reg = ParseSpecialReg(RegKind, RegNum, RegWidth);
+    Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
     if (Reg == AMDGPU::NoRegister)
-      Reg = ParseRegularReg(RegKind, RegNum, RegWidth);
+      Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
   } else {
-    Reg = ParseRegList(RegKind, RegNum, RegWidth);
+    Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
   }
 
   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
   return Reg != AMDGPU::NoRegister && subtargetHasRegister(*TRI, Reg);
 }
 
+bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
+                                          unsigned &RegNum, unsigned &RegWidth,
+                                          bool RestoreOnFailure) {
+  Reg = AMDGPU::NoRegister;
+
+  SmallVector<AsmToken, 1> Tokens;
+  if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) {
+    if (RestoreOnFailure) {
+      while (!Tokens.empty()) {
+        getLexer().UnLex(Tokens.pop_back_val());
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
 Optional<StringRef>
 AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
   switch (RegKind) {
@@ -2321,7 +2363,8 @@ bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
   return true;
 }
 
-std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
+std::unique_ptr<AMDGPUOperand>
+AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
   const auto &Tok = Parser.getTok();
   SMLoc StartLoc = Tok.getLoc();
   SMLoc EndLoc = Tok.getEndLoc();
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b52fcdc536382ad4bdeac6153d237f97c5d43b5e..bce539dfb6bcef945c17f5b7e67f1ca9b79336af 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -64,7 +64,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUPromoteAlloca.cpp
   AMDGPUPropagateAttributes.cpp
   AMDGPURegisterBankInfo.cpp
-  AMDGPURegisterInfo.cpp
   AMDGPURewriteOutArguments.cpp
   AMDGPUSubtarget.cpp
   AMDGPUTargetMachine.cpp
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 8945b576ae7836a13d07a03262e135f36e999891..f563e169ace8df2846d02cb4c3c7071c4b76855c 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -18,7 +18,6 @@
 
 #include "Disassembler/AMDGPUDisassembler.h"
 #include "AMDGPU.h"
-#include "AMDGPURegisterInfo.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIDefines.h"
 #include "TargetInfo/AMDGPUTargetInfo.h"
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
index f8ec3c36f01908fd274fbf36b09681c26b57886e..05c8ff53dd3ed0ff61a2ee7ef3ff0c9d4f354b51 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
@@ -13,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
-#include "AMDGPURegisterInfo.h"
 #include "MCTargetDesc/AMDGPUFixupKinds.h"
 #include "MCTargetDesc/AMDGPUMCCodeEmitter.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b6966e66c36be3e16244e8c1128faea39eb54dd0..55003521b8b25b826b19a2c7936609975198ee37 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1251,9 +1251,11 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
     // If we have an uniform constant load, it still requires using a slow
     // buffer instruction if unaligned.
     if (IsFast) {
+      // Accesses can really be issued as 1-byte aligned or 4-byte aligned, so
+      // 2-byte alignment is worse than 1 unless doing a 2-byte accesss.
       *IsFast = (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
                  AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ?
-        (Align % 4 == 0) : true;
+        Align >= 4 : Align != 2;
     }
 
     return true;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index bb685568f14c5090a9dbe029518178bf517f1d4b..2bb4d5979780e629bd8788da7ed98b6a1bc11e5d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -169,7 +169,8 @@ def WWM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;
 } // End let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC]
 
 def ENTER_WWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> {
-  let Defs = [EXEC];
+  let Uses = [EXEC];
+  let Defs = [EXEC, SCC];
   let hasSideEffects = 0;
   let mayLoad = 0;
   let mayStore = 0;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 685fccbaad6b470c681e07570f7f10cfbf1f0ea5..3b07fc1346e278fb09fa88c53621352611bc30f3 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -29,24 +29,8 @@
 
 using namespace llvm;
 
-static bool hasPressureSet(const int *PSets, unsigned PSetID) {
-  for (unsigned i = 0; PSets[i] != -1; ++i) {
-    if (PSets[i] == (int)PSetID)
-      return true;
-  }
-  return false;
-}
-
-void SIRegisterInfo::classifyPressureSet(unsigned PSetID, unsigned Reg,
-                                         BitVector &PressureSets) const {
-  for (MCRegUnitIterator U(Reg, this); U.isValid(); ++U) {
-    const int *PSets = getRegUnitPressureSets(*U);
-    if (hasPressureSet(PSets, PSetID)) {
-      PressureSets.set(PSetID);
-      break;
-    }
-  }
-}
+#define GET_REGINFO_TARGET_DESC
+#include "AMDGPUGenRegisterInfo.inc"
 
 static cl::opt<bool> EnableSpillSGPRToVGPR(
   "amdgpu-spill-sgpr-to-vgpr",
@@ -55,7 +39,7 @@ static cl::opt<bool> EnableSpillSGPRToVGPR(
   cl::init(true));
 
 SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) :
-  AMDGPURegisterInfo(),
+  AMDGPUGenRegisterInfo(0),
   ST(ST),
   SGPRPressureSets(getNumRegPressureSets()),
   VGPRPressureSets(getNumRegPressureSets()),
@@ -106,6 +90,83 @@ SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) :
          AGPRSetID < NumRegPressureSets);
 }
 
+void SIRegisterInfo::reserveRegisterTuples(BitVector &Reserved,
+                                           unsigned Reg) const {
+  MCRegAliasIterator R(Reg, this, true);
+
+  for (; R.isValid(); ++R)
+    Reserved.set(*R);
+}
+
+// Forced to be here by one .inc
+const MCPhysReg *SIRegisterInfo::getCalleeSavedRegs(
+  const MachineFunction *MF) const {
+  CallingConv::ID CC = MF->getFunction().getCallingConv();
+  switch (CC) {
+  case CallingConv::C:
+  case CallingConv::Fast:
+  case CallingConv::Cold:
+    return CSR_AMDGPU_HighRegs_SaveList;
+  default: {
+    // Dummy to not crash RegisterClassInfo.
+    static const MCPhysReg NoCalleeSavedReg = AMDGPU::NoRegister;
+    return &NoCalleeSavedReg;
+  }
+  }
+}
+
+const MCPhysReg *
+SIRegisterInfo::getCalleeSavedRegsViaCopy(const MachineFunction *MF) const {
+  return nullptr;
+}
+
+const uint32_t *SIRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
+                                                     CallingConv::ID CC) const {
+  switch (CC) {
+  case CallingConv::C:
+  case CallingConv::Fast:
+  case CallingConv::Cold:
+    return CSR_AMDGPU_HighRegs_RegMask;
+  default:
+    return nullptr;
+  }
+}
+
+Register SIRegisterInfo::getFrameRegister(const MachineFunction &MF) const {
+  const SIFrameLowering *TFI =
+      MF.getSubtarget<GCNSubtarget>().getFrameLowering();
+  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+  return TFI->hasFP(MF) ? FuncInfo->getFrameOffsetReg()
+                        : FuncInfo->getStackPtrOffsetReg();
+}
+
+const uint32_t *SIRegisterInfo::getAllVGPRRegMask() const {
+  return CSR_AMDGPU_AllVGPRs_RegMask;
+}
+
+const uint32_t *SIRegisterInfo::getAllAllocatableSRegMask() const {
+  return CSR_AMDGPU_AllAllocatableSRegs_RegMask;
+}
+
+static bool hasPressureSet(const int *PSets, unsigned PSetID) {
+  for (unsigned i = 0; PSets[i] != -1; ++i) {
+    if (PSets[i] == (int)PSetID)
+      return true;
+  }
+  return false;
+}
+
+void SIRegisterInfo::classifyPressureSet(unsigned PSetID, unsigned Reg,
+                                         BitVector &PressureSets) const {
+  for (MCRegUnitIterator U(Reg, this); U.isValid(); ++U) {
+    const int *PSets = getRegUnitPressureSets(*U);
+    if (hasPressureSet(PSets, PSetID)) {
+      PressureSets.set(PSetID);
+      break;
+    }
+  }
+}
+
 // FIXME: TableGen should generate something to make this manageable for all
 // register classes. At a minimum we could use the opposite of
 // composeSubRegIndices and go up from the base 32-bit subreg.
@@ -1789,7 +1850,7 @@ unsigned SIRegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
                                                        MF.getFunction());
   switch (RC->getID()) {
   default:
-    return AMDGPURegisterInfo::getRegPressureLimit(RC, MF);
+    return AMDGPUGenRegisterInfo::getRegPressureLimit(RC, MF);
   case AMDGPU::VGPR_32RegClassID:
     return std::min(ST.getMaxNumVGPRs(Occupancy), ST.getMaxNumVGPRs(MF));
   case AMDGPU::SGPR_32RegClassID:
@@ -1807,7 +1868,7 @@ unsigned SIRegisterInfo::getRegPressureSetLimit(const MachineFunction &MF,
     return getRegPressureLimit(&AMDGPU::SGPR_32RegClass,
                                const_cast<MachineFunction &>(MF));
 
-  return AMDGPURegisterInfo::getRegPressureSetLimit(MF, Idx);
+  return AMDGPUGenRegisterInfo::getRegPressureSetLimit(MF, Idx);
 }
 
 const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const {
@@ -1815,7 +1876,7 @@ const int *SIRegisterInfo::getRegUnitPressureSets(unsigned RegUnit) const {
 
   if (hasRegUnit(AMDGPU::M0, RegUnit))
     return Empty;
-  return AMDGPURegisterInfo::getRegUnitPressureSets(RegUnit);
+  return AMDGPUGenRegisterInfo::getRegUnitPressureSets(RegUnit);
 }
 
 unsigned SIRegisterInfo::getReturnAddressReg(const MachineFunction &MF) const {
@@ -1899,7 +1960,7 @@ SIRegisterInfo::getRegClass(unsigned RCID) const {
   case -1:
     return nullptr;
   default:
-    return AMDGPURegisterInfo::getRegClass(RCID);
+    return AMDGPUGenRegisterInfo::getRegClass(RCID);
   }
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index ac6d936154d07f969499c1446c7cd9f6361441cb..cf4cf47bf0fdf7c2fa47f14fcc897a4fe2b845fe 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -14,7 +14,9 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_SIREGISTERINFO_H
 #define LLVM_LIB_TARGET_AMDGPU_SIREGISTERINFO_H
 
-#include "AMDGPURegisterInfo.h"
+#define GET_REGINFO_HEADER
+#include "AMDGPUGenRegisterInfo.inc"
+
 #include "SIDefines.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 
@@ -25,7 +27,7 @@ class LiveIntervals;
 class MachineRegisterInfo;
 class SIMachineFunctionInfo;
 
-class SIRegisterInfo final : public AMDGPURegisterInfo {
+class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
 private:
   const GCNSubtarget &ST;
   unsigned SGPRSetID;
@@ -37,6 +39,8 @@ private:
   bool SpillSGPRToVGPR;
   bool isWave32;
 
+  void reserveRegisterTuples(BitVector &, unsigned Reg) const;
+
   void classifyPressureSet(unsigned PSetID, unsigned Reg,
                            BitVector &PressureSets) const;
 public:
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index 558e964ec3f0794cb58cf0908d85ca0eed8a119e..927cf6eafb522281bddfdc8e0ceea73d9e929754 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -6,6 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
+//  Subregister declarations
+//===----------------------------------------------------------------------===//
+
+let Namespace = "AMDGPU" in {
+foreach Index = 0-31 in {
+  def sub#Index : SubRegIndex<32, !shl(Index, 5)>;
+}
+}
+
 //===----------------------------------------------------------------------===//
 //  Helpers
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index bd82c8f6440035ce7f3664e46866ebf61fad8baf..f6ee9c07bd40fe12c8bd3518b34d00717b52a91e 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2223,7 +2223,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         isThisReturn = true;
       }
       const TargetOptions &Options = DAG.getTarget().Options;
-      if (Options.EnableDebugEntryValues)
+      if (Options.SupportsDebugEntryValues)
         CSInfo.emplace_back(VA.getLocReg(), i);
       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
     } else if (isByVal) {
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 84876eda33a6f58e33a50cb04211a4e855684bd8..63aa65267ef26680b12f6b4a41b88961b5bd0081 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -243,6 +243,9 @@ ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
     this->Options.NoTrapAfterNoreturn = true;
   }
 
+  // ARM supports the debug entry values.
+  setSupportsDebugEntryValues(true);
+
   initAsmInfo();
 }
 
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 8921eb7306bb4d51b0b44c8e0c33c4f2e3869dbe..e4f375c6f0431846bcde036de55411b2e72471bc 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -628,6 +628,8 @@ public:
 
   // Implementation of the MCTargetAsmParser interface:
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
   bool ParseDirective(AsmToken DirectiveID) override;
@@ -3885,6 +3887,14 @@ bool ARMAsmParser::ParseRegister(unsigned &RegNo,
   return (RegNo == (unsigned)-1);
 }
 
+OperandMatchResultTy ARMAsmParser::tryParseRegister(unsigned &RegNo,
+                                                    SMLoc &StartLoc,
+                                                    SMLoc &EndLoc) {
+  if (ParseRegister(RegNo, StartLoc, EndLoc))
+    return MatchOperand_NoMatch;
+  return MatchOperand_Success;
+}
+
 /// Try to parse a register name.  The token must be an Identifier when called,
 /// and if it is a register name the token is eaten and the register number is
 /// returned.  Otherwise return -1.
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index 151385de7850369b785be6bd7d94964c3400bbf4..9b8c437c053b45d2dec5bb1450ec155b5201748f 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -260,13 +260,18 @@ bool MVETailPredication::isTailPredicate(TripCountPattern &TCP) {
   // %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert,
   //                                  <4 x i32> undef,
   //                                  <4 x i32> zeroinitializer
-  // %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  // %induction = [add|or] <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
   // %pred = icmp ule <4 x i32> %induction, %broadcast.splat11
-
+  //
+  // Please note that the 'or' is equivalent to the 'and' here, this relies on
+  // BroadcastSplat being the IV which we know is a phi with 0 start and Lanes
+  // increment, which is all being checked below.
   Instruction *BroadcastSplat = nullptr;
   Constant *Const = nullptr;
   if (!match(TCP.Induction,
-             m_Add(m_Instruction(BroadcastSplat), m_Constant(Const))))
+             m_Add(m_Instruction(BroadcastSplat), m_Constant(Const))) &&
+      !match(TCP.Induction,
+             m_Or(m_Instruction(BroadcastSplat), m_Constant(Const))))
     return false;
 
   // Check that we're adding <0, 1, 2, 3...
diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
index fc34583ae5736af22f8b77772d4e45768d3ac780..9d0dee8cc293a5aee0a975ae082116a7e8f9f650 100644
--- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
+++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
@@ -53,6 +53,8 @@ class AVRAsmParser : public MCTargetAsmParser {
                                bool MatchingInlineAsm) override;
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
@@ -64,7 +66,7 @@ class AVRAsmParser : public MCTargetAsmParser {
   bool parseOperand(OperandVector &Operands);
   int parseRegisterName(unsigned (*matchFn)(StringRef));
   int parseRegisterName();
-  int parseRegister();
+  int parseRegister(bool RestoreOnFailure = false);
   bool tryParseRegisterOperand(OperandVector &Operands);
   bool tryParseExpression(OperandVector &Operands);
   bool tryParseRelocExpression(OperandVector &Operands);
@@ -359,19 +361,25 @@ int AVRAsmParser::parseRegisterName() {
   return RegNum;
 }
 
-int AVRAsmParser::parseRegister() {
+int AVRAsmParser::parseRegister(bool RestoreOnFailure) {
   int RegNum = AVR::NoRegister;
 
   if (Parser.getTok().is(AsmToken::Identifier)) {
     // Check for register pair syntax
     if (Parser.getLexer().peekTok().is(AsmToken::Colon)) {
+      AsmToken HighTok = Parser.getTok();
       Parser.Lex();
+      AsmToken ColonTok = Parser.getTok();
       Parser.Lex(); // Eat high (odd) register and colon
 
       if (Parser.getTok().is(AsmToken::Identifier)) {
         // Convert lower (even) register to DREG
         RegNum = toDREG(parseRegisterName());
       }
+      if (RegNum == AVR::NoRegister && RestoreOnFailure) {
+        getLexer().UnLex(std::move(ColonTok));
+        getLexer().UnLex(std::move(HighTok));
+      }
     } else {
       RegNum = parseRegisterName();
     }
@@ -580,12 +588,24 @@ AVRAsmParser::parseMemriOperand(OperandVector &Operands) {
 bool AVRAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                  SMLoc &EndLoc) {
   StartLoc = Parser.getTok().getLoc();
-  RegNo = parseRegister();
+  RegNo = parseRegister(/*RestoreOnFailure=*/false);
   EndLoc = Parser.getTok().getLoc();
 
   return (RegNo == AVR::NoRegister);
 }
 
+OperandMatchResultTy AVRAsmParser::tryParseRegister(unsigned &RegNo,
+                                                    SMLoc &StartLoc,
+                                                    SMLoc &EndLoc) {
+  StartLoc = Parser.getTok().getLoc();
+  RegNo = parseRegister(/*RestoreOnFailure=*/true);
+  EndLoc = Parser.getTok().getLoc();
+
+  if (RegNo == AVR::NoRegister)
+    return MatchOperand_NoMatch;
+  return MatchOperand_Success;
+}
+
 void AVRAsmParser::eatComma() {
   if (getLexer().is(AsmToken::Comma)) {
     Parser.Lex();
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index 1f5d5025bc7bcbae552ca0a5230fc0ee9f084dbf..dbcb156a43fd859061ebd99f536aa3199bda3d8e 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -39,6 +39,8 @@ class BPFAsmParser : public MCTargetAsmParser {
                                bool MatchingInlineAsm) override;
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
@@ -322,6 +324,14 @@ bool BPFAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
 
 bool BPFAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                  SMLoc &EndLoc) {
+  if (tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success)
+    return Error(StartLoc, "invalid register name");
+  return false;
+}
+
+OperandMatchResultTy BPFAsmParser::tryParseRegister(unsigned &RegNo,
+                                                    SMLoc &StartLoc,
+                                                    SMLoc &EndLoc) {
   const AsmToken &Tok = getParser().getTok();
   StartLoc = Tok.getLoc();
   EndLoc = Tok.getEndLoc();
@@ -330,10 +340,10 @@ bool BPFAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
 
   if (!MatchRegisterName(Name)) {
     getParser().Lex(); // Eat identifier token.
-    return false;
+    return MatchOperand_Success;
   }
 
-  return Error(StartLoc, "invalid register name");
+  return MatchOperand_NoMatch;
 }
 
 OperandMatchResultTy
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index 56e0288f26c9fd96465e423af7c60f8ad7af14c2..799a60c6cc92b95a60e35c2d49258ef835b4d304 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -171,6 +171,38 @@ bool BPFTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) cons
   return false;
 }
 
+bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
+  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
+    return false;
+  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
+  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
+  return NumBits1 > NumBits2;
+}
+
+bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
+  if (!VT1.isInteger() || !VT2.isInteger())
+    return false;
+  unsigned NumBits1 = VT1.getSizeInBits();
+  unsigned NumBits2 = VT2.getSizeInBits();
+  return NumBits1 > NumBits2;
+}
+
+bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
+  if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
+    return false;
+  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
+  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
+  return NumBits1 == 32 && NumBits2 == 64;
+}
+
+bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
+  if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
+    return false;
+  unsigned NumBits1 = VT1.getSizeInBits();
+  unsigned NumBits2 = VT2.getSizeInBits();
+  return NumBits1 == 32 && NumBits2 == 64;
+}
+
 std::pair<unsigned, const TargetRegisterClass *>
 BPFTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
                                                 StringRef Constraint,
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
index 0e9ec1398aeb0ead5b5e6b6454b9125d899111ea..cc752dda87b0282f13d1bbee68997fdc51d8b0fa 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.h
+++ b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -122,6 +122,16 @@ private:
     return false;
   }
 
+  // isTruncateFree - Return true if it's free to truncate a value of
+  // type Ty1 to type Ty2. e.g. On BPF at alu32 mode, it's free to truncate
+  // a i64 value in register R1 to i32 by referencing its sub-register W1.
+  bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
+  bool isTruncateFree(EVT VT1, EVT VT2) const override;
+
+  // For 32bit ALU result zext to 64bit is free.
+  bool isZExtFree(Type *Ty1, Type *Ty2) const override;
+  bool isZExtFree(EVT VT1, EVT VT2) const override;
+
   unsigned EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB, unsigned Reg,
                          bool isSigned) const;
 
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index 5976a811c978af409b69e9145fb47b3f76ecbc8a..ae19b138b0c7f0c7ea27e1cdf43f02c220eddd8e 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -116,6 +116,8 @@ class HexagonAsmParser : public MCTargetAsmParser {
   bool ParseDirectiveFalign(unsigned Size, SMLoc L);
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
   bool ParseDirectiveSubsection(SMLoc L);
   bool ParseDirectiveComm(bool IsLocal, SMLoc L);
   bool RegisterMatchesArch(unsigned MatchNum) const;
@@ -964,6 +966,12 @@ bool HexagonAsmParser::handleNoncontigiousRegister(bool Contigious,
 
 bool HexagonAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                      SMLoc &EndLoc) {
+  return tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success;
+}
+
+OperandMatchResultTy HexagonAsmParser::tryParseRegister(unsigned &RegNo,
+                                                        SMLoc &StartLoc,
+                                                        SMLoc &EndLoc) {
   MCAsmLexer &Lexer = getLexer();
   StartLoc = getLexer().getLoc();
   SmallVector<AsmToken, 5> Lookahead;
@@ -998,8 +1006,8 @@ bool HexagonAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
       RegNo = DotReg;
       EndLoc = Lexer.getLoc();
       if (handleNoncontigiousRegister(!NeededWorkaround, StartLoc))
-        return true;
-      return false;
+        return MatchOperand_NoMatch;
+      return MatchOperand_Success;
     } else {
       RegNo = DotReg;
       size_t First = RawString.find('.');
@@ -1007,28 +1015,26 @@ bool HexagonAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
       Lexer.UnLex(AsmToken(AsmToken::Identifier, DotString));
       EndLoc = Lexer.getLoc();
       if (handleNoncontigiousRegister(!NeededWorkaround, StartLoc))
-        return true;
-      return false;
+        return MatchOperand_NoMatch;
+      return MatchOperand_Success;
     }
   }
   std::pair<StringRef, StringRef> ColonSplit = StringRef(FullString).split(':');
   unsigned ColonReg = matchRegister(ColonSplit.first.lower());
   if (ColonReg != Hexagon::NoRegister && RegisterMatchesArch(DotReg)) {
     do {
-      Lexer.UnLex(Lookahead.back());
-      Lookahead.pop_back();
-    } while (!Lookahead.empty () && !Lexer.is(AsmToken::Colon));
+      Lexer.UnLex(Lookahead.pop_back_val());
+    } while (!Lookahead.empty() && !Lexer.is(AsmToken::Colon));
     RegNo = ColonReg;
     EndLoc = Lexer.getLoc();
     if (handleNoncontigiousRegister(!NeededWorkaround, StartLoc))
-      return true;
-    return false;
+      return MatchOperand_NoMatch;
+    return MatchOperand_Success;
   }
   while (!Lookahead.empty()) {
-    Lexer.UnLex(Lookahead.back());
-    Lookahead.pop_back();
+    Lexer.UnLex(Lookahead.pop_back_val());
   }
-  return true;
+  return MatchOperand_NoMatch;
 }
 
 bool HexagonAsmParser::implicitExpressionLocation(OperandVector &Operands) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
index 481869cb8f2c9e74a33e44bf075dcb4a9fc87ed2..8632250fa7f567b8f7b43d23c2803c3b7d4a0afb 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
@@ -1187,7 +1187,7 @@ void HexagonDAGToDAGISel::ppHoistZextI1(std::vector<SDNode*> &&Nodes) {
         Ops[i] = U->getOperand(i);
       EVT BVT = Ops[I1N].getValueType();
 
-      SDLoc dl(U);
+      const SDLoc &dl(U);
       SDValue C0 = DAG.getConstant(0, dl, BVT);
       SDValue C1 = DAG.getConstant(1, dl, BVT);
       SDValue If0, If1;
@@ -1205,8 +1205,15 @@ void HexagonDAGToDAGISel::ppHoistZextI1(std::vector<SDNode*> &&Nodes) {
         Ops[I1N] = C1;
         If1 = DAG.getNode(UseOpc, dl, UVT, Ops);
       }
-      SDValue Sel = DAG.getNode(ISD::SELECT, dl, UVT, OpI1, If1, If0);
-      DAG.ReplaceAllUsesWith(U, Sel.getNode());
+      // We're generating a SELECT way after legalization, so keep the types
+      // simple.
+      unsigned UW = UVT.getSizeInBits();
+      EVT SVT = (UW == 32 || UW == 64) ? MVT::getIntegerVT(UW) : UVT;
+      SDValue Sel = DAG.getNode(ISD::SELECT, dl, SVT, OpI1,
+                                DAG.getBitcast(SVT, If1),
+                                DAG.getBitcast(SVT, If0));
+      SDValue Ret = DAG.getBitcast(UVT, Sel);
+      DAG.ReplaceAllUsesWith(U, Ret.getNode());
     }
   }
 }
diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index 8b8504978c755834e7a9db416e487c6051e90530..9028f4ad93d9820985104c433e01e046fa5ba8ae 100644
--- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -47,7 +47,7 @@ struct LanaiOperand;
 
 class LanaiAsmParser : public MCTargetAsmParser {
   // Parse operands
-  std::unique_ptr<LanaiOperand> parseRegister();
+  std::unique_ptr<LanaiOperand> parseRegister(bool RestoreOnFailure = false);
 
   std::unique_ptr<LanaiOperand> parseImmediate();
 
@@ -67,6 +67,8 @@ class LanaiAsmParser : public MCTargetAsmParser {
                         SMLoc NameLoc, OperandVector &Operands) override;
 
   bool ParseRegister(unsigned &RegNum, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool MatchAndEmitInstruction(SMLoc IdLoc, unsigned &Opcode,
                                OperandVector &Operands, MCStreamer &Out,
@@ -687,21 +689,30 @@ bool LanaiAsmParser::MatchAndEmitInstruction(SMLoc IdLoc, unsigned &Opcode,
 // backwards compatible with GCC and the different ways inline assembly is
 // handled.
 // TODO: see if there isn't a better way to do this.
-std::unique_ptr<LanaiOperand> LanaiAsmParser::parseRegister() {
+std::unique_ptr<LanaiOperand>
+LanaiAsmParser::parseRegister(bool RestoreOnFailure) {
   SMLoc Start = Parser.getTok().getLoc();
   SMLoc End = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+  Optional<AsmToken> PercentTok;
 
   unsigned RegNum;
   // Eat the '%'.
-  if (Lexer.getKind() == AsmToken::Percent)
+  if (Lexer.getKind() == AsmToken::Percent) {
+    PercentTok = Parser.getTok();
     Parser.Lex();
+  }
   if (Lexer.getKind() == AsmToken::Identifier) {
     RegNum = MatchRegisterName(Lexer.getTok().getIdentifier());
-    if (RegNum == 0)
+    if (RegNum == 0) {
+      if (PercentTok.hasValue() && RestoreOnFailure)
+        Lexer.UnLex(PercentTok.getValue());
       return nullptr;
+    }
     Parser.Lex(); // Eat identifier token
     return LanaiOperand::createReg(RegNum, Start, End);
   }
+  if (PercentTok.hasValue() && RestoreOnFailure)
+    Lexer.UnLex(PercentTok.getValue());
   return nullptr;
 }
 
@@ -710,12 +721,25 @@ bool LanaiAsmParser::ParseRegister(unsigned &RegNum, SMLoc &StartLoc,
   const AsmToken &Tok = getParser().getTok();
   StartLoc = Tok.getLoc();
   EndLoc = Tok.getEndLoc();
-  std::unique_ptr<LanaiOperand> Op = parseRegister();
+  std::unique_ptr<LanaiOperand> Op = parseRegister(/*RestoreOnFailure=*/false);
   if (Op != nullptr)
     RegNum = Op->getReg();
   return (Op == nullptr);
 }
 
+OperandMatchResultTy LanaiAsmParser::tryParseRegister(unsigned &RegNum,
+                                                      SMLoc &StartLoc,
+                                                      SMLoc &EndLoc) {
+  const AsmToken &Tok = getParser().getTok();
+  StartLoc = Tok.getLoc();
+  EndLoc = Tok.getEndLoc();
+  std::unique_ptr<LanaiOperand> Op = parseRegister(/*RestoreOnFailure=*/true);
+  if (Op == nullptr)
+    return MatchOperand_NoMatch;
+  RegNum = Op->getReg();
+  return MatchOperand_Success;
+}
+
 std::unique_ptr<LanaiOperand> LanaiAsmParser::parseIdentifier() {
   SMLoc Start = Parser.getTok().getLoc();
   SMLoc End = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
index 0995e80a0a0972c7a2a1b43b1479aabf655f00ed..daa1fb4b52cfc3f25d721c172592219e4d795033 100644
--- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
+++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
@@ -46,6 +46,8 @@ class MSP430AsmParser : public MCTargetAsmParser {
                                bool MatchingInlineAsm) override;
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
@@ -288,13 +290,28 @@ static unsigned MatchRegisterAltName(StringRef Name);
 
 bool MSP430AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                     SMLoc &EndLoc) {
+  switch (tryParseRegister(RegNo, StartLoc, EndLoc)) {
+  case MatchOperand_ParseFail:
+    return Error(StartLoc, "invalid register name");
+  case MatchOperand_Success:
+    return false;
+  case MatchOperand_NoMatch:
+    return true;
+  }
+
+  llvm_unreachable("unknown match result type");
+}
+
+OperandMatchResultTy MSP430AsmParser::tryParseRegister(unsigned &RegNo,
+                                                       SMLoc &StartLoc,
+                                                       SMLoc &EndLoc) {
   if (getLexer().getKind() == AsmToken::Identifier) {
     auto Name = getLexer().getTok().getIdentifier().lower();
     RegNo = MatchRegisterName(Name);
     if (RegNo == MSP430::NoRegister) {
       RegNo = MatchRegisterAltName(Name);
       if (RegNo == MSP430::NoRegister)
-        return true;
+        return MatchOperand_NoMatch;
     }
 
     AsmToken const &T = getParser().getTok();
@@ -302,10 +319,10 @@ bool MSP430AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
     EndLoc = T.getEndLoc();
     getLexer().Lex(); // eat register token
 
-    return false;
+    return MatchOperand_Success;
   }
 
-  return Error(StartLoc, "invalid register name");
+  return MatchOperand_ParseFail;
 }
 
 bool MSP430AsmParser::parseJccInstruction(ParseInstructionInfo &Info,
diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index e467ed36938bffce35b28a34c3d6d7a2baf16c44..21b8d503741eb011ad7dc8d0c30b768619cea0fc 100644
--- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -179,6 +179,8 @@ class MipsAsmParser : public MCTargetAsmParser {
 
   /// Parse a register as used in CFI directives
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool parseParenSuffix(StringRef Name, OperandVector &Operands);
 
@@ -6202,6 +6204,12 @@ bool MipsAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
 
 bool MipsAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                   SMLoc &EndLoc) {
+  return tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success;
+}
+
+OperandMatchResultTy MipsAsmParser::tryParseRegister(unsigned &RegNo,
+                                                     SMLoc &StartLoc,
+                                                     SMLoc &EndLoc) {
   SmallVector<std::unique_ptr<MCParsedAsmOperand>, 1> Operands;
   OperandMatchResultTy ResTy = parseAnyRegister(Operands);
   if (ResTy == MatchOperand_Success) {
@@ -6219,11 +6227,12 @@ bool MipsAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
       RegNo = isGP64bit() ? Operand.getGPR64Reg() : Operand.getGPR32Reg();
     }
 
-    return (RegNo == (unsigned)-1);
+    return (RegNo == (unsigned)-1) ? MatchOperand_NoMatch
+                                   : MatchOperand_Success;
   }
 
   assert(Operands.size() == 0);
-  return (RegNo == (unsigned)-1);
+  return (RegNo == (unsigned)-1) ? MatchOperand_NoMatch : MatchOperand_Success;
 }
 
 bool MipsAsmParser::parseMemOffset(const MCExpr *&Res, bool isParenExpr) {
diff --git a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
index a0b68ed936e54b72053c4f004a6f43116e4f72ae..3ddba25fcef619d52ec268cdcfe83f2414104bee 100644
--- a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
@@ -860,6 +860,7 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) {
       const SDValue &Constant = Node->getOperand(3);
 
       assert(Chain.getValueType() == MVT::Other);
+      (void)Intrinsic;
       assert(Intrinsic.getOpcode() == ISD::TargetConstant &&
              Constant.getOpcode() == ISD::Constant &&
              "Invalid instruction operand.");
@@ -931,6 +932,7 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) {
       const SDValue &Constant = Node->getOperand(4);
 
       assert(Chain.getValueType() == MVT::Other);
+      (void)Intrinsic;
       assert(Intrinsic.getOpcode() == ISD::TargetConstant &&
              Constant.getOpcode() == ISD::Constant &&
              "Invalid instruction operand.");
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index ed98852088e7de8c02d292f46420a83eeadbcee8..7859b13ecfaa82b780eb4c254bdab6c9a8c07d61 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -109,6 +109,8 @@ class PPCAsmParser : public MCTargetAsmParser {
   bool MatchRegisterName(unsigned &RegNo, int64_t &IntVal);
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   const MCExpr *ExtractModifierFromExpr(const MCExpr *E,
                                         PPCMCExpr::VariantKind &Variant);
@@ -1221,14 +1223,22 @@ bool PPCAsmParser::MatchRegisterName(unsigned &RegNo, int64_t &IntVal) {
 
 bool PPCAsmParser::
 ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
+  if (tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success)
+    return TokError("invalid register name");
+  return false;
+}
+
+OperandMatchResultTy PPCAsmParser::tryParseRegister(unsigned &RegNo,
+                                                    SMLoc &StartLoc,
+                                                    SMLoc &EndLoc) {
   const AsmToken &Tok = getParser().getTok();
   StartLoc = Tok.getLoc();
   EndLoc = Tok.getEndLoc();
   RegNo = 0;
   int64_t IntVal;
   if (MatchRegisterName(RegNo, IntVal))
-    return TokError("invalid register name");
-  return false;
+    return MatchOperand_NoMatch;
+  return MatchOperand_Success;
 }
 
 /// Extract \code @l/@ha \endcode modifier from expression.  Recursively scan
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index cde40e800e72235b87212f6ba8c59f5a82d50cf6..36acbc22e2141b2360a6d8dd097310232345778c 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -76,6 +76,8 @@ class RISCVAsmParser : public MCTargetAsmParser {
                                bool MatchingInlineAsm) override;
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
@@ -1201,17 +1203,25 @@ static bool matchRegisterNameHelper(bool IsRV32E, Register &RegNo,
 
 bool RISCVAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                    SMLoc &EndLoc) {
+  if (tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success)
+    return Error(StartLoc, "invalid register name");
+  return false;
+}
+
+OperandMatchResultTy RISCVAsmParser::tryParseRegister(unsigned &RegNo,
+                                                      SMLoc &StartLoc,
+                                                      SMLoc &EndLoc) {
   const AsmToken &Tok = getParser().getTok();
   StartLoc = Tok.getLoc();
   EndLoc = Tok.getEndLoc();
   RegNo = 0;
   StringRef Name = getLexer().getTok().getIdentifier();
 
-  if (matchRegisterNameHelper(isRV32E(), (Register&)RegNo, Name))
-    return Error(StartLoc, "invalid register name");
+  if (matchRegisterNameHelper(isRV32E(), (Register &)RegNo, Name))
+    return MatchOperand_NoMatch;
 
   getParser().Lex(); // Eat identifier token.
-  return false;
+  return MatchOperand_Success;
 }
 
 OperandMatchResultTy RISCVAsmParser::parseRegister(OperandVector &Operands,
diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td
index f1d2b9e260a7e854dd7a79f2e2dd4337fef8d5e4..70214ed840e6ab41e0ce46d3f18cdd1d669352f9 100644
--- a/llvm/lib/Target/RISCV/RISCV.td
+++ b/llvm/lib/Target/RISCV/RISCV.td
@@ -90,6 +90,9 @@ foreach i = {1-31} in
         SubtargetFeature<"reserve-x"#i, "UserReservedRegister[RISCV::X"#i#"]",
                          "true", "Reserve X"#i>;
 
+def FeatureSaveRestore : SubtargetFeature<"save-restore", "EnableSaveRestore",
+                                          "true", "Enable save/restore.">;
+
 //===----------------------------------------------------------------------===//
 // Named operands for CSR instructions.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 8551bd781990496de52cfde360a6749f0a78b9eb..c85a1f1cba25f32c71f7c30d20d3be1cf87f78d5 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -26,6 +26,100 @@
 
 using namespace llvm;
 
+// Get the ID of the libcall used for spilling and restoring callee saved
+// registers. The ID is representative of the number of registers saved or
+// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
+// single register.
+static int getLibCallID(const MachineFunction &MF,
+                        const std::vector<CalleeSavedInfo> &CSI) {
+  const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
+
+  if (CSI.empty() || !RVFI->useSaveRestoreLibCalls())
+    return -1;
+
+  Register MaxReg = RISCV::NoRegister;
+  for (auto &CS : CSI)
+    // RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to
+    // registers which can be saved by libcall.
+    if (CS.getFrameIdx() < 0)
+      MaxReg = std::max(MaxReg.id(), CS.getReg());
+
+  if (MaxReg == RISCV::NoRegister)
+    return -1;
+
+  switch (MaxReg) {
+  default:
+    llvm_unreachable("Something has gone wrong!");
+  case /*s11*/ RISCV::X27: return 12;
+  case /*s10*/ RISCV::X26: return 11;
+  case /*s9*/  RISCV::X25: return 10;
+  case /*s8*/  RISCV::X24: return 9;
+  case /*s7*/  RISCV::X23: return 8;
+  case /*s6*/  RISCV::X22: return 7;
+  case /*s5*/  RISCV::X21: return 6;
+  case /*s4*/  RISCV::X20: return 5;
+  case /*s3*/  RISCV::X19: return 4;
+  case /*s2*/  RISCV::X18: return 3;
+  case /*s1*/  RISCV::X9:  return 2;
+  case /*s0*/  RISCV::X8:  return 1;
+  case /*ra*/  RISCV::X1:  return 0;
+  }
+}
+
+// Get the name of the libcall used for spilling callee saved registers.
+// If this function will not use save/restore libcalls, then return a nullptr.
+static const char *
+getSpillLibCallName(const MachineFunction &MF,
+                    const std::vector<CalleeSavedInfo> &CSI) {
+  static const char *const SpillLibCalls[] = {
+    "__riscv_save_0",
+    "__riscv_save_1",
+    "__riscv_save_2",
+    "__riscv_save_3",
+    "__riscv_save_4",
+    "__riscv_save_5",
+    "__riscv_save_6",
+    "__riscv_save_7",
+    "__riscv_save_8",
+    "__riscv_save_9",
+    "__riscv_save_10",
+    "__riscv_save_11",
+    "__riscv_save_12"
+  };
+
+  int LibCallID = getLibCallID(MF, CSI);
+  if (LibCallID == -1)
+    return nullptr;
+  return SpillLibCalls[LibCallID];
+}
+
+// Get the name of the libcall used for restoring callee saved registers.
+// If this function will not use save/restore libcalls, then return a nullptr.
+static const char *
+getRestoreLibCallName(const MachineFunction &MF,
+                      const std::vector<CalleeSavedInfo> &CSI) {
+  static const char *const RestoreLibCalls[] = {
+    "__riscv_restore_0",
+    "__riscv_restore_1",
+    "__riscv_restore_2",
+    "__riscv_restore_3",
+    "__riscv_restore_4",
+    "__riscv_restore_5",
+    "__riscv_restore_6",
+    "__riscv_restore_7",
+    "__riscv_restore_8",
+    "__riscv_restore_9",
+    "__riscv_restore_10",
+    "__riscv_restore_11",
+    "__riscv_restore_12"
+  };
+
+  int LibCallID = getLibCallID(MF, CSI);
+  if (LibCallID == -1)
+    return nullptr;
+  return RestoreLibCalls[LibCallID];
+}
+
 bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const {
   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
@@ -179,6 +273,17 @@ static Register getBPReg(const RISCVSubtarget &STI) { return RISCV::X9; }
 // Returns the register used to hold the stack pointer.
 static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; }
 
+static SmallVector<CalleeSavedInfo, 8>
+getNonLibcallCSI(const std::vector<CalleeSavedInfo> &CSI) {
+  SmallVector<CalleeSavedInfo, 8> NonLibcallCSI;
+
+  for (auto &CS : CSI)
+    if (CS.getFrameIdx() >= 0)
+      NonLibcallCSI.push_back(CS);
+
+  return NonLibcallCSI;
+}
+
 void RISCVFrameLowering::alignSP(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MBBI,
                                  const DebugLoc &DL, int64_t Alignment) const {
@@ -236,6 +341,11 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   Register SPReg = getSPReg(STI);
   Register BPReg = RISCVABI::getBPReg();
 
+  // Since spillCalleeSavedRegisters may have inserted a libcall, skip past
+  // any instructions marked as FrameSetup
+  while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
+    ++MBBI;
+
   // Debug location must be unknown since the first debug location is used
   // to determine the end of the prologue.
   DebugLoc DL;
@@ -243,12 +353,38 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   // Determine the correct frame layout
   determineFrameLayout(MF);
 
+  // If libcalls are used to spill and restore callee-saved registers, the frame
+  // has two sections; the opaque section managed by the libcalls, and the
+  // section managed by MachineFrameInfo which can also hold callee saved
+  // registers in fixed stack slots, both of which have negative frame indices.
+  // This gets even more complicated when incoming arguments are passed via the
+  // stack, as these too have negative frame indices. An example is detailed
+  // below:
+  //
+  //  | incoming arg | <- FI[-3]
+  //  | libcallspill |
+  //  | calleespill  | <- FI[-2]
+  //  | calleespill  | <- FI[-1]
+  //  | this_frame   | <- FI[0]
+  //
+  // For negative frame indices, the offset from the frame pointer will differ
+  // depending on which of these groups the frame index applies to.
+  // The following calculates the correct offset knowing the number of callee
+  // saved registers spilt by the two methods.
+  if (int LibCallRegs = getLibCallID(MF, MFI.getCalleeSavedInfo()) + 1) {
+    // Calculate the size of the frame managed by the libcall. The libcalls are
+    // implemented such that the stack will always be 16 byte aligned.
+    unsigned LibCallFrameSize = alignTo((STI.getXLen() / 8) * LibCallRegs, 16);
+    RVFI->setLibCallStackSize(LibCallFrameSize);
+  }
+
   // FIXME (note copied from Lanai): This appears to be overallocating.  Needs
   // investigation. Get the number of bytes to allocate from the FrameInfo.
   uint64_t StackSize = MFI.getStackSize();
+  uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize();
 
   // Early exit if there is no need to allocate on the stack
-  if (StackSize == 0 && !MFI.adjustsStack())
+  if (RealStackSize == 0 && !MFI.adjustsStack())
     return;
 
   // If the stack pointer has been marked as reserved, then produce an error if
@@ -259,19 +395,23 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
 
   uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
   // Split the SP adjustment to reduce the offsets of callee saved spill.
-  if (FirstSPAdjustAmount)
+  if (FirstSPAdjustAmount) {
     StackSize = FirstSPAdjustAmount;
+    RealStackSize = FirstSPAdjustAmount;
+  }
 
   // Allocate space on the stack if necessary.
   adjustReg(MBB, MBBI, DL, SPReg, SPReg, -StackSize, MachineInstr::FrameSetup);
 
-  // Emit ".cfi_def_cfa_offset StackSize"
+  // Emit ".cfi_def_cfa_offset RealStackSize"
   unsigned CFIIndex = MF.addFrameInst(
-      MCCFIInstruction::createDefCfaOffset(nullptr, -StackSize));
+      MCCFIInstruction::createDefCfaOffset(nullptr, -RealStackSize));
   BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
       .addCFIIndex(CFIIndex)
       .setMIFlag(MachineInstr::FrameSetup);
 
+  const auto &CSI = MFI.getCalleeSavedInfo();
+
   // The frame pointer is callee-saved, and code has been generated for us to
   // save it to the stack. We need to skip over the storing of callee-saved
   // registers as the frame pointer must be modified after it has been saved
@@ -280,8 +420,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   // register.
   // EPI registers break this assumption but they are to be handled after we
   // adjust the FP.
-  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
-  int InsnToSkip = CSI.size();
+  int InsnToSkip = getNonLibcallCSI(CSI).size();
   for (auto &CS : CSI) {
     if (RISCV::VRRegClass.contains(CS.getReg()))
       InsnToSkip--;
@@ -291,7 +430,15 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
   // Iterate over list of callee-saved registers and emit .cfi_offset
   // directives.
   for (const auto &Entry : CSI) {
-    int64_t Offset = MFI.getObjectOffset(Entry.getFrameIdx());
+    int FrameIdx = Entry.getFrameIdx();
+    int64_t Offset;
+    // Offsets for objects with fixed locations (IE: those saved by libcall) are
+    // simply calculated from the frame index.
+    if (FrameIdx < 0)
+      Offset = FrameIdx * (int64_t) STI.getXLen() / 8;
+    else
+      Offset = MFI.getObjectOffset(Entry.getFrameIdx()) -
+               RVFI->getLibCallStackSize();
     Register Reg = Entry.getReg();
     // We don't have sensible DWARF for VRs yet
     if (RISCV::VRRegClass.contains(Reg))
@@ -310,7 +457,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
           MF.getFunction(), "Frame pointer required, but has been reserved."});
 
     adjustReg(MBB, MBBI, DL, FPReg, SPReg,
-              StackSize - RVFI->getVarArgsSaveSize(), MachineInstr::FrameSetup);
+              RealStackSize - RVFI->getVarArgsSaveSize(),
+              MachineInstr::FrameSetup);
 
     // Emit ".cfi_def_cfa $fp, -RVFI->getVarArgsSaveSize()"
     unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createDefCfa(
@@ -443,22 +591,33 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
     // last instruction.
     if (!MBBI->isTerminator())
       MBBI = std::next(MBBI);
+
+    // If callee-saved registers are saved via libcall, place stack adjustment
+    // before this call.
+    while (MBBI != MBB.begin() &&
+           std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy))
+      --MBBI;
   }
 
+  const auto &CSI = getNonLibcallCSI(MFI.getCalleeSavedInfo());
+
   // Skip to before the restores of callee-saved registers
   // FIXME: assumes exactly one instruction is used to restore each
   // callee-saved register.
-  // Ignore the VRs as we did in the prologue.
-  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
-  int InsnToSkip = CSI.size();
-  for (auto &CS : CSI) {
-    if (RISCV::VRRegClass.contains(CS.getReg()))
-      InsnToSkip--;
+  auto LastFrameDestroy = MBBI;
+  if (!CSI.empty()) {
+    // Ignore the VRs as we did in the prologue.
+    int InsnToSkip = CSI.size();
+    for (auto &CS : CSI) {
+      if (RISCV::VRRegClass.contains(CS.getReg()))
+        InsnToSkip--;
+    }
+    LastFrameDestroy = std::prev(MBBI, InsnToSkip);
   }
-  auto LastFrameDestroy = std::prev(MBBI, InsnToSkip);
 
   uint64_t StackSize = MFI.getStackSize();
-  uint64_t FPOffset = StackSize - RVFI->getVarArgsSaveSize();
+  uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize();
+  uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
 
   // Restore the stack pointer using the value of the frame pointer. Only
   // necessary if the stack pointer was modified, meaning the stack size is
@@ -500,7 +659,7 @@ int RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF,
   // (negative offset) unless the offset from FP is not known at compile time,
   // as it happens when we have to align the stack or we have variably sized
   // data.
-  const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
+  const auto &CSI = MFI.getCalleeSavedInfo();
   int MinCSFI = 0;
   int MaxCSFI = -1;
 
@@ -521,7 +680,7 @@ int RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF,
     if (FirstSPAdjustAmount)
       Offset += FirstSPAdjustAmount;
     else
-      Offset += MF.getFrameInfo().getStackSize();
+      Offset += MFI.getStackSize();
   } else if (RI->needsStackRealignment(MF) && !MFI.isFixedObjectIndex(FI)) {
     // If the stack was realigned, the frame pointer is set in order to allow
     // SP to be restored, so we need another base register to record the stack
@@ -530,13 +689,20 @@ int RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF,
       FrameReg = RISCVABI::getBPReg();
     else
       FrameReg = RISCV::X2;
-    Offset += MF.getFrameInfo().getStackSize();
+    Offset += MFI.getStackSize();
+    if (FI < 0)
+      Offset += RVFI->getLibCallStackSize();
   } else {
     FrameReg = RI->getFrameRegister(MF);
-    if (hasFP(MF))
+    if (hasFP(MF)) {
       Offset += RVFI->getVarArgsSaveSize();
-    else
-      Offset += MF.getFrameInfo().getStackSize();
+      if (FI >= 0)
+        Offset -= RVFI->getLibCallStackSize();
+    } else {
+      Offset += MFI.getStackSize();
+      if (FI < 0)
+        Offset += RVFI->getLibCallStackSize();
+    }
   }
   return Offset;
 }
@@ -681,16 +847,18 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(
 //   add     sp,sp,-64
 uint64_t
 RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
+  const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
   uint64_t StackSize = MFI.getStackSize();
   uint64_t StackAlign = getStackAlignment();
 
-  // FIXME: Disable SplitSPAdjust if save-restore libcall enabled when the patch
-  //        landing. The callee saved registers will be pushed by the
-  //        save-restore libcalls, so we don't have to split the SP adjustment
-  //        in this case.
-  //
+  // Disable SplitSPAdjust if save-restore libcall used. The callee saved
+  // registers will be pushed by the save-restore libcalls, so we don't have to
+  // split the SP adjustment in this case.
+  if (RVFI->getLibCallStackSize())
+    return 0;
+
   // Return the FirstSPAdjustAmount if the StackSize can not fit in signed
   // 12-bit and there exists a callee saved register need to be pushed.
   if (!isInt<12>(StackSize) && (CSI.size() > 0)) {
@@ -704,3 +872,124 @@ RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
   }
   return 0;
 }
+
+bool RISCVFrameLowering::spillCalleeSavedRegisters(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+    ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
+  if (CSI.empty())
+    return true;
+
+  MachineFunction *MF = MBB.getParent();
+  const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
+  DebugLoc DL;
+  if (MI != MBB.end() && !MI->isDebugInstr())
+    DL = MI->getDebugLoc();
+
+  const char *SpillLibCall = getSpillLibCallName(*MF, CSI);
+  if (SpillLibCall) {
+    // Add spill libcall via non-callee-saved register t0.
+    BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5)
+        .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL)
+        .setMIFlag(MachineInstr::FrameSetup);
+
+    // Add registers spilled in libcall as liveins.
+    for (auto &CS : CSI)
+      MBB.addLiveIn(CS.getReg());
+  }
+
+  // Manually spill values not spilled by libcall.
+  const auto &NonLibcallCSI = getNonLibcallCSI(CSI);
+  for (auto &CS : NonLibcallCSI) {
+    // Insert the spill to the stack frame.
+    Register Reg = CS.getReg();
+    const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+    TII.storeRegToStackSlot(MBB, MI, Reg, true, CS.getFrameIdx(), RC, TRI);
+  }
+
+  return true;
+}
+
+bool RISCVFrameLowering::restoreCalleeSavedRegisters(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+    std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI) const {
+  if (CSI.empty())
+    return true;
+
+  MachineFunction *MF = MBB.getParent();
+  const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
+  DebugLoc DL;
+  if (MI != MBB.end() && !MI->isDebugInstr())
+    DL = MI->getDebugLoc();
+
+  // Manually restore values not restored by libcall. Insert in reverse order.
+  // loadRegFromStackSlot can insert multiple instructions.
+  const auto &NonLibcallCSI = getNonLibcallCSI(CSI);
+  for (auto &CS : reverse(NonLibcallCSI)) {
+    Register Reg = CS.getReg();
+    const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+    TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, TRI);
+    assert(MI != MBB.begin() && "loadRegFromStackSlot didn't insert any code!");
+  }
+
+  const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI);
+  if (RestoreLibCall) {
+    // Add restore libcall via tail call.
+    MachineBasicBlock::iterator NewMI =
+        BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL))
+            .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL)
+            .setMIFlag(MachineInstr::FrameDestroy);
+
+    // Remove trailing returns, since the terminator is now a tail call to the
+    // restore function.
+    if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
+      NewMI->copyImplicitOps(*MF, *MI);
+      MI->eraseFromParent();
+    }
+  }
+
+  return true;
+}
+
+bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
+  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
+  const auto *RVFI = MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
+
+  if (!RVFI->useSaveRestoreLibCalls())
+    return true;
+
+  // Inserting a call to a __riscv_save libcall requires the use of the register
+  // t0 (X5) to hold the return address. Therefore if this register is already
+  // used we can't insert the call.
+
+  RegScavenger RS;
+  RS.enterBasicBlock(*TmpMBB);
+  return !RS.isRegUsed(RISCV::X5);
+}
+
+bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
+  MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
+  const auto *RVFI = MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
+
+  if (!RVFI->useSaveRestoreLibCalls())
+    return true;
+
+  // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
+  // This means if we still need to continue executing code within this function
+  // the restore cannot take place in this basic block.
+
+  if (MBB.succ_size() > 1)
+    return false;
+
+  MachineBasicBlock *SuccMBB =
+      MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
+
+  // Doing a tail call should be safe if there are no successors, because either
+  // we have a returning block or the end of the block is unreachable, so the
+  // restore will be eliminated regardless.
+  if (!SuccMBB)
+    return true;
+
+  // The successor can only contain a return, since we would effectively be
+  // replacing the successor with our own tail return at the end of our block.
+  return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
index 92609ce432b3c92be1acd9ce8f3173a5289b6122..f7a5b3abb2cab8552cf27bd74df30772b5b3ad78 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h
@@ -48,12 +48,24 @@ public:
   MachineBasicBlock::iterator
   eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator MI) const override;
+  bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
+                                 MachineBasicBlock::iterator MI,
+                                 ArrayRef<CalleeSavedInfo> CSI,
+                                 const TargetRegisterInfo *TRI) const override;
+  bool
+  restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+                              MachineBasicBlock::iterator MI,
+                              std::vector<CalleeSavedInfo> &CSI,
+                              const TargetRegisterInfo *TRI) const override;
 
   // Get the first stack adjustment amount for SplitSPAdjust.
   // Return 0 if we don't want to to split the SP adjustment in prologue and
   // epilogue.
   uint64_t getFirstSPAdjustAmount(const MachineFunction &MF) const;
 
+  bool canUseAsPrologue(const MachineBasicBlock &MBB) const override;
+  bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override;
+
 protected:
   const RISCVSubtarget &STI;
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index fb4c6206efc96fbaf8ef0394f82797d19e640e57..7b2a89aac5de117ce704f6ecb592c8e2c8f58458 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -144,6 +144,20 @@ def simm12 : Operand<XLenVT>, ImmLeaf<XLenVT, [{return isInt<12>(Imm);}]> {
   let OperandNamespace = "RISCVOp";
 }
 
+// A 12-bit signed immediate plus one where the imm range will be -2047~2048.
+def simm12_plus1 : Operand<XLenVT>, ImmLeaf<XLenVT,
+  [{return (isInt<12>(Imm) && Imm != -2048) || Imm == 2048;}]> {
+  let ParserMatchClass = SImmAsmOperand<12>;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeSImmOperand<12>";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return (isInt<12>(Imm) && Imm != -2048) || Imm == 2048;
+    return MCOp.isBareSymbolRef();
+  }];
+}
+
 // A 13-bit signed immediate where the least significant bit is zero.
 def simm13_lsb0 : Operand<OtherVT> {
   let ParserMatchClass = SImmAsmOperand<13, "Lsb0">;
@@ -316,6 +330,13 @@ def HHI20 : SDNodeXForm<imm, [{
                                    SDLoc(N), N->getValueType(0));
 }]>;
 
+
+// Return the negation of an immediate value.
+def NegImm : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(-N->getSExtValue(), SDLoc(N),
+                                   N->getValueType(0));
+}]>;
+
 //===----------------------------------------------------------------------===//
 // Instruction Formats
 //===----------------------------------------------------------------------===//
@@ -877,12 +898,12 @@ def : PatGprSimm12<setult, SLTIU>;
 // handled by a RISC-V instruction.
 def : Pat<(seteq GPR:$rs1, 0), (SLTIU GPR:$rs1, 1)>;
 def : Pat<(seteq GPR:$rs1, GPR:$rs2), (SLTIU (XOR GPR:$rs1, GPR:$rs2), 1)>;
-def : Pat<(seteq GPR:$rs1, simm12:$imm12),
-          (SLTIU (XORI GPR:$rs1, simm12:$imm12), 1)>;
+def : Pat<(seteq GPR:$rs1, simm12_plus1:$imm12),
+          (SLTIU (ADDI GPR:$rs1, (NegImm simm12_plus1:$imm12)), 1)>;
 def : Pat<(setne GPR:$rs1, 0), (SLTU X0, GPR:$rs1)>;
 def : Pat<(setne GPR:$rs1, GPR:$rs2), (SLTU X0, (XOR GPR:$rs1, GPR:$rs2))>;
-def : Pat<(setne GPR:$rs1, simm12:$imm12),
-          (SLTU X0, (XORI GPR:$rs1, simm12:$imm12))>;
+def : Pat<(setne GPR:$rs1, simm12_plus1:$imm12),
+          (SLTU X0, (ADDI GPR:$rs1, (NegImm simm12_plus1:$imm12)))>;
 def : Pat<(setugt GPR:$rs1, GPR:$rs2), (SLTU GPR:$rs2, GPR:$rs1)>;
 def : Pat<(setuge GPR:$rs1, GPR:$rs2), (XORI (SLTU GPR:$rs1, GPR:$rs2), 1)>;
 def : Pat<(setule GPR:$rs1, GPR:$rs2), (XORI (SLTU GPR:$rs2, GPR:$rs1), 1)>;
diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index edddb65d8fbd2337654227fd883c7ee537bdaa17..99e9376663d919165bb36c6a182502a6c73f5874 100644
--- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_LIB_TARGET_RISCV_RISCVMACHINEFUNCTIONINFO_H
 #define LLVM_LIB_TARGET_RISCV_RISCVMACHINEFUNCTIONINFO_H
 
+#include "RISCVSubtarget.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 
@@ -30,6 +31,8 @@ private:
   /// FrameIndex used for transferring values between 64-bit FPRs and a pair
   /// of 32-bit GPRs via the stack.
   int MoveF64FrameIndex = -1;
+  /// Size of any opaque stack adjustment due to save/restore libcalls.
+  unsigned LibCallStackSize = 0;
 
   // True if this function spills VR
   bool HasSpilledVR = false;
@@ -49,6 +52,16 @@ public:
     return MoveF64FrameIndex;
   }
 
+  unsigned getLibCallStackSize() const { return LibCallStackSize; }
+  void setLibCallStackSize(unsigned Size) { LibCallStackSize = Size; }
+
+  bool useSaveRestoreLibCalls() const {
+    // We cannot use fixed locations for the callee saved spill slots if the
+    // function uses a varargs save area.
+    return MF.getSubtarget<RISCVSubtarget>().enableSaveRestore() &&
+           VarArgsSaveSize == 0 && !MF.getFrameInfo().hasTailCall();
+  }
+
   bool hasSpilledVR() const { return HasSpilledVR; }
   void setHasSpilledVR(bool V = true) { HasSpilledVR = V; }
 };
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index ef6356f4426f2e5280f180ad04b4f075b2f28744..965e42e215e3296c7853522f5fee7164635e15f6 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -115,6 +115,39 @@ const uint32_t *RISCVRegisterInfo::getNoPreservedMask() const {
   return CSR_NoRegs_RegMask;
 }
 
+// Frame indexes representing locations of CSRs which are given a fixed location
+// by save/restore libcalls.
+static const std::map<unsigned, int> FixedCSRFIMap = {
+  {/*ra*/  RISCV::X1,   -1},
+  {/*s0*/  RISCV::X8,   -2},
+  {/*s1*/  RISCV::X9,   -3},
+  {/*s2*/  RISCV::X18,  -4},
+  {/*s3*/  RISCV::X19,  -5},
+  {/*s4*/  RISCV::X20,  -6},
+  {/*s5*/  RISCV::X21,  -7},
+  {/*s6*/  RISCV::X22,  -8},
+  {/*s7*/  RISCV::X23,  -9},
+  {/*s8*/  RISCV::X24,  -10},
+  {/*s9*/  RISCV::X25,  -11},
+  {/*s10*/ RISCV::X26,  -12},
+  {/*s11*/ RISCV::X27,  -13}
+};
+
+bool RISCVRegisterInfo::hasReservedSpillSlot(const MachineFunction &MF,
+                                             unsigned Reg,
+                                             int &FrameIdx) const {
+  const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
+  if (!RVFI->useSaveRestoreLibCalls())
+    return false;
+
+  auto FII = FixedCSRFIMap.find(Reg);
+  if (FII == FixedCSRFIMap.end())
+    return false;
+
+  FrameIdx = FII->second;
+  return true;
+}
+
 void RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                             int SPAdj, unsigned FIOperandNum,
                                             RegScavenger *RS) const {
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
index e3ff098708b1688f72de1dfbdf2cdfd4bea267c4..b4e0161ae5b6b6ec095b599d2e261ae1d49e77a9 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
@@ -37,6 +37,9 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
 
   const uint32_t *getNoPreservedMask() const override;
 
+  bool hasReservedSpillSlot(const MachineFunction &MF, unsigned Reg,
+                            int &FrameIdx) const override;
+
   void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
                            unsigned FIOperandNum,
                            RegScavenger *RS = nullptr) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 300f2c0dbce8f510ac5344730107f371e681d32b..c0e24f8b62d2ed50287ae9839eccfa1e30894f58 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -44,6 +44,7 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   bool IsRV32E = false;
   bool EnableLinkerRelax = false;
   bool EnableRVCHintInstrs = false;
+  bool EnableSaveRestore = false;
   unsigned XLen = 32;
   MVT XLenVT = MVT::i32;
   RISCVABI::ABI TargetABI = RISCVABI::ABI_Unknown;
@@ -93,6 +94,7 @@ public:
   bool isRV32E() const { return IsRV32E; }
   bool enableLinkerRelax() const { return EnableLinkerRelax; }
   bool enableRVCHintInstrs() const { return EnableRVCHintInstrs; }
+  bool enableSaveRestore() const { return EnableSaveRestore; }
   MVT getXLenVT() const { return XLenVT; }
   unsigned getXLen() const { return XLen; }
 
diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index 2d3137f38821759a81fba9b1590e937a7d084af0..dc75f5ccf973d45cd0d7862a2228cf7eaba42110 100644
--- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -68,6 +68,8 @@ class SparcAsmParser : public MCTargetAsmParser {
                                uint64_t &ErrorInfo,
                                bool MatchingInlineAsm) override;
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
   bool ParseDirective(AsmToken DirectiveID) override;
@@ -630,20 +632,29 @@ bool SparcAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
 
 bool SparcAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                    SMLoc &EndLoc) {
+  if (tryParseRegister(RegNo, StartLoc, EndLoc) != MatchOperand_Success)
+    return Error(StartLoc, "invalid register name");
+  return false;
+}
+
+OperandMatchResultTy SparcAsmParser::tryParseRegister(unsigned &RegNo,
+                                                      SMLoc &StartLoc,
+                                                      SMLoc &EndLoc) {
   const AsmToken &Tok = Parser.getTok();
   StartLoc = Tok.getLoc();
   EndLoc = Tok.getEndLoc();
   RegNo = 0;
   if (getLexer().getKind() != AsmToken::Percent)
-    return false;
+    return MatchOperand_Success;
   Parser.Lex();
   unsigned regKind = SparcOperand::rk_None;
   if (matchRegisterName(Tok, RegNo, regKind)) {
     Parser.Lex();
-    return false;
+    return MatchOperand_Success;
   }
 
-  return Error(StartLoc, "invalid register name");
+  getLexer().UnLex(Tok);
+  return MatchOperand_NoMatch;
 }
 
 static void applyMnemonicAliases(StringRef &Mnemonic,
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 607266d552a6478236694b3d0d010b1edde18daa..00fdb9e846d6a4ca4236485e4367e294dc058541 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -405,7 +405,7 @@ private:
     SMLoc StartLoc, EndLoc;
   };
 
-  bool parseRegister(Register &Reg);
+  bool parseRegister(Register &Reg, bool RestoreOnFailure = false);
 
   bool parseRegister(Register &Reg, RegisterGroup Group, const unsigned *Regs,
                      bool IsAddress = false);
@@ -449,6 +449,10 @@ public:
   // Override MCTargetAsmParser.
   bool ParseDirective(AsmToken DirectiveID) override;
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
+                     bool RestoreOnFailure);
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
   bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override;
   bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
@@ -691,27 +695,37 @@ void SystemZOperand::print(raw_ostream &OS) const {
 }
 
 // Parse one register of the form %<prefix><number>.
-bool SystemZAsmParser::parseRegister(Register &Reg) {
+bool SystemZAsmParser::parseRegister(Register &Reg, bool RestoreOnFailure) {
   Reg.StartLoc = Parser.getTok().getLoc();
 
   // Eat the % prefix.
   if (Parser.getTok().isNot(AsmToken::Percent))
     return Error(Parser.getTok().getLoc(), "register expected");
+  const AsmToken &PercentTok = Parser.getTok();
   Parser.Lex();
 
   // Expect a register name.
-  if (Parser.getTok().isNot(AsmToken::Identifier))
+  if (Parser.getTok().isNot(AsmToken::Identifier)) {
+    if (RestoreOnFailure)
+      getLexer().UnLex(PercentTok);
     return Error(Reg.StartLoc, "invalid register");
+  }
 
   // Check that there's a prefix.
   StringRef Name = Parser.getTok().getString();
-  if (Name.size() < 2)
+  if (Name.size() < 2) {
+    if (RestoreOnFailure)
+      getLexer().UnLex(PercentTok);
     return Error(Reg.StartLoc, "invalid register");
+  }
   char Prefix = Name[0];
 
   // Treat the rest of the register name as a register number.
-  if (Name.substr(1).getAsInteger(10, Reg.Num))
+  if (Name.substr(1).getAsInteger(10, Reg.Num)) {
+    if (RestoreOnFailure)
+      getLexer().UnLex(PercentTok);
     return Error(Reg.StartLoc, "invalid register");
+  }
 
   // Look for valid combinations of prefix and number.
   if (Prefix == 'r' && Reg.Num < 16)
@@ -724,8 +738,11 @@ bool SystemZAsmParser::parseRegister(Register &Reg) {
     Reg.Group = RegAR;
   else if (Prefix == 'c' && Reg.Num < 16)
     Reg.Group = RegCR;
-  else
+  else {
+    if (RestoreOnFailure)
+      getLexer().UnLex(PercentTok);
     return Error(Reg.StartLoc, "invalid register");
+  }
 
   Reg.EndLoc = Parser.getTok().getLoc();
   Parser.Lex();
@@ -1124,9 +1141,9 @@ bool SystemZAsmParser::ParseDirectiveInsn(SMLoc L) {
 }
 
 bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
-                                     SMLoc &EndLoc) {
+                                     SMLoc &EndLoc, bool RestoreOnFailure) {
   Register Reg;
-  if (parseRegister(Reg))
+  if (parseRegister(Reg, RestoreOnFailure))
     return true;
   if (Reg.Group == RegGR)
     RegNo = SystemZMC::GR64Regs[Reg.Num];
@@ -1143,6 +1160,25 @@ bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
   return false;
 }
 
+bool SystemZAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                     SMLoc &EndLoc) {
+  return ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
+}
+
+OperandMatchResultTy SystemZAsmParser::tryParseRegister(unsigned &RegNo,
+                                                        SMLoc &StartLoc,
+                                                        SMLoc &EndLoc) {
+  bool Result =
+      ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
+  bool PendingErrors = getParser().hasPendingError();
+  getParser().clearPendingErrors();
+  if (PendingErrors)
+    return MatchOperand_ParseFail;
+  if (Result)
+    return MatchOperand_NoMatch;
+  return MatchOperand_Success;
+}
+
 bool SystemZAsmParser::ParseInstruction(ParseInstructionInfo &Info,
                                         StringRef Name, SMLoc NameLoc,
                                         OperandVector &Operands) {
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 362bf19963d9e34a59fe618c709abb6f8f724cf0..8cb01709e9e79b76a30700c84433d958999485a3 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -6886,8 +6886,6 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
   for (MachineBasicBlock::iterator NextMIIt =
          std::next(MachineBasicBlock::iterator(MI));
        NextMIIt != MBB->end(); ++NextMIIt) {
-    if (NextMIIt->definesRegister(SystemZ::CC))
-      break;
     if (isSelectPseudo(*NextMIIt)) {
       assert(NextMIIt->getOperand(3).getImm() == CCValid &&
              "Bad CCValid operands since CC was not redefined.");
@@ -6898,6 +6896,9 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
       }
       break;
     }
+    if (NextMIIt->definesRegister(SystemZ::CC) ||
+        NextMIIt->usesCustomInsertionHook())
+      break;
     bool User = false;
     for (auto SelMI : Selects)
       if (NextMIIt->readsVirtualRegister(SelMI->getOperand(0).getReg())) {
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 92d0fc2419d4b718b00c3f29cdf80b548dd19631..e0411dae1727d92175a7b1038fcbc0bd44302b94 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -556,13 +556,28 @@ VETargetLowering::VETargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::VAEND, MVT::Other, Expand);
   /// } VAARG handling
 
-  // VE has no REM or DIVREM operations.
-  for (MVT IntVT : MVT::integer_valuetypes()) {
+  /// Int Ops {
+  for (MVT IntVT : {MVT::i32, MVT::i64}) {
+    // VE has no REM or DIVREM operations.
     setOperationAction(ISD::UREM, IntVT, Expand);
     setOperationAction(ISD::SREM, IntVT, Expand);
     setOperationAction(ISD::SDIVREM, IntVT, Expand);
     setOperationAction(ISD::UDIVREM, IntVT, Expand);
+
+    setOperationAction(ISD::CTTZ, IntVT, Expand);
+    setOperationAction(ISD::ROTL, IntVT, Expand);
+    setOperationAction(ISD::ROTR, IntVT, Expand);
+
+    // Use isel patterns for i32 and i64
+    setOperationAction(ISD::BSWAP, IntVT, Legal);
+    setOperationAction(ISD::CTLZ, IntVT, Legal);
+    setOperationAction(ISD::CTPOP, IntVT, Legal);
+
+    // Use isel patterns for i64, Promote i32
+    LegalizeAction Act = (IntVT == MVT::i32) ? Promote : Legal;
+    setOperationAction(ISD::BITREVERSE, IntVT, Act);
   }
+  /// } Int Ops
 
   /// Conversion {
   // VE doesn't have instructions for fp<->uint, so expand them by llvm
diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td
index a64fa5f5454724701092b4d5adf0f750400535de..f788b3232f7e6b8d822698ec4da3ea27f62bdf31 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.td
+++ b/llvm/lib/Target/VE/VEInstrInfo.td
@@ -533,6 +533,41 @@ multiclass RRCMOVm<string opcStr, bits<8>opc,
   }
 }
 
+// Multiclass for RR type instructions with only 2 operands
+//   Used by pcnt, brv
+let hasSideEffects = 0 in
+multiclass RRI2m<string opcStr, bits<8>opc, RegisterClass RC, ValueType Ty,
+                 Operand immOp2, SDPatternOperator OpNode=null_frag> {
+  def r : RR<
+    opc, (outs RC:$sx), (ins RC:$sz),
+    !strconcat(opcStr, " $sx, $sz"),
+    [(set Ty:$sx, (OpNode Ty:$sz))]> {
+    let cy = 1;
+    let cz = 1;
+  }
+  def i : RR<
+    opc, (outs RC:$sx), (ins RC:$sz),
+    !strconcat(opcStr, " $sx, $sz"),
+    [(set Ty:$sx, (OpNode Ty:$sz))]> {
+    let cy = 0;
+    let cz = 1;
+  }
+  def m0 : RR<
+    opc, (outs RC:$sx), (ins immOp2:$sz),
+    !strconcat(opcStr, " $sx, (${sz})0")> {
+    let cy = 1;
+    let cz = 0;
+    let sz{6} = 1;
+  }
+  def m1 : RR<
+    opc, (outs RC:$sx), (ins immOp2:$sz),
+    !strconcat(opcStr, " $sx, (${sz})1")> {
+    let cy = 1;
+    let cz = 0;
+  }
+}
+
+
 // Branch multiclass
 let isBranch = 1, isTerminator = 1, hasDelaySlot = 1 in
 multiclass BCRm<string opcStr, string opcStrAt, bits<8> opc,
@@ -760,6 +795,16 @@ let cx = 0 in {
   }
 }
 
+// Bits operations
+
+let cx = 0 in {
+  defm PCNT : RRI2m<"pcnt", 0x38, I64, i64, uimm6Op64, ctpop>;
+  defm BRV : RRI2m<"brv", 0x39, I64, i64, uimm6Op64, bitreverse>;
+  defm LDZ : RRI2m<"ldz", 0x67, I64, i64, uimm6Op64, ctlz>;
+  defm BSWP : RRIm<"bswp", 0x2B, I64, i64, simm7Op64, uimm6Op64>;
+}
+
+
 
 // 5.3.2.4 Shift Instructions
 
@@ -1426,6 +1471,19 @@ def : Pat<(f32 (bitconvert i32:$op)),
           (EXTRACT_SUBREG (SLLri (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
             $op, sub_i32), 32), sub_f32)>;
 
+// Bits operations pattern matchings.
+def : Pat<(i32 (ctpop i32:$src)),
+          (EXTRACT_SUBREG (PCNTr (ANDrm0 (INSERT_SUBREG
+            (i64 (IMPLICIT_DEF)), $src, sub_i32), 32)), sub_i32)>;
+def : Pat<(i32 (ctlz i32:$src)),
+          (EXTRACT_SUBREG (LDZr (SLLri (INSERT_SUBREG
+            (i64 (IMPLICIT_DEF)), $src, sub_i32), 32)), sub_i32)>;
+def : Pat<(i64 (bswap i64:$src)),
+          (BSWPri $src, 0)>;
+def : Pat<(i32 (bswap i32:$src)),
+          (EXTRACT_SUBREG (BSWPri (INSERT_SUBREG
+            (i64 (IMPLICIT_DEF)), $src, sub_i32), 1), sub_i32)>;
+
 // Several special pattern matches to optimize code
 
 def : Pat<(i32 (and i32:$lhs, 0xff)),
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index a46bf0649ac75b10037498a21de599b8eb441c2b..edd20900d810e46ffd1f1a203d50b11c00a89977 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -214,6 +214,11 @@ public:
                      SMLoc & /*EndLoc*/) override {
     llvm_unreachable("ParseRegister is not implemented.");
   }
+  OperandMatchResultTy tryParseRegister(unsigned & /*RegNo*/,
+                                        SMLoc & /*StartLoc*/,
+                                        SMLoc & /*EndLoc*/) override {
+    llvm_unreachable("tryParseRegister is not implemented.");
+  }
 
   bool error(const Twine &Msg, const AsmToken &Tok) {
     return Parser.Error(Tok.getLoc(), Msg + Tok.getString());
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index d37d812df485e315397464de05cfebe68a2afe42..ea4d23d03f292fda82b4ecf1197a192f965c08ce 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -858,6 +858,9 @@ private:
     return nullptr;
   }
 
+  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
+                     bool RestoreOnFailure);
+
   std::unique_ptr<X86Operand> DefaultMemSIOperand(SMLoc Loc);
   std::unique_ptr<X86Operand> DefaultMemDIOperand(SMLoc Loc);
   bool IsSIReg(unsigned Reg);
@@ -1023,6 +1026,8 @@ public:
   }
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
+  OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                        SMLoc &EndLoc) override;
 
   bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override;
 
@@ -1129,22 +1134,36 @@ static bool CheckBaseRegAndIndexRegAndScale(unsigned BaseReg, unsigned IndexReg,
   return checkScale(Scale, ErrMsg);
 }
 
-bool X86AsmParser::ParseRegister(unsigned &RegNo,
-                                 SMLoc &StartLoc, SMLoc &EndLoc) {
+bool X86AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                 SMLoc &EndLoc, bool RestoreOnFailure) {
   MCAsmParser &Parser = getParser();
+  MCAsmLexer &Lexer = getLexer();
   RegNo = 0;
+
+  SmallVector<AsmToken, 5> Tokens;
+  auto OnFailure = [RestoreOnFailure, &Lexer, &Tokens]() {
+    if (RestoreOnFailure) {
+      while (!Tokens.empty()) {
+        Lexer.UnLex(Tokens.pop_back_val());
+      }
+    }
+  };
+
   const AsmToken &PercentTok = Parser.getTok();
   StartLoc = PercentTok.getLoc();
 
   // If we encounter a %, ignore it. This code handles registers with and
   // without the prefix, unprefixed registers can occur in cfi directives.
-  if (!isParsingIntelSyntax() && PercentTok.is(AsmToken::Percent))
+  if (!isParsingIntelSyntax() && PercentTok.is(AsmToken::Percent)) {
+    Tokens.push_back(PercentTok);
     Parser.Lex(); // Eat percent token.
+  }
 
   const AsmToken &Tok = Parser.getTok();
   EndLoc = Tok.getEndLoc();
 
   if (Tok.isNot(AsmToken::Identifier)) {
+    OnFailure();
     if (isParsingIntelSyntax()) return true;
     return Error(StartLoc, "invalid register name",
                  SMRange(StartLoc, EndLoc));
@@ -1173,7 +1192,10 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
         X86II::isX86_64NonExtLowByteReg(RegNo) ||
         X86II::isX86_64ExtendedReg(RegNo)) {
       StringRef RegName = Tok.getString();
-      Parser.Lex(); // Eat register name.
+      OnFailure();
+      if (!RestoreOnFailure) {
+        Parser.Lex(); // Eat register name.
+      }
       return Error(StartLoc,
                    "register %" + RegName + " is only available in 64-bit mode",
                    SMRange(StartLoc, EndLoc));
@@ -1182,17 +1204,21 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
 
   // Parse "%st" as "%st(0)" and "%st(1)", which is multiple tokens.
   if (RegNo == X86::ST0) {
+    Tokens.push_back(Tok);
     Parser.Lex(); // Eat 'st'
 
     // Check to see if we have '(4)' after %st.
-    if (getLexer().isNot(AsmToken::LParen))
+    if (Lexer.isNot(AsmToken::LParen))
       return false;
     // Lex the paren.
-    getParser().Lex();
+    Tokens.push_back(Parser.getTok());
+    Parser.Lex();
 
     const AsmToken &IntTok = Parser.getTok();
-    if (IntTok.isNot(AsmToken::Integer))
+    if (IntTok.isNot(AsmToken::Integer)) {
+      OnFailure();
       return Error(IntTok.getLoc(), "expected stack index");
+    }
     switch (IntTok.getIntVal()) {
     case 0: RegNo = X86::ST0; break;
     case 1: RegNo = X86::ST1; break;
@@ -1202,11 +1228,18 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
     case 5: RegNo = X86::ST5; break;
     case 6: RegNo = X86::ST6; break;
     case 7: RegNo = X86::ST7; break;
-    default: return Error(IntTok.getLoc(), "invalid stack index");
+    default:
+      OnFailure();
+      return Error(IntTok.getLoc(), "invalid stack index");
     }
 
-    if (getParser().Lex().isNot(AsmToken::RParen))
+    // Lex IntTok
+    Tokens.push_back(IntTok);
+    Parser.Lex();
+    if (Lexer.isNot(AsmToken::RParen)) {
+      OnFailure();
       return Error(Parser.getTok().getLoc(), "expected ')'");
+    }
 
     EndLoc = Parser.getTok().getEndLoc();
     Parser.Lex(); // Eat ')'
@@ -1250,6 +1283,7 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
   }
 
   if (RegNo == 0) {
+    OnFailure();
     if (isParsingIntelSyntax()) return true;
     return Error(StartLoc, "invalid register name",
                  SMRange(StartLoc, EndLoc));
@@ -1259,6 +1293,25 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
   return false;
 }
 
+bool X86AsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                 SMLoc &EndLoc) {
+  return ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
+}
+
+OperandMatchResultTy X86AsmParser::tryParseRegister(unsigned &RegNo,
+                                                    SMLoc &StartLoc,
+                                                    SMLoc &EndLoc) {
+  bool Result =
+      ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
+  bool PendingErrors = getParser().hasPendingError();
+  getParser().clearPendingErrors();
+  if (PendingErrors)
+    return MatchOperand_ParseFail;
+  if (Result)
+    return MatchOperand_NoMatch;
+  return MatchOperand_Success;
+}
+
 std::unique_ptr<X86Operand> X86AsmParser::DefaultMemSIOperand(SMLoc Loc) {
   bool Parse32 = is32BitMode() || Code16GCC;
   unsigned Basereg = is64BitMode() ? X86::RSI : (Parse32 ? X86::ESI : X86::SI);
diff --git a/llvm/lib/Target/X86/X86CmovConversion.cpp b/llvm/lib/Target/X86/X86CmovConversion.cpp
index fe43bf4cbbce1ab2fae0fd60937bafe2d78cb9c8..fe5cb3ae2bf64468c6b42277601a9ad6dcdd59eb 100644
--- a/llvm/lib/Target/X86/X86CmovConversion.cpp
+++ b/llvm/lib/Target/X86/X86CmovConversion.cpp
@@ -364,12 +364,13 @@ bool X86CmovConverterPass::collectCmovCandidates(
 /// \param TrueOpDepth depth cost of CMOV true value operand.
 /// \param FalseOpDepth depth cost of CMOV false value operand.
 static unsigned getDepthOfOptCmov(unsigned TrueOpDepth, unsigned FalseOpDepth) {
-  //===--------------------------------------------------------------------===//
-  // With no info about branch weight, we assume 50% for each value operand.
-  // Thus, depth of optimized CMOV instruction is the rounded up average of
-  // its True-Operand-Value-Depth and False-Operand-Value-Depth.
-  //===--------------------------------------------------------------------===//
-  return (TrueOpDepth + FalseOpDepth + 1) / 2;
+  // The depth of the result after branch conversion is
+  // TrueOpDepth * TrueOpProbability + FalseOpDepth * FalseOpProbability.
+  // As we have no info about branch weight, we assume 75% for one and 25% for
+  // the other, and pick the result with the largest resulting depth.
+  return std::max(
+      divideCeil(TrueOpDepth * 3 + FalseOpDepth, 4),
+      divideCeil(FalseOpDepth * 3 + TrueOpDepth, 4));
 }
 
 bool X86CmovConverterPass::checkForProfitableCmovCandidates(
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b0187eb729202bab8499dd573a32c8c44792925d..60b06546f544dadfcc4d76e8341c95dbf39595b0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -374,20 +374,30 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   // If we don't have F16C support, then lower half float conversions
   // into library calls.
   if (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) {
-    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
-    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
+    setOperationAction(ISD::FP16_TO_FP,        MVT::f32, Custom);
+    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
+    setOperationAction(ISD::FP_TO_FP16,        MVT::f32, Custom);
+    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
   } else {
-    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
-    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
+    setOperationAction(ISD::FP16_TO_FP,        MVT::f32, Expand);
+    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Expand);
+    setOperationAction(ISD::FP_TO_FP16,        MVT::f32, Expand);
+    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Expand);
   }
 
   // There's never any support for operations beyond MVT::f32.
   setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
   setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand);
   setOperationAction(ISD::FP16_TO_FP, MVT::f128, Expand);
+  setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
+  setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f80, Expand);
+  setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f128, Expand);
   setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
   setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand);
   setOperationAction(ISD::FP_TO_FP16, MVT::f128, Expand);
+  setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Expand);
+  setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f80, Expand);
+  setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f128, Expand);
 
   setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
@@ -4010,7 +4020,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     } else if (VA.isRegLoc()) {
       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
       const TargetOptions &Options = DAG.getTarget().Options;
-      if (Options.EnableDebugEntryValues)
+      if (Options.SupportsDebugEntryValues)
         CSInfo.emplace_back(VA.getLocReg(), I);
       if (isVarArg && IsWin64) {
         // Win64 ABI requires argument XMM reg to be copied to the corresponding
@@ -20553,29 +20563,64 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
 }
 
 static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) {
-  assert(Op.getOperand(0).getValueType() == MVT::i16 &&
-         Op.getValueType() == MVT::f32 && "Unexpected VT!");
+  bool IsStrict = Op->isStrictFPOpcode();
+  SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
+  assert(Src.getValueType() == MVT::i16 && Op.getValueType() == MVT::f32 &&
+         "Unexpected VT!");
 
   SDLoc dl(Op);
   SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,
-                            DAG.getConstant(0, dl, MVT::v8i16),
-                            Op.getOperand(0), DAG.getIntPtrConstant(0, dl));
-  Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
-  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
-                     DAG.getIntPtrConstant(0, dl));
+                            DAG.getConstant(0, dl, MVT::v8i16), Src,
+                            DAG.getIntPtrConstant(0, dl));
+
+  SDValue Chain;
+  if (IsStrict) {
+    Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other},
+                      {Op.getOperand(0), Res});
+    Chain = Res.getValue(1);
+  } else {
+    Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);
+  }
+
+  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
+                    DAG.getIntPtrConstant(0, dl));
+
+  if (IsStrict)
+    return DAG.getMergeValues({Res, Chain}, dl);
+
+  return Res;
 }
 
 static SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) {
-  assert(Op.getOperand(0).getValueType() == MVT::f32 &&
-         Op.getValueType() == MVT::i16 && "Unexpected VT!");
+  bool IsStrict = Op->isStrictFPOpcode();
+  SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
+  assert(Src.getValueType() == MVT::f32 && Op.getValueType() == MVT::i16 &&
+         "Unexpected VT!");
 
   SDLoc dl(Op);
-  SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,
-                            Op.getOperand(0));
-  Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
-                    DAG.getTargetConstant(4, dl, MVT::i32));
-  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,
-                     DAG.getIntPtrConstant(0, dl));
+  SDValue Res, Chain;
+  if (IsStrict) {
+    Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32,
+                      DAG.getConstantFP(0, dl, MVT::v4f32), Src,
+                      DAG.getIntPtrConstant(0, dl));
+    Res = DAG.getNode(
+        X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},
+        {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)});
+    Chain = Res.getValue(1);
+  } else {
+    // FIXME: Should we use zeros for upper elements for non-strict?
+    Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src);
+    Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,
+                      DAG.getTargetConstant(4, dl, MVT::i32));
+  }
+
+  Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,
+                    DAG.getIntPtrConstant(0, dl));
+
+  if (IsStrict)
+    return DAG.getMergeValues({Res, Chain}, dl);
+
+  return Res;
 }
 
 /// Depending on uarch and/or optimizing for size, we might prefer to use a
@@ -28821,8 +28866,10 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::STRICT_FP_EXTEND:   return LowerFP_EXTEND(Op, DAG);
   case ISD::FP_ROUND:
   case ISD::STRICT_FP_ROUND:    return LowerFP_ROUND(Op, DAG);
-  case ISD::FP16_TO_FP:         return LowerFP16_TO_FP(Op, DAG);
-  case ISD::FP_TO_FP16:         return LowerFP_TO_FP16(Op, DAG);
+  case ISD::FP16_TO_FP:
+  case ISD::STRICT_FP16_TO_FP:  return LowerFP16_TO_FP(Op, DAG);
+  case ISD::FP_TO_FP16:
+  case ISD::STRICT_FP_TO_FP16:  return LowerFP_TO_FP16(Op, DAG);
   case ISD::LOAD:               return LowerLoad(Op, Subtarget, DAG);
   case ISD::STORE:              return LowerStore(Op, Subtarget, DAG);
   case ISD::FADD:
@@ -30162,8 +30209,10 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(SCALAR_UINT_TO_FP)
   NODE_NAME_CASE(SCALAR_UINT_TO_FP_RND)
   NODE_NAME_CASE(CVTPS2PH)
+  NODE_NAME_CASE(STRICT_CVTPS2PH)
   NODE_NAME_CASE(MCVTPS2PH)
   NODE_NAME_CASE(CVTPH2PS)
+  NODE_NAME_CASE(STRICT_CVTPH2PS)
   NODE_NAME_CASE(CVTPH2PS_SAE)
   NODE_NAME_CASE(CVTP2SI)
   NODE_NAME_CASE(CVTP2UI)
@@ -37359,10 +37408,7 @@ static SDValue combineHorizontalPredicateResult(SDNode *Extract,
       return SDValue();
     Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
   } else {
-    // Bail with AVX512VL (which uses predicate registers).
-    if (Subtarget.hasVLX())
-      return SDValue();
-
+    // FIXME: Better handling of k-registers or 512-bit vectors?
     unsigned MatchSizeInBits = Match.getValueSizeInBits();
     if (!(MatchSizeInBits == 128 ||
           (MatchSizeInBits == 256 && Subtarget.hasAVX())))
@@ -45931,6 +45977,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     return getZeroVector(VT, Subtarget, DAG, DL);
 
   SDValue Op0 = Ops[0];
+  bool IsSplat = llvm::all_of(Ops, [&Op0](SDValue Op) { return Op == Op0; });
 
   // Fold subvector loads into one.
   // If needed, look through bitcasts to get to the load.
@@ -45947,7 +45994,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
   }
 
   // Repeated subvectors.
-  if (llvm::all_of(Ops, [Op0](SDValue Op) { return Op == Op0; })) {
+  if (IsSplat) {
     // If this broadcast/subv_broadcast is inserted into both halves, use a
     // larger broadcast/subv_broadcast.
     if (Op0.getOpcode() == X86ISD::VBROADCAST ||
@@ -45970,8 +46017,6 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
   }
 
-  bool IsSplat = llvm::all_of(Ops, [&Op0](SDValue Op) { return Op == Op0; });
-
   // Repeated opcode.
   // TODO - combineX86ShufflesRecursively should handle shuffle concatenation
   // but it currently struggles with different vector widths.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index cf88ec8ae41341598e9a5c63ee2a6b5cfa1a3dbf..8c085c7f77f34c6d91b865e36f3d2ade1bc4d85d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -627,6 +627,9 @@ namespace llvm {
       // Strict FMA nodes.
       STRICT_FNMADD, STRICT_FMSUB, STRICT_FNMSUB,
 
+      // Conversions between float and half-float.
+      STRICT_CVTPS2PH, STRICT_CVTPH2PS,
+
       // Compare and swap.
       LCMPXCHG_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE,
       LCMPXCHG8_DAG,
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 9ce695730046edfe93ea13757d9b6f7b9e8b5541..53a9294c9fef4cce221219ca04e0bb6248d4faa8 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -8568,14 +8568,15 @@ let Uses = [MXCSR], mayRaiseFPException = 1 in
 multiclass avx512_cvtph2ps<X86VectorVTInfo _dest, X86VectorVTInfo _src,
                            X86MemOperand x86memop, PatFrag ld_frag,
                            X86FoldableSchedWrite sched> {
-  defm rr : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst),
+  defm rr : AVX512_maskable_split<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst),
                             (ins _src.RC:$src), "vcvtph2ps", "$src", "$src",
+                            (X86any_cvtph2ps (_src.VT _src.RC:$src)),
                             (X86cvtph2ps (_src.VT _src.RC:$src))>,
                             T8PD, Sched<[sched]>;
-  defm rm : AVX512_maskable<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst),
+  defm rm : AVX512_maskable_split<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst),
                             (ins x86memop:$src), "vcvtph2ps", "$src", "$src",
-                            (X86cvtph2ps (_src.VT
-                                          (ld_frag addr:$src)))>,
+                            (X86any_cvtph2ps (_src.VT (ld_frag addr:$src))),
+                            (X86cvtph2ps (_src.VT (ld_frag addr:$src)))>,
                             T8PD, Sched<[sched.Folded]>;
 }
 
@@ -8604,9 +8605,9 @@ let Predicates = [HasVLX] in {
                        EVEX_CD8<32, CD8VH>;
 
   // Pattern match vcvtph2ps of a scalar i64 load.
-  def : Pat<(v4f32 (X86cvtph2ps (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),
+  def : Pat<(v4f32 (X86any_cvtph2ps (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),
             (VCVTPH2PSZ128rm addr:$src)>;
-  def : Pat<(v4f32 (X86cvtph2ps (v8i16 (bitconvert
+  def : Pat<(v4f32 (X86any_cvtph2ps (v8i16 (bitconvert
               (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
             (VCVTPH2PSZ128rm addr:$src)>;
 }
@@ -8618,7 +8619,7 @@ let ExeDomain = GenericDomain, Uses = [MXCSR], mayRaiseFPException = 1 in {
              (ins _src.RC:$src1, i32u8imm:$src2),
              "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
              [(set _dest.RC:$dst,
-                   (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2)))]>,
+                   (X86any_cvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2)))]>,
              Sched<[RR]>;
   let Constraints = "$src0 = $dst" in
   def rrk : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),
@@ -8663,27 +8664,29 @@ let Predicates = [HasAVX512] in {
                                     WriteCvtPS2PHZ, WriteCvtPS2PHZSt>,
                     avx512_cvtps2ph_sae<v16i16x_info, v16f32_info, WriteCvtPS2PHZ>,
                                         EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
-  let Predicates = [HasVLX] in {
-    defm VCVTPS2PHZ256 : avx512_cvtps2ph<v8i16x_info, v8f32x_info, f128mem,
-                                         WriteCvtPS2PHY, WriteCvtPS2PHYSt>,
-                                         EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
-    defm VCVTPS2PHZ128 : avx512_cvtps2ph<v8i16x_info, v4f32x_info, f64mem,
-                                         WriteCvtPS2PH, WriteCvtPS2PHSt>,
-                                         EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
-  }
+
+  def : Pat<(store (v16i16 (X86any_cvtps2ph VR512:$src1, timm:$src2)), addr:$dst),
+            (VCVTPS2PHZmr addr:$dst, VR512:$src1, timm:$src2)>;
+}
+
+let Predicates = [HasVLX] in {
+  defm VCVTPS2PHZ256 : avx512_cvtps2ph<v8i16x_info, v8f32x_info, f128mem,
+                                       WriteCvtPS2PHY, WriteCvtPS2PHYSt>,
+                                       EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
+  defm VCVTPS2PHZ128 : avx512_cvtps2ph<v8i16x_info, v4f32x_info, f64mem,
+                                       WriteCvtPS2PH, WriteCvtPS2PHSt>,
+                                       EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
 
   def : Pat<(store (f64 (extractelt
-                         (bc_v2f64 (v8i16 (X86cvtps2ph VR128X:$src1, timm:$src2))),
+                         (bc_v2f64 (v8i16 (X86any_cvtps2ph VR128X:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
             (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, timm:$src2)>;
   def : Pat<(store (i64 (extractelt
-                         (bc_v2i64 (v8i16 (X86cvtps2ph VR128X:$src1, timm:$src2))),
+                         (bc_v2i64 (v8i16 (X86any_cvtps2ph VR128X:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
             (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, timm:$src2)>;
-  def : Pat<(store (v8i16 (X86cvtps2ph VR256X:$src1, timm:$src2)), addr:$dst),
+  def : Pat<(store (v8i16 (X86any_cvtps2ph VR256X:$src1, timm:$src2)), addr:$dst),
             (VCVTPS2PHZ256mr addr:$dst, VR256X:$src1, timm:$src2)>;
-  def : Pat<(store (v16i16 (X86cvtps2ph VR512:$src1, timm:$src2)), addr:$dst),
-            (VCVTPS2PHZmr addr:$dst, VR512:$src1, timm:$src2)>;
 }
 
 //  Unordered/Ordered scalar fp compare with Sae and set EFLAGS
diff --git a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index 5c5ddae0d0292cb53998d13457880c43b641cec8..3fc63e85fee497019fc15e85cada0770d60dcceb 100644
--- a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -721,19 +721,27 @@ def X86mcvtp2UInt    : SDNode<"X86ISD::MCVTP2UI",  SDTMFloatToInt>;
 def X86mcvttp2si     : SDNode<"X86ISD::MCVTTP2SI", SDTMFloatToInt>;
 def X86mcvttp2ui     : SDNode<"X86ISD::MCVTTP2UI", SDTMFloatToInt>;
 
+def SDTcvtph2ps : SDTypeProfile<1, 1, [SDTCVecEltisVT<0, f32>,
+                                       SDTCVecEltisVT<1, i16>]>;
+def X86cvtph2ps        : SDNode<"X86ISD::CVTPH2PS", SDTcvtph2ps>;
+def X86strict_cvtph2ps : SDNode<"X86ISD::STRICT_CVTPH2PS", SDTcvtph2ps,
+                                [SDNPHasChain]>;
+def X86any_cvtph2ps : PatFrags<(ops node:$src),
+                               [(X86strict_cvtph2ps node:$src),
+                                (X86cvtph2ps node:$src)]>;
+
+def X86cvtph2psSAE     : SDNode<"X86ISD::CVTPH2PS_SAE", SDTcvtph2ps>;
+
+def SDTcvtps2ph : SDTypeProfile<1, 2, [SDTCVecEltisVT<0, i16>,
+                                       SDTCVecEltisVT<1, f32>,
+                                       SDTCisVT<2, i32>]>;
+def X86cvtps2ph        : SDNode<"X86ISD::CVTPS2PH", SDTcvtps2ph>;
+def X86strict_cvtps2ph : SDNode<"X86ISD::STRICT_CVTPS2PH", SDTcvtps2ph,
+                                [SDNPHasChain]>;
+def X86any_cvtps2ph : PatFrags<(ops node:$src1, node:$src2),
+                               [(X86strict_cvtps2ph node:$src1, node:$src2),
+                                (X86cvtps2ph node:$src1, node:$src2)]>;
 
-def X86cvtph2ps     : SDNode<"X86ISD::CVTPH2PS",
-                              SDTypeProfile<1, 1, [SDTCVecEltisVT<0, f32>,
-                                                   SDTCVecEltisVT<1, i16>]> >;
-
-def X86cvtph2psSAE  : SDNode<"X86ISD::CVTPH2PS_SAE",
-                              SDTypeProfile<1, 1, [SDTCVecEltisVT<0, f32>,
-                                                   SDTCVecEltisVT<1, i16>]> >;
-
-def X86cvtps2ph   : SDNode<"X86ISD::CVTPS2PH",
-                        SDTypeProfile<1, 2, [SDTCVecEltisVT<0, i16>,
-                                             SDTCVecEltisVT<1, f32>,
-                                             SDTCisVT<2, i32>]> >;
 def X86mcvtps2ph   : SDNode<"X86ISD::MCVTPS2PH",
                         SDTypeProfile<1, 4, [SDTCVecEltisVT<0, i16>,
                                              SDTCVecEltisVT<1, f32>,
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index b519e4a42e9d6cee65e0b388c53393abc4d57fa9..a8c285cfe5f5b5234b6bc90a436cbafa009e96da 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -7337,12 +7337,11 @@ multiclass f16c_ph2ps<RegisterClass RC, X86MemOperand x86memop,
                       X86FoldableSchedWrite sched> {
   def rr : I<0x13, MRMSrcReg, (outs RC:$dst), (ins VR128:$src),
              "vcvtph2ps\t{$src, $dst|$dst, $src}",
-             [(set RC:$dst, (X86cvtph2ps VR128:$src))]>,
+             [(set RC:$dst, (X86any_cvtph2ps VR128:$src))]>,
              T8PD, VEX, Sched<[sched]>;
-  let hasSideEffects = 0, mayLoad = 1 in
   def rm : I<0x13, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
              "vcvtph2ps\t{$src, $dst|$dst, $src}",
-             [(set RC:$dst, (X86cvtph2ps (loadv8i16 addr:$src)))]>,
+             [(set RC:$dst, (X86any_cvtph2ps (loadv8i16 addr:$src)))]>,
              T8PD, VEX, Sched<[sched.Folded]>;
 }
 
@@ -7351,7 +7350,7 @@ multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop,
   def rr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst),
                (ins RC:$src1, i32u8imm:$src2),
                "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-               [(set VR128:$dst, (X86cvtps2ph RC:$src1, timm:$src2))]>,
+               [(set VR128:$dst, (X86any_cvtps2ph RC:$src1, timm:$src2))]>,
                TAPD, VEX, Sched<[RR]>;
   let hasSideEffects = 0, mayStore = 1 in
   def mr : Ii8<0x1D, MRMDestMem, (outs),
@@ -7369,21 +7368,21 @@ let Predicates = [HasF16C, NoVLX] in {
                                WriteCvtPS2PHYSt>, VEX_L, SIMD_EXC;
 
   // Pattern match vcvtph2ps of a scalar i64 load.
-  def : Pat<(v4f32 (X86cvtph2ps (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),
+  def : Pat<(v4f32 (X86any_cvtph2ps (bc_v8i16 (v2i64 (X86vzload64 addr:$src))))),
             (VCVTPH2PSrm addr:$src)>;
-  def : Pat<(v4f32 (X86cvtph2ps (bc_v8i16
+  def : Pat<(v4f32 (X86any_cvtph2ps (bc_v8i16
               (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
             (VCVTPH2PSrm addr:$src)>;
 
   def : Pat<(store (f64 (extractelt
-                         (bc_v2f64 (v8i16 (X86cvtps2ph VR128:$src1, timm:$src2))),
+                         (bc_v2f64 (v8i16 (X86any_cvtps2ph VR128:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
             (VCVTPS2PHmr addr:$dst, VR128:$src1, timm:$src2)>;
   def : Pat<(store (i64 (extractelt
-                         (bc_v2i64 (v8i16 (X86cvtps2ph VR128:$src1, timm:$src2))),
+                         (bc_v2i64 (v8i16 (X86any_cvtps2ph VR128:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
             (VCVTPS2PHmr addr:$dst, VR128:$src1, timm:$src2)>;
-  def : Pat<(store (v8i16 (X86cvtps2ph VR256:$src1, timm:$src2)), addr:$dst),
+  def : Pat<(store (v8i16 (X86any_cvtps2ph VR256:$src1, timm:$src2)), addr:$dst),
             (VCVTPS2PHYmr addr:$dst, VR256:$src1, timm:$src2)>;
 }
 
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
index 0950203801afa21860637260a0046319d2c26571..febef9cc32b98921a4baa7c3c2ca3cfe2d28e94e 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
@@ -362,9 +362,9 @@ defm : X86WriteResPairUnsupported<WriteVecLogicZ>;
 defm : SKLWriteResPair<WriteVecTest,  [SKLPort0,SKLPort5], 3, [1,1], 2, 6>; // Vector integer TEST instructions.
 defm : SKLWriteResPair<WriteVecTestY, [SKLPort0,SKLPort5], 3, [1,1], 2, 7>;
 defm : X86WriteResPairUnsupported<WriteVecTestZ>;
-defm : SKLWriteResPair<WriteVecIMul,  [SKLPort0] ,  4, [1], 1, 5>; // Vector integer multiply.
-defm : SKLWriteResPair<WriteVecIMulX, [SKLPort01],  4, [1], 1, 6>;
-defm : SKLWriteResPair<WriteVecIMulY, [SKLPort01],  4, [1], 1, 7>;
+defm : SKLWriteResPair<WriteVecIMul,  [SKLPort0] ,  5, [1], 1, 5>; // Vector integer multiply.
+defm : SKLWriteResPair<WriteVecIMulX, [SKLPort01],  5, [1], 1, 6>;
+defm : SKLWriteResPair<WriteVecIMulY, [SKLPort01],  5, [1], 1, 7>;
 defm : X86WriteResPairUnsupported<WriteVecIMulZ>;
 defm : SKLWriteResPair<WritePMULLD,   [SKLPort01], 10, [2], 2, 6>; // Vector PMULLD.
 defm : SKLWriteResPair<WritePMULLDY,  [SKLPort01], 10, [2], 2, 7>;
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index 2556f06c64c3b4d3ee1c9749747d41f97e6ac2c6..55893945638e9d691f2674132cc02546b7302016 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -362,10 +362,10 @@ defm : SKXWriteResPair<WriteVecLogicZ,[SKXPort05], 1, [1], 1, 7>;
 defm : SKXWriteResPair<WriteVecTest,  [SKXPort0,SKXPort5], 3, [1,1], 2, 6>; // Vector integer TEST instructions.
 defm : SKXWriteResPair<WriteVecTestY, [SKXPort0,SKXPort5], 3, [1,1], 2, 7>;
 defm : SKXWriteResPair<WriteVecTestZ, [SKXPort0,SKXPort5], 3, [1,1], 2, 7>;
-defm : SKXWriteResPair<WriteVecIMul,  [SKXPort0],    4, [1], 1, 5>; // Vector integer multiply.
-defm : SKXWriteResPair<WriteVecIMulX, [SKXPort01],  4, [1], 1, 6>;
-defm : SKXWriteResPair<WriteVecIMulY, [SKXPort01],  4, [1], 1, 7>;
-defm : SKXWriteResPair<WriteVecIMulZ, [SKXPort05],  4, [1], 1, 7>;
+defm : SKXWriteResPair<WriteVecIMul,  [SKXPort0],   5, [1], 1, 5>; // Vector integer multiply.
+defm : SKXWriteResPair<WriteVecIMulX, [SKXPort01],  5, [1], 1, 6>;
+defm : SKXWriteResPair<WriteVecIMulY, [SKXPort01],  5, [1], 1, 7>;
+defm : SKXWriteResPair<WriteVecIMulZ, [SKXPort05],  5, [1], 1, 7>;
 defm : SKXWriteResPair<WritePMULLD,   [SKXPort01], 10, [2], 2, 6>; // Vector PMULLD.
 defm : SKXWriteResPair<WritePMULLDY,  [SKXPort01], 10, [2], 2, 7>;
 defm : SKXWriteResPair<WritePMULLDZ,  [SKXPort05], 10, [2], 2, 7>;
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 22b4e2805a5ea321911d09d4b442383a21e5ab47..0cfa7bb04771eff41980d081104b57a56db27cdf 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -232,6 +232,9 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
 
   setMachineOutliner(true);
 
+  // x86 supports the debug entry values.
+  setSupportsDebugEntryValues(true);
+
   initAsmInfo();
 }
 
diff --git a/llvm/lib/TextAPI/MachO/TextStubCommon.cpp b/llvm/lib/TextAPI/MachO/TextStubCommon.cpp
index e4e58cd66f3fce18e9650792cbd23f84fc40d7ce..21be654e130c0b7ab2219f9b76732d90ce1d2e78 100644
--- a/llvm/lib/TextAPI/MachO/TextStubCommon.cpp
+++ b/llvm/lib/TextAPI/MachO/TextStubCommon.cpp
@@ -62,12 +62,18 @@ void ScalarTraits<PlatformSet>::output(const PlatformSet &Values, void *IO,
   case PlatformKind::macOS:
     OS << "macosx";
     break;
+  case PlatformKind::iOSSimulator:
+    LLVM_FALLTHROUGH;
   case PlatformKind::iOS:
     OS << "ios";
     break;
+  case PlatformKind::watchOSSimulator:
+    LLVM_FALLTHROUGH;
   case PlatformKind::watchOS:
     OS << "watchos";
     break;
+  case PlatformKind::tvOSSimulator:
+    LLVM_FALLTHROUGH;
   case PlatformKind::tvOS:
     OS << "tvos";
     break;
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 31a7e93b239adeb1af8ac673579ef09b03d8b638..86968d1f7761bae725f96a1de00a1725f4d6ab33 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -374,9 +374,11 @@ static Value *constructPointer(Type *ResTy, Value *Ptr, int64_t Offset,
 /// will be done by looking through cast instructions, selects, phis, and calls
 /// with the "returned" attribute. Once we cannot look through the value any
 /// further, the callback \p VisitValueCB is invoked and passed the current
-/// value, the \p State, and a flag to indicate if we stripped anything. To
-/// limit how much effort is invested, we will never visit more values than
-/// specified by \p MaxValues.
+/// value, the \p State, and a flag to indicate if we stripped anything.
+/// Stripped means that we unpacked the value associated with \p IRP at least
+/// once. Note that the value used for the callback may still be the value
+/// associated with \p IRP (due to PHIs). To limit how much effort is invested,
+/// we will never visit more values than specified by \p MaxValues.
 template <typename AAType, typename StateTy>
 static bool genericValueTraversal(
     Attributor &A, IRPosition IRP, const AAType &QueryingAA, StateTy &State,
@@ -1192,8 +1194,9 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) {
   auto ReplaceCallSiteUsersWith = [&A](CallBase &CB, Constant &C) {
     if (CB.getNumUses() == 0 || CB.isMustTailCall())
       return ChangeStatus::UNCHANGED;
-    A.replaceAllUsesWith(CB, C);
-    return ChangeStatus::CHANGED;
+    if (A.changeValueAfterManifest(CB, C))
+      return ChangeStatus::CHANGED;
+    return ChangeStatus::UNCHANGED;
   };
 
   // If the assumed unique return value is an argument, annotate it.
@@ -1771,6 +1774,8 @@ struct AANoFreeFloating : AANoFreeImpl {
         Follow = true;
         return true;
       }
+      if (isa<ReturnInst>(UserI))
+        return true;
 
       // Unknown user.
       return false;
@@ -4443,8 +4448,8 @@ struct AAValueSimplifyImpl : AAValueSimplify {
       if (!V.user_empty() && &V != C && V.getType() == C->getType()) {
         LLVM_DEBUG(dbgs() << "[ValueSimplify] " << V << " -> " << *C
                           << " :: " << *this << "\n");
-        A.changeValueAfterManifest(V, *C);
-        Changed = ChangeStatus::CHANGED;
+        if (A.changeValueAfterManifest(V, *C))
+          Changed = ChangeStatus::CHANGED;
       }
     }
 
@@ -4696,7 +4701,7 @@ struct AAHeapToStackImpl : public AAHeapToStack {
         AI = new BitCastInst(AI, MallocCall->getType(), "malloc_bc",
                              AI->getNextNode());
 
-      A.replaceAllUsesWith(*MallocCall, *AI);
+      A.changeValueAfterManifest(*MallocCall, *AI);
 
       if (auto *II = dyn_cast<InvokeInst>(MallocCall)) {
         auto *NBB = II->getNormalDest();
@@ -6485,14 +6490,27 @@ bool Attributor::checkForAllUses(
     const Use *U = Worklist.pop_back_val();
     if (!Visited.insert(U).second)
       continue;
-    LLVM_DEBUG(dbgs() << "[Attributor] Check use: " << **U << "\n");
-    if (Instruction *UserI = dyn_cast<Instruction>(U->getUser()))
-      if (LivenessAA && LivenessAA->isAssumedDead(UserI)) {
-        LLVM_DEBUG(dbgs() << "[Attributor] Dead user: " << *UserI << ": "
-                          << *LivenessAA << "\n");
-        AnyDead = true;
-        continue;
+    LLVM_DEBUG(dbgs() << "[Attributor] Check use: " << **U << " [" << LivenessAA
+                      << "]\n");
+    if (LivenessAA) {
+      if (Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
+        if (LivenessAA->isAssumedDead(UserI)) {
+          LLVM_DEBUG(dbgs() << "[Attributor] Dead user: " << *UserI << ": "
+                            << *LivenessAA << "\n");
+          AnyDead = true;
+          continue;
+        }
+        if (PHINode *PHI = dyn_cast<PHINode>(UserI)) {
+          BasicBlock *IncomingBB = PHI->getIncomingBlock(*U);
+          if (LivenessAA->isAssumedDead(IncomingBB->getTerminator())) {
+            LLVM_DEBUG(dbgs() << "[Attributor] Dead user: " << *UserI << ": "
+                              << *LivenessAA << "\n");
+            AnyDead = true;
+            continue;
+          }
+        }
       }
+    }
 
     bool Follow = false;
     if (!Pred(*U, Follow))
@@ -7705,10 +7723,12 @@ static bool runAttributorOnFunctions(InformationCache &InfoCache,
     A.identifyDefaultAbstractAttributes(*F);
   }
 
-  bool Changed = A.run() == ChangeStatus::CHANGED;
+  ChangeStatus Changed = A.run();
   assert(!verifyModule(*Functions.front()->getParent(), &errs()) &&
          "Module verification failed!");
-  return Changed;
+  LLVM_DEBUG(dbgs() << "[Attributor] Done with " << Functions.size()
+                    << " functions, result: " << Changed << ".\n");
+  return Changed == ChangeStatus::CHANGED;
 }
 
 PreservedAnalyses AttributorPass::run(Module &M, ModuleAnalysisManager &AM) {
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 6f38a3123932cd5f13f8b45c0487af497823f142..7c26f156d4c90b85d8cbf28b858ad9d5cd6ce3da 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -735,6 +735,9 @@ static bool isKnownTypeIdMember(Metadata *TypeId, const DataLayout &DL,
 /// replace the call with.
 Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,
                                                const TypeIdLowering &TIL) {
+  // Delay lowering if the resolution is currently unknown.
+  if (TIL.TheKind == TypeTestResolution::Unknown)
+    return nullptr;
   if (TIL.TheKind == TypeTestResolution::Unsat)
     return ConstantInt::getFalse(M.getContext());
 
@@ -1043,8 +1046,10 @@ void LowerTypeTestsModule::importTypeTest(CallInst *CI) {
 
   TypeIdLowering TIL = importTypeId(TypeIdStr->getString());
   Value *Lowered = lowerTypeTestCall(TypeIdStr, CI, TIL);
-  CI->replaceAllUsesWith(Lowered);
-  CI->eraseFromParent();
+  if (Lowered) {
+    CI->replaceAllUsesWith(Lowered);
+    CI->eraseFromParent();
+  }
 }
 
 // ThinLTO backend: the function F has a jump table entry; update this module
@@ -1167,8 +1172,10 @@ void LowerTypeTestsModule::lowerTypeTestCalls(
     for (CallInst *CI : TIUI.CallSites) {
       ++NumTypeTestCallsLowered;
       Value *Lowered = lowerTypeTestCall(TypeId, CI, TIL);
-      CI->replaceAllUsesWith(Lowered);
-      CI->eraseFromParent();
+      if (Lowered) {
+        CI->replaceAllUsesWith(Lowered);
+        CI->eraseFromParent();
+      }
     }
   }
 }
diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
index 61ec141835c9033511ac940d042de98d4e8604e8..6f155dfebb56a3eee637d1aabd1e4e161dfb1169 100644
--- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -504,6 +504,7 @@ void PassManagerBuilder::populateModulePassManager(
       MPM.add(createBarrierNoopPass());
 
     if (PerformThinLTO) {
+      MPM.add(createLowerTypeTestsPass(nullptr, nullptr, true));
       // Drop available_externally and unreferenced globals. This is necessary
       // with ThinLTO in order to avoid leaving undefined references to dead
       // globals in the object file.
@@ -537,9 +538,11 @@ void PassManagerBuilder::populateModulePassManager(
   // inter-module indirect calls. For that we perform indirect call promotion
   // earlier in the pass pipeline, here before globalopt. Otherwise imported
   // available_externally functions look unreferenced and are removed.
-  if (PerformThinLTO)
+  if (PerformThinLTO) {
     MPM.add(createPGOIndirectCallPromotionLegacyPass(/*InLTO = */ true,
                                                      !PGOSampleUse.empty()));
+    MPM.add(createLowerTypeTestsPass(nullptr, nullptr, true));
+  }
 
   // For SamplePGO in ThinLTO compile phase, we do not want to unroll loops
   // as it will change the CFG too much to make the 2nd profile annotation
@@ -1060,8 +1063,8 @@ void PassManagerBuilder::populateThinLTOPassManager(
     PM.add(createVerifierPass());
 
   if (ImportSummary) {
-    // These passes import type identifier resolutions for whole-program
-    // devirtualization and CFI. They must run early because other passes may
+    // This pass imports type identifier resolutions for whole-program
+    // devirtualization and CFI. It must run early because other passes may
     // disturb the specific instruction patterns that these passes look for,
     // creating dependencies on resolutions that may not appear in the summary.
     //
@@ -1109,6 +1112,9 @@ void PassManagerBuilder::populateLTOPassManager(legacy::PassManagerBase &PM) {
   // control flow integrity mechanisms (-fsanitize=cfi*) and needs to run at
   // link time if CFI is enabled. The pass does nothing if CFI is disabled.
   PM.add(createLowerTypeTestsPass(ExportSummary, nullptr));
+  // Run a second time to clean up any type tests left behind by WPD for use
+  // in ICP (which is performed earlier than this in the regular LTO pipeline).
+  PM.add(createLowerTypeTestsPass(nullptr, nullptr, true));
 
   if (OptLevel != 0)
     addLateLTOOptimizationPasses(PM);
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index bbc1433a22e895c1646154635e9be6cd6b457a69..26beb54c205c509ed63542ca67a887c4914275e4 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -510,7 +510,9 @@ struct DevirtModule {
 
   bool areRemarksEnabled();
 
-  void scanTypeTestUsers(Function *TypeTestFunc);
+  void
+  scanTypeTestUsers(Function *TypeTestFunc,
+                    DenseMap<Metadata *, std::set<TypeMemberInfo>> &TypeIdMap);
   void scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc);
 
   void buildTypeIdentifierMap(
@@ -1666,7 +1668,9 @@ bool DevirtModule::areRemarksEnabled() {
   return false;
 }
 
-void DevirtModule::scanTypeTestUsers(Function *TypeTestFunc) {
+void DevirtModule::scanTypeTestUsers(
+    Function *TypeTestFunc,
+    DenseMap<Metadata *, std::set<TypeMemberInfo>> &TypeIdMap) {
   // Find all virtual calls via a virtual table pointer %p under an assumption
   // of the form llvm.assume(llvm.type.test(%p, %md)). This indicates that %p
   // points to a member of the type identifier %md. Group calls by (type ID,
@@ -1686,10 +1690,10 @@ void DevirtModule::scanTypeTestUsers(Function *TypeTestFunc) {
     auto &DT = LookupDomTree(*CI->getFunction());
     findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
 
+    Metadata *TypeId =
+        cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
     // If we found any, add them to CallSlots.
     if (!Assumes.empty()) {
-      Metadata *TypeId =
-          cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
       Value *Ptr = CI->getArgOperand(0)->stripPointerCasts();
       for (DevirtCallSite Call : DevirtCalls) {
         // Only add this CallSite if we haven't seen it before. The vtable
@@ -1702,6 +1706,13 @@ void DevirtModule::scanTypeTestUsers(Function *TypeTestFunc) {
       }
     }
 
+    // If we have any uses on type metadata, keep the type test assumes for
+    // later analysis. Otherwise remove as they aren't useful, and
+    // LowerTypeTests will think they are Unsat and lower to False, which
+    // breaks any uses on assumes.
+    if (TypeIdMap.count(TypeId))
+      continue;
+
     // We no longer need the assumes or the type test.
     for (auto Assume : Assumes)
       Assume->eraseFromParent();
@@ -1900,8 +1911,13 @@ bool DevirtModule::run() {
       (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()))
     return false;
 
+  // Rebuild type metadata into a map for easy lookup.
+  std::vector<VTableBits> Bits;
+  DenseMap<Metadata *, std::set<TypeMemberInfo>> TypeIdMap;
+  buildTypeIdentifierMap(Bits, TypeIdMap);
+
   if (TypeTestFunc && AssumeFunc)
-    scanTypeTestUsers(TypeTestFunc);
+    scanTypeTestUsers(TypeTestFunc, TypeIdMap);
 
   if (TypeCheckedLoadFunc)
     scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
@@ -1923,10 +1939,6 @@ bool DevirtModule::run() {
     return true;
   }
 
-  // Rebuild type metadata into a map for easy lookup.
-  std::vector<VTableBits> Bits;
-  DenseMap<Metadata *, std::set<TypeMemberInfo>> TypeIdMap;
-  buildTypeIdentifierMap(Bits, TypeIdMap);
   if (TypeIdMap.empty())
     return true;
 
@@ -1983,14 +1995,18 @@ bool DevirtModule::run() {
     // function implementation at offset S.first.ByteOffset, and add to
     // TargetsForSlot.
     std::vector<VirtualCallTarget> TargetsForSlot;
+    WholeProgramDevirtResolution *Res = nullptr;
+    if (ExportSummary && isa<MDString>(S.first.TypeID) &&
+        TypeIdMap.count(S.first.TypeID))
+      // For any type id used on a global's type metadata, create the type id
+      // summary resolution regardless of whether we can devirtualize, so that
+      // lower type tests knows the type id is not Unsat.
+      Res = &ExportSummary
+                 ->getOrInsertTypeIdSummary(
+                     cast<MDString>(S.first.TypeID)->getString())
+                 .WPDRes[S.first.ByteOffset];
     if (tryFindVirtualCallTargets(TargetsForSlot, TypeIdMap[S.first.TypeID],
                                   S.first.ByteOffset)) {
-      WholeProgramDevirtResolution *Res = nullptr;
-      if (ExportSummary && isa<MDString>(S.first.TypeID))
-        Res = &ExportSummary
-                   ->getOrInsertTypeIdSummary(
-                       cast<MDString>(S.first.TypeID)->getString())
-                   .WPDRes[S.first.ByteOffset];
 
       if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {
         DidVirtualConstProp |=
@@ -2104,11 +2120,14 @@ void DevirtIndex::run() {
     std::vector<ValueInfo> TargetsForSlot;
     auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);
     assert(TidSummary);
+    // Create the type id summary resolution regardlness of whether we can
+    // devirtualize, so that lower type tests knows the type id is used on
+    // a global and not Unsat.
+    WholeProgramDevirtResolution *Res =
+        &ExportSummary.getOrInsertTypeIdSummary(S.first.TypeID)
+             .WPDRes[S.first.ByteOffset];
     if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
                                   S.first.ByteOffset)) {
-      WholeProgramDevirtResolution *Res =
-          &ExportSummary.getOrInsertTypeIdSummary(S.first.TypeID)
-               .WPDRes[S.first.ByteOffset];
 
       if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
                                DevirtTargets))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 57466280094108d4596adfdba9655706f8c0f8c1..d1b820f93660c74ca8d07aaf61c376e68d099321 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1380,8 +1380,9 @@ Instruction *InstCombiner::visitAdd(BinaryOperator &I) {
   // (add (and A, B) (or A, B)) --> (add A, B)
   if (match(&I, m_c_BinOp(m_Or(m_Value(A), m_Value(B)),
                           m_c_And(m_Deferred(A), m_Deferred(B))))) {
-    I.setOperand(0, A);
-    I.setOperand(1, B);
+    // Replacing operands in-place to preserve nuw/nsw flags.
+    replaceOperand(I, 0, A);
+    replaceOperand(I, 1, B);
     return &I;
   }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
index 825f4b468b0a7f95dcfb0662052cde647fabe7fc..9d3ecba559c3eb27874d9ac36eda587ff5a1af5a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -138,13 +138,11 @@ Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
   if (RMWI.getType()->isIntegerTy() &&
       RMWI.getOperation() != AtomicRMWInst::Or) {
     RMWI.setOperation(AtomicRMWInst::Or);
-    RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0));
-    return &RMWI;
+    return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));
   } else if (RMWI.getType()->isFloatingPointTy() &&
              RMWI.getOperation() != AtomicRMWInst::FAdd) {
     RMWI.setOperation(AtomicRMWInst::FAdd);
-    RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType()));
-    return &RMWI;
+    return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));
   }
 
   // Check if the required ordering is compatible with an atomic load.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 0f8767873cf12b8166fcbaf00166b10843433f9d..37e04e68ec5f6890efa4d2bb96ce1ebd0c7d1a5f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -6030,14 +6030,11 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
   // If we're just checking for a NaN (ORD/UNO) and have a non-NaN operand,
   // then canonicalize the operand to 0.0.
   if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
-    if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI)) {
-      I.setOperand(0, ConstantFP::getNullValue(OpType));
-      return &I;
-    }
-    if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI)) {
-      I.setOperand(1, ConstantFP::getNullValue(OpType));
-      return &I;
-    }
+    if (!match(Op0, m_PosZeroFP()) && isKnownNeverNaN(Op0, &TLI))
+      return replaceOperand(I, 0, ConstantFP::getNullValue(OpType));
+
+    if (!match(Op1, m_PosZeroFP()) && isKnownNeverNaN(Op1, &TLI))
+      return replaceOperand(I, 1, ConstantFP::getNullValue(OpType));
   }
 
   // fcmp pred (fneg X), (fneg Y) -> fcmp swap(pred) X, Y
@@ -6062,10 +6059,8 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
 
   // The sign of 0.0 is ignored by fcmp, so canonicalize to +0.0:
   // fcmp Pred X, -0.0 --> fcmp Pred X, 0.0
-  if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP())) {
-    I.setOperand(1, ConstantFP::getNullValue(OpType));
-    return &I;
-  }
+  if (match(Op1, m_AnyZeroFP()) && !match(Op1, m_PosZeroFP()))
+    return replaceOperand(I, 1, ConstantFP::getNullValue(OpType));
 
   // Handle fcmp with instruction LHS and constant RHS.
   Instruction *LHSI;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index c086b152c09b4660481f9243b05bfcf38de58058..c9baa8b87fafb008926ec5d49c68bd358a354b9d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -56,7 +56,8 @@ static Value *createMinMax(InstCombiner::BuilderTy &Builder,
 /// Replace a select operand based on an equality comparison with the identity
 /// constant of a binop.
 static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
-                                            const TargetLibraryInfo &TLI) {
+                                            const TargetLibraryInfo &TLI,
+                                            InstCombiner &IC) {
   // The select condition must be an equality compare with a constant operand.
   Value *X;
   Constant *C;
@@ -107,8 +108,7 @@ static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
   // S = { select (cmp eq X, C), BO, ? } or { select (cmp ne X, C), ?, BO }
   // =>
   // S = { select (cmp eq X, C),  Y, ? } or { select (cmp ne X, C), ?,  Y }
-  Sel.setOperand(IsEq ? 1 : 2, Y);
-  return &Sel;
+  return IC.replaceOperand(Sel, IsEq ? 1 : 2, Y);
 }
 
 /// This folds:
@@ -997,7 +997,7 @@ static bool adjustMinMax(SelectInst &Sel, ICmpInst &Cmp) {
 /// constant operand of the select.
 static Instruction *
 canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp,
-                               InstCombiner::BuilderTy &Builder) {
+                               InstCombiner &IC) {
   if (!Cmp.hasOneUse() || !isa<Constant>(Cmp.getOperand(1)))
     return nullptr;
 
@@ -1020,7 +1020,7 @@ canonicalizeMinMaxWithConstant(SelectInst &Sel, ICmpInst &Cmp,
     return nullptr;
 
   // Create the canonical compare and plug it into the select.
-  Sel.setCondition(Builder.CreateICmp(CanonicalPred, LHS, RHS));
+  IC.replaceOperand(Sel, 0, IC.Builder.CreateICmp(CanonicalPred, LHS, RHS));
 
   // If the select operands did not change, we're done.
   if (Sel.getTrueValue() == LHS && Sel.getFalseValue() == RHS)
@@ -1329,7 +1329,7 @@ static Instruction *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
 // and swap the hands of select.
 static Instruction *
 tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
-                                         InstCombiner::BuilderTy &Builder) {
+                                         InstCombiner &IC) {
   ICmpInst::Predicate Pred;
   Value *X;
   Constant *C0;
@@ -1381,13 +1381,13 @@ tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
     return nullptr;
 
   // It matched! Lets insert the new comparison just before select.
-  InstCombiner::BuilderTy::InsertPointGuard Guard(Builder);
-  Builder.SetInsertPoint(&Sel);
+  InstCombiner::BuilderTy::InsertPointGuard Guard(IC.Builder);
+  IC.Builder.SetInsertPoint(&Sel);
 
   Pred = ICmpInst::getSwappedPredicate(Pred); // Yes, swapped.
-  Value *NewCmp = Builder.CreateICmp(Pred, X, FlippedStrictness->second,
-                                     Cmp.getName() + ".inv");
-  Sel.setCondition(NewCmp);
+  Value *NewCmp = IC.Builder.CreateICmp(Pred, X, FlippedStrictness->second,
+                                        Cmp.getName() + ".inv");
+  IC.replaceOperand(Sel, 0, NewCmp);
   Sel.swapValues();
   Sel.swapProfMetadata();
 
@@ -1400,7 +1400,7 @@ Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI,
   if (Value *V = foldSelectValueEquivalence(SI, *ICI, SQ))
     return replaceInstUsesWith(SI, V);
 
-  if (Instruction *NewSel = canonicalizeMinMaxWithConstant(SI, *ICI, Builder))
+  if (Instruction *NewSel = canonicalizeMinMaxWithConstant(SI, *ICI, *this))
     return NewSel;
 
   if (Instruction *NewAbs = canonicalizeAbsNabs(SI, *ICI, Builder))
@@ -1410,7 +1410,7 @@ Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI,
     return NewAbs;
 
   if (Instruction *NewSel =
-          tryToReuseConstantFromSelectInComparison(SI, *ICI, Builder))
+          tryToReuseConstantFromSelectInComparison(SI, *ICI, *this))
     return NewSel;
 
   bool Changed = adjustMinMax(SI, *ICI);
@@ -1973,7 +1973,7 @@ static Instruction *canonicalizeSelectToShuffle(SelectInst &SI) {
 /// other operations in IR and having all operands of a select be vector types
 /// is likely better for vector codegen.
 static Instruction *canonicalizeScalarSelectOfVecs(
-    SelectInst &Sel, InstCombiner::BuilderTy &Builder) {
+    SelectInst &Sel, InstCombiner &IC) {
   Type *Ty = Sel.getType();
   if (!Ty->isVectorTy())
     return nullptr;
@@ -1987,9 +1987,7 @@ static Instruction *canonicalizeScalarSelectOfVecs(
   // Splatting the extracted condition reduces code (we could directly create a
   // splat shuffle of the source vector to eliminate the intermediate step).
   unsigned NumElts = Ty->getVectorNumElements();
-  Value *SplatCond = Builder.CreateVectorSplat(NumElts, Cond);
-  Sel.setCondition(SplatCond);
-  return &Sel;
+  return IC.replaceOperand(Sel, 0, IC.Builder.CreateVectorSplat(NumElts, Cond));
 }
 
 /// Reuse bitcasted operands between a compare and select:
@@ -2395,7 +2393,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   if (Instruction *I = canonicalizeSelectToShuffle(SI))
     return I;
 
-  if (Instruction *I = canonicalizeScalarSelectOfVecs(SI, Builder))
+  if (Instruction *I = canonicalizeScalarSelectOfVecs(SI, *this))
     return I;
 
   // Canonicalize a one-use integer compare with a non-canonical predicate by
@@ -2698,8 +2696,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       if (TrueSI->getCondition() == CondVal) {
         if (SI.getTrueValue() == TrueSI->getTrueValue())
           return nullptr;
-        SI.setOperand(1, TrueSI->getTrueValue());
-        return &SI;
+        return replaceOperand(SI, 1, TrueSI->getTrueValue());
       }
       // select(C0, select(C1, a, b), b) -> select(C0&C1, a, b)
       // We choose this as normal form to enable folding on the And and shortening
@@ -2718,8 +2715,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
       if (FalseSI->getCondition() == CondVal) {
         if (SI.getFalseValue() == FalseSI->getFalseValue())
           return nullptr;
-        SI.setOperand(2, FalseSI->getFalseValue());
-        return &SI;
+        return replaceOperand(SI, 2, FalseSI->getFalseValue());
       }
       // select(C0, a, select(C1, a, b)) -> select(C0|C1, a, b)
       if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) {
@@ -2788,7 +2784,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
 
   Value *NotCond;
   if (match(CondVal, m_Not(m_Value(NotCond)))) {
-    SI.setOperand(0, NotCond);
+    replaceOperand(SI, 0, NotCond);
     SI.swapValues();
     SI.swapProfMetadata();
     return &SI;
@@ -2826,7 +2822,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   if (Instruction *Select = foldSelectCmpXchg(SI))
     return Select;
 
-  if (Instruction *Select = foldSelectBinOpIdentity(SI, TLI))
+  if (Instruction *Select = foldSelectBinOpIdentity(SI, TLI, *this))
     return Select;
 
   if (Instruction *Rot = foldSelectRotate(SI))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 43be42a409694e1a01e96bcbb578b23002d0933e..49d6443d22770772a45082b00896accee89dfe58 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -388,8 +388,7 @@ Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) {
     // demand the sign bit (and many others) here??
     Value *Rem = Builder.CreateAnd(A, ConstantInt::get(I.getType(), *B - 1),
                                    Op1->getName());
-    I.setOperand(1, Rem);
-    return &I;
+    return replaceOperand(I, 1, Rem);
   }
 
   if (Instruction *Logic = foldShiftOfShiftedLogic(I, Builder))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 57194fecfba6cf6986b1441e0b75e34120ee617c..b9a85e3b4fe5df0bcabdf1f67a4817ebf0fad0b8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -1745,7 +1745,8 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
 
 /// Try to replace a shuffle with an insertelement or try to replace a shuffle
 /// operand with the operand of an insertelement.
-static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf) {
+static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf,
+                                          InstCombiner &IC) {
   // Do not fold if shuffle is a scalable vector.
   if (Shuf.getType()->getVectorIsScalable())
     return nullptr;
@@ -1769,20 +1770,16 @@ static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf) {
   uint64_t IdxC;
   if (match(V0, m_InsertElement(m_Value(X), m_Value(), m_ConstantInt(IdxC)))) {
     // shuf (inselt X, ?, IdxC), ?, Mask --> shuf X, ?, Mask
-    if (none_of(Mask, [IdxC](int MaskElt) { return MaskElt == (int)IdxC; })) {
-      Shuf.setOperand(0, X);
-      return &Shuf;
-    }
+    if (none_of(Mask, [IdxC](int MaskElt) { return MaskElt == (int)IdxC; }))
+      return IC.replaceOperand(Shuf, 0, X);
   }
   if (match(V1, m_InsertElement(m_Value(X), m_Value(), m_ConstantInt(IdxC)))) {
     // Offset the index constant by the vector width because we are checking for
     // accesses to the 2nd vector input of the shuffle.
     IdxC += NumElts;
     // shuf ?, (inselt X, ?, IdxC), Mask --> shuf ?, X, Mask
-    if (none_of(Mask, [IdxC](int MaskElt) { return MaskElt == (int)IdxC; })) {
-      Shuf.setOperand(1, X);
-      return &Shuf;
-    }
+    if (none_of(Mask, [IdxC](int MaskElt) { return MaskElt == (int)IdxC; }))
+      return IC.replaceOperand(Shuf, 1, X);
   }
 
   // shuffle (insert ?, Scalar, IndexC), V1, Mask --> insert V1, Scalar, IndexC'
@@ -1963,7 +1960,7 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
 
   // These transforms have the potential to lose undef knowledge, so they are
   // intentionally placed after SimplifyDemandedVectorElts().
-  if (Instruction *I = foldShuffleWithInsert(SVI))
+  if (Instruction *I = foldShuffleWithInsert(SVI, *this))
     return I;
   if (Instruction *I = foldIdentityPaddedShuffles(SVI))
     return I;
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index e039d97139db30a1e2fedf147723beaa77494d8a..f56a4c5f89200b4627fea21b2ce9afafb541623b 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -29,10 +29,7 @@
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/Analysis/MemorySSA.h"
-#include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/OrderedBasicBlock.h"
-#include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Argument.h"
@@ -43,7 +40,6 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -91,20 +87,11 @@ EnablePartialStoreMerging("enable-dse-partial-store-merging",
   cl::init(true), cl::Hidden,
   cl::desc("Enable partial store merging in DSE"));
 
+// Temporary dummy option for tests.
 static cl::opt<bool>
     EnableMemorySSA("enable-dse-memoryssa", cl::init(false), cl::Hidden,
                     cl::desc("Use the new MemorySSA-backed DSE."));
 
-static cl::opt<unsigned>
-    MemorySSAScanLimit("dse-memoryssa-scanlimit", cl::init(100), cl::Hidden,
-                       cl::desc("The number of memory instructions to scan for "
-                                "dead store elimination (default = 100)"));
-
-static cl::opt<unsigned> MemorySSADefsPerBlockLimit(
-    "dse-memoryssa-defs-per-block-limit", cl::init(5000), cl::Hidden,
-    cl::desc("The number of MemoryDefs we consider as candidates to eliminated "
-             "other stores per basic block (default = 5000)"));
-
 //===----------------------------------------------------------------------===//
 // Helper functions
 //===----------------------------------------------------------------------===//
@@ -1367,490 +1354,22 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis *AA,
   return MadeChange;
 }
 
-namespace {
-//=============================================================================
-// MemorySSA backed dead store elimination.
-//
-// The code below implements dead store elimination using MemorySSA. It uses
-// the following general approach: given a MemoryDef, walk upwards to find
-// clobbering MemoryDefs that may be killed by the starting def. Then check
-// that there are no uses that may read the location of the original MemoryDef
-// in between both MemoryDefs. A bit more concretely:
-//
-// For all MemoryDefs StartDef:
-// 1. Get the next dominating clobbering MemoryDef (DomAccess) by walking
-//    upwards.
-// 2. Check that there are no reads between DomAccess and the StartDef by
-//    checking all uses starting at DomAccess and walking until we see StartDef.
-// 3. For each found DomDef, check that:
-//   1. There are no barrier instructions between DomDef and StartDef (like
-//       throws or stores with ordering constraints).
-//   2. StartDef is executed whenever DomDef is executed.
-//   3. StartDef completely overwrites DomDef.
-// 4. Erase DomDef from the function and MemorySSA.
-
-// Returns true if \p M is an intrisnic that does not read or write memory.
-bool isNoopIntrinsic(MemoryUseOrDef *M) {
-  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(M->getMemoryInst())) {
-    switch (II->getIntrinsicID()) {
-    case Intrinsic::lifetime_start:
-    case Intrinsic::lifetime_end:
-    case Intrinsic::invariant_end:
-    case Intrinsic::launder_invariant_group:
-    case Intrinsic::assume:
-      return true;
-    case Intrinsic::dbg_addr:
-    case Intrinsic::dbg_declare:
-    case Intrinsic::dbg_label:
-    case Intrinsic::dbg_value:
-      llvm_unreachable("Intrinsic should not be modeled in MemorySSA");
-    default:
-      return false;
-    }
-  }
-  return false;
-}
-
-// Check if we can ignore \p D for DSE.
-bool canSkipDef(MemoryDef *D, bool DefVisibleToCaller) {
-  Instruction *DI = D->getMemoryInst();
-  // Calls that only access inaccessible memory cannot read or write any memory
-  // locations we consider for elimination.
-  if (auto CS = CallSite(DI))
-    if (CS.onlyAccessesInaccessibleMemory())
-      return true;
-
-  // We can eliminate stores to locations not visible to the caller across
-  // throwing instructions.
-  if (DI->mayThrow() && !DefVisibleToCaller)
-    return true;
-
-  // We can remove the dead stores, irrespective of the fence and its ordering
-  // (release/acquire/seq_cst). Fences only constraints the ordering of
-  // already visible stores, it does not make a store visible to other
-  // threads. So, skipping over a fence does not change a store from being
-  // dead.
-  if (isa<FenceInst>(DI))
-    return true;
-
-  // Skip intrinsics that do not really read or modify memory.
-  if (isNoopIntrinsic(D))
-    return true;
-
-  return false;
-}
-
-struct DSEState {
-  Function &F;
-  AliasAnalysis &AA;
-  MemorySSA &MSSA;
-  DominatorTree &DT;
-  PostDominatorTree &PDT;
-  const TargetLibraryInfo &TLI;
-
-  // All MemoryDefs that potentially could kill other MemDefs.
-  SmallVector<MemoryDef *, 64> MemDefs;
-  // Any that should be skipped as they are already deleted
-  SmallPtrSet<MemoryAccess *, 4> SkipStores;
-  // Keep track of all of the objects that are invisible to the caller until the
-  // function returns.
-  SmallPtrSet<const Value *, 16> InvisibleToCaller;
-  // Keep track of blocks with throwing instructions not modeled in MemorySSA.
-  SmallPtrSet<BasicBlock *, 16> ThrowingBlocks;
-
-  DSEState(Function &F, AliasAnalysis &AA, MemorySSA &MSSA, DominatorTree &DT,
-           PostDominatorTree &PDT, const TargetLibraryInfo &TLI)
-      : F(F), AA(AA), MSSA(MSSA), DT(DT), PDT(PDT), TLI(TLI) {}
-
-  static DSEState get(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
-                      DominatorTree &DT, PostDominatorTree &PDT,
-                      const TargetLibraryInfo &TLI) {
-    DSEState State(F, AA, MSSA, DT, PDT, TLI);
-    // Collect blocks with throwing instructions not modeled in MemorySSA and
-    // alloc-like objects.
-    for (Instruction &I : instructions(F)) {
-      if (I.mayThrow() && !MSSA.getMemoryAccess(&I))
-        State.ThrowingBlocks.insert(I.getParent());
-
-      auto *MD = dyn_cast_or_null<MemoryDef>(MSSA.getMemoryAccess(&I));
-      if (MD && State.MemDefs.size() < MemorySSADefsPerBlockLimit &&
-          hasAnalyzableMemoryWrite(&I, TLI) && isRemovable(&I))
-        State.MemDefs.push_back(MD);
-
-      // Track alloca and alloca-like objects. Here we care about objects not
-      // visible to the caller during function execution. Alloca objects are
-      // invalid in the caller, for alloca-like objects we ensure that they are
-      // not captured throughout the function.
-      if (isa<AllocaInst>(&I) ||
-          (isAllocLikeFn(&I, &TLI) && !PointerMayBeCaptured(&I, false, true)))
-        State.InvisibleToCaller.insert(&I);
-    }
-    // Treat byval or inalloca arguments the same as Allocas, stores to them are
-    // dead at the end of the function.
-    for (Argument &AI : F.args())
-      if (AI.hasByValOrInAllocaAttr())
-        State.InvisibleToCaller.insert(&AI);
-    return State;
-  }
-
-  Optional<MemoryLocation> getLocForWriteEx(Instruction *I) const {
-    if (!I->mayWriteToMemory())
-      return None;
-
-    if (auto *MTI = dyn_cast<AnyMemIntrinsic>(I))
-      return {MemoryLocation::getForDest(MTI)};
-
-    if (auto CS = CallSite(I)) {
-      if (Function *F = CS.getCalledFunction()) {
-        StringRef FnName = F->getName();
-        if (TLI.has(LibFunc_strcpy) && FnName == TLI.getName(LibFunc_strcpy))
-          return {MemoryLocation(CS.getArgument(0))};
-        if (TLI.has(LibFunc_strncpy) && FnName == TLI.getName(LibFunc_strncpy))
-          return {MemoryLocation(CS.getArgument(0))};
-        if (TLI.has(LibFunc_strcat) && FnName == TLI.getName(LibFunc_strcat))
-          return {MemoryLocation(CS.getArgument(0))};
-        if (TLI.has(LibFunc_strncat) && FnName == TLI.getName(LibFunc_strncat))
-          return {MemoryLocation(CS.getArgument(0))};
-      }
-      return None;
-    }
-
-    return MemoryLocation::getOrNone(I);
-  }
-
-  /// Returns true if \p Use completely overwrites \p DefLoc.
-  bool isCompleteOverwrite(MemoryLocation DefLoc, Instruction *UseInst) const {
-    // UseInst has a MemoryDef associated in MemorySSA. It's possible for a
-    // MemoryDef to not write to memory, e.g. a volatile load is modeled as a
-    // MemoryDef.
-    if (!UseInst->mayWriteToMemory())
-      return false;
-
-    if (auto CS = CallSite(UseInst))
-      if (CS.onlyAccessesInaccessibleMemory())
-        return false;
-
-    ModRefInfo MR = AA.getModRefInfo(UseInst, DefLoc);
-    // If necessary, perform additional analysis.
-    if (isModSet(MR))
-      MR = AA.callCapturesBefore(UseInst, DefLoc, &DT);
-
-    Optional<MemoryLocation> UseLoc = getLocForWriteEx(UseInst);
-    return isModSet(MR) && isMustSet(MR) &&
-           UseLoc->Size.getValue() >= DefLoc.Size.getValue();
-  }
-
-  /// Returns true if \p Use may read from \p DefLoc.
-  bool isReadClobber(MemoryLocation DefLoc, Instruction *UseInst) const {
-    if (!UseInst->mayReadFromMemory())
-      return false;
-
-    if (auto CS = CallSite(UseInst))
-      if (CS.onlyAccessesInaccessibleMemory())
-        return false;
-
-    ModRefInfo MR = AA.getModRefInfo(UseInst, DefLoc);
-    // If necessary, perform additional analysis.
-    if (isRefSet(MR))
-      MR = AA.callCapturesBefore(UseInst, DefLoc, &DT);
-    return isRefSet(MR);
-  }
-
-  // Find a MemoryDef writing to \p DefLoc and dominating \p Current, with no
-  // read access in between or return None otherwise. The returned value may not
-  // (completely) overwrite \p DefLoc. Currently we bail out when we encounter
-  // any of the following
-  //  * An aliasing MemoryUse (read).
-  //  * A MemoryPHI.
-  Optional<MemoryAccess *> getDomMemoryDef(MemoryDef *KillingDef,
-                                           MemoryAccess *Current,
-                                           MemoryLocation DefLoc,
-                                           bool DefVisibleToCaller,
-                                           int &ScanLimit) const {
-    MemoryDef *DomDef;
-    MemoryAccess *StartDef = Current;
-    bool StepAgain;
-    LLVM_DEBUG(dbgs() << "  trying to get dominating access for " << *Current
-                      << "\n");
-    // Find the next clobbering Mod access for DefLoc, starting at Current.
-    do {
-      StepAgain = false;
-      // Reached TOP.
-      if (MSSA.isLiveOnEntryDef(Current))
-        return None;
-
-      MemoryUseOrDef *CurrentUD = dyn_cast<MemoryUseOrDef>(Current);
-      if (!CurrentUD)
-        return None;
-
-      // Look for access that clobber DefLoc.
-      MemoryAccess *DomAccess =
-          MSSA.getSkipSelfWalker()->getClobberingMemoryAccess(
-              CurrentUD->getDefiningAccess(), DefLoc);
-      DomDef = dyn_cast<MemoryDef>(DomAccess);
-      if (!DomDef || MSSA.isLiveOnEntryDef(DomDef))
-        return None;
-
-      // Check if we can skip DomDef for DSE. We also require the KillingDef
-      // execute whenever DomDef executes and use post-dominance to ensure that.
-      if (canSkipDef(DomDef, DefVisibleToCaller) ||
-          !PDT.dominates(KillingDef->getBlock(), DomDef->getBlock())) {
-        StepAgain = true;
-        Current = DomDef;
-      }
-
-    } while (StepAgain);
-
-    LLVM_DEBUG(dbgs() << "  Checking for reads of " << *DomDef << " ("
-                      << *DomDef->getMemoryInst() << ")\n");
-
-    SmallSetVector<MemoryAccess *, 32> WorkList;
-    auto PushMemUses = [&WorkList](MemoryAccess *Acc) {
-      for (Use &U : Acc->uses())
-        WorkList.insert(cast<MemoryAccess>(U.getUser()));
-    };
-    PushMemUses(DomDef);
-
-    // Check if DomDef may be read.
-    for (unsigned I = 0; I < WorkList.size(); I++) {
-      MemoryAccess *UseAccess = WorkList[I];
-
-      LLVM_DEBUG(dbgs() << "   Checking use " << *UseAccess);
-      if (--ScanLimit == 0) {
-        LLVM_DEBUG(dbgs() << "  ...  hit scan limit\n");
-        return None;
-      }
-
-      // Bail out on MemoryPhis for now.
-      if (isa<MemoryPhi>(UseAccess)) {
-        LLVM_DEBUG(dbgs() << "  ...  hit MemoryPhi\n");
-        return None;
-      }
-
-      Instruction *UseInst = cast<MemoryUseOrDef>(UseAccess)->getMemoryInst();
-      LLVM_DEBUG(dbgs() << " (" << *UseInst << ")\n");
-
-      if (isNoopIntrinsic(cast<MemoryUseOrDef>(UseAccess))) {
-        PushMemUses(UseAccess);
-        continue;
-      }
-
-      // Uses which may read the original MemoryDef mean we cannot eliminate the
-      // original MD. Stop walk.
-      if (isReadClobber(DefLoc, UseInst)) {
-        LLVM_DEBUG(dbgs() << "  ... found read clobber\n");
-        return None;
-      }
-
-      if (StartDef == UseAccess)
-        continue;
-
-      // Check all uses for MemoryDefs, except for defs completely overwriting
-      // the original location. Otherwise we have to check uses of *all*
-      // MemoryDefs we discover, including non-aliasing ones. Otherwise we might
-      // miss cases like the following
-      //   1 = Def(LoE) ; <----- DomDef stores [0,1]
-      //   2 = Def(1)   ; (2, 1) = NoAlias,   stores [2,3]
-      //   Use(2)       ; MayAlias 2 *and* 1, loads [0, 3].
-      //                  (The Use points to the *first* Def it may alias)
-      //   3 = Def(1)   ; <---- Current  (3, 2) = NoAlias, (3,1) = MayAlias,
-      //                  stores [0,1]
-      if (MemoryDef *UseDef = dyn_cast<MemoryDef>(UseAccess)) {
-        if (!isCompleteOverwrite(DefLoc, UseInst))
-          PushMemUses(UseDef);
-      }
-    }
-
-    // No aliasing MemoryUses of DomDef found, DomDef is potentially dead.
-    return {DomDef};
-  }
-
-  // Delete dead memory defs
-  void deleteDeadInstruction(Instruction *SI) {
-    MemorySSAUpdater Updater(&MSSA);
-    SmallVector<Instruction *, 32> NowDeadInsts;
-    NowDeadInsts.push_back(SI);
-    --NumFastOther;
-
-    while (!NowDeadInsts.empty()) {
-      Instruction *DeadInst = NowDeadInsts.pop_back_val();
-      ++NumFastOther;
-
-      // Try to preserve debug information attached to the dead instruction.
-      salvageDebugInfo(*DeadInst);
-
-      // Remove the Instruction from MSSA.
-      if (MemoryAccess *MA = MSSA.getMemoryAccess(DeadInst)) {
-        Updater.removeMemoryAccess(MA);
-        if (MemoryDef *MD = dyn_cast<MemoryDef>(MA)) {
-          SkipStores.insert(MD);
-        }
-      }
-
-      // Remove its operands
-      for (Use &O : DeadInst->operands())
-        if (Instruction *OpI = dyn_cast<Instruction>(O)) {
-          O = nullptr;
-          if (isInstructionTriviallyDead(OpI, &TLI))
-            NowDeadInsts.push_back(OpI);
-        }
-
-      DeadInst->eraseFromParent();
-    }
-  }
-
-  // Check for any extra throws between SI and NI that block DSE.  This only
-  // checks extra maythrows (those that aren't MemoryDef's). MemoryDef that may
-  // throw are handled during the walk from one def to the next.
-  bool mayThrowBetween(Instruction *SI, Instruction *NI,
-                       const Value *SILocUnd) const {
-    // First see if we can ignore it by using the fact that SI is an
-    // alloca/alloca like object that is not visible to the caller during
-    // execution of the function.
-    if (SILocUnd && InvisibleToCaller.count(SILocUnd))
-      return false;
-
-    if (SI->getParent() == NI->getParent())
-      return ThrowingBlocks.find(SI->getParent()) != ThrowingBlocks.end();
-    return !ThrowingBlocks.empty();
-  }
-
-  // Check if \p NI acts as a DSE barrier for \p SI. The following instructions
-  // act as barriers:
-  //  * A memory instruction that may throw and \p SI accesses a non-stack
-  //  object.
-  //  * Atomic stores stronger that monotonic.
-  bool isDSEBarrier(Instruction *SI, MemoryLocation &SILoc,
-                    const Value *SILocUnd, Instruction *NI,
-                    MemoryLocation &NILoc) const {
-    // If NI may throw it acts as a barrier, unless we are to an alloca/alloca
-    // like object that does not escape.
-    if (NI->mayThrow() && !InvisibleToCaller.count(SILocUnd))
-      return true;
-
-    if (NI->isAtomic()) {
-      if (auto *NSI = dyn_cast<StoreInst>(NI)) {
-        if (isStrongerThanMonotonic(NSI->getOrdering()))
-          return true;
-      } else
-        llvm_unreachable(
-            "Other instructions should be modeled/skipped in MemorySSA");
-    }
-
-    return false;
-  }
-};
-
-bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
-                                  MemorySSA &MSSA, DominatorTree &DT,
-                                  PostDominatorTree &PDT,
-                                  const TargetLibraryInfo &TLI) {
-  const DataLayout &DL = F.getParent()->getDataLayout();
-  bool MadeChange = false;
-
-  DSEState State = DSEState::get(F, AA, MSSA, DT, PDT, TLI);
-  // For each store:
-  for (unsigned I = 0; I < State.MemDefs.size(); I++) {
-    MemoryDef *Current = State.MemDefs[I];
-    if (State.SkipStores.count(Current))
-      continue;
-    Instruction *SI = cast<MemoryDef>(Current)->getMemoryInst();
-    auto MaybeSILoc = State.getLocForWriteEx(SI);
-    if (!MaybeSILoc) {
-      LLVM_DEBUG(dbgs() << "Failed to find analyzable write location for "
-                        << *SI << "\n");
-      continue;
-    }
-    MemoryLocation SILoc = *MaybeSILoc;
-    assert(SILoc.Ptr && "SILoc should not be null");
-    const Value *SILocUnd = GetUnderlyingObject(SILoc.Ptr, DL);
-    Instruction *DefObj =
-        const_cast<Instruction *>(dyn_cast<Instruction>(SILocUnd));
-    bool DefVisibleToCaller = !State.InvisibleToCaller.count(SILocUnd);
-    if (DefObj && ((isAllocLikeFn(DefObj, &TLI) &&
-                    !PointerMayBeCapturedBefore(DefObj, false, true, SI, &DT))))
-      DefVisibleToCaller = false;
-
-    LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by " << *SI
-                      << "\n");
-
-    int ScanLimit = MemorySSAScanLimit;
-    MemoryDef *StartDef = Current;
-    // Walk MemorySSA upward to find MemoryDefs that might be killed by SI.
-    while (Optional<MemoryAccess *> Next = State.getDomMemoryDef(
-               StartDef, Current, SILoc, DefVisibleToCaller, ScanLimit)) {
-      MemoryAccess *DomAccess = *Next;
-      LLVM_DEBUG(dbgs() << " Checking if we can kill " << *DomAccess << "\n");
-      MemoryDef *NextDef = dyn_cast<MemoryDef>(DomAccess);
-      Instruction *NI = NextDef->getMemoryInst();
-      LLVM_DEBUG(dbgs() << "  def " << *NI << "\n");
-
-      if (!hasAnalyzableMemoryWrite(NI, TLI))
-        break;
-      MemoryLocation NILoc = *State.getLocForWriteEx(NI);
-      // Check for anything that looks like it will be a barrier to further
-      // removal
-      if (State.isDSEBarrier(SI, SILoc, SILocUnd, NI, NILoc)) {
-        LLVM_DEBUG(dbgs() << "  stop, barrier\n");
-        break;
-      }
-
-      // Before we try to remove anything, check for any extra throwing
-      // instructions that block us from DSEing
-      if (State.mayThrowBetween(SI, NI, SILocUnd)) {
-        LLVM_DEBUG(dbgs() << " stop, may throw!\n");
-        break;
-      }
-
-      // Check if NI overwrites SI.
-      int64_t InstWriteOffset, DepWriteOffset;
-      InstOverlapIntervalsTy IOL;
-      OverwriteResult OR = isOverwrite(SILoc, NILoc, DL, TLI, DepWriteOffset,
-                                       InstWriteOffset, NI, IOL, AA, &F);
-
-      if (OR == OW_Complete) {
-        LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *NI
-                          << "\n  KILLER: " << *SI << '\n');
-        State.deleteDeadInstruction(NI);
-        ++NumFastStores;
-        MadeChange = true;
-      } else
-        Current = NextDef;
-    }
-  }
-
-  return MadeChange;
-}
-} // end anonymous namespace
-
 //===----------------------------------------------------------------------===//
 // DSE Pass
 //===----------------------------------------------------------------------===//
 PreservedAnalyses DSEPass::run(Function &F, FunctionAnalysisManager &AM) {
-  AliasAnalysis &AA = AM.getResult<AAManager>(F);
-  const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
-  DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
-
-  if (EnableMemorySSA) {
-    MemorySSA &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
-    PostDominatorTree &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    if (!eliminateDeadStoresMemorySSA(F, AA, MSSA, DT, PDT, TLI))
-      return PreservedAnalyses::all();
-  } else {
-    MemoryDependenceResults &MD = AM.getResult<MemoryDependenceAnalysis>(F);
+  AliasAnalysis *AA = &AM.getResult<AAManager>(F);
+  DominatorTree *DT = &AM.getResult<DominatorTreeAnalysis>(F);
+  MemoryDependenceResults *MD = &AM.getResult<MemoryDependenceAnalysis>(F);
+  const TargetLibraryInfo *TLI = &AM.getResult<TargetLibraryAnalysis>(F);
 
-    if (!eliminateDeadStores(F, &AA, &MD, &DT, &TLI))
-      return PreservedAnalyses::all();
-  }
+  if (!eliminateDeadStores(F, AA, MD, DT, TLI))
+    return PreservedAnalyses::all();
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
   PA.preserve<GlobalsAA>();
-  if (EnableMemorySSA)
-    PA.preserve<MemorySSAAnalysis>();
-  else
-    PA.preserve<MemoryDependenceAnalysis>();
+  PA.preserve<MemoryDependenceAnalysis>();
   return PA;
 }
 
@@ -1869,42 +1388,25 @@ public:
     if (skipFunction(F))
       return false;
 
-    AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
-    DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-    const TargetLibraryInfo &TLI =
-        getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-
-    if (EnableMemorySSA) {
-      MemorySSA &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA();
-      PostDominatorTree &PDT =
-          getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
-
-      return eliminateDeadStoresMemorySSA(F, AA, MSSA, DT, PDT, TLI);
-    } else {
-      MemoryDependenceResults &MD =
-          getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
+    DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    AliasAnalysis *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+    MemoryDependenceResults *MD =
+        &getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
+    const TargetLibraryInfo *TLI =
+        &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
 
-      return eliminateDeadStores(F, &AA, &MD, &DT, &TLI);
-    }
+    return eliminateDeadStores(F, AA, MD, DT, TLI);
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
+    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addRequired<AAResultsWrapperPass>();
+    AU.addRequired<MemoryDependenceWrapperPass>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
-    AU.addPreserved<GlobalsAAWrapperPass>();
-    AU.addRequired<DominatorTreeWrapperPass>();
     AU.addPreserved<DominatorTreeWrapperPass>();
-
-    if (EnableMemorySSA) {
-      AU.addRequired<PostDominatorTreeWrapperPass>();
-      AU.addRequired<MemorySSAWrapperPass>();
-      AU.addPreserved<PostDominatorTreeWrapperPass>();
-      AU.addPreserved<MemorySSAWrapperPass>();
-    } else {
-      AU.addRequired<MemoryDependenceWrapperPass>();
-      AU.addPreserved<MemoryDependenceWrapperPass>();
-    }
+    AU.addPreserved<GlobalsAAWrapperPass>();
+    AU.addPreserved<MemoryDependenceWrapperPass>();
   }
 };
 
@@ -1915,10 +1417,8 @@ char DSELegacyPass::ID = 0;
 INITIALIZE_PASS_BEGIN(DSELegacyPass, "dse", "Dead Store Elimination", false,
                       false)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(DSELegacyPass, "dse", "Dead Store Elimination", false,
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 67c54be684b50bf4c9e757ae230f9a2ff3e9858d..7c572795c29fd2c378a5afe6fa1fdf2e9178e973 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1029,8 +1029,10 @@ bool Vectorizer::vectorizeStoreChain(
     unsigned NewAlign = getOrEnforceKnownAlignment(S0->getPointerOperand(),
                                                    StackAdjustedAlignment,
                                                    DL, S0, nullptr, &DT);
-    if (NewAlign != 0)
+    if (NewAlign >= Alignment.value())
       Alignment = Align(NewAlign);
+    else
+      return false;
   }
 
   if (!TTI.isLegalToVectorizeStoreChain(SzInBytes, Alignment.value(), AS)) {
@@ -1170,8 +1172,12 @@ bool Vectorizer::vectorizeLoadChain(
              vectorizeLoadChain(Chains.second, InstructionsProcessed);
     }
 
-    Alignment = getOrEnforceKnownAlignment(
-        L0->getPointerOperand(), StackAdjustedAlignment, DL, L0, nullptr, &DT);
+    unsigned NewAlign = getOrEnforceKnownAlignment(
+      L0->getPointerOperand(), StackAdjustedAlignment, DL, L0, nullptr, &DT);
+    if (NewAlign >= Alignment)
+      Alignment = NewAlign;
+    else
+      return false;
   }
 
   if (!TTI.isLegalToVectorizeLoadChain(SzInBytes, Alignment, AS)) {
diff --git a/llvm/lib/XRay/FDRTraceWriter.cpp b/llvm/lib/XRay/FDRTraceWriter.cpp
index f50dc19b4be8d91deba67860b6615295f83d9af4..71c09bd4fce45bf300bec34079edd4f602908fd5 100644
--- a/llvm/lib/XRay/FDRTraceWriter.cpp
+++ b/llvm/lib/XRay/FDRTraceWriter.cpp
@@ -20,10 +20,9 @@ namespace {
 template <size_t Index> struct IndexedWriter {
   template <
       class Tuple,
-      typename std::enable_if<
-          (Index <
-           std::tuple_size<typename std::remove_reference<Tuple>::type>::value),
-          int>::type = 0>
+      std::enable_if_t<(Index <
+                        std::tuple_size<std::remove_reference_t<Tuple>>::value),
+                       int> = 0>
   static size_t write(support::endian::Writer &OS, Tuple &&T) {
     OS.write(std::get<Index>(T));
     return sizeof(std::get<Index>(T)) + IndexedWriter<Index + 1>::write(OS, T);
@@ -31,10 +30,9 @@ template <size_t Index> struct IndexedWriter {
 
   template <
       class Tuple,
-      typename std::enable_if<
-          (Index >=
-           std::tuple_size<typename std::remove_reference<Tuple>::type>::value),
-          int>::type = 0>
+      std::enable_if_t<(Index >=
+                        std::tuple_size<std::remove_reference_t<Tuple>>::value),
+                       int> = 0>
   static size_t write(support::endian::Writer &OS, Tuple &&) {
     return 0;
   }
diff --git a/llvm/test/Analysis/BasicAA/phi-values-usage.ll b/llvm/test/Analysis/BasicAA/phi-values-usage.ll
index 22743881e93963d75b5c25080d9faf944856158a..2ed5ea54283bddcd0570d1d1b5175327820c358e 100644
--- a/llvm/test/Analysis/BasicAA/phi-values-usage.ll
+++ b/llvm/test/Analysis/BasicAA/phi-values-usage.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -debug-pass=Executions -memdep -instcombine -disable-output < %s 2>&1 | FileCheck %s -check-prefix=CHECK
 
 ; Check that phi values is not run when it's not already available, and that
-; basicaa is freed after a pass that preserves CFG.
+; basicaa is not freed after a pass that preserves CFG, as it preserves CFG.
 
 ; CHECK: Executing Pass 'Phi Values Analysis'
 ; CHECK: Executing Pass 'Basic Alias Analysis (stateless AA impl)'
@@ -11,9 +11,9 @@
 ; CHECK-MEMCPY-DAG: Freeing Pass 'MemCpy Optimization'
 ; CHECK-DAG: Freeing Pass 'Phi Values Analysis'
 ; CHECK-DAG: Freeing Pass 'Memory Dependence Analysis'
-; CHECK-DAG: Freeing Pass 'Basic Alias Analysis (stateless AA impl)'
+; CHECK-MEMCPY-NOT: Freeing Pass 'Basic Alias Analysis (stateless AA impl)'
 ; CHECK-NOT: Executing Pass 'Phi Values Analysis'
-; CHECK-MEMCPY: Executing Pass 'Basic Alias Analysis (stateless AA impl)'
+; CHECK-NOT: Executing Pass 'Basic Alias Analysis (stateless AA impl)'
 ; CHECK: Executing Pass 'Combine redundant instructions'
 
 target datalayout = "p:8:8-n8"
diff --git a/llvm/test/Analysis/ConstantFolding/vscale.ll b/llvm/test/Analysis/ConstantFolding/vscale.ll
index 2491308910dfb266df770596f8e5dd23a545254f..3a3d6a16c7d664a0e0cb3b6535a197db0a523bf5 100644
--- a/llvm/test/Analysis/ConstantFolding/vscale.ll
+++ b/llvm/test/Analysis/ConstantFolding/vscale.ll
@@ -210,3 +210,14 @@ define <vscale x 4 x i32> @select() {
   %r = select <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> undef
   ret <vscale x 4 x i32> %r
 }
+
+declare <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>)
+
+define <vscale x 16 x i8> @call() {
+; CHECK-LABEL: @call(
+; CHECK-NEXT:    [[R:%.*]] = call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[R]]
+;
+  %r =  call <vscale x 16 x i8> @llvm.sadd.sat.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+  ret <vscale x 16 x i8> %r
+}
diff --git a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/intrinsics.ll
index 923ce600a8bf5100aa19332e474fcbe3b7692378..e9c753f027ab632cb0c4a306d9fc190ed3797df4 100644
--- a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/intrinsics.ll
@@ -42,12 +42,20 @@ define amdgpu_kernel void @mov_dpp8(i32 addrspace(1)* %out, i32 %in) #0 {
   ret void
 }
 
+; CHECK: DIVERGENT: %tmp0 = call i32 @llvm.amdgcn.writelane(i32 0, i32 1, i32 2)
+define amdgpu_kernel void @writelane(i32 addrspace(1)* %out) #0 {
+  %tmp0 = call i32 @llvm.amdgcn.writelane(i32 0, i32 1, i32 2)
+  store i32 %tmp0, i32 addrspace(1)* %out
+  ret void
+}
+
 declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #1
 declare i32 @llvm.amdgcn.permlane16(i32, i32, i32, i32, i1, i1) #1
 declare i32 @llvm.amdgcn.permlanex16(i32, i32, i32, i32, i1, i1) #1
 declare i32 @llvm.amdgcn.mov.dpp.i32(i32, i32, i32, i32, i1) #1
 declare i32 @llvm.amdgcn.mov.dpp8.i32(i32, i32) #1
 declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #1
+declare i32 @llvm.amdgcn.writelane(i32, i32, i32) #1
 
 attributes #0 = { nounwind convergent }
 attributes #1 = { nounwind readnone convergent }
diff --git a/llvm/test/Bitcode/summary_version.ll b/llvm/test/Bitcode/summary_version.ll
index 2a67073713c0b9077175ba06ed6d48740872f04a..98feab6fe2f9950d4f2fa1915b060ccdf5a84a6c 100644
--- a/llvm/test/Bitcode/summary_version.ll
+++ b/llvm/test/Bitcode/summary_version.ll
@@ -2,7 +2,7 @@
 ; RUN: opt  -module-summary  %s -o - | llvm-bcanalyzer -dump | FileCheck %s
 
 ; CHECK: <GLOBALVAL_SUMMARY_BLOCK
-; CHECK: <VERSION op0=8/>
+; CHECK: <VERSION op0=9/>
 
 
 
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 2f19963fe1b6e3bc59ce3f13859ef6aa050d2fe3..fb19741de44ba7a343658a0772b20dc42afd8240 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -122,10 +122,6 @@ if(TARGET LLVMgold)
   set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} LLVMgold)
 endif()
 
-if(TARGET llvm-go)
-  set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} llvm-go)
-endif()
-
 if(TARGET LTO)
   set(LLVM_TEST_DEPENDS ${LLVM_TEST_DEPENDS} LTO)
 endif()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll
index 32bef20c9ffdb2873e921d7e449cc0e28c991539..3cad9f11bdbac4c341eb62e80c94a7a2621ee26d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.dec.ll
@@ -1141,10 +1141,9 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 8, v0
 ; CI-NEXT:    v_mov_b32_e32 v1, 9
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_dec_rtn_u32 v3, v0, v1
+; CI-NEXT:    ds_dec_rtn_u32 v3, v0, v1 offset:8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1160,10 +1159,9 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
 ; VI-NEXT:    v_mov_b32_e32 v1, 9
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    ds_dec_rtn_u32 v3, v0, v1
+; VI-NEXT:    ds_dec_rtn_u32 v3, v0, v1 offset:8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -1708,15 +1706,14 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(i64 addrspa
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
-; CI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 0, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, 9
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1]
+; CI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
@@ -1728,15 +1725,14 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out,
 ;
 ; VI-LABEL: atomic_dec_shl_base_lds_0_i64:
 ; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 16, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 0, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, 9
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1]
+; VI-NEXT:    ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
index 1f42aa8a67bc0a4d33a4a2b216c0ae02b82b92ef..f4b01506ad450053e63a16f19b4fa860f45b0af1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
@@ -506,10 +506,9 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(i32 addrspace(1)* %out,
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 8, v0
 ; CI-NEXT:    v_mov_b32_e32 v1, 9
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_inc_rtn_u32 v3, v0, v1
+; CI-NEXT:    ds_inc_rtn_u32 v3, v0, v1 offset:8
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v0, s2
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
@@ -525,10 +524,9 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(i32 addrspace(1)* %out,
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, 2, v0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 8, v0
 ; VI-NEXT:    v_mov_b32_e32 v1, 9
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    ds_inc_rtn_u32 v3, v0, v1
+; VI-NEXT:    ds_inc_rtn_u32 v3, v0, v1 offset:8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
@@ -544,9 +542,8 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(i32 addrspace(1)* %out,
 ; GFX9-NEXT:    v_add_u32_e32 v2, 2, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
-; GFX9-NEXT:    v_add_u32_e32 v0, 8, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 9
-; GFX9-NEXT:    ds_inc_rtn_u32 v3, v0, v1
+; GFX9-NEXT:    ds_inc_rtn_u32 v3, v0, v1 offset:8
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
@@ -1279,15 +1276,14 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(i32* %ptr) #0
 define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
 ; CI-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; CI:       ; %bb.0:
+; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; CI-NEXT:    v_add_i32_e32 v4, vcc, 2, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; CI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; CI-NEXT:    v_add_i32_e32 v0, vcc, 0, v0
-; CI-NEXT:    v_add_i32_e32 v2, vcc, 16, v0
+; CI-NEXT:    v_add_i32_e32 v2, vcc, 0, v0
 ; CI-NEXT:    v_mov_b32_e32 v0, 9
 ; CI-NEXT:    v_mov_b32_e32 v1, 0
 ; CI-NEXT:    s_mov_b32 m0, -1
-; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
+; CI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v2, s2
 ; CI-NEXT:    v_mov_b32_e32 v3, s3
@@ -1299,15 +1295,14 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(i64 addrspace(1)* %out,
 ;
 ; VI-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v0
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 0, v0
-; VI-NEXT:    v_add_u32_e32 v2, vcc, 16, v0
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 0, v0
 ; VI-NEXT:    v_mov_b32_e32 v0, 9
 ; VI-NEXT:    v_mov_b32_e32 v1, 0
 ; VI-NEXT:    s_mov_b32 m0, -1
-; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
+; VI-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s2
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
@@ -1319,14 +1314,13 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(i64 addrspace(1)* %out,
 ;
 ; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64:
 ; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GFX9-NEXT:    v_add_u32_e32 v4, 2, v0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX9-NEXT:    v_add_u32_e32 v0, 0, v0
-; GFX9-NEXT:    v_add_u32_e32 v2, 16, v0
+; GFX9-NEXT:    v_add_u32_e32 v2, 0, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 9
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1]
+; GFX9-NEXT:    ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:16
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
index 11bb1c27b1d5052c6f51cbffc117b6dc34684322..1b6a1a7427d6f59b85921628a66967b4dae81971 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fold-binop-select.ll
@@ -92,7 +92,7 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) {
 ; IR-NEXT:    [[TMP3:%.*]] = add i32 [[SELECT]], [[TMP1]]
 ; IR-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP3]], [[TMP1]]
 ; IR-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP4]] to float
-; IR-NEXT:    [[TMP6:%.*]] = fdiv fast float 1.000000e+00, [[TMP5]]
+; IR-NEXT:    [[TMP6:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP5]])
 ; IR-NEXT:    [[TMP7:%.*]] = fmul fast float [[TMP6]], 0x41F0000000000000
 ; IR-NEXT:    [[TMP8:%.*]] = fptoui float [[TMP7]] to i32
 ; IR-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
@@ -121,18 +121,15 @@ define i32 @select_sdiv_lhs_opaque_const0_i32(i1 %cond) {
 ; IR-NEXT:    [[TMP32:%.*]] = mul i32 [[TMP31]], [[TMP4]]
 ; IR-NEXT:    [[TMP33:%.*]] = sub i32 1000000, [[TMP32]]
 ; IR-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP33]], [[TMP4]]
-; IR-NEXT:    [[TMP35:%.*]] = select i1 [[TMP34]], i32 -1, i32 0
-; IR-NEXT:    [[TMP36:%.*]] = icmp uge i32 1000000, [[TMP32]]
-; IR-NEXT:    [[TMP37:%.*]] = select i1 [[TMP36]], i32 -1, i32 0
-; IR-NEXT:    [[TMP38:%.*]] = and i32 [[TMP35]], [[TMP37]]
-; IR-NEXT:    [[TMP39:%.*]] = icmp eq i32 [[TMP38]], 0
-; IR-NEXT:    [[TMP40:%.*]] = add i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP41:%.*]] = sub i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP42:%.*]] = select i1 [[TMP39]], i32 [[TMP31]], i32 [[TMP40]]
-; IR-NEXT:    [[TMP43:%.*]] = select i1 [[TMP36]], i32 [[TMP42]], i32 [[TMP41]]
-; IR-NEXT:    [[TMP44:%.*]] = xor i32 [[TMP43]], [[TMP2]]
-; IR-NEXT:    [[TMP45:%.*]] = sub i32 [[TMP44]], [[TMP2]]
-; IR-NEXT:    ret i32 [[TMP45]]
+; IR-NEXT:    [[TMP35:%.*]] = icmp uge i32 1000000, [[TMP32]]
+; IR-NEXT:    [[TMP36:%.*]] = and i1 [[TMP34]], [[TMP35]]
+; IR-NEXT:    [[TMP37:%.*]] = add i32 [[TMP31]], 1
+; IR-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP31]], 1
+; IR-NEXT:    [[TMP39:%.*]] = select i1 [[TMP36]], i32 [[TMP37]], i32 [[TMP31]]
+; IR-NEXT:    [[TMP40:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP38]]
+; IR-NEXT:    [[TMP41:%.*]] = xor i32 [[TMP40]], [[TMP2]]
+; IR-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP41]], [[TMP2]]
+; IR-NEXT:    ret i32 [[TMP42]]
 ;
 ; GCN-LABEL: select_sdiv_lhs_opaque_const0_i32:
 ; GCN:       ; %bb.0:
@@ -190,7 +187,7 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) {
 ; IR-NEXT:    [[TMP3:%.*]] = add i32 [[SELECT]], [[TMP1]]
 ; IR-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP3]], [[TMP1]]
 ; IR-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP4]] to float
-; IR-NEXT:    [[TMP6:%.*]] = fdiv fast float 1.000000e+00, [[TMP5]]
+; IR-NEXT:    [[TMP6:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP5]])
 ; IR-NEXT:    [[TMP7:%.*]] = fmul fast float [[TMP6]], 0x41F0000000000000
 ; IR-NEXT:    [[TMP8:%.*]] = fptoui float [[TMP7]] to i32
 ; IR-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
@@ -219,18 +216,15 @@ define i32 @select_sdiv_lhs_opaque_const1_i32(i1 %cond) {
 ; IR-NEXT:    [[TMP32:%.*]] = mul i32 [[TMP31]], [[TMP4]]
 ; IR-NEXT:    [[TMP33:%.*]] = sub i32 1000000, [[TMP32]]
 ; IR-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP33]], [[TMP4]]
-; IR-NEXT:    [[TMP35:%.*]] = select i1 [[TMP34]], i32 -1, i32 0
-; IR-NEXT:    [[TMP36:%.*]] = icmp uge i32 1000000, [[TMP32]]
-; IR-NEXT:    [[TMP37:%.*]] = select i1 [[TMP36]], i32 -1, i32 0
-; IR-NEXT:    [[TMP38:%.*]] = and i32 [[TMP35]], [[TMP37]]
-; IR-NEXT:    [[TMP39:%.*]] = icmp eq i32 [[TMP38]], 0
-; IR-NEXT:    [[TMP40:%.*]] = add i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP41:%.*]] = sub i32 [[TMP31]], 1
-; IR-NEXT:    [[TMP42:%.*]] = select i1 [[TMP39]], i32 [[TMP31]], i32 [[TMP40]]
-; IR-NEXT:    [[TMP43:%.*]] = select i1 [[TMP36]], i32 [[TMP42]], i32 [[TMP41]]
-; IR-NEXT:    [[TMP44:%.*]] = xor i32 [[TMP43]], [[TMP2]]
-; IR-NEXT:    [[TMP45:%.*]] = sub i32 [[TMP44]], [[TMP2]]
-; IR-NEXT:    ret i32 [[TMP45]]
+; IR-NEXT:    [[TMP35:%.*]] = icmp uge i32 1000000, [[TMP32]]
+; IR-NEXT:    [[TMP36:%.*]] = and i1 [[TMP34]], [[TMP35]]
+; IR-NEXT:    [[TMP37:%.*]] = add i32 [[TMP31]], 1
+; IR-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP31]], 1
+; IR-NEXT:    [[TMP39:%.*]] = select i1 [[TMP36]], i32 [[TMP37]], i32 [[TMP31]]
+; IR-NEXT:    [[TMP40:%.*]] = select i1 [[TMP35]], i32 [[TMP39]], i32 [[TMP38]]
+; IR-NEXT:    [[TMP41:%.*]] = xor i32 [[TMP40]], [[TMP2]]
+; IR-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP41]], [[TMP2]]
+; IR-NEXT:    ret i32 [[TMP42]]
 ;
 ; GCN-LABEL: select_sdiv_lhs_opaque_const1_i32:
 ; GCN:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 9faaa1001ab3fd4bf23d9e6a300368afecd0d435..daf56e41522a20a036b73da951ffd33bc1d3adef 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -1,10 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-- -amdgpu-codegenprepare %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -amdgpu-codegenprepare %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti < %s | FileCheck -check-prefix=GCN %s
 
 define amdgpu_kernel void @udiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-LABEL: @udiv_i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = uitofp i32 [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP2:%.*]] = fdiv fast float 1.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptoui float [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
@@ -34,18 +36,47 @@ define amdgpu_kernel void @udiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], [[Y]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[X]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[X]], [[TMP29]]
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP32]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[TMP35]], 0
-; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP28]], 1
-; CHECK-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP28]], 1
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP36]], i32 [[TMP28]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP33]], i32 [[TMP39]], i32 [[TMP38]]
-; CHECK-NEXT:    store i32 [[TMP40]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp uge i32 [[X]], [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = and i1 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = add i32 [[TMP28]], 1
+; CHECK-NEXT:    [[TMP35:%.*]] = sub i32 [[TMP28]], 1
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP33]], i32 [[TMP34]], i32 [[TMP28]]
+; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP35]]
+; CHECK-NEXT:    store i32 [[TMP37]], i32 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s8, v1
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, s8, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v4
+; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv i32 %x, %y
   store i32 %r, i32 addrspace(1)* %out
   ret void
@@ -54,7 +85,7 @@ define amdgpu_kernel void @udiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 define amdgpu_kernel void @urem_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-LABEL: @urem_i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = uitofp i32 [[Y:%.*]] to float
-; CHECK-NEXT:    [[TMP2:%.*]] = fdiv fast float 1.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP2]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptoui float [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
@@ -84,18 +115,47 @@ define amdgpu_kernel void @urem_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], [[Y]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[X]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp uge i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[X]], [[TMP29]]
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP35:%.*]] = and i32 [[TMP32]], [[TMP34]]
-; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i32 [[TMP35]], 0
-; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP38:%.*]] = add i32 [[TMP30]], [[Y]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP36]], i32 [[TMP30]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP33]], i32 [[TMP39]], i32 [[TMP38]]
-; CHECK-NEXT:    store i32 [[TMP40]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp uge i32 [[X]], [[TMP29]]
+; CHECK-NEXT:    [[TMP33:%.*]] = and i1 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP30]], [[Y]]
+; CHECK-NEXT:    [[TMP35:%.*]] = add i32 [[TMP30]], [[Y]]
+; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP33]], i32 [[TMP34]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP32]], i32 [[TMP36]], i32 [[TMP35]]
+; CHECK-NEXT:    store i32 [[TMP37]], i32 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s9
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s8, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s8, v0
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s9, v1
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s9, v1
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[2:3]
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = urem i32 %x, %y
   store i32 %r, i32 addrspace(1)* %out
   ret void
@@ -111,7 +171,7 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 [[TMP5]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = uitofp i32 [[TMP7]] to float
-; CHECK-NEXT:    [[TMP9:%.*]] = fdiv fast float 1.000000e+00, [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul fast float [[TMP9]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP11:%.*]] = fptoui float [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
@@ -141,20 +201,58 @@ define amdgpu_kernel void @sdiv_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP36:%.*]] = mul i32 [[TMP35]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP6]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = icmp uge i32 [[TMP37]], [[TMP7]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP38]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP6]], [[TMP36]]
-; CHECK-NEXT:    [[TMP41:%.*]] = select i1 [[TMP40]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP42:%.*]] = and i32 [[TMP39]], [[TMP41]]
-; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i32 [[TMP42]], 0
-; CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[TMP35]], 1
-; CHECK-NEXT:    [[TMP45:%.*]] = sub i32 [[TMP35]], 1
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP43]], i32 [[TMP35]], i32 [[TMP44]]
-; CHECK-NEXT:    [[TMP47:%.*]] = select i1 [[TMP40]], i32 [[TMP46]], i32 [[TMP45]]
-; CHECK-NEXT:    [[TMP48:%.*]] = xor i32 [[TMP47]], [[TMP3]]
-; CHECK-NEXT:    [[TMP49:%.*]] = sub i32 [[TMP48]], [[TMP3]]
-; CHECK-NEXT:    store i32 [[TMP49]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP6]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and i1 [[TMP38]], [[TMP39]]
+; CHECK-NEXT:    [[TMP41:%.*]] = add i32 [[TMP35]], 1
+; CHECK-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP35]], 1
+; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP40]], i32 [[TMP41]], i32 [[TMP35]]
+; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP39]], i32 [[TMP43]], i32 [[TMP42]]
+; CHECK-NEXT:    [[TMP45:%.*]] = xor i32 [[TMP44]], [[TMP3]]
+; CHECK-NEXT:    [[TMP46:%.*]] = sub i32 [[TMP45]], [[TMP3]]
+; CHECK-NEXT:    store i32 [[TMP46]], i32 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s8, s3, 31
+; GCN-NEXT:    s_add_i32 s3, s3, s8
+; GCN-NEXT:    s_xor_b32 s9, s3, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    s_ashr_i32 s3, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_xor_b32 s2, s2, s3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s3, s3, s8
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s2, v1
+; GCN-NEXT:    v_cmp_ge_u32_e32 vcc, s2, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v4
+; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s3, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s3, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv i32 %x, %y
   store i32 %r, i32 addrspace(1)* %out
   ret void
@@ -169,7 +267,7 @@ define amdgpu_kernel void @srem_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = uitofp i32 [[TMP6]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP8]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP10:%.*]] = fptoui float [[TMP9]] to i32
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
@@ -199,20 +297,58 @@ define amdgpu_kernel void @srem_i32(i32 addrspace(1)* %out, i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP35:%.*]] = mul i32 [[TMP34]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = sub i32 [[TMP5]], [[TMP35]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = icmp uge i32 [[TMP36]], [[TMP6]]
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP5]], [[TMP35]]
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP41:%.*]] = and i32 [[TMP38]], [[TMP40]]
-; CHECK-NEXT:    [[TMP42:%.*]] = icmp eq i32 [[TMP41]], 0
-; CHECK-NEXT:    [[TMP43:%.*]] = sub i32 [[TMP36]], [[TMP6]]
-; CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[TMP36]], [[TMP6]]
-; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[TMP36]], i32 [[TMP43]]
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP39]], i32 [[TMP45]], i32 [[TMP44]]
-; CHECK-NEXT:    [[TMP47:%.*]] = xor i32 [[TMP46]], [[TMP1]]
-; CHECK-NEXT:    [[TMP48:%.*]] = sub i32 [[TMP47]], [[TMP1]]
-; CHECK-NEXT:    store i32 [[TMP48]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp uge i32 [[TMP5]], [[TMP35]]
+; CHECK-NEXT:    [[TMP39:%.*]] = and i1 [[TMP37]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = sub i32 [[TMP36]], [[TMP6]]
+; CHECK-NEXT:    [[TMP41:%.*]] = add i32 [[TMP36]], [[TMP6]]
+; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP39]], i32 [[TMP40]], i32 [[TMP36]]
+; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP38]], i32 [[TMP42]], i32 [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = xor i32 [[TMP43]], [[TMP1]]
+; CHECK-NEXT:    [[TMP45:%.*]] = sub i32 [[TMP44]], [[TMP1]]
+; CHECK-NEXT:    store i32 [[TMP45]], i32 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s5, 31
+; GCN-NEXT:    s_add_i32 s3, s5, s2
+; GCN-NEXT:    s_xor_b32 s10, s3, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; GCN-NEXT:    s_ashr_i32 s8, s4, 31
+; GCN-NEXT:    s_add_i32 s4, s4, s8
+; GCN-NEXT:    s_xor_b32 s9, s4, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s10
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s10
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s9
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s10
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s9, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v0
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s10, v1
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v1
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[2:3]
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = srem i32 %x, %y
   store i32 %r, i32 addrspace(1)* %out
   ret void
@@ -224,7 +360,7 @@ define amdgpu_kernel void @udiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
@@ -240,6 +376,26 @@ define amdgpu_kernel void @udiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    store i16 [[TMP17]], i16 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s3, s2, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GCN-NEXT:    s_and_b32 s2, s2, 0xffff
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s2
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv i16 %x, %y
   store i16 %r, i16 addrspace(1)* %out
   ret void
@@ -251,7 +407,7 @@ define amdgpu_kernel void @urem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
@@ -269,6 +425,28 @@ define amdgpu_kernel void @urem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    store i16 [[TMP19]], i16 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s2, s4, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    s_and_b32 s3, s4, 0xffff
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %r = urem i16 %x, %y
   store i16 %r, i16 addrspace(1)* %out
   ret void
@@ -283,7 +461,7 @@ define amdgpu_kernel void @sdiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
@@ -300,6 +478,31 @@ define amdgpu_kernel void @sdiv_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    store i16 [[TMP21]], i16 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s1, s0, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_sext_i32_i16 s0, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv i16 %x, %y
   store i16 %r, i16 addrspace(1)* %out
   ret void
@@ -314,7 +517,7 @@ define amdgpu_kernel void @srem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
@@ -333,6 +536,33 @@ define amdgpu_kernel void @srem_i16(i16 addrspace(1)* %out, i16 %x, i16 %y) {
 ; CHECK-NEXT:    store i16 [[TMP23]], i16 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s4, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s2
+; GCN-NEXT:    s_sext_i32_i16 s3, s4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
+; GCN-NEXT:    s_xor_b32 s3, s3, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s3, s3, 30
+; GCN-NEXT:    s_or_b32 s3, s3, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %r = srem i16 %x, %y
   store i16 %r, i16 addrspace(1)* %out
   ret void
@@ -344,7 +574,7 @@ define amdgpu_kernel void @udiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
@@ -360,6 +590,24 @@ define amdgpu_kernel void @udiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    store i8 [[TMP17]], i8 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_i8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v0, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v2, s0
+; GCN-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v1
+; GCN-NEXT:    v_mad_f32 v1, -v1, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv i8 %x, %y
   store i8 %r, i8 addrspace(1)* %out
   ret void
@@ -371,7 +619,7 @@ define amdgpu_kernel void @urem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
@@ -389,6 +637,27 @@ define amdgpu_kernel void @urem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    store i8 [[TMP19]], i8 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_i8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_ubyte1_e32 v0, s4
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v2, s4
+; GCN-NEXT:    s_lshr_b32 s2, s4, 8
+; GCN-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v1
+; GCN-NEXT:    v_mad_f32 v1, -v1, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %r = urem i8 %x, %y
   store i8 %r, i8 addrspace(1)* %out
   ret void
@@ -403,7 +672,7 @@ define amdgpu_kernel void @sdiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
@@ -420,6 +689,31 @@ define amdgpu_kernel void @sdiv_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    store i8 [[TMP21]], i8 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_i8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bfe_i32 s1, s0, 0x80008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_sext_i32_i8 s0, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv i8 %x, %y
   store i8 %r, i8 addrspace(1)* %out
   ret void
@@ -434,7 +728,7 @@ define amdgpu_kernel void @srem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
@@ -453,6 +747,34 @@ define amdgpu_kernel void @srem_i8(i8 addrspace(1)* %out, i8 %x, i8 %y) {
 ; CHECK-NEXT:    store i8 [[TMP23]], i8 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_i8:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bfe_i32 s1, s0, 0x80008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_sext_i32_i8 s3, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
+; GCN-NEXT:    s_xor_b32 s1, s3, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    s_lshr_b32 s2, s0, 8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = srem i8 %x, %y
   store i8 %r, i8 addrspace(1)* %out
   ret void
@@ -463,7 +785,7 @@ define amdgpu_kernel void @udiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP4:%.*]] = fdiv fast float 1.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
@@ -493,148 +815,238 @@ define amdgpu_kernel void @udiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP34]], [[TMP36]]
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i32 [[TMP37]], 0
-; CHECK-NEXT:    [[TMP39:%.*]] = add i32 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP40:%.*]] = sub i32 [[TMP30]], 1
-; CHECK-NEXT:    [[TMP41:%.*]] = select i1 [[TMP38]], i32 [[TMP30]], i32 [[TMP39]]
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], i32 [[TMP41]], i32 [[TMP40]]
-; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x i32> undef, i32 [[TMP42]], i64 0
-; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP46:%.*]] = uitofp i32 [[TMP45]] to float
-; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
-; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP47]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP49:%.*]] = fptoui float [[TMP48]] to i32
-; CHECK-NEXT:    [[TMP50:%.*]] = zext i32 [[TMP49]] to i64
-; CHECK-NEXT:    [[TMP51:%.*]] = zext i32 [[TMP45]] to i64
-; CHECK-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP50]], [[TMP51]]
-; CHECK-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i32
-; CHECK-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP52]], 32
-; CHECK-NEXT:    [[TMP55:%.*]] = trunc i64 [[TMP54]] to i32
-; CHECK-NEXT:    [[TMP56:%.*]] = sub i32 0, [[TMP53]]
-; CHECK-NEXT:    [[TMP57:%.*]] = icmp eq i32 [[TMP55]], 0
-; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], i32 [[TMP56]], i32 [[TMP53]]
-; CHECK-NEXT:    [[TMP59:%.*]] = zext i32 [[TMP58]] to i64
-; CHECK-NEXT:    [[TMP60:%.*]] = zext i32 [[TMP49]] to i64
-; CHECK-NEXT:    [[TMP61:%.*]] = mul i64 [[TMP59]], [[TMP60]]
-; CHECK-NEXT:    [[TMP62:%.*]] = trunc i64 [[TMP61]] to i32
-; CHECK-NEXT:    [[TMP63:%.*]] = lshr i64 [[TMP61]], 32
-; CHECK-NEXT:    [[TMP64:%.*]] = trunc i64 [[TMP63]] to i32
-; CHECK-NEXT:    [[TMP65:%.*]] = add i32 [[TMP49]], [[TMP64]]
-; CHECK-NEXT:    [[TMP66:%.*]] = sub i32 [[TMP49]], [[TMP64]]
-; CHECK-NEXT:    [[TMP67:%.*]] = select i1 [[TMP57]], i32 [[TMP65]], i32 [[TMP66]]
-; CHECK-NEXT:    [[TMP68:%.*]] = zext i32 [[TMP67]] to i64
-; CHECK-NEXT:    [[TMP69:%.*]] = zext i32 [[TMP44]] to i64
-; CHECK-NEXT:    [[TMP70:%.*]] = mul i64 [[TMP68]], [[TMP69]]
-; CHECK-NEXT:    [[TMP71:%.*]] = trunc i64 [[TMP70]] to i32
-; CHECK-NEXT:    [[TMP72:%.*]] = lshr i64 [[TMP70]], 32
-; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP72]] to i32
-; CHECK-NEXT:    [[TMP74:%.*]] = mul i32 [[TMP73]], [[TMP45]]
-; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 [[TMP44]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = icmp uge i32 [[TMP75]], [[TMP45]]
-; CHECK-NEXT:    [[TMP77:%.*]] = select i1 [[TMP76]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP78:%.*]] = icmp uge i32 [[TMP44]], [[TMP74]]
-; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP78]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP80:%.*]] = and i32 [[TMP77]], [[TMP79]]
-; CHECK-NEXT:    [[TMP81:%.*]] = icmp eq i32 [[TMP80]], 0
-; CHECK-NEXT:    [[TMP82:%.*]] = add i32 [[TMP73]], 1
-; CHECK-NEXT:    [[TMP83:%.*]] = sub i32 [[TMP73]], 1
-; CHECK-NEXT:    [[TMP84:%.*]] = select i1 [[TMP81]], i32 [[TMP73]], i32 [[TMP82]]
-; CHECK-NEXT:    [[TMP85:%.*]] = select i1 [[TMP78]], i32 [[TMP84]], i32 [[TMP83]]
-; CHECK-NEXT:    [[TMP86:%.*]] = insertelement <4 x i32> [[TMP43]], i32 [[TMP85]], i64 1
-; CHECK-NEXT:    [[TMP87:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP89:%.*]] = uitofp i32 [[TMP88]] to float
-; CHECK-NEXT:    [[TMP90:%.*]] = fdiv fast float 1.000000e+00, [[TMP89]]
-; CHECK-NEXT:    [[TMP91:%.*]] = fmul fast float [[TMP90]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP92:%.*]] = fptoui float [[TMP91]] to i32
-; CHECK-NEXT:    [[TMP93:%.*]] = zext i32 [[TMP92]] to i64
-; CHECK-NEXT:    [[TMP94:%.*]] = zext i32 [[TMP88]] to i64
-; CHECK-NEXT:    [[TMP95:%.*]] = mul i64 [[TMP93]], [[TMP94]]
-; CHECK-NEXT:    [[TMP96:%.*]] = trunc i64 [[TMP95]] to i32
-; CHECK-NEXT:    [[TMP97:%.*]] = lshr i64 [[TMP95]], 32
-; CHECK-NEXT:    [[TMP98:%.*]] = trunc i64 [[TMP97]] to i32
-; CHECK-NEXT:    [[TMP99:%.*]] = sub i32 0, [[TMP96]]
-; CHECK-NEXT:    [[TMP100:%.*]] = icmp eq i32 [[TMP98]], 0
-; CHECK-NEXT:    [[TMP101:%.*]] = select i1 [[TMP100]], i32 [[TMP99]], i32 [[TMP96]]
-; CHECK-NEXT:    [[TMP102:%.*]] = zext i32 [[TMP101]] to i64
-; CHECK-NEXT:    [[TMP103:%.*]] = zext i32 [[TMP92]] to i64
-; CHECK-NEXT:    [[TMP104:%.*]] = mul i64 [[TMP102]], [[TMP103]]
-; CHECK-NEXT:    [[TMP105:%.*]] = trunc i64 [[TMP104]] to i32
-; CHECK-NEXT:    [[TMP106:%.*]] = lshr i64 [[TMP104]], 32
-; CHECK-NEXT:    [[TMP107:%.*]] = trunc i64 [[TMP106]] to i32
-; CHECK-NEXT:    [[TMP108:%.*]] = add i32 [[TMP92]], [[TMP107]]
-; CHECK-NEXT:    [[TMP109:%.*]] = sub i32 [[TMP92]], [[TMP107]]
-; CHECK-NEXT:    [[TMP110:%.*]] = select i1 [[TMP100]], i32 [[TMP108]], i32 [[TMP109]]
-; CHECK-NEXT:    [[TMP111:%.*]] = zext i32 [[TMP110]] to i64
-; CHECK-NEXT:    [[TMP112:%.*]] = zext i32 [[TMP87]] to i64
-; CHECK-NEXT:    [[TMP113:%.*]] = mul i64 [[TMP111]], [[TMP112]]
-; CHECK-NEXT:    [[TMP114:%.*]] = trunc i64 [[TMP113]] to i32
-; CHECK-NEXT:    [[TMP115:%.*]] = lshr i64 [[TMP113]], 32
-; CHECK-NEXT:    [[TMP116:%.*]] = trunc i64 [[TMP115]] to i32
-; CHECK-NEXT:    [[TMP117:%.*]] = mul i32 [[TMP116]], [[TMP88]]
-; CHECK-NEXT:    [[TMP118:%.*]] = sub i32 [[TMP87]], [[TMP117]]
-; CHECK-NEXT:    [[TMP119:%.*]] = icmp uge i32 [[TMP118]], [[TMP88]]
-; CHECK-NEXT:    [[TMP120:%.*]] = select i1 [[TMP119]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP121:%.*]] = icmp uge i32 [[TMP87]], [[TMP117]]
-; CHECK-NEXT:    [[TMP122:%.*]] = select i1 [[TMP121]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP123:%.*]] = and i32 [[TMP120]], [[TMP122]]
-; CHECK-NEXT:    [[TMP124:%.*]] = icmp eq i32 [[TMP123]], 0
-; CHECK-NEXT:    [[TMP125:%.*]] = add i32 [[TMP116]], 1
-; CHECK-NEXT:    [[TMP126:%.*]] = sub i32 [[TMP116]], 1
-; CHECK-NEXT:    [[TMP127:%.*]] = select i1 [[TMP124]], i32 [[TMP116]], i32 [[TMP125]]
-; CHECK-NEXT:    [[TMP128:%.*]] = select i1 [[TMP121]], i32 [[TMP127]], i32 [[TMP126]]
-; CHECK-NEXT:    [[TMP129:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP128]], i64 2
-; CHECK-NEXT:    [[TMP130:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP131:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP132:%.*]] = uitofp i32 [[TMP131]] to float
-; CHECK-NEXT:    [[TMP133:%.*]] = fdiv fast float 1.000000e+00, [[TMP132]]
-; CHECK-NEXT:    [[TMP134:%.*]] = fmul fast float [[TMP133]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP135:%.*]] = fptoui float [[TMP134]] to i32
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP30]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <4 x i32> undef, i32 [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
+; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
+; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
+; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
+; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
+; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
+; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
+; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
+; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
+; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
+; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
+; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
+; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP70]], 1
+; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP70]], 1
+; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP70]]
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
+; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP79]], i64 1
+; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP83:%.*]] = uitofp i32 [[TMP82]] to float
+; CHECK-NEXT:    [[TMP84:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP83]])
+; CHECK-NEXT:    [[TMP85:%.*]] = fmul fast float [[TMP84]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP86:%.*]] = fptoui float [[TMP85]] to i32
+; CHECK-NEXT:    [[TMP87:%.*]] = zext i32 [[TMP86]] to i64
+; CHECK-NEXT:    [[TMP88:%.*]] = zext i32 [[TMP82]] to i64
+; CHECK-NEXT:    [[TMP89:%.*]] = mul i64 [[TMP87]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = trunc i64 [[TMP89]] to i32
+; CHECK-NEXT:    [[TMP91:%.*]] = lshr i64 [[TMP89]], 32
+; CHECK-NEXT:    [[TMP92:%.*]] = trunc i64 [[TMP91]] to i32
+; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 0, [[TMP90]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp eq i32 [[TMP92]], 0
+; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP94]], i32 [[TMP93]], i32 [[TMP90]]
+; CHECK-NEXT:    [[TMP96:%.*]] = zext i32 [[TMP95]] to i64
+; CHECK-NEXT:    [[TMP97:%.*]] = zext i32 [[TMP86]] to i64
+; CHECK-NEXT:    [[TMP98:%.*]] = mul i64 [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP99:%.*]] = trunc i64 [[TMP98]] to i32
+; CHECK-NEXT:    [[TMP100:%.*]] = lshr i64 [[TMP98]], 32
+; CHECK-NEXT:    [[TMP101:%.*]] = trunc i64 [[TMP100]] to i32
+; CHECK-NEXT:    [[TMP102:%.*]] = add i32 [[TMP86]], [[TMP101]]
+; CHECK-NEXT:    [[TMP103:%.*]] = sub i32 [[TMP86]], [[TMP101]]
+; CHECK-NEXT:    [[TMP104:%.*]] = select i1 [[TMP94]], i32 [[TMP102]], i32 [[TMP103]]
+; CHECK-NEXT:    [[TMP105:%.*]] = zext i32 [[TMP104]] to i64
+; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP81]] to i64
+; CHECK-NEXT:    [[TMP107:%.*]] = mul i64 [[TMP105]], [[TMP106]]
+; CHECK-NEXT:    [[TMP108:%.*]] = trunc i64 [[TMP107]] to i32
+; CHECK-NEXT:    [[TMP109:%.*]] = lshr i64 [[TMP107]], 32
+; CHECK-NEXT:    [[TMP110:%.*]] = trunc i64 [[TMP109]] to i32
+; CHECK-NEXT:    [[TMP111:%.*]] = mul i32 [[TMP110]], [[TMP82]]
+; CHECK-NEXT:    [[TMP112:%.*]] = sub i32 [[TMP81]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = icmp uge i32 [[TMP112]], [[TMP82]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp uge i32 [[TMP81]], [[TMP111]]
+; CHECK-NEXT:    [[TMP115:%.*]] = and i1 [[TMP113]], [[TMP114]]
+; CHECK-NEXT:    [[TMP116:%.*]] = add i32 [[TMP110]], 1
+; CHECK-NEXT:    [[TMP117:%.*]] = sub i32 [[TMP110]], 1
+; CHECK-NEXT:    [[TMP118:%.*]] = select i1 [[TMP115]], i32 [[TMP116]], i32 [[TMP110]]
+; CHECK-NEXT:    [[TMP119:%.*]] = select i1 [[TMP114]], i32 [[TMP118]], i32 [[TMP117]]
+; CHECK-NEXT:    [[TMP120:%.*]] = insertelement <4 x i32> [[TMP80]], i32 [[TMP119]], i64 2
+; CHECK-NEXT:    [[TMP121:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP123:%.*]] = uitofp i32 [[TMP122]] to float
+; CHECK-NEXT:    [[TMP124:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP123]])
+; CHECK-NEXT:    [[TMP125:%.*]] = fmul fast float [[TMP124]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP126:%.*]] = fptoui float [[TMP125]] to i32
+; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
+; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP122]] to i64
+; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
+; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
+; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
+; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
+; CHECK-NEXT:    [[TMP133:%.*]] = sub i32 0, [[TMP130]]
+; CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i32 [[TMP132]], 0
+; CHECK-NEXT:    [[TMP135:%.*]] = select i1 [[TMP134]], i32 [[TMP133]], i32 [[TMP130]]
 ; CHECK-NEXT:    [[TMP136:%.*]] = zext i32 [[TMP135]] to i64
-; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP131]] to i64
+; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP126]] to i64
 ; CHECK-NEXT:    [[TMP138:%.*]] = mul i64 [[TMP136]], [[TMP137]]
 ; CHECK-NEXT:    [[TMP139:%.*]] = trunc i64 [[TMP138]] to i32
 ; CHECK-NEXT:    [[TMP140:%.*]] = lshr i64 [[TMP138]], 32
 ; CHECK-NEXT:    [[TMP141:%.*]] = trunc i64 [[TMP140]] to i32
-; CHECK-NEXT:    [[TMP142:%.*]] = sub i32 0, [[TMP139]]
-; CHECK-NEXT:    [[TMP143:%.*]] = icmp eq i32 [[TMP141]], 0
-; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP143]], i32 [[TMP142]], i32 [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = add i32 [[TMP126]], [[TMP141]]
+; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP126]], [[TMP141]]
+; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP134]], i32 [[TMP142]], i32 [[TMP143]]
 ; CHECK-NEXT:    [[TMP145:%.*]] = zext i32 [[TMP144]] to i64
-; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP135]] to i64
+; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP121]] to i64
 ; CHECK-NEXT:    [[TMP147:%.*]] = mul i64 [[TMP145]], [[TMP146]]
 ; CHECK-NEXT:    [[TMP148:%.*]] = trunc i64 [[TMP147]] to i32
 ; CHECK-NEXT:    [[TMP149:%.*]] = lshr i64 [[TMP147]], 32
 ; CHECK-NEXT:    [[TMP150:%.*]] = trunc i64 [[TMP149]] to i32
-; CHECK-NEXT:    [[TMP151:%.*]] = add i32 [[TMP135]], [[TMP150]]
-; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP135]], [[TMP150]]
-; CHECK-NEXT:    [[TMP153:%.*]] = select i1 [[TMP143]], i32 [[TMP151]], i32 [[TMP152]]
-; CHECK-NEXT:    [[TMP154:%.*]] = zext i32 [[TMP153]] to i64
-; CHECK-NEXT:    [[TMP155:%.*]] = zext i32 [[TMP130]] to i64
-; CHECK-NEXT:    [[TMP156:%.*]] = mul i64 [[TMP154]], [[TMP155]]
-; CHECK-NEXT:    [[TMP157:%.*]] = trunc i64 [[TMP156]] to i32
-; CHECK-NEXT:    [[TMP158:%.*]] = lshr i64 [[TMP156]], 32
-; CHECK-NEXT:    [[TMP159:%.*]] = trunc i64 [[TMP158]] to i32
-; CHECK-NEXT:    [[TMP160:%.*]] = mul i32 [[TMP159]], [[TMP131]]
-; CHECK-NEXT:    [[TMP161:%.*]] = sub i32 [[TMP130]], [[TMP160]]
-; CHECK-NEXT:    [[TMP162:%.*]] = icmp uge i32 [[TMP161]], [[TMP131]]
-; CHECK-NEXT:    [[TMP163:%.*]] = select i1 [[TMP162]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP164:%.*]] = icmp uge i32 [[TMP130]], [[TMP160]]
-; CHECK-NEXT:    [[TMP165:%.*]] = select i1 [[TMP164]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP166:%.*]] = and i32 [[TMP163]], [[TMP165]]
-; CHECK-NEXT:    [[TMP167:%.*]] = icmp eq i32 [[TMP166]], 0
-; CHECK-NEXT:    [[TMP168:%.*]] = add i32 [[TMP159]], 1
-; CHECK-NEXT:    [[TMP169:%.*]] = sub i32 [[TMP159]], 1
-; CHECK-NEXT:    [[TMP170:%.*]] = select i1 [[TMP167]], i32 [[TMP159]], i32 [[TMP168]]
-; CHECK-NEXT:    [[TMP171:%.*]] = select i1 [[TMP164]], i32 [[TMP170]], i32 [[TMP169]]
-; CHECK-NEXT:    [[TMP172:%.*]] = insertelement <4 x i32> [[TMP129]], i32 [[TMP171]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP172]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP151:%.*]] = mul i32 [[TMP150]], [[TMP122]]
+; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP121]], [[TMP151]]
+; CHECK-NEXT:    [[TMP153:%.*]] = icmp uge i32 [[TMP152]], [[TMP122]]
+; CHECK-NEXT:    [[TMP154:%.*]] = icmp uge i32 [[TMP121]], [[TMP151]]
+; CHECK-NEXT:    [[TMP155:%.*]] = and i1 [[TMP153]], [[TMP154]]
+; CHECK-NEXT:    [[TMP156:%.*]] = add i32 [[TMP150]], 1
+; CHECK-NEXT:    [[TMP157:%.*]] = sub i32 [[TMP150]], 1
+; CHECK-NEXT:    [[TMP158:%.*]] = select i1 [[TMP155]], i32 [[TMP156]], i32 [[TMP150]]
+; CHECK-NEXT:    [[TMP159:%.*]] = select i1 [[TMP154]], i32 [[TMP158]], i32 [[TMP157]]
+; CHECK-NEXT:    [[TMP160:%.*]] = insertelement <4 x i32> [[TMP120]], i32 [[TMP159]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP160]], <4 x i32> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_v4i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s6, 0x4f800000
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s19, 0xf000
+; GCN-NEXT:    s_mov_b32 s18, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GCN-NEXT:    v_cvt_f32_u32_e32 v7, s15
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, s6, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s6, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s12
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s12
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, s13
+; GCN-NEXT:    v_mul_lo_u32 v5, v1, s13
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v6, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v6, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s12
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, -1, v0
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s8, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[4:5], s12, v5
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s14
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s8, v3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
+; GCN-NEXT:    s_and_b64 vcc, s[4:5], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-NEXT:    v_mul_f32_e32 v2, s6, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, s13
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, s14
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, s14
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s9, v3
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v3
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v3, v3, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, -1, v1
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v1
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v3, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, s10
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_mul_f32_e32 v3, s6, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, s14
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v7, v3, s15
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, s15
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v5
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v7
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, -1, v2
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v4, v3
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v3, v3, s11
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s10, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v5, v3, s15
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v5
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v4
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s11, v5
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, -1, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 1, v3
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[2:3]
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
   ret void
@@ -645,7 +1057,7 @@ define amdgpu_kernel void @urem_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[Y:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP4:%.*]] = fdiv fast float 1.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
@@ -675,148 +1087,238 @@ define amdgpu_kernel void @urem_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP37:%.*]] = and i32 [[TMP34]], [[TMP36]]
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i32 [[TMP37]], 0
-; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP40:%.*]] = add i32 [[TMP32]], [[TMP2]]
-; CHECK-NEXT:    [[TMP41:%.*]] = select i1 [[TMP38]], i32 [[TMP32]], i32 [[TMP39]]
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP35]], i32 [[TMP41]], i32 [[TMP40]]
-; CHECK-NEXT:    [[TMP43:%.*]] = insertelement <4 x i32> undef, i32 [[TMP42]], i64 0
-; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP46:%.*]] = uitofp i32 [[TMP45]] to float
-; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
-; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP47]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP49:%.*]] = fptoui float [[TMP48]] to i32
-; CHECK-NEXT:    [[TMP50:%.*]] = zext i32 [[TMP49]] to i64
-; CHECK-NEXT:    [[TMP51:%.*]] = zext i32 [[TMP45]] to i64
-; CHECK-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP50]], [[TMP51]]
-; CHECK-NEXT:    [[TMP53:%.*]] = trunc i64 [[TMP52]] to i32
-; CHECK-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP52]], 32
-; CHECK-NEXT:    [[TMP55:%.*]] = trunc i64 [[TMP54]] to i32
-; CHECK-NEXT:    [[TMP56:%.*]] = sub i32 0, [[TMP53]]
-; CHECK-NEXT:    [[TMP57:%.*]] = icmp eq i32 [[TMP55]], 0
-; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], i32 [[TMP56]], i32 [[TMP53]]
-; CHECK-NEXT:    [[TMP59:%.*]] = zext i32 [[TMP58]] to i64
-; CHECK-NEXT:    [[TMP60:%.*]] = zext i32 [[TMP49]] to i64
-; CHECK-NEXT:    [[TMP61:%.*]] = mul i64 [[TMP59]], [[TMP60]]
-; CHECK-NEXT:    [[TMP62:%.*]] = trunc i64 [[TMP61]] to i32
-; CHECK-NEXT:    [[TMP63:%.*]] = lshr i64 [[TMP61]], 32
-; CHECK-NEXT:    [[TMP64:%.*]] = trunc i64 [[TMP63]] to i32
-; CHECK-NEXT:    [[TMP65:%.*]] = add i32 [[TMP49]], [[TMP64]]
-; CHECK-NEXT:    [[TMP66:%.*]] = sub i32 [[TMP49]], [[TMP64]]
-; CHECK-NEXT:    [[TMP67:%.*]] = select i1 [[TMP57]], i32 [[TMP65]], i32 [[TMP66]]
-; CHECK-NEXT:    [[TMP68:%.*]] = zext i32 [[TMP67]] to i64
-; CHECK-NEXT:    [[TMP69:%.*]] = zext i32 [[TMP44]] to i64
-; CHECK-NEXT:    [[TMP70:%.*]] = mul i64 [[TMP68]], [[TMP69]]
-; CHECK-NEXT:    [[TMP71:%.*]] = trunc i64 [[TMP70]] to i32
-; CHECK-NEXT:    [[TMP72:%.*]] = lshr i64 [[TMP70]], 32
-; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP72]] to i32
-; CHECK-NEXT:    [[TMP74:%.*]] = mul i32 [[TMP73]], [[TMP45]]
-; CHECK-NEXT:    [[TMP75:%.*]] = sub i32 [[TMP44]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = icmp uge i32 [[TMP75]], [[TMP45]]
-; CHECK-NEXT:    [[TMP77:%.*]] = select i1 [[TMP76]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP78:%.*]] = icmp uge i32 [[TMP44]], [[TMP74]]
-; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP78]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP80:%.*]] = and i32 [[TMP77]], [[TMP79]]
-; CHECK-NEXT:    [[TMP81:%.*]] = icmp eq i32 [[TMP80]], 0
-; CHECK-NEXT:    [[TMP82:%.*]] = sub i32 [[TMP75]], [[TMP45]]
-; CHECK-NEXT:    [[TMP83:%.*]] = add i32 [[TMP75]], [[TMP45]]
-; CHECK-NEXT:    [[TMP84:%.*]] = select i1 [[TMP81]], i32 [[TMP75]], i32 [[TMP82]]
-; CHECK-NEXT:    [[TMP85:%.*]] = select i1 [[TMP78]], i32 [[TMP84]], i32 [[TMP83]]
-; CHECK-NEXT:    [[TMP86:%.*]] = insertelement <4 x i32> [[TMP43]], i32 [[TMP85]], i64 1
-; CHECK-NEXT:    [[TMP87:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP89:%.*]] = uitofp i32 [[TMP88]] to float
-; CHECK-NEXT:    [[TMP90:%.*]] = fdiv fast float 1.000000e+00, [[TMP89]]
-; CHECK-NEXT:    [[TMP91:%.*]] = fmul fast float [[TMP90]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP92:%.*]] = fptoui float [[TMP91]] to i32
-; CHECK-NEXT:    [[TMP93:%.*]] = zext i32 [[TMP92]] to i64
-; CHECK-NEXT:    [[TMP94:%.*]] = zext i32 [[TMP88]] to i64
-; CHECK-NEXT:    [[TMP95:%.*]] = mul i64 [[TMP93]], [[TMP94]]
-; CHECK-NEXT:    [[TMP96:%.*]] = trunc i64 [[TMP95]] to i32
-; CHECK-NEXT:    [[TMP97:%.*]] = lshr i64 [[TMP95]], 32
-; CHECK-NEXT:    [[TMP98:%.*]] = trunc i64 [[TMP97]] to i32
-; CHECK-NEXT:    [[TMP99:%.*]] = sub i32 0, [[TMP96]]
-; CHECK-NEXT:    [[TMP100:%.*]] = icmp eq i32 [[TMP98]], 0
-; CHECK-NEXT:    [[TMP101:%.*]] = select i1 [[TMP100]], i32 [[TMP99]], i32 [[TMP96]]
-; CHECK-NEXT:    [[TMP102:%.*]] = zext i32 [[TMP101]] to i64
-; CHECK-NEXT:    [[TMP103:%.*]] = zext i32 [[TMP92]] to i64
-; CHECK-NEXT:    [[TMP104:%.*]] = mul i64 [[TMP102]], [[TMP103]]
-; CHECK-NEXT:    [[TMP105:%.*]] = trunc i64 [[TMP104]] to i32
-; CHECK-NEXT:    [[TMP106:%.*]] = lshr i64 [[TMP104]], 32
-; CHECK-NEXT:    [[TMP107:%.*]] = trunc i64 [[TMP106]] to i32
-; CHECK-NEXT:    [[TMP108:%.*]] = add i32 [[TMP92]], [[TMP107]]
-; CHECK-NEXT:    [[TMP109:%.*]] = sub i32 [[TMP92]], [[TMP107]]
-; CHECK-NEXT:    [[TMP110:%.*]] = select i1 [[TMP100]], i32 [[TMP108]], i32 [[TMP109]]
-; CHECK-NEXT:    [[TMP111:%.*]] = zext i32 [[TMP110]] to i64
-; CHECK-NEXT:    [[TMP112:%.*]] = zext i32 [[TMP87]] to i64
-; CHECK-NEXT:    [[TMP113:%.*]] = mul i64 [[TMP111]], [[TMP112]]
-; CHECK-NEXT:    [[TMP114:%.*]] = trunc i64 [[TMP113]] to i32
-; CHECK-NEXT:    [[TMP115:%.*]] = lshr i64 [[TMP113]], 32
-; CHECK-NEXT:    [[TMP116:%.*]] = trunc i64 [[TMP115]] to i32
-; CHECK-NEXT:    [[TMP117:%.*]] = mul i32 [[TMP116]], [[TMP88]]
-; CHECK-NEXT:    [[TMP118:%.*]] = sub i32 [[TMP87]], [[TMP117]]
-; CHECK-NEXT:    [[TMP119:%.*]] = icmp uge i32 [[TMP118]], [[TMP88]]
-; CHECK-NEXT:    [[TMP120:%.*]] = select i1 [[TMP119]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP121:%.*]] = icmp uge i32 [[TMP87]], [[TMP117]]
-; CHECK-NEXT:    [[TMP122:%.*]] = select i1 [[TMP121]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP123:%.*]] = and i32 [[TMP120]], [[TMP122]]
-; CHECK-NEXT:    [[TMP124:%.*]] = icmp eq i32 [[TMP123]], 0
-; CHECK-NEXT:    [[TMP125:%.*]] = sub i32 [[TMP118]], [[TMP88]]
-; CHECK-NEXT:    [[TMP126:%.*]] = add i32 [[TMP118]], [[TMP88]]
-; CHECK-NEXT:    [[TMP127:%.*]] = select i1 [[TMP124]], i32 [[TMP118]], i32 [[TMP125]]
-; CHECK-NEXT:    [[TMP128:%.*]] = select i1 [[TMP121]], i32 [[TMP127]], i32 [[TMP126]]
-; CHECK-NEXT:    [[TMP129:%.*]] = insertelement <4 x i32> [[TMP86]], i32 [[TMP128]], i64 2
-; CHECK-NEXT:    [[TMP130:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP131:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP132:%.*]] = uitofp i32 [[TMP131]] to float
-; CHECK-NEXT:    [[TMP133:%.*]] = fdiv fast float 1.000000e+00, [[TMP132]]
-; CHECK-NEXT:    [[TMP134:%.*]] = fmul fast float [[TMP133]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP135:%.*]] = fptoui float [[TMP134]] to i32
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = sub i32 [[TMP32]], [[TMP2]]
+; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <4 x i32> undef, i32 [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
+; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
+; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
+; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
+; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
+; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
+; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
+; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
+; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
+; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
+; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
+; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
+; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = sub i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP77:%.*]] = add i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP72]]
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
+; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP79]], i64 1
+; CHECK-NEXT:    [[TMP81:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP83:%.*]] = uitofp i32 [[TMP82]] to float
+; CHECK-NEXT:    [[TMP84:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP83]])
+; CHECK-NEXT:    [[TMP85:%.*]] = fmul fast float [[TMP84]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP86:%.*]] = fptoui float [[TMP85]] to i32
+; CHECK-NEXT:    [[TMP87:%.*]] = zext i32 [[TMP86]] to i64
+; CHECK-NEXT:    [[TMP88:%.*]] = zext i32 [[TMP82]] to i64
+; CHECK-NEXT:    [[TMP89:%.*]] = mul i64 [[TMP87]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = trunc i64 [[TMP89]] to i32
+; CHECK-NEXT:    [[TMP91:%.*]] = lshr i64 [[TMP89]], 32
+; CHECK-NEXT:    [[TMP92:%.*]] = trunc i64 [[TMP91]] to i32
+; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 0, [[TMP90]]
+; CHECK-NEXT:    [[TMP94:%.*]] = icmp eq i32 [[TMP92]], 0
+; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP94]], i32 [[TMP93]], i32 [[TMP90]]
+; CHECK-NEXT:    [[TMP96:%.*]] = zext i32 [[TMP95]] to i64
+; CHECK-NEXT:    [[TMP97:%.*]] = zext i32 [[TMP86]] to i64
+; CHECK-NEXT:    [[TMP98:%.*]] = mul i64 [[TMP96]], [[TMP97]]
+; CHECK-NEXT:    [[TMP99:%.*]] = trunc i64 [[TMP98]] to i32
+; CHECK-NEXT:    [[TMP100:%.*]] = lshr i64 [[TMP98]], 32
+; CHECK-NEXT:    [[TMP101:%.*]] = trunc i64 [[TMP100]] to i32
+; CHECK-NEXT:    [[TMP102:%.*]] = add i32 [[TMP86]], [[TMP101]]
+; CHECK-NEXT:    [[TMP103:%.*]] = sub i32 [[TMP86]], [[TMP101]]
+; CHECK-NEXT:    [[TMP104:%.*]] = select i1 [[TMP94]], i32 [[TMP102]], i32 [[TMP103]]
+; CHECK-NEXT:    [[TMP105:%.*]] = zext i32 [[TMP104]] to i64
+; CHECK-NEXT:    [[TMP106:%.*]] = zext i32 [[TMP81]] to i64
+; CHECK-NEXT:    [[TMP107:%.*]] = mul i64 [[TMP105]], [[TMP106]]
+; CHECK-NEXT:    [[TMP108:%.*]] = trunc i64 [[TMP107]] to i32
+; CHECK-NEXT:    [[TMP109:%.*]] = lshr i64 [[TMP107]], 32
+; CHECK-NEXT:    [[TMP110:%.*]] = trunc i64 [[TMP109]] to i32
+; CHECK-NEXT:    [[TMP111:%.*]] = mul i32 [[TMP110]], [[TMP82]]
+; CHECK-NEXT:    [[TMP112:%.*]] = sub i32 [[TMP81]], [[TMP111]]
+; CHECK-NEXT:    [[TMP113:%.*]] = icmp uge i32 [[TMP112]], [[TMP82]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp uge i32 [[TMP81]], [[TMP111]]
+; CHECK-NEXT:    [[TMP115:%.*]] = and i1 [[TMP113]], [[TMP114]]
+; CHECK-NEXT:    [[TMP116:%.*]] = sub i32 [[TMP112]], [[TMP82]]
+; CHECK-NEXT:    [[TMP117:%.*]] = add i32 [[TMP112]], [[TMP82]]
+; CHECK-NEXT:    [[TMP118:%.*]] = select i1 [[TMP115]], i32 [[TMP116]], i32 [[TMP112]]
+; CHECK-NEXT:    [[TMP119:%.*]] = select i1 [[TMP114]], i32 [[TMP118]], i32 [[TMP117]]
+; CHECK-NEXT:    [[TMP120:%.*]] = insertelement <4 x i32> [[TMP80]], i32 [[TMP119]], i64 2
+; CHECK-NEXT:    [[TMP121:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP123:%.*]] = uitofp i32 [[TMP122]] to float
+; CHECK-NEXT:    [[TMP124:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP123]])
+; CHECK-NEXT:    [[TMP125:%.*]] = fmul fast float [[TMP124]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP126:%.*]] = fptoui float [[TMP125]] to i32
+; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
+; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP122]] to i64
+; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
+; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
+; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
+; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
+; CHECK-NEXT:    [[TMP133:%.*]] = sub i32 0, [[TMP130]]
+; CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i32 [[TMP132]], 0
+; CHECK-NEXT:    [[TMP135:%.*]] = select i1 [[TMP134]], i32 [[TMP133]], i32 [[TMP130]]
 ; CHECK-NEXT:    [[TMP136:%.*]] = zext i32 [[TMP135]] to i64
-; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP131]] to i64
+; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP126]] to i64
 ; CHECK-NEXT:    [[TMP138:%.*]] = mul i64 [[TMP136]], [[TMP137]]
 ; CHECK-NEXT:    [[TMP139:%.*]] = trunc i64 [[TMP138]] to i32
 ; CHECK-NEXT:    [[TMP140:%.*]] = lshr i64 [[TMP138]], 32
 ; CHECK-NEXT:    [[TMP141:%.*]] = trunc i64 [[TMP140]] to i32
-; CHECK-NEXT:    [[TMP142:%.*]] = sub i32 0, [[TMP139]]
-; CHECK-NEXT:    [[TMP143:%.*]] = icmp eq i32 [[TMP141]], 0
-; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP143]], i32 [[TMP142]], i32 [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = add i32 [[TMP126]], [[TMP141]]
+; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP126]], [[TMP141]]
+; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP134]], i32 [[TMP142]], i32 [[TMP143]]
 ; CHECK-NEXT:    [[TMP145:%.*]] = zext i32 [[TMP144]] to i64
-; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP135]] to i64
+; CHECK-NEXT:    [[TMP146:%.*]] = zext i32 [[TMP121]] to i64
 ; CHECK-NEXT:    [[TMP147:%.*]] = mul i64 [[TMP145]], [[TMP146]]
 ; CHECK-NEXT:    [[TMP148:%.*]] = trunc i64 [[TMP147]] to i32
 ; CHECK-NEXT:    [[TMP149:%.*]] = lshr i64 [[TMP147]], 32
 ; CHECK-NEXT:    [[TMP150:%.*]] = trunc i64 [[TMP149]] to i32
-; CHECK-NEXT:    [[TMP151:%.*]] = add i32 [[TMP135]], [[TMP150]]
-; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP135]], [[TMP150]]
-; CHECK-NEXT:    [[TMP153:%.*]] = select i1 [[TMP143]], i32 [[TMP151]], i32 [[TMP152]]
-; CHECK-NEXT:    [[TMP154:%.*]] = zext i32 [[TMP153]] to i64
-; CHECK-NEXT:    [[TMP155:%.*]] = zext i32 [[TMP130]] to i64
-; CHECK-NEXT:    [[TMP156:%.*]] = mul i64 [[TMP154]], [[TMP155]]
-; CHECK-NEXT:    [[TMP157:%.*]] = trunc i64 [[TMP156]] to i32
-; CHECK-NEXT:    [[TMP158:%.*]] = lshr i64 [[TMP156]], 32
-; CHECK-NEXT:    [[TMP159:%.*]] = trunc i64 [[TMP158]] to i32
-; CHECK-NEXT:    [[TMP160:%.*]] = mul i32 [[TMP159]], [[TMP131]]
-; CHECK-NEXT:    [[TMP161:%.*]] = sub i32 [[TMP130]], [[TMP160]]
-; CHECK-NEXT:    [[TMP162:%.*]] = icmp uge i32 [[TMP161]], [[TMP131]]
-; CHECK-NEXT:    [[TMP163:%.*]] = select i1 [[TMP162]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP164:%.*]] = icmp uge i32 [[TMP130]], [[TMP160]]
-; CHECK-NEXT:    [[TMP165:%.*]] = select i1 [[TMP164]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP166:%.*]] = and i32 [[TMP163]], [[TMP165]]
-; CHECK-NEXT:    [[TMP167:%.*]] = icmp eq i32 [[TMP166]], 0
-; CHECK-NEXT:    [[TMP168:%.*]] = sub i32 [[TMP161]], [[TMP131]]
-; CHECK-NEXT:    [[TMP169:%.*]] = add i32 [[TMP161]], [[TMP131]]
-; CHECK-NEXT:    [[TMP170:%.*]] = select i1 [[TMP167]], i32 [[TMP161]], i32 [[TMP168]]
-; CHECK-NEXT:    [[TMP171:%.*]] = select i1 [[TMP164]], i32 [[TMP170]], i32 [[TMP169]]
-; CHECK-NEXT:    [[TMP172:%.*]] = insertelement <4 x i32> [[TMP129]], i32 [[TMP171]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP172]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP151:%.*]] = mul i32 [[TMP150]], [[TMP122]]
+; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP121]], [[TMP151]]
+; CHECK-NEXT:    [[TMP153:%.*]] = icmp uge i32 [[TMP152]], [[TMP122]]
+; CHECK-NEXT:    [[TMP154:%.*]] = icmp uge i32 [[TMP121]], [[TMP151]]
+; CHECK-NEXT:    [[TMP155:%.*]] = and i1 [[TMP153]], [[TMP154]]
+; CHECK-NEXT:    [[TMP156:%.*]] = sub i32 [[TMP152]], [[TMP122]]
+; CHECK-NEXT:    [[TMP157:%.*]] = add i32 [[TMP152]], [[TMP122]]
+; CHECK-NEXT:    [[TMP158:%.*]] = select i1 [[TMP155]], i32 [[TMP156]], i32 [[TMP152]]
+; CHECK-NEXT:    [[TMP159:%.*]] = select i1 [[TMP154]], i32 [[TMP158]], i32 [[TMP157]]
+; CHECK-NEXT:    [[TMP160:%.*]] = insertelement <4 x i32> [[TMP120]], i32 [[TMP159]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP160]], <4 x i32> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_v4i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s6, 0x4f800000
+; GCN-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s19, 0xf000
+; GCN-NEXT:    s_mov_b32 s18, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GCN-NEXT:    v_cvt_f32_u32_e32 v7, s15
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, s6, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s6, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s12
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s12
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, s13
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, s13
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s12
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s14
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[4:5], s8, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s13
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s12, v3
+; GCN-NEXT:    v_mul_f32_e32 v2, s6, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s12, v3
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v3
+; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[4:5]
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, s14
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, s14
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[4:5]
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s9, v1
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v1
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, 0, v5
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s13, v3
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s13, v3
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v1, v2
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v1, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s10
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v7
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, v1, s14
+; GCN-NEXT:    v_mul_f32_e32 v1, s6, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v4, v3, s[2:3]
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v5
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v3
+; GCN-NEXT:    v_mul_lo_u32 v4, v2, s15
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, s15
+; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, v2
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, s14, v3
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v4, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v7, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, s11
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s10, v5
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s14, v3
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, s15
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[2:3]
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s11, v5
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s11, v5
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s15, v3
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, s15, v3
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v4, v3, s[2:3]
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GCN-NEXT:    s_endpgm
   %r = urem <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
   ret void
@@ -834,7 +1336,7 @@ define amdgpu_kernel void @sdiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP6]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = xor i32 [[TMP7]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = uitofp i32 [[TMP9]] to float
-; CHECK-NEXT:    [[TMP11:%.*]] = fdiv fast float 1.000000e+00, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP10]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP13:%.*]] = fptoui float [[TMP12]] to i32
 ; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
@@ -864,177 +1366,303 @@ define amdgpu_kernel void @sdiv_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP38:%.*]] = mul i32 [[TMP37]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 [[TMP8]], [[TMP38]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP39]], [[TMP9]]
-; CHECK-NEXT:    [[TMP41:%.*]] = select i1 [[TMP40]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP42:%.*]] = icmp uge i32 [[TMP8]], [[TMP38]]
-; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP42]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP44:%.*]] = and i32 [[TMP41]], [[TMP43]]
-; CHECK-NEXT:    [[TMP45:%.*]] = icmp eq i32 [[TMP44]], 0
-; CHECK-NEXT:    [[TMP46:%.*]] = add i32 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP47:%.*]] = sub i32 [[TMP37]], 1
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP45]], i32 [[TMP37]], i32 [[TMP46]]
-; CHECK-NEXT:    [[TMP49:%.*]] = select i1 [[TMP42]], i32 [[TMP48]], i32 [[TMP47]]
-; CHECK-NEXT:    [[TMP50:%.*]] = xor i32 [[TMP49]], [[TMP5]]
-; CHECK-NEXT:    [[TMP51:%.*]] = sub i32 [[TMP50]], [[TMP5]]
-; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i32> undef, i32 [[TMP51]], i64 0
-; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP55:%.*]] = ashr i32 [[TMP53]], 31
-; CHECK-NEXT:    [[TMP56:%.*]] = ashr i32 [[TMP54]], 31
-; CHECK-NEXT:    [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP56]]
-; CHECK-NEXT:    [[TMP58:%.*]] = add i32 [[TMP53]], [[TMP55]]
-; CHECK-NEXT:    [[TMP59:%.*]] = add i32 [[TMP54]], [[TMP56]]
-; CHECK-NEXT:    [[TMP60:%.*]] = xor i32 [[TMP58]], [[TMP55]]
-; CHECK-NEXT:    [[TMP61:%.*]] = xor i32 [[TMP59]], [[TMP56]]
-; CHECK-NEXT:    [[TMP62:%.*]] = uitofp i32 [[TMP61]] to float
-; CHECK-NEXT:    [[TMP63:%.*]] = fdiv fast float 1.000000e+00, [[TMP62]]
-; CHECK-NEXT:    [[TMP64:%.*]] = fmul fast float [[TMP63]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP65:%.*]] = fptoui float [[TMP64]] to i32
-; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP65]] to i64
-; CHECK-NEXT:    [[TMP67:%.*]] = zext i32 [[TMP61]] to i64
-; CHECK-NEXT:    [[TMP68:%.*]] = mul i64 [[TMP66]], [[TMP67]]
-; CHECK-NEXT:    [[TMP69:%.*]] = trunc i64 [[TMP68]] to i32
-; CHECK-NEXT:    [[TMP70:%.*]] = lshr i64 [[TMP68]], 32
-; CHECK-NEXT:    [[TMP71:%.*]] = trunc i64 [[TMP70]] to i32
-; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 0, [[TMP69]]
-; CHECK-NEXT:    [[TMP73:%.*]] = icmp eq i32 [[TMP71]], 0
-; CHECK-NEXT:    [[TMP74:%.*]] = select i1 [[TMP73]], i32 [[TMP72]], i32 [[TMP69]]
-; CHECK-NEXT:    [[TMP75:%.*]] = zext i32 [[TMP74]] to i64
-; CHECK-NEXT:    [[TMP76:%.*]] = zext i32 [[TMP65]] to i64
-; CHECK-NEXT:    [[TMP77:%.*]] = mul i64 [[TMP75]], [[TMP76]]
-; CHECK-NEXT:    [[TMP78:%.*]] = trunc i64 [[TMP77]] to i32
-; CHECK-NEXT:    [[TMP79:%.*]] = lshr i64 [[TMP77]], 32
-; CHECK-NEXT:    [[TMP80:%.*]] = trunc i64 [[TMP79]] to i32
-; CHECK-NEXT:    [[TMP81:%.*]] = add i32 [[TMP65]], [[TMP80]]
-; CHECK-NEXT:    [[TMP82:%.*]] = sub i32 [[TMP65]], [[TMP80]]
-; CHECK-NEXT:    [[TMP83:%.*]] = select i1 [[TMP73]], i32 [[TMP81]], i32 [[TMP82]]
-; CHECK-NEXT:    [[TMP84:%.*]] = zext i32 [[TMP83]] to i64
-; CHECK-NEXT:    [[TMP85:%.*]] = zext i32 [[TMP60]] to i64
-; CHECK-NEXT:    [[TMP86:%.*]] = mul i64 [[TMP84]], [[TMP85]]
-; CHECK-NEXT:    [[TMP87:%.*]] = trunc i64 [[TMP86]] to i32
-; CHECK-NEXT:    [[TMP88:%.*]] = lshr i64 [[TMP86]], 32
-; CHECK-NEXT:    [[TMP89:%.*]] = trunc i64 [[TMP88]] to i32
-; CHECK-NEXT:    [[TMP90:%.*]] = mul i32 [[TMP89]], [[TMP61]]
-; CHECK-NEXT:    [[TMP91:%.*]] = sub i32 [[TMP60]], [[TMP90]]
-; CHECK-NEXT:    [[TMP92:%.*]] = icmp uge i32 [[TMP91]], [[TMP61]]
-; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP92]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP94:%.*]] = icmp uge i32 [[TMP60]], [[TMP90]]
-; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP94]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP96:%.*]] = and i32 [[TMP93]], [[TMP95]]
-; CHECK-NEXT:    [[TMP97:%.*]] = icmp eq i32 [[TMP96]], 0
-; CHECK-NEXT:    [[TMP98:%.*]] = add i32 [[TMP89]], 1
-; CHECK-NEXT:    [[TMP99:%.*]] = sub i32 [[TMP89]], 1
-; CHECK-NEXT:    [[TMP100:%.*]] = select i1 [[TMP97]], i32 [[TMP89]], i32 [[TMP98]]
-; CHECK-NEXT:    [[TMP101:%.*]] = select i1 [[TMP94]], i32 [[TMP100]], i32 [[TMP99]]
-; CHECK-NEXT:    [[TMP102:%.*]] = xor i32 [[TMP101]], [[TMP57]]
-; CHECK-NEXT:    [[TMP103:%.*]] = sub i32 [[TMP102]], [[TMP57]]
-; CHECK-NEXT:    [[TMP104:%.*]] = insertelement <4 x i32> [[TMP52]], i32 [[TMP103]], i64 1
-; CHECK-NEXT:    [[TMP105:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP106:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP107:%.*]] = ashr i32 [[TMP105]], 31
-; CHECK-NEXT:    [[TMP108:%.*]] = ashr i32 [[TMP106]], 31
-; CHECK-NEXT:    [[TMP109:%.*]] = xor i32 [[TMP107]], [[TMP108]]
-; CHECK-NEXT:    [[TMP110:%.*]] = add i32 [[TMP105]], [[TMP107]]
-; CHECK-NEXT:    [[TMP111:%.*]] = add i32 [[TMP106]], [[TMP108]]
-; CHECK-NEXT:    [[TMP112:%.*]] = xor i32 [[TMP110]], [[TMP107]]
-; CHECK-NEXT:    [[TMP113:%.*]] = xor i32 [[TMP111]], [[TMP108]]
-; CHECK-NEXT:    [[TMP114:%.*]] = uitofp i32 [[TMP113]] to float
-; CHECK-NEXT:    [[TMP115:%.*]] = fdiv fast float 1.000000e+00, [[TMP114]]
-; CHECK-NEXT:    [[TMP116:%.*]] = fmul fast float [[TMP115]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP117:%.*]] = fptoui float [[TMP116]] to i32
-; CHECK-NEXT:    [[TMP118:%.*]] = zext i32 [[TMP117]] to i64
-; CHECK-NEXT:    [[TMP119:%.*]] = zext i32 [[TMP113]] to i64
-; CHECK-NEXT:    [[TMP120:%.*]] = mul i64 [[TMP118]], [[TMP119]]
-; CHECK-NEXT:    [[TMP121:%.*]] = trunc i64 [[TMP120]] to i32
-; CHECK-NEXT:    [[TMP122:%.*]] = lshr i64 [[TMP120]], 32
-; CHECK-NEXT:    [[TMP123:%.*]] = trunc i64 [[TMP122]] to i32
-; CHECK-NEXT:    [[TMP124:%.*]] = sub i32 0, [[TMP121]]
-; CHECK-NEXT:    [[TMP125:%.*]] = icmp eq i32 [[TMP123]], 0
-; CHECK-NEXT:    [[TMP126:%.*]] = select i1 [[TMP125]], i32 [[TMP124]], i32 [[TMP121]]
-; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
-; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP117]] to i64
-; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
-; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
-; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
-; CHECK-NEXT:    [[TMP133:%.*]] = add i32 [[TMP117]], [[TMP132]]
-; CHECK-NEXT:    [[TMP134:%.*]] = sub i32 [[TMP117]], [[TMP132]]
-; CHECK-NEXT:    [[TMP135:%.*]] = select i1 [[TMP125]], i32 [[TMP133]], i32 [[TMP134]]
-; CHECK-NEXT:    [[TMP136:%.*]] = zext i32 [[TMP135]] to i64
-; CHECK-NEXT:    [[TMP137:%.*]] = zext i32 [[TMP112]] to i64
-; CHECK-NEXT:    [[TMP138:%.*]] = mul i64 [[TMP136]], [[TMP137]]
-; CHECK-NEXT:    [[TMP139:%.*]] = trunc i64 [[TMP138]] to i32
-; CHECK-NEXT:    [[TMP140:%.*]] = lshr i64 [[TMP138]], 32
-; CHECK-NEXT:    [[TMP141:%.*]] = trunc i64 [[TMP140]] to i32
-; CHECK-NEXT:    [[TMP142:%.*]] = mul i32 [[TMP141]], [[TMP113]]
-; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP112]], [[TMP142]]
-; CHECK-NEXT:    [[TMP144:%.*]] = icmp uge i32 [[TMP143]], [[TMP113]]
-; CHECK-NEXT:    [[TMP145:%.*]] = select i1 [[TMP144]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP146:%.*]] = icmp uge i32 [[TMP112]], [[TMP142]]
-; CHECK-NEXT:    [[TMP147:%.*]] = select i1 [[TMP146]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP148:%.*]] = and i32 [[TMP145]], [[TMP147]]
-; CHECK-NEXT:    [[TMP149:%.*]] = icmp eq i32 [[TMP148]], 0
-; CHECK-NEXT:    [[TMP150:%.*]] = add i32 [[TMP141]], 1
-; CHECK-NEXT:    [[TMP151:%.*]] = sub i32 [[TMP141]], 1
-; CHECK-NEXT:    [[TMP152:%.*]] = select i1 [[TMP149]], i32 [[TMP141]], i32 [[TMP150]]
-; CHECK-NEXT:    [[TMP153:%.*]] = select i1 [[TMP146]], i32 [[TMP152]], i32 [[TMP151]]
-; CHECK-NEXT:    [[TMP154:%.*]] = xor i32 [[TMP153]], [[TMP109]]
-; CHECK-NEXT:    [[TMP155:%.*]] = sub i32 [[TMP154]], [[TMP109]]
-; CHECK-NEXT:    [[TMP156:%.*]] = insertelement <4 x i32> [[TMP104]], i32 [[TMP155]], i64 2
-; CHECK-NEXT:    [[TMP157:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP158:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP159:%.*]] = ashr i32 [[TMP157]], 31
-; CHECK-NEXT:    [[TMP160:%.*]] = ashr i32 [[TMP158]], 31
-; CHECK-NEXT:    [[TMP161:%.*]] = xor i32 [[TMP159]], [[TMP160]]
-; CHECK-NEXT:    [[TMP162:%.*]] = add i32 [[TMP157]], [[TMP159]]
-; CHECK-NEXT:    [[TMP163:%.*]] = add i32 [[TMP158]], [[TMP160]]
-; CHECK-NEXT:    [[TMP164:%.*]] = xor i32 [[TMP162]], [[TMP159]]
-; CHECK-NEXT:    [[TMP165:%.*]] = xor i32 [[TMP163]], [[TMP160]]
-; CHECK-NEXT:    [[TMP166:%.*]] = uitofp i32 [[TMP165]] to float
-; CHECK-NEXT:    [[TMP167:%.*]] = fdiv fast float 1.000000e+00, [[TMP166]]
-; CHECK-NEXT:    [[TMP168:%.*]] = fmul fast float [[TMP167]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP169:%.*]] = fptoui float [[TMP168]] to i32
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp uge i32 [[TMP8]], [[TMP38]]
+; CHECK-NEXT:    [[TMP42:%.*]] = and i1 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP37]], 1
+; CHECK-NEXT:    [[TMP44:%.*]] = sub i32 [[TMP37]], 1
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[TMP43]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP41]], i32 [[TMP45]], i32 [[TMP44]]
+; CHECK-NEXT:    [[TMP47:%.*]] = xor i32 [[TMP46]], [[TMP5]]
+; CHECK-NEXT:    [[TMP48:%.*]] = sub i32 [[TMP47]], [[TMP5]]
+; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <4 x i32> undef, i32 [[TMP48]], i64 0
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP53:%.*]] = ashr i32 [[TMP51]], 31
+; CHECK-NEXT:    [[TMP54:%.*]] = xor i32 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = add i32 [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[TMP51]], [[TMP53]]
+; CHECK-NEXT:    [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP52]]
+; CHECK-NEXT:    [[TMP58:%.*]] = xor i32 [[TMP56]], [[TMP53]]
+; CHECK-NEXT:    [[TMP59:%.*]] = uitofp i32 [[TMP58]] to float
+; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP59]])
+; CHECK-NEXT:    [[TMP61:%.*]] = fmul fast float [[TMP60]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP62:%.*]] = fptoui float [[TMP61]] to i32
+; CHECK-NEXT:    [[TMP63:%.*]] = zext i32 [[TMP62]] to i64
+; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP58]] to i64
+; CHECK-NEXT:    [[TMP65:%.*]] = mul i64 [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
+; CHECK-NEXT:    [[TMP67:%.*]] = lshr i64 [[TMP65]], 32
+; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP69:%.*]] = sub i32 0, [[TMP66]]
+; CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i32 [[TMP68]], 0
+; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP69]], i32 [[TMP66]]
+; CHECK-NEXT:    [[TMP72:%.*]] = zext i32 [[TMP71]] to i64
+; CHECK-NEXT:    [[TMP73:%.*]] = zext i32 [[TMP62]] to i64
+; CHECK-NEXT:    [[TMP74:%.*]] = mul i64 [[TMP72]], [[TMP73]]
+; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
+; CHECK-NEXT:    [[TMP76:%.*]] = lshr i64 [[TMP74]], 32
+; CHECK-NEXT:    [[TMP77:%.*]] = trunc i64 [[TMP76]] to i32
+; CHECK-NEXT:    [[TMP78:%.*]] = add i32 [[TMP62]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = sub i32 [[TMP62]], [[TMP77]]
+; CHECK-NEXT:    [[TMP80:%.*]] = select i1 [[TMP70]], i32 [[TMP78]], i32 [[TMP79]]
+; CHECK-NEXT:    [[TMP81:%.*]] = zext i32 [[TMP80]] to i64
+; CHECK-NEXT:    [[TMP82:%.*]] = zext i32 [[TMP57]] to i64
+; CHECK-NEXT:    [[TMP83:%.*]] = mul i64 [[TMP81]], [[TMP82]]
+; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
+; CHECK-NEXT:    [[TMP85:%.*]] = lshr i64 [[TMP83]], 32
+; CHECK-NEXT:    [[TMP86:%.*]] = trunc i64 [[TMP85]] to i32
+; CHECK-NEXT:    [[TMP87:%.*]] = mul i32 [[TMP86]], [[TMP58]]
+; CHECK-NEXT:    [[TMP88:%.*]] = sub i32 [[TMP57]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = icmp uge i32 [[TMP88]], [[TMP58]]
+; CHECK-NEXT:    [[TMP90:%.*]] = icmp uge i32 [[TMP57]], [[TMP87]]
+; CHECK-NEXT:    [[TMP91:%.*]] = and i1 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = add i32 [[TMP86]], 1
+; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 [[TMP86]], 1
+; CHECK-NEXT:    [[TMP94:%.*]] = select i1 [[TMP91]], i32 [[TMP92]], i32 [[TMP86]]
+; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP90]], i32 [[TMP94]], i32 [[TMP93]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor i32 [[TMP95]], [[TMP54]]
+; CHECK-NEXT:    [[TMP97:%.*]] = sub i32 [[TMP96]], [[TMP54]]
+; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <4 x i32> [[TMP49]], i32 [[TMP97]], i64 1
+; CHECK-NEXT:    [[TMP99:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP101:%.*]] = ashr i32 [[TMP99]], 31
+; CHECK-NEXT:    [[TMP102:%.*]] = ashr i32 [[TMP100]], 31
+; CHECK-NEXT:    [[TMP103:%.*]] = xor i32 [[TMP101]], [[TMP102]]
+; CHECK-NEXT:    [[TMP104:%.*]] = add i32 [[TMP99]], [[TMP101]]
+; CHECK-NEXT:    [[TMP105:%.*]] = add i32 [[TMP100]], [[TMP102]]
+; CHECK-NEXT:    [[TMP106:%.*]] = xor i32 [[TMP104]], [[TMP101]]
+; CHECK-NEXT:    [[TMP107:%.*]] = xor i32 [[TMP105]], [[TMP102]]
+; CHECK-NEXT:    [[TMP108:%.*]] = uitofp i32 [[TMP107]] to float
+; CHECK-NEXT:    [[TMP109:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP108]])
+; CHECK-NEXT:    [[TMP110:%.*]] = fmul fast float [[TMP109]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP111:%.*]] = fptoui float [[TMP110]] to i32
+; CHECK-NEXT:    [[TMP112:%.*]] = zext i32 [[TMP111]] to i64
+; CHECK-NEXT:    [[TMP113:%.*]] = zext i32 [[TMP107]] to i64
+; CHECK-NEXT:    [[TMP114:%.*]] = mul i64 [[TMP112]], [[TMP113]]
+; CHECK-NEXT:    [[TMP115:%.*]] = trunc i64 [[TMP114]] to i32
+; CHECK-NEXT:    [[TMP116:%.*]] = lshr i64 [[TMP114]], 32
+; CHECK-NEXT:    [[TMP117:%.*]] = trunc i64 [[TMP116]] to i32
+; CHECK-NEXT:    [[TMP118:%.*]] = sub i32 0, [[TMP115]]
+; CHECK-NEXT:    [[TMP119:%.*]] = icmp eq i32 [[TMP117]], 0
+; CHECK-NEXT:    [[TMP120:%.*]] = select i1 [[TMP119]], i32 [[TMP118]], i32 [[TMP115]]
+; CHECK-NEXT:    [[TMP121:%.*]] = zext i32 [[TMP120]] to i64
+; CHECK-NEXT:    [[TMP122:%.*]] = zext i32 [[TMP111]] to i64
+; CHECK-NEXT:    [[TMP123:%.*]] = mul i64 [[TMP121]], [[TMP122]]
+; CHECK-NEXT:    [[TMP124:%.*]] = trunc i64 [[TMP123]] to i32
+; CHECK-NEXT:    [[TMP125:%.*]] = lshr i64 [[TMP123]], 32
+; CHECK-NEXT:    [[TMP126:%.*]] = trunc i64 [[TMP125]] to i32
+; CHECK-NEXT:    [[TMP127:%.*]] = add i32 [[TMP111]], [[TMP126]]
+; CHECK-NEXT:    [[TMP128:%.*]] = sub i32 [[TMP111]], [[TMP126]]
+; CHECK-NEXT:    [[TMP129:%.*]] = select i1 [[TMP119]], i32 [[TMP127]], i32 [[TMP128]]
+; CHECK-NEXT:    [[TMP130:%.*]] = zext i32 [[TMP129]] to i64
+; CHECK-NEXT:    [[TMP131:%.*]] = zext i32 [[TMP106]] to i64
+; CHECK-NEXT:    [[TMP132:%.*]] = mul i64 [[TMP130]], [[TMP131]]
+; CHECK-NEXT:    [[TMP133:%.*]] = trunc i64 [[TMP132]] to i32
+; CHECK-NEXT:    [[TMP134:%.*]] = lshr i64 [[TMP132]], 32
+; CHECK-NEXT:    [[TMP135:%.*]] = trunc i64 [[TMP134]] to i32
+; CHECK-NEXT:    [[TMP136:%.*]] = mul i32 [[TMP135]], [[TMP107]]
+; CHECK-NEXT:    [[TMP137:%.*]] = sub i32 [[TMP106]], [[TMP136]]
+; CHECK-NEXT:    [[TMP138:%.*]] = icmp uge i32 [[TMP137]], [[TMP107]]
+; CHECK-NEXT:    [[TMP139:%.*]] = icmp uge i32 [[TMP106]], [[TMP136]]
+; CHECK-NEXT:    [[TMP140:%.*]] = and i1 [[TMP138]], [[TMP139]]
+; CHECK-NEXT:    [[TMP141:%.*]] = add i32 [[TMP135]], 1
+; CHECK-NEXT:    [[TMP142:%.*]] = sub i32 [[TMP135]], 1
+; CHECK-NEXT:    [[TMP143:%.*]] = select i1 [[TMP140]], i32 [[TMP141]], i32 [[TMP135]]
+; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP139]], i32 [[TMP143]], i32 [[TMP142]]
+; CHECK-NEXT:    [[TMP145:%.*]] = xor i32 [[TMP144]], [[TMP103]]
+; CHECK-NEXT:    [[TMP146:%.*]] = sub i32 [[TMP145]], [[TMP103]]
+; CHECK-NEXT:    [[TMP147:%.*]] = insertelement <4 x i32> [[TMP98]], i32 [[TMP146]], i64 2
+; CHECK-NEXT:    [[TMP148:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP149:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP150:%.*]] = ashr i32 [[TMP148]], 31
+; CHECK-NEXT:    [[TMP151:%.*]] = ashr i32 [[TMP149]], 31
+; CHECK-NEXT:    [[TMP152:%.*]] = xor i32 [[TMP150]], [[TMP151]]
+; CHECK-NEXT:    [[TMP153:%.*]] = add i32 [[TMP148]], [[TMP150]]
+; CHECK-NEXT:    [[TMP154:%.*]] = add i32 [[TMP149]], [[TMP151]]
+; CHECK-NEXT:    [[TMP155:%.*]] = xor i32 [[TMP153]], [[TMP150]]
+; CHECK-NEXT:    [[TMP156:%.*]] = xor i32 [[TMP154]], [[TMP151]]
+; CHECK-NEXT:    [[TMP157:%.*]] = uitofp i32 [[TMP156]] to float
+; CHECK-NEXT:    [[TMP158:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP157]])
+; CHECK-NEXT:    [[TMP159:%.*]] = fmul fast float [[TMP158]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP160:%.*]] = fptoui float [[TMP159]] to i32
+; CHECK-NEXT:    [[TMP161:%.*]] = zext i32 [[TMP160]] to i64
+; CHECK-NEXT:    [[TMP162:%.*]] = zext i32 [[TMP156]] to i64
+; CHECK-NEXT:    [[TMP163:%.*]] = mul i64 [[TMP161]], [[TMP162]]
+; CHECK-NEXT:    [[TMP164:%.*]] = trunc i64 [[TMP163]] to i32
+; CHECK-NEXT:    [[TMP165:%.*]] = lshr i64 [[TMP163]], 32
+; CHECK-NEXT:    [[TMP166:%.*]] = trunc i64 [[TMP165]] to i32
+; CHECK-NEXT:    [[TMP167:%.*]] = sub i32 0, [[TMP164]]
+; CHECK-NEXT:    [[TMP168:%.*]] = icmp eq i32 [[TMP166]], 0
+; CHECK-NEXT:    [[TMP169:%.*]] = select i1 [[TMP168]], i32 [[TMP167]], i32 [[TMP164]]
 ; CHECK-NEXT:    [[TMP170:%.*]] = zext i32 [[TMP169]] to i64
-; CHECK-NEXT:    [[TMP171:%.*]] = zext i32 [[TMP165]] to i64
+; CHECK-NEXT:    [[TMP171:%.*]] = zext i32 [[TMP160]] to i64
 ; CHECK-NEXT:    [[TMP172:%.*]] = mul i64 [[TMP170]], [[TMP171]]
 ; CHECK-NEXT:    [[TMP173:%.*]] = trunc i64 [[TMP172]] to i32
 ; CHECK-NEXT:    [[TMP174:%.*]] = lshr i64 [[TMP172]], 32
 ; CHECK-NEXT:    [[TMP175:%.*]] = trunc i64 [[TMP174]] to i32
-; CHECK-NEXT:    [[TMP176:%.*]] = sub i32 0, [[TMP173]]
-; CHECK-NEXT:    [[TMP177:%.*]] = icmp eq i32 [[TMP175]], 0
-; CHECK-NEXT:    [[TMP178:%.*]] = select i1 [[TMP177]], i32 [[TMP176]], i32 [[TMP173]]
+; CHECK-NEXT:    [[TMP176:%.*]] = add i32 [[TMP160]], [[TMP175]]
+; CHECK-NEXT:    [[TMP177:%.*]] = sub i32 [[TMP160]], [[TMP175]]
+; CHECK-NEXT:    [[TMP178:%.*]] = select i1 [[TMP168]], i32 [[TMP176]], i32 [[TMP177]]
 ; CHECK-NEXT:    [[TMP179:%.*]] = zext i32 [[TMP178]] to i64
-; CHECK-NEXT:    [[TMP180:%.*]] = zext i32 [[TMP169]] to i64
+; CHECK-NEXT:    [[TMP180:%.*]] = zext i32 [[TMP155]] to i64
 ; CHECK-NEXT:    [[TMP181:%.*]] = mul i64 [[TMP179]], [[TMP180]]
 ; CHECK-NEXT:    [[TMP182:%.*]] = trunc i64 [[TMP181]] to i32
 ; CHECK-NEXT:    [[TMP183:%.*]] = lshr i64 [[TMP181]], 32
 ; CHECK-NEXT:    [[TMP184:%.*]] = trunc i64 [[TMP183]] to i32
-; CHECK-NEXT:    [[TMP185:%.*]] = add i32 [[TMP169]], [[TMP184]]
-; CHECK-NEXT:    [[TMP186:%.*]] = sub i32 [[TMP169]], [[TMP184]]
-; CHECK-NEXT:    [[TMP187:%.*]] = select i1 [[TMP177]], i32 [[TMP185]], i32 [[TMP186]]
-; CHECK-NEXT:    [[TMP188:%.*]] = zext i32 [[TMP187]] to i64
-; CHECK-NEXT:    [[TMP189:%.*]] = zext i32 [[TMP164]] to i64
-; CHECK-NEXT:    [[TMP190:%.*]] = mul i64 [[TMP188]], [[TMP189]]
-; CHECK-NEXT:    [[TMP191:%.*]] = trunc i64 [[TMP190]] to i32
-; CHECK-NEXT:    [[TMP192:%.*]] = lshr i64 [[TMP190]], 32
-; CHECK-NEXT:    [[TMP193:%.*]] = trunc i64 [[TMP192]] to i32
-; CHECK-NEXT:    [[TMP194:%.*]] = mul i32 [[TMP193]], [[TMP165]]
-; CHECK-NEXT:    [[TMP195:%.*]] = sub i32 [[TMP164]], [[TMP194]]
-; CHECK-NEXT:    [[TMP196:%.*]] = icmp uge i32 [[TMP195]], [[TMP165]]
-; CHECK-NEXT:    [[TMP197:%.*]] = select i1 [[TMP196]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP198:%.*]] = icmp uge i32 [[TMP164]], [[TMP194]]
-; CHECK-NEXT:    [[TMP199:%.*]] = select i1 [[TMP198]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP200:%.*]] = and i32 [[TMP197]], [[TMP199]]
-; CHECK-NEXT:    [[TMP201:%.*]] = icmp eq i32 [[TMP200]], 0
-; CHECK-NEXT:    [[TMP202:%.*]] = add i32 [[TMP193]], 1
-; CHECK-NEXT:    [[TMP203:%.*]] = sub i32 [[TMP193]], 1
-; CHECK-NEXT:    [[TMP204:%.*]] = select i1 [[TMP201]], i32 [[TMP193]], i32 [[TMP202]]
-; CHECK-NEXT:    [[TMP205:%.*]] = select i1 [[TMP198]], i32 [[TMP204]], i32 [[TMP203]]
-; CHECK-NEXT:    [[TMP206:%.*]] = xor i32 [[TMP205]], [[TMP161]]
-; CHECK-NEXT:    [[TMP207:%.*]] = sub i32 [[TMP206]], [[TMP161]]
-; CHECK-NEXT:    [[TMP208:%.*]] = insertelement <4 x i32> [[TMP156]], i32 [[TMP207]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP208]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP185:%.*]] = mul i32 [[TMP184]], [[TMP156]]
+; CHECK-NEXT:    [[TMP186:%.*]] = sub i32 [[TMP155]], [[TMP185]]
+; CHECK-NEXT:    [[TMP187:%.*]] = icmp uge i32 [[TMP186]], [[TMP156]]
+; CHECK-NEXT:    [[TMP188:%.*]] = icmp uge i32 [[TMP155]], [[TMP185]]
+; CHECK-NEXT:    [[TMP189:%.*]] = and i1 [[TMP187]], [[TMP188]]
+; CHECK-NEXT:    [[TMP190:%.*]] = add i32 [[TMP184]], 1
+; CHECK-NEXT:    [[TMP191:%.*]] = sub i32 [[TMP184]], 1
+; CHECK-NEXT:    [[TMP192:%.*]] = select i1 [[TMP189]], i32 [[TMP190]], i32 [[TMP184]]
+; CHECK-NEXT:    [[TMP193:%.*]] = select i1 [[TMP188]], i32 [[TMP192]], i32 [[TMP191]]
+; CHECK-NEXT:    [[TMP194:%.*]] = xor i32 [[TMP193]], [[TMP152]]
+; CHECK-NEXT:    [[TMP195:%.*]] = sub i32 [[TMP194]], [[TMP152]]
+; CHECK-NEXT:    [[TMP196:%.*]] = insertelement <4 x i32> [[TMP147]], i32 [[TMP195]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP196]], <4 x i32> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_v4i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[12:19], s[0:1], 0xd
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s16, 31
+; GCN-NEXT:    s_add_i32 s3, s16, s2
+; GCN-NEXT:    s_xor_b32 s5, s3, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GCN-NEXT:    s_mov_b32 s16, 0x4f800000
+; GCN-NEXT:    s_ashr_i32 s6, s17, 31
+; GCN-NEXT:    s_add_i32 s0, s17, s6
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s17, s0, s6
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s17
+; GCN-NEXT:    s_ashr_i32 s3, s12, 31
+; GCN-NEXT:    v_mul_f32_e32 v0, s16, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    s_add_i32 s4, s12, s3
+; GCN-NEXT:    s_xor_b32 s4, s4, s3
+; GCN-NEXT:    s_xor_b32 s7, s3, s2
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s5
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s5
+; GCN-NEXT:    s_ashr_i32 s12, s13, 31
+; GCN-NEXT:    s_add_i32 s13, s13, s12
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v3
+; GCN-NEXT:    s_xor_b32 s13, s13, s12
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s4
+; GCN-NEXT:    v_mul_f32_e32 v1, s16, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s5
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, s17
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s4, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s5, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s17
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s4, v2
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    s_ashr_i32 s5, s18, 31
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[0:1]
+; GCN-NEXT:    s_add_i32 s0, s18, s5
+; GCN-NEXT:    s_xor_b32 s4, s12, s6
+; GCN-NEXT:    s_xor_b32 s12, s0, s5
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s12
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s13
+; GCN-NEXT:    v_xor_b32_e32 v0, s7, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s7, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s17
+; GCN-NEXT:    s_ashr_i32 s6, s19, 31
+; GCN-NEXT:    v_mul_f32_e32 v4, s16, v4
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s13, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s17, v3
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s13, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v2, v4, s12
+; GCN-NEXT:    v_mul_hi_u32 v3, v4, s12
+; GCN-NEXT:    s_ashr_i32 s2, s14, 31
+; GCN-NEXT:    s_add_i32 s3, s14, s2
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, 0, v2
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v4
+; GCN-NEXT:    s_xor_b32 s3, s3, s2
+; GCN-NEXT:    v_xor_b32_e32 v1, s4, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v2, v4
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:    s_add_i32 s0, s19, s6
+; GCN-NEXT:    s_xor_b32 s14, s0, s6
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s14
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, s3
+; GCN-NEXT:    s_xor_b32 s7, s2, s5
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v4
+; GCN-NEXT:    v_mul_lo_u32 v3, v2, s12
+; GCN-NEXT:    v_mul_f32_e32 v4, s16, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s3, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v5
+; GCN-NEXT:    s_ashr_i32 s12, s15, 31
+; GCN-NEXT:    v_mul_lo_u32 v6, v4, s14
+; GCN-NEXT:    v_mul_hi_u32 v7, v4, s14
+; GCN-NEXT:    s_add_i32 s13, s15, s12
+; GCN-NEXT:    s_xor_b32 s13, s13, s12
+; GCN-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v6, v6, v4
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s3, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, -1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v2
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v6, v4
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v6, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, s13
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v3, v4, s14
+; GCN-NEXT:    v_xor_b32_e32 v2, s7, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s7, v2
+; GCN-NEXT:    s_xor_b32 s4, s12, s6
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s13, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v5
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s13, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, -1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
+; GCN-NEXT:    v_xor_b32_e32 v3, s4, v3
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s4, v3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
   ret void
@@ -1051,7 +1679,7 @@ define amdgpu_kernel void @srem_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 [[TMP5]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP6]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP10]], 0x41F0000000000000
 ; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP11]] to i32
 ; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
@@ -1081,174 +1709,296 @@ define amdgpu_kernel void @srem_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %x
 ; CHECK-NEXT:    [[TMP37:%.*]] = mul i32 [[TMP36]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP7]], [[TMP37]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp uge i32 [[TMP7]], [[TMP37]]
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP43:%.*]] = and i32 [[TMP40]], [[TMP42]]
-; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i32 [[TMP43]], 0
-; CHECK-NEXT:    [[TMP45:%.*]] = sub i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP46:%.*]] = add i32 [[TMP38]], [[TMP8]]
-; CHECK-NEXT:    [[TMP47:%.*]] = select i1 [[TMP44]], i32 [[TMP38]], i32 [[TMP45]]
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP41]], i32 [[TMP47]], i32 [[TMP46]]
-; CHECK-NEXT:    [[TMP49:%.*]] = xor i32 [[TMP48]], [[TMP3]]
-; CHECK-NEXT:    [[TMP50:%.*]] = sub i32 [[TMP49]], [[TMP3]]
-; CHECK-NEXT:    [[TMP51:%.*]] = insertelement <4 x i32> undef, i32 [[TMP50]], i64 0
-; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <4 x i32> [[X]], i64 1
-; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <4 x i32> [[Y]], i64 1
-; CHECK-NEXT:    [[TMP54:%.*]] = ashr i32 [[TMP52]], 31
-; CHECK-NEXT:    [[TMP55:%.*]] = ashr i32 [[TMP53]], 31
-; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[TMP52]], [[TMP54]]
-; CHECK-NEXT:    [[TMP57:%.*]] = add i32 [[TMP53]], [[TMP55]]
-; CHECK-NEXT:    [[TMP58:%.*]] = xor i32 [[TMP56]], [[TMP54]]
-; CHECK-NEXT:    [[TMP59:%.*]] = xor i32 [[TMP57]], [[TMP55]]
-; CHECK-NEXT:    [[TMP60:%.*]] = uitofp i32 [[TMP59]] to float
-; CHECK-NEXT:    [[TMP61:%.*]] = fdiv fast float 1.000000e+00, [[TMP60]]
-; CHECK-NEXT:    [[TMP62:%.*]] = fmul fast float [[TMP61]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP63:%.*]] = fptoui float [[TMP62]] to i32
-; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP63]] to i64
-; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP59]] to i64
-; CHECK-NEXT:    [[TMP66:%.*]] = mul i64 [[TMP64]], [[TMP65]]
-; CHECK-NEXT:    [[TMP67:%.*]] = trunc i64 [[TMP66]] to i32
-; CHECK-NEXT:    [[TMP68:%.*]] = lshr i64 [[TMP66]], 32
-; CHECK-NEXT:    [[TMP69:%.*]] = trunc i64 [[TMP68]] to i32
-; CHECK-NEXT:    [[TMP70:%.*]] = sub i32 0, [[TMP67]]
-; CHECK-NEXT:    [[TMP71:%.*]] = icmp eq i32 [[TMP69]], 0
-; CHECK-NEXT:    [[TMP72:%.*]] = select i1 [[TMP71]], i32 [[TMP70]], i32 [[TMP67]]
-; CHECK-NEXT:    [[TMP73:%.*]] = zext i32 [[TMP72]] to i64
-; CHECK-NEXT:    [[TMP74:%.*]] = zext i32 [[TMP63]] to i64
-; CHECK-NEXT:    [[TMP75:%.*]] = mul i64 [[TMP73]], [[TMP74]]
-; CHECK-NEXT:    [[TMP76:%.*]] = trunc i64 [[TMP75]] to i32
-; CHECK-NEXT:    [[TMP77:%.*]] = lshr i64 [[TMP75]], 32
-; CHECK-NEXT:    [[TMP78:%.*]] = trunc i64 [[TMP77]] to i32
-; CHECK-NEXT:    [[TMP79:%.*]] = add i32 [[TMP63]], [[TMP78]]
-; CHECK-NEXT:    [[TMP80:%.*]] = sub i32 [[TMP63]], [[TMP78]]
-; CHECK-NEXT:    [[TMP81:%.*]] = select i1 [[TMP71]], i32 [[TMP79]], i32 [[TMP80]]
-; CHECK-NEXT:    [[TMP82:%.*]] = zext i32 [[TMP81]] to i64
-; CHECK-NEXT:    [[TMP83:%.*]] = zext i32 [[TMP58]] to i64
-; CHECK-NEXT:    [[TMP84:%.*]] = mul i64 [[TMP82]], [[TMP83]]
-; CHECK-NEXT:    [[TMP85:%.*]] = trunc i64 [[TMP84]] to i32
-; CHECK-NEXT:    [[TMP86:%.*]] = lshr i64 [[TMP84]], 32
-; CHECK-NEXT:    [[TMP87:%.*]] = trunc i64 [[TMP86]] to i32
-; CHECK-NEXT:    [[TMP88:%.*]] = mul i32 [[TMP87]], [[TMP59]]
-; CHECK-NEXT:    [[TMP89:%.*]] = sub i32 [[TMP58]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = icmp uge i32 [[TMP89]], [[TMP59]]
-; CHECK-NEXT:    [[TMP91:%.*]] = select i1 [[TMP90]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP92:%.*]] = icmp uge i32 [[TMP58]], [[TMP88]]
-; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP92]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP94:%.*]] = and i32 [[TMP91]], [[TMP93]]
-; CHECK-NEXT:    [[TMP95:%.*]] = icmp eq i32 [[TMP94]], 0
-; CHECK-NEXT:    [[TMP96:%.*]] = sub i32 [[TMP89]], [[TMP59]]
-; CHECK-NEXT:    [[TMP97:%.*]] = add i32 [[TMP89]], [[TMP59]]
-; CHECK-NEXT:    [[TMP98:%.*]] = select i1 [[TMP95]], i32 [[TMP89]], i32 [[TMP96]]
-; CHECK-NEXT:    [[TMP99:%.*]] = select i1 [[TMP92]], i32 [[TMP98]], i32 [[TMP97]]
-; CHECK-NEXT:    [[TMP100:%.*]] = xor i32 [[TMP99]], [[TMP54]]
-; CHECK-NEXT:    [[TMP101:%.*]] = sub i32 [[TMP100]], [[TMP54]]
-; CHECK-NEXT:    [[TMP102:%.*]] = insertelement <4 x i32> [[TMP51]], i32 [[TMP101]], i64 1
-; CHECK-NEXT:    [[TMP103:%.*]] = extractelement <4 x i32> [[X]], i64 2
-; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <4 x i32> [[Y]], i64 2
-; CHECK-NEXT:    [[TMP105:%.*]] = ashr i32 [[TMP103]], 31
-; CHECK-NEXT:    [[TMP106:%.*]] = ashr i32 [[TMP104]], 31
-; CHECK-NEXT:    [[TMP107:%.*]] = add i32 [[TMP103]], [[TMP105]]
-; CHECK-NEXT:    [[TMP108:%.*]] = add i32 [[TMP104]], [[TMP106]]
-; CHECK-NEXT:    [[TMP109:%.*]] = xor i32 [[TMP107]], [[TMP105]]
-; CHECK-NEXT:    [[TMP110:%.*]] = xor i32 [[TMP108]], [[TMP106]]
-; CHECK-NEXT:    [[TMP111:%.*]] = uitofp i32 [[TMP110]] to float
-; CHECK-NEXT:    [[TMP112:%.*]] = fdiv fast float 1.000000e+00, [[TMP111]]
-; CHECK-NEXT:    [[TMP113:%.*]] = fmul fast float [[TMP112]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP114:%.*]] = fptoui float [[TMP113]] to i32
-; CHECK-NEXT:    [[TMP115:%.*]] = zext i32 [[TMP114]] to i64
-; CHECK-NEXT:    [[TMP116:%.*]] = zext i32 [[TMP110]] to i64
-; CHECK-NEXT:    [[TMP117:%.*]] = mul i64 [[TMP115]], [[TMP116]]
-; CHECK-NEXT:    [[TMP118:%.*]] = trunc i64 [[TMP117]] to i32
-; CHECK-NEXT:    [[TMP119:%.*]] = lshr i64 [[TMP117]], 32
-; CHECK-NEXT:    [[TMP120:%.*]] = trunc i64 [[TMP119]] to i32
-; CHECK-NEXT:    [[TMP121:%.*]] = sub i32 0, [[TMP118]]
-; CHECK-NEXT:    [[TMP122:%.*]] = icmp eq i32 [[TMP120]], 0
-; CHECK-NEXT:    [[TMP123:%.*]] = select i1 [[TMP122]], i32 [[TMP121]], i32 [[TMP118]]
-; CHECK-NEXT:    [[TMP124:%.*]] = zext i32 [[TMP123]] to i64
-; CHECK-NEXT:    [[TMP125:%.*]] = zext i32 [[TMP114]] to i64
-; CHECK-NEXT:    [[TMP126:%.*]] = mul i64 [[TMP124]], [[TMP125]]
-; CHECK-NEXT:    [[TMP127:%.*]] = trunc i64 [[TMP126]] to i32
-; CHECK-NEXT:    [[TMP128:%.*]] = lshr i64 [[TMP126]], 32
-; CHECK-NEXT:    [[TMP129:%.*]] = trunc i64 [[TMP128]] to i32
-; CHECK-NEXT:    [[TMP130:%.*]] = add i32 [[TMP114]], [[TMP129]]
-; CHECK-NEXT:    [[TMP131:%.*]] = sub i32 [[TMP114]], [[TMP129]]
-; CHECK-NEXT:    [[TMP132:%.*]] = select i1 [[TMP122]], i32 [[TMP130]], i32 [[TMP131]]
-; CHECK-NEXT:    [[TMP133:%.*]] = zext i32 [[TMP132]] to i64
-; CHECK-NEXT:    [[TMP134:%.*]] = zext i32 [[TMP109]] to i64
-; CHECK-NEXT:    [[TMP135:%.*]] = mul i64 [[TMP133]], [[TMP134]]
-; CHECK-NEXT:    [[TMP136:%.*]] = trunc i64 [[TMP135]] to i32
-; CHECK-NEXT:    [[TMP137:%.*]] = lshr i64 [[TMP135]], 32
-; CHECK-NEXT:    [[TMP138:%.*]] = trunc i64 [[TMP137]] to i32
-; CHECK-NEXT:    [[TMP139:%.*]] = mul i32 [[TMP138]], [[TMP110]]
-; CHECK-NEXT:    [[TMP140:%.*]] = sub i32 [[TMP109]], [[TMP139]]
-; CHECK-NEXT:    [[TMP141:%.*]] = icmp uge i32 [[TMP140]], [[TMP110]]
-; CHECK-NEXT:    [[TMP142:%.*]] = select i1 [[TMP141]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP143:%.*]] = icmp uge i32 [[TMP109]], [[TMP139]]
-; CHECK-NEXT:    [[TMP144:%.*]] = select i1 [[TMP143]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP145:%.*]] = and i32 [[TMP142]], [[TMP144]]
-; CHECK-NEXT:    [[TMP146:%.*]] = icmp eq i32 [[TMP145]], 0
-; CHECK-NEXT:    [[TMP147:%.*]] = sub i32 [[TMP140]], [[TMP110]]
-; CHECK-NEXT:    [[TMP148:%.*]] = add i32 [[TMP140]], [[TMP110]]
-; CHECK-NEXT:    [[TMP149:%.*]] = select i1 [[TMP146]], i32 [[TMP140]], i32 [[TMP147]]
-; CHECK-NEXT:    [[TMP150:%.*]] = select i1 [[TMP143]], i32 [[TMP149]], i32 [[TMP148]]
-; CHECK-NEXT:    [[TMP151:%.*]] = xor i32 [[TMP150]], [[TMP105]]
-; CHECK-NEXT:    [[TMP152:%.*]] = sub i32 [[TMP151]], [[TMP105]]
-; CHECK-NEXT:    [[TMP153:%.*]] = insertelement <4 x i32> [[TMP102]], i32 [[TMP152]], i64 2
-; CHECK-NEXT:    [[TMP154:%.*]] = extractelement <4 x i32> [[X]], i64 3
-; CHECK-NEXT:    [[TMP155:%.*]] = extractelement <4 x i32> [[Y]], i64 3
-; CHECK-NEXT:    [[TMP156:%.*]] = ashr i32 [[TMP154]], 31
-; CHECK-NEXT:    [[TMP157:%.*]] = ashr i32 [[TMP155]], 31
-; CHECK-NEXT:    [[TMP158:%.*]] = add i32 [[TMP154]], [[TMP156]]
-; CHECK-NEXT:    [[TMP159:%.*]] = add i32 [[TMP155]], [[TMP157]]
-; CHECK-NEXT:    [[TMP160:%.*]] = xor i32 [[TMP158]], [[TMP156]]
-; CHECK-NEXT:    [[TMP161:%.*]] = xor i32 [[TMP159]], [[TMP157]]
-; CHECK-NEXT:    [[TMP162:%.*]] = uitofp i32 [[TMP161]] to float
-; CHECK-NEXT:    [[TMP163:%.*]] = fdiv fast float 1.000000e+00, [[TMP162]]
-; CHECK-NEXT:    [[TMP164:%.*]] = fmul fast float [[TMP163]], 0x41F0000000000000
-; CHECK-NEXT:    [[TMP165:%.*]] = fptoui float [[TMP164]] to i32
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP7]], [[TMP37]]
+; CHECK-NEXT:    [[TMP41:%.*]] = and i1 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP38]], [[TMP8]]
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP38]], [[TMP8]]
+; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP41]], i32 [[TMP42]], i32 [[TMP38]]
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP40]], i32 [[TMP44]], i32 [[TMP43]]
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP45]], [[TMP3]]
+; CHECK-NEXT:    [[TMP47:%.*]] = sub i32 [[TMP46]], [[TMP3]]
+; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <4 x i32> undef, i32 [[TMP47]], i64 0
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <4 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <4 x i32> [[Y]], i64 1
+; CHECK-NEXT:    [[TMP51:%.*]] = ashr i32 [[TMP49]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP53:%.*]] = add i32 [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[TMP54:%.*]] = add i32 [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP55:%.*]] = xor i32 [[TMP53]], [[TMP51]]
+; CHECK-NEXT:    [[TMP56:%.*]] = xor i32 [[TMP54]], [[TMP52]]
+; CHECK-NEXT:    [[TMP57:%.*]] = uitofp i32 [[TMP56]] to float
+; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
+; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP58]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP60:%.*]] = fptoui float [[TMP59]] to i32
+; CHECK-NEXT:    [[TMP61:%.*]] = zext i32 [[TMP60]] to i64
+; CHECK-NEXT:    [[TMP62:%.*]] = zext i32 [[TMP56]] to i64
+; CHECK-NEXT:    [[TMP63:%.*]] = mul i64 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = trunc i64 [[TMP63]] to i32
+; CHECK-NEXT:    [[TMP65:%.*]] = lshr i64 [[TMP63]], 32
+; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
+; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 0, [[TMP64]]
+; CHECK-NEXT:    [[TMP68:%.*]] = icmp eq i32 [[TMP66]], 0
+; CHECK-NEXT:    [[TMP69:%.*]] = select i1 [[TMP68]], i32 [[TMP67]], i32 [[TMP64]]
+; CHECK-NEXT:    [[TMP70:%.*]] = zext i32 [[TMP69]] to i64
+; CHECK-NEXT:    [[TMP71:%.*]] = zext i32 [[TMP60]] to i64
+; CHECK-NEXT:    [[TMP72:%.*]] = mul i64 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP72]] to i32
+; CHECK-NEXT:    [[TMP74:%.*]] = lshr i64 [[TMP72]], 32
+; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
+; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP60]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP60]], [[TMP75]]
+; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP68]], i32 [[TMP76]], i32 [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = zext i32 [[TMP78]] to i64
+; CHECK-NEXT:    [[TMP80:%.*]] = zext i32 [[TMP55]] to i64
+; CHECK-NEXT:    [[TMP81:%.*]] = mul i64 [[TMP79]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = trunc i64 [[TMP81]] to i32
+; CHECK-NEXT:    [[TMP83:%.*]] = lshr i64 [[TMP81]], 32
+; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
+; CHECK-NEXT:    [[TMP85:%.*]] = mul i32 [[TMP84]], [[TMP56]]
+; CHECK-NEXT:    [[TMP86:%.*]] = sub i32 [[TMP55]], [[TMP85]]
+; CHECK-NEXT:    [[TMP87:%.*]] = icmp uge i32 [[TMP86]], [[TMP56]]
+; CHECK-NEXT:    [[TMP88:%.*]] = icmp uge i32 [[TMP55]], [[TMP85]]
+; CHECK-NEXT:    [[TMP89:%.*]] = and i1 [[TMP87]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = sub i32 [[TMP86]], [[TMP56]]
+; CHECK-NEXT:    [[TMP91:%.*]] = add i32 [[TMP86]], [[TMP56]]
+; CHECK-NEXT:    [[TMP92:%.*]] = select i1 [[TMP89]], i32 [[TMP90]], i32 [[TMP86]]
+; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP88]], i32 [[TMP92]], i32 [[TMP91]]
+; CHECK-NEXT:    [[TMP94:%.*]] = xor i32 [[TMP93]], [[TMP51]]
+; CHECK-NEXT:    [[TMP95:%.*]] = sub i32 [[TMP94]], [[TMP51]]
+; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <4 x i32> [[TMP48]], i32 [[TMP95]], i64 1
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <4 x i32> [[X]], i64 2
+; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <4 x i32> [[Y]], i64 2
+; CHECK-NEXT:    [[TMP99:%.*]] = ashr i32 [[TMP97]], 31
+; CHECK-NEXT:    [[TMP100:%.*]] = ashr i32 [[TMP98]], 31
+; CHECK-NEXT:    [[TMP101:%.*]] = add i32 [[TMP97]], [[TMP99]]
+; CHECK-NEXT:    [[TMP102:%.*]] = add i32 [[TMP98]], [[TMP100]]
+; CHECK-NEXT:    [[TMP103:%.*]] = xor i32 [[TMP101]], [[TMP99]]
+; CHECK-NEXT:    [[TMP104:%.*]] = xor i32 [[TMP102]], [[TMP100]]
+; CHECK-NEXT:    [[TMP105:%.*]] = uitofp i32 [[TMP104]] to float
+; CHECK-NEXT:    [[TMP106:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP105]])
+; CHECK-NEXT:    [[TMP107:%.*]] = fmul fast float [[TMP106]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP108:%.*]] = fptoui float [[TMP107]] to i32
+; CHECK-NEXT:    [[TMP109:%.*]] = zext i32 [[TMP108]] to i64
+; CHECK-NEXT:    [[TMP110:%.*]] = zext i32 [[TMP104]] to i64
+; CHECK-NEXT:    [[TMP111:%.*]] = mul i64 [[TMP109]], [[TMP110]]
+; CHECK-NEXT:    [[TMP112:%.*]] = trunc i64 [[TMP111]] to i32
+; CHECK-NEXT:    [[TMP113:%.*]] = lshr i64 [[TMP111]], 32
+; CHECK-NEXT:    [[TMP114:%.*]] = trunc i64 [[TMP113]] to i32
+; CHECK-NEXT:    [[TMP115:%.*]] = sub i32 0, [[TMP112]]
+; CHECK-NEXT:    [[TMP116:%.*]] = icmp eq i32 [[TMP114]], 0
+; CHECK-NEXT:    [[TMP117:%.*]] = select i1 [[TMP116]], i32 [[TMP115]], i32 [[TMP112]]
+; CHECK-NEXT:    [[TMP118:%.*]] = zext i32 [[TMP117]] to i64
+; CHECK-NEXT:    [[TMP119:%.*]] = zext i32 [[TMP108]] to i64
+; CHECK-NEXT:    [[TMP120:%.*]] = mul i64 [[TMP118]], [[TMP119]]
+; CHECK-NEXT:    [[TMP121:%.*]] = trunc i64 [[TMP120]] to i32
+; CHECK-NEXT:    [[TMP122:%.*]] = lshr i64 [[TMP120]], 32
+; CHECK-NEXT:    [[TMP123:%.*]] = trunc i64 [[TMP122]] to i32
+; CHECK-NEXT:    [[TMP124:%.*]] = add i32 [[TMP108]], [[TMP123]]
+; CHECK-NEXT:    [[TMP125:%.*]] = sub i32 [[TMP108]], [[TMP123]]
+; CHECK-NEXT:    [[TMP126:%.*]] = select i1 [[TMP116]], i32 [[TMP124]], i32 [[TMP125]]
+; CHECK-NEXT:    [[TMP127:%.*]] = zext i32 [[TMP126]] to i64
+; CHECK-NEXT:    [[TMP128:%.*]] = zext i32 [[TMP103]] to i64
+; CHECK-NEXT:    [[TMP129:%.*]] = mul i64 [[TMP127]], [[TMP128]]
+; CHECK-NEXT:    [[TMP130:%.*]] = trunc i64 [[TMP129]] to i32
+; CHECK-NEXT:    [[TMP131:%.*]] = lshr i64 [[TMP129]], 32
+; CHECK-NEXT:    [[TMP132:%.*]] = trunc i64 [[TMP131]] to i32
+; CHECK-NEXT:    [[TMP133:%.*]] = mul i32 [[TMP132]], [[TMP104]]
+; CHECK-NEXT:    [[TMP134:%.*]] = sub i32 [[TMP103]], [[TMP133]]
+; CHECK-NEXT:    [[TMP135:%.*]] = icmp uge i32 [[TMP134]], [[TMP104]]
+; CHECK-NEXT:    [[TMP136:%.*]] = icmp uge i32 [[TMP103]], [[TMP133]]
+; CHECK-NEXT:    [[TMP137:%.*]] = and i1 [[TMP135]], [[TMP136]]
+; CHECK-NEXT:    [[TMP138:%.*]] = sub i32 [[TMP134]], [[TMP104]]
+; CHECK-NEXT:    [[TMP139:%.*]] = add i32 [[TMP134]], [[TMP104]]
+; CHECK-NEXT:    [[TMP140:%.*]] = select i1 [[TMP137]], i32 [[TMP138]], i32 [[TMP134]]
+; CHECK-NEXT:    [[TMP141:%.*]] = select i1 [[TMP136]], i32 [[TMP140]], i32 [[TMP139]]
+; CHECK-NEXT:    [[TMP142:%.*]] = xor i32 [[TMP141]], [[TMP99]]
+; CHECK-NEXT:    [[TMP143:%.*]] = sub i32 [[TMP142]], [[TMP99]]
+; CHECK-NEXT:    [[TMP144:%.*]] = insertelement <4 x i32> [[TMP96]], i32 [[TMP143]], i64 2
+; CHECK-NEXT:    [[TMP145:%.*]] = extractelement <4 x i32> [[X]], i64 3
+; CHECK-NEXT:    [[TMP146:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[TMP147:%.*]] = ashr i32 [[TMP145]], 31
+; CHECK-NEXT:    [[TMP148:%.*]] = ashr i32 [[TMP146]], 31
+; CHECK-NEXT:    [[TMP149:%.*]] = add i32 [[TMP145]], [[TMP147]]
+; CHECK-NEXT:    [[TMP150:%.*]] = add i32 [[TMP146]], [[TMP148]]
+; CHECK-NEXT:    [[TMP151:%.*]] = xor i32 [[TMP149]], [[TMP147]]
+; CHECK-NEXT:    [[TMP152:%.*]] = xor i32 [[TMP150]], [[TMP148]]
+; CHECK-NEXT:    [[TMP153:%.*]] = uitofp i32 [[TMP152]] to float
+; CHECK-NEXT:    [[TMP154:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP153]])
+; CHECK-NEXT:    [[TMP155:%.*]] = fmul fast float [[TMP154]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP156:%.*]] = fptoui float [[TMP155]] to i32
+; CHECK-NEXT:    [[TMP157:%.*]] = zext i32 [[TMP156]] to i64
+; CHECK-NEXT:    [[TMP158:%.*]] = zext i32 [[TMP152]] to i64
+; CHECK-NEXT:    [[TMP159:%.*]] = mul i64 [[TMP157]], [[TMP158]]
+; CHECK-NEXT:    [[TMP160:%.*]] = trunc i64 [[TMP159]] to i32
+; CHECK-NEXT:    [[TMP161:%.*]] = lshr i64 [[TMP159]], 32
+; CHECK-NEXT:    [[TMP162:%.*]] = trunc i64 [[TMP161]] to i32
+; CHECK-NEXT:    [[TMP163:%.*]] = sub i32 0, [[TMP160]]
+; CHECK-NEXT:    [[TMP164:%.*]] = icmp eq i32 [[TMP162]], 0
+; CHECK-NEXT:    [[TMP165:%.*]] = select i1 [[TMP164]], i32 [[TMP163]], i32 [[TMP160]]
 ; CHECK-NEXT:    [[TMP166:%.*]] = zext i32 [[TMP165]] to i64
-; CHECK-NEXT:    [[TMP167:%.*]] = zext i32 [[TMP161]] to i64
+; CHECK-NEXT:    [[TMP167:%.*]] = zext i32 [[TMP156]] to i64
 ; CHECK-NEXT:    [[TMP168:%.*]] = mul i64 [[TMP166]], [[TMP167]]
 ; CHECK-NEXT:    [[TMP169:%.*]] = trunc i64 [[TMP168]] to i32
 ; CHECK-NEXT:    [[TMP170:%.*]] = lshr i64 [[TMP168]], 32
 ; CHECK-NEXT:    [[TMP171:%.*]] = trunc i64 [[TMP170]] to i32
-; CHECK-NEXT:    [[TMP172:%.*]] = sub i32 0, [[TMP169]]
-; CHECK-NEXT:    [[TMP173:%.*]] = icmp eq i32 [[TMP171]], 0
-; CHECK-NEXT:    [[TMP174:%.*]] = select i1 [[TMP173]], i32 [[TMP172]], i32 [[TMP169]]
+; CHECK-NEXT:    [[TMP172:%.*]] = add i32 [[TMP156]], [[TMP171]]
+; CHECK-NEXT:    [[TMP173:%.*]] = sub i32 [[TMP156]], [[TMP171]]
+; CHECK-NEXT:    [[TMP174:%.*]] = select i1 [[TMP164]], i32 [[TMP172]], i32 [[TMP173]]
 ; CHECK-NEXT:    [[TMP175:%.*]] = zext i32 [[TMP174]] to i64
-; CHECK-NEXT:    [[TMP176:%.*]] = zext i32 [[TMP165]] to i64
+; CHECK-NEXT:    [[TMP176:%.*]] = zext i32 [[TMP151]] to i64
 ; CHECK-NEXT:    [[TMP177:%.*]] = mul i64 [[TMP175]], [[TMP176]]
 ; CHECK-NEXT:    [[TMP178:%.*]] = trunc i64 [[TMP177]] to i32
 ; CHECK-NEXT:    [[TMP179:%.*]] = lshr i64 [[TMP177]], 32
 ; CHECK-NEXT:    [[TMP180:%.*]] = trunc i64 [[TMP179]] to i32
-; CHECK-NEXT:    [[TMP181:%.*]] = add i32 [[TMP165]], [[TMP180]]
-; CHECK-NEXT:    [[TMP182:%.*]] = sub i32 [[TMP165]], [[TMP180]]
-; CHECK-NEXT:    [[TMP183:%.*]] = select i1 [[TMP173]], i32 [[TMP181]], i32 [[TMP182]]
-; CHECK-NEXT:    [[TMP184:%.*]] = zext i32 [[TMP183]] to i64
-; CHECK-NEXT:    [[TMP185:%.*]] = zext i32 [[TMP160]] to i64
-; CHECK-NEXT:    [[TMP186:%.*]] = mul i64 [[TMP184]], [[TMP185]]
-; CHECK-NEXT:    [[TMP187:%.*]] = trunc i64 [[TMP186]] to i32
-; CHECK-NEXT:    [[TMP188:%.*]] = lshr i64 [[TMP186]], 32
-; CHECK-NEXT:    [[TMP189:%.*]] = trunc i64 [[TMP188]] to i32
-; CHECK-NEXT:    [[TMP190:%.*]] = mul i32 [[TMP189]], [[TMP161]]
-; CHECK-NEXT:    [[TMP191:%.*]] = sub i32 [[TMP160]], [[TMP190]]
-; CHECK-NEXT:    [[TMP192:%.*]] = icmp uge i32 [[TMP191]], [[TMP161]]
-; CHECK-NEXT:    [[TMP193:%.*]] = select i1 [[TMP192]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP194:%.*]] = icmp uge i32 [[TMP160]], [[TMP190]]
-; CHECK-NEXT:    [[TMP195:%.*]] = select i1 [[TMP194]], i32 -1, i32 0
-; CHECK-NEXT:    [[TMP196:%.*]] = and i32 [[TMP193]], [[TMP195]]
-; CHECK-NEXT:    [[TMP197:%.*]] = icmp eq i32 [[TMP196]], 0
-; CHECK-NEXT:    [[TMP198:%.*]] = sub i32 [[TMP191]], [[TMP161]]
-; CHECK-NEXT:    [[TMP199:%.*]] = add i32 [[TMP191]], [[TMP161]]
-; CHECK-NEXT:    [[TMP200:%.*]] = select i1 [[TMP197]], i32 [[TMP191]], i32 [[TMP198]]
-; CHECK-NEXT:    [[TMP201:%.*]] = select i1 [[TMP194]], i32 [[TMP200]], i32 [[TMP199]]
-; CHECK-NEXT:    [[TMP202:%.*]] = xor i32 [[TMP201]], [[TMP156]]
-; CHECK-NEXT:    [[TMP203:%.*]] = sub i32 [[TMP202]], [[TMP156]]
-; CHECK-NEXT:    [[TMP204:%.*]] = insertelement <4 x i32> [[TMP153]], i32 [[TMP203]], i64 3
-; CHECK-NEXT:    store <4 x i32> [[TMP204]], <4 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    [[TMP181:%.*]] = mul i32 [[TMP180]], [[TMP152]]
+; CHECK-NEXT:    [[TMP182:%.*]] = sub i32 [[TMP151]], [[TMP181]]
+; CHECK-NEXT:    [[TMP183:%.*]] = icmp uge i32 [[TMP182]], [[TMP152]]
+; CHECK-NEXT:    [[TMP184:%.*]] = icmp uge i32 [[TMP151]], [[TMP181]]
+; CHECK-NEXT:    [[TMP185:%.*]] = and i1 [[TMP183]], [[TMP184]]
+; CHECK-NEXT:    [[TMP186:%.*]] = sub i32 [[TMP182]], [[TMP152]]
+; CHECK-NEXT:    [[TMP187:%.*]] = add i32 [[TMP182]], [[TMP152]]
+; CHECK-NEXT:    [[TMP188:%.*]] = select i1 [[TMP185]], i32 [[TMP186]], i32 [[TMP182]]
+; CHECK-NEXT:    [[TMP189:%.*]] = select i1 [[TMP184]], i32 [[TMP188]], i32 [[TMP187]]
+; CHECK-NEXT:    [[TMP190:%.*]] = xor i32 [[TMP189]], [[TMP147]]
+; CHECK-NEXT:    [[TMP191:%.*]] = sub i32 [[TMP190]], [[TMP147]]
+; CHECK-NEXT:    [[TMP192:%.*]] = insertelement <4 x i32> [[TMP144]], i32 [[TMP191]], i64 3
+; CHECK-NEXT:    store <4 x i32> [[TMP192]], <4 x i32> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_v4i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[12:19], s[0:1], 0xd
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s16, 31
+; GCN-NEXT:    s_add_i32 s3, s16, s2
+; GCN-NEXT:    s_xor_b32 s5, s3, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s5
+; GCN-NEXT:    s_mov_b32 s16, 0x4f800000
+; GCN-NEXT:    s_ashr_i32 s6, s12, 31
+; GCN-NEXT:    s_ashr_i32 s2, s17, 31
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_add_i32 s0, s12, s6
+; GCN-NEXT:    s_add_i32 s3, s17, s2
+; GCN-NEXT:    s_xor_b32 s4, s0, s6
+; GCN-NEXT:    v_mul_f32_e32 v0, s16, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s17, s3, s2
+; GCN-NEXT:    s_ashr_i32 s7, s13, 31
+; GCN-NEXT:    s_add_i32 s12, s13, s7
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s5
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s5
+; GCN-NEXT:    s_xor_b32 s12, s12, s7
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s17
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s4
+; GCN-NEXT:    v_mul_f32_e32 v1, s16, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s5
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s17
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, s17
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s4, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s4, v0
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s5, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s5, v2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s5, v2
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    s_ashr_i32 s0, s18, 31
+; GCN-NEXT:    s_add_i32 s1, s18, s0
+; GCN-NEXT:    s_xor_b32 s13, s1, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s13
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s12
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[2:3]
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GCN-NEXT:    v_xor_b32_e32 v0, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s17
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT:    v_mul_f32_e32 v2, s16, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s12, v1
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s12, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s17, v3
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, s13
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, s13
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s17, v3
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s17, v3
+; GCN-NEXT:    v_sub_i32_e32 v7, vcc, 0, v5
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v5, v5, v2
+; GCN-NEXT:    s_ashr_i32 s6, s14, 31
+; GCN-NEXT:    s_add_i32 s12, s14, s6
+; GCN-NEXT:    s_xor_b32 s12, s12, s6
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v5, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    s_ashr_i32 s0, s19, 31
+; GCN-NEXT:    s_add_i32 s1, s19, s0
+; GCN-NEXT:    s_xor_b32 s14, s1, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s14
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, s12
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[2:3]
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v3
+; GCN-NEXT:    v_xor_b32_e32 v1, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, s13
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s7, v1
+; GCN-NEXT:    v_mul_f32_e32 v3, s16, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GCN-NEXT:    s_ashr_i32 s7, s15, 31
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s12, v2
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s12, v2
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, s14
+; GCN-NEXT:    v_mul_hi_u32 v7, v3, s14
+; GCN-NEXT:    s_add_i32 s12, s15, s7
+; GCN-NEXT:    s_xor_b32 s12, s12, s7
+; GCN-NEXT:    v_sub_i32_e32 v8, vcc, 0, v6
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v6, v8, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v6, v6, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s13, v4
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s13, v4
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v6, v3
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v3, v7, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v3, v3, s12
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v2, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s14
+; GCN-NEXT:    v_xor_b32_e32 v2, s6, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s12, v3
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s12, v3
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, s14, v4
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s14, v4
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v5, v3, s[2:3]
+; GCN-NEXT:    v_xor_b32_e32 v3, s7, v3
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s7, v3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
   %r = srem <4 x i32> %x, %y
   store <4 x i32> %r, <4 x i32> addrspace(1)* %out
   ret void
@@ -1262,7 +2012,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP6:%.*]] = uitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
@@ -1282,7 +2032,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP22]] to i32
 ; CHECK-NEXT:    [[TMP25:%.*]] = uitofp i32 [[TMP23]] to float
 ; CHECK-NEXT:    [[TMP26:%.*]] = uitofp i32 [[TMP24]] to float
-; CHECK-NEXT:    [[TMP27:%.*]] = fdiv fast float 1.000000e+00, [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
 ; CHECK-NEXT:    [[TMP30:%.*]] = fneg fast float [[TMP29]]
@@ -1302,7 +2052,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP44:%.*]] = zext i16 [[TMP42]] to i32
 ; CHECK-NEXT:    [[TMP45:%.*]] = uitofp i32 [[TMP43]] to float
 ; CHECK-NEXT:    [[TMP46:%.*]] = uitofp i32 [[TMP44]] to float
-; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
+; CHECK-NEXT:    [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
 ; CHECK-NEXT:    [[TMP50:%.*]] = fneg fast float [[TMP49]]
@@ -1322,7 +2072,7 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP64:%.*]] = zext i16 [[TMP62]] to i32
 ; CHECK-NEXT:    [[TMP65:%.*]] = uitofp i32 [[TMP63]] to float
 ; CHECK-NEXT:    [[TMP66:%.*]] = uitofp i32 [[TMP64]] to float
-; CHECK-NEXT:    [[TMP67:%.*]] = fdiv fast float 1.000000e+00, [[TMP66]]
+; CHECK-NEXT:    [[TMP67:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP66]])
 ; CHECK-NEXT:    [[TMP68:%.*]] = fmul fast float [[TMP65]], [[TMP67]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = call fast float @llvm.trunc.f32(float [[TMP68]])
 ; CHECK-NEXT:    [[TMP70:%.*]] = fneg fast float [[TMP69]]
@@ -1339,6 +2089,66 @@ define amdgpu_kernel void @udiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    store <4 x i16> [[TMP80]], <4 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_v4i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s8, 0xffff
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s9, s2, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    s_lshr_b32 s9, s0, 16
+; GCN-NEXT:    s_and_b32 s0, s0, s8
+; GCN-NEXT:    s_lshr_b32 s2, s2, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s9
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    s_and_b32 s2, s3, s8
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GCN-NEXT:    v_mad_f32 v2, -v1, v3, v4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s2
+; GCN-NEXT:    s_lshr_b32 s0, s1, 16
+; GCN-NEXT:    s_and_b32 s1, s1, s8
+; GCN-NEXT:    s_lshr_b32 s10, s3, 16
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s10
+; GCN-NEXT:    v_cvt_f32_u32_e32 v5, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v1, vcc
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v3
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_mul_f32_e32 v1, v5, v6
+; GCN-NEXT:    v_cvt_f32_u32_e32 v6, s0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mad_f32 v5, -v1, v4, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v4, v6, v7
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v4
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mad_f32 v4, -v4, v3, v6
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_and_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GCN-NEXT:    v_and_b32_e32 v1, s8, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv <4 x i16> %x, %y
   store <4 x i16> %r, <4 x i16> addrspace(1)* %out
   ret void
@@ -1352,7 +2162,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP6:%.*]] = uitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
@@ -1374,7 +2184,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP26:%.*]] = zext i16 [[TMP24]] to i32
 ; CHECK-NEXT:    [[TMP27:%.*]] = uitofp i32 [[TMP25]] to float
 ; CHECK-NEXT:    [[TMP28:%.*]] = uitofp i32 [[TMP26]] to float
-; CHECK-NEXT:    [[TMP29:%.*]] = fdiv fast float 1.000000e+00, [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
 ; CHECK-NEXT:    [[TMP32:%.*]] = fneg fast float [[TMP31]]
@@ -1396,7 +2206,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP48:%.*]] = zext i16 [[TMP46]] to i32
 ; CHECK-NEXT:    [[TMP49:%.*]] = uitofp i32 [[TMP47]] to float
 ; CHECK-NEXT:    [[TMP50:%.*]] = uitofp i32 [[TMP48]] to float
-; CHECK-NEXT:    [[TMP51:%.*]] = fdiv fast float 1.000000e+00, [[TMP50]]
+; CHECK-NEXT:    [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
 ; CHECK-NEXT:    [[TMP54:%.*]] = fneg fast float [[TMP53]]
@@ -1418,7 +2228,7 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP70:%.*]] = zext i16 [[TMP68]] to i32
 ; CHECK-NEXT:    [[TMP71:%.*]] = uitofp i32 [[TMP69]] to float
 ; CHECK-NEXT:    [[TMP72:%.*]] = uitofp i32 [[TMP70]] to float
-; CHECK-NEXT:    [[TMP73:%.*]] = fdiv fast float 1.000000e+00, [[TMP72]]
+; CHECK-NEXT:    [[TMP73:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP72]])
 ; CHECK-NEXT:    [[TMP74:%.*]] = fmul fast float [[TMP71]], [[TMP73]]
 ; CHECK-NEXT:    [[TMP75:%.*]] = call fast float @llvm.trunc.f32(float [[TMP74]])
 ; CHECK-NEXT:    [[TMP76:%.*]] = fneg fast float [[TMP75]]
@@ -1437,6 +2247,74 @@ define amdgpu_kernel void @urem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    store <4 x i16> [[TMP88]], <4 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_v4i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s8, 0xffff
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s9, s2, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    s_and_b32 s10, s0, s8
+; GCN-NEXT:    s_lshr_b32 s11, s2, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s10
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s11
+; GCN-NEXT:    s_lshr_b32 s9, s0, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s9
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v1
+; GCN-NEXT:    v_mad_f32 v1, -v1, v3, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v3
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    s_and_b32 s2, s3, s8
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s2
+; GCN-NEXT:    s_and_b32 s2, s1, s8
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s11
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GCN-NEXT:    s_lshr_b32 s12, s3, 16
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s9, v1
+; GCN-NEXT:    s_lshr_b32 s10, s1, 16
+; GCN-NEXT:    v_mul_f32_e32 v1, v3, v4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v6, s10
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v4
+; GCN-NEXT:    v_mad_f32 v3, -v1, v2, v3
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v2, v6, v7
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mad_f32 v2, -v2, v4, v6
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, s12
+; GCN-NEXT:    v_and_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s10, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_and_b32_e32 v1, s8, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v5
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = urem <4 x i16> %x, %y
   store <4 x i16> %r, <4 x i16> addrspace(1)* %out
   ret void
@@ -1453,7 +2331,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
@@ -1477,7 +2355,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP30]], 1
 ; CHECK-NEXT:    [[TMP32:%.*]] = sitofp i32 [[TMP27]] to float
 ; CHECK-NEXT:    [[TMP33:%.*]] = sitofp i32 [[TMP28]] to float
-; CHECK-NEXT:    [[TMP34:%.*]] = fdiv fast float 1.000000e+00, [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
 ; CHECK-NEXT:    [[TMP37:%.*]] = fneg fast float [[TMP36]]
@@ -1501,7 +2379,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP54]], 1
 ; CHECK-NEXT:    [[TMP56:%.*]] = sitofp i32 [[TMP51]] to float
 ; CHECK-NEXT:    [[TMP57:%.*]] = sitofp i32 [[TMP52]] to float
-; CHECK-NEXT:    [[TMP58:%.*]] = fdiv fast float 1.000000e+00, [[TMP57]]
+; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
 ; CHECK-NEXT:    [[TMP61:%.*]] = fneg fast float [[TMP60]]
@@ -1525,7 +2403,7 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP79:%.*]] = or i32 [[TMP78]], 1
 ; CHECK-NEXT:    [[TMP80:%.*]] = sitofp i32 [[TMP75]] to float
 ; CHECK-NEXT:    [[TMP81:%.*]] = sitofp i32 [[TMP76]] to float
-; CHECK-NEXT:    [[TMP82:%.*]] = fdiv fast float 1.000000e+00, [[TMP81]]
+; CHECK-NEXT:    [[TMP82:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP81]])
 ; CHECK-NEXT:    [[TMP83:%.*]] = fmul fast float [[TMP80]], [[TMP82]]
 ; CHECK-NEXT:    [[TMP84:%.*]] = call fast float @llvm.trunc.f32(float [[TMP83]])
 ; CHECK-NEXT:    [[TMP85:%.*]] = fneg fast float [[TMP84]]
@@ -1543,6 +2421,86 @@ define amdgpu_kernel void @sdiv_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    store <4 x i16> [[TMP96]], <4 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_v4i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_sext_i32_i16 s8, s2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
+; GCN-NEXT:    s_sext_i32_i16 s9, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
+; GCN-NEXT:    s_xor_b32 s8, s9, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s2, s2, 16
+; GCN-NEXT:    s_ashr_i32 s8, s8, 30
+; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    s_ashr_i32 s0, s0, 16
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
+; GCN-NEXT:    v_trunc_f32_e32 v3, v3
+; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
+; GCN-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-NEXT:    s_sext_i32_i16 s0, s3
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
+; GCN-NEXT:    s_sext_i32_i16 s2, s1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v4, v1, v4
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v1, -v4, v2, v1
+; GCN-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-NEXT:    s_ashr_i32 s0, s3, 16
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
+; GCN-NEXT:    s_ashr_i32 s1, s1, 16
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
+; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_mov_b32_e32 v6, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v2|
+; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    s_mov_b32 s0, 0xffff
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_and_b32_e32 v1, s0, v1
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv <4 x i16> %x, %y
   store <4 x i16> %r, <4 x i16> addrspace(1)* %out
   ret void
@@ -1559,7 +2517,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
@@ -1585,7 +2543,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP32]], 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = sitofp i32 [[TMP29]] to float
 ; CHECK-NEXT:    [[TMP35:%.*]] = sitofp i32 [[TMP30]] to float
-; CHECK-NEXT:    [[TMP36:%.*]] = fdiv fast float 1.000000e+00, [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
 ; CHECK-NEXT:    [[TMP39:%.*]] = fneg fast float [[TMP38]]
@@ -1611,7 +2569,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP58]], 1
 ; CHECK-NEXT:    [[TMP60:%.*]] = sitofp i32 [[TMP55]] to float
 ; CHECK-NEXT:    [[TMP61:%.*]] = sitofp i32 [[TMP56]] to float
-; CHECK-NEXT:    [[TMP62:%.*]] = fdiv fast float 1.000000e+00, [[TMP61]]
+; CHECK-NEXT:    [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
 ; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
 ; CHECK-NEXT:    [[TMP65:%.*]] = fneg fast float [[TMP64]]
@@ -1637,7 +2595,7 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    [[TMP85:%.*]] = or i32 [[TMP84]], 1
 ; CHECK-NEXT:    [[TMP86:%.*]] = sitofp i32 [[TMP81]] to float
 ; CHECK-NEXT:    [[TMP87:%.*]] = sitofp i32 [[TMP82]] to float
-; CHECK-NEXT:    [[TMP88:%.*]] = fdiv fast float 1.000000e+00, [[TMP87]]
+; CHECK-NEXT:    [[TMP88:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP87]])
 ; CHECK-NEXT:    [[TMP89:%.*]] = fmul fast float [[TMP86]], [[TMP88]]
 ; CHECK-NEXT:    [[TMP90:%.*]] = call fast float @llvm.trunc.f32(float [[TMP89]])
 ; CHECK-NEXT:    [[TMP91:%.*]] = fneg fast float [[TMP90]]
@@ -1657,6 +2615,94 @@ define amdgpu_kernel void @srem_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %x
 ; CHECK-NEXT:    store <4 x i16> [[TMP104]], <4 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_v4i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_sext_i32_i16 s8, s2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
+; GCN-NEXT:    s_sext_i32_i16 s9, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
+; GCN-NEXT:    s_xor_b32 s8, s9, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s8, s8, 30
+; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    s_ashr_i32 s2, s2, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s2
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
+; GCN-NEXT:    s_xor_b32 s8, s0, s2
+; GCN-NEXT:    s_ashr_i32 s8, s8, 30
+; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
+; GCN-NEXT:    v_trunc_f32_e32 v3, v3
+; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
+; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; GCN-NEXT:    v_mov_b32_e32 v4, s8
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s2
+; GCN-NEXT:    s_sext_i32_i16 s2, s3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s0, v1
+; GCN-NEXT:    s_sext_i32_i16 s0, s1
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GCN-NEXT:    s_xor_b32 s0, s0, s2
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v4, v1, v4
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v1, -v4, v2, v1
+; GCN-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-NEXT:    s_ashr_i32 s0, s3, 16
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v2|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v5, vcc
+; GCN-NEXT:    s_ashr_i32 s2, s1, 16
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
+; GCN-NEXT:    s_xor_b32 s3, s2, s0
+; GCN-NEXT:    s_ashr_i32 s3, s3, 30
+; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mad_f32 v4, -v5, v2, v4
+; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    s_or_b32 s3, s3, 1
+; GCN-NEXT:    v_mov_b32_e32 v6, s3
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v2|
+; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, s0
+; GCN-NEXT:    s_mov_b32 s0, 0xffff
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v1
+; GCN-NEXT:    v_and_b32_e32 v1, s0, v1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GCN-NEXT:    v_or_b32_e32 v1, v1, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 16, v3
+; GCN-NEXT:    v_and_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v2
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = srem <4 x i16> %x, %y
   store <4 x i16> %r, <4 x i16> addrspace(1)* %out
   ret void
@@ -1668,7 +2714,7 @@ define amdgpu_kernel void @udiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i3 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
@@ -1684,6 +2730,27 @@ define amdgpu_kernel void @udiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    store i3 [[TMP17]], i3 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_i3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bfe_u32 s1, s0, 0x30008
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GCN-NEXT:    s_and_b32 s0, s0, 7
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v2, s0
+; GCN-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v1
+; GCN-NEXT:    v_mad_f32 v1, -v1, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv i3 %x, %y
   store i3 %r, i3 addrspace(1)* %out
   ret void
@@ -1695,7 +2762,7 @@ define amdgpu_kernel void @urem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i3 [[Y:%.*]] to i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = uitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP5:%.*]] = fdiv fast float 1.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP4]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast float [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.trunc.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fneg fast float [[TMP7]]
@@ -1713,6 +2780,30 @@ define amdgpu_kernel void @urem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    store i3 [[TMP19]], i3 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_i3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bfe_u32 s1, s0, 0x30008
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v0
+; GCN-NEXT:    s_and_b32 s2, s0, 7
+; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v2, s2
+; GCN-NEXT:    s_lshr_b32 s1, s0, 8
+; GCN-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v1
+; GCN-NEXT:    v_mad_f32 v1, -v1, v0, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v3, vcc
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = urem i3 %x, %y
   store i3 %r, i3 addrspace(1)* %out
   ret void
@@ -1727,7 +2818,7 @@ define amdgpu_kernel void @sdiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
@@ -1744,6 +2835,32 @@ define amdgpu_kernel void @sdiv_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    store i3 [[TMP21]], i3 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_i3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bfe_i32 s1, s0, 0x30008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_bfe_i32 s0, s0, 0x30000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    s_xor_b32 s0, s0, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s0
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv i3 %x, %y
   store i3 %r, i3 addrspace(1)* %out
   ret void
@@ -1758,7 +2875,7 @@ define amdgpu_kernel void @srem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp i32 [[TMP2]] to float
-; CHECK-NEXT:    [[TMP8:%.*]] = fdiv fast float 1.000000e+00, [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.trunc.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fneg fast float [[TMP10]]
@@ -1777,6 +2894,35 @@ define amdgpu_kernel void @srem_i3(i3 addrspace(1)* %out, i3 %x, i3 %y) {
 ; CHECK-NEXT:    store i3 [[TMP23]], i3 addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_i3:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bfe_i32 s1, s0, 0x30008
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s1
+; GCN-NEXT:    s_bfe_i32 s3, s0, 0x30000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s3
+; GCN-NEXT:    s_xor_b32 s1, s3, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    s_lshr_b32 s2, s0, 8
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s2
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_and_b32_e32 v0, 7, v0
+; GCN-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = srem i3 %x, %y
   store i3 %r, i3 addrspace(1)* %out
   ret void
@@ -1790,7 +2936,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP6:%.*]] = uitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
@@ -1810,7 +2956,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP22]] to i32
 ; CHECK-NEXT:    [[TMP25:%.*]] = uitofp i32 [[TMP23]] to float
 ; CHECK-NEXT:    [[TMP26:%.*]] = uitofp i32 [[TMP24]] to float
-; CHECK-NEXT:    [[TMP27:%.*]] = fdiv fast float 1.000000e+00, [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
 ; CHECK-NEXT:    [[TMP30:%.*]] = fneg fast float [[TMP29]]
@@ -1830,7 +2976,7 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP44:%.*]] = zext i16 [[TMP42]] to i32
 ; CHECK-NEXT:    [[TMP45:%.*]] = uitofp i32 [[TMP43]] to float
 ; CHECK-NEXT:    [[TMP46:%.*]] = uitofp i32 [[TMP44]] to float
-; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
+; CHECK-NEXT:    [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
 ; CHECK-NEXT:    [[TMP50:%.*]] = fneg fast float [[TMP49]]
@@ -1847,6 +2993,54 @@ define amdgpu_kernel void @udiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    store <3 x i16> [[TMP60]], <3 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_v3i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s8, 0xffff
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s6, s0, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GCN-NEXT:    s_and_b32 s6, s2, s8
+; GCN-NEXT:    s_lshr_b32 s0, s0, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s6
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_lshr_b32 s0, s2, 16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    s_and_b32 s0, s1, s8
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GCN-NEXT:    v_mad_f32 v2, -v1, v3, v4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, s0
+; GCN-NEXT:    s_and_b32 s0, s3, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v5, s0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v3
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_f32_e32 v2, v5, v6
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v2
+; GCN-NEXT:    v_mad_f32 v2, -v2, v4, v5
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_and_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = udiv <3 x i16> %x, %y
   store <3 x i16> %r, <3 x i16> addrspace(1)* %out
   ret void
@@ -1860,7 +3054,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i16 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP6:%.*]] = uitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
@@ -1882,7 +3076,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP26:%.*]] = zext i16 [[TMP24]] to i32
 ; CHECK-NEXT:    [[TMP27:%.*]] = uitofp i32 [[TMP25]] to float
 ; CHECK-NEXT:    [[TMP28:%.*]] = uitofp i32 [[TMP26]] to float
-; CHECK-NEXT:    [[TMP29:%.*]] = fdiv fast float 1.000000e+00, [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
 ; CHECK-NEXT:    [[TMP32:%.*]] = fneg fast float [[TMP31]]
@@ -1904,7 +3098,7 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP48:%.*]] = zext i16 [[TMP46]] to i32
 ; CHECK-NEXT:    [[TMP49:%.*]] = uitofp i32 [[TMP47]] to float
 ; CHECK-NEXT:    [[TMP50:%.*]] = uitofp i32 [[TMP48]] to float
-; CHECK-NEXT:    [[TMP51:%.*]] = fdiv fast float 1.000000e+00, [[TMP50]]
+; CHECK-NEXT:    [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
 ; CHECK-NEXT:    [[TMP54:%.*]] = fneg fast float [[TMP53]]
@@ -1923,6 +3117,64 @@ define amdgpu_kernel void @urem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    store <3 x i16> [[TMP66]], <3 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_v3i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s8, 0xffff
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    s_and_b32 s6, s0, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GCN-NEXT:    s_and_b32 s6, s2, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s6
+; GCN-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v0
+; GCN-NEXT:    v_alignbit_b32 v4, s1, v4, 16
+; GCN-NEXT:    v_and_b32_e32 v5, s8, v4
+; GCN-NEXT:    v_alignbit_b32 v1, s3, v1, 16
+; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
+; GCN-NEXT:    v_trunc_f32_e32 v3, v3
+; GCN-NEXT:    v_mad_f32 v2, -v3, v0, v2
+; GCN-NEXT:    v_cvt_u32_f32_e32 v6, v3
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v5
+; GCN-NEXT:    v_and_b32_e32 v3, s8, v1
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GCN-NEXT:    s_and_b32 s0, s1, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, v3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v6, s0
+; GCN-NEXT:    s_and_b32 s0, s3, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v7, s0
+; GCN-NEXT:    v_mul_f32_e32 v5, v3, v5
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v6
+; GCN-NEXT:    v_mad_f32 v3, -v5, v2, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v2
+; GCN-NEXT:    v_mul_f32_e32 v3, v7, v8
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, 0, v5, vcc
+; GCN-NEXT:    v_trunc_f32_e32 v3, v3
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v3
+; GCN-NEXT:    v_mad_f32 v3, -v3, v6, v7
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v6
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_and_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s3, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = urem <3 x i16> %x, %y
   store <3 x i16> %r, <3 x i16> addrspace(1)* %out
   ret void
@@ -1939,7 +3191,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
@@ -1963,7 +3215,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP30]], 1
 ; CHECK-NEXT:    [[TMP32:%.*]] = sitofp i32 [[TMP27]] to float
 ; CHECK-NEXT:    [[TMP33:%.*]] = sitofp i32 [[TMP28]] to float
-; CHECK-NEXT:    [[TMP34:%.*]] = fdiv fast float 1.000000e+00, [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
 ; CHECK-NEXT:    [[TMP37:%.*]] = fneg fast float [[TMP36]]
@@ -1987,7 +3239,7 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP54]], 1
 ; CHECK-NEXT:    [[TMP56:%.*]] = sitofp i32 [[TMP51]] to float
 ; CHECK-NEXT:    [[TMP57:%.*]] = sitofp i32 [[TMP52]] to float
-; CHECK-NEXT:    [[TMP58:%.*]] = fdiv fast float 1.000000e+00, [[TMP57]]
+; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
 ; CHECK-NEXT:    [[TMP61:%.*]] = fneg fast float [[TMP60]]
@@ -2005,6 +3257,68 @@ define amdgpu_kernel void @sdiv_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    store <3 x i16> [[TMP72]], <3 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_v3i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_sext_i32_i16 s9, s2
+; GCN-NEXT:    s_sext_i32_i16 s8, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s8
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s9
+; GCN-NEXT:    s_xor_b32 s8, s9, s8
+; GCN-NEXT:    s_ashr_i32 s0, s0, 16
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_ashr_i32 s8, s8, 30
+; GCN-NEXT:    s_or_b32 s8, s8, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s8
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    s_ashr_i32 s2, s2, 16
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v3, v1
+; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v3, v2, v3
+; GCN-NEXT:    v_trunc_f32_e32 v3, v3
+; GCN-NEXT:    v_mad_f32 v2, -v3, v1, v2
+; GCN-NEXT:    v_mov_b32_e32 v4, s0
+; GCN-NEXT:    s_sext_i32_i16 s0, s1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v2|, |v1|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v1, 0, v4, vcc
+; GCN-NEXT:    s_sext_i32_i16 s1, s3
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_mov_b32_e32 v5, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = sdiv <3 x i16> %x, %y
   store <3 x i16> %r, <3 x i16> addrspace(1)* %out
   ret void
@@ -2021,7 +3335,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
@@ -2047,7 +3361,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP32]], 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = sitofp i32 [[TMP29]] to float
 ; CHECK-NEXT:    [[TMP35:%.*]] = sitofp i32 [[TMP30]] to float
-; CHECK-NEXT:    [[TMP36:%.*]] = fdiv fast float 1.000000e+00, [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
 ; CHECK-NEXT:    [[TMP39:%.*]] = fneg fast float [[TMP38]]
@@ -2073,7 +3387,7 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP58]], 1
 ; CHECK-NEXT:    [[TMP60:%.*]] = sitofp i32 [[TMP55]] to float
 ; CHECK-NEXT:    [[TMP61:%.*]] = sitofp i32 [[TMP56]] to float
-; CHECK-NEXT:    [[TMP62:%.*]] = fdiv fast float 1.000000e+00, [[TMP61]]
+; CHECK-NEXT:    [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
 ; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
 ; CHECK-NEXT:    [[TMP65:%.*]] = fneg fast float [[TMP64]]
@@ -2093,6 +3407,77 @@ define amdgpu_kernel void @srem_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %x
 ; CHECK-NEXT:    store <3 x i16> [[TMP78]], <3 x i16> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_v3i16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_sext_i32_i16 s8, s2
+; GCN-NEXT:    s_sext_i32_i16 s6, s0
+; GCN-NEXT:    v_cvt_f32_i32_e32 v0, s6
+; GCN-NEXT:    v_cvt_f32_i32_e32 v1, s8
+; GCN-NEXT:    s_xor_b32 s6, s8, s6
+; GCN-NEXT:    s_ashr_i32 s6, s6, 30
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v2, v0
+; GCN-NEXT:    s_or_b32 s6, s6, 1
+; GCN-NEXT:    v_mov_b32_e32 v3, s6
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mul_f32_e32 v2, v1, v2
+; GCN-NEXT:    v_trunc_f32_e32 v2, v2
+; GCN-NEXT:    v_mad_f32 v1, -v2, v0, v1
+; GCN-NEXT:    v_cvt_i32_f32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, |v0|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_alignbit_b32 v2, s1, v2, 16
+; GCN-NEXT:    v_bfe_i32 v3, v2, 0, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, v3
+; GCN-NEXT:    v_alignbit_b32 v1, s3, v1, 16
+; GCN-NEXT:    v_bfe_i32 v5, v1, 0, 16
+; GCN-NEXT:    v_cvt_f32_i32_e32 v6, v5
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v4
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s0
+; GCN-NEXT:    v_xor_b32_e32 v3, v5, v3
+; GCN-NEXT:    s_sext_i32_i16 s0, s1
+; GCN-NEXT:    v_mul_f32_e32 v5, v6, v7
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    v_mad_f32 v6, -v5, v4, v6
+; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_ashrrev_i32_e32 v3, 30, v3
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v6|, |v4|
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s0
+; GCN-NEXT:    v_or_b32_e32 v3, 1, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    s_sext_i32_i16 s2, s3
+; GCN-NEXT:    v_mul_lo_u32 v2, v3, v2
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v4
+; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v5, v3, v5
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mad_f32 v3, -v5, v4, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_mov_b32_e32 v6, s0
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v4|
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s3, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    buffer_store_short v2, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %r = srem <3 x i16> %x, %y
   store <3 x i16> %r, <3 x i16> addrspace(1)* %out
   ret void
@@ -2106,7 +3491,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i15 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP6:%.*]] = uitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
@@ -2126,7 +3511,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext i15 [[TMP22]] to i32
 ; CHECK-NEXT:    [[TMP25:%.*]] = uitofp i32 [[TMP23]] to float
 ; CHECK-NEXT:    [[TMP26:%.*]] = uitofp i32 [[TMP24]] to float
-; CHECK-NEXT:    [[TMP27:%.*]] = fdiv fast float 1.000000e+00, [[TMP26]]
+; CHECK-NEXT:    [[TMP27:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP26]])
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul fast float [[TMP25]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.trunc.f32(float [[TMP28]])
 ; CHECK-NEXT:    [[TMP30:%.*]] = fneg fast float [[TMP29]]
@@ -2146,7 +3531,7 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP44:%.*]] = zext i15 [[TMP42]] to i32
 ; CHECK-NEXT:    [[TMP45:%.*]] = uitofp i32 [[TMP43]] to float
 ; CHECK-NEXT:    [[TMP46:%.*]] = uitofp i32 [[TMP44]] to float
-; CHECK-NEXT:    [[TMP47:%.*]] = fdiv fast float 1.000000e+00, [[TMP46]]
+; CHECK-NEXT:    [[TMP47:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP46]])
 ; CHECK-NEXT:    [[TMP48:%.*]] = fmul fast float [[TMP45]], [[TMP47]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call fast float @llvm.trunc.f32(float [[TMP48]])
 ; CHECK-NEXT:    [[TMP50:%.*]] = fneg fast float [[TMP49]]
@@ -2163,6 +3548,62 @@ define amdgpu_kernel void @udiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    store <3 x i15> [[TMP60]], <3 x i15> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: udiv_v3i15:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_alignbit_b32 v0, s3, v0, 30
+; GCN-NEXT:    s_movk_i32 s3, 0x7fff
+; GCN-NEXT:    s_and_b32 s9, s0, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s9
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    s_and_b32 s8, s2, s3
+; GCN-NEXT:    s_bfe_u32 s0, s0, 0xf000f
+; GCN-NEXT:    v_cvt_f32_u32_e32 v5, s0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v1
+; GCN-NEXT:    s_bfe_u32 s2, s2, 0xf000f
+; GCN-NEXT:    v_alignbit_b32 v2, s1, v2, 30
+; GCN-NEXT:    v_cvt_f32_u32_e32 v6, s2
+; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v7, v5
+; GCN-NEXT:    v_and_b32_e32 v2, s3, v2
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v3, -v4, v1, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
+; GCN-NEXT:    v_mul_f32_e32 v1, v6, v7
+; GCN-NEXT:    v_and_b32_e32 v0, s3, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GCN-NEXT:    v_mad_f32 v4, -v1, v5, v6
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v2
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, v5
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_f32_e32 v1, v0, v6
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v1
+; GCN-NEXT:    v_mad_f32 v0, -v1, v2, v0
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, v2
+; GCN-NEXT:    v_and_b32_e32 v2, s3, v3
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, 0, v5, vcc
+; GCN-NEXT:    v_and_b32_e32 v3, s3, v4
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x1fff, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
+; GCN-NEXT:    s_endpgm
   %r = udiv <3 x i15> %x, %y
   store <3 x i15> %r, <3 x i15> addrspace(1)* %out
   ret void
@@ -2176,7 +3617,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i15 [[TMP2]] to i32
 ; CHECK-NEXT:    [[TMP5:%.*]] = uitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP6:%.*]] = uitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP7:%.*]] = fdiv fast float 1.000000e+00, [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP6]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.trunc.f32(float [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = fneg fast float [[TMP9]]
@@ -2198,7 +3639,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP26:%.*]] = zext i15 [[TMP24]] to i32
 ; CHECK-NEXT:    [[TMP27:%.*]] = uitofp i32 [[TMP25]] to float
 ; CHECK-NEXT:    [[TMP28:%.*]] = uitofp i32 [[TMP26]] to float
-; CHECK-NEXT:    [[TMP29:%.*]] = fdiv fast float 1.000000e+00, [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP28]])
 ; CHECK-NEXT:    [[TMP30:%.*]] = fmul fast float [[TMP27]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call fast float @llvm.trunc.f32(float [[TMP30]])
 ; CHECK-NEXT:    [[TMP32:%.*]] = fneg fast float [[TMP31]]
@@ -2220,7 +3661,7 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP48:%.*]] = zext i15 [[TMP46]] to i32
 ; CHECK-NEXT:    [[TMP49:%.*]] = uitofp i32 [[TMP47]] to float
 ; CHECK-NEXT:    [[TMP50:%.*]] = uitofp i32 [[TMP48]] to float
-; CHECK-NEXT:    [[TMP51:%.*]] = fdiv fast float 1.000000e+00, [[TMP50]]
+; CHECK-NEXT:    [[TMP51:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP50]])
 ; CHECK-NEXT:    [[TMP52:%.*]] = fmul fast float [[TMP49]], [[TMP51]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call fast float @llvm.trunc.f32(float [[TMP52]])
 ; CHECK-NEXT:    [[TMP54:%.*]] = fneg fast float [[TMP53]]
@@ -2239,6 +3680,70 @@ define amdgpu_kernel void @urem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    store <3 x i15> [[TMP66]], <3 x i15> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: urem_v3i15:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_alignbit_b32 v0, s3, v0, 30
+; GCN-NEXT:    s_movk_i32 s3, 0x7fff
+; GCN-NEXT:    s_and_b32 s10, s0, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s10
+; GCN-NEXT:    s_and_b32 s9, s2, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s9
+; GCN-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v1
+; GCN-NEXT:    v_alignbit_b32 v2, s1, v2, 30
+; GCN-NEXT:    s_bfe_u32 s1, s0, 0xf000f
+; GCN-NEXT:    v_cvt_f32_u32_e32 v5, s1
+; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v3, -v4, v1, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v4, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v1
+; GCN-NEXT:    s_bfe_u32 s10, s2, 0xf000f
+; GCN-NEXT:    v_cvt_f32_u32_e32 v3, s10
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v5
+; GCN-NEXT:    v_and_b32_e32 v2, s3, v2
+; GCN-NEXT:    v_and_b32_e32 v0, s3, v0
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, s2, v1
+; GCN-NEXT:    v_mul_f32_e32 v1, v3, v4
+; GCN-NEXT:    v_cvt_f32_u32_e32 v4, v2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v7, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mad_f32 v3, -v1, v5, v3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v5
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_lshr_b32 s0, s0, 15
+; GCN-NEXT:    v_mul_f32_e32 v3, v7, v8
+; GCN-NEXT:    v_trunc_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v3
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mad_f32 v3, -v3, v4, v7
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, v4
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s0
+; GCN-NEXT:    v_mul_lo_u32 v2, v3, v2
+; GCN-NEXT:    s_lshr_b32 s8, s2, 15
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v1
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_and_b32_e32 v3, s3, v3
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
+; GCN-NEXT:    v_and_b32_e32 v2, s3, v6
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x1fff, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
+; GCN-NEXT:    s_endpgm
   %r = urem <3 x i15> %x, %y
   store <3 x i15> %r, <3 x i15> addrspace(1)* %out
   ret void
@@ -2255,7 +3760,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
@@ -2279,7 +3784,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP31:%.*]] = or i32 [[TMP30]], 1
 ; CHECK-NEXT:    [[TMP32:%.*]] = sitofp i32 [[TMP27]] to float
 ; CHECK-NEXT:    [[TMP33:%.*]] = sitofp i32 [[TMP28]] to float
-; CHECK-NEXT:    [[TMP34:%.*]] = fdiv fast float 1.000000e+00, [[TMP33]]
+; CHECK-NEXT:    [[TMP34:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP33]])
 ; CHECK-NEXT:    [[TMP35:%.*]] = fmul fast float [[TMP32]], [[TMP34]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.trunc.f32(float [[TMP35]])
 ; CHECK-NEXT:    [[TMP37:%.*]] = fneg fast float [[TMP36]]
@@ -2303,7 +3808,7 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP55:%.*]] = or i32 [[TMP54]], 1
 ; CHECK-NEXT:    [[TMP56:%.*]] = sitofp i32 [[TMP51]] to float
 ; CHECK-NEXT:    [[TMP57:%.*]] = sitofp i32 [[TMP52]] to float
-; CHECK-NEXT:    [[TMP58:%.*]] = fdiv fast float 1.000000e+00, [[TMP57]]
+; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
 ; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP56]], [[TMP58]]
 ; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.trunc.f32(float [[TMP59]])
 ; CHECK-NEXT:    [[TMP61:%.*]] = fneg fast float [[TMP60]]
@@ -2321,6 +3826,76 @@ define amdgpu_kernel void @sdiv_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    store <3 x i15> [[TMP72]], <3 x i15> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: sdiv_v3i15:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_alignbit_b32 v0, s3, v0, 30
+; GCN-NEXT:    s_bfe_i32 s3, s0, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s3
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
+; GCN-NEXT:    s_bfe_i32 s1, s2, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GCN-NEXT:    s_xor_b32 s1, s1, s3
+; GCN-NEXT:    s_bfe_i32 s0, s0, 0xf000f
+; GCN-NEXT:    s_ashr_i32 s1, s1, 30
+; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
+; GCN-NEXT:    s_or_b32 s1, s1, 1
+; GCN-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
+; GCN-NEXT:    s_bfe_i32 s1, s2, 0xf000f
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s1
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    s_xor_b32 s0, s1, s0
+; GCN-NEXT:    v_bfe_i32 v1, v1, 0, 15
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mad_f32 v4, -v5, v3, v4
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
+; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, v1
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mov_b32_e32 v6, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    v_bfe_i32 v0, v0, 0, 15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_cvt_f32_i32_e32 v5, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v6, v4
+; GCN-NEXT:    v_xor_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
+; GCN-NEXT:    v_or_b32_e32 v0, 1, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, v5, v6
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mad_f32 v5, -v1, v4, v5
+; GCN-NEXT:    v_cvt_i32_f32_e32 v1, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v5|, |v4|
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN-NEXT:    s_movk_i32 s0, 0x7fff
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_and_b32_e32 v3, s0, v3
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
+; GCN-NEXT:    v_and_b32_e32 v2, s0, v2
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x1fff, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
+; GCN-NEXT:    s_endpgm
   %r = sdiv <3 x i15> %x, %y
   store <3 x i15> %r, <3 x i15> addrspace(1)* %out
   ret void
@@ -2337,7 +3912,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sitofp i32 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP9:%.*]] = sitofp i32 [[TMP4]] to float
-; CHECK-NEXT:    [[TMP10:%.*]] = fdiv fast float 1.000000e+00, [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.trunc.f32(float [[TMP11]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = fneg fast float [[TMP12]]
@@ -2363,7 +3938,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP33:%.*]] = or i32 [[TMP32]], 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = sitofp i32 [[TMP29]] to float
 ; CHECK-NEXT:    [[TMP35:%.*]] = sitofp i32 [[TMP30]] to float
-; CHECK-NEXT:    [[TMP36:%.*]] = fdiv fast float 1.000000e+00, [[TMP35]]
+; CHECK-NEXT:    [[TMP36:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP35]])
 ; CHECK-NEXT:    [[TMP37:%.*]] = fmul fast float [[TMP34]], [[TMP36]]
 ; CHECK-NEXT:    [[TMP38:%.*]] = call fast float @llvm.trunc.f32(float [[TMP37]])
 ; CHECK-NEXT:    [[TMP39:%.*]] = fneg fast float [[TMP38]]
@@ -2389,7 +3964,7 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    [[TMP59:%.*]] = or i32 [[TMP58]], 1
 ; CHECK-NEXT:    [[TMP60:%.*]] = sitofp i32 [[TMP55]] to float
 ; CHECK-NEXT:    [[TMP61:%.*]] = sitofp i32 [[TMP56]] to float
-; CHECK-NEXT:    [[TMP62:%.*]] = fdiv fast float 1.000000e+00, [[TMP61]]
+; CHECK-NEXT:    [[TMP62:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP61]])
 ; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP60]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP64:%.*]] = call fast float @llvm.trunc.f32(float [[TMP63]])
 ; CHECK-NEXT:    [[TMP65:%.*]] = fneg fast float [[TMP64]]
@@ -2409,7 +3984,3343 @@ define amdgpu_kernel void @srem_v3i15(<3 x i15> addrspace(1)* %out, <3 x i15> %x
 ; CHECK-NEXT:    store <3 x i15> [[TMP78]], <3 x i15> addrspace(1)* [[OUT:%.*]]
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: srem_v3i15:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_alignbit_b32 v0, s3, v0, 30
+; GCN-NEXT:    s_movk_i32 s3, 0x7fff
+; GCN-NEXT:    s_and_b32 s11, s0, s3
+; GCN-NEXT:    s_bfe_i32 s11, s11, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v2, s11
+; GCN-NEXT:    s_and_b32 s9, s2, s3
+; GCN-NEXT:    s_bfe_i32 s9, s9, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s9
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v4, v2
+; GCN-NEXT:    s_xor_b32 s9, s9, s11
+; GCN-NEXT:    s_ashr_i32 s9, s9, 30
+; GCN-NEXT:    s_or_b32 s9, s9, 1
+; GCN-NEXT:    v_mul_f32_e32 v4, v3, v4
+; GCN-NEXT:    v_trunc_f32_e32 v4, v4
+; GCN-NEXT:    v_mad_f32 v3, -v4, v2, v3
+; GCN-NEXT:    v_cvt_i32_f32_e32 v4, v4
+; GCN-NEXT:    v_mov_b32_e32 v5, s9
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v2|
+; GCN-NEXT:    v_cndmask_b32_e32 v2, 0, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    s_bfe_u32 s12, s0, 0xf000f
+; GCN-NEXT:    v_alignbit_b32 v1, s1, v1, 30
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, s0
+; GCN-NEXT:    s_lshr_b32 s1, s0, 15
+; GCN-NEXT:    s_bfe_i32 s0, s12, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v3, s0
+; GCN-NEXT:    s_bfe_u32 s10, s2, 0xf000f
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s2, v2
+; GCN-NEXT:    s_lshr_b32 s8, s2, 15
+; GCN-NEXT:    s_bfe_i32 s2, s10, 0xf0000
+; GCN-NEXT:    v_cvt_f32_i32_e32 v4, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v5, v3
+; GCN-NEXT:    s_xor_b32 s0, s2, s0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 30
+; GCN-NEXT:    s_or_b32 s0, s0, 1
+; GCN-NEXT:    v_mul_f32_e32 v5, v4, v5
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mad_f32 v4, -v5, v3, v4
+; GCN-NEXT:    v_cvt_i32_f32_e32 v5, v5
+; GCN-NEXT:    v_and_b32_e32 v1, s3, v1
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v4|, |v3|
+; GCN-NEXT:    v_mov_b32_e32 v6, s0
+; GCN-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
+; GCN-NEXT:    v_bfe_i32 v4, v1, 0, 15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_cvt_f32_i32_e32 v5, v4
+; GCN-NEXT:    v_and_b32_e32 v0, s3, v0
+; GCN-NEXT:    v_bfe_i32 v6, v0, 0, 15
+; GCN-NEXT:    v_cvt_f32_i32_e32 v7, v6
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v8, v5
+; GCN-NEXT:    v_xor_b32_e32 v4, v6, v4
+; GCN-NEXT:    v_ashrrev_i32_e32 v4, 30, v4
+; GCN-NEXT:    v_or_b32_e32 v4, 1, v4
+; GCN-NEXT:    v_mul_f32_e32 v6, v7, v8
+; GCN-NEXT:    v_trunc_f32_e32 v6, v6
+; GCN-NEXT:    v_mad_f32 v7, -v6, v5, v7
+; GCN-NEXT:    v_cvt_i32_f32_e32 v6, v6
+; GCN-NEXT:    v_cmp_ge_f32_e64 vcc, |v7|, |v5|
+; GCN-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, s1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_lo_u32 v1, v4, v1
+; GCN-NEXT:    v_and_b32_e32 v2, s3, v2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v3
+; GCN-NEXT:    v_and_b32_e32 v3, s3, v3
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_lshl_b64 v[0:1], v[0:1], 30
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 15, v3
+; GCN-NEXT:    v_or_b32_e32 v2, v2, v3
+; GCN-NEXT:    v_or_b32_e32 v0, v2, v0
+; GCN-NEXT:    v_and_b32_e32 v1, 0x1fff, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    buffer_store_short v1, off, s[4:7], 0 offset:4
+; GCN-NEXT:    s_endpgm
   %r = srem <3 x i15> %x, %y
   store <3 x i15> %r, <3 x i15> addrspace(1)* %out
   ret void
 }
+
+define amdgpu_kernel void @udiv_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @udiv_i32_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], 1235195
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_i32_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    v_mov_b32_e32 v0, 0xb2a50881
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s0, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 20, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv i32 %x, 1235195
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @udiv_i32_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], 4096
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s0, 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv i32 %x, 4096
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %x, i32 %y) {
+; CHECK-LABEL: @udiv_i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = udiv i32 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_i32 s1, s1, 12
+; GCN-NEXT:    s_lshr_b32 s0, s0, s1
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i32 4096, %y
+  %r = udiv i32 %x, %shl.y
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x) {
+; CHECK-LABEL: @udiv_v2i32_pow2k_denom(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], 4096
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_v2i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b32 s0, s0, 12
+; GCN-NEXT:    s_lshr_b32 s1, s1, 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv <2 x i32> %x, <i32 4096, i32 4096>
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_v2i32_mixed_pow2k_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x) {
+; CHECK-LABEL: @udiv_v2i32_mixed_pow2k_denom(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], 4095
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_v2i32_mixed_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x100101
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GCN-NEXT:    s_lshr_b32 s0, s0, 12
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s1, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 11, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv <2 x i32> %x, <i32 4096, i32 4095>
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @udiv_v2i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i32> <i32 4096, i32 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
+; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP15]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP6]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP6]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP14]], i32 [[TMP22]], i32 [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP27]], 32
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = add i32 [[TMP30]], 1
+; CHECK-NEXT:    [[TMP37:%.*]] = sub i32 [[TMP30]], 1
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <2 x i32> undef, i32 [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
+; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
+; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
+; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
+; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
+; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
+; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
+; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
+; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
+; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
+; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
+; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
+; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP70]], 1
+; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP70]], 1
+; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP70]]
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
+; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <2 x i32> [[TMP40]], i32 [[TMP79]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP80]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_v2i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, s4, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    s_lshl_b32 s10, s4, s3
+; GCN-NEXT:    s_mov_b32 s3, 0x4f800000
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s10
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, s3, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s3, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, s10
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, s10
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v5
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s2, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s10
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s8, v5
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[2:3]
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
+  %r = udiv <2 x i32> %x, %shl.y
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @urem_i32_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = urem i32 [[X:%.*]], 1235195
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_i32_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    v_mov_b32_e32 v0, 0xb2a50881
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, s0, v0
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s0, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 1, v1
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_lshrrev_b32_e32 v0, 20, v0
+; GCN-NEXT:    v_mul_u32_u24_e32 v0, 0x12d8fb, v0
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = urem i32 %x, 1235195
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @urem_i32_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = urem i32 [[X:%.*]], 4096
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s0, s0, 0xfff
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = urem i32 %x, 4096
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %x, i32 %y) {
+; CHECK-LABEL: @urem_i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = urem i32 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s1, 0x1000, s1
+; GCN-NEXT:    s_add_i32 s1, s1, -1
+; GCN-NEXT:    s_and_b32 s0, s0, s1
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i32 4096, %y
+  %r = urem i32 %x, %shl.y
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x) {
+; CHECK-LABEL: @urem_v2i32_pow2k_denom(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = urem i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = urem i32 [[TMP4]], 4096
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_v2i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    s_movk_i32 s2, 0xfff
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s0, s0, s2
+; GCN-NEXT:    s_and_b32 s1, s1, s2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = urem <2 x i32> %x, <i32 4096, i32 4096>
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @urem_v2i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i32> <i32 4096, i32 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = uitofp i32 [[TMP2]] to float
+; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[TMP4]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP6:%.*]] = fptoui float [[TMP5]] to i32
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = lshr i64 [[TMP9]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 0, [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP12]], 0
+; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP15]] to i64
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = lshr i64 [[TMP18]], 32
+; CHECK-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[TMP6]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP6]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP14]], i32 [[TMP22]], i32 [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP25]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP29:%.*]] = lshr i64 [[TMP27]], 32
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP32:%.*]] = sub i32 [[TMP1]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp uge i32 [[TMP32]], [[TMP2]]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp uge i32 [[TMP1]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and i1 [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = sub i32 [[TMP32]], [[TMP2]]
+; CHECK-NEXT:    [[TMP37:%.*]] = add i32 [[TMP32]], [[TMP2]]
+; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP35]], i32 [[TMP36]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP34]], i32 [[TMP38]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <2 x i32> undef, i32 [[TMP39]], i64 0
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP43:%.*]] = uitofp i32 [[TMP42]] to float
+; CHECK-NEXT:    [[TMP44:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP43]])
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast float [[TMP44]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP46:%.*]] = fptoui float [[TMP45]] to i32
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i32 [[TMP42]] to i64
+; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = trunc i64 [[TMP49]] to i32
+; CHECK-NEXT:    [[TMP51:%.*]] = lshr i64 [[TMP49]], 32
+; CHECK-NEXT:    [[TMP52:%.*]] = trunc i64 [[TMP51]] to i32
+; CHECK-NEXT:    [[TMP53:%.*]] = sub i32 0, [[TMP50]]
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp eq i32 [[TMP52]], 0
+; CHECK-NEXT:    [[TMP55:%.*]] = select i1 [[TMP54]], i32 [[TMP53]], i32 [[TMP50]]
+; CHECK-NEXT:    [[TMP56:%.*]] = zext i32 [[TMP55]] to i64
+; CHECK-NEXT:    [[TMP57:%.*]] = zext i32 [[TMP46]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP56]], [[TMP57]]
+; CHECK-NEXT:    [[TMP59:%.*]] = trunc i64 [[TMP58]] to i32
+; CHECK-NEXT:    [[TMP60:%.*]] = lshr i64 [[TMP58]], 32
+; CHECK-NEXT:    [[TMP61:%.*]] = trunc i64 [[TMP60]] to i32
+; CHECK-NEXT:    [[TMP62:%.*]] = add i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = sub i32 [[TMP46]], [[TMP61]]
+; CHECK-NEXT:    [[TMP64:%.*]] = select i1 [[TMP54]], i32 [[TMP62]], i32 [[TMP63]]
+; CHECK-NEXT:    [[TMP65:%.*]] = zext i32 [[TMP64]] to i64
+; CHECK-NEXT:    [[TMP66:%.*]] = zext i32 [[TMP41]] to i64
+; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP65]], [[TMP66]]
+; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP69:%.*]] = lshr i64 [[TMP67]], 32
+; CHECK-NEXT:    [[TMP70:%.*]] = trunc i64 [[TMP69]] to i32
+; CHECK-NEXT:    [[TMP71:%.*]] = mul i32 [[TMP70]], [[TMP42]]
+; CHECK-NEXT:    [[TMP72:%.*]] = sub i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp uge i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP74:%.*]] = icmp uge i32 [[TMP41]], [[TMP71]]
+; CHECK-NEXT:    [[TMP75:%.*]] = and i1 [[TMP73]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = sub i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP77:%.*]] = add i32 [[TMP72]], [[TMP42]]
+; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP75]], i32 [[TMP76]], i32 [[TMP72]]
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP74]], i32 [[TMP78]], i32 [[TMP77]]
+; CHECK-NEXT:    [[TMP80:%.*]] = insertelement <2 x i32> [[TMP40]], i32 [[TMP79]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP80]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_v2i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s10, s4, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; GCN-NEXT:    s_mov_b32 s2, 0x4f800000
+; GCN-NEXT:    s_lshl_b32 s11, s4, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s11
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xb
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_f32_e32 v0, s2, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s2, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s10
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s10
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v2
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, s11
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v2, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v2, v0
+; GCN-NEXT:    v_mul_hi_u32 v2, v1, s11
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v4, s[0:1]
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, 0, v3
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s8
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v2, v2, v1
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s10
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s8, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s8, v0
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s10, v3
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s11
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, s10, v3
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v3
+; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s9, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s[0:1]
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s9, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s11, v2
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s11, v2
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
+  %r = urem <2 x i32> %x, %shl.y
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @sdiv_i32_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], 1235195
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_i32_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    v_mov_b32_e32 v0, 0xd9528441
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_i32 v0, s0, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 31, v0
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 20, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv i32 %x, 1235195
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @sdiv_i32_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], 4096
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s1, s0, 31
+; GCN-NEXT:    s_lshr_b32 s1, s1, 20
+; GCN-NEXT:    s_add_i32 s0, s0, s1
+; GCN-NEXT:    s_ashr_i32 s0, s0, 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv i32 %x, 4096
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %x, i32 %y) {
+; CHECK-LABEL: @sdiv_i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sdiv i32 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s3, 0x1000, s3
+; GCN-NEXT:    s_ashr_i32 s8, s3, 31
+; GCN-NEXT:    s_add_i32 s3, s3, s8
+; GCN-NEXT:    s_xor_b32 s9, s3, s8
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s9
+; GCN-NEXT:    s_ashr_i32 s3, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_xor_b32 s2, s2, s3
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s3, s3, s8
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s9
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s2
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s9
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s2, v1
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s2, v1
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; GCN-NEXT:    s_and_b64 vcc, vcc, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[0:1]
+; GCN-NEXT:    v_xor_b32_e32 v0, s3, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s3, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i32 4096, %y
+  %r = sdiv i32 %x, %shl.y
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x) {
+; CHECK-LABEL: @sdiv_v2i32_pow2k_denom(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sdiv i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i32 [[TMP4]], 4096
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_v2i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s0, 31
+; GCN-NEXT:    s_lshr_b32 s2, s2, 20
+; GCN-NEXT:    s_ashr_i32 s3, s1, 31
+; GCN-NEXT:    s_add_i32 s0, s0, s2
+; GCN-NEXT:    s_lshr_b32 s2, s3, 20
+; GCN-NEXT:    s_add_i32 s1, s1, s2
+; GCN-NEXT:    s_ashr_i32 s0, s0, 12
+; GCN-NEXT:    s_ashr_i32 s1, s1, 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv <2 x i32> %x, <i32 4096, i32 4096>
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @ssdiv_v2i32_mixed_pow2k_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x) {
+; CHECK-LABEL: @ssdiv_v2i32_mixed_pow2k_denom(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = sdiv i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i32 [[TMP4]], 4095
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: ssdiv_v2i32_mixed_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x80080081
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_i32 v0, s1, v0
+; GCN-NEXT:    s_ashr_i32 s2, s0, 31
+; GCN-NEXT:    s_lshr_b32 s2, s2, 20
+; GCN-NEXT:    s_add_i32 s0, s0, s2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s1, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 31, v0
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 11, v0
+; GCN-NEXT:    s_ashr_i32 s0, s0, 12
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv <2 x i32> %x, <i32 4096, i32 4095>
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @sdiv_v2i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i32> <i32 4096, i32 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i32 [[TMP1]], 31
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i32 [[TMP2]], 31
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP6]], [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i32 [[TMP7]], [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = uitofp i32 [[TMP9]] to float
+; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP10]])
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul fast float [[TMP11]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP13:%.*]] = fptoui float [[TMP12]] to i32
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = lshr i64 [[TMP16]], 32
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = sub i32 0, [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[TMP19]], 0
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP17]]
+; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP22]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP25]] to i32
+; CHECK-NEXT:    [[TMP27:%.*]] = lshr i64 [[TMP25]], 32
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP29:%.*]] = add i32 [[TMP13]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP13]], [[TMP28]]
+; CHECK-NEXT:    [[TMP31:%.*]] = select i1 [[TMP21]], i32 [[TMP29]], i32 [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[TMP31]] to i64
+; CHECK-NEXT:    [[TMP33:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP35:%.*]] = trunc i64 [[TMP34]] to i32
+; CHECK-NEXT:    [[TMP36:%.*]] = lshr i64 [[TMP34]], 32
+; CHECK-NEXT:    [[TMP37:%.*]] = trunc i64 [[TMP36]] to i32
+; CHECK-NEXT:    [[TMP38:%.*]] = mul i32 [[TMP37]], [[TMP9]]
+; CHECK-NEXT:    [[TMP39:%.*]] = sub i32 [[TMP8]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP39]], [[TMP9]]
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp uge i32 [[TMP8]], [[TMP38]]
+; CHECK-NEXT:    [[TMP42:%.*]] = and i1 [[TMP40]], [[TMP41]]
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP37]], 1
+; CHECK-NEXT:    [[TMP44:%.*]] = sub i32 [[TMP37]], 1
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP42]], i32 [[TMP43]], i32 [[TMP37]]
+; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP41]], i32 [[TMP45]], i32 [[TMP44]]
+; CHECK-NEXT:    [[TMP47:%.*]] = xor i32 [[TMP46]], [[TMP5]]
+; CHECK-NEXT:    [[TMP48:%.*]] = sub i32 [[TMP47]], [[TMP5]]
+; CHECK-NEXT:    [[TMP49:%.*]] = insertelement <2 x i32> undef, i32 [[TMP48]], i64 0
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP53:%.*]] = ashr i32 [[TMP51]], 31
+; CHECK-NEXT:    [[TMP54:%.*]] = xor i32 [[TMP52]], [[TMP53]]
+; CHECK-NEXT:    [[TMP55:%.*]] = add i32 [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP56:%.*]] = add i32 [[TMP51]], [[TMP53]]
+; CHECK-NEXT:    [[TMP57:%.*]] = xor i32 [[TMP55]], [[TMP52]]
+; CHECK-NEXT:    [[TMP58:%.*]] = xor i32 [[TMP56]], [[TMP53]]
+; CHECK-NEXT:    [[TMP59:%.*]] = uitofp i32 [[TMP58]] to float
+; CHECK-NEXT:    [[TMP60:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP59]])
+; CHECK-NEXT:    [[TMP61:%.*]] = fmul fast float [[TMP60]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP62:%.*]] = fptoui float [[TMP61]] to i32
+; CHECK-NEXT:    [[TMP63:%.*]] = zext i32 [[TMP62]] to i64
+; CHECK-NEXT:    [[TMP64:%.*]] = zext i32 [[TMP58]] to i64
+; CHECK-NEXT:    [[TMP65:%.*]] = mul i64 [[TMP63]], [[TMP64]]
+; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
+; CHECK-NEXT:    [[TMP67:%.*]] = lshr i64 [[TMP65]], 32
+; CHECK-NEXT:    [[TMP68:%.*]] = trunc i64 [[TMP67]] to i32
+; CHECK-NEXT:    [[TMP69:%.*]] = sub i32 0, [[TMP66]]
+; CHECK-NEXT:    [[TMP70:%.*]] = icmp eq i32 [[TMP68]], 0
+; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP69]], i32 [[TMP66]]
+; CHECK-NEXT:    [[TMP72:%.*]] = zext i32 [[TMP71]] to i64
+; CHECK-NEXT:    [[TMP73:%.*]] = zext i32 [[TMP62]] to i64
+; CHECK-NEXT:    [[TMP74:%.*]] = mul i64 [[TMP72]], [[TMP73]]
+; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
+; CHECK-NEXT:    [[TMP76:%.*]] = lshr i64 [[TMP74]], 32
+; CHECK-NEXT:    [[TMP77:%.*]] = trunc i64 [[TMP76]] to i32
+; CHECK-NEXT:    [[TMP78:%.*]] = add i32 [[TMP62]], [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = sub i32 [[TMP62]], [[TMP77]]
+; CHECK-NEXT:    [[TMP80:%.*]] = select i1 [[TMP70]], i32 [[TMP78]], i32 [[TMP79]]
+; CHECK-NEXT:    [[TMP81:%.*]] = zext i32 [[TMP80]] to i64
+; CHECK-NEXT:    [[TMP82:%.*]] = zext i32 [[TMP57]] to i64
+; CHECK-NEXT:    [[TMP83:%.*]] = mul i64 [[TMP81]], [[TMP82]]
+; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
+; CHECK-NEXT:    [[TMP85:%.*]] = lshr i64 [[TMP83]], 32
+; CHECK-NEXT:    [[TMP86:%.*]] = trunc i64 [[TMP85]] to i32
+; CHECK-NEXT:    [[TMP87:%.*]] = mul i32 [[TMP86]], [[TMP58]]
+; CHECK-NEXT:    [[TMP88:%.*]] = sub i32 [[TMP57]], [[TMP87]]
+; CHECK-NEXT:    [[TMP89:%.*]] = icmp uge i32 [[TMP88]], [[TMP58]]
+; CHECK-NEXT:    [[TMP90:%.*]] = icmp uge i32 [[TMP57]], [[TMP87]]
+; CHECK-NEXT:    [[TMP91:%.*]] = and i1 [[TMP89]], [[TMP90]]
+; CHECK-NEXT:    [[TMP92:%.*]] = add i32 [[TMP86]], 1
+; CHECK-NEXT:    [[TMP93:%.*]] = sub i32 [[TMP86]], 1
+; CHECK-NEXT:    [[TMP94:%.*]] = select i1 [[TMP91]], i32 [[TMP92]], i32 [[TMP86]]
+; CHECK-NEXT:    [[TMP95:%.*]] = select i1 [[TMP90]], i32 [[TMP94]], i32 [[TMP93]]
+; CHECK-NEXT:    [[TMP96:%.*]] = xor i32 [[TMP95]], [[TMP54]]
+; CHECK-NEXT:    [[TMP97:%.*]] = sub i32 [[TMP96]], [[TMP54]]
+; CHECK-NEXT:    [[TMP98:%.*]] = insertelement <2 x i32> [[TMP49]], i32 [[TMP97]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP98]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_v2i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_mov_b32 s14, 0x4f800000
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, s4, s2
+; GCN-NEXT:    s_ashr_i32 s5, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s5
+; GCN-NEXT:    s_xor_b32 s13, s2, s5
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s13
+; GCN-NEXT:    s_ashr_i32 s2, s6, 31
+; GCN-NEXT:    s_lshl_b32 s0, s4, s3
+; GCN-NEXT:    s_add_i32 s1, s6, s2
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_ashr_i32 s6, s0, 31
+; GCN-NEXT:    s_add_i32 s4, s0, s6
+; GCN-NEXT:    s_xor_b32 s3, s1, s2
+; GCN-NEXT:    v_mul_f32_e32 v0, s14, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s15, s4, s6
+; GCN-NEXT:    s_xor_b32 s12, s2, s5
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s13
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s13
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s15
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[0:1]
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v2
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s3
+; GCN-NEXT:    v_mul_f32_e32 v1, s14, v1
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s13
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v0
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s3, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v4
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s15
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, s15
+; GCN-NEXT:    s_ashr_i32 s13, s7, 31
+; GCN-NEXT:    s_add_i32 s7, s7, s13
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
+; GCN-NEXT:    s_xor_b32 s7, s7, s13
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s7
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s15
+; GCN-NEXT:    v_xor_b32_e32 v0, s12, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
+; GCN-NEXT:    s_xor_b32 s4, s13, s6
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s7, v2
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v3
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s7, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, -1, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, 1, v1
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
+; GCN-NEXT:    v_xor_b32_e32 v1, s4, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s4, v1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
+  %r = sdiv <2 x i32> %x, %shl.y
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_i32_oddk_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @srem_i32_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = srem i32 [[X:%.*]], 1235195
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_i32_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    v_mov_b32_e32 v0, 0xd9528441
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_i32 v0, s0, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_lshrrev_b32_e32 v1, 31, v0
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 20, v0
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_i32_i24_e32 v0, 0x12d8fb, v0
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = srem i32 %x, 1235195
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_i32_pow2k_denom(i32 addrspace(1)* %out, i32 %x) {
+; CHECK-LABEL: @srem_i32_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = srem i32 [[X:%.*]], 4096
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s1, s0, 31
+; GCN-NEXT:    s_lshr_b32 s1, s1, 20
+; GCN-NEXT:    s_add_i32 s1, s0, s1
+; GCN-NEXT:    s_and_b32 s1, s1, 0xfffff000
+; GCN-NEXT:    s_sub_i32 s0, s0, s1
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = srem i32 %x, 4096
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_i32_pow2_shl_denom(i32 addrspace(1)* %out, i32 %x, i32 %y) {
+; CHECK-LABEL: @srem_i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i32 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = srem i32 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i32 [[R]], i32 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xb
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, 0x1000, s5
+; GCN-NEXT:    s_ashr_i32 s3, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s3
+; GCN-NEXT:    s_xor_b32 s10, s2, s3
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s10
+; GCN-NEXT:    s_ashr_i32 s8, s4, 31
+; GCN-NEXT:    s_add_i32 s4, s4, s8
+; GCN-NEXT:    s_xor_b32 s9, s4, s8
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x4f800000, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s10
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s10
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s9
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s10
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s9, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[0:1], s9, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, s10, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[2:3], s10, v1
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s10, v1
+; GCN-NEXT:    s_and_b64 vcc, s[2:3], s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, v0, s[0:1]
+; GCN-NEXT:    v_xor_b32_e32 v0, s8, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s8, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i32 4096, %y
+  %r = srem i32 %x, %shl.y
+  store i32 %r, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_v2i32_pow2k_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x) {
+; CHECK-LABEL: @srem_v2i32_pow2k_denom(
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = srem i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = srem i32 [[TMP4]], 4096
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP5]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_v2i32_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GCN-NEXT:    s_movk_i32 s2, 0xf000
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s3, s0, 31
+; GCN-NEXT:    s_lshr_b32 s3, s3, 20
+; GCN-NEXT:    s_add_i32 s3, s0, s3
+; GCN-NEXT:    s_and_b32 s3, s3, s2
+; GCN-NEXT:    s_sub_i32 s0, s0, s3
+; GCN-NEXT:    s_ashr_i32 s3, s1, 31
+; GCN-NEXT:    s_lshr_b32 s3, s3, 20
+; GCN-NEXT:    s_add_i32 s3, s1, s3
+; GCN-NEXT:    s_and_b32 s2, s3, s2
+; GCN-NEXT:    s_sub_i32 s1, s1, s2
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = srem <2 x i32> %x, <i32 4096, i32 4096>
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(<2 x i32> addrspace(1)* %out, <2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @srem_v2i32_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i32> <i32 4096, i32 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = ashr i32 [[TMP1]], 31
+; CHECK-NEXT:    [[TMP4:%.*]] = ashr i32 [[TMP2]], 31
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 [[TMP5]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP6]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
+; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast float [[TMP10]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP12:%.*]] = fptoui float [[TMP11]] to i32
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = lshr i64 [[TMP15]], 32
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP17]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 0, [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP16]]
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP24]] to i32
+; CHECK-NEXT:    [[TMP26:%.*]] = lshr i64 [[TMP24]], 32
+; CHECK-NEXT:    [[TMP27:%.*]] = trunc i64 [[TMP26]] to i32
+; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[TMP12]], [[TMP27]]
+; CHECK-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP12]], [[TMP27]]
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP20]], i32 [[TMP28]], i32 [[TMP29]]
+; CHECK-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP30]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = trunc i64 [[TMP33]] to i32
+; CHECK-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP33]], 32
+; CHECK-NEXT:    [[TMP36:%.*]] = trunc i64 [[TMP35]] to i32
+; CHECK-NEXT:    [[TMP37:%.*]] = mul i32 [[TMP36]], [[TMP8]]
+; CHECK-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP7]], [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = icmp uge i32 [[TMP38]], [[TMP8]]
+; CHECK-NEXT:    [[TMP40:%.*]] = icmp uge i32 [[TMP7]], [[TMP37]]
+; CHECK-NEXT:    [[TMP41:%.*]] = and i1 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP38]], [[TMP8]]
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP38]], [[TMP8]]
+; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP41]], i32 [[TMP42]], i32 [[TMP38]]
+; CHECK-NEXT:    [[TMP45:%.*]] = select i1 [[TMP40]], i32 [[TMP44]], i32 [[TMP43]]
+; CHECK-NEXT:    [[TMP46:%.*]] = xor i32 [[TMP45]], [[TMP3]]
+; CHECK-NEXT:    [[TMP47:%.*]] = sub i32 [[TMP46]], [[TMP3]]
+; CHECK-NEXT:    [[TMP48:%.*]] = insertelement <2 x i32> undef, i32 [[TMP47]], i64 0
+; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <2 x i32> [[X]], i64 1
+; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <2 x i32> [[SHL_Y]], i64 1
+; CHECK-NEXT:    [[TMP51:%.*]] = ashr i32 [[TMP49]], 31
+; CHECK-NEXT:    [[TMP52:%.*]] = ashr i32 [[TMP50]], 31
+; CHECK-NEXT:    [[TMP53:%.*]] = add i32 [[TMP49]], [[TMP51]]
+; CHECK-NEXT:    [[TMP54:%.*]] = add i32 [[TMP50]], [[TMP52]]
+; CHECK-NEXT:    [[TMP55:%.*]] = xor i32 [[TMP53]], [[TMP51]]
+; CHECK-NEXT:    [[TMP56:%.*]] = xor i32 [[TMP54]], [[TMP52]]
+; CHECK-NEXT:    [[TMP57:%.*]] = uitofp i32 [[TMP56]] to float
+; CHECK-NEXT:    [[TMP58:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP57]])
+; CHECK-NEXT:    [[TMP59:%.*]] = fmul fast float [[TMP58]], 0x41F0000000000000
+; CHECK-NEXT:    [[TMP60:%.*]] = fptoui float [[TMP59]] to i32
+; CHECK-NEXT:    [[TMP61:%.*]] = zext i32 [[TMP60]] to i64
+; CHECK-NEXT:    [[TMP62:%.*]] = zext i32 [[TMP56]] to i64
+; CHECK-NEXT:    [[TMP63:%.*]] = mul i64 [[TMP61]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = trunc i64 [[TMP63]] to i32
+; CHECK-NEXT:    [[TMP65:%.*]] = lshr i64 [[TMP63]], 32
+; CHECK-NEXT:    [[TMP66:%.*]] = trunc i64 [[TMP65]] to i32
+; CHECK-NEXT:    [[TMP67:%.*]] = sub i32 0, [[TMP64]]
+; CHECK-NEXT:    [[TMP68:%.*]] = icmp eq i32 [[TMP66]], 0
+; CHECK-NEXT:    [[TMP69:%.*]] = select i1 [[TMP68]], i32 [[TMP67]], i32 [[TMP64]]
+; CHECK-NEXT:    [[TMP70:%.*]] = zext i32 [[TMP69]] to i64
+; CHECK-NEXT:    [[TMP71:%.*]] = zext i32 [[TMP60]] to i64
+; CHECK-NEXT:    [[TMP72:%.*]] = mul i64 [[TMP70]], [[TMP71]]
+; CHECK-NEXT:    [[TMP73:%.*]] = trunc i64 [[TMP72]] to i32
+; CHECK-NEXT:    [[TMP74:%.*]] = lshr i64 [[TMP72]], 32
+; CHECK-NEXT:    [[TMP75:%.*]] = trunc i64 [[TMP74]] to i32
+; CHECK-NEXT:    [[TMP76:%.*]] = add i32 [[TMP60]], [[TMP75]]
+; CHECK-NEXT:    [[TMP77:%.*]] = sub i32 [[TMP60]], [[TMP75]]
+; CHECK-NEXT:    [[TMP78:%.*]] = select i1 [[TMP68]], i32 [[TMP76]], i32 [[TMP77]]
+; CHECK-NEXT:    [[TMP79:%.*]] = zext i32 [[TMP78]] to i64
+; CHECK-NEXT:    [[TMP80:%.*]] = zext i32 [[TMP55]] to i64
+; CHECK-NEXT:    [[TMP81:%.*]] = mul i64 [[TMP79]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = trunc i64 [[TMP81]] to i32
+; CHECK-NEXT:    [[TMP83:%.*]] = lshr i64 [[TMP81]], 32
+; CHECK-NEXT:    [[TMP84:%.*]] = trunc i64 [[TMP83]] to i32
+; CHECK-NEXT:    [[TMP85:%.*]] = mul i32 [[TMP84]], [[TMP56]]
+; CHECK-NEXT:    [[TMP86:%.*]] = sub i32 [[TMP55]], [[TMP85]]
+; CHECK-NEXT:    [[TMP87:%.*]] = icmp uge i32 [[TMP86]], [[TMP56]]
+; CHECK-NEXT:    [[TMP88:%.*]] = icmp uge i32 [[TMP55]], [[TMP85]]
+; CHECK-NEXT:    [[TMP89:%.*]] = and i1 [[TMP87]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = sub i32 [[TMP86]], [[TMP56]]
+; CHECK-NEXT:    [[TMP91:%.*]] = add i32 [[TMP86]], [[TMP56]]
+; CHECK-NEXT:    [[TMP92:%.*]] = select i1 [[TMP89]], i32 [[TMP90]], i32 [[TMP86]]
+; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP88]], i32 [[TMP92]], i32 [[TMP91]]
+; CHECK-NEXT:    [[TMP94:%.*]] = xor i32 [[TMP93]], [[TMP51]]
+; CHECK-NEXT:    [[TMP95:%.*]] = sub i32 [[TMP94]], [[TMP51]]
+; CHECK-NEXT:    [[TMP96:%.*]] = insertelement <2 x i32> [[TMP48]], i32 [[TMP95]], i64 1
+; CHECK-NEXT:    store <2 x i32> [[TMP96]], <2 x i32> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_v2i32_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0xd
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_mov_b32 s14, 0x4f800000
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, s4, s2
+; GCN-NEXT:    s_ashr_i32 s5, s2, 31
+; GCN-NEXT:    s_add_i32 s2, s2, s5
+; GCN-NEXT:    s_xor_b32 s13, s2, s5
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s13
+; GCN-NEXT:    s_lshl_b32 s2, s4, s3
+; GCN-NEXT:    s_ashr_i32 s12, s6, 31
+; GCN-NEXT:    s_add_i32 s3, s6, s12
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v0, v0
+; GCN-NEXT:    s_ashr_i32 s4, s2, 31
+; GCN-NEXT:    s_add_i32 s6, s2, s4
+; GCN-NEXT:    s_xor_b32 s5, s3, s12
+; GCN-NEXT:    v_mul_f32_e32 v0, s14, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    s_xor_b32 s15, s6, s4
+; GCN-NEXT:    s_ashr_i32 s6, s7, 31
+; GCN-NEXT:    s_add_i32 s7, s7, s6
+; GCN-NEXT:    v_mul_lo_u32 v1, v0, s13
+; GCN-NEXT:    v_mul_hi_u32 v2, v0, s13
+; GCN-NEXT:    s_xor_b32 s7, s7, s6
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, 0, v1
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[2:3], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, v0
+; GCN-NEXT:    v_cvt_f32_u32_e32 v2, s15
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v1, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_rcp_iflag_f32_e32 v1, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v3, s[2:3]
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, s5
+; GCN-NEXT:    v_mul_f32_e32 v1, s14, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s13
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s15
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, s15
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s5, v0
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s5, v0
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, 0, v4
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v6, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v4, v4, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s13, v2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s13, v2
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v4, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, v4, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[4:5]
+; GCN-NEXT:    v_mul_hi_u32 v1, v1, s7
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v3, v0, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s15
+; GCN-NEXT:    v_xor_b32_e32 v0, s12, v0
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s12, v0
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s7, v1
+; GCN-NEXT:    v_cmp_ge_u32_e64 s[2:3], s7, v1
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, s15, v2
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s15, v2
+; GCN-NEXT:    s_and_b64 vcc, s[0:1], s[2:3]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[2:3]
+; GCN-NEXT:    v_xor_b32_e32 v1, s6, v1
+; GCN-NEXT:    v_subrev_i32_e32 v1, vcc, s6, v1
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
+  %r = srem <2 x i32> %x, %shl.y
+  store <2 x i32> %r, <2 x i32> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @udiv_i64_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = udiv i64 [[X:%.*]], 1235195949943
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_i64_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f176a73
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; GCN-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_movk_i32 s2, 0xfee0
+; GCN-NEXT:    s_mov_b32 s3, 0x68958c89
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    s_mov_b32 s4, 0x976a7376
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v5, v0, s3
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v6, v2, s3
+; GCN-NEXT:    s_movk_i32 s2, 0x11f
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
+; GCN-NEXT:    s_mov_b32 s3, 0x976a7377
+; GCN-NEXT:    s_mov_b32 s9, s5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
+; GCN-NEXT:    v_mov_b32_e32 v5, s2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s7, v2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s6, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s3, v3
+; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
+; GCN-NEXT:    s_movk_i32 s3, 0x11e
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s3, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s4, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s2, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
+; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
+; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v6, s7
+; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s2, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv i64 %x, 1235195949943
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @udiv_i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = udiv i64 [[X:%.*]], 4096
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_lshr_b64 s[4:5], s[6:7], 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv i64 %x, 4096
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %x, i64 %y) {
+; CHECK-LABEL: @udiv_i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = udiv i64 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s8, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_add_i32 s8, s8, 12
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_lshr_b64 s[4:5], s[6:7], s8
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i64 4096, %y
+  %r = udiv i64 %x, %shl.y
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x) {
+; CHECK-LABEL: @udiv_v2i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = udiv <2 x i64> [[X:%.*]], <i64 4096, i64 4096>
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_v2i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshr_b64 s[2:3], s[2:3], 12
+; GCN-NEXT:    s_lshr_b64 s[0:1], s[0:1], 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv <2 x i64> %x, <i64 4096, i64 4096>
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x) {
+; CHECK-LABEL: @udiv_v2i64_mixed_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = udiv <2 x i64> [[X:%.*]], <i64 4096, i64 4095>
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_v2i64_mixed_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
+; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x457ff000
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_movk_i32 s4, 0xf001
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s4
+; GCN-NEXT:    v_mul_lo_u32 v5, v1, s4
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s4
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, v0, v3
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v3
+; GCN-NEXT:    v_mul_hi_u32 v8, v0, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, v1, v3
+; GCN-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v7, v8, vcc
+; GCN-NEXT:    v_mul_lo_u32 v8, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v4, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_add_i32_e64 v0, s[2:3], v0, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
+; GCN-NEXT:    v_mul_hi_u32 v5, v0, s4
+; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v1, v4, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v6, v3, s4
+; GCN-NEXT:    v_mul_lo_u32 v8, v0, s4
+; GCN-NEXT:    v_subrev_i32_e32 v5, vcc, v0, v5
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v8
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v3, v5
+; GCN-NEXT:    s_movk_i32 s0, 0xfff
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v7, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v3, v8
+; GCN-NEXT:    v_mul_hi_u32 v8, v3, v8
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, v5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v9, v8, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v4
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v3, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v4, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v6, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v4, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v6, v2, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s0
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s0
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s10, v4
+; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s0, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
+; GCN-NEXT:    s_movk_i32 s0, 0xffe
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v3, v1, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v7, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v0, v1, s[0:1]
+; GCN-NEXT:    s_lshr_b64 s[0:1], s[8:9], 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = udiv <2 x i64> %x, <i64 4096, i64 4095>
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @udiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @udiv_v2i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i64> <i64 4096, i64 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = udiv <2 x i64> [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: udiv_v2i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x11
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_i32 s1, s2, 12
+; GCN-NEXT:    s_add_i32 s0, s0, 12
+; GCN-NEXT:    s_lshr_b64 s[2:3], s[10:11], s1
+; GCN-NEXT:    s_lshr_b64 s[0:1], s[8:9], s0
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = udiv <2 x i64> %x, %shl.y
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @urem_i64_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = urem i64 [[X:%.*]], 1235195393993
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_i64_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f1761f8
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; GCN-NEXT:    v_madmk_f32 v0, v1, 0x438f8000, v0
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_movk_i32 s2, 0xfee0
+; GCN-NEXT:    s_mov_b32 s3, 0x689e0837
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s3
+; GCN-NEXT:    v_mul_lo_u32 v4, v1, s3
+; GCN-NEXT:    s_mov_b32 s12, 0x9761f7c9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s8, s4
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    s_movk_i32 s4, 0x11f
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v3
+; GCN-NEXT:    v_mul_hi_u32 v3, v1, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
+; GCN-NEXT:    v_mul_hi_u32 v5, v0, s3
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v6, v2, s3
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v10, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v9, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v2, v4
+; GCN-NEXT:    s_mov_b32 s9, s5
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v9, v6
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v8, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v2, v5
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v9, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v11, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s6, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, s7, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s7, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s7, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v0, s4
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, s12
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s12
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s12
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s7, v1
+; GCN-NEXT:    v_mov_b32_e32 v3, s4
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
+; GCN-NEXT:    v_subb_u32_e64 v3, s[2:3], v2, v3, s[0:1]
+; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    s_movk_i32 s2, 0x11e
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s2, v2
+; GCN-NEXT:    s_mov_b32 s3, 0x9761f7c8
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s3, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s4, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v6, s[0:1], s12, v4
+; GCN-NEXT:    v_subbrev_u32_e64 v3, s[0:1], 0, v3, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s2, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s3, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s4, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
+  %r = urem i64 %x, 1235195393993
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @urem_i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = urem i64 [[X:%.*]], 4096
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_and_b32 s4, s6, 0xfff
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+  %r = urem i64 %x, 4096
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %x, i64 %y) {
+; CHECK-LABEL: @urem_i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = urem i64 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_load_dword s8, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    s_movk_i32 s4, 0x1000
+; GCN-NEXT:    s_lshl_b64 s[4:5], s[4:5], s8
+; GCN-NEXT:    s_add_u32 s4, s4, -1
+; GCN-NEXT:    s_addc_u32 s5, s5, -1
+; GCN-NEXT:    s_and_b64 s[4:5], s[6:7], s[4:5]
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i64 4096, %y
+  %r = urem i64 %x, %shl.y
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x) {
+; CHECK-LABEL: @urem_v2i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = urem <2 x i64> [[X:%.*]], <i64 4096, i64 4096>
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_v2i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
+; GCN-NEXT:    s_movk_i32 s8, 0xfff
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s1, s2, s8
+; GCN-NEXT:    s_and_b32 s0, s0, s8
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_mov_b32_e32 v3, v1
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = urem <2 x i64> %x, <i64 4096, i64 4096>
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @urem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @urem_v2i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i64> <i64 4096, i64 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = urem <2 x i64> [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: urem_v2i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x11
+; GCN-NEXT:    s_mov_b32 s13, 0
+; GCN-NEXT:    s_movk_i32 s12, 0x1000
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[0:1], s[12:13], s0
+; GCN-NEXT:    s_lshl_b64 s[2:3], s[12:13], s2
+; GCN-NEXT:    s_add_u32 s2, s2, -1
+; GCN-NEXT:    s_addc_u32 s3, s3, -1
+; GCN-NEXT:    s_and_b64 s[2:3], s[10:11], s[2:3]
+; GCN-NEXT:    s_add_u32 s0, s0, -1
+; GCN-NEXT:    s_addc_u32 s1, s1, -1
+; GCN-NEXT:    s_and_b64 s[0:1], s[8:9], s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = urem <2 x i64> %x, %shl.y
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @sdiv_i64_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = sdiv i64 [[X:%.*]], 1235195
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_i64_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
+; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x4996c7d8
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_mov_b32 s2, 0xffed2705
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s2
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v3, vcc
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v4, v2, s2
+; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v5
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v8, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    s_ashr_i32 s2, s11, 31
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s10, s2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_mov_b32 s3, s2
+; GCN-NEXT:    s_addc_u32 s1, s11, s2
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s0, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, s1, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s1, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, s1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GCN-NEXT:    s_mov_b32 s3, 0x12d8fb
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s3
+; GCN-NEXT:    v_mul_hi_u32 v3, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
+; GCN-NEXT:    s_mov_b32 s0, 0x12d8fa
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv i64 %x, 1235195
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @sdiv_i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = sdiv i64 [[X:%.*]], 4096
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_ashr_i32 s4, s7, 31
+; GCN-NEXT:    s_lshr_b32 s4, s4, 20
+; GCN-NEXT:    s_add_u32 s4, s6, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_addc_u32 s5, s7, 0
+; GCN-NEXT:    s_ashr_i64 s[4:5], s[4:5], 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv i64 %x, 4096
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %x, i64 %y) {
+; CHECK-LABEL: @sdiv_i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sdiv i64 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_movk_i32 s2, 0x1000
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
+; GCN-NEXT:    s_ashr_i32 s12, s3, 31
+; GCN-NEXT:    s_add_u32 s2, s2, s12
+; GCN-NEXT:    s_mov_b32 s13, s12
+; GCN-NEXT:    s_addc_u32 s3, s3, s12
+; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[12:13]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s3
+; GCN-NEXT:    s_sub_u32 s4, 0, s2
+; GCN-NEXT:    s_subb_u32 s5, 0, s3
+; GCN-NEXT:    s_ashr_i32 s14, s11, 31
+; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s4, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s5, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s4, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s4, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s4, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s5, v0
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s4, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s10, s14
+; GCN-NEXT:    s_addc_u32 s1, s11, s14
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s3, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s11, v2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s2, v3
+; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s3, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s2, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s3, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v5, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v5, s[0:1], 2, v0
+; GCN-NEXT:    v_addc_u32_e64 v6, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 1, v0
+; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v8, v6, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v6, s11
+; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v6, v2, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s2, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s3, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v6, v3, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v7, v5, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[12:13]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s0, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s1, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i64 4096, %y
+  %r = sdiv i64 %x, %shl.y
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x) {
+; CHECK-LABEL: @sdiv_v2i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = sdiv <2 x i64> [[X:%.*]], <i64 4096, i64 4096>
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_v2i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s8, s3, 31
+; GCN-NEXT:    s_lshr_b32 s8, s8, 20
+; GCN-NEXT:    s_add_u32 s2, s2, s8
+; GCN-NEXT:    s_addc_u32 s3, s3, 0
+; GCN-NEXT:    s_ashr_i32 s8, s1, 31
+; GCN-NEXT:    s_ashr_i64 s[2:3], s[2:3], 12
+; GCN-NEXT:    s_lshr_b32 s8, s8, 20
+; GCN-NEXT:    s_add_u32 s0, s0, s8
+; GCN-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv <2 x i64> %x, <i64 4096, i64 4096>
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @ssdiv_v2i64_mixed_pow2k_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x) {
+; CHECK-LABEL: @ssdiv_v2i64_mixed_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = sdiv <2 x i64> [[X:%.*]], <i64 4096, i64 4095>
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: ssdiv_v2i64_mixed_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
+; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x457ff000
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_movk_i32 s6, 0xf001
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s6
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s6
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v8, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v7, v8, vcc
+; GCN-NEXT:    v_mul_lo_u32 v8, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v8, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v4, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e64 v0, s[2:3], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v7, v4, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v4, v2, s6
+; GCN-NEXT:    v_mul_hi_u32 v6, s6, v0
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v6, v4
+; GCN-NEXT:    v_mul_lo_u32 v6, v0, s6
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v6
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v6
+; GCN-NEXT:    v_mul_lo_u32 v6, v2, v6
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v7, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[2:3]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s11, 31
+; GCN-NEXT:    s_add_u32 s0, s10, s2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_mov_b32 s3, s2
+; GCN-NEXT:    s_addc_u32 s1, s11, s2
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s0, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v6, s1, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s1, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v7, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, s1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GCN-NEXT:    s_movk_i32 s3, 0xfff
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v7, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s3
+; GCN-NEXT:    v_mul_hi_u32 v3, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s0, v4
+; GCN-NEXT:    v_mov_b32_e32 v3, s1
+; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v3, v2, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v3, vcc, s3, v4
+; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v2, vcc
+; GCN-NEXT:    s_movk_i32 s0, 0xffe
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v3, -1, v3, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, 2, v0
+; GCN-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, 1, v0
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v4
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v2, -1, v4, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v8, v6, vcc
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v7, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    s_ashr_i32 s0, s9, 31
+; GCN-NEXT:    s_lshr_b32 s0, s0, 20
+; GCN-NEXT:    s_add_u32 s0, s8, s0
+; GCN-NEXT:    s_addc_u32 s1, s9, 0
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
+; GCN-NEXT:    v_mov_b32_e32 v3, s2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s2, v0
+; GCN-NEXT:    s_ashr_i64 s[0:1], s[0:1], 12
+; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = sdiv <2 x i64> %x, <i64 4096, i64 4095>
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @sdiv_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @sdiv_v2i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i64> <i64 4096, i64 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = sdiv <2 x i64> [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: sdiv_v2i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x11
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_movk_i32 s2, 0x1000
+; GCN-NEXT:    s_mov_b32 s18, 0x4f800000
+; GCN-NEXT:    s_mov_b32 s19, 0x5f7ffffc
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[12:13], s[2:3], s4
+; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s6
+; GCN-NEXT:    s_ashr_i32 s16, s3, 31
+; GCN-NEXT:    s_add_u32 s2, s2, s16
+; GCN-NEXT:    s_mov_b32 s17, s16
+; GCN-NEXT:    s_addc_u32 s3, s3, s16
+; GCN-NEXT:    s_xor_b64 s[14:15], s[2:3], s[16:17]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s14
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s15
+; GCN-NEXT:    s_mov_b32 s20, 0x2f800000
+; GCN-NEXT:    s_mov_b32 s21, 0xcf800000
+; GCN-NEXT:    s_sub_u32 s6, 0, s14
+; GCN-NEXT:    v_mac_f32_e32 v0, s18, v1
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_subb_u32 s7, 0, s15
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; GCN-NEXT:    v_mul_f32_e32 v0, s19, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s20, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, s21, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
+; GCN-NEXT:    v_mul_lo_u32 v5, s6, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[2:3], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v5, s6, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s7, v0
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s6, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[2:3]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s2, s11, 31
+; GCN-NEXT:    s_add_u32 s0, s10, s2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_mov_b32 s3, s2
+; GCN-NEXT:    s_addc_u32 s1, s11, s2
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[2:3]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    s_xor_b64 s[2:3], s[2:3], s[16:17]
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s14, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s14, v0
+; GCN-NEXT:    v_mul_lo_u32 v5, s15, v0
+; GCN-NEXT:    v_mov_b32_e32 v7, s15
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mul_lo_u32 v3, s14, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_sub_i32_e32 v5, vcc, s11, v2
+; GCN-NEXT:    v_sub_i32_e32 v3, vcc, s10, v3
+; GCN-NEXT:    v_subb_u32_e64 v5, s[0:1], v5, v7, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v7, s[0:1], s14, v3
+; GCN-NEXT:    v_subbrev_u32_e64 v5, s[0:1], 0, v5, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s15, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s14, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s15, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v8, v7, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 2, v0
+; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v9, s[0:1], 1, v0
+; GCN-NEXT:    v_addc_u32_e64 v10, s[0:1], 0, v1, s[0:1]
+; GCN-NEXT:    s_ashr_i32 s10, s13, 31
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
+; GCN-NEXT:    s_add_u32 s12, s12, s10
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v10, v8, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v8, s11
+; GCN-NEXT:    s_mov_b32 s11, s10
+; GCN-NEXT:    s_addc_u32 s13, s13, s10
+; GCN-NEXT:    s_xor_b64 s[12:13], s[12:13], s[10:11]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v10, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v11, s13
+; GCN-NEXT:    v_subb_u32_e32 v2, vcc, v8, v2, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s15, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s14, v3
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s15, v2
+; GCN-NEXT:    v_mac_f32_e32 v10, s18, v11
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v8, v3, vcc
+; GCN-NEXT:    v_rcp_f32_e32 v3, v10
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GCN-NEXT:    s_sub_u32 s14, 0, s12
+; GCN-NEXT:    v_mul_f32_e32 v3, s19, v3
+; GCN-NEXT:    v_mul_f32_e32 v5, s20, v3
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mac_f32_e32 v3, s21, v5
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v9, v7, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_mul_hi_u32 v2, s14, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s14, v5
+; GCN-NEXT:    s_subb_u32 s15, 0, s13
+; GCN-NEXT:    v_mul_lo_u32 v8, s15, v3
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, s14, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; GCN-NEXT:    v_mul_lo_u32 v8, v3, v2
+; GCN-NEXT:    v_mul_hi_u32 v10, v3, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, v3, v7
+; GCN-NEXT:    v_mul_hi_u32 v11, v5, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v5, v2
+; GCN-NEXT:    v_xor_b32_e32 v1, s3, v1
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v5, v7
+; GCN-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v11, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_add_i32_e64 v2, s[0:1], v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v6, v8, vcc
+; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v5, v7, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v8, s14, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, s14, v2
+; GCN-NEXT:    v_mul_lo_u32 v10, s15, v2
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_mul_lo_u32 v9, s14, v2
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GCN-NEXT:    v_mul_lo_u32 v12, v2, v8
+; GCN-NEXT:    v_mul_hi_u32 v14, v2, v8
+; GCN-NEXT:    v_mul_hi_u32 v13, v2, v9
+; GCN-NEXT:    v_mul_hi_u32 v11, v3, v9
+; GCN-NEXT:    v_mul_lo_u32 v9, v3, v9
+; GCN-NEXT:    v_mul_hi_u32 v10, v3, v8
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v14, vcc
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v13, v11, vcc
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v10, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v9, v3
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v6, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GCN-NEXT:    s_ashr_i32 s14, s9, 31
+; GCN-NEXT:    v_addc_u32_e64 v5, vcc, v5, v8, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s8, s14
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_addc_u32 s1, s9, s14
+; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[14:15]
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s8, v3
+; GCN-NEXT:    v_mul_hi_u32 v7, s8, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, s8, v3
+; GCN-NEXT:    v_mul_hi_u32 v10, s9, v3
+; GCN-NEXT:    v_mul_lo_u32 v3, s9, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v9, vcc
+; GCN-NEXT:    v_mul_lo_u32 v9, s9, v2
+; GCN-NEXT:    v_mul_hi_u32 v2, s9, v2
+; GCN-NEXT:    v_mov_b32_e32 v8, s3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v2, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v10, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, s12, v4
+; GCN-NEXT:    v_mul_hi_u32 v7, s12, v5
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, s13, v5
+; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v1, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v7, v6
+; GCN-NEXT:    v_mov_b32_e32 v7, s13
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, s12, v5
+; GCN-NEXT:    v_sub_i32_e32 v6, vcc, s9, v0
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s8, v1
+; GCN-NEXT:    v_subb_u32_e64 v6, s[0:1], v6, v7, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v7, s[0:1], s12, v1
+; GCN-NEXT:    v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s13, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v8, v7, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v7, s[0:1], 2, v5
+; GCN-NEXT:    v_addc_u32_e64 v8, s[0:1], 0, v4, s[0:1]
+; GCN-NEXT:    v_add_i32_e64 v9, s[0:1], 1, v5
+; GCN-NEXT:    v_addc_u32_e64 v10, s[0:1], 0, v4, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v6, v10, v8, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v8, s9
+; GCN-NEXT:    v_subb_u32_e32 v0, vcc, v8, v0, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v8, v1, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v9, v7, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GCN-NEXT:    s_xor_b64 s[0:1], s[14:15], s[10:11]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, s0, v1
+; GCN-NEXT:    v_xor_b32_e32 v4, s1, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s1
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v1
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v4, v5, vcc
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = sdiv <2 x i64> %x, %shl.y
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_i64_oddk_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @srem_i64_oddk_denom(
+; CHECK-NEXT:    [[R:%.*]] = srem i64 [[X:%.*]], 1235195
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_i64_oddk_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x4f800000
+; GCN-NEXT:    v_madak_f32 v0, 0, v0, 0x4996c7d8
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_mov_b32 s2, 0xffed2705
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, s2
+; GCN-NEXT:    v_mul_lo_u32 v4, v0, s2
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, v0, v2
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v3, vcc
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v4, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v9, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v4, v2, s2
+; GCN-NEXT:    v_mul_hi_u32 v5, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, s2
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v4
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v5
+; GCN-NEXT:    v_mul_lo_u32 v5, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, v8, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v6, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    s_ashr_i32 s2, s11, 31
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v4, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s10, s2
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_mov_b32 s3, s2
+; GCN-NEXT:    s_addc_u32 s1, s11, s2
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s0, v0
+; GCN-NEXT:    v_mul_hi_u32 v4, s0, v1
+; GCN-NEXT:    v_mul_hi_u32 v5, s1, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s1, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v8, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, s1, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s1, v0
+; GCN-NEXT:    s_mov_b32 s3, 0x12d8fb
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v4, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v5, v7, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v8, v2, vcc
+; GCN-NEXT:    v_mul_hi_u32 v2, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, s3
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, s3
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s0, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, s1
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s3, v0
+; GCN-NEXT:    v_subbrev_u32_e32 v3, vcc, 0, v1, vcc
+; GCN-NEXT:    v_subrev_i32_e32 v4, vcc, s3, v2
+; GCN-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v3, vcc
+; GCN-NEXT:    s_mov_b32 s0, 0x12d8fa
+; GCN-NEXT:    v_cmp_lt_u32_e32 vcc, s0, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v6, -1, v6, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v6
+; GCN-NEXT:    v_cmp_lt_u32_e64 s[0:1], s0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v1, v3, s[0:1]
+; GCN-NEXT:    v_xor_b32_e32 v0, s2, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s2, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = srem i64 %x, 1235195
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_i64_pow2k_denom(i64 addrspace(1)* %out, i64 %x) {
+; CHECK-LABEL: @srem_i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = srem i64 [[X:%.*]], 4096
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b32 s0, s4
+; GCN-NEXT:    s_ashr_i32 s4, s7, 31
+; GCN-NEXT:    s_lshr_b32 s4, s4, 20
+; GCN-NEXT:    s_add_u32 s4, s6, s4
+; GCN-NEXT:    s_mov_b32 s1, s5
+; GCN-NEXT:    s_addc_u32 s5, s7, 0
+; GCN-NEXT:    s_and_b32 s4, s4, 0xfffff000
+; GCN-NEXT:    s_sub_u32 s4, s6, s4
+; GCN-NEXT:    s_subb_u32 s5, s7, s5
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
+  %r = srem i64 %x, 4096
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_i64_pow2_shl_denom(i64 addrspace(1)* %out, i64 %x, i64 %y) {
+; CHECK-LABEL: @srem_i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl i64 4096, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = srem i64 [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store i64 [[R]], i64 addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xd
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_movk_i32 s2, 0x1000
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s4
+; GCN-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-NEXT:    s_add_u32 s2, s2, s4
+; GCN-NEXT:    s_mov_b32 s5, s4
+; GCN-NEXT:    s_addc_u32 s3, s3, s4
+; GCN-NEXT:    s_xor_b64 s[12:13], s[2:3], s[4:5]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s12
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s13
+; GCN-NEXT:    s_sub_u32 s2, 0, s12
+; GCN-NEXT:    s_subb_u32 s3, 0, s13
+; GCN-NEXT:    s_ashr_i32 s14, s11, 31
+; GCN-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s4, s8
+; GCN-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, 0xcf800000, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_mov_b32 s5, s9
+; GCN-NEXT:    v_mul_hi_u32 v3, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s2, v1
+; GCN-NEXT:    v_mul_lo_u32 v5, s3, v0
+; GCN-NEXT:    v_mul_lo_u32 v4, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; GCN-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-NEXT:    v_mul_lo_u32 v5, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v4
+; GCN-NEXT:    v_mul_hi_u32 v4, v1, v4
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[0:1], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v5, s2, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s2, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s10, s14
+; GCN-NEXT:    s_addc_u32 s1, s11, s14
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[14:15]
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, s12, v1
+; GCN-NEXT:    v_mul_hi_u32 v2, s12, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s13, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, s12, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
+; GCN-NEXT:    v_mov_b32_e32 v3, s13
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v4, s[0:1], s12, v0
+; GCN-NEXT:    v_subb_u32_e64 v3, s[2:3], v2, v3, s[0:1]
+; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s13, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s12, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v6, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s13, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v5, v5, v6, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v6, s[0:1], s12, v4
+; GCN-NEXT:    v_subbrev_u32_e64 v3, s[0:1], 0, v3, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s13, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s12, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s13, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v5, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v4, v6, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_xor_b32_e32 v0, s14, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s14, v1
+; GCN-NEXT:    v_mov_b32_e32 v2, s14
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s14, v0
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl i64 4096, %y
+  %r = srem i64 %x, %shl.y
+  store i64 %r, i64 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_v2i64_pow2k_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x) {
+; CHECK-LABEL: @srem_v2i64_pow2k_denom(
+; CHECK-NEXT:    [[R:%.*]] = srem <2 x i64> [[X:%.*]], <i64 4096, i64 4096>
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_v2i64_pow2k_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0xd
+; GCN-NEXT:    s_movk_i32 s8, 0xf000
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s9, s3, 31
+; GCN-NEXT:    s_lshr_b32 s9, s9, 20
+; GCN-NEXT:    s_add_u32 s9, s2, s9
+; GCN-NEXT:    s_addc_u32 s10, s3, 0
+; GCN-NEXT:    s_and_b32 s9, s9, s8
+; GCN-NEXT:    s_sub_u32 s2, s2, s9
+; GCN-NEXT:    s_subb_u32 s3, s3, s10
+; GCN-NEXT:    s_ashr_i32 s9, s1, 31
+; GCN-NEXT:    s_lshr_b32 s9, s9, 20
+; GCN-NEXT:    s_add_u32 s9, s0, s9
+; GCN-NEXT:    s_addc_u32 s10, s1, 0
+; GCN-NEXT:    s_and_b32 s8, s9, s8
+; GCN-NEXT:    s_sub_u32 s0, s0, s8
+; GCN-NEXT:    s_subb_u32 s1, s1, s10
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s3
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %r = srem <2 x i64> %x, <i64 4096, i64 4096>
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @srem_v2i64_pow2_shl_denom(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @srem_v2i64_pow2_shl_denom(
+; CHECK-NEXT:    [[SHL_Y:%.*]] = shl <2 x i64> <i64 4096, i64 4096>, [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = srem <2 x i64> [[X:%.*]], [[SHL_Y]]
+; CHECK-NEXT:    store <2 x i64> [[R]], <2 x i64> addrspace(1)* [[OUT:%.*]]
+; CHECK-NEXT:    ret void
+;
+; GCN-LABEL: srem_v2i64_pow2_shl_denom:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x11
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_movk_i32 s2, 0x1000
+; GCN-NEXT:    s_mov_b32 s18, 0x4f800000
+; GCN-NEXT:    s_mov_b32 s19, 0x5f7ffffc
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b64 s[14:15], s[2:3], s4
+; GCN-NEXT:    s_lshl_b64 s[2:3], s[2:3], s6
+; GCN-NEXT:    s_ashr_i32 s4, s3, 31
+; GCN-NEXT:    s_add_u32 s2, s2, s4
+; GCN-NEXT:    s_mov_b32 s5, s4
+; GCN-NEXT:    s_addc_u32 s3, s3, s4
+; GCN-NEXT:    s_xor_b64 s[16:17], s[2:3], s[4:5]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v0, s16
+; GCN-NEXT:    v_cvt_f32_u32_e32 v1, s17
+; GCN-NEXT:    s_mov_b32 s20, 0x2f800000
+; GCN-NEXT:    s_mov_b32 s21, 0xcf800000
+; GCN-NEXT:    s_sub_u32 s6, 0, s16
+; GCN-NEXT:    v_mac_f32_e32 v0, s18, v1
+; GCN-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-NEXT:    s_subb_u32 s7, 0, s17
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0xd
+; GCN-NEXT:    v_mul_f32_e32 v0, s19, v0
+; GCN-NEXT:    v_mul_f32_e32 v1, s20, v0
+; GCN-NEXT:    v_trunc_f32_e32 v1, v1
+; GCN-NEXT:    v_mac_f32_e32 v0, s21, v1
+; GCN-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GCN-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_ashr_i32 s12, s11, 31
+; GCN-NEXT:    s_add_u32 s0, s10, s12
+; GCN-NEXT:    v_mul_hi_u32 v3, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v2, s6, v1
+; GCN-NEXT:    v_mul_lo_u32 v4, s7, v0
+; GCN-NEXT:    v_mul_lo_u32 v5, s6, v0
+; GCN-NEXT:    s_mov_b32 s13, s12
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
+; GCN-NEXT:    v_mul_lo_u32 v3, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v4, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v6, v0, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, v1, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v1, v2
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GCN-NEXT:    v_mul_lo_u32 v6, v1, v5
+; GCN-NEXT:    v_mul_hi_u32 v5, v1, v5
+; GCN-NEXT:    s_addc_u32 s1, s11, s12
+; GCN-NEXT:    s_xor_b64 s[10:11], s[0:1], s[12:13]
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v4, v5, vcc
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-NEXT:    v_add_i32_e64 v0, s[2:3], v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v6, v5, vcc
+; GCN-NEXT:    v_addc_u32_e64 v2, vcc, v1, v3, s[2:3]
+; GCN-NEXT:    v_mul_lo_u32 v5, s6, v2
+; GCN-NEXT:    v_mul_hi_u32 v7, s6, v0
+; GCN-NEXT:    v_mul_lo_u32 v8, s7, v0
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_mul_lo_u32 v7, s6, v0
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v8, v5
+; GCN-NEXT:    v_mul_lo_u32 v10, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v12, v0, v5
+; GCN-NEXT:    v_mul_hi_u32 v11, v0, v7
+; GCN-NEXT:    v_mul_hi_u32 v9, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, v2, v7
+; GCN-NEXT:    v_mul_hi_u32 v8, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
+; GCN-NEXT:    v_addc_u32_e32 v11, vcc, 0, v12, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, v2, v5
+; GCN-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v11, v9, vcc
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v8, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_addc_u32_e32 v5, vcc, v6, v5, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_addc_u32_e64 v1, vcc, v1, v5, s[2:3]
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN-NEXT:    v_mul_lo_u32 v2, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v3, s10, v0
+; GCN-NEXT:    v_mul_hi_u32 v5, s10, v1
+; GCN-NEXT:    v_mul_hi_u32 v7, s11, v1
+; GCN-NEXT:    v_mul_lo_u32 v1, s11, v1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s11, v0
+; GCN-NEXT:    v_mul_hi_u32 v0, s11, v0
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
+; GCN-NEXT:    v_addc_u32_e32 v0, vcc, v3, v0, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v6, v2, vcc
+; GCN-NEXT:    v_mul_lo_u32 v1, s16, v1
+; GCN-NEXT:    v_mul_hi_u32 v2, s16, v0
+; GCN-NEXT:    v_mul_lo_u32 v3, s17, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, s16, v0
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
+; GCN-NEXT:    v_sub_i32_e32 v2, vcc, s11, v1
+; GCN-NEXT:    v_mov_b32_e32 v3, s17
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
+; GCN-NEXT:    v_subb_u32_e64 v2, s[0:1], v2, v3, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v5, s[0:1], s16, v0
+; GCN-NEXT:    v_subb_u32_e64 v3, s[2:3], v2, v3, s[0:1]
+; GCN-NEXT:    v_subbrev_u32_e64 v2, s[0:1], 0, v2, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s17, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s16, v5
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s17, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v8, s[0:1], s16, v5
+; GCN-NEXT:    s_ashr_i32 s2, s15, 31
+; GCN-NEXT:    v_subbrev_u32_e64 v3, s[0:1], 0, v3, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v7
+; GCN-NEXT:    s_add_u32 s10, s14, s2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v3, s11
+; GCN-NEXT:    s_mov_b32 s3, s2
+; GCN-NEXT:    s_addc_u32 s11, s15, s2
+; GCN-NEXT:    s_xor_b64 s[10:11], s[10:11], s[2:3]
+; GCN-NEXT:    v_cvt_f32_u32_e32 v7, s10
+; GCN-NEXT:    v_cvt_f32_u32_e32 v9, s11
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s17, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GCN-NEXT:    v_mac_f32_e32 v7, s18, v9
+; GCN-NEXT:    v_rcp_f32_e32 v7, v7
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s16, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v10, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s17, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v3, v3, v10, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
+; GCN-NEXT:    v_mul_f32_e32 v3, s19, v7
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v2, v5, v8, s[0:1]
+; GCN-NEXT:    v_mul_f32_e32 v5, s20, v3
+; GCN-NEXT:    v_trunc_f32_e32 v5, v5
+; GCN-NEXT:    v_mac_f32_e32 v3, s21, v5
+; GCN-NEXT:    v_cvt_u32_f32_e32 v3, v3
+; GCN-NEXT:    v_cvt_u32_f32_e32 v5, v5
+; GCN-NEXT:    s_sub_u32 s2, 0, s10
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_mul_hi_u32 v2, s2, v3
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v5
+; GCN-NEXT:    s_subb_u32 s3, 0, s11
+; GCN-NEXT:    v_mul_lo_u32 v8, s3, v3
+; GCN-NEXT:    s_ashr_i32 s14, s9, 31
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v7
+; GCN-NEXT:    v_mul_lo_u32 v7, s2, v3
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v8
+; GCN-NEXT:    v_mul_lo_u32 v8, v3, v2
+; GCN-NEXT:    v_mul_hi_u32 v10, v3, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, v3, v7
+; GCN-NEXT:    v_mul_hi_u32 v11, v5, v2
+; GCN-NEXT:    v_mul_lo_u32 v2, v5, v2
+; GCN-NEXT:    s_mov_b32 s15, s14
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
+; GCN-NEXT:    v_mul_lo_u32 v10, v5, v7
+; GCN-NEXT:    v_mul_hi_u32 v7, v5, v7
+; GCN-NEXT:    v_xor_b32_e32 v0, s12, v0
+; GCN-NEXT:    v_xor_b32_e32 v1, s12, v1
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v9, v7, vcc
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v11, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v7, v2
+; GCN-NEXT:    v_add_i32_e64 v2, s[0:1], v3, v2
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, v6, v8, vcc
+; GCN-NEXT:    v_addc_u32_e64 v3, vcc, v5, v7, s[0:1]
+; GCN-NEXT:    v_mul_lo_u32 v8, s2, v3
+; GCN-NEXT:    v_mul_hi_u32 v9, s2, v2
+; GCN-NEXT:    v_mul_lo_u32 v10, s3, v2
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; GCN-NEXT:    v_mul_lo_u32 v9, s2, v2
+; GCN-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
+; GCN-NEXT:    v_mul_lo_u32 v12, v2, v8
+; GCN-NEXT:    v_mul_hi_u32 v14, v2, v8
+; GCN-NEXT:    v_mul_hi_u32 v13, v2, v9
+; GCN-NEXT:    v_mul_hi_u32 v11, v3, v9
+; GCN-NEXT:    v_mul_lo_u32 v9, v3, v9
+; GCN-NEXT:    v_mul_hi_u32 v10, v3, v8
+; GCN-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
+; GCN-NEXT:    v_addc_u32_e32 v13, vcc, 0, v14, vcc
+; GCN-NEXT:    v_mul_lo_u32 v3, v3, v8
+; GCN-NEXT:    v_add_i32_e32 v9, vcc, v9, v12
+; GCN-NEXT:    v_addc_u32_e32 v9, vcc, v13, v11, vcc
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v10, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v3, vcc, v9, v3
+; GCN-NEXT:    v_addc_u32_e32 v8, vcc, v6, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
+; GCN-NEXT:    v_addc_u32_e64 v5, vcc, v5, v8, s[0:1]
+; GCN-NEXT:    s_add_u32 s0, s8, s14
+; GCN-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GCN-NEXT:    s_addc_u32 s1, s9, s14
+; GCN-NEXT:    s_xor_b64 s[8:9], s[0:1], s[14:15]
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GCN-NEXT:    v_mul_lo_u32 v5, s8, v3
+; GCN-NEXT:    v_mul_hi_u32 v7, s8, v2
+; GCN-NEXT:    v_mul_hi_u32 v9, s8, v3
+; GCN-NEXT:    v_mul_hi_u32 v10, s9, v3
+; GCN-NEXT:    v_mul_lo_u32 v3, s9, v3
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
+; GCN-NEXT:    v_addc_u32_e32 v7, vcc, 0, v9, vcc
+; GCN-NEXT:    v_mul_lo_u32 v9, s9, v2
+; GCN-NEXT:    v_mul_hi_u32 v2, s9, v2
+; GCN-NEXT:    v_mov_b32_e32 v8, s12
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v7, v2, vcc
+; GCN-NEXT:    v_addc_u32_e32 v4, vcc, v10, v4, vcc
+; GCN-NEXT:    v_add_i32_e32 v5, vcc, v2, v3
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v6, v4, vcc
+; GCN-NEXT:    v_mul_lo_u32 v4, s10, v2
+; GCN-NEXT:    v_mul_hi_u32 v6, s10, v5
+; GCN-NEXT:    v_subrev_i32_e32 v2, vcc, s12, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, s11, v5
+; GCN-NEXT:    v_subb_u32_e32 v3, vcc, v1, v8, vcc
+; GCN-NEXT:    v_add_i32_e32 v1, vcc, v6, v4
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
+; GCN-NEXT:    v_mul_lo_u32 v1, s10, v5
+; GCN-NEXT:    v_sub_i32_e32 v4, vcc, s9, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s11
+; GCN-NEXT:    v_sub_i32_e32 v1, vcc, s8, v1
+; GCN-NEXT:    v_subb_u32_e64 v4, s[0:1], v4, v5, vcc
+; GCN-NEXT:    v_subrev_i32_e64 v6, s[0:1], s10, v1
+; GCN-NEXT:    v_subb_u32_e64 v5, s[2:3], v4, v5, s[0:1]
+; GCN-NEXT:    v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s11, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_le_u32_e64 s[0:1], s10, v6
+; GCN-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[0:1]
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], s11, v4
+; GCN-NEXT:    v_cndmask_b32_e64 v7, v7, v8, s[0:1]
+; GCN-NEXT:    v_subrev_i32_e64 v8, s[0:1], s10, v6
+; GCN-NEXT:    v_subbrev_u32_e64 v5, s[0:1], 0, v5, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v7
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v4, v5, s[0:1]
+; GCN-NEXT:    v_mov_b32_e32 v5, s9
+; GCN-NEXT:    v_subb_u32_e32 v0, vcc, v5, v0, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s11, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
+; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s10, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, s11, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v5
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-NEXT:    v_xor_b32_e32 v1, s14, v1
+; GCN-NEXT:    v_xor_b32_e32 v4, s14, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, s14
+; GCN-NEXT:    v_subrev_i32_e32 v0, vcc, s14, v1
+; GCN-NEXT:    v_subb_u32_e32 v1, vcc, v4, v5, vcc
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
+  %shl.y = shl <2 x i64> <i64 4096, i64 4096>, %y
+  %r = srem <2 x i64> %x, %shl.y
+  store <2 x i64> %r, <2 x i64> addrspace(1)* %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/bug-sdag-scheduler-cycle.ll b/llvm/test/CodeGen/AMDGPU/bug-sdag-scheduler-cycle.ll
new file mode 100644
index 0000000000000000000000000000000000000000..50ba7e19f46e071de011b37bf94e695b9e7053f7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bug-sdag-scheduler-cycle.ll
@@ -0,0 +1,27 @@
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
+
+; This used to cause a circular chain dependency during
+; SelectionDAG instruction scheduling.
+
+; CHECK-LABEL: {{^}}_amdgpu_gs_main:
+; CHECK: ds_read_b32
+; CHECK: ds_read_b32
+; CHECK: ds_read_b32
+; CHECK: ds_read_b32
+define amdgpu_gs float @_amdgpu_gs_main(i8 addrspace(3)* %arg0, i8 addrspace(3)* %arg1, i8 addrspace(3)* %arg2) #0 {
+  %tmp0 = bitcast i8 addrspace(3)* %arg0 to i32 addrspace(3)* addrspace(3)*
+  %tmp = load volatile i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* %tmp0, align 4
+
+  %tmp3 = load volatile i32, i32 addrspace(3)* %tmp, align 4
+
+  %tmp4a = bitcast i8 addrspace(3)* %arg1 to i32 addrspace(3)*
+  %tmp4 = load volatile i32, i32 addrspace(3)* %tmp4a, align 4
+
+  %tmp7a = getelementptr i32, i32 addrspace(3)* %tmp, i32 8
+  %tmp8 = load volatile i32, i32 addrspace(3)* %tmp7a, align 4
+
+  %tmp9 = add i32 %tmp3, %tmp8
+  %tmp10 = add i32 %tmp9, %tmp4
+  %tmp14 = bitcast i32 %tmp10 to float
+  ret float %tmp14
+}
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index 9ec8b7573ceb48986bcaad6b158443e496039ddc..0df32537808ac6b4b9be1b95c7af537c54be5e9f 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -199,14 +199,17 @@ define amdgpu_kernel void @vload2_private(i16 addrspace(1)* nocapture readonly %
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v2, s4
 ; GCN-NEXT:    v_mov_b32_e32 v3, s5
-; GCN-NEXT:    global_load_ushort v4, v[2:3], off offset:4
-; GCN-NEXT:    global_load_dword v2, v[2:3], off
+; GCN-NEXT:    global_load_ushort v4, v[2:3], off
 ; GCN-NEXT:    v_mov_b32_e32 v0, s6
 ; GCN-NEXT:    v_mov_b32_e32 v1, s7
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    buffer_store_short v2, off, s[0:3], s9 offset:4
-; GCN-NEXT:    buffer_store_short_d16_hi v2, off, s[0:3], s9 offset:6
-; GCN-NEXT:    buffer_store_short v4, off, s[0:3], s9 offset:8
+; GCN-NEXT:    buffer_store_short v4, off, s[0:3], s9 offset:4
+; GCN-NEXT:    global_load_ushort v4, v[2:3], off offset:2
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_short v4, off, s[0:3], s9 offset:6
+; GCN-NEXT:    global_load_ushort v2, v[2:3], off offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_store_short v2, off, s[0:3], s9 offset:8
 ; GCN-NEXT:    buffer_load_ushort v2, off, s[0:3], s9 offset:4
 ; GCN-NEXT:    buffer_load_ushort v4, off, s[0:3], s9 offset:6
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
index e32c93bdf4078d5dc39d967fba61d2669dadd400..81bccd608c648d2638b430f1600d858f169c0d71 100644
--- a/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
+++ b/llvm/test/CodeGen/AMDGPU/divrem24-assume.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @divrem24_assume(i32 addrspace(1)* %arg, i32 %arg1) {
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[TMP2]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = uitofp i32 [[TMP]] to float
 ; CHECK-NEXT:    [[TMP1:%.*]] = uitofp i32 [[ARG1]] to float
-; CHECK-NEXT:    [[TMP2:%.*]] = fdiv fast float 1.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call fast float @llvm.amdgcn.rcp.f32(float [[TMP1]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast float [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call fast float @llvm.trunc.f32(float [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = fneg fast float [[TMP4]]
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
new file mode 100644
index 0000000000000000000000000000000000000000..34f8706ac66c562b3bd7fa9f99305ee21795197d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.global.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -mattr=-unaligned-buffer-access < %s | FileCheck -check-prefixes=GCN,GFX7-ALIGNED %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -mattr=+unaligned-buffer-access < %s | FileCheck -check-prefixes=GCN,GFX7-UNALIGNED %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+unaligned-buffer-access < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+; Should not merge this to a dword load
+define i32 @global_load_2xi16_align2(i16 addrspace(1)* %p) #0 {
+; GFX7-ALIGNED-LABEL: global_load_2xi16_align2:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
+; GFX7-ALIGNED-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX7-ALIGNED-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX7-ALIGNED-NEXT:    flat_load_ushort v1, v[2:3]
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: global_load_2xi16_align2:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
+; GFX7-UNALIGNED-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX7-UNALIGNED-NEXT:    flat_load_ushort v0, v[0:1]
+; GFX7-UNALIGNED-NEXT:    flat_load_ushort v1, v[2:3]
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-UNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_load_2xi16_align2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NEXT:    global_load_ushort v0, v[0:1], off offset:2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.p = getelementptr i16, i16 addrspace(1)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(1)* %p, align 2
+  %p.1 = load i16, i16 addrspace(1)* %gep.p, align 2
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; Should not merge this to a dword store
+define amdgpu_kernel void @global_store_2xi16_align2(i16 addrspace(1)* %p, i16 addrspace(1)* %r) #0 {
+; GFX7-ALIGNED-LABEL: global_store_2xi16_align2:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 1
+; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-ALIGNED-NEXT:    s_add_u32 s2, s0, 2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-ALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-ALIGNED-NEXT:    s_addc_u32 s3, s1, 0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-ALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-ALIGNED-NEXT:    s_endpgm
+;
+; GFX7-UNALIGNED-LABEL: global_store_2xi16_align2:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 1
+; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-UNALIGNED-NEXT:    s_add_u32 s2, s0, 2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-UNALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-UNALIGNED-NEXT:    s_addc_u32 s3, s1, 0
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-UNALIGNED-NEXT:    flat_store_short v[0:1], v2
+; GFX7-UNALIGNED-NEXT:    s_endpgm
+;
+; GFX9-LABEL: global_store_2xi16_align2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX9-NEXT:    v_mov_b32_e32 v2, 1
+; GFX9-NEXT:    v_mov_b32_e32 v3, 2
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-NEXT:    global_store_short v[0:1], v3, off offset:2
+; GFX9-NEXT:    s_endpgm
+  %gep.r = getelementptr i16, i16 addrspace(1)* %r, i64 1
+  store i16 1, i16 addrspace(1)* %r, align 2
+  store i16 2, i16 addrspace(1)* %gep.r, align 2
+  ret void
+}
+
+; Should produce align 1 dword when legal
+define i32 @global_load_2xi16_align1(i16 addrspace(1)* %p) #0 {
+; GFX7-ALIGNED-LABEL: global_load_2xi16_align1:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
+; GFX7-ALIGNED-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v4, vcc, 1, v0
+; GFX7-ALIGNED-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-ALIGNED-NEXT:    flat_load_ubyte v6, v[0:1]
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; GFX7-ALIGNED-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-ALIGNED-NEXT:    flat_load_ubyte v2, v[2:3]
+; GFX7-ALIGNED-NEXT:    flat_load_ubyte v3, v[4:5]
+; GFX7-ALIGNED-NEXT:    flat_load_ubyte v0, v[0:1]
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(1) lgkmcnt(1)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 8, v3
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v1, v1, v6
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: global_load_2xi16_align1:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_load_2xi16_align1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v1, 0, v0
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.p = getelementptr i16, i16 addrspace(1)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(1)* %p, align 1
+  %p.1 = load i16, i16 addrspace(1)* %gep.p, align 1
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; Should produce align 1 dword when legal
+define amdgpu_kernel void @global_store_2xi16_align1(i16 addrspace(1)* %p, i16 addrspace(1)* %r) #0 {
+; GFX7-ALIGNED-LABEL: global_store_2xi16_align1:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, 1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v5, 0
+; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    s_add_u32 s2, s0, 2
+; GFX7-ALIGNED-NEXT:    s_addc_u32 s3, s1, 0
+; GFX7-ALIGNED-NEXT:    s_add_u32 s4, s0, 1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-ALIGNED-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-ALIGNED-NEXT:    s_add_u32 s0, s0, 3
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, s5
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v4
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[2:3], v5
+; GFX7-ALIGNED-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, 2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[0:1], v5
+; GFX7-ALIGNED-NEXT:    flat_store_byte v[2:3], v4
+; GFX7-ALIGNED-NEXT:    s_endpgm
+;
+; GFX7-UNALIGNED-LABEL: global_store_2xi16_align1:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-UNALIGNED-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-UNALIGNED-NEXT:    s_endpgm
+;
+; GFX9-LABEL: global_store_2xi16_align1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+  %gep.r = getelementptr i16, i16 addrspace(1)* %r, i64 1
+  store i16 1, i16 addrspace(1)* %r, align 1
+  store i16 2, i16 addrspace(1)* %gep.r, align 1
+  ret void
+}
+
+; Should merge this to a dword load
+define i32 @global_load_2xi16_align4(i16 addrspace(1)* %p) #0 {
+; GFX7-LABEL: load_2xi16_align4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-ALIGNED-LABEL: global_load_2xi16_align4:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: global_load_2xi16_align4:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_load_2xi16_align4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v1, 0, v0
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.p = getelementptr i16, i16 addrspace(1)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(1)* %p, align 4
+  %p.1 = load i16, i16 addrspace(1)* %gep.p, align 2
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; Should merge this to a dword store
+define amdgpu_kernel void @global_store_2xi16_align4(i16 addrspace(1)* %p, i16 addrspace(1)* %r) #0 {
+; GFX7-LABEL: global_store_2xi16_align4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX7-ALIGNED-LABEL: global_store_2xi16_align4:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX7-ALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-ALIGNED-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-ALIGNED-NEXT:    s_endpgm
+;
+; GFX7-UNALIGNED-LABEL: global_store_2xi16_align4:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX7-UNALIGNED-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-UNALIGNED-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-UNALIGNED-NEXT:    s_endpgm
+;
+; GFX9-LABEL: global_store_2xi16_align4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+  %gep.r = getelementptr i16, i16 addrspace(1)* %r, i64 1
+  store i16 1, i16 addrspace(1)* %r, align 4
+  store i16 2, i16 addrspace(1)* %gep.r, align 2
+  ret void
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0053d2f3019dfba593e6b46e8298a1135dffd0e8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
@@ -0,0 +1,245 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -mattr=-unaligned-scratch-access < %s | FileCheck -check-prefixes=GCN,GFX7-ALIGNED %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -mattr=+unaligned-scratch-access < %s | FileCheck -check-prefixes=GCN,GFX7-UNALIGNED %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+unaligned-scratch-access < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+
+; Should not merge this to a dword load
+define i32 @private_load_2xi16_align2(i16 addrspace(5)* %p) #0 {
+; GFX7-ALIGNED-LABEL: private_load_2xi16_align2:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v1, vcc, 2, v0
+; GFX7-ALIGNED-NEXT:    buffer_load_ushort v1, v1, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_load_ushort v0, v0, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: private_load_2xi16_align2:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_add_i32_e32 v1, vcc, 2, v0
+; GFX7-UNALIGNED-NEXT:    buffer_load_ushort v1, v1, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    buffer_load_ushort v0, v0, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; GFX7-UNALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: private_load_2xi16_align2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_ushort v1, v0, s[0:3], s33 offen
+; GFX9-NEXT:    buffer_load_ushort v0, v0, s[0:3], s33 offen offset:2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshl_or_b32 v0, v0, 16, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(5)* %p, align 2
+  %p.1 = load i16, i16 addrspace(5)* %gep.p, align 2
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; Should not merge this to a dword store
+define void @private_store_2xi16_align2(i16 addrspace(5)* %p, i16 addrspace(5)* %r) #0 {
+; GFX7-ALIGNED-LABEL: private_store_2xi16_align2:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, 1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, 2
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v1
+; GFX7-ALIGNED-NEXT:    buffer_store_short v3, v1, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_store_short v0, v2, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: private_store_2xi16_align2:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v3, 1
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, 2
+; GFX7-UNALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v1
+; GFX7-UNALIGNED-NEXT:    buffer_store_short v3, v1, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    buffer_store_short v0, v2, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: private_store_2xi16_align2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 1
+; GFX9-NEXT:    buffer_store_short v0, v1, s[0:3], s33 offen
+; GFX9-NEXT:    v_mov_b32_e32 v0, 2
+; GFX9-NEXT:    buffer_store_short v0, v1, s[0:3], s33 offen offset:2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.r = getelementptr i16, i16 addrspace(5)* %r, i64 1
+  store i16 1, i16 addrspace(5)* %r, align 2
+  store i16 2, i16 addrspace(5)* %gep.r, align 2
+  ret void
+}
+
+; Should produce align 1 dword when legal
+define i32 @private_load_2xi16_align1(i16 addrspace(5)* %p) #0 {
+; GFX7-ALIGNED-LABEL: private_load_2xi16_align1:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v1, vcc, 3, v0
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v0
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v3, vcc, 1, v0
+; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v1, v1, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v3, v3, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v2, v2, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_load_ubyte v0, v0, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(3)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(2)
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(1)
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX7-ALIGNED-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ALIGNED-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: private_load_2xi16_align1:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    buffer_load_dword v0, v0, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: private_load_2xi16_align1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v0, v0, s[0:3], s33 offen
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v1, 0, v0
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(5)* %p, align 1
+  %p.1 = load i16, i16 addrspace(5)* %gep.p, align 1
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; Should produce align 1 dword when legal
+define void @private_store_2xi16_align1(i16 addrspace(5)* %p, i16 addrspace(5)* %r) #0 {
+; GFX7-ALIGNED-LABEL: private_store_2xi16_align1:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v3, 1
+; GFX7-ALIGNED-NEXT:    buffer_store_byte v3, v1, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v2, vcc, 2, v1
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7-ALIGNED-NEXT:    v_add_i32_e32 v1, vcc, 3, v1
+; GFX7-ALIGNED-NEXT:    v_mov_b32_e32 v0, 2
+; GFX7-ALIGNED-NEXT:    buffer_store_byte v4, v3, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_store_byte v4, v1, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    buffer_store_byte v0, v2, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: private_store_2xi16_align1:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    v_mov_b32_e32 v0, 0x20001
+; GFX7-UNALIGNED-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: private_store_2xi16_align1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0x20001
+; GFX9-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.r = getelementptr i16, i16 addrspace(5)* %r, i64 1
+  store i16 1, i16 addrspace(5)* %r, align 1
+  store i16 2, i16 addrspace(5)* %gep.r, align 1
+  ret void
+}
+
+; Should merge this to a dword load
+define i32 @private_load_2xi16_align4(i16 addrspace(5)* %p) #0 {
+; GFX7-LABEL: load_2xi16_align4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-ALIGNED-LABEL: private_load_2xi16_align4:
+; GFX7-ALIGNED:       ; %bb.0:
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ALIGNED-NEXT:    buffer_load_dword v0, v0, s[0:3], s33 offen
+; GFX7-ALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-ALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX7-UNALIGNED-LABEL: private_load_2xi16_align4:
+; GFX7-UNALIGNED:       ; %bb.0:
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-UNALIGNED-NEXT:    buffer_load_dword v0, v0, s[0:3], s33 offen
+; GFX7-UNALIGNED-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-UNALIGNED-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: private_load_2xi16_align4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v0, v0, s[0:3], s33 offen
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT:    s_mov_b32 s4, 0xffff
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_bfi_b32 v1, v1, 0, v0
+; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(5)* %p, align 4
+  %p.1 = load i16, i16 addrspace(5)* %gep.p, align 2
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; Should merge this to a dword store
+define void @private_store_2xi16_align4(i16 addrspace(5)* %p, i16 addrspace(5)* %r) #0 {
+; GFX7-LABEL: private_store_2xi16_align4:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    v_mov_b32_e32 v2, 0x20001
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GCN-LABEL: private_store_2xi16_align4:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, 0x20001
+; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], s33 offen
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %gep.r = getelementptr i16, i16 addrspace(5)* %r, i64 1
+  store i16 1, i16 addrspace(5)* %r, align 4
+  store i16 2, i16 addrspace(5)* %gep.r, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
index 6691696924b42b4b6f9cca73a5763dc9d96266b4..8a33e92557cb11e3f16f26e07966b539e38d8d79 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-argument-dag-lowering.ll
@@ -130,3 +130,25 @@ define amdgpu_kernel void @array_3xi16(i8 %arg0, [3 x i16] %arg1) {
   store volatile [3 x i16] %arg1, [3 x i16] addrspace(1)* undef
   ret void
 }
+
+; GCN-LABEL: {{^}}v2i15_arg:
+; GCN: s_load_dword [[DWORD:s[0-9]+]]
+; GCN-DAG: s_bfe_u32 [[BFE:s[0-9]+]], [[DWORD]], 0x100010{{$}}
+; GCN-DAG: s_and_b32 [[AND:s[0-9]+]], [[DWORD]], 0x7fff{{$}}
+define amdgpu_kernel void @v2i15_arg(<2 x i15> addrspace(1)* nocapture %out, <2 x i15> %in) {
+entry:
+  store <2 x i15> %in, <2 x i15> addrspace(1)* %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}v3i15_arg:
+; GCN: s_load_dword [[DWORD:s[0-9]+]]
+; GCN: s_lshl_b64
+; GCN: s_and_b32
+; GCN: s_and_b32
+; GCN: s_or_b32
+define amdgpu_kernel void @v3i15_arg(<3 x i15> addrspace(1)* nocapture %out, <3 x i15> %in) {
+entry:
+  store <3 x i15> %in, <3 x i15> addrspace(1)* %out, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-scalar-ops.mir b/llvm/test/CodeGen/AMDGPU/sdwa-scalar-ops.mir
index 2e96d2129ec5d7cff7c9af1c49ecb1ddee4e9d3d..ed12cdd9d25b502770044a8d7b811fa4bce4435a 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-scalar-ops.mir
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-scalar-ops.mir
@@ -219,7 +219,7 @@ body:             |
     %13 = COPY %7.sub1
     %14 = S_ADD_U32 %7.sub0, %0.sub0, implicit-def $scc
     %15 = S_ADDC_U32 %7.sub1, %0.sub1, implicit-def dead $scc, implicit $scc
-    %16 = REG_SEQUENCE %14, 1, %15, 2
+    %16 = REG_SEQUENCE %14, %subreg.sub0, %15, %subreg.sub1
     %18 = COPY %16
     %17 = FLAT_LOAD_DWORD %18, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.uglygep45)
     %60 = V_BFE_U32 %17, 8, 8, implicit $exec
@@ -227,23 +227,23 @@ body:             |
     %70 = V_ADD_I32_e32 %7.sub0, %61, implicit-def $vcc, implicit $exec
     %66 = COPY %13
     %65 = V_ADDC_U32_e32 0, %66, implicit-def $vcc, implicit $vcc, implicit $exec
-    %67 = REG_SEQUENCE %70, 1, killed %65, 2
+    %67 = REG_SEQUENCE %70, %subreg.sub0, killed %65, %subreg.sub1
     FLAT_STORE_DWORD %67, %30, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4 into %ir.tmp9)
     %37 = S_ADD_U32 %14, 4, implicit-def $scc
     %38 = S_ADDC_U32 %15, 0, implicit-def dead $scc, implicit $scc
     %71 = COPY killed %37
     %72 = COPY killed %38
-    %41 = REG_SEQUENCE killed %71, 1, killed %72, 2
+    %41 = REG_SEQUENCE killed %71, %subreg.sub0, killed %72, %subreg.sub1
     %40 = FLAT_LOAD_DWORD killed %41, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.scevgep)
     %73 = V_BFE_U32 %40, 8, 8, implicit $exec
     %74 = V_LSHLREV_B32_e32 2, killed %73, implicit $exec
     %83 = V_ADD_I32_e32 %7.sub0, %74, implicit-def $vcc, implicit $exec
     %78 = V_ADDC_U32_e32 0, %66, implicit-def $vcc, implicit $vcc, implicit $exec
-    %80 = REG_SEQUENCE %83, 1, killed %78, 2
+    %80 = REG_SEQUENCE %83, %subreg.sub0, killed %78, %subreg.sub1
     FLAT_STORE_DWORD %80, %30, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4 into %ir.tmp17)
     %55 = S_ADD_U32 %0.sub0, 8, implicit-def $scc
     %56 = S_ADDC_U32 %0.sub1, 0, implicit-def dead $scc, implicit $scc
-    %57 = REG_SEQUENCE %55, 1, killed %56, 2
+    %57 = REG_SEQUENCE %55, %subreg.sub0, killed %56, %subreg.sub1
     %1 = COPY %57
     S_CMPK_EQ_I32 %55, 4096, implicit-def $scc
     S_CBRANCH_SCC1 %bb.1.bb1, implicit $scc
@@ -382,7 +382,7 @@ body:             |
     %13 = COPY %7.sub1
     %14 = S_ADD_U32 %7.sub0, %0.sub0, implicit-def $scc
     %15 = S_ADDC_U32 %7.sub1, %0.sub1, implicit-def dead $scc, implicit $scc
-    %16 = REG_SEQUENCE %14, 1, %15, 2
+    %16 = REG_SEQUENCE %14, %subreg.sub0, %15, %subreg.sub1
     %18 = COPY %16
     %17 = FLAT_LOAD_DWORD %18, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.uglygep45)
     %60 = V_BFE_U32 %17, 8, 8, implicit $exec
@@ -390,23 +390,23 @@ body:             |
     %70 = V_ADD_I32_e32 %7.sub0, %61, implicit-def $vcc, implicit $exec
     %66 = COPY %13
     %65 = V_ADDC_U32_e32 0, %66, implicit-def $vcc, implicit $vcc, implicit $exec
-    %67 = REG_SEQUENCE %70, 1, killed %65, 2
+    %67 = REG_SEQUENCE %70, %subreg.sub0, killed %65, %subreg.sub1
     FLAT_STORE_DWORD %67, %30, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4 into %ir.tmp9)
     %37 = S_ADD_U32 %14, 4, implicit-def $scc
     %38 = S_ADDC_U32 %15, 0, implicit-def dead $scc, implicit $scc
     %71 = COPY killed %37
     %72 = COPY killed %38
-    %41 = REG_SEQUENCE killed %71, 1, killed %72, 2
+    %41 = REG_SEQUENCE killed %71, %subreg.sub0, killed %72, %subreg.sub1
     %40 = FLAT_LOAD_DWORD killed %41, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %ir.scevgep)
     %73 = V_BFE_U32 %40, 8, 8, implicit $exec
     %74 = V_LSHLREV_B32_e32 %84, killed %73, implicit $exec
     %83 = V_ADD_I32_e32 %7.sub0, %74, implicit-def $vcc, implicit $exec
     %78 = V_ADDC_U32_e32 0, %66, implicit-def $vcc, implicit $vcc, implicit $exec
-    %80 = REG_SEQUENCE %83, 1, killed %78, 2
+    %80 = REG_SEQUENCE %83, %subreg.sub0, killed %78, %subreg.sub1
     FLAT_STORE_DWORD %80, %30, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4 into %ir.tmp17)
     %55 = S_ADD_U32 %0.sub0, 8, implicit-def $scc
     %56 = S_ADDC_U32 %0.sub1, 0, implicit-def dead $scc, implicit $scc
-    %57 = REG_SEQUENCE %55, 1, killed %56, 2
+    %57 = REG_SEQUENCE %55, %subreg.sub0, killed %56, %subreg.sub1
     %1 = COPY %57
     S_CMPK_EQ_I32 %55, 4096, implicit-def $scc
     S_CBRANCH_SCC1 %bb.1.bb1, implicit $scc
diff --git a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
index 9bcf35e13a1b0713837fe6e6fd2c94799376924c..020f677ee3cfa92b2e3ce36fee4f9e7fed7f5e9c 100644
--- a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
@@ -665,4 +665,25 @@ define void @private_store_align2_f64(double addrspace(5)* %out, double %x) #0 {
   ret void
 }
 
+; Should not merge this to a dword store
+define amdgpu_kernel void @global_store_2xi16_align2(i16 addrspace(1)* %p, i16 addrspace(1)* %r) #0 {
+  %gep.r = getelementptr i16, i16 addrspace(1)* %r, i64 1
+  %v = load i16, i16 addrspace(1)* %p, align 2
+  store i16 1, i16 addrspace(1)* %r, align 2
+  store i16 2, i16 addrspace(1)* %gep.r, align 2
+  ret void
+}
+
+; Should not merge this to a word load
+define i32 @load_2xi16_align2(i16 addrspace(1)* %p) #0 {
+  %gep.p = getelementptr i16, i16 addrspace(1)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(1)* %p, align 2
+  %p.1 = load i16, i16 addrspace(1)* %gep.p, align 2
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/ARM/smml.ll b/llvm/test/CodeGen/ARM/smml.ll
index 712aaa4392f60fe90858f1affb2bd95ea1a87a9a..9ddb1a28301dfc6fda7ae49eca65d4e9d1834a8d 100644
--- a/llvm/test/CodeGen/ARM/smml.ll
+++ b/llvm/test/CodeGen/ARM/smml.ll
@@ -10,7 +10,7 @@
 
 ; Next test would previously trigger an assertion responsible for verification of
 ; call site info state.
-; RUN: llc -stop-after=if-converter -debug-entry-values -mtriple=thumbv6t2-eabi %s -o -| FileCheck %s -check-prefix=CHECK-CALLSITE
+; RUN: llc -stop-after=if-converter -mtriple=thumbv6t2-eabi %s -o -| FileCheck %s -check-prefix=CHECK-CALLSITE
 ; CHECK-CALLSITE: name:  test_used_flags
 ; CHECK-CALLSITE: callSites:
 
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll
index d7e48d3904161023595d29983cf8b932540179bf..d871cb35311b79eb02e69ac997e2ca311a767731 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-1.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
-; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK64 %s
+; RUN: llc -march=bpfel -mattr=+alu32 -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK,CHECK32 %s
 ; Source code:
 ;   struct s {
 ;     int a;
@@ -74,8 +74,10 @@ entry:
 ; CHECK:             r{{[0-9]+}} = 4
 ; CHECK:             r{{[0-9]+}} = 4
 ; CHECK:             r{{[0-9]+}} <<= 51
-; CHECK:             r{{[0-9]+}} s>>= 60
-; CHECK:             r{{[0-9]+}} >>= 60
+; CHECK64:           r{{[0-9]+}} s>>= 60
+; CHECK64:           r{{[0-9]+}} >>= 60
+; CHECK32:           r{{[0-9]+}} >>= 60
+; CHECK32:           r{{[0-9]+}} s>>= 60
 ; CHECK:             r{{[0-9]+}} = 1
 
 ; CHECK:             .byte   115                     # string offset=1
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
index 01af9d8a697ba99131c7d965f276f015ae27392a..45d5ae1e1f30cb04a4b10f4a2e90ec6e01852579 100644
--- a/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
@@ -116,8 +116,10 @@ sw.epilog:                                        ; preds = %entry, %sw.bb9, %sw
 ; CHECK:             r{{[0-9]+}} = 4
 ; CHECK-EL:          r{{[0-9]+}} <<= 51
 ; CHECK-EB:          r{{[0-9]+}} <<= 41
-; CHECK:             r{{[0-9]+}} s>>= 60
-; CHECK:             r{{[0-9]+}} >>= 60
+; CHECK64:           r{{[0-9]+}} s>>= 60
+; CHECK64:           r{{[0-9]+}} >>= 60
+; CHECK32:           r{{[0-9]+}} >>= 60
+; CHECK32:           r{{[0-9]+}} s>>= 60
 ; CHECK:             r{{[0-9]+}} = 1
 
 ; CHECK:             .long   1                       # BTF_KIND_STRUCT(id = 2)
@@ -127,8 +129,11 @@ sw.epilog:                                        ; preds = %entry, %sw.bb9, %sw
 
 ; CHECK:             .long   16                      # FieldReloc
 ; CHECK-NEXT:        .long   30                      # Field reloc section string offset=30
-; CHECK32:           .long   6
-; CHECK64:           .long   7
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   36
@@ -136,11 +141,11 @@ sw.epilog:                                        ; preds = %entry, %sw.bb9, %sw
 ; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   36
-; CHECK-NEXT:        .long   1
-; CHECK64:           .long   .Ltmp{{[0-9]+}}
-; CHECK64:           .long   2
-; CHECK64:           .long   36
-; CHECK64:           .long   0
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
 ; CHECK-NEXT:        .long   2
 ; CHECK-NEXT:        .long   36
diff --git a/llvm/test/CodeGen/BPF/is_trunc_free.ll b/llvm/test/CodeGen/BPF/is_trunc_free.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cc6bef1a92a91719248dc82d2953a098946a7b3c
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/is_trunc_free.ll
@@ -0,0 +1,80 @@
+; RUN: llc -march=bpfel -mattr=+alu32 < %s | FileCheck %s
+; Source:
+;   struct env_t {
+;     unsigned data;
+;     unsigned data_end;
+;   };
+;   extern int work(struct env_t *skb, unsigned offset);
+;   int test(struct env_t *skb)
+;   {
+;     void *cursor, *data_end;
+;     struct env_t *srh, *ip;
+;
+;     data_end = (void *)(long)skb->data_end;
+;     cursor = (void *)(long)skb->data;
+;
+;     ip = cursor; cursor += sizeof(*ip);
+;     if ((void *)ip + sizeof(*ip) > data_end)
+;       return 0;
+;
+;     srh = cursor; cursor += sizeof(*srh);
+;     if ((void *)srh + sizeof(*srh) > data_end)
+;       return 0;
+;
+;     return work(skb, (char *)srh - (char *)(long)skb->data);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -emit-llvm -S test.c
+
+%struct.env_t = type { i32, i32 }
+
+; Function Attrs: nounwind
+define dso_local i32 @test(%struct.env_t* %skb) local_unnamed_addr #0 {
+entry:
+  %data_end1 = getelementptr inbounds %struct.env_t, %struct.env_t* %skb, i64 0, i32 1
+  %0 = load i32, i32* %data_end1, align 4, !tbaa !2
+  %conv = zext i32 %0 to i64
+  %1 = inttoptr i64 %conv to i8*
+  %data = getelementptr inbounds %struct.env_t, %struct.env_t* %skb, i64 0, i32 0
+  %2 = load i32, i32* %data, align 4, !tbaa !7
+  %conv2 = zext i32 %2 to i64
+  %3 = inttoptr i64 %conv2 to i8*
+  %add.ptr = getelementptr i8, i8* %3, i64 8
+  %cmp = icmp ugt i8* %add.ptr, %1
+  %add.ptr6 = getelementptr i8, i8* %3, i64 16
+  %cmp7 = icmp ugt i8* %add.ptr6, %1
+  %or.cond = or i1 %cmp, %cmp7
+  br i1 %or.cond, label %cleanup, label %if.end10
+
+if.end10:                                         ; preds = %entry
+  %sub.ptr.lhs.cast = ptrtoint i8* %add.ptr to i64
+  %4 = trunc i64 %sub.ptr.lhs.cast to i32
+  %conv13 = sub i32 %4, %2
+  %call = tail call i32 @work(%struct.env_t* nonnull %skb, i32 %conv13) #2
+  br label %cleanup
+
+cleanup:                                          ; preds = %entry, %if.end10
+  %retval.0 = phi i32 [ %call, %if.end10 ], [ 0, %entry ]
+  ret i32 %retval.0
+}
+
+; CHECK: w{{[0-9]+}} = *(u32 *)(r{{[0-9]+}} + 0)
+; CHECK-NOT: w{{[0-9]+}} = w{{[0-9]+}}
+
+declare dso_local i32 @work(%struct.env_t*, i32) local_unnamed_addr #1
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git 016d3ce1f4b07ee3056f7c10fedb24c441c4870f)"}
+!2 = !{!3, !4, i64 4}
+!3 = !{!"env_t", !4, i64 0, !4, i64 4}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!3, !4, i64 0}
diff --git a/llvm/test/CodeGen/BPF/is_zext_free.ll b/llvm/test/CodeGen/BPF/is_zext_free.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2a62dbefd62d285023e57a3f7ae970222bef82a8
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/is_zext_free.ll
@@ -0,0 +1,26 @@
+; RUN: llc -march=bpfel -mattr=+alu32 < %s | FileCheck %s
+; Source:
+;   unsigned test(unsigned long x, unsigned long y) {
+;     return x & y;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -emit-llvm -S test.c
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local i32 @test(i64 %x, i64 %y) local_unnamed_addr #0 {
+entry:
+  %and = and i64 %y, %x
+  %conv = trunc i64 %and to i32
+  ret i32 %conv
+}
+
+; CHECK: r[[REG1:[0-9]+]] = r{{[0-9]+}}
+; CHECK: w[[REG1]] &= w{{[0-9]+}}
+
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 11.0.0 (https://github.com/llvm/llvm-project.git b3ab5b2e7ffe9964ddf75a92fd7a444fe5aaa426)"}
diff --git a/llvm/test/CodeGen/Hexagon/isel-select-v4i8.ll b/llvm/test/CodeGen/Hexagon/isel-select-v4i8.ll
new file mode 100644
index 0000000000000000000000000000000000000000..58f72a15497a5e741e06e5892104a599def0db75
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/isel-select-v4i8.ll
@@ -0,0 +1,35 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; This used to fail:
+; LLVM ERROR: Cannot select: t54: v4i8 = select t50, t53, t52
+
+; CHECK: jumpr r31
+
+target triple = "hexagon"
+
+@g0 = external dso_local unnamed_addr constant [41 x i8], align 1
+define dso_local void @f0() local_unnamed_addr #0 {
+b0:
+  %v0 = load <16 x i32>, <16 x i32>* undef, align 16
+  %v1 = icmp eq <16 x i32> %v0, zeroinitializer
+  %v2 = or <16 x i1> %v1, zeroinitializer
+  %v3 = or <16 x i1> %v2, zeroinitializer
+  %v4 = or <16 x i1> %v3, zeroinitializer
+  %v5 = shufflevector <16 x i1> %v4, <16 x i1> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %v6 = or <16 x i1> %v4, %v5
+  %v7 = extractelement <16 x i1> %v6, i32 0
+  %v8 = or i1 %v7, undef
+  %v9 = or i1 %v8, undef
+  br i1 %v9, label %b2, label %b1
+
+b1:                                               ; preds = %b0
+  call void (i8*, ...) @f1(i8* getelementptr inbounds ([41 x i8], [41 x i8]* @g0, i32 0, i32 0))
+  unreachable
+
+b2:                                               ; preds = %b0
+  ret void
+}
+declare dso_local void @f1(i8*, ...) local_unnamed_addr #1
+
+attributes #0 = { "target-cpu"="hexagonv66" "target-features"="+hvx-length64b,+hvxv66,+v66,-long-calls" }
+attributes #1 = { "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/MIR/Hexagon/bundled-call-site-info.mir b/llvm/test/CodeGen/MIR/Hexagon/bundled-call-site-info.mir
index 5ffa0293a2e1baba6273eaeb3823016ce9361bef..96a9411af31135754a6f34aeac97b6ec87687c75 100644
--- a/llvm/test/CodeGen/MIR/Hexagon/bundled-call-site-info.mir
+++ b/llvm/test/CodeGen/MIR/Hexagon/bundled-call-site-info.mir
@@ -1,3 +1,5 @@
+# We do not support the call site info for the target now, so we use the experimental option (-debug-entry-values).
+
 # RUN: llc -debug-entry-values -run-pass=none -verify-machineinstrs -o - %s | FileCheck %s
 
 # Verify that it is possible to read and write MIR where a callSites entry
diff --git a/llvm/test/CodeGen/MIR/X86/call-site-info-error1.mir b/llvm/test/CodeGen/MIR/X86/call-site-info-error1.mir
index 096a80f77dbb65f330da1112fddc12756daa5de4..c28e1d7084ff986c7dd9aefaa4671c8a670d6f55 100644
--- a/llvm/test/CodeGen/MIR/X86/call-site-info-error1.mir
+++ b/llvm/test/CodeGen/MIR/X86/call-site-info-error1.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=x86_64-- -run-pass none -debug-entry-values %s -o - 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=x86_64-- -run-pass none %s -o - 2>&1 | FileCheck %s
 # CHECK: baa call instruction block out of range. Unable to reference bb:1
 --- |
   define dso_local i32 @baa(i32 %a) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir b/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir
index bd5b2451a8d768cbc3a0201187e3599322f7a1b3..ca26a738f6e578eec990c31c5aa4bb40d1265b31 100644
--- a/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir
+++ b/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=x86_64-- -run-pass none -debug-entry-values %s -o - 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=x86_64-- -run-pass none %s -o - 2>&1 | FileCheck %s
 # CHECK: baa call instruction offset out of range. Unable to reference instruction at bb: 0 at offset:1
 --- |
   define dso_local i32 @baa(i32 %a) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/MIR/X86/call-site-info-error3.mir b/llvm/test/CodeGen/MIR/X86/call-site-info-error3.mir
index ded9fd2bb1bc12059d363963dd69bdc5820a1a46..3271a8b3a80cee7b0d7cd97171206bd6b77d2fd4 100644
--- a/llvm/test/CodeGen/MIR/X86/call-site-info-error3.mir
+++ b/llvm/test/CodeGen/MIR/X86/call-site-info-error3.mir
@@ -1,4 +1,4 @@
-# RUN: not llc -mtriple=x86_64-- -run-pass none -debug-entry-values %s -o - 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=x86_64-- -run-pass none %s -o - 2>&1 | FileCheck %s
 # CHECK: baa call site info should reference call instruction. Instruction at bb:0 at offset:0 is not a call instruction
 --- |
   define dso_local i32 @baa(i32 %a) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/MIR/X86/call-site-info-error4.mir b/llvm/test/CodeGen/MIR/X86/call-site-info-error4.mir
index 2472aa707e16948cd28add731d78ef2223fa6952..8f71d2b21da39bb7a3ea2e974a32bb8d401d45f9 100644
--- a/llvm/test/CodeGen/MIR/X86/call-site-info-error4.mir
+++ b/llvm/test/CodeGen/MIR/X86/call-site-info-error4.mir
@@ -1,5 +1,5 @@
-# RUN: not llc -mtriple=x86_64-- -run-pass none %s -o - 2>&1 | FileCheck %s
-# CHECK: Call site info provided but not used
+# RUN: llc -mtriple=x86_64-- -run-pass none %s -o - 2>&1 | FileCheck %s
+# CHECK-NOT: Call site info provided but not used
 --- |
   define dso_local i32 @baa(i32 %a) local_unnamed_addr {
   entry:
diff --git a/llvm/test/CodeGen/RISCV/i32-icmp.ll b/llvm/test/CodeGen/RISCV/i32-icmp.ll
index 04b1eeaad21a0d648b8e0367cc949e7ce918c714..34557696cc5715ab4b1baf0d3263aa82ce32bf4c 100644
--- a/llvm/test/CodeGen/RISCV/i32-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/i32-icmp.ll
@@ -19,7 +19,7 @@ define i32 @icmp_eq(i32 %a, i32 %b) nounwind {
 define i32 @icmp_eq_constant(i32 %a) nounwind {
 ; RV32I-LABEL: icmp_eq_constant:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xori a0, a0, 42
+; RV32I-NEXT:    addi a0, a0, -42
 ; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    ret
   %1 = icmp eq i32 %a, 42
@@ -27,6 +27,40 @@ define i32 @icmp_eq_constant(i32 %a) nounwind {
   ret i32 %2
 }
 
+define i32 @icmp_eq_constant_2048(i32 %a) nounwind {
+; RV32I-LABEL: icmp_eq_constant_2048:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, -2048
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+  %1 = icmp eq i32 %a, 2048
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @icmp_eq_constant_neg_2048(i32 %a) nounwind {
+; RV32I-LABEL: icmp_eq_constant_neg_2048:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a1, zero, -2048
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+  %1 = icmp eq i32 %a, -2048
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @icmp_eq_constant_neg_2047(i32 %a) nounwind {
+; RV32I-LABEL: icmp_eq_constant_neg_2047:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, 2047
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+  %1 = icmp eq i32 %a, -2047
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
 define i32 @icmp_eqz(i32 %a) nounwind {
 ; RV32I-LABEL: icmp_eqz:
 ; RV32I:       # %bb.0:
@@ -51,7 +85,7 @@ define i32 @icmp_ne(i32 %a, i32 %b) nounwind {
 define i32 @icmp_ne_constant(i32 %a) nounwind {
 ; RV32I-LABEL: icmp_ne_constant:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xori a0, a0, 42
+; RV32I-NEXT:    addi a0, a0, -42
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    ret
   %1 = icmp ne i32 %a, 42
@@ -59,6 +93,29 @@ define i32 @icmp_ne_constant(i32 %a) nounwind {
   ret i32 %2
 }
 
+define i32 @icmp_ne_constant_2048(i32 %a) nounwind {
+; RV32I-LABEL: icmp_ne_constant_2048:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a0, a0, -2048
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+  %1 = icmp ne i32 %a, 2048
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @icmp_ne_constant_neg_2048(i32 %a) nounwind {
+; RV32I-LABEL: icmp_ne_constant_neg_2048:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a1, zero, -2048
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+  %1 = icmp ne i32 %a, -2048
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
 define i32 @icmp_nez(i32 %a) nounwind {
 ; RV32I-LABEL: icmp_nez:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/saverestore.ll b/llvm/test/CodeGen/RISCV/saverestore.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ef1b2a52afa91bda90fa4a60960cfc4b637e56e3
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/saverestore.ll
@@ -0,0 +1,299 @@
+; RUN: llc -mtriple=riscv32 < %s | FileCheck %s -check-prefix=RV32I
+; RUN: llc -mtriple=riscv64 < %s | FileCheck %s -check-prefix=RV64I
+; RUN: llc -mtriple=riscv32 -mattr=+save-restore < %s | FileCheck %s -check-prefix=RV32I-SR
+; RUN: llc -mtriple=riscv64 -mattr=+save-restore < %s | FileCheck %s -check-prefix=RV64I-SR
+; RUN: llc -mtriple=riscv32 -mattr=+f,+save-restore -target-abi=ilp32f < %s | FileCheck %s -check-prefix=RV32I-FP-SR
+; RUN: llc -mtriple=riscv64 -mattr=+f,+d,+save-restore -target-abi=lp64d < %s | FileCheck %s -check-prefix=RV64I-FP-SR
+
+; Check that the correct save/restore libcalls are generated.
+
+@var0 = global [18 x i32] zeroinitializer
+@var1 = global [24 x i32] zeroinitializer
+@var2 = global [30 x i32] zeroinitializer
+
+define void @callee_saved0() nounwind {
+; RV32I-LABEL: callee_saved0:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: callee_saved0:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: callee_saved0:
+; RV32I-SR:         call t0, __riscv_save_5
+; RV32I-SR:         tail __riscv_restore_5
+;
+; RV64I-SR-LABEL: callee_saved0:
+; RV64I-SR:         call t0, __riscv_save_5
+; RV64I-SR:         tail __riscv_restore_5
+;
+; RV32I-FP-SR-LABEL: callee_saved0:
+; RV32I-FP-SR:         call t0, __riscv_save_5
+; RV32I-FP-SR:         tail __riscv_restore_5
+;
+; RV64I-FP-SR-LABEL: callee_saved0:
+; RV64I-FP-SR:         call t0, __riscv_save_5
+; RV64I-FP-SR:         tail __riscv_restore_5
+  %val = load [18 x i32], [18 x i32]* @var0
+  store volatile [18 x i32] %val, [18 x i32]* @var0
+  ret void
+}
+
+define void @callee_saved1() nounwind {
+; RV32I-LABEL: callee_saved1:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: callee_saved1:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: callee_saved1:
+; RV32I-SR:         call t0, __riscv_save_11
+; RV32I-SR:         tail __riscv_restore_11
+;
+; RV64I-SR-LABEL: callee_saved1:
+; RV64I-SR:         call t0, __riscv_save_11
+; RV64I-SR:         tail __riscv_restore_11
+;
+; RV32I-FP-SR-LABEL: callee_saved1:
+; RV32I-FP-SR:         call t0, __riscv_save_11
+; RV32I-FP-SR:         tail __riscv_restore_11
+;
+; RV64I-FP-SR-LABEL: callee_saved1:
+; RV64I-FP-SR:         call t0, __riscv_save_11
+; RV64I-FP-SR:         tail __riscv_restore_11
+  %val = load [24 x i32], [24 x i32]* @var1
+  store volatile [24 x i32] %val, [24 x i32]* @var1
+  ret void
+}
+
+define void @callee_saved2() nounwind {
+; RV32I-LABEL: callee_saved2:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: callee_saved2:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: callee_saved2:
+; RV32I-SR:         call t0, __riscv_save_12
+; RV32I-SR:         tail __riscv_restore_12
+;
+; RV64I-SR-LABEL: callee_saved2:
+; RV64I-SR:         call t0, __riscv_save_12
+; RV64I-SR:         tail __riscv_restore_12
+;
+; RV32I-FP-SR-LABEL: callee_saved2:
+; RV32I-FP-SR:         call t0, __riscv_save_12
+; RV32I-FP-SR:         tail __riscv_restore_12
+;
+; RV64I-FP-SR-LABEL: callee_saved2:
+; RV64I-FP-SR:         call t0, __riscv_save_12
+; RV64I-FP-SR:         tail __riscv_restore_12
+  %val = load [30 x i32], [30 x i32]* @var2
+  store volatile [30 x i32] %val, [30 x i32]* @var2
+  ret void
+}
+
+; Check that floating point callee saved registers are still manually saved and
+; restored.
+
+define void @callee_saved_fp() nounwind {
+; RV32I-LABEL: callee_saved_fp:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: callee_saved_fp:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: callee_saved_fp:
+; RV32I-SR:         call t0, __riscv_save_7
+; RV32I-SR:         tail __riscv_restore_7
+;
+; RV64I-SR-LABEL: callee_saved_fp:
+; RV64I-SR:         call t0, __riscv_save_7
+; RV64I-SR:         tail __riscv_restore_7
+;
+; RV32I-FP-SR-LABEL: callee_saved_fp:
+; RV32I-FP-SR:         call t0, __riscv_save_7
+; RV32I-FP-SR-NEXT:    addi sp, sp, -16
+; RV32I-FP-SR-NEXT:    fsw fs0, 12(sp)
+; RV32I-FP-SR:         flw fs0, 12(sp)
+; RV32I-FP-SR-NEXT:    addi sp, sp, 16
+; RV32I-FP-SR-NEXT:    tail __riscv_restore_7
+;
+; RV64I-FP-SR-LABEL: callee_saved_fp:
+; RV64I-FP-SR:         call t0, __riscv_save_7
+; RV64I-FP-SR-NEXT:    addi sp, sp, -16
+; RV64I-FP-SR-NEXT:    fsd fs0, 8(sp)
+; RV64I-FP-SR:         fld fs0, 8(sp)
+; RV64I-FP-SR-NEXT:    addi sp, sp, 16
+; RV64I-FP-SR-NEXT:    tail __riscv_restore_7
+  call void asm sideeffect "", "~{f8},~{x9},~{x18},~{x19},~{x20},~{x21},~{x22}"()
+  ret void
+}
+
+; Check that preserving tail calls is preferred over save/restore
+
+declare i32 @tail_callee(i32 %i)
+
+define i32 @tail_call(i32 %i) nounwind {
+; RV32I-LABEL: tail_call:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I:         tail tail_callee
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: tail_call:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I:         tail tail_callee
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: tail_call:
+; RV32I-SR-NOT:     call t0, __riscv_save
+; RV32I-SR:         tail tail_callee
+; RV32I-SR-NOT:     tail __riscv_restore
+;
+; RV64I-SR-LABEL: tail_call:
+; RV64I-SR-NOT:     call t0, __riscv_save
+; RV64I-SR:         tail tail_callee
+; RV64I-SR-NOT:     tail __riscv_restore
+;
+; RV32I-FP-SR-LABEL: tail_call:
+; RV32I-FP-SR-NOT:     call t0, __riscv_save
+; RV32I-FP-SR:         tail tail_callee
+; RV32I-FP-SR-NOT:     tail __riscv_restore
+;
+; RV64I-FP-SR-LABEL: tail_call:
+; RV64I-FP-SR-NOT:     call t0, __riscv_save
+; RV64I-FP-SR:         tail tail_callee
+; RV64I-FP-SR-NOT:     tail __riscv_restore
+entry:
+  %val = load [18 x i32], [18 x i32]* @var0
+  store volatile [18 x i32] %val, [18 x i32]* @var0
+  %r = tail call i32 @tail_callee(i32 %i)
+  ret i32 %r
+}
+
+; Check that functions with varargs do not use save/restore code
+
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_end(i8*)
+
+define i32 @varargs(i8* %fmt, ...) nounwind {
+; RV32I-LABEL: varargs:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: varargs:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: varargs:
+; RV32I-SR-NOT:     call t0, __riscv_save
+; RV32I-SR-NOT:     tail __riscv_restore
+;
+; RV64I-SR-LABEL: varargs:
+; RV64I-SR-NOT:     call t0, __riscv_save
+; RV64I-SR-NOT:     tail __riscv_restore
+;
+; RV32I-FP-SR-LABEL: varargs:
+; RV32I-FP-SR-NOT:     call t0, __riscv_save
+; RV32I-FP-SR-NOT:     tail __riscv_restore
+;
+; RV64I-FP-SR-LABEL: varargs:
+; RV64I-FP-SR-NOT:     call t0, __riscv_save
+; RV64I-FP-SR-NOT:     tail __riscv_restore
+  %va = alloca i8*, align 4
+  %1 = bitcast i8** %va to i8*
+  call void @llvm.va_start(i8* %1)
+  %argp.cur = load i8*, i8** %va, align 4
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+  store i8* %argp.next, i8** %va, align 4
+  %2 = bitcast i8* %argp.cur to i32*
+  %3 = load i32, i32* %2, align 4
+  call void @llvm.va_end(i8* %1)
+  ret i32 %3
+}
+
+define void @many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32) nounwind {
+; RV32I-LABEL: many_args:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: many_args:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: many_args:
+; RV32I-SR:         call t0, __riscv_save_5
+; RV32I-SR:         tail __riscv_restore_5
+;
+; RV64I-SR-LABEL: many_args:
+; RV64I-SR:         call t0, __riscv_save_5
+; RV64I-SR:         tail __riscv_restore_5
+;
+; RV32I-FP-SR-LABEL: many_args:
+; RV32I-FP-SR:         call t0, __riscv_save_5
+; RV32I-FP-SR:         tail __riscv_restore_5
+;
+; RV64I-FP-SR-LABEL: many_args:
+; RV64I-FP-SR:         call t0, __riscv_save_5
+; RV64I-FP-SR:         tail __riscv_restore_5
+entry:
+  %val = load [18 x i32], [18 x i32]* @var0
+  store volatile [18 x i32] %val, [18 x i32]* @var0
+  ret void
+}
+
+; Check that dynamic allocation calculations remain correct
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+declare void @notdead(i8*)
+
+define void @alloca(i32 %n) nounwind {
+; RV32I-LABEL: alloca:
+; RV32I-NOT:     call t0, __riscv_save
+; RV32I:         addi s0, sp, 16
+; RV32I:         addi sp, s0, -16
+; RV32I-NOT:     tail __riscv_restore
+;
+; RV64I-LABEL: alloca:
+; RV64I-NOT:     call t0, __riscv_save
+; RV64I:         addi s0, sp, 32
+; RV64I:         addi sp, s0, -32
+; RV64I-NOT:     tail __riscv_restore
+;
+; RV32I-SR-LABEL: alloca:
+; RV32I-SR:         call t0, __riscv_save_2
+; RV32I-SR:         addi s0, sp, 16
+; RV32I-SR:         addi sp, s0, -16
+; RV32I-SR:         tail __riscv_restore_2
+;
+; RV64I-SR-LABEL: alloca:
+; RV64I-SR:         call t0, __riscv_save_2
+; RV64I-SR:         addi s0, sp, 32
+; RV64I-SR:         addi sp, s0, -32
+; RV64I-SR:         tail __riscv_restore_2
+;
+; RV32I-FP-SR-LABEL: alloca:
+; RV32I-FP-SR:         call t0, __riscv_save_2
+; RV32I-FP-SR:         addi s0, sp, 16
+; RV32I-FP-SR:         addi sp, s0, -16
+; RV32I-FP-SR:         tail __riscv_restore_2
+;
+; RV64I-FP-SR-LABEL: alloca:
+; RV64I-FP-SR:         call t0, __riscv_save_2
+; RV64I-FP-SR:         addi s0, sp, 32
+; RV64I-FP-SR:         addi sp, s0, -32
+; RV64I-FP-SR:         tail __riscv_restore_2
+  %sp = call i8* @llvm.stacksave()
+  %addr = alloca i8, i32 %n
+  call void @notdead(i8* %addr)
+  call void @llvm.stackrestore(i8* %sp)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/setcc-logic.ll b/llvm/test/CodeGen/RISCV/setcc-logic.ll
index 0c29bf505234ed52b6eb4606511cea12b9b5b353..dfaee187db9cc480ef0e4d0092004d6ffd38c726 100644
--- a/llvm/test/CodeGen/RISCV/setcc-logic.ll
+++ b/llvm/test/CodeGen/RISCV/setcc-logic.ll
@@ -102,9 +102,9 @@ define i1 @and_icmps_const_1bit_diff(i32 %x) nounwind {
 define i1 @and_icmps_const_not1bit_diff(i32 %x) nounwind {
 ; RV32I-LABEL: and_icmps_const_not1bit_diff:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xori a1, a0, 44
+; RV32I-NEXT:    addi a1, a0, -44
 ; RV32I-NEXT:    snez a1, a1
-; RV32I-NEXT:    xori a0, a0, 92
+; RV32I-NEXT:    addi a0, a0, -92
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    and a0, a1, a0
 ; RV32I-NEXT:    ret
@@ -113,9 +113,9 @@ define i1 @and_icmps_const_not1bit_diff(i32 %x) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    xori a1, a0, 44
+; RV64I-NEXT:    addi a1, a0, -44
 ; RV64I-NEXT:    snez a1, a1
-; RV64I-NEXT:    xori a0, a0, 92
+; RV64I-NEXT:    addi a0, a0, -92
 ; RV64I-NEXT:    snez a0, a0
 ; RV64I-NEXT:    and a0, a1, a0
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/shrinkwrap.ll b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
index 88ff585a7a20a01b3678ce0ce332238a1ccc4943..d8e9fdf0b7d061a3c75589d774bedf0aec1afebb 100644
--- a/llvm/test/CodeGen/RISCV/shrinkwrap.ll
+++ b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple riscv32 < %s | FileCheck %s -check-prefix=RV32I-NOSW
 ; RUN: llc -mtriple riscv32 -enable-shrink-wrap < %s | FileCheck %s -check-prefix=RV32I-SW
+; RUN: llc -mtriple riscv32 -enable-shrink-wrap -mattr=+save-restore < %s \
+; RUN:     | FileCheck %s -check-prefix=RV32I-SW-SR
 
 
 declare void @abort()
@@ -29,6 +31,16 @@ define void @eliminate_restore(i32 %n) nounwind {
 ; RV32I-SW-NEXT:    addi sp, sp, -16
 ; RV32I-SW-NEXT:    sw ra, 12(sp)
 ; RV32I-SW-NEXT:    call abort
+;
+; RV32I-SW-SR-LABEL: eliminate_restore:
+; RV32I-SW-SR:       # %bb.0:
+; RV32I-SW-SR-NEXT:    addi a1, zero, 32
+; RV32I-SW-SR-NEXT:    bgeu a1, a0, .LBB0_2
+; RV32I-SW-SR-NEXT:  # %bb.1: # %if.end
+; RV32I-SW-SR-NEXT:    ret
+; RV32I-SW-SR-NEXT:  .LBB0_2: # %if.then
+; RV32I-SW-SR-NEXT:    call t0, __riscv_save_0
+; RV32I-SW-SR-NEXT:    call abort
   %cmp = icmp ule i32 %n, 32
   br i1 %cmp, label %if.then, label %if.end
 
@@ -84,6 +96,23 @@ define void @conditional_alloca(i32 %n) nounwind {
 ; RV32I-SW-NEXT:    addi sp, sp, 16
 ; RV32I-SW-NEXT:  .LBB1_2: # %if.end
 ; RV32I-SW-NEXT:    ret
+;
+; RV32I-SW-SR-LABEL: conditional_alloca:
+; RV32I-SW-SR:       # %bb.0:
+; RV32I-SW-SR-NEXT:    addi a1, zero, 32
+; RV32I-SW-SR-NEXT:    bltu a1, a0, .LBB1_2
+; RV32I-SW-SR-NEXT:  # %bb.1: # %if.then
+; RV32I-SW-SR-NEXT:    call t0, __riscv_save_1
+; RV32I-SW-SR-NEXT:    addi s0, sp, 16
+; RV32I-SW-SR-NEXT:    addi a0, a0, 15
+; RV32I-SW-SR-NEXT:    andi a0, a0, -16
+; RV32I-SW-SR-NEXT:    sub a0, sp, a0
+; RV32I-SW-SR-NEXT:    mv sp, a0
+; RV32I-SW-SR-NEXT:    call notdead
+; RV32I-SW-SR-NEXT:    addi sp, s0, -16
+; RV32I-SW-SR-NEXT:    tail __riscv_restore_1
+; RV32I-SW-SR-NEXT:  .LBB1_2: # %if.end
+; RV32I-SW-SR-NEXT:    ret
   %cmp = icmp ule i32 %n, 32
   br i1 %cmp, label %if.then, label %if.end
 
diff --git a/llvm/test/CodeGen/SystemZ/multiselect-02.mir b/llvm/test/CodeGen/SystemZ/multiselect-02.mir
new file mode 100644
index 0000000000000000000000000000000000000000..abb6a01c9eb548beaca3b2dec8930926784dd99f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/multiselect-02.mir
@@ -0,0 +1,44 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z10 -run-pass=finalize-isel -o - %s \
+# RUN:   | FileCheck %s
+#
+# Test that an instruction (ZEXT128) that uses custom insertion gets treated
+# correctly also when it lies between two Select instructions that could
+# potentially be handled together.
+#
+# CHECK-LABEL: bb.0.entry:
+# CHECK-NOT: ZEXT128
+
+--- |
+  declare void @bar(i32)
+  define i32 @fun() { entry: ret i32 0 }
+---
+name:            fun
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    %1:addr64bit = IMPLICIT_DEF
+    %0:gr32bit = LLC %1, 0, $noreg :: (load 1 from `i8* undef`)
+    CHI killed %0, 0, implicit-def $cc
+    %2:gr32bit = LHI 2
+    %3:gr32bit = LHI 8
+    %4:gr32bit = Select32 killed %3, killed %2, 14, 8, implicit $cc
+    %5:gr32bit = LHI 128
+    %7:gr64bit = IMPLICIT_DEF
+    %6:gr64bit = INSERT_SUBREG %7, killed %5, %subreg.subreg_l32
+    %8:gr128bit = ZEXT128 killed %6
+    %10:addr64bit = IMPLICIT_DEF
+    %9:gr128bit = DL %8, %10, 0, $noreg :: (load 4 from `i64* undef` + 4)
+    %11:gr32bit = COPY %9.subreg_l32
+    %12:gr64bit = LGHI 2
+    %13:gr64bit = LGHI 8
+    %14:gr64bit = Select64 killed %13, killed %12, 14, 8, implicit $cc
+    CR %4, %11, implicit-def $cc
+    %15:gr32bit = Select32 %11, %4, 14, 4, implicit $cc
+    ADJCALLSTACKDOWN 0, 0
+    $r2d = COPY %14
+    CallBRASL @bar, $r2d, csr_systemz, implicit-def dead $r14d, implicit-def dead $cc, implicit $fpc
+    ADJCALLSTACKUP 0, 0
+    $r2l = COPY %15
+    Return implicit $r2l
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
index 257d950c60fb364826ce25471b4c3d4fc3054568..ad7920007267e712ce0ad8038feec74d942689a2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
@@ -32,7 +32,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
   %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0
   %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
-  %induction = add <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %induction = or <16 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp = getelementptr inbounds i8, i8* %a, i32 %index
   %tmp1 = icmp ule <16 x i32> %induction, %broadcast.splat11
   %tmp2 = bitcast i8* %tmp to <16 x i8>*
@@ -137,7 +137,7 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %tmp14 = phi i32 [ %tmp13, %vector.ph ], [ %tmp15, %vector.body ]
   %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
   %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
-  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %induction = or <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
   %tmp = getelementptr inbounds i32, i32* %a, i32 %index
   %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat11
   %tmp2 = bitcast i32* %tmp to <4 x i32>*
diff --git a/llvm/test/CodeGen/VE/bitreverse.ll b/llvm/test/CodeGen/VE/bitreverse.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fce969af657e2498b91bdce2ad7db239c2191030
--- /dev/null
+++ b/llvm/test/CodeGen/VE/bitreverse.ll
@@ -0,0 +1,100 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %p) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.bitreverse.i64(i64 %p)
+  ret i64 %r
+}
+
+declare i64 @llvm.bitreverse.i64(i64)
+
+define i32 @func2(i32 %p) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    srl %s0, %s0, 32
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.bitreverse.i32(i32 %p)
+  ret i32 %r
+}
+
+declare i32 @llvm.bitreverse.i32(i32)
+
+define signext i16 @func3(i16 signext %p) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    sra.l %s0, %s0, 48
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.bitreverse.i16(i16 %p)
+  ret i16 %r
+}
+
+declare i16 @llvm.bitreverse.i16(i16)
+
+define signext i8 @func4(i8 signext %p) {
+; CHECK-LABEL: func4:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    sra.l %s0, %s0, 56
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i8 @llvm.bitreverse.i8(i8 %p)
+  ret i8 %r
+}
+
+declare i8 @llvm.bitreverse.i8(i8)
+
+define i64 @func5(i64 %p) {
+; CHECK-LABEL: func5:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.bitreverse.i64(i64 %p)
+  ret i64 %r
+}
+
+define i32 @func6(i32 %p) {
+; CHECK-LABEL: func6:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    srl %s0, %s0, 32
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.bitreverse.i32(i32 %p)
+  ret i32 %r
+}
+
+define zeroext i16 @func7(i16 zeroext %p) {
+; CHECK-LABEL: func7:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    srl %s0, %s0, 48
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.bitreverse.i16(i16 %p)
+  ret i16 %r
+}
+
+define zeroext i8 @func8(i8 zeroext %p) {
+; CHECK-LABEL: func8:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    brv %s0, %s0
+; CHECK-NEXT:    srl %s0, %s0, 56
+; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i8 @llvm.bitreverse.i8(i8 %p)
+  ret i8 %r
+}
+
diff --git a/llvm/test/CodeGen/VE/bswap.ll b/llvm/test/CodeGen/VE/bswap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..274085462856f618f48fb64045684bcac787180a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/bswap.ll
@@ -0,0 +1,71 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %p) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    bswp %s0, %s0, 0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.bswap.i64(i64 %p)
+  ret i64 %r
+}
+
+declare i64 @llvm.bswap.i64(i64)
+
+define i32 @func2(i32 %p) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    bswp %s0, %s0, 1
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.bswap.i32(i32 %p)
+  ret i32 %r
+}
+
+declare i32 @llvm.bswap.i32(i32)
+
+define signext i16 @func3(i16 signext %p) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    bswp %s0, %s0, 1
+; CHECK-NEXT:    sra.w.sx %s0, %s0, 16
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.bswap.i16(i16 %p)
+  ret i16 %r
+}
+
+declare i16 @llvm.bswap.i16(i16)
+
+define i64 @func4(i64 %p) {
+; CHECK-LABEL: func4:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    bswp %s0, %s0, 0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.bswap.i64(i64 %p)
+  ret i64 %r
+}
+
+define i32 @func5(i32 %p) {
+; CHECK-LABEL: func5:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    bswp %s0, %s0, 1
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.bswap.i32(i32 %p)
+  ret i32 %r
+}
+
+define zeroext i16 @func6(i16 zeroext %p) {
+; CHECK-LABEL: func6:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    bswp %s0, %s0, 1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, 16
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.bswap.i16(i16 %p)
+  ret i16 %r
+}
diff --git a/llvm/test/CodeGen/VE/ctlz.ll b/llvm/test/CodeGen/VE/ctlz.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0d0b2c3f6b0a7859da0cc10030662c55fb819917
--- /dev/null
+++ b/llvm/test/CodeGen/VE/ctlz.ll
@@ -0,0 +1,54 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %p) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    ldz %s0, %s0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.ctlz.i64(i64 %p, i1 true)
+  ret i64 %r
+}
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+
+define i32 @func2(i32 %p) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    sll %s0, %s0, 32
+; CHECK-NEXT:    ldz %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.ctlz.i32(i32 %p, i1 true)
+  ret i32 %r
+}
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+
+define i16 @func3(i16 %p) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    and %s0, %s0, (48)0
+; CHECK-NEXT:    sll %s0, %s0, 32
+; CHECK-NEXT:    ldz %s0, %s0
+; CHECK-NEXT:    lea %s0, -16(%s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.ctlz.i16(i16 %p, i1 true)
+  ret i16 %r
+}
+
+declare i16 @llvm.ctlz.i16(i16, i1)
+
+define i8 @func4(i8 %p) {
+; CHECK-LABEL: func4:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    and %s0, %s0, (56)0
+; CHECK-NEXT:    sll %s0, %s0, 32
+; CHECK-NEXT:    ldz %s0, %s0
+; CHECK-NEXT:    lea %s0, -24(%s0)
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i8 @llvm.ctlz.i8(i8 %p, i1 true)
+  ret i8 %r
+}
+
+declare i8 @llvm.ctlz.i8(i8, i1)
diff --git a/llvm/test/CodeGen/VE/ctpop.ll b/llvm/test/CodeGen/VE/ctpop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3d25909ab25cbd1a73385ec83c569f5cf8324a55
--- /dev/null
+++ b/llvm/test/CodeGen/VE/ctpop.ll
@@ -0,0 +1,54 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %p) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.ctpop.i64(i64 %p)
+  ret i64 %r
+}
+
+declare i64 @llvm.ctpop.i64(i64 %p)
+
+define i32 @func2(i32 %p) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.ctpop.i32(i32 %p)
+  ret i32 %r
+}
+
+declare i32 @llvm.ctpop.i32(i32 %p)
+
+define i16 @func3(i16 %p) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    and %s0, %s0, (48)0
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.ctpop.i16(i16 %p)
+  ret i16 %r
+}
+
+declare i16 @llvm.ctpop.i16(i16 %p)
+
+define i8 @func4(i8 %p) {
+; CHECK-LABEL: func4:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    and %s0, %s0, (56)0
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i8 @llvm.ctpop.i8(i8 %p)
+  ret i8 %r
+}
+
+declare i8 @llvm.ctpop.i8(i8)
diff --git a/llvm/test/CodeGen/VE/cttz.ll b/llvm/test/CodeGen/VE/cttz.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0b0399523ab7eefdd22fffb9ab91a57e0b782bc8
--- /dev/null
+++ b/llvm/test/CodeGen/VE/cttz.ll
@@ -0,0 +1,63 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %p) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s1, -1(%s0)
+; CHECK-NEXT:    xor %s0, -1, %s0
+; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i64 @llvm.cttz.i64(i64 %p, i1 true)
+  ret i64 %r
+}
+
+declare i64 @llvm.cttz.i64(i64, i1)
+
+define i32 @func2(i32 %p) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s1, -1(%s0)
+; CHECK-NEXT:    xor %s0, -1, %s0
+; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i32 @llvm.cttz.i32(i32 %p, i1 true)
+  ret i32 %r
+}
+
+declare i32 @llvm.cttz.i32(i32, i1)
+
+define i16 @func3(i16 %p) {
+; CHECK-LABEL: func3:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s1, -1(%s0)
+; CHECK-NEXT:    xor %s0, -1, %s0
+; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i16 @llvm.cttz.i16(i16 %p, i1 true)
+  ret i16 %r
+}
+
+declare i16 @llvm.cttz.i16(i16, i1)
+
+define i8 @func4(i8 %p) {
+; CHECK-LABEL: func4:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    lea %s1, -1(%s0)
+; CHECK-NEXT:    xor %s0, -1, %s0
+; CHECK-NEXT:    and %s0, %s0, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    pcnt %s0, %s0
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 killed $sx0
+; CHECK-NEXT:    or %s11, 0, %s9
+  %r = tail call i8 @llvm.cttz.i8(i8 %p, i1 true)
+  ret i8 %r
+}
+
+declare i8 @llvm.cttz.i8(i8, i1)
diff --git a/llvm/test/CodeGen/VE/rotl.ll b/llvm/test/CodeGen/VE/rotl.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e7c498f1d34d2eb798c19566a4032e1cad4a7b31
--- /dev/null
+++ b/llvm/test/CodeGen/VE/rotl.ll
@@ -0,0 +1,37 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %a, i32 %b) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    sll %s2, %s0, %s1
+; CHECK-NEXT:    lea %s3, 64
+; CHECK-NEXT:    subs.w.sx %s1, %s3, %s1
+; CHECK-NEXT:    srl %s0, %s0, %s1
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %b64 = zext i32 %b to i64
+  %a.sl = shl i64 %a, %b64
+  %b.inv = sub nsw i32 64, %b
+  %b.inv64 = zext i32 %b.inv to i64
+  %a.sr = lshr i64 %a, %b.inv64
+  %r = or i64 %a.sr, %a.sl
+  ret i64 %r
+}
+
+define i32 @func2(i32 %a, i32 %b) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    sla.w.sx %s2, %s0, %s1
+; CHECK-NEXT:    subs.w.sx %s1, 32, %s1
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    srl %s0, %s0, %s1
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %a.sl = shl i32 %a, %b
+  %b.inv = sub nsw i32 32, %b
+  %a.sr = lshr i32 %a, %b.inv
+  %r = or i32 %a.sr, %a.sl
+  ret i32 %r
+}
+
diff --git a/llvm/test/CodeGen/VE/rotr.ll b/llvm/test/CodeGen/VE/rotr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..40734a3d5178a575c026fbb08261d4c18f0ad821
--- /dev/null
+++ b/llvm/test/CodeGen/VE/rotr.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
+
+define i64 @func1(i64 %a, i32 %b) {
+; CHECK-LABEL: func1:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    srl %s2, %s0, %s1
+; CHECK-NEXT:    lea %s3, 64
+; CHECK-NEXT:    subs.w.sx %s1, %s3, %s1
+; CHECK-NEXT:    sll %s0, %s0, %s1
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %b64 = zext i32 %b to i64
+  %a.lr = lshr i64 %a, %b64
+  %b.inv = sub nsw i32 64, %b
+  %b.inv64 = zext i32 %b.inv to i64
+  %a.sl = shl i64 %a, %b.inv64
+  %r = or i64 %a.sl, %a.lr
+  ret i64 %r
+}
+
+define i32 @func2(i32 %a, i32 %b) {
+; CHECK-LABEL: func2:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    # kill: def $sw0 killed $sw0 def $sx0
+; CHECK-NEXT:    and %s2, %s0, (32)0
+; CHECK-NEXT:    srl %s2, %s2, %s1
+; CHECK-NEXT:    subs.w.sx %s1, 32, %s1
+; CHECK-NEXT:    sla.w.sx %s0, %s0, %s1
+; CHECK-NEXT:    or %s0, %s0, %s2
+; CHECK-NEXT:    or %s11, 0, %s9
+  %a.lr = lshr i32 %a, %b
+  %b.inv = sub nsw i32 32, %b
+  %a.sl = shl i32 %a, %b.inv
+  %r = or i32 %a.sl, %a.lr
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/call-site-info-output.ll b/llvm/test/CodeGen/X86/call-site-info-output.ll
index 4b1e236aadfe0b9f75ab63d4986c1ccfb787c1e5..8586d34762db7fef3d8ca4a29f21ecae133af429 100644
--- a/llvm/test/CodeGen/X86/call-site-info-output.ll
+++ b/llvm/test/CodeGen/X86/call-site-info-output.ll
@@ -1,6 +1,6 @@
 ; Test call site info MIR printer and parser.Parser assertions and machine
 ; verifier will check the rest;
-; RUN: llc -debug-entry-values %s -stop-before=finalize-isel -o %t.mir
+; RUN: llc %s -stop-before=finalize-isel -o %t.mir
 ; RUN: cat %t.mir | FileCheck %s
 ; CHECK: name: fn2
 ; CHECK: callSites:
@@ -10,7 +10,7 @@
 ; CHECK-NEXT:   arg: 0, reg: '$edi'
 ; CHECK-NEXT:   arg: 1, reg: '$esi'
 ; CHECK-NEXT:   arg: 2, reg: '$edx'
-; RUN: llc -debug-entry-values %t.mir -run-pass=finalize-isel -o -| FileCheck %s --check-prefix=PARSER
+; RUN: llc %t.mir -run-pass=finalize-isel -o -| FileCheck %s --check-prefix=PARSER
 ; Verify that we are able to parse output mir and that we are getting the same result.
 ; PARSER: name: fn2
 ; PARSER: callSites:
diff --git a/llvm/test/CodeGen/X86/half-constrained.ll b/llvm/test/CodeGen/X86/half-constrained.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2082e1df67bf7ab78739c50e280fac6365597d03
--- /dev/null
+++ b/llvm/test/CodeGen/X86/half-constrained.ll
@@ -0,0 +1,379 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-apple-darwin | FileCheck %s --check-prefix=X32-NOF16C
+; RUN: llc < %s -mtriple=i686-apple-darwin | FileCheck %s --check-prefix=X32-F16C
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s --check-prefix=X64-NOF16C
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=f16c | FileCheck %s --check-prefix=X64-F16C
+
+@a = global half 0xH0000, align 2
+@b = global half 0xH0000, align 2
+@c = global half 0xH0000, align 2
+
+define float @half_to_float() strictfp {
+; X32-NOF16C-LABEL: half_to_float:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $12, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-NOF16C-NEXT:    movzwl _a, %eax
+; X32-NOF16C-NEXT:    movl %eax, (%esp)
+; X32-NOF16C-NEXT:    calll ___extendhfsf2
+; X32-NOF16C-NEXT:    addl $12, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: half_to_float:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $12, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-F16C-NEXT:    movzwl _a, %eax
+; X32-F16C-NEXT:    movl %eax, (%esp)
+; X32-F16C-NEXT:    calll ___extendhfsf2
+; X32-F16C-NEXT:    addl $12, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: half_to_float:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    pushq %rax
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOF16C-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-NOF16C-NEXT:    callq ___extendhfsf2
+; X64-NOF16C-NEXT:    popq %rax
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: half_to_float:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    movzwl {{.*}}(%rip), %eax
+; X64-F16C-NEXT:    vmovd %eax, %xmm0
+; X64-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; X64-F16C-NEXT:    retq
+  %1 = load half, half* @a, align 2
+  %2 = tail call float @llvm.experimental.constrained.fpext.f32.f16(half %1, metadata !"fpexcept.strict") #0
+  ret float %2
+}
+
+define double @half_to_double() strictfp {
+; X32-NOF16C-LABEL: half_to_double:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $12, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-NOF16C-NEXT:    movzwl _a, %eax
+; X32-NOF16C-NEXT:    movl %eax, (%esp)
+; X32-NOF16C-NEXT:    calll ___extendhfsf2
+; X32-NOF16C-NEXT:    addl $12, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: half_to_double:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $12, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-F16C-NEXT:    movzwl _a, %eax
+; X32-F16C-NEXT:    movl %eax, (%esp)
+; X32-F16C-NEXT:    calll ___extendhfsf2
+; X32-F16C-NEXT:    addl $12, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: half_to_double:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    pushq %rax
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOF16C-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-NOF16C-NEXT:    callq ___extendhfsf2
+; X64-NOF16C-NEXT:    cvtss2sd %xmm0, %xmm0
+; X64-NOF16C-NEXT:    popq %rax
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: half_to_double:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    movzwl {{.*}}(%rip), %eax
+; X64-F16C-NEXT:    vmovd %eax, %xmm0
+; X64-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; X64-F16C-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
+; X64-F16C-NEXT:    retq
+  %1 = load half, half* @a, align 2
+  %2 = tail call double @llvm.experimental.constrained.fpext.f64.f16(half %1, metadata !"fpexcept.strict") #0
+  ret double %2
+}
+
+define x86_fp80 @half_to_fp80() strictfp {
+; X32-NOF16C-LABEL: half_to_fp80:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $12, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-NOF16C-NEXT:    movzwl _a, %eax
+; X32-NOF16C-NEXT:    movl %eax, (%esp)
+; X32-NOF16C-NEXT:    calll ___extendhfsf2
+; X32-NOF16C-NEXT:    addl $12, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: half_to_fp80:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $12, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-F16C-NEXT:    movzwl _a, %eax
+; X32-F16C-NEXT:    movl %eax, (%esp)
+; X32-F16C-NEXT:    calll ___extendhfsf2
+; X32-F16C-NEXT:    addl $12, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: half_to_fp80:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    pushq %rax
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOF16C-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-NOF16C-NEXT:    callq ___extendhfsf2
+; X64-NOF16C-NEXT:    movss %xmm0, {{[0-9]+}}(%rsp)
+; X64-NOF16C-NEXT:    flds {{[0-9]+}}(%rsp)
+; X64-NOF16C-NEXT:    wait
+; X64-NOF16C-NEXT:    popq %rax
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: half_to_fp80:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    movzwl {{.*}}(%rip), %eax
+; X64-F16C-NEXT:    vmovd %eax, %xmm0
+; X64-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; X64-F16C-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp)
+; X64-F16C-NEXT:    flds -{{[0-9]+}}(%rsp)
+; X64-F16C-NEXT:    wait
+; X64-F16C-NEXT:    retq
+  %1 = load half, half* @a, align 2
+  %2 = tail call x86_fp80 @llvm.experimental.constrained.fpext.f80.f16(half %1, metadata !"fpexcept.strict") #0
+  ret x86_fp80 %2
+}
+
+define void @float_to_half(float %0) strictfp {
+; X32-NOF16C-LABEL: float_to_half:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $12, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-NOF16C-NEXT:    flds {{[0-9]+}}(%esp)
+; X32-NOF16C-NEXT:    fstps (%esp)
+; X32-NOF16C-NEXT:    wait
+; X32-NOF16C-NEXT:    calll ___truncsfhf2
+; X32-NOF16C-NEXT:    movw %ax, _a
+; X32-NOF16C-NEXT:    addl $12, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: float_to_half:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $12, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-F16C-NEXT:    flds {{[0-9]+}}(%esp)
+; X32-F16C-NEXT:    fstps (%esp)
+; X32-F16C-NEXT:    wait
+; X32-F16C-NEXT:    calll ___truncsfhf2
+; X32-F16C-NEXT:    movw %ax, _a
+; X32-F16C-NEXT:    addl $12, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: float_to_half:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    pushq %rax
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOF16C-NEXT:    callq ___truncsfhf2
+; X64-NOF16C-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-NOF16C-NEXT:    popq %rax
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: float_to_half:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; X64-F16C-NEXT:    vpextrw $0, %xmm0, {{.*}}(%rip)
+; X64-F16C-NEXT:    retq
+  %2 = tail call half @llvm.experimental.constrained.fptrunc.f16.f32(float %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  store half %2, half* @a, align 2
+  ret void
+}
+
+define void @double_to_half(double %0) strictfp {
+; X32-NOF16C-LABEL: double_to_half:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $12, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-NOF16C-NEXT:    fldl {{[0-9]+}}(%esp)
+; X32-NOF16C-NEXT:    fstpl (%esp)
+; X32-NOF16C-NEXT:    wait
+; X32-NOF16C-NEXT:    calll ___truncdfhf2
+; X32-NOF16C-NEXT:    movw %ax, _a
+; X32-NOF16C-NEXT:    addl $12, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: double_to_half:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $12, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-F16C-NEXT:    fldl {{[0-9]+}}(%esp)
+; X32-F16C-NEXT:    fstpl (%esp)
+; X32-F16C-NEXT:    wait
+; X32-F16C-NEXT:    calll ___truncdfhf2
+; X32-F16C-NEXT:    movw %ax, _a
+; X32-F16C-NEXT:    addl $12, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: double_to_half:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    pushq %rax
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOF16C-NEXT:    callq ___truncdfhf2
+; X64-NOF16C-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-NOF16C-NEXT:    popq %rax
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: double_to_half:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    pushq %rax
+; X64-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-F16C-NEXT:    callq ___truncdfhf2
+; X64-F16C-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-F16C-NEXT:    popq %rax
+; X64-F16C-NEXT:    retq
+  %2 = tail call half @llvm.experimental.constrained.fptrunc.f16.f64(double %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  store half %2, half* @a, align 2
+  ret void
+}
+
+define void @fp80_to_half(x86_fp80 %0) strictfp {
+; X32-NOF16C-LABEL: fp80_to_half:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $28, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 32
+; X32-NOF16C-NEXT:    fldt {{[0-9]+}}(%esp)
+; X32-NOF16C-NEXT:    fstpt (%esp)
+; X32-NOF16C-NEXT:    wait
+; X32-NOF16C-NEXT:    calll ___truncxfhf2
+; X32-NOF16C-NEXT:    movw %ax, _a
+; X32-NOF16C-NEXT:    addl $28, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: fp80_to_half:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $28, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 32
+; X32-F16C-NEXT:    fldt {{[0-9]+}}(%esp)
+; X32-F16C-NEXT:    fstpt (%esp)
+; X32-F16C-NEXT:    wait
+; X32-F16C-NEXT:    calll ___truncxfhf2
+; X32-F16C-NEXT:    movw %ax, _a
+; X32-F16C-NEXT:    addl $28, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: fp80_to_half:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    subq $24, %rsp
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 32
+; X64-NOF16C-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-NOF16C-NEXT:    fstpt (%rsp)
+; X64-NOF16C-NEXT:    wait
+; X64-NOF16C-NEXT:    callq ___truncxfhf2
+; X64-NOF16C-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-NOF16C-NEXT:    addq $24, %rsp
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: fp80_to_half:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    subq $24, %rsp
+; X64-F16C-NEXT:    .cfi_def_cfa_offset 32
+; X64-F16C-NEXT:    fldt {{[0-9]+}}(%rsp)
+; X64-F16C-NEXT:    fstpt (%rsp)
+; X64-F16C-NEXT:    wait
+; X64-F16C-NEXT:    callq ___truncxfhf2
+; X64-F16C-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-F16C-NEXT:    addq $24, %rsp
+; X64-F16C-NEXT:    retq
+  %2 = tail call half @llvm.experimental.constrained.fptrunc.f16.f80(x86_fp80 %0, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  store half %2, half* @a, align 2
+  ret void
+}
+
+define void @add() strictfp {
+; X32-NOF16C-LABEL: add:
+; X32-NOF16C:       ## %bb.0:
+; X32-NOF16C-NEXT:    subl $12, %esp
+; X32-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-NOF16C-NEXT:    movzwl _a, %eax
+; X32-NOF16C-NEXT:    movl %eax, (%esp)
+; X32-NOF16C-NEXT:    calll ___extendhfsf2
+; X32-NOF16C-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Spill
+; X32-NOF16C-NEXT:    wait
+; X32-NOF16C-NEXT:    movzwl _b, %eax
+; X32-NOF16C-NEXT:    movl %eax, (%esp)
+; X32-NOF16C-NEXT:    calll ___extendhfsf2
+; X32-NOF16C-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Reload
+; X32-NOF16C-NEXT:    faddp %st, %st(1)
+; X32-NOF16C-NEXT:    fstps (%esp)
+; X32-NOF16C-NEXT:    wait
+; X32-NOF16C-NEXT:    calll ___truncsfhf2
+; X32-NOF16C-NEXT:    movw %ax, _c
+; X32-NOF16C-NEXT:    addl $12, %esp
+; X32-NOF16C-NEXT:    retl
+;
+; X32-F16C-LABEL: add:
+; X32-F16C:       ## %bb.0:
+; X32-F16C-NEXT:    subl $12, %esp
+; X32-F16C-NEXT:    .cfi_def_cfa_offset 16
+; X32-F16C-NEXT:    movzwl _a, %eax
+; X32-F16C-NEXT:    movl %eax, (%esp)
+; X32-F16C-NEXT:    calll ___extendhfsf2
+; X32-F16C-NEXT:    fstps {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Spill
+; X32-F16C-NEXT:    wait
+; X32-F16C-NEXT:    movzwl _b, %eax
+; X32-F16C-NEXT:    movl %eax, (%esp)
+; X32-F16C-NEXT:    calll ___extendhfsf2
+; X32-F16C-NEXT:    flds {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Folded Reload
+; X32-F16C-NEXT:    faddp %st, %st(1)
+; X32-F16C-NEXT:    fstps (%esp)
+; X32-F16C-NEXT:    wait
+; X32-F16C-NEXT:    calll ___truncsfhf2
+; X32-F16C-NEXT:    movw %ax, _c
+; X32-F16C-NEXT:    addl $12, %esp
+; X32-F16C-NEXT:    retl
+;
+; X64-NOF16C-LABEL: add:
+; X64-NOF16C:       ## %bb.0:
+; X64-NOF16C-NEXT:    pushq %rax
+; X64-NOF16C-NEXT:    .cfi_def_cfa_offset 16
+; X64-NOF16C-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-NOF16C-NEXT:    callq ___extendhfsf2
+; X64-NOF16C-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
+; X64-NOF16C-NEXT:    movzwl {{.*}}(%rip), %edi
+; X64-NOF16C-NEXT:    callq ___extendhfsf2
+; X64-NOF16C-NEXT:    addss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 4-byte Folded Reload
+; X64-NOF16C-NEXT:    callq ___truncsfhf2
+; X64-NOF16C-NEXT:    movw %ax, {{.*}}(%rip)
+; X64-NOF16C-NEXT:    popq %rax
+; X64-NOF16C-NEXT:    retq
+;
+; X64-F16C-LABEL: add:
+; X64-F16C:       ## %bb.0:
+; X64-F16C-NEXT:    movzwl {{.*}}(%rip), %eax
+; X64-F16C-NEXT:    vmovd %eax, %xmm0
+; X64-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; X64-F16C-NEXT:    movzwl {{.*}}(%rip), %eax
+; X64-F16C-NEXT:    vmovd %eax, %xmm1
+; X64-F16C-NEXT:    vcvtph2ps %xmm1, %xmm1
+; X64-F16C-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; X64-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-F16C-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; X64-F16C-NEXT:    vpextrw $0, %xmm0, {{.*}}(%rip)
+; X64-F16C-NEXT:    retq
+  %1 = load half, half* @a, align 2
+  %2 = tail call float @llvm.experimental.constrained.fpext.f32.f16(half %1, metadata !"fpexcept.strict") #0
+  %3 = load half, half* @b, align 2
+  %4 = tail call float @llvm.experimental.constrained.fpext.f32.f16(half %3, metadata !"fpexcept.strict") #0
+  %5 = tail call float @llvm.experimental.constrained.fadd.f32(float %2, float %4, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  %6 = tail call half @llvm.experimental.constrained.fptrunc.f16.f32(float %5, metadata !"round.tonearest", metadata !"fpexcept.strict") #0
+  store half %6, half* @c, align 2
+  ret void
+}
+
+declare float @llvm.experimental.constrained.fpext.f32.f16(half, metadata)
+declare double @llvm.experimental.constrained.fpext.f64.f16(half, metadata)
+declare x86_fp80 @llvm.experimental.constrained.fpext.f80.f16(half, metadata)
+declare float @llvm.experimental.constrained.fadd.f32(float, float, metadata, metadata)
+declare half @llvm.experimental.constrained.fptrunc.f16.f32(float, metadata, metadata)
+declare half @llvm.experimental.constrained.fptrunc.f16.f64(double, metadata, metadata)
+declare half @llvm.experimental.constrained.fptrunc.f16.f80(x86_fp80, metadata, metadata)
+
+attributes #0 = { strictfp }
+
diff --git a/llvm/test/CodeGen/X86/linux-preemption.ll b/llvm/test/CodeGen/X86/linux-preemption.ll
index 5de8587038e74252ef7a254138858c4e4a031e8a..3bf97112a840b998234cfc3393c078aa8dff2644 100644
--- a/llvm/test/CodeGen/X86/linux-preemption.ll
+++ b/llvm/test/CodeGen/X86/linux-preemption.ll
@@ -228,3 +228,16 @@ define void()* @get_external_preemptable_function() {
 
 ; COMMON:     {{^}}strong_local_global:
 ; COMMON-NEXT .Lstrong_local_global:
+
+; COMMON:      .globl strong_default_alias
+; COMMON-NEXT: .set strong_default_alias, aliasee
+; COMMON-NEXT: .weak weak_default_alias
+; COMMON-NEXT: .set weak_default_alias, aliasee
+; COMMON-NEXT: .globl strong_local_alias
+; COMMON-NEXT: .set strong_local_alias, aliasee
+; COMMON-NEXT: .weak weak_local_alias
+; COMMON-NEXT: .set weak_local_alias, aliasee
+; COMMON-NEXT: .globl strong_preemptable_alias
+; COMMON-NEXT: .set strong_preemptable_alias, aliasee
+; COMMON-NEXT: .weak weak_preemptable_alias
+; COMMON-NEXT: .set weak_preemptable_alias, aliasee
diff --git a/llvm/test/CodeGen/X86/vector-compare-all_of.ll b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
index 02c9f4f17fd4e883056e4e601fb196442d47b877..4348edf5b7c26382054413f71225a9588625f93c 100644
--- a/llvm/test/CodeGen/X86/vector-compare-all_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-all_of.ll
@@ -28,9 +28,11 @@ define i64 @test_v2f64_sext(<2 x double> %a0, <2 x double> %a1) {
 ; AVX512-LABEL: test_v2f64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vandpd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $3, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negq %rax
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <2 x double> %a0, %a1
   %s = sext <2 x i1> %c to <2 x i64>
@@ -67,11 +69,11 @@ define i64 @test_v4f64_sext(<4 x double> %a0, <4 x double> %a1) {
 ; AVX512-LABEL: test_v4f64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm0
-; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vandpd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vandpd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %ymm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $15, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negq %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <4 x double> %a0, %a1
@@ -115,12 +117,11 @@ define i64 @test_v4f64_legal_sext(<4 x double> %a0, <4 x double> %a1) {
 ; AVX512-NEXT:    vcmpltpd %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltq
+; AVX512-NEXT:    vmovmskps %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $15, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negq %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <4 x double> %a0, %a1
@@ -158,11 +159,11 @@ define i32 @test_v4f32_sext(<4 x float> %a0, <4 x float> %a1) {
 ; AVX512-LABEL: test_v4f32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $15, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <4 x float> %a0, %a1
   %s = sext <4 x i1> %c to <4 x i32>
@@ -201,13 +202,11 @@ define i32 @test_v8f32_sext(<8 x float> %a0, <8 x float> %a1) {
 ; AVX512-LABEL: test_v8f32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
-; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %ymm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $255, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <8 x float> %a0, %a1
@@ -252,14 +251,11 @@ define i32 @test_v8f32_legal_sext(<8 x float> %a0, <8 x float> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %ymm0, %ymm1, %k0
 ; AVX512-NEXT:    vpmovm2w %k0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cwtl
+; AVX512-NEXT:    vpmovmskb %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <8 x float> %a0, %a1
@@ -299,9 +295,11 @@ define i64 @test_v2i64_sext(<2 x i64> %a0, <2 x i64> %a1) {
 ; AVX512-LABEL: test_v2i64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $3, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negq %rax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <2 x i64> %a0, %a1
   %s = sext <2 x i1> %c to <2 x i64>
@@ -353,11 +351,11 @@ define i64 @test_v4i64_sext(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX512-LABEL: test_v4i64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %ymm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $15, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negq %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <4 x i64> %a0, %a1
@@ -416,12 +414,11 @@ define i64 @test_v4i64_legal_sext(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltq
+; AVX512-NEXT:    vmovmskps %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $15, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negq %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <4 x i64> %a0, %a1
@@ -459,11 +456,11 @@ define i32 @test_v4i32_sext(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX512-LABEL: test_v4i32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $15, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <4 x i32> %a0, %a1
   %s = sext <4 x i1> %c to <4 x i32>
@@ -517,13 +514,11 @@ define i32 @test_v8i32_sext(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX512-LABEL: test_v8i32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %ymm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $255, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <8 x i32> %a0, %a1
@@ -583,14 +578,11 @@ define i32 @test_v8i32_legal_sext(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k0
 ; AVX512-NEXT:    vpmovm2w %k0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cwtl
+; AVX512-NEXT:    vpmovmskb %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <8 x i32> %a0, %a1
@@ -632,13 +624,11 @@ define i16 @test_v8i16_sext(<8 x i16> %a0, <8 x i16> %a1) {
 ; AVX512-LABEL: test_v8i16_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $65535, %ecx # imm = 0xFFFF
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <8 x i16> %a0, %a1
@@ -698,15 +688,11 @@ define i16 @test_v16i16_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX512-LABEL: test_v16i16_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %ymm0, %ecx
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    cmpl $-1, %ecx
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negl %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -772,15 +758,10 @@ define i16 @test_v16i16_legal_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %ymm1, %ymm0, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negb %al
 ; AVX512-NEXT:    movsbl %al, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
@@ -822,16 +803,10 @@ define i8 @test_v16i8_sext(<16 x i8> %a0, <16 x i8> %a1) {
 ; AVX512-LABEL: test_v16i8_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negb %al
 ; AVX512-NEXT:    retq
   %c = icmp sgt <16 x i8> %a0, %a1
   %s = sext <16 x i1> %c to <16 x i8>
@@ -886,18 +861,10 @@ define i8 @test_v32i8_sext(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX512-LABEL: test_v32i8_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    vpmovmskb %ymm0, %eax
+; AVX512-NEXT:    cmpl $-1, %eax
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    negb %al
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c  = icmp sgt <32 x i8> %a0, %a1
diff --git a/llvm/test/CodeGen/X86/vector-compare-any_of.ll b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
index 084de61dd08654d4bf97701a3de4dbdf6e72f900..b3443b9707b7fcdf8b5c29463bc1881700c84bd5 100644
--- a/llvm/test/CodeGen/X86/vector-compare-any_of.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-any_of.ll
@@ -24,9 +24,9 @@ define i64 @test_v2f64_sext(<2 x double> %a0, <2 x double> %a1) {
 ; AVX512-LABEL: test_v2f64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vorpd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbq %rax, %rax
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <2 x double> %a0, %a1
   %s = sext <2 x i1> %c to <2 x i64>
@@ -59,11 +59,9 @@ define i64 @test_v4f64_sext(<4 x double> %a0, <4 x double> %a1) {
 ; AVX512-LABEL: test_v4f64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm0
-; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vorpd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vorpd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %ymm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbq %rax, %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <4 x double> %a0, %a1
@@ -103,12 +101,9 @@ define i64 @test_v4f64_legal_sext(<4 x double> %a0, <4 x double> %a1) {
 ; AVX512-NEXT:    vcmpltpd %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltq
+; AVX512-NEXT:    vmovmskps %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbq %rax, %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <4 x double> %a0, %a1
@@ -142,11 +137,9 @@ define i32 @test_v4f32_sext(<4 x float> %a0, <4 x float> %a1) {
 ; AVX512-LABEL: test_v4f32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <4 x float> %a0, %a1
   %s = sext <4 x i1> %c to <4 x i32>
@@ -181,13 +174,9 @@ define i32 @test_v8f32_sext(<8 x float> %a0, <8 x float> %a1) {
 ; AVX512-LABEL: test_v8f32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
-; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %ymm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <8 x float> %a0, %a1
@@ -228,14 +217,9 @@ define i32 @test_v8f32_legal_sext(<8 x float> %a0, <8 x float> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %ymm0, %ymm1, %k0
 ; AVX512-NEXT:    vpmovm2w %k0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cwtl
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = fcmp ogt <8 x float> %a0, %a1
@@ -271,9 +255,9 @@ define i64 @test_v2i64_sext(<2 x i64> %a0, <2 x i64> %a1) {
 ; AVX512-LABEL: test_v2i64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbq %rax, %rax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <2 x i64> %a0, %a1
   %s = sext <2 x i1> %c to <2 x i64>
@@ -319,11 +303,9 @@ define i64 @test_v4i64_sext(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX512-LABEL: test_v4i64_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rax
+; AVX512-NEXT:    vmovmskpd %ymm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbq %rax, %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <4 x i64> %a0, %a1
@@ -376,12 +358,9 @@ define i64 @test_v4i64_legal_sext(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cltq
+; AVX512-NEXT:    vmovmskps %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbq %rax, %rax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <4 x i64> %a0, %a1
@@ -415,11 +394,9 @@ define i32 @test_v4i32_sext(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX512-LABEL: test_v4i32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <4 x i32> %a0, %a1
   %s = sext <4 x i1> %c to <4 x i32>
@@ -467,13 +444,9 @@ define i32 @test_v8i32_sext(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX512-LABEL: test_v8i32_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vmovmskps %ymm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <8 x i32> %a0, %a1
@@ -527,14 +500,9 @@ define i32 @test_v8i32_legal_sext(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k0
 ; AVX512-NEXT:    vpmovm2w %k0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    cwtl
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %c = icmp sgt <8 x i32> %a0, %a1
@@ -572,13 +540,9 @@ define i16 @test_v8i16_sext(<8 x i16> %a0, <8 x i16> %a1) {
 ; AVX512-LABEL: test_v8i16_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <8 x i16> %a0, %a1
@@ -632,15 +596,9 @@ define i16 @test_v16i16_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX512-LABEL: test_v16i16_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %ymm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -700,16 +658,9 @@ define i16 @test_v16i16_legal_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %ymm1, %ymm0, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512-NEXT:    movsbl %al, %eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -750,15 +701,9 @@ define i8 @test_v16i8_sext(<16 x i8> %a0, <16 x i8> %a1) {
 ; AVX512-LABEL: test_v16i8_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %c = icmp sgt <16 x i8> %a0, %a1
@@ -814,17 +759,9 @@ define i8 @test_v32i8_sext(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX512-LABEL: test_v32i8_sext:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    vpmovmskb %ymm0, %eax
+; AVX512-NEXT:    negl %eax
+; AVX512-NEXT:    sbbl %eax, %eax
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/llvm/test/DebugInfo/AArch64/call-site-info-output.ll b/llvm/test/DebugInfo/AArch64/call-site-info-output.ll
index d52d6962f3c41517761ededd429052810a1c4826..6817aabd602be03e927d03809adfcb5084a65786 100644
--- a/llvm/test/DebugInfo/AArch64/call-site-info-output.ll
+++ b/llvm/test/DebugInfo/AArch64/call-site-info-output.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple aarch64-linux-gnu -debug-entry-values %s -o - -stop-before=finalize-isel | FileCheck %s
+; RUN: llc -mtriple aarch64-linux-gnu %s -o - -stop-before=finalize-isel | FileCheck %s
 ; Verify that Selection DAG knows how to recognize simple function parameter forwarding registers.
 ; Produced from:
 ; extern int fn1(int,int,int);
diff --git a/llvm/test/DebugInfo/ARM/call-site-info-output.ll b/llvm/test/DebugInfo/ARM/call-site-info-output.ll
index 9255a7d57dde98ee16b1f19b2be96ce7238cabd5..e1cd900aefb1721f0d3ea7d7464f1977ec57b73d 100644
--- a/llvm/test/DebugInfo/ARM/call-site-info-output.ll
+++ b/llvm/test/DebugInfo/ARM/call-site-info-output.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple arm-linux-gnu -debug-entry-values %s -o - -stop-before=finalize-isel | FileCheck %s
+; RUN: llc -mtriple arm-linux-gnu %s -o - -stop-before=finalize-isel | FileCheck %s
 ; Verify that Selection DAG knows how to recognize simple function parameter forwarding registers.
 ; Produced from:
 ; extern int fn1(int,int,int);
diff --git a/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll b/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll
index 71cf417827201d06b21ca60f8e4fb044bebc89da..4c5a38161c70f6eb46392f50442e97cd63eb3c42 100644
--- a/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll
+++ b/llvm/test/DebugInfo/ARM/entry-value-multi-byte-expr.ll
@@ -1,4 +1,4 @@
-; RUN: llc -debug-entry-values -filetype=asm -o - %s | FileCheck %s
+; RUN: llc -filetype=asm -o - %s | FileCheck %s
 
 ; Verify that the size operands of the DW_OP_GNU_entry_value operations are
 ; correct for the multi-byte DW_OP_regx expressions.
diff --git a/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpret-movzxi.mir b/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpret-movzxi.mir
index dc7561ca64008cb787f03b2c50052b95d033f41a..8fdfe37d7db5fcedbf428f96e9663a864b201587 100644
--- a/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpret-movzxi.mir
+++ b/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpret-movzxi.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple aarch64-linux-gnu -debug-entry-values -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
+# RUN: llc -mtriple aarch64-linux-gnu -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
 #
 # Based on the following C reproducer:
 #
diff --git a/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpretation.mir b/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpretation.mir
index 0371ccef603e8fcf146ac11a6d203310accc3d54..559a11d3d53671f6e2f069dc586e3665c0a08d63 100644
--- a/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpretation.mir
+++ b/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-interpretation.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple aarch64-linux-gnu -debug-entry-values -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
+# RUN: llc -mtriple aarch64-linux-gnu -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
 # Following code is used for producing this test case. Note that
 # some of argument loading instruction are modified in order to
 # cover certain cases.
diff --git a/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-orr-moves.mir b/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-orr-moves.mir
index 9d4feaccf9e735bef78e6e472d8b83517fafa6cb..72540049612f53533e3032b738cad3e191162368 100644
--- a/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-orr-moves.mir
+++ b/llvm/test/DebugInfo/MIR/AArch64/dbgcall-site-orr-moves.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-after=livedebugvalues -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+# RUN: llc -start-after=livedebugvalues -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
 
 # Based on the following C reproducer:
 #
diff --git a/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-interpretation.mir b/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-interpretation.mir
index ce8dc97f0e72c9853426338ba3d48fefb499d262..507250d0e5a13ce27c3c8d5042c16aa4b1016442 100644
--- a/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-interpretation.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-interpretation.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=arm-linux-gnueabi -debug-entry-values -filetype=obj -start-after=machineverifier %s -o -| llvm-dwarfdump -| FileCheck %s
+# RUN: llc -mtriple=arm-linux-gnueabi -filetype=obj -start-after=machineverifier %s -o -| llvm-dwarfdump -| FileCheck %s
 # Following code is used for producing this test case. Note that
 # some of argument loading instruction are modified in order to
 # cover certain cases.
diff --git a/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-propagated-value.mir b/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-propagated-value.mir
index 9001c8ba8eea2a4ec771c163ae60f14ed93d976d..13012a7dc106f021dfc18be98bd2dcd3e06c0254 100644
--- a/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-propagated-value.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/dbgcall-site-propagated-value.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -o - %s | FileCheck %s
 
 # Based on the following C reproducer:
 #
@@ -20,7 +20,7 @@
 #
 # Generated using:
 #
-# clang -Os -fno-inline -Xclang -femit-debug-entry-values -g --target=armeb.
+# clang -Os -fno-inline -g --target=armeb.
 
 --- |
   target datalayout = "E-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/llvm/test/DebugInfo/MIR/ARM/if-coverter-call-site-info.mir b/llvm/test/DebugInfo/MIR/ARM/if-coverter-call-site-info.mir
index aa7b54c1e5bbe73524dcf1eb7b87fa92bd111df5..b2b9580cdc7bdb4162e6c0a6c391ec15ebcfcfb1 100644
--- a/llvm/test/DebugInfo/MIR/ARM/if-coverter-call-site-info.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/if-coverter-call-site-info.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=arm-linux-gnu -debug-entry-values -run-pass if-converter %s -o -| FileCheck %s
+# RUN: llc -mtriple=arm-linux-gnu -run-pass if-converter %s -o -| FileCheck %s
 
 # Vefify that the call site info will be updated after the optimization.
 # This test case would previously trigger an assertion when
@@ -19,8 +19,7 @@
 #
 # With slight change of MIR - substitution of BL instruction with BL_pred 
 # in order to trigger optimization.
-# clang -target arm-linux-gnu -g -O2 -Xclang -femit-debug-entry-values 
-#       %s -stop-before=if-convert
+# clang -target arm-linux-gnu -g -O2 %s -stop-before=if-convert
 #
 # CHECK: callSites:
 # CHECK-NEXT: - { bb: {{.*}}, offset: {{.*}}, fwdArgRegs:
diff --git a/llvm/test/DebugInfo/MIR/Hexagon/dbgcall-site-instr-before-bundled-call.mir b/llvm/test/DebugInfo/MIR/Hexagon/dbgcall-site-instr-before-bundled-call.mir
index 8ae628af2c099444724f1281200f1534b97a8434..a86cd7f7a440e5c604f147398a5e738e9b2c0396 100644
--- a/llvm/test/DebugInfo/MIR/Hexagon/dbgcall-site-instr-before-bundled-call.mir
+++ b/llvm/test/DebugInfo/MIR/Hexagon/dbgcall-site-instr-before-bundled-call.mir
@@ -1,4 +1,6 @@
-# RUN: llc -mtriple hexagon -debug-entry-values -start-after=machineverifier -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
+# We do not support the call site info for the target now, so we use the experimental option (-debug-entry-values).
+
+# RUN: llc -debug-entry-values -mtriple hexagon -start-after=machineverifier -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
 
 # Based on the following C reproducer:
 #
diff --git a/llvm/test/DebugInfo/MIR/Hexagon/live-debug-values-bundled-entry-values.mir b/llvm/test/DebugInfo/MIR/Hexagon/live-debug-values-bundled-entry-values.mir
index ff0a539dd15dc4d00e2c31d8ea724428aff72cef..956db574c59d161f732050708e0f16a4d14d70d8 100644
--- a/llvm/test/DebugInfo/MIR/Hexagon/live-debug-values-bundled-entry-values.mir
+++ b/llvm/test/DebugInfo/MIR/Hexagon/live-debug-values-bundled-entry-values.mir
@@ -1,3 +1,5 @@
+# We do not support the call site info for the target now, so we use the experimental option (-debug-entry-values).
+
 # RUN: llc -debug-entry-values -run-pass=livedebugvalues -o - %s | FileCheck %s
 
 # Verify that the entry values for the input parameters are inserted after the
diff --git a/llvm/test/DebugInfo/MIR/SystemZ/call-site-lzer.mir b/llvm/test/DebugInfo/MIR/SystemZ/call-site-lzer.mir
index 8a4e8b5632c26feaec4f03ba2ce4c0dda36d1d54..4f06b1a639e5535b93e2e82455303d4faf245529 100644
--- a/llvm/test/DebugInfo/MIR/SystemZ/call-site-lzer.mir
+++ b/llvm/test/DebugInfo/MIR/SystemZ/call-site-lzer.mir
@@ -1,3 +1,5 @@
+# We do not support the call site info for the target now, so we use the experimental option (-debug-entry-values).
+
 # RUN: llc -debug-entry-values -start-after=livedebugvalues -o - %s | FileCheck %s
 
 # This test would previously trigger an assertion when trying to describe the
diff --git a/llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir b/llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir
index e6fe5d2de878177b06ba9630aafe830d9edde8e7..f7f74b628d1660bb0f1c86334ea7fe86e2acd0df 100644
--- a/llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir
+++ b/llvm/test/DebugInfo/MIR/X86/DW_OP_entry_value.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-before=livedebugvalues -mtriple=x86_64-apple-darwin -o %t %s -filetype=obj
+# RUN: llc -start-before=livedebugvalues -mtriple=x86_64-apple-darwin -o %t %s -filetype=obj
 # RUN: llvm-dwarfdump %t | FileCheck %s
 #
 # int global;
diff --git a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
index 1c5922ce76f0d67d838908e04f270105dd44af02..0718878e0faa4d19c3945c1495478d07568a7a6e 100644
--- a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
@@ -1,14 +1,18 @@
 # Test the call site encoding in DWARF5 vs GNU extensions.
 #
-# RUN: llc -dwarf-version 4 -debugger-tune=gdb -debug-entry-values -filetype=obj \
+# RUN: llc -dwarf-version 4 -debugger-tune=gdb -filetype=obj \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s  \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-GNU
 #
-# RUN: llc -dwarf-version 5 -debugger-tune=lldb -debug-entry-values -filetype=obj \
+# RUN: llc -dwarf-version 5 -debugger-tune=lldb -filetype=obj \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s   \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
 #
-# RUN: llc -dwarf-version 5 -debug-entry-values -filetype=obj \
+# RUN: llc -dwarf-version 5 -filetype=obj \
+# RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s   \
+# RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
+#
+# RUN: llc -dwarf-version 5 -filetype=obj -debugger-tune=sce -emit-debug-entry-values \
 # RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s   \
 # RUN:     | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
 #
diff --git a/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir b/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir
index c32a1155d0389150b4fa1bc29e63ec7209cfd338..0d161fe26c0633437c5020c51ec37fd32f2be919 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir
@@ -1,9 +1,9 @@
 # Check that llvm can describe a call site parameter which resides in a spill slot.
 #
-# RUN: llc -debug-entry-values -start-after=machineverifier -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
+# RUN: llc -start-after=machineverifier -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
 # 
 # Command:
-# $ ~/src/builds/llvm-project-master-RA/bin/clang -g -Xclang -femit-debug-entry-values -O2 -c -o spill.o spill.cc -mllvm -stop-before=machineverifier -o spill.mir
+# $ ~/src/builds/llvm-project-master-RA/bin/clang -g -O2 -c -o spill.o spill.cc -mllvm -stop-before=machineverifier -o spill.mir
 #
 # Source:
 ## extern void callee(int);
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-copy-super-sub.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-copy-super-sub.mir
index a2d51a20351215cfca050401c2c700e7d809d9f5..271a9c101e77f0658daf0cf4bae27340efb7a0b5 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-copy-super-sub.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-copy-super-sub.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-after=livedebugvalues -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
+# RUN: llc -start-after=livedebugvalues -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
 
 # Based on the following reproducer:
 #
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir
index dc3a425e569036c6d650f35280ef20fe5d267d7a..5204bae67aed60c386fb84e1851ed7b3e48df54e 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
+# RUN: llc -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
 #
 # CHECK:        DW_TAG_GNU_call_site
 # CHECK-NEXT:     DW_AT_abstract_origin {{.*}} "foo"
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-lea-interpretation.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-lea-interpretation.mir
index 1bb70f6d453087050a60a48181643e8f47178ada..4530ab6eae991a1e40fbc124a3dde0771c2d1623 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-lea-interpretation.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-lea-interpretation.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
+# RUN: llc -start-after=machineverifier -filetype=obj %s -o -| llvm-dwarfdump -| FileCheck %s
 # CHECK:        DW_TAG_GNU_call_site
 # CHECK-NEXT:     DW_AT_abstract_origin {{.*}} "foo")
 # CHECK-NEXT:     DW_AT_low_pc {{.*}}
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe.mir
index 377409518432c266ada95a4ddca4c0f74d2ad6ce..580eefb9234f920bf72f7c66c63f76575edba5bd 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-partial-describe.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-before=livedebugvalues -filetype=obj -o - %s \
+# RUN: llc -start-before=livedebugvalues -filetype=obj -o - %s \
 # RUN:     | llvm-dwarfdump - | FileCheck %s --implicit-check-not=DW_TAG_GNU_call_site_parameter
 
 --- |
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reference.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reference.mir
index 235787573f51c90fdd7e8962955df8fa9a510ec3..73927772ca085fa54f012dbfadc6499a0249c2ab 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reference.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reference.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-before=livedebugvalues -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+# RUN: llc -start-before=livedebugvalues -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
 
 # Based on the following C++ code:
 # struct A { A(A &) {} };
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reg-shuffle.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reg-shuffle.mir
index 5b934f6dc5337c6e6587e6b15444e91f247a14e9..c9bed340bbcdbe8b58172b1fbf24cc6088b339cf 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reg-shuffle.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-reg-shuffle.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -start-before=livedebugvalues -filetype=obj -o - %s \
+# RUN: llc -start-before=livedebugvalues -filetype=obj -o - %s \
 # RUN:     | llvm-dwarfdump - | FileCheck %s --implicit-check-not=DW_TAG_GNU_call_site_parameter
 
 --- |
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-two-fwd-reg-defs.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-two-fwd-reg-defs.mir
index 4e1b8fdb5bef694072ca2fe3a3e24ccf18bde5f9..77b26c398ba7b4a813caab6fc3f4208c92ef6d6d 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-two-fwd-reg-defs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-two-fwd-reg-defs.mir
@@ -1,4 +1,4 @@
-# RUN: llc -O1 -debug-entry-values -start-after=livedebugvalues -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
+# RUN: llc -O1 -start-after=livedebugvalues -filetype=obj %s -o - | llvm-dwarfdump - | FileCheck %s
 
 # Based on the following C reproducer:
 #
diff --git a/llvm/test/DebugInfo/MIR/X86/dbginfo-entryvals.mir b/llvm/test/DebugInfo/MIR/X86/dbginfo-entryvals.mir
index 5d203029936e5b1b5b3cc26d019d9154bdfeaf2d..6c6b4ddb46d7d51295913e514f0fa08264fabf83 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbginfo-entryvals.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbginfo-entryvals.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
 #
 #extern void fn2(int);
 #
diff --git a/llvm/test/DebugInfo/MIR/X86/debug-call-site-param.mir b/llvm/test/DebugInfo/MIR/X86/debug-call-site-param.mir
index caee15c2a9e646d2514a0d4fbe22f2f7584e3050..dfbca43e571f9fe85fbfc10196e9b9cb2ca16909 100644
--- a/llvm/test/DebugInfo/MIR/X86/debug-call-site-param.mir
+++ b/llvm/test/DebugInfo/MIR/X86/debug-call-site-param.mir
@@ -2,8 +2,8 @@
 # When the debugger tuning is set to gdb, use GNU opcodes.
 # For lldb, use the standard DWARF5 opcodes.
 
-# RUN: llc -debug-entry-values -debugger-tune=gdb -filetype=obj -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-GNU
-# RUN: llc -debug-entry-values -debugger-tune=lldb -filetype=obj -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
+# RUN: llc -debugger-tune=gdb -filetype=obj -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-GNU
+# RUN: llc -debugger-tune=lldb -filetype=obj -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
 #
 # extern void foo(int *a, int b, int c, int d, int e, int f);
 # extern int getVal();
diff --git a/llvm/test/DebugInfo/MIR/X86/entry-value-of-modified-param.mir b/llvm/test/DebugInfo/MIR/X86/entry-value-of-modified-param.mir
index adc43d96a961bf5acb0a05872137bc4175e48652..b37f6557ae68e0ff25d924e2e639567b319e04da 100644
--- a/llvm/test/DebugInfo/MIR/X86/entry-value-of-modified-param.mir
+++ b/llvm/test/DebugInfo/MIR/X86/entry-value-of-modified-param.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
 #
 #extern void fn1 (int, int, int);
 #
diff --git a/llvm/test/DebugInfo/MIR/X86/entry-values-diamond-bbs.mir b/llvm/test/DebugInfo/MIR/X86/entry-values-diamond-bbs.mir
index 2396daada876e883e44f6e13848afef8d815c537..f2f02e80952237459e91c549007d1151d4c534f1 100644
--- a/llvm/test/DebugInfo/MIR/X86/entry-values-diamond-bbs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/entry-values-diamond-bbs.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
 #
 # The test case was artificially adjusted, in order to make proper diamond basic
 # block structure relevant to the debug entry values propagation.
diff --git a/llvm/test/DebugInfo/MIR/X86/kill-entry-value-after-diamond-bbs.mir b/llvm/test/DebugInfo/MIR/X86/kill-entry-value-after-diamond-bbs.mir
index 0109dc47ef36d8ddb580eaa5b34c4260dfd3687c..7152080923c3961b1192443028c85f4a0e8514c4 100644
--- a/llvm/test/DebugInfo/MIR/X86/kill-entry-value-after-diamond-bbs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/kill-entry-value-after-diamond-bbs.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
 #
 # The test case was artificially adjusted, in order to make proper diamond basic
 # block structure relevant to the debug entry values clobbering.
diff --git a/llvm/test/DebugInfo/MIR/X86/multiple-param-dbg-value-entry.mir b/llvm/test/DebugInfo/MIR/X86/multiple-param-dbg-value-entry.mir
index fd154ed6a731b292f9fb6cab4d4e8e418d0b7dc3..be3fc00a1b78f477b22571d9b6b347938ef14658 100644
--- a/llvm/test/DebugInfo/MIR/X86/multiple-param-dbg-value-entry.mir
+++ b/llvm/test/DebugInfo/MIR/X86/multiple-param-dbg-value-entry.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -march=x86-64 -o - %s| FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -march=x86-64 -o - %s| FileCheck %s
 #
 #int global;
 #int foo(int p, int q, int r) {
diff --git a/llvm/test/DebugInfo/MIR/X86/propagate-entry-value-cross-bbs.mir b/llvm/test/DebugInfo/MIR/X86/propagate-entry-value-cross-bbs.mir
index 86b1cddaa462b44bad8f2e884335b38b9efeb27e..e24f90ff27062fae9ae2559e3a512f61c4e9f24a 100644
--- a/llvm/test/DebugInfo/MIR/X86/propagate-entry-value-cross-bbs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/propagate-entry-value-cross-bbs.mir
@@ -1,4 +1,4 @@
-# RUN: llc -debug-entry-values -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
+# RUN: llc -run-pass=livedebugvalues -march=x86-64 -o - %s | FileCheck %s
 #
 #extern void fn1 (int, int, int);
 #__attribute__((noinline))
diff --git a/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir b/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir
index d282d796f6d721ead229bcf6fa29ed9840fd6f76..66d799162e044ddd5ac5b672cb95b1236b1007c6 100644
--- a/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir
+++ b/llvm/test/DebugInfo/MIR/X86/unreachable-block-call-site.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=x86_64-pc-linux -debug-entry-values -run-pass=unreachable-mbb-elimination -o - %s | FileCheck %s
+# RUN: llc -mtriple=x86_64-pc-linux -run-pass=unreachable-mbb-elimination -o - %s | FileCheck %s
 
 # Verify that the call site information for the call residing in the eliminated
 # block is removed. This test case would previously trigger an assertion when
diff --git a/llvm/test/DebugInfo/RISCV/saverestore.ll b/llvm/test/DebugInfo/RISCV/saverestore.ll
new file mode 100644
index 0000000000000000000000000000000000000000..77ece5e55ac59fe53c794aaeff102a48094a57c5
--- /dev/null
+++ b/llvm/test/DebugInfo/RISCV/saverestore.ll
@@ -0,0 +1,28 @@
+; RUN: llc -mtriple=riscv32 -mattr=+save-restore < %s
+
+; Ensure that the addition of framesetup instructions which call save/restore
+; libcalls do not cause a crash when DIFlagAllCallsDescribed is set.
+
+define i32 @main() noreturn nounwind !dbg !7 {
+entry:
+  tail call void @exit(i32 0)
+  unreachable
+}
+
+declare void @exit(i32) noreturn
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "saverestore.c", directory: ".")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
diff --git a/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll b/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
index 0af5619b75d78a6dbc473e1a98fa0dd8ca58428a..cfe8a3d0da0484cdb1dd44aea54c280ceb376cd7 100644
--- a/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
+++ b/llvm/test/DebugInfo/Sparc/entry-value-complex-reg-expr.ll
@@ -1,3 +1,5 @@
+; We do not support the call site info for the target now, so we use the experimental option (-debug-entry-values).
+
 ; RUN: llc -debug-entry-values -filetype=asm -o - %s | FileCheck %s
 
 ; Verify that the entry value covers both of the DW_OP_regx pieces. Previously
diff --git a/llvm/test/DebugInfo/X86/dbg-value-range.ll b/llvm/test/DebugInfo/X86/dbg-value-range.ll
index e0cfe5f15ee95952588d629d29b4ccbae4ab81e1..9159d2aac780c26808c465339c212a317c302628 100644
--- a/llvm/test/DebugInfo/X86/dbg-value-range.ll
+++ b/llvm/test/DebugInfo/X86/dbg-value-range.ll
@@ -56,6 +56,6 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
 ;CHECK-NEXT:	.quad	[[CLOBBER_OFF]]
 ;CHECK-NEXT:  .short 1 ## Loc expr size
 ;CHECK-NEXT:	.byte	85 ## DW_OP_reg
-;CHECK-NEXT:	.quad	0
+;CHECK:        .quad  0
 ;CHECK-NEXT:	.quad	0
 !24 = !{i32 1, !"Debug Info Version", i32 3}
diff --git a/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll b/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll
index 440498a9d8dd454e408ee081d5811a66ebdbe224..425a6cb38c4105e10cd982b6759aefbb1ae6cd5a 100644
--- a/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll
+++ b/llvm/test/DebugInfo/X86/dbg-value-regmask-clobber.ll
@@ -9,8 +9,7 @@
 ; ASM: movl $1, x(%rip)
 ; ASM: callq clobber
 ; ASM-NEXT: [[argc_range_end:.Ltmp[0-9]+]]:
-; Previously LiveDebugValues would claim argc was still in ecx after the call.
-; ASM-NOT: #DEBUG_VALUE: main:argc
+; ASM: #DEBUG_VALUE: main:argc <- [DW_OP_LLVM_entry_value 1] $ecx
 
 ; argc is the first debug location.
 ; ASM: .Ldebug_loc1:
@@ -23,7 +22,8 @@
 ; DWARF: .debug_info contents:
 ; DWARF:  DW_TAG_formal_parameter
 ; DWARF-NEXT:    DW_AT_location ({{0x.*}}
-; DWARF-NEXT:      [0x0000000000000000, 0x0000000000000013): DW_OP_reg2 RCX)
+; DWARF-NEXT:    [0x0000000000000000, 0x0000000000000013): DW_OP_reg2 RCX
+; DWARF-NEXT:    [0x0000000000000013, 0x0000000000000043): DW_OP_GNU_entry_value(DW_OP_reg2 RCX), DW_OP_stack_value
 ; DWARF-NEXT:    DW_AT_name ("argc")
 
 ; ModuleID = 't.cpp'
diff --git a/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll b/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll
index b698f1cdbfe853b3a9972cacff948c0356abc46c..c25d614f9d55630ea68a624756ab09bfe1a0d1c6 100644
--- a/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll
+++ b/llvm/test/DebugInfo/X86/dbgcall-site-64-bit-imms.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O1 -debug-entry-values -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+; RUN: llc -O1 -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
 
 ; Verify that the 64-bit call site immediates are not truncated.
 ;
diff --git a/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll b/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll
index 9fe67f82a2b459e6e2f159907b58f9547e5e0dab..1900a6ae0023a19ea66529fb39318ff05986d6c7 100644
--- a/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll
+++ b/llvm/test/DebugInfo/X86/dbgcall-site-zero-valued-imms.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O3 -debug-entry-values -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
+; RUN: llc -O3 -filetype=obj -o - %s | llvm-dwarfdump - | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/DebugInfo/X86/debug_addr.ll b/llvm/test/DebugInfo/X86/debug_addr.ll
index 1fa797efd5fbb0f0bea8426a1257f95dd6c403cb..a9efba0d1bc83e3a18cb01babe270b7c62601518 100644
--- a/llvm/test/DebugInfo/X86/debug_addr.ll
+++ b/llvm/test/DebugInfo/X86/debug_addr.ll
@@ -22,7 +22,6 @@
 ; DWARF4: DW_TAG_GNU_call_site
 ; DWARF4:   DW_AT_low_pc [DW_FORM_GNU_addr_index] (indexed (00000002) address = 0x0000000000000018 ".text")
 ; DWARF4: .debug_addr contents:
-; DWARF4-NEXT: 0x00000000: Addr Section: length = 0x00000000, version = 0x0004, addr_size = 0x04, seg_size = 0x00
 ; DWARF4-NEXT: Addrs: [
 ; DWARF4-NEXT: 0x00000000
 ; DWARF4-NEXT: 0x00000010
@@ -39,7 +38,7 @@
 ; DWARF5: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x0000000000000000 ".text")
 ; DWARF5: DW_AT_call_return_pc [DW_FORM_addrx] (indexed (00000002) address = 0x0000000000000018 ".text")
 ; DWARF5: .debug_addr contents:
-; DWARF5-NEXT: 0x00000000: Addr Section: length = 0x00000010, version = 0x0005, addr_size = 0x04, seg_size = 0x00
+; DWARF5-NEXT: 0x00000000: Address table header: length = 0x00000010, version = 0x0005, addr_size = 0x04, seg_size = 0x00
 ; DWARF5-NEXT: Addrs: [
 ; DWARF5-NEXT: 0x00000000
 ; DWARF5-NEXT: 0x00000010
diff --git a/llvm/test/DebugInfo/X86/loclists-dwp.ll b/llvm/test/DebugInfo/X86/loclists-dwp.ll
index 91f83887633865768ecb16a2fc8739fb6a1664fb..a972c8094c5f1586b811c1504d603ded670969b6 100644
--- a/llvm/test/DebugInfo/X86/loclists-dwp.ll
+++ b/llvm/test/DebugInfo/X86/loclists-dwp.ll
@@ -19,10 +19,12 @@
 ; void b(int i) { asm("" : : : "rdi"); }
 
 ; CHECK:      DW_AT_location [DW_FORM_sec_offset]   (0x00000000
-; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000000, 0x0000000000000006): DW_OP_reg5 RDI)
+; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000000, 0x0000000000000006): DW_OP_reg5 RDI
+; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000001, 0x0000000000000002): DW_OP_GNU_entry_value(DW_OP_reg5 RDI), DW_OP_stack_value)
 
 ; CHECK:      DW_AT_location [DW_FORM_sec_offset]   (0x00000000
-; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000000, 0x0000000000000000): DW_OP_reg5 RDI)
+; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000000, 0x0000000000000000): DW_OP_reg5 RDI
+; CHECK-NEXT: DW_LLE_startx_length (0x0000000000000001, 0x0000000000000001): DW_OP_GNU_entry_value(DW_OP_reg5 RDI), DW_OP_stack_value)
 
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/DebugInfo/X86/no-entry-values-with-O0.ll b/llvm/test/DebugInfo/X86/no-entry-values-with-O0.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8ba22b7b6e5103798f49a77f6498be7217b48fbd
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/no-entry-values-with-O0.ll
@@ -0,0 +1,88 @@
+; RUN: llc -O0  -dwarf-version=5 -debugger-tune=lldb -march=x86-64 -filetype=obj < %s \
+; RUN:  | llvm-dwarfdump - | FileCheck --implicit-check-not=DW_OP_entry_value %s
+; RUN: llc -O0  -dwarf-version=5 -debugger-tune=gdb -march=x86-64 -filetype=obj < %s \
+; RUN:  | llvm-dwarfdump - | FileCheck --implicit-check-not=DW_OP_entry_value %s
+
+; The call-site-params are created iff corresponding DISubprogram contains
+; the AllCallsDescribed DIFlag.
+; CHECK-NOT: DW_TAG_call_site_param
+
+; Genarated with:
+; clang -gdwarf-5 -O0 test.c -S -emit-llvm
+;
+; ModuleID = 'test.c'
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local void @fn1(i32 %x, i32 %y) !dbg !7 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  %u = alloca i32, align 4
+  %a = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %x.addr, metadata !11, metadata !DIExpression()), !dbg !12
+  store i32 %y, i32* %y.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %y.addr, metadata !13, metadata !DIExpression()), !dbg !14
+  call void @llvm.dbg.declare(metadata i32* %u, metadata !15, metadata !DIExpression()), !dbg !16
+  %0 = load i32, i32* %x.addr, align 4, !dbg !16
+  %1 = load i32, i32* %y.addr, align 4, !dbg !16
+  %add = add nsw i32 %0, %1, !dbg !16
+  store i32 %add, i32* %u, align 4, !dbg !16
+  %2 = load i32, i32* %x.addr, align 4, !dbg !17
+  %cmp = icmp sgt i32 %2, 1, !dbg !17
+  br i1 %cmp, label %if.then, label %if.else, !dbg !16
+
+if.then:                                          ; preds = %entry
+  %3 = load i32, i32* %u, align 4, !dbg !17
+  %add1 = add nsw i32 %3, 1, !dbg !17
+  store i32 %add1, i32* %u, align 4, !dbg !17
+  br label %if.end, !dbg !17
+
+if.else:                                          ; preds = %entry
+  %4 = load i32, i32* %u, align 4, !dbg !17
+  %add2 = add nsw i32 %4, 2, !dbg !17
+  store i32 %add2, i32* %u, align 4, !dbg !17
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  call void @llvm.dbg.declare(metadata i32* %a, metadata !19, metadata !DIExpression()), !dbg !16
+  store i32 7, i32* %a, align 4, !dbg !16
+  %5 = load i32, i32* %a, align 4, !dbg !16
+  call void @fn2(i32 %5), !dbg !16
+  %6 = load i32, i32* %u, align 4, !dbg !16
+  %dec = add nsw i32 %6, -1, !dbg !16
+  store i32 %dec, i32* %u, align 4, !dbg !16
+  ret void, !dbg !16
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+declare dso_local void @fn2(i32)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 11.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0"}
+!7 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !1, line: 5, type: !10)
+!12 = !DILocation(line: 5, column: 10, scope: !7)
+!13 = !DILocalVariable(name: "y", arg: 2, scope: !7, file: !1, line: 5, type: !10)
+!14 = !DILocation(line: 5, column: 17, scope: !7)
+!15 = !DILocalVariable(name: "u", scope: !7, file: !1, line: 6, type: !10)
+!16 = !DILocation(line: 6, column: 7, scope: !7)
+!17 = !DILocation(line: 7, column: 7, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !7, file: !1, line: 7, column: 7)
+!19 = !DILocalVariable(name: "a", scope: !7, file: !1, line: 11, type: !10)
diff --git a/llvm/test/ExecutionEngine/JITLink/X86/LocalDependencyPropagation.s b/llvm/test/ExecutionEngine/JITLink/X86/LocalDependencyPropagation.s
new file mode 100644
index 0000000000000000000000000000000000000000..e44e39f67fae214f6ef028a913fd38d154672685
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/X86/LocalDependencyPropagation.s
@@ -0,0 +1,31 @@
+# REQUIRES: asserts
+# RUN: llvm-mc -triple=x86_64-apple-macosx10.9 -filetype=obj -o %t %s
+# RUN: llvm-jitlink -debug-only=orc -noexec -define-abs _external_func=0x1 \
+# RUN:   -entry=_foo %t 2>&1 | FileCheck %s
+#
+# Verify that symbol dependencies are correctly propagated through local
+# symbols: _baz depends on _foo indirectly via the local symbol _bar. We expect
+# _baz to depend on _foo, and _foo on _external_func.
+
+# CHECK-DAG: In <main> adding dependencies for _foo: { (<main>, { _external_func }) }
+# CHECK-DAG: In <main> adding dependencies for _baz: { (<main>, { _foo }) }
+
+        .section	__TEXT,__text,regular,pure_instructions
+
+	.globl	_foo
+	.p2align	4, 0x90
+_foo:
+	jmp	_external_func
+
+	.p2align	4, 0x90
+_bar:
+
+	jmp	_foo
+
+	.globl	_baz
+	.p2align	4, 0x90
+_baz:
+
+	jmp	_bar
+
+.subsections_via_symbols
diff --git a/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s b/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s
index 3a25999c2b8c680f0f6e98978125e73481313d44..c5e022671dcef7a16743dbcfae3c5e13ada19848 100644
--- a/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s
+++ b/llvm/test/MC/MachO/darwin-x86_64-diff-relocs.s
@@ -147,7 +147,7 @@ L3:
 // CHECK:     Size: 0x5E
 // CHECK:     Offset: 384
 // CHECK:     Alignment: 0
-// CHECK:     RelocationOffset: 0x26C
+// CHECK:     RelocationOffset: 0x270
 // CHECK:     RelocationCount: 12
 // CHECK:     Type: Regular (0x0)
 // CHECK:     Attributes [ (0x800004)
@@ -174,7 +174,7 @@ L3:
 // CHECK:     Size: 0x8E
 // CHECK:     Offset: 478
 // CHECK:     Alignment: 0
-// CHECK:     RelocationOffset: 0x2CC
+// CHECK:     RelocationOffset: 0x2D0
 // CHECK:     RelocationCount: 16
 // CHECK:     Type: Regular (0x0)
 // CHECK:     Attributes [ (0x4)
diff --git a/llvm/test/MC/MachO/variable-exprs.s b/llvm/test/MC/MachO/variable-exprs.s
index c2eb05dceaeeff949a5ee83b64983d645fb4376e..b4b57811529d88220f3496141406b83052eef9fb 100644
--- a/llvm/test/MC/MachO/variable-exprs.s
+++ b/llvm/test/MC/MachO/variable-exprs.s
@@ -298,7 +298,7 @@ Lt0_x = Lt0_a - Lt0_b
 // CHECK-X86_64:     Size: 0x38
 // CHECK-X86_64:     Offset: 385
 // CHECK-X86_64:     Alignment: 0
-// CHECK-X86_64:     RelocationOffset: 0x1BC
+// CHECK-X86_64:     RelocationOffset: 0x1C0
 // CHECK-X86_64:     RelocationCount: 9
 // CHECK-X86_64:     Type: Regular (0x0)
 // CHECK-X86_64:     Attributes [ (0x0)
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 4bfee73720f7463a7a60d25177a09febeb83b7b0..bab23c924d64b82054172f50d4b095aa820a72a5 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -92,6 +92,7 @@
 ; CHECK-O2-NEXT: Running analysis: DemandedBitsAnalysis
 ; CHECK-O2-NEXT: Running pass: CrossDSOCFIPass
 ; CHECK-O2-NEXT: Running pass: LowerTypeTestsPass
+; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O2-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}SimplifyCFGPass>
 ; CHECK-O2-NEXT: Running pass: EliminateAvailableExternallyPass
 ; CHECK-O2-NEXT: Running pass: GlobalDCEPass
diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll
index 74daa0505b8c94d10796ac462e34b09e27603d17..cc2dff646faf081cae7e3534a1a45d114c036ef8 100644
--- a/llvm/test/Other/new-pm-thinlto-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -80,6 +80,7 @@
 ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass
 ; CHECK-O-NEXT: Finished llvm::Function pass manager run.
 ; CHECK-O-NEXT: Running pass: AttributorPass
+; CHECK-POSTLINK-O-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index d31767d8fe798442797a7db0ce4bfd19871e59a1..eb6caff8c9fa23e4c319571a06577f3b1402d9c2 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -49,6 +49,7 @@
 ; CHECK-O3-NEXT: Running pass: CallSiteSplittingPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
 ; CHECK-O-NEXT: Running pass: AttributorPass
+; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index d9a2d4b41291e76e14c0e5bc8bd44e1610d73fff..435e29fdff0e75aad03b7eabb074aff180b15abf 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -60,6 +60,7 @@
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-O-NEXT: Running pass: PGOIndirectCallPromotion
 ; CHECK-O-NEXT: Running pass: AttributorPass
+; CHECK-O-NEXT: Running pass: LowerTypeTestsPass
 ; CHECK-O-NEXT: Running pass: IPSCCPPass
 ; CHECK-O-NEXT: Running pass: CalledValuePropagationPass
 ; CHECK-O-NEXT: Running pass: GlobalOptPass
diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll
index 4a4cdbb52b999c2cc9d2e85714a0b7ed550cc311..49144beec3afc48927a95807f304ba48b801677a 100644
--- a/llvm/test/Other/opt-O2-pipeline.ll
+++ b/llvm/test/Other/opt-O2-pipeline.ll
@@ -70,7 +70,6 @@
 ; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Early CSE w/ MemorySSA
 ; CHECK-NEXT:         Speculatively execute instructions if target has divergent branches
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Lazy Value Information Analysis
 ; CHECK-NEXT:         Jump Threading
@@ -134,7 +133,6 @@
 ; CHECK-NEXT:           Unroll loops
 ; CHECK-NEXT:         MergedLoadStoreMotion
 ; CHECK-NEXT:         Phi Values Analysis
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
@@ -149,7 +147,6 @@
 ; CHECK-NEXT:         Sparse Conditional Constant Propagation
 ; CHECK-NEXT:         Demanded bits analysis
 ; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
 ; CHECK-NEXT:         Lazy Block Frequency Analysis
@@ -163,7 +160,6 @@
 ; CHECK-NEXT:         Phi Values Analysis
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Dead Store Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Natural Loop Information
@@ -231,7 +227,6 @@
 ; CHECK-NEXT:       Loop Vectorization
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Loop Access Analysis
 ; CHECK-NEXT:       Lazy Branch Probability Analysis
@@ -256,7 +251,6 @@
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       SLP Vectorizer
 ; CHECK-NEXT:       Optimize scalar/vector ops
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       Combine redundant instructions
diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll
index 5a9e85efaa8102b31c31bdce616d9ea1db42cd42..cbb90eef5ef7c886415470d8a478bc1bb7a0eb26 100644
--- a/llvm/test/Other/opt-O3-pipeline.ll
+++ b/llvm/test/Other/opt-O3-pipeline.ll
@@ -74,7 +74,6 @@
 ; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Early CSE w/ MemorySSA
 ; CHECK-NEXT:         Speculatively execute instructions if target has divergent branches
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Lazy Value Information Analysis
 ; CHECK-NEXT:         Jump Threading
@@ -139,7 +138,6 @@
 ; CHECK-NEXT:           Unroll loops
 ; CHECK-NEXT:         MergedLoadStoreMotion
 ; CHECK-NEXT:         Phi Values Analysis
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
@@ -154,7 +152,6 @@
 ; CHECK-NEXT:         Sparse Conditional Constant Propagation
 ; CHECK-NEXT:         Demanded bits analysis
 ; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
 ; CHECK-NEXT:         Lazy Block Frequency Analysis
@@ -168,7 +165,6 @@
 ; CHECK-NEXT:         Phi Values Analysis
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Dead Store Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Natural Loop Information
@@ -236,7 +232,6 @@
 ; CHECK-NEXT:       Loop Vectorization
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Loop Access Analysis
 ; CHECK-NEXT:       Lazy Branch Probability Analysis
@@ -261,7 +256,6 @@
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       SLP Vectorizer
 ; CHECK-NEXT:       Optimize scalar/vector ops
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       Combine redundant instructions
diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll
index 3e25010b6eec528ce0642ee976118328989c1d26..ce3801388b1e6811c608e8013d61cfcd0f77e1e6 100644
--- a/llvm/test/Other/opt-Os-pipeline.ll
+++ b/llvm/test/Other/opt-Os-pipeline.ll
@@ -70,7 +70,6 @@
 ; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Early CSE w/ MemorySSA
 ; CHECK-NEXT:         Speculatively execute instructions if target has divergent branches
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Lazy Value Information Analysis
 ; CHECK-NEXT:         Jump Threading
@@ -121,7 +120,6 @@
 ; CHECK-NEXT:           Unroll loops
 ; CHECK-NEXT:         MergedLoadStoreMotion
 ; CHECK-NEXT:         Phi Values Analysis
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
@@ -136,7 +134,6 @@
 ; CHECK-NEXT:         Sparse Conditional Constant Propagation
 ; CHECK-NEXT:         Demanded bits analysis
 ; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Lazy Branch Probability Analysis
 ; CHECK-NEXT:         Lazy Block Frequency Analysis
@@ -150,7 +147,6 @@
 ; CHECK-NEXT:         Phi Values Analysis
 ; CHECK-NEXT:         Memory Dependence Analysis
 ; CHECK-NEXT:         Dead Store Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:         Function Alias Analysis Results
 ; CHECK-NEXT:         Memory SSA
 ; CHECK-NEXT:         Natural Loop Information
@@ -218,7 +214,6 @@
 ; CHECK-NEXT:       Loop Vectorization
 ; CHECK-NEXT:       Canonicalize natural loops
 ; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Loop Access Analysis
 ; CHECK-NEXT:       Lazy Branch Probability Analysis
@@ -243,7 +238,6 @@
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       SLP Vectorizer
 ; CHECK-NEXT:       Optimize scalar/vector ops
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Optimization Remark Emitter
 ; CHECK-NEXT:       Combine redundant instructions
diff --git a/llvm/test/ThinLTO/X86/Inputs/cfi-unsat.ll b/llvm/test/ThinLTO/X86/Inputs/cfi-unsat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bc7a0e36dfa3b5f6e0314e35e92117d674451033
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/cfi-unsat.ll
@@ -0,0 +1,50 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { i32 (...)** }
+
+@_ZTV1B = linkonce_odr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1nEi to i8*)] }, !type !0
+
+$test = comdat any
+
+; CHECK-IR-LABEL: define i32 @test
+define linkonce_odr i32 @test(%struct.A* %obj, i32 %a) comdat {
+entry:
+  %0 = bitcast %struct.A* %obj to i8**
+  %vtable5 = load i8*, i8** %0
+
+  %1 = tail call { i8*, i1 } @llvm.type.checked.load(i8* %vtable5, i32 8, metadata !"_ZTS1A")
+  %2 = extractvalue { i8*, i1 } %1, 1
+  br i1 %2, label %cont, label %trap
+
+trap:
+  tail call void @llvm.trap()
+  unreachable
+
+cont:
+  %3 = extractvalue { i8*, i1 } %1, 0
+  %4 = bitcast i8* %3 to i32 (%struct.A*, i32)*
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR: %call = tail call i32 @_ZN1A1nEi
+  %call = tail call i32 %4(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+}
+; CHECK-IR-LABEL: ret i32
+; CHECK-IR-LABEL: }
+
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
+declare void @llvm.trap()
+
+define internal i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) {
+entry:
+   ret i32 0
+}
+define internal i32 @_ZN1B1nEi(%struct.B* %this, i32 %a) {
+entry:
+   ret i32 0
+}
+
+!0 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/cfi-unsat.ll b/llvm/test/ThinLTO/X86/cfi-unsat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..24e837303c2ade6cd1c4cac8bd0c22ee05bd9b34
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/cfi-unsat.ll
@@ -0,0 +1,74 @@
+; REQUIRES: x86-registered-target
+
+; Test CFI devirtualization through the thin link and backend when
+; a type id is Unsat (not used on any global's type metadata).
+;
+; In this test case, the first module is split and will import a resolution
+; for its type test. The resolution would be exported by the second
+; module, which is set up so that it does not get split (treated as regular
+; LTO because it does not have any external globals from which to create
+; a unique module ID). We should not actually get any resolution for the
+; type id in this case, since no globals include it in their type metadata,
+; so the resolution is Unsat and the type.checked.load instructions are
+; converted to type tests that evaluate to false.
+
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t.o %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t1.o %p/Inputs/cfi-unsat.ll
+
+; RUN: llvm-lto2 run %t.o %t1.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -whole-program-visibility \
+; RUN:   -o %t3 \
+; RUN:   -r=%t.o,test2,px \
+; RUN:   -r=%t1.o,_ZTV1B,px \
+; RUN:   -r=%t1.o,test,px
+; RUN: llvm-dis %t3.index.bc -o - | FileCheck %s --check-prefix=INDEX
+; RUN: llvm-dis %t3.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR0
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+
+; INDEX-NOT: "typeid:"
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+
+$test2 = comdat any
+
+; CHECK-IR0: define weak_odr i32 @test
+define linkonce_odr i32 @test2(%struct.A* %obj, i32 %a) comdat {
+; CHECK-IR0-NEXT: entry:
+entry:
+; CHECK-IR0-NEXT: %0 = bitcast
+  %0 = bitcast %struct.A* %obj to i8**
+; CHECK-IR0-NEXT: %vtable5 =
+  %vtable5 = load i8*, i8** %0
+
+; CHECK-IR0-NEXT: tail call void @llvm.trap()
+; CHECK-IR0-NEXT: unreachable
+
+  %1 = tail call { i8*, i1 } @llvm.type.checked.load(i8* %vtable5, i32 8, metadata !"_ZTS1A")
+  %2 = extractvalue { i8*, i1 } %1, 1
+  br i1 %2, label %cont, label %trap
+
+trap:
+  tail call void @llvm.trap()
+  unreachable
+
+cont:
+  %3 = extractvalue { i8*, i1 } %1, 0
+  %4 = bitcast i8* %3 to i32 (%struct.A*, i32)*
+
+  %call = tail call i32 %4(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+; CHECK-IR0-NEXT: }
+}
+
+; CHECK-IR1: define weak_odr i32 @test2
+; CHECK-IR1-NEXT:   entry:
+; CHECK-IR1-NEXT:     tail call void @llvm.trap()
+; CHECK-IR1-NEXT:     unreachable
+; CHECK-IR1-NEXT:   }
+
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
+declare void @llvm.trap()
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
index 6677cb5467251d891a4ae10e4e3e30a36bc221b6..3396cf3b4ac74b936210f08f17a5f13eef186419 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt -S -basicaa -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,OLDPM_MODULE
-; RUN: opt -S -basicaa -attributor-cgscc -attributor-disable=false < %s | FileCheck %s --check-prefixes=CHECK,OLDPM_CGSCC
-; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,NEWPM_MODULE
-; RUN: opt -S -passes='attributor-cgscc' -aa-pipeline='basic-aa' -attributor-disable=false < %s | FileCheck %s --check-prefixes=CHECK,NEWPM_CGSCC
+; RUN: opt -S -basicaa -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,OLDPM,OLDPM_MODULE
+; RUN: opt -S -basicaa -attributor-cgscc -attributor-disable=false < %s | FileCheck %s --check-prefixes=CHECK,OLDPM,OLDPM_CGSCC
+; RUN: opt -S -passes='attributor' -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 < %s | FileCheck %s --check-prefixes=CHECK,NEWPM,NEWPM_MODULE
+; RUN: opt -S -passes='attributor-cgscc' -aa-pipeline='basic-aa' -attributor-disable=false < %s | FileCheck %s --check-prefixes=CHECK,NEWPM,NEWPM_CGSCC
 
 ; OLDPM_MODULE-NOT: @dead
 ; NEWPM_MODULE-NOT: @dead
@@ -15,15 +15,36 @@ define internal void @dead() {
 }
 
 define internal i32 @test(i32* %X, i32* %Y) {
-; CHECK-LABEL: define {{[^@]+}}@test()
-; CHECK-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
-; CHECK:       live:
-; CHECK-NEXT:    ret i32 0
-; CHECK:       dead:
-; CHECK-NEXT:    unreachable
+; OLDPM-LABEL: define {{[^@]+}}@test
+; OLDPM-SAME: (i32* noalias nocapture nofree writeonly align 4 [[X:%.*]])
+; OLDPM-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
+; OLDPM:       live:
+; OLDPM-NEXT:    store i32 0, i32* [[X]], align 4
+; OLDPM-NEXT:    ret i32 0
+; OLDPM:       dead:
+; OLDPM-NEXT:    unreachable
+;
+; NEWPM_MODULE-LABEL: define {{[^@]+}}@test
+; NEWPM_MODULE-SAME: (i32* noalias nocapture nofree writeonly align 4 [[X:%.*]])
+; NEWPM_MODULE-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
+; NEWPM_MODULE:       live:
+; NEWPM_MODULE-NEXT:    store i32 0, i32* [[X]], align 4
+; NEWPM_MODULE-NEXT:    ret i32 0
+; NEWPM_MODULE:       dead:
+; NEWPM_MODULE-NEXT:    unreachable
+;
+; NEWPM_CGSCC-LABEL: define {{[^@]+}}@test
+; NEWPM_CGSCC-SAME: (i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[X:%.*]])
+; NEWPM_CGSCC-NEXT:    br i1 true, label [[LIVE:%.*]], label [[DEAD:%.*]]
+; NEWPM_CGSCC:       live:
+; NEWPM_CGSCC-NEXT:    store i32 0, i32* [[X]], align 4
+; NEWPM_CGSCC-NEXT:    ret i32 0
+; NEWPM_CGSCC:       dead:
+; NEWPM_CGSCC-NEXT:    unreachable
 ;
   br i1 true, label %live, label %dead
 live:
+  store i32 0, i32* %X
   ret i32 0
 dead:
   call i32 @caller(i32* null)
@@ -35,7 +56,7 @@ define internal i32 @caller(i32* %B) {
 ; CHECK-LABEL: define {{[^@]+}}@caller()
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32
 ; CHECK-NEXT:    store i32 1, i32* [[A]], align 4
-; CHECK-NEXT:    [[C:%.*]] = call i32 @test()
+; CHECK-NEXT:    [[C:%.*]] = call i32 @test(i32* noalias nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[A]])
 ; CHECK-NEXT:    ret i32 0
 ;
   %A = alloca i32
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
index 2c83cbb4aff4f566e900997f59975323cecc27b6..ac0fc2b9951cadad108fd0ea4d924165a24a6ccd 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/musttail.ll
@@ -45,11 +45,6 @@ define i32 @foo(%T* %p, i32 %v) {
 }
 
 define internal i32 @test2(%T* %p, i32 %p2) {
-; CHECK-LABEL: define {{[^@]+}}@test2
-; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
-; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @foo(%T* undef, i32 undef)
-; CHECK-NEXT:    ret i32 [[CA]]
-;
   %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
   %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
   %a = load i32, i32* %a.gep
@@ -62,9 +57,54 @@ define internal i32 @test2(%T* %p, i32 %p2) {
 define i32 @caller2(%T* %g) {
 ; CHECK-LABEL: define {{[^@]+}}@caller2
 ; CHECK-SAME: (%T* nocapture nofree readonly [[G:%.*]])
-; CHECK-NEXT:    [[V:%.*]] = call i32 @test2(%T* nocapture nofree readonly undef, i32 undef)
 ; CHECK-NEXT:    ret i32 0
 ;
   %v = call i32 @test2(%T* %g, i32 0)
   ret i32 %v
 }
+
+; In the version above we can remove the call to foo completely.
+; In the version below we keep the call and verify the return value
+; is kept as well.
+
+define i32 @bar(%T* %p, i32 %v) {
+; CHECK-LABEL: define {{[^@]+}}@bar
+; CHECK-SAME: (%T* nocapture nofree nonnull writeonly dereferenceable(4) [[P:%.*]], i32 [[V:%.*]])
+; CHECK-NEXT:    [[I32PTR:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 0
+; CHECK-NEXT:    store i32 [[V]], i32* [[I32PTR]]
+; CHECK-NEXT:    ret i32 0
+;
+  %i32ptr = getelementptr %T, %T* %p, i64 0, i32 0
+  store i32 %v, i32* %i32ptr
+  ret i32 0
+}
+
+define internal i32 @test2b(%T* %p, i32 %p2) {
+; CHECK-LABEL: define {{[^@]+}}@test2b
+; CHECK-SAME: (%T* nocapture nofree readonly [[P:%.*]], i32 [[P2:%.*]])
+; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr [[T:%.*]], %T* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[B_GEP:%.*]] = getelementptr [[T]], %T* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[A:%.*]] = load i32, i32* [[A_GEP]]
+; CHECK-NEXT:    [[B:%.*]] = load i32, i32* [[B_GEP]]
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT:    [[CA:%.*]] = musttail call i32 @bar(%T* undef, i32 [[V]])
+; CHECK-NEXT:    ret i32 [[CA]]
+;
+  %a.gep = getelementptr %T, %T* %p, i64 0, i32 3
+  %b.gep = getelementptr %T, %T* %p, i64 0, i32 2
+  %a = load i32, i32* %a.gep
+  %b = load i32, i32* %b.gep
+  %v = add i32 %a, %b
+  %ca = musttail call i32 @bar(%T* undef, i32 %v)
+  ret i32 %ca
+}
+
+define i32 @caller2b(%T* %g) {
+; CHECK-LABEL: define {{[^@]+}}@caller2b
+; CHECK-SAME: (%T* nocapture nofree readonly [[G:%.*]])
+; CHECK-NEXT:    [[V:%.*]] = call i32 @test2b(%T* nocapture nofree readonly [[G]], i32 undef)
+; CHECK-NEXT:    ret i32 0
+;
+  %v = call i32 @test2b(%T* %g, i32 0)
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
index 2f50a9c3d203d434f75771b7a81826c22a71b425..2fcde7a92288948020bdd7e0b9a3d1e477f699a8 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/comdat-ipo.ll
@@ -15,7 +15,6 @@ define i32 @baz() {
 
 define linkonce_odr i32 @foo() {
 ; CHECK-LABEL: define {{[^@]+}}@foo()
-; CHECK-NEXT:    [[VAL:%.*]] = call i32 @baz()
 ; CHECK-NEXT:    ret i32 10
 ;
 
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
index 3e0b575f94f5e6ae3a62b1816827e041fe72d2a4..339a9fc4b76728e842a6814ea6ca13e5af64003e 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/remove-call-inst.ll
@@ -10,7 +10,6 @@
 define i32 @main() noreturn nounwind {
 ; CHECK-LABEL: define {{[^@]+}}@main()
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @wwrite(i64 0)
 ; CHECK-NEXT:    ret i32 123
 ;
 entry:
@@ -19,18 +18,6 @@ entry:
 }
 
 define internal i32 @wwrite(i64 %i) nounwind readnone {
-; CHECK-LABEL: define {{[^@]+}}@wwrite
-; CHECK-SAME: (i64 [[I:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    switch i64 0, label [[SW_DEFAULT:%.*]] [
-; CHECK-NEXT:    i64 3, label [[RETURN:%.*]]
-; CHECK-NEXT:    i64 10, label [[RETURN]]
-; CHECK-NEXT:    ]
-; CHECK:       sw.default:
-; CHECK-NEXT:    ret i32 123
-; CHECK:       return:
-; CHECK-NEXT:    unreachable
-;
 entry:
   switch i64 %i, label %sw.default [
   i64 3, label %return
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
index 4bc1b25572d4f80fb92517bed42e832bf89e8271..cfbef759d41c411ef85ff0ef240f20612e9a2a55 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-constant.ll
@@ -44,7 +44,6 @@ F:              ; preds = %0
 define i1 @caller(i1 %C) {
 ; CHECK-LABEL: define {{[^@]+}}@caller
 ; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    [[X:%.*]] = call i32 @foo(i1 [[C]])
 ; CHECK-NEXT:    ret i1 true
 ;
   %X = call i32 @foo( i1 %C )             ; <i32> [#uses=1]
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
index ff33eccb22550ad70e853bf34c9b84d3355ef289..ba3f9f929cd5cd68a9a3d0d58427dd5fe212b22f 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/solve-after-each-resolving-undefs-for-function.ll
@@ -2,17 +2,6 @@
 ; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 < %s | FileCheck %s
 
 define internal i32 @testf(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@testf
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C]], label [[IF_COND:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.cond:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.then:
-; CHECK-NEXT:    unreachable
-; CHECK:       if.end:
-; CHECK-NEXT:    ret i32 10
-;
 entry:
   br i1 %c, label %if.cond, label %if.end
 
@@ -27,18 +16,6 @@ if.end:                                          ; preds = %if.then1, %entry
 }
 
 define internal i32 @test1(i1 %c) {
-; CHECK-LABEL: define {{[^@]+}}@test1
-; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @testf(i1 [[C]])
-; CHECK-NEXT:    br label [[RET1:%.*]]
-; CHECK:       ret1:
-; CHECK-NEXT:    ret i32 99
-; CHECK:       ret2:
-; CHECK-NEXT:    unreachable
-;
 entry:
   br label %if.then
 
@@ -57,7 +34,6 @@ ret2:                                           ; preds = %if.then, %entry
 define i32 @main(i1 %c) {
 ; CHECK-LABEL: define {{[^@]+}}@main
 ; CHECK-SAME: (i1 [[C:%.*]])
-; CHECK-NEXT:    [[RES:%.*]] = call i32 @test1(i1 [[C]])
 ; CHECK-NEXT:    ret i32 99
 ;
   %res = call i32 @test1(i1 %c)
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
index 13f524d9b05911a0692370a758493a0e279a6061..3d4a761d081202fdd27fa423979316605ced10b9 100644
--- a/llvm/test/Transforms/Attributor/align.ll
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -133,7 +133,7 @@ define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
   ret i8* %6
 }
 
-; UTC_ARGS: --turn=on
+; UTC_ARGS: --enable
 
 ; TEST 7
 ; Better than IR information
@@ -264,7 +264,7 @@ define align 4 i32* @test7b(i32* align 32 %p) #0 {
   ret i32* %p
 }
 
-; UTC_ARGS: --turn=off
+; UTC_ARGS: --disable
 
 ; TEST 8
 define void @test8_helper() {
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
index d8e879e02279928cd1ee224e1a6896e2ce7a9aa9..92142f06d4ee236190b84ea9bd2e81201233befc 100644
--- a/llvm/test/Transforms/Attributor/liveness.ll
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -3,7 +3,7 @@
 ; RUN: opt -attributor-cgscc --attributor-disable=false -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,CGSCC
 ; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,MODULE,ALL_BUT_OLD_CGSCCC
 ; RUN: opt -passes='attributor-cgscc' --attributor-disable=false -attributor-annotate-decl-cs -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefixes=CHECK,CGSCC,ALL_BUT_OLD_CGSCCC
-; UTC_ARGS: --turn off
+; UTC_ARGS: --disable
 
 ; ALL_BUT_OLD_CGSCCC: @dead_with_blockaddress_users.l = constant [2 x i8*] [i8* inttoptr (i32 1 to i8*), i8* inttoptr (i32 1 to i8*)]
 @dead_with_blockaddress_users.l = constant [2 x i8*] [i8* blockaddress(@dead_with_blockaddress_users, %lab0), i8* blockaddress(@dead_with_blockaddress_users, %end)]
@@ -280,7 +280,7 @@ cleanup:
   ret i32 0
 }
 
-; UTC_ARGS: --turn on
+; UTC_ARGS: --enable
 
 ; TEST 5.4 unounwind invoke instruction replaced by a call and a branch instruction put after it.
 define i32 @invoke_nounwind_phi(i32 %a) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
@@ -370,7 +370,7 @@ cleanup:
   ret i32 0
 }
 
-; UTC_ARGS: --turn off
+; UTC_ARGS: --disable
 
 ; TEST 6: Undefined behvior, taken from LangRef.
 ; FIXME: Should be able to detect undefined behavior.
@@ -544,6 +544,7 @@ define internal i8* @f3(i8* readnone %0) local_unnamed_addr #0 {
   ret i8* %6
 }
 
+declare void @sink() nofree nosync nounwind willreturn
 define void @test_unreachable() {
 ; CHECK:       define void @test_unreachable()
 ; CHECK-NEXT:    call void @test_unreachable()
@@ -893,7 +894,7 @@ define void @useless_arg_ext_int_ext(i32* %a) {
   ret void
 }
 
-; UTC_ARGS: --turn on
+; UTC_ARGS: --enable
 
 ; FIXME: We should fold terminators.
 
@@ -906,6 +907,7 @@ define internal i32 @switch_default(i64 %i) nounwind {
 ; CHECK-NEXT:    i64 10, label [[RETURN]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.default:
+; CHECK-NEXT:    call void @sink()
 ; CHECK-NEXT:    ret i32 123
 ; CHECK:       return:
 ; CHECK-NEXT:    unreachable
@@ -917,6 +919,7 @@ entry:
   ]
 
 sw.default:
+  call void @sink()
   ret i32 123
 
 return:
@@ -924,14 +927,37 @@ return:
 }
 
 define i32 @switch_default_caller() {
-; CHECK-LABEL: define {{[^@]+}}@switch_default_caller()
-; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @switch_default(i64 0)
-; CHECK-NEXT:    ret i32 123
+; CGSCC-LABEL: define {{[^@]+}}@switch_default_caller()
+; CGSCC-NEXT:    [[CALL2:%.*]] = tail call i32 @switch_default(i64 0)
+; CGSCC-NEXT:    ret i32 123
 ;
   %call2 = tail call i32 @switch_default(i64 0)
   ret i32 %call2
 }
-; UTC_ARGS: --turn off
+
+define internal i32 @switch_default_dead(i64 %i) nounwind {
+entry:
+  switch i64 %i, label %sw.default [
+  i64 3, label %return
+  i64 10, label %return
+  ]
+
+sw.default:
+  ret i32 123
+
+return:
+  ret i32 0
+}
+
+define i32 @switch_default_dead_caller() {
+; CHECK-LABEL: define {{[^@]+}}@switch_default_dead_caller()
+; CHECK-NEXT:    ret i32 123
+;
+  %call2 = tail call i32 @switch_default_dead(i64 0)
+  ret i32 %call2
+}
+
+; UTC_ARGS: --disable
 
 ; Allow blockaddress users
 ; ALL_BUT_OLD_CGSCCC-NOT @dead_with_blockaddress_users
diff --git a/llvm/test/Transforms/Attributor/nonnull.ll b/llvm/test/Transforms/Attributor/nonnull.ll
index 7420a39b8907ef3906aa01b2cb424b6ca5e029a8..f11f3941aaa8f30b61a1a9a72fe983687a8c976b 100644
--- a/llvm/test/Transforms/Attributor/nonnull.ll
+++ b/llvm/test/Transforms/Attributor/nonnull.ll
@@ -525,12 +525,14 @@ define i32 addrspace(3)* @as(i32 addrspace(3)* dereferenceable(4) %p) {
   ret i32 addrspace(3)* %p
 }
 
-; ATTRIBUTOR: define internal nonnull align 4 i32* @g2()
+; ATTRIBUTOR-NOT: @g2()
 define internal i32* @g2() {
   ret i32* inttoptr (i64 4 to i32*)
 }
 
-define  i32* @g1() {
+; ATTRIBUTOR: define nonnull align 4 i32* @g1()
+; ATTRIBUTOR:   ret i32* inttoptr (i64 4 to i32*)
+define i32* @g1() {
  %c = call i32* @g2()
   ret i32* %c
 }
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 5712c947ccf9f0b9ed3e58c627aa5407f2ac65f1..efdb8a5c00195dca45b48ed0b43b843d68027db3 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -705,22 +705,6 @@ define internal i32 @r1(i32) local_unnamed_addr {
 ; OLD_PM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 100
 ; OLD_PM-NEXT:    br i1 [[TMP9]], label [[TMP1:%.*]], label [[TMP4]]
 ;
-; NEW_PM-LABEL: define {{[^@]+}}@r1() local_unnamed_addr
-; NEW_PM-NEXT:    br label [[TMP3:%.*]]
-; NEW_PM:       1:
-; NEW_PM-NEXT:    br label [[F:%.*]]
-; NEW_PM:       2:
-; NEW_PM-NEXT:    unreachable
-; NEW_PM:       f:
-; NEW_PM-NEXT:    ret i32 10
-; NEW_PM:       3:
-; NEW_PM-NEXT:    [[TMP4:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP7:%.*]], [[TMP3]] ]
-; NEW_PM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP0]] ], [ [[TMP6:%.*]], [[TMP3]] ]
-; NEW_PM-NEXT:    [[TMP6]] = add nuw nsw i32 [[TMP4]], [[TMP5]]
-; NEW_PM-NEXT:    [[TMP7]] = add nuw nsw i32 [[TMP4]], 1
-; NEW_PM-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 100
-; NEW_PM-NEXT:    br i1 [[TMP8]], label [[TMP1:%.*]], label [[TMP3]]
-;
 ; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@r1
 ; CGSCC_OLD_PM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
 ; CGSCC_OLD_PM-NEXT:    br label [[TMP5:%.*]]
@@ -738,23 +722,6 @@ define internal i32 @r1(i32) local_unnamed_addr {
 ; CGSCC_OLD_PM-NEXT:    [[TMP9]] = add nuw nsw i32 [[TMP6]], 1
 ; CGSCC_OLD_PM-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[TMP9]], 100
 ; CGSCC_OLD_PM-NEXT:    br i1 [[TMP10]], label [[TMP2:%.*]], label [[TMP5]]
-;
-; CGSCC_NEW_PM-LABEL: define {{[^@]+}}@r1
-; CGSCC_NEW_PM-SAME: (i32 [[TMP0:%.*]]) local_unnamed_addr
-; CGSCC_NEW_PM-NEXT:    br label [[TMP4:%.*]]
-; CGSCC_NEW_PM:       2:
-; CGSCC_NEW_PM-NEXT:    br label [[F:%.*]]
-; CGSCC_NEW_PM:       3:
-; CGSCC_NEW_PM-NEXT:    unreachable
-; CGSCC_NEW_PM:       f:
-; CGSCC_NEW_PM-NEXT:    ret i32 10
-; CGSCC_NEW_PM:       4:
-; CGSCC_NEW_PM-NEXT:    [[TMP5:%.*]] = phi i32 [ 0, [[TMP1:%.*]] ], [ [[TMP8:%.*]], [[TMP4]] ]
-; CGSCC_NEW_PM-NEXT:    [[TMP6:%.*]] = phi i32 [ 0, [[TMP1]] ], [ [[TMP7:%.*]], [[TMP4]] ]
-; CGSCC_NEW_PM-NEXT:    [[TMP7]] = add nuw nsw i32 [[TMP5]], [[TMP6]]
-; CGSCC_NEW_PM-NEXT:    [[TMP8]] = add nuw nsw i32 [[TMP5]], 1
-; CGSCC_NEW_PM-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 100
-; CGSCC_NEW_PM-NEXT:    br i1 [[TMP9]], label [[TMP2:%.*]], label [[TMP4]]
 ;
   br label %5
 
@@ -788,11 +755,10 @@ define void @f1(i32){
 ;
 ; NEW_PM-LABEL: define {{[^@]+}}@f1
 ; NEW_PM-SAME: (i32 [[TMP0:%.*]])
-; NEW_PM-NEXT:    [[TMP2:%.*]] = tail call i32 @r1()
-; NEW_PM-NEXT:    br label [[TMP4:%.*]]
-; NEW_PM:       3:
+; NEW_PM-NEXT:    br label [[TMP3:%.*]]
+; NEW_PM:       2:
 ; NEW_PM-NEXT:    unreachable
-; NEW_PM:       4:
+; NEW_PM:       3:
 ; NEW_PM-NEXT:    ret void
 ;
 ; CGSCC_OLD_PM-LABEL: define {{[^@]+}}@f1
@@ -808,11 +774,10 @@ define void @f1(i32){
 ;
 ; CGSCC_NEW_PM-LABEL: define {{[^@]+}}@f1
 ; CGSCC_NEW_PM-SAME: (i32 [[TMP0:%.*]])
-; CGSCC_NEW_PM-NEXT:    [[TMP2:%.*]] = tail call i32 @r1(i32 [[TMP0]])
-; CGSCC_NEW_PM-NEXT:    br label [[TMP4:%.*]]
-; CGSCC_NEW_PM:       3:
+; CGSCC_NEW_PM-NEXT:    br label [[TMP3:%.*]]
+; CGSCC_NEW_PM:       2:
 ; CGSCC_NEW_PM-NEXT:    unreachable
-; CGSCC_NEW_PM:       4:
+; CGSCC_NEW_PM:       3:
 ; CGSCC_NEW_PM-NEXT:    ret void
 ;
   %2 = tail call i32 @r1(i32 %0)
diff --git a/llvm/test/Transforms/Attributor/returned.ll b/llvm/test/Transforms/Attributor/returned.ll
index 85ab69a0d99a90784c5ddda9d1af14de63828efa..3efa581c9b772ba4fa96b172cbf356d15b4cb482 100644
--- a/llvm/test/Transforms/Attributor/returned.ll
+++ b/llvm/test/Transforms/Attributor/returned.ll
@@ -334,8 +334,8 @@ if.end:
 ; BOTH: declare void @unknown_fn(i32* (i32*)*)
 ;
 ; BOTH:       Function Attrs: noinline nounwind uwtable
-; BOTH-NEXT:  define i32* @calls_unknown_fn(i32* readnone returned "no-capture-maybe-returned" %r)
-; ATTRIBUTOR: define i32* @calls_unknown_fn(i32* readnone returned "no-capture-maybe-returned" %r)
+; BOTH-NEXT:  define i32* @calls_unknown_fn(i32* nofree readnone returned "no-capture-maybe-returned" %r)
+; ATTRIBUTOR: define i32* @calls_unknown_fn(i32* nofree readnone returned "no-capture-maybe-returned" %r)
 declare void @unknown_fn(i32* (i32*)*) #0
 
 define i32* @calls_unknown_fn(i32* %r) #0 {
diff --git a/llvm/test/Transforms/Attributor/undefined_behavior.ll b/llvm/test/Transforms/Attributor/undefined_behavior.ll
index 8e16a919f483a7fc1f84e45313c0011acca7463f..1d2f6fb430db605fac60d9d3523d9d73f992f58f 100644
--- a/llvm/test/Transforms/Attributor/undefined_behavior.ll
+++ b/llvm/test/Transforms/Attributor/undefined_behavior.ll
@@ -223,7 +223,6 @@ define i1 @ret_undef() {
 
 define void @cond_br_on_undef_interproc() {
 ; ATTRIBUTOR-LABEL: @cond_br_on_undef_interproc(
-; ATTRIBUTOR-NEXT:    %cond = call i1 @ret_undef()
 ; ATTRIBUTOR-NEXT:    unreachable
 ; ATTRIBUTOR:       t:
 ; ATTRIBUTOR-NEXT:    unreachable
@@ -249,7 +248,6 @@ e:
 ; More complicated interproc deduction of undef
 define void @cond_br_on_undef_interproc2() {
 ; ATTRIBUTOR-LABEL: @cond_br_on_undef_interproc2(
-; ATTRIBUTOR-NEXT:    %cond = call i1 @ret_undef2()
 ; ATTRIBUTOR-NEXT:    unreachable
 ; ATTRIBUTOR:       t:
 ; ATTRIBUTOR-NEXT:    unreachable
diff --git a/llvm/test/Transforms/CodeGenPrepare/PowerPC/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/PowerPC/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..091332439b186719ce70be1df6973b9bd6f14c51
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/PowerPC/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5bc7d3a3b3b4c1d692bdcce4d44cb9a56073263b
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/PowerPC/split-store-alignment.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare -mtriple=powerpc64-unknown-linux-gnu -data-layout="E-m:e-i64:64-n32:64" -force-split-store < %s  | FileCheck --check-prefixes=ALL,BE %s
+; RUN: opt -S -codegenprepare -mtriple=powerpc64le-unknown-linux-gnu -data-layout="e-m:e-i64:64-n32:64" -force-split-store < %s  | FileCheck --check-prefixes=ALL,LE %s
+
+define void @split_store_align1(float %x, i64* %p) {
+; BE-LABEL: @split_store_align1(
+; BE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; BE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; BE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; BE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; BE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 1
+; BE-NEXT:    store i32 [[B]], i32* [[TMP2]], align 1
+; BE-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P]] to i32*
+; BE-NEXT:    store i32 0, i32* [[TMP3]], align 1
+; BE-NEXT:    ret void
+;
+; LE-LABEL: @split_store_align1(
+; LE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; LE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; LE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; LE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; LE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; LE-NEXT:    store i32 [[B]], i32* [[TMP1]], align 1
+; LE-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; LE-NEXT:    store i32 0, i32* [[TMP3]], align 1
+; LE-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 1
+  ret void
+}
+
+define void @split_store_align2(float %x, i64* %p) {
+; BE-LABEL: @split_store_align2(
+; BE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; BE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; BE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; BE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; BE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 1
+; BE-NEXT:    store i32 [[B]], i32* [[TMP2]], align 2
+; BE-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P]] to i32*
+; BE-NEXT:    store i32 0, i32* [[TMP3]], align 2
+; BE-NEXT:    ret void
+;
+; LE-LABEL: @split_store_align2(
+; LE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; LE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; LE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; LE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; LE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; LE-NEXT:    store i32 [[B]], i32* [[TMP1]], align 2
+; LE-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; LE-NEXT:    store i32 0, i32* [[TMP3]], align 2
+; LE-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 2
+  ret void
+}
+
+define void @split_store_align8(float %x, i64* %p) {
+; BE-LABEL: @split_store_align8(
+; BE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; BE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; BE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; BE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; BE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; BE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; BE-NEXT:    [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], i32 1
+; BE-NEXT:    store i32 [[B]], i32* [[TMP2]], align 4
+; BE-NEXT:    [[TMP3:%.*]] = bitcast i64* [[P]] to i32*
+; BE-NEXT:    store i32 0, i32* [[TMP3]], align 8
+; BE-NEXT:    ret void
+;
+; LE-LABEL: @split_store_align8(
+; LE-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; LE-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; LE-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; LE-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; LE-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; LE-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; LE-NEXT:    store i32 [[B]], i32* [[TMP1]], align 8
+; LE-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; LE-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; LE-NEXT:    store i32 0, i32* [[TMP3]], align 4
+; LE-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7eb8cb8ebfe1bffc0e562a23b751a4e63fe020b4
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/split-store-alignment.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -mtriple=x86_64-unknown-unknown -force-split-store -S < %s | FileCheck %s
+
+target datalayout = "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-w64-windows-gnu"
+
+define void @split_store_align1(float %x, i64* %p) {
+; CHECK-LABEL: @split_store_align1(
+; CHECK-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; CHECK-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[B]], i32* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 1
+; CHECK-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 1
+  ret void
+}
+
+define void @split_store_align2(float %x, i64* %p) {
+; CHECK-LABEL: @split_store_align2(
+; CHECK-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; CHECK-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[B]], i32* [[TMP1]], align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 2
+; CHECK-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 2
+  ret void
+}
+
+define void @split_store_align8(float %x, i64* %p) {
+; CHECK-LABEL: @split_store_align8(
+; CHECK-NEXT:    [[B:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[S:%.*]] = shl nuw nsw i64 [[Z]], 32
+; CHECK-NEXT:    [[Z2:%.*]] = zext i32 [[B]] to i64
+; CHECK-NEXT:    [[O:%.*]] = or i64 [[S]], [[Z2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[P:%.*]] to i32*
+; CHECK-NEXT:    store i32 [[B]], i32* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[P]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, i32* [[TMP2]], i32 1
+; CHECK-NEXT:    store i32 0, i32* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+  %b = bitcast float %x to i32
+  %z = zext i32 0 to i64
+  %s = shl nuw nsw i64 %z, 32
+  %z2 = zext i32 %b to i64
+  %o = or i64 %s, %z2
+  store i64 %o, i64* %p, align 8
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll
index 0d84e9cf9d51bfd538784789cf71b4d5af0453c4..b9a0ea76d7fbb637d6cd856cf22d7d621bfb9481 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/2011-09-06-EndOfFunction.ll
@@ -1,4 +1,3 @@
-; XFAIL: *
 ; RUN: opt -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll
index 1028c8ed47f643b324c6383365012685efd8e9d9..a7278d56e04cff9a8643ff6e2fa9b322a5fe241b 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreBegin.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 define void @write4to7(i32* nocapture %p) {
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll
index 0711855532fe85859fc0a46aa57d820917161a57..d8791aa5e930b8591de23a66572ef4700d619142 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/OverwriteStoreEnd.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll
index 9558033b37c7c0f95fd7f1554a28c1dc945dfb75..b21183c1ac2b3cb158a4f23b2065f6f12e3517d2 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/atomic.ll
@@ -1,4 +1,3 @@
-; XFAIL: *
 ; RUN: opt -basicaa -dse -enable-dse-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll
index 397c5947ae5c9c3c581ca0c034d8bd2bce958086..19ebaa93e7d3f7f7f2476c3725190856f6e103e9 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/calloc-store.ll
@@ -1,5 +1,3 @@
-; XFAIL: *
-
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 declare noalias i8* @calloc(i64, i64)
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll
deleted file mode 100644
index 4ee295821e227ed51b54b09bae5632fd16e408e3..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence-todo.ll
+++ /dev/null
@@ -1,50 +0,0 @@
-; XFAIL: *
-
-; RUN: opt -S -basicaa -dse -enable-dse-memoryssa < %s | FileCheck %s
-
-; We DSE stack alloc'ed and byval locations, in the presence of fences.
-; Fence does not make an otherwise thread local store visible.
-; Right now the DSE in presence of fence is only done in end blocks (with no successors),
-; but the same logic applies to other basic blocks as well.
-; The store to %addr.i can be removed since it is a byval attribute
-define void @test3(i32* byval %addr.i) {
-; CHECK-LABEL: @test3
-; CHECK-NOT: store
-; CHECK: fence
-; CHECK: ret
-  store i32 5, i32* %addr.i, align 4
-  fence release
-  ret void
-}
-
-declare void @foo(i8* nocapture %p)
-
-declare noalias i8* @malloc(i32)
-
-; DSE of stores in locations allocated through library calls.
-define void @test_nocapture() {
-; CHECK-LABEL: @test_nocapture
-; CHECK: malloc
-; CHECK: foo
-; CHECK-NOT: store
-; CHECK: fence
-  %m  =  call i8* @malloc(i32 24)
-  call void @foo(i8* %m)
-  store i8 4, i8* %m
-  fence release
-  ret void
-}
-
-
-; This is a full fence, but it does not make a thread local store visible.
-; We can DSE the store in presence of the fence.
-define void @fence_seq_cst() {
-; CHECK-LABEL: @fence_seq_cst
-; CHECK-NEXT: fence seq_cst
-; CHECK-NEXT: ret void
-  %P1 = alloca i32
-  store i32 0, i32* %P1, align 4
-  fence seq_cst
-  store i32 4, i32* %P1, align 4
-  ret void
-}
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll
index b22fc9c92004d919c3e43fc9e6b2f49b915273c7..9b43d376b295a1928b5e7c53ff45e76e4b43fb21 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/fence.ll
@@ -46,3 +46,51 @@ define void @test2(i32* %addr.i) {
   store i32 5, i32* %addr.i, align 4
   ret void
 }
+
+; We DSE stack alloc'ed and byval locations, in the presence of fences.
+; Fence does not make an otherwise thread local store visible.
+; Right now the DSE in presence of fence is only done in end blocks (with no successors),
+; but the same logic applies to other basic blocks as well.
+; The store to %addr.i can be removed since it is a byval attribute
+define void @test3(i32* byval %addr.i) {
+; CHECK-LABEL: @test3
+; CHECK-NOT: store
+; CHECK: fence
+; CHECK: ret
+  store i32 5, i32* %addr.i, align 4
+  fence release
+  ret void
+}
+
+declare void @foo(i8* nocapture %p)
+
+declare noalias i8* @malloc(i32)
+
+; DSE of stores in locations allocated through library calls.
+define void @test_nocapture() {
+; CHECK-LABEL: @test_nocapture
+; CHECK: malloc
+; CHECK: foo
+; CHECK-NOT: store
+; CHECK: fence
+  %m  =  call i8* @malloc(i32 24)
+  call void @foo(i8* %m)
+  store i8 4, i8* %m
+  fence release
+  ret void
+}
+
+
+; This is a full fence, but it does not make a thread local store visible.
+; We can DSE the store in presence of the fence.
+define void @fence_seq_cst() {
+; CHECK-LABEL: @fence_seq_cst
+; CHECK-NEXT: fence seq_cst
+; CHECK-NEXT: ret void
+  %P1 = alloca i32
+  store i32 0, i32* %P1, align 4
+  fence seq_cst
+  store i32 4, i32* %P1, align 4
+  ret void
+}
+
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll
index a6e47d7bb107c5d049a35d847319f1c83242c246..a9a32348cb6b81cd89f0ec99428a4ee252db0c97 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/free.ll
@@ -1,5 +1,3 @@
-; XFAIL: *
-
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/inst-limits.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/inst-limits.ll
index 638571f6f41728bdc1721caeadec5d86e0d8e77f..78fe292d0cf53d4bc5295d4419b0c5ebe1f7d4a9 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/inst-limits.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/inst-limits.ll
@@ -1,9 +1,10 @@
 ; RUN: opt -S -dse -enable-dse-memoryssa < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
-; This test is not relevant for DSE with MemorySSA. Non-memory instructions
-; are ignored anyways. The limits for the MemorySSA traversal are tested in
-; llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll
+; If there are two stores to the same location, DSE should be able to remove
+; the first store if the two stores are separated by no more than 98
+; instructions. The existence of debug intrinsics between the stores should
+; not affect this instruction limit.
 
 @x = global i32 0, align 4
 
@@ -128,7 +129,7 @@ entry:
 define i32 @test_outside_limit() {
 entry:
   ; The first store; later there is a second store to the same location
-  ; CHECK-NOT: store i32 1, i32* @x, align 4
+  ; CHECK: store i32 1, i32* @x, align 4
   store i32 1, i32* @x, align 4
 
   ; Insert 99 dummy instructions between the two stores; this is
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll
index e4d4722677a798be5922be6c55dc88dd4760dc17..3b3f4d498e9fe843d13a043d0461e4d4ab934608 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/lifetime.ll
@@ -1,5 +1,3 @@
-; XFAIL: *
-
 ; RUN: opt -S -basicaa -dse -enable-dse-memoryssa < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll
index 06bcc346a4351774da19ab5f66ee1f3cdac53537..52e2ecdc7290b42f5a41e469225f0f1f0f8075ee 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/mda-with-dbg-values.ll
@@ -1,12 +1,13 @@
-; RUN: opt -S -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=2 < %s | FileCheck %s
-; RUN: opt -S -strip-debug -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=2 < %s | FileCheck %s
+; RUN: opt -S -dse -enable-dse-memoryssa -memdep-block-scan-limit=3 < %s | FileCheck %s
+; RUN: opt -S -strip-debug -dse -enable-dse-memoryssa -memdep-block-scan-limit=3 < %s | FileCheck %s
 
-; Test case to check that DSE gets the same result even if we have a dbg value
-; between the memcpy.
+; Test case to check that the memory dependency analysis gets the same
+; result even if we have a dbg value between the memcpy and
+; store. The memory dependency is then used by DSE to remove the store.
 
-; This test case is less relevant for the MemorySSA backed version of DSE, as
-; debug values are not modeled in MemorySSA and are skipped regardless of the
-; exploration limit.
+; We use -memdep-block-scan-limit=3 to be able to create a small test case.
+; Without it, we would need to squeeze in 100 instructions since the default
+; limit is 100.
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -19,11 +20,6 @@ entry:
   %i = alloca i8, align 1
   store i8 1, i8* %i, align 1, !dbg !19
   call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
-  call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !17, metadata !DIExpression()), !dbg !18
   %0 = bitcast [1 x i8]* @g to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %i, i8* %0, i64 1, i1 false), !dbg !20
   br label %bb2
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll
index 931597265e414a375ec6925aac2ee36e8b597169..38cb8a38511b2c3658fd45b0efed5ffb0b44b000 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memcpy-complete-overwrite.ll
@@ -1,6 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-
-; XFAIL: *
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll
index c22ad19198b45cf3170954786eb5e98bf7af2271..51c45c32d8c5c71deb8bc25e4e8572db8b887ee2 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memintrinsics.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt -S -dse -enable-dse-memoryssa < %s | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p0i8.i8(i8* nocapture, i8* nocapture, i8, i1) nounwind
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll
deleted file mode 100644
index 43a9f0c92f1b4c4c1779f08ff161a765f5cfda3e..0000000000000000000000000000000000000000
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memoryssa-scan-limit.ll
+++ /dev/null
@@ -1,72 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck --check-prefix=NO-LIMIT %s
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=0 -S | FileCheck --check-prefix=LIMIT-0 %s
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=3 -S | FileCheck --check-prefix=LIMIT-3 %s
-; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -dse-memoryssa-scanlimit=4 -S | FileCheck --check-prefix=LIMIT-4 %s
-
-target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
-
-
-define void @test2(i32* noalias %P, i32* noalias %Q, i32* noalias %R) {
-;
-; NO-LIMIT-LABEL: @test2(
-; NO-LIMIT-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
-; NO-LIMIT:       bb1:
-; NO-LIMIT-NEXT:    br label [[BB3:%.*]]
-; NO-LIMIT:       bb2:
-; NO-LIMIT-NEXT:    br label [[BB3]]
-; NO-LIMIT:       bb3:
-; NO-LIMIT-NEXT:    store i32 0, i32* [[Q:%.*]]
-; NO-LIMIT-NEXT:    store i32 0, i32* [[R:%.*]]
-; NO-LIMIT-NEXT:    store i32 0, i32* [[P:%.*]]
-; NO-LIMIT-NEXT:    ret void
-;
-; LIMIT-0-LABEL: @test2(
-; LIMIT-0-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
-; LIMIT-0:       bb1:
-; LIMIT-0-NEXT:    br label [[BB3:%.*]]
-; LIMIT-0:       bb2:
-; LIMIT-0-NEXT:    br label [[BB3]]
-; LIMIT-0:       bb3:
-; LIMIT-0-NEXT:    store i32 0, i32* [[Q:%.*]]
-; LIMIT-0-NEXT:    store i32 0, i32* [[R:%.*]]
-; LIMIT-0-NEXT:    store i32 0, i32* [[P:%.*]]
-; LIMIT-0-NEXT:    ret void
-;
-; LIMIT-3-LABEL: @test2(
-; LIMIT-3-NEXT:    store i32 1, i32* [[P:%.*]]
-; LIMIT-3-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
-; LIMIT-3:       bb1:
-; LIMIT-3-NEXT:    br label [[BB3:%.*]]
-; LIMIT-3:       bb2:
-; LIMIT-3-NEXT:    br label [[BB3]]
-; LIMIT-3:       bb3:
-; LIMIT-3-NEXT:    store i32 0, i32* [[Q:%.*]]
-; LIMIT-3-NEXT:    store i32 0, i32* [[R:%.*]]
-; LIMIT-3-NEXT:    store i32 0, i32* [[P]]
-; LIMIT-3-NEXT:    ret void
-;
-; LIMIT-4-LABEL: @test2(
-; LIMIT-4-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
-; LIMIT-4:       bb1:
-; LIMIT-4-NEXT:    br label [[BB3:%.*]]
-; LIMIT-4:       bb2:
-; LIMIT-4-NEXT:    br label [[BB3]]
-; LIMIT-4:       bb3:
-; LIMIT-4-NEXT:    store i32 0, i32* [[Q:%.*]]
-; LIMIT-4-NEXT:    store i32 0, i32* [[R:%.*]]
-; LIMIT-4-NEXT:    store i32 0, i32* [[P:%.*]]
-; LIMIT-4-NEXT:    ret void
-;
-  store i32 1, i32* %P
-  br i1 true, label %bb1, label %bb2
-bb1:
-  br label %bb3
-bb2:
-  br label %bb3
-bb3:
-  store i32 0, i32* %Q
-  store i32 0, i32* %R
-  store i32 0, i32* %P
-  ret void
-}
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll
index c142ea6cf8deb7840ba73a834f279c965d44f262..be4574e54cb1d0f5abbda6444ab31db1e6d0ce26 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-and-memcpy.ll
@@ -59,7 +59,8 @@ define void @test17_atomic_weaker_2(i8* %P, i8* noalias %Q) nounwind ssp {
 ; Should not delete the volatile memset.
 define void @test17v(i8* %P, i8* %Q) nounwind ssp {
 ; CHECK-LABEL: @test17v(
-; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memset.p0i8.i64(i8* [[P:%.*]], i8 42, i64 8, i1 true)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[Q:%.*]], i64 12, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.memset.p0i8.i64(i8* %P, i8 42, i64 8, i1 true)
@@ -73,7 +74,8 @@ define void @test17v(i8* %P, i8* %Q) nounwind ssp {
 ; Previously this was not allowed (PR8728), also discussed in PR11763.
 define void @test18(i8* %P, i8* %Q, i8* %R) nounwind ssp {
 ; CHECK-LABEL: @test18(
-; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[R:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[R:%.*]], i64 12, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %Q, i64 12, i1 false)
@@ -83,7 +85,8 @@ define void @test18(i8* %P, i8* %Q, i8* %R) nounwind ssp {
 
 define void @test18_atomic(i8* %P, i8* %Q, i8* %R) nounwind ssp {
 ; CHECK-LABEL: @test18_atomic(
-; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[R:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[R:%.*]], i64 12, i32 1)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %P, i8* align 1 %Q, i64 12, i32 1)
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll
index d1aaab9a3d49b49977311875b8a8332c236397c1..11d155c5fc1ef8dc558f869e40f49811c3471741 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/memset-missing-debugloc.ll
@@ -1,7 +1,6 @@
 ; Test that the getelementptr generated when the dse pass determines that
 ; a memset can be shortened has the debugloc carried over from the memset.
 
-; XFAIL: *
 ; RUN: opt -S -march=native -dse -enable-dse-memoryssa < %s| FileCheck %s
 ; CHECK: bitcast [5 x i64]* %{{[a-zA-Z_][a-zA-Z0-9_]*}} to i8*, !dbg
 ; CHECK-NEXT: %{{[0-9]+}} = getelementptr inbounds i8, i8* %0, i64 32, !dbg ![[DBG:[0-9]+]]
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores-big-endian.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores-big-endian.ll
index 055f117c4b5cdfa6f7076a0165720af6388593b9..8acc29f3f62e4a099e6d5d8821c1667a28749bef 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores-big-endian.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores-big-endian.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt -dse -enable-dse-memoryssa -enable-dse-partial-store-merging -S < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores.ll
index b09904e5a27de5a7554a7b8552e91df365aa0867..feb1f31c99ddf128c8574f7a9974b0d2f72bfe34 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/merge-stores.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt -dse -enable-dse-memoryssa -enable-dse-partial-store-merging -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64"
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll
index c1f73923419633c98f28b2986f92c30fbf59e5a1..7caf48148156164c2562c81c08975372056983f2 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-captures.ll
@@ -25,6 +25,7 @@ define i8* @test_return_captures_1() {
 define i8* @test_return_captures_2() {
 ; CHECK-LABEL: @test_return_captures_2(
 ; CHECK-NEXT:    [[M:%.*]] = call i8* @malloc(i64 24)
+; CHECK-NEXT:    store i8 0, i8* [[M]]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    store i8 1, i8* [[M]]
@@ -90,6 +91,7 @@ exit:
 define i8* @test_malloc_capture_3() {
 ; CHECK-LABEL: @test_malloc_capture_3(
 ; CHECK-NEXT:    [[M:%.*]] = call i8* @malloc(i64 24)
+; CHECK-NEXT:    store i8 0, i8* [[M]]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    store i8 1, i8* [[M]]
@@ -189,6 +191,7 @@ exit:
 define i8* @test_malloc_capture_7() {
 ; CHECK-LABEL: @test_malloc_capture_7(
 ; CHECK-NEXT:    [[M:%.*]] = call i8* @malloc(i64 24)
+; CHECK-NEXT:    store i8 0, i8* [[M]]
 ; CHECK-NEXT:    call void @may_throw()
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
@@ -212,10 +215,10 @@ exit:
 define void @test_alloca_nocapture_1() {
 ; CHECK-LABEL: @test_alloca_nocapture_1(
 ; CHECK-NEXT:    [[M:%.*]] = alloca i8
+; CHECK-NEXT:    store i8 0, i8* [[M]]
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    ret void
 ;
   %m = alloca i8
@@ -237,7 +240,6 @@ define void @test_alloca_capture_1() {
 ; CHECK-NEXT:    call void @capture(i8* [[M]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    ret void
 ;
   %m = alloca i8
@@ -260,7 +262,6 @@ define void @test_alloca_capture_2(%S1* %E) {
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[F_PTR:%.*]] = getelementptr [[S1:%.*]], %S1* [[E:%.*]], i32 0, i32 0
 ; CHECK-NEXT:    store i8* [[M]], i8** [[F_PTR]]
-; CHECK-NEXT:    store i8 1, i8* [[M]]
 ; CHECK-NEXT:    ret void
 ;
   %m = alloca i8
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll
index a719e273e227217755c560834dd6836d955ea138..41d4d019cb94e6db0f61ea1d355def497e41ca0d 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-exceptions.ll
@@ -30,7 +30,6 @@ define void @test12(i32* %p) personality i32 (...)* @__CxxFrameHandler3 {
 ; CHECK-NEXT:    [[C1:%.*]] = cleanuppad within none []
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    store i32 40, i32* [[SV]]
 ; CHECK-NEXT:    ret void
 ;
 block1:
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll
index 1ec4434fb20a6dc82336c5232648f118dbcf1cac..bcf75e3860da563a9731dda3e252c07e937b12c2 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-loops.ll
@@ -111,6 +111,7 @@ define void @test_loop(i32 %N, i32* noalias nocapture readonly %A, i32* noalias
 ; CHECK:       for.body4.lr.ph:
 ; CHECK-NEXT:    [[I_028:%.*]] = phi i32 [ [[INC11:%.*]], [[FOR_COND_CLEANUP3:%.*]] ], [ 0, [[FOR_BODY4_LR_PH_PREHEADER]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 [[I_028]]
+; CHECK-NEXT:    store i32 0, i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I_028]], [[N]]
 ; CHECK-NEXT:    br label [[FOR_BODY4:%.*]]
 ; CHECK:       for.body4:
@@ -169,116 +170,3 @@ for.cond.cleanup3:                                ; preds = %for.body4
   %exitcond29 = icmp eq i32 %inc11, %N
   br i1 %exitcond29, label %for.cond.cleanup, label %for.body4.lr.ph
 }
-
-declare i1 @cond() readnone
-
-; TODO: We can eliminate the store in for.header, but we currently hit a MemoryPhi.
-define void @loop_multiple_def_uses(i32* noalias %P) {
-; CHECK-LABEL: @loop_multiple_def_uses(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_HEADER:%.*]]
-; CHECK:       for.header:
-; CHECK-NEXT:    store i32 1, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    [[C1:%.*]] = call i1 @cond()
-; CHECK-NEXT:    br i1 [[C1]], label [[FOR_BODY:%.*]], label [[END:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    store i32 1, i32* [[P]], align 4
-; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    br label [[FOR_HEADER]]
-; CHECK:       end:
-; CHECK-NEXT:    store i32 3, i32* [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  br label %for.header
-
-for.header:
-  store i32 1, i32* %P, align 4
-  %c1 = call i1 @cond()
-  br i1 %c1, label %for.body, label %end
-
-for.body:
-  store i32 1, i32* %P, align 4
-  %lv = load i32, i32* %P
-  br label %for.header
-
-end:
-  store i32 3, i32* %P, align 4
-  ret void
-}
-
-; We cannot eliminate the store in for.header, as it is only partially
-; overwritten in for.body and read afterwards.
-define void @loop_multiple_def_uses_partial_write(i32* noalias %p) {
-; CHECK-LABEL: @loop_multiple_def_uses_partial_write(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_HEADER:%.*]]
-; CHECK:       for.header:
-; CHECK-NEXT:    store i32 1239491, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    [[C1:%.*]] = call i1 @cond()
-; CHECK-NEXT:    br i1 [[C1]], label [[FOR_BODY:%.*]], label [[END:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[C:%.*]] = bitcast i32* [[P]] to i8*
-; CHECK-NEXT:    store i8 1, i8* [[C]], align 4
-; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    br label [[FOR_HEADER]]
-; CHECK:       end:
-; CHECK-NEXT:    store i32 3, i32* [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  br label %for.header
-
-for.header:
-  store i32 1239491, i32* %p, align 4
-  %c1 = call i1 @cond()
-  br i1 %c1, label %for.body, label %end
-
-for.body:
-  %c = bitcast i32* %p to i8*
-  store i8 1, i8* %c, align 4
-  %lv = load i32, i32* %p
-  br label %for.header
-
-end:
-  store i32 3, i32* %p, align 4
-  ret void
-}
-
-; We cannot eliminate the store in for.header, as the location is not overwritten
-; in for.body and read afterwards.
-define void @loop_multiple_def_uses_mayalias_write(i32* %p, i32* %q) {
-
-; CHECK-LABEL: @loop_multiple_def_uses_mayalias_write(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_HEADER:%.*]]
-; CHECK:       for.header:
-; CHECK-NEXT:    store i32 1239491, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    [[C1:%.*]] = call i1 @cond()
-; CHECK-NEXT:    br i1 [[C1]], label [[FOR_BODY:%.*]], label [[END:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    store i32 1, i32* [[Q:%.*]], align 4
-; CHECK-NEXT:    [[LV:%.*]] = load i32, i32* [[P]]
-; CHECK-NEXT:    br label [[FOR_HEADER]]
-; CHECK:       end:
-; CHECK-NEXT:    store i32 3, i32* [[P]], align 4
-; CHECK-NEXT:    ret void
-;
-entry:
-  br label %for.header
-
-for.header:
-  store i32 1239491, i32* %p, align 4
-  %c1 = call i1 @cond()
-  br i1 %c1, label %for.body, label %end
-
-for.body:
-  store i32 1, i32* %q, align 4
-  %lv = load i32, i32* %p
-  br label %for.header
-
-end:
-  store i32 3, i32* %p, align 4
-  ret void
-}
-
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll
index ce34986ff4e3b6249eea11e84a92beb6c0c82830..ddfc4d43d51a2bfda12ead2eab16f15247a967ec 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-memoryphis.ll
@@ -103,73 +103,3 @@ bb3:
   store i8 1, i8* %P2
   ret void
 }
-
-declare void @hoge()
-
-; Check a function with a MemoryPhi with 3 incoming values.
-define void @widget(i32* %Ptr, i1 %c1, i1 %c2, i32 %v1, i32 %v2, i32 %v3) {
-; CHECK-LABEL: @widget(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    tail call void @hoge()
-; CHECK-NEXT:    br i1 [[C1:%.*]], label [[BB3:%.*]], label [[BB1:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br i1 [[C2:%.*]], label [[BB2:%.*]], label [[BB3]]
-; CHECK:       bb2:
-; CHECK-NEXT:    store i32 -1, i32* [[PTR:%.*]], align 4
-; CHECK-NEXT:    br label [[BB3]]
-; CHECK:       bb3:
-; CHECK-NEXT:    br label [[BB4:%.*]]
-; CHECK:       bb4:
-; CHECK-NEXT:    switch i32 [[V1:%.*]], label [[BB8:%.*]] [
-; CHECK-NEXT:    i32 0, label [[BB5:%.*]]
-; CHECK-NEXT:    i32 1, label [[BB6:%.*]]
-; CHECK-NEXT:    i32 2, label [[BB7:%.*]]
-; CHECK-NEXT:    ]
-; CHECK:       bb5:
-; CHECK-NEXT:    store i32 0, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[BB8]]
-; CHECK:       bb6:
-; CHECK-NEXT:    store i32 1, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[BB8]]
-; CHECK:       bb7:
-; CHECK-NEXT:    store i32 2, i32* [[PTR]], align 4
-; CHECK-NEXT:    br label [[BB8]]
-; CHECK:       bb8:
-; CHECK-NEXT:    br label [[BB4]]
-;
-bb:
-  tail call void @hoge()
-  br i1 %c1, label %bb3, label %bb1
-
-bb1:                                              ; preds = %bb
-  br i1 %c2, label %bb2, label %bb3
-
-bb2:                                              ; preds = %bb1
-  store i32 -1, i32* %Ptr, align 4
-  br label %bb3
-
-bb3:                                              ; preds = %bb2, %bb1, %bb
-  br label %bb4
-
-bb4:                                              ; preds = %bb8, %bb3
-  switch i32 %v1, label %bb8 [
-  i32 0, label %bb5
-  i32 1, label %bb6
-  i32 2, label %bb7
-  ]
-
-bb5:                                              ; preds = %bb4
-  store i32 0, i32* %Ptr, align 4
-  br label %bb8
-
-bb6:                                              ; preds = %bb4
-  store i32 1, i32* %Ptr, align 4
-  br label %bb8
-
-bb7:                                              ; preds = %bb4
-  store i32 2, i32* %Ptr, align 4
-  br label %bb8
-
-bb8:                                              ; preds = %bb7, %bb6, %bb5, %bb4
-  br label %bb4
-}
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll
index cbd7dae56da1d2112b291d01cec8ce047c1d8f0a..17bd8a6fcc879a09b6425e5a2bab4df93165e07a 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-partial.ll
@@ -32,13 +32,14 @@ bb3:
 
 define void @second_store_bigger(i32* noalias %P) {
 ; CHECK-LABEL: @second_store_bigger(
+; CHECK-NEXT:    store i32 1, i32* [[P:%.*]]
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[P_I64:%.*]] = bitcast i32* [[P:%.*]] to i64*
+; CHECK-NEXT:    [[P_I64:%.*]] = bitcast i32* [[P]] to i64*
 ; CHECK-NEXT:    store i64 0, i64* [[P_I64]]
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll
index 520a6eaccf09f1baa946dc560084cb015440b99f..2b0227751b2a60685c4c96b150423cb67a0cf70e 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/multiblock-simple.ll
@@ -6,13 +6,14 @@ target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
 define void @test2(i32* noalias %P) {
 ; CHECK-LABEL: @test2(
+; CHECK-NEXT:    store i32 1, i32* [[P:%.*]]
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    store i32 0, i32* [[P:%.*]]
+; CHECK-NEXT:    store i32 0, i32* [[P]]
 ; CHECK-NEXT:    ret void
 ;
   store i32 1, i32* %P
@@ -52,6 +53,7 @@ bb3:
 
 define void @test7(i32* noalias %P, i32* noalias %Q) {
 ; CHECK-LABEL: @test7(
+; CHECK-NEXT:    store i32 1, i32* [[Q:%.*]]
 ; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[P:%.*]]
@@ -59,7 +61,7 @@ define void @test7(i32* noalias %P, i32* noalias %Q) {
 ; CHECK:       bb2:
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    store i32 0, i32* [[Q:%.*]]
+; CHECK-NEXT:    store i32 0, i32* [[Q]]
 ; CHECK-NEXT:    store i32 0, i32* [[P]]
 ; CHECK-NEXT:    ret void
 ;
@@ -112,38 +114,3 @@ bb3:
   store i32 0, i32* %P
   ret void
 }
-
-; We cannot eliminate `store i32 0, i32* %P`, as it is read by the later load.
-; Make sure that we check the uses of `store i32 1, i32* %P.1 which does not
-; alias %P. Note that uses point to the *first* def that may alias.
-define void @overlapping_read(i32* %P) {
-; CHECK-LABEL: @overlapping_read(
-; CHECK-NEXT:    store i32 0, i32* [[P:%.*]]
-; CHECK-NEXT:    [[P_1:%.*]] = getelementptr i32, i32* [[P]], i32 1
-; CHECK-NEXT:    store i32 1, i32* [[P_1]]
-; CHECK-NEXT:    [[P_64:%.*]] = bitcast i32* [[P]] to i64*
-; CHECK-NEXT:    [[LV:%.*]] = load i64, i64* [[P_64]]
-; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB2:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB3:%.*]]
-; CHECK:       bb2:
-; CHECK-NEXT:    ret void
-; CHECK:       bb3:
-; CHECK-NEXT:    store i32 2, i32* [[P]]
-; CHECK-NEXT:    ret void
-;
-  store i32 0, i32* %P
-  %P.1 = getelementptr i32, i32* %P, i32 1
-  store i32 1, i32* %P.1
-
-  %P.64 = bitcast i32* %P to i64*
-  %lv = load i64, i64* %P.64
-  br i1 true, label %bb1, label %bb2
-bb1:
-  br label %bb3
-bb2:
-  ret void
-bb3:
-  store i32 2, i32* %P
-  ret void
-}
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll
index 3a59286c53f2c44cc226a49f251e7896d35d0f56..9e88651bdb2ea27511bf2dd60c766a55b48f7e4f 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/operand-bundles.ll
@@ -1,4 +1,3 @@
-; XFAIL: *
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 
 declare noalias i8* @malloc(i64) "malloc-like"
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll
index d0a3cef71a7ea94cd4b82907cf025e806c4d5390..b96cc9f33c20b347c3e9c32de666a6dd262e59d7 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple-todo.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; XFAIL: *
 ; RUN: opt < %s -basicaa -dse -enable-dse-memoryssa -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=dse -enable-dse-memoryssa -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
@@ -10,6 +9,19 @@ declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) n
 declare void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32) nounwind
 declare void @llvm.init.trampoline(i8*, i8*, i8*)
 
+; PR8576 - Should delete store of 10 even though p/q are may aliases.
+define void @test2(i32 *%p, i32 *%q) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    store i32 20, i32* [[Q:%.*]], align 4
+; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 10, i32* %p, align 4
+  store i32 20, i32* %q, align 4
+  store i32 30, i32* %p, align 4
+  ret void
+}
+
 define void @test5(i32* %Q) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    [[A:%.*]] = load volatile i32, i32* [[Q:%.*]]
@@ -20,6 +32,62 @@ define void @test5(i32* %Q) {
   ret void
 }
 
+; Should delete store of 10 even though memset is a may-store to P (P and Q may
+; alias).
+define void @test6(i32 *%p, i8 *%q) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[Q:%.*]], i8 42, i64 900, i1 false)
+; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 10, i32* %p, align 4       ;; dead.
+  call void @llvm.memset.p0i8.i64(i8* %q, i8 42, i64 900, i1 false)
+  store i32 30, i32* %p, align 4
+  ret void
+}
+
+; Should delete store of 10 even though memset is a may-store to P (P and Q may
+; alias).
+define void @test6_atomic(i32* align 4 %p, i8* align 4 %q) {
+; CHECK-LABEL: @test6_atomic(
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[Q:%.*]], i8 42, i64 900, i32 4)
+; CHECK-NEXT:    store atomic i32 30, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 10, i32* %p unordered, align 4       ;; dead.
+  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %q, i8 42, i64 900, i32 4)
+  store atomic i32 30, i32* %p unordered, align 4
+  ret void
+}
+
+; Should delete store of 10 even though memcpy is a may-store to P (P and Q may
+; alias).
+define void @test7(i32 *%p, i8 *%q, i8* noalias %r) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[Q:%.*]], i8* [[R:%.*]], i64 900, i1 false)
+; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
+  store i32 10, i32* %p, align 4       ;; dead.
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %q, i8* %r, i64 900, i1 false)
+  store i32 30, i32* %p, align 4
+  ret void
+}
+
+; Should delete store of 10 even though memcpy is a may-store to P (P and Q may
+; alias).
+define void @test7_atomic(i32* align 4 %p, i8* align 4 %q, i8* noalias align 4 %r) {
+; CHECK-LABEL: @test7_atomic(
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 [[Q:%.*]], i8* align 4 [[R:%.*]], i64 900, i32 4)
+; CHECK-NEXT:    store atomic i32 30, i32* [[P:%.*]] unordered, align 4
+; CHECK-NEXT:    ret void
+;
+  store atomic i32 10, i32* %p unordered, align 4       ;; dead.
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 %q, i8* align 4 %r, i64 900, i32 4)
+  store atomic i32 30, i32* %p unordered, align 4
+  ret void
+}
+
 ; Do not delete stores that are only partially killed.
 define i32 @test8() {
 ; CHECK-LABEL: @test8(
@@ -90,6 +158,46 @@ define void @test12({ i32, i32 }* %x) nounwind  {
 }
 
 
+; %P doesn't escape, the DEAD instructions should be removed.
+declare void @test13f()
+define i32* @test13() {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT:    [[PTR:%.*]] = tail call i8* @malloc(i32 4)
+; CHECK-NEXT:    [[P:%.*]] = bitcast i8* [[PTR]] to i32*
+; CHECK-NEXT:    call void @test13f()
+; CHECK-NEXT:    store i32 0, i32* [[P]]
+; CHECK-NEXT:    ret i32* [[P]]
+;
+  %ptr = tail call i8* @malloc(i32 4)
+  %P = bitcast i8* %ptr to i32*
+  %DEAD = load i32, i32* %P
+  %DEAD2 = add i32 %DEAD, 1
+  store i32 %DEAD2, i32* %P
+  call void @test13f( )
+  store i32 0, i32* %P
+  ret i32* %P
+}
+
+define i32 addrspace(1)* @test13_addrspacecast() {
+; CHECK-LABEL: @test13_addrspacecast(
+; CHECK-NEXT:    [[P:%.*]] = tail call i8* @malloc(i32 4)
+; CHECK-NEXT:    [[P_BC:%.*]] = bitcast i8* [[P]] to i32*
+; CHECK-NEXT:    [[P:%.*]] = addrspacecast i32* [[P_BC]] to i32 addrspace(1)*
+; CHECK-NEXT:    call void @test13f()
+; CHECK-NEXT:    store i32 0, i32 addrspace(1)* [[P]]
+; CHECK-NEXT:    ret i32 addrspace(1)* [[P]]
+;
+  %p = tail call i8* @malloc(i32 4)
+  %p.bc = bitcast i8* %p to i32*
+  %P = addrspacecast i32* %p.bc to i32 addrspace(1)*
+  %DEAD = load i32, i32 addrspace(1)* %P
+  %DEAD2 = add i32 %DEAD, 1
+  store i32 %DEAD2, i32 addrspace(1)* %P
+  call void @test13f( )
+  store i32 0, i32 addrspace(1)* %P
+  ret i32 addrspace(1)* %P
+}
+
 declare noalias i8* @malloc(i32)
 declare noalias i8* @calloc(i32, i32)
 
@@ -133,6 +241,26 @@ define void @test22(i1 %i, i32 %k, i32 %m) nounwind {
   ret void
 }
 
+; Make sure same sized store to later element is deleted
+define void @test24([2 x i32]* %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: @test24(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[A:%.*]], i64 0, i64 0
+; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[A]], i64 0, i64 1
+; CHECK-NEXT:    store i32 [[C:%.*]], i32* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
+  %1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 0
+  store i32 0, i32* %1, align 4
+  %2 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 1
+  store i32 0, i32* %2, align 4
+  %3 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 0
+  store i32 %b, i32* %3, align 4
+  %4 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 1
+  store i32 %c, i32* %4, align 4
+  ret void
+}
+
 ; Remove redundant store if loaded value is in another block.
 define i32 @test26(i1 %c, i32* %p) {
 ; CHECK-LABEL: @test26(
@@ -183,6 +311,35 @@ bb3:
 
 declare void @unknown_func()
 
+; Don't remove redundant store because of unknown call.
+define i32 @test30(i1 %c, i32* %p, i32 %i) {
+; CHECK-LABEL: @test30(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @unknown_func()
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    store i32 [[V]], i32* [[P]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %v = load i32, i32* %p, align 4
+  br i1 %c, label %bb1, label %bb2
+bb1:
+  br label %bb3
+bb2:
+  ; Might overwrite value at %p
+  call void @unknown_func()
+  br label %bb3
+bb3:
+  store i32 %v, i32* %p, align 4
+  ret i32 0
+}
+
 ; Remove redundant store if loaded value is in another block inside a loop.
 define i32 @test31(i1 %c, i32* %p, i32 %i) {
 ; CHECK-LABEL: @test31(
diff --git a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll
index ef05504b30f80274bd802de442b8885be5fc461b..b2c66f2994596995265a0c5b08bef93ee8f37140 100644
--- a/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/MSSA/simple.ll
@@ -50,76 +50,6 @@ define void @test4(i32* %Q) {
   ret void
 }
 
-; PR8576 - Should delete store of 10 even though p/q are may aliases.
-define void @test2(i32 *%p, i32 *%q) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:    store i32 20, i32* [[Q:%.*]], align 4
-; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    ret void
-;
-  store i32 10, i32* %p, align 4
-  store i32 20, i32* %q, align 4
-  store i32 30, i32* %p, align 4
-  ret void
-}
-
-; Should delete store of 10 even though memset is a may-store to P (P and Q may
-; alias).
-define void @test6(i32 *%p, i8 *%q) {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[Q:%.*]], i8 42, i64 900, i1 false)
-; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    ret void
-;
-  store i32 10, i32* %p, align 4       ;; dead.
-  call void @llvm.memset.p0i8.i64(i8* %q, i8 42, i64 900, i1 false)
-  store i32 30, i32* %p, align 4
-  ret void
-}
-
-; Should delete store of 10 even though memset is a may-store to P (P and Q may
-; alias).
-define void @test6_atomic(i32* align 4 %p, i8* align 4 %q) {
-; CHECK-LABEL: @test6_atomic(
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 [[Q:%.*]], i8 42, i64 900, i32 4)
-; CHECK-NEXT:    store atomic i32 30, i32* [[P:%.*]] unordered, align 4
-; CHECK-NEXT:    ret void
-;
-  store atomic i32 10, i32* %p unordered, align 4       ;; dead.
-  call void @llvm.memset.element.unordered.atomic.p0i8.i64(i8* align 4 %q, i8 42, i64 900, i32 4)
-  store atomic i32 30, i32* %p unordered, align 4
-  ret void
-}
-
-; Should delete store of 10 even though memcpy is a may-store to P (P and Q may
-; alias).
-define void @test7(i32 *%p, i8 *%q, i8* noalias %r) {
-; CHECK-LABEL: @test7(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[Q:%.*]], i8* [[R:%.*]], i64 900, i1 false)
-; CHECK-NEXT:    store i32 30, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    ret void
-;
-  store i32 10, i32* %p, align 4       ;; dead.
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %q, i8* %r, i64 900, i1 false)
-  store i32 30, i32* %p, align 4
-  ret void
-}
-
-; Should delete store of 10 even though memcpy is a may-store to P (P and Q may
-; alias).
-define void @test7_atomic(i32* align 4 %p, i8* align 4 %q, i8* noalias align 4 %r) {
-; CHECK-LABEL: @test7_atomic(
-; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 [[Q:%.*]], i8* align 4 [[R:%.*]], i64 900, i32 4)
-; CHECK-NEXT:    store atomic i32 30, i32* [[P:%.*]] unordered, align 4
-; CHECK-NEXT:    ret void
-;
-  store atomic i32 10, i32* %p unordered, align 4       ;; dead.
-  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 4 %q, i8* align 4 %r, i64 900, i32 4)
-  store atomic i32 30, i32* %p unordered, align 4
-  ret void
-}
-
-
 ; va_arg has fuzzy dependence, the store shouldn't be zapped.
 define double @test10(i8* %X) {
 ; CHECK-LABEL: @test10(
@@ -134,46 +64,6 @@ define double @test10(i8* %X) {
   ret double %tmp.0
 }
 
-; %P doesn't escape, the DEAD instructions should be removed.
-declare void @test13f()
-define i32* @test13() {
-; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[PTR:%.*]] = tail call i8* @malloc(i32 4)
-; CHECK-NEXT:    [[P:%.*]] = bitcast i8* [[PTR]] to i32*
-; CHECK-NEXT:    call void @test13f()
-; CHECK-NEXT:    store i32 0, i32* [[P]]
-; CHECK-NEXT:    ret i32* [[P]]
-;
-  %ptr = tail call i8* @malloc(i32 4)
-  %P = bitcast i8* %ptr to i32*
-  %DEAD = load i32, i32* %P
-  %DEAD2 = add i32 %DEAD, 1
-  store i32 %DEAD2, i32* %P
-  call void @test13f( )
-  store i32 0, i32* %P
-  ret i32* %P
-}
-
-define i32 addrspace(1)* @test13_addrspacecast() {
-; CHECK-LABEL: @test13_addrspacecast(
-; CHECK-NEXT:    [[P:%.*]] = tail call i8* @malloc(i32 4)
-; CHECK-NEXT:    [[P_BC:%.*]] = bitcast i8* [[P]] to i32*
-; CHECK-NEXT:    [[P:%.*]] = addrspacecast i32* [[P_BC]] to i32 addrspace(1)*
-; CHECK-NEXT:    call void @test13f()
-; CHECK-NEXT:    store i32 0, i32 addrspace(1)* [[P]]
-; CHECK-NEXT:    ret i32 addrspace(1)* [[P]]
-;
-  %p = tail call i8* @malloc(i32 4)
-  %p.bc = bitcast i8* %p to i32*
-  %P = addrspacecast i32* %p.bc to i32 addrspace(1)*
-  %DEAD = load i32, i32 addrspace(1)* %P
-  %DEAD2 = add i32 %DEAD, 1
-  store i32 %DEAD2, i32 addrspace(1)* %P
-  call void @test13f( )
-  store i32 0, i32 addrspace(1)* %P
-  ret i32 addrspace(1)* %P
-}
-
 
 declare noalias i8* @malloc(i32)
 declare noalias i8* @calloc(i32, i32)
@@ -218,26 +108,6 @@ define noalias i8* @test23() nounwind uwtable ssp {
   ret i8* %call
 }
 
-; Make sure same sized store to later element is deleted
-define void @test24([2 x i32]* %a, i32 %b, i32 %c) nounwind {
-; CHECK-LABEL: @test24(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[A:%.*]], i64 0, i64 0
-; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [2 x i32], [2 x i32]* [[A]], i64 0, i64 1
-; CHECK-NEXT:    store i32 [[C:%.*]], i32* [[TMP2]], align 4
-; CHECK-NEXT:    ret void
-;
-  %1 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 0
-  store i32 0, i32* %1, align 4
-  %2 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 1
-  store i32 0, i32* %2, align 4
-  %3 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 0
-  store i32 %b, i32* %3, align 4
-  %4 = getelementptr inbounds [2 x i32], [2 x i32]* %a, i64 0, i64 1
-  store i32 %c, i32* %4, align 4
-  ret void
-}
-
 ; Check another case like PR13547 where strdup is not like malloc.
 define i8* @test25(i8* %p) nounwind {
 ; CHECK-LABEL: @test25(
@@ -317,35 +187,6 @@ bb3:
 
 declare void @unknown_func()
 
-; Don't remove redundant store because of unknown call.
-define i32 @test30(i1 %c, i32* %p, i32 %i) {
-; CHECK-LABEL: @test30(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V:%.*]] = load i32, i32* [[P:%.*]], align 4
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BB1:%.*]], label [[BB2:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    br label [[BB3:%.*]]
-; CHECK:       bb2:
-; CHECK-NEXT:    call void @unknown_func()
-; CHECK-NEXT:    br label [[BB3]]
-; CHECK:       bb3:
-; CHECK-NEXT:    store i32 [[V]], i32* [[P]], align 4
-; CHECK-NEXT:    ret i32 0
-;
-entry:
-  %v = load i32, i32* %p, align 4
-  br i1 %c, label %bb1, label %bb2
-bb1:
-  br label %bb3
-bb2:
-  ; Might overwrite value at %p
-  call void @unknown_func()
-  br label %bb3
-bb3:
-  store i32 %v, i32* %p, align 4
-  ret i32 0
-}
-
 ; Don't remove redundant store in a loop with a may-alias store.
 define i32 @test32(i1 %c, i32* %p, i32 %i) {
 ; CHECK-LABEL: @test32(
@@ -453,7 +294,8 @@ define void @test37_atomic(i8* %P, i8* %Q, i8* %R) {
 ; The memmove is dead, because memcpy arguments cannot overlap.
 define void @test38(i8* %P, i8* %Q, i8* %R) {
 ; CHECK-LABEL: @test38(
-; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[R:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[Q:%.*]], i64 12, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P]], i8* [[R:%.*]], i64 12, i1 false)
 ; CHECK-NEXT:    ret void
 ;
 
@@ -465,7 +307,8 @@ define void @test38(i8* %P, i8* %Q, i8* %R) {
 ; The memmove is dead, because memcpy arguments cannot overlap.
 define void @test38_atomic(i8* %P, i8* %Q, i8* %R) {
 ; CHECK-LABEL: @test38_atomic(
-; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[R:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P:%.*]], i8* align 1 [[Q:%.*]], i64 12, i32 1)
+; CHECK-NEXT:    tail call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 [[P]], i8* align 1 [[R:%.*]], i64 12, i32 1)
 ; CHECK-NEXT:    ret void
 ;
 
@@ -536,9 +379,7 @@ declare void @free(i8* nocapture)
 define void @test41(i32* noalias %P) {
 ; CHECK-LABEL: @test41(
 ; CHECK-NEXT:    [[P2:%.*]] = bitcast i32* [[P:%.*]] to i8*
-; CHECK-NEXT:    store i32 1, i32* [[P]]
 ; CHECK-NEXT:    call void @unknown_func()
-; CHECK-NEXT:    store i32 2, i32* [[P]]
 ; CHECK-NEXT:    call void @free(i8* [[P2]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
index b0dd5d185c77b7fd9376361aacaa3d7d162cc6fe..9f85fec33ba143be3cfc80724908e2820b9019fd 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
@@ -207,4 +207,55 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() {
   ret void
 }
 
+; Make sure we don't think the alignment will increase if the base address isn't an alloca
+; ALL-LABEL: @private_store_2xi16_align2_not_alloca(
+; ALL: store i16
+; ALL: store i16
+define void @private_store_2xi16_align2_not_alloca(i16 addrspace(5)* %p, i16 addrspace(5)* %r) #0 {
+  %gep.r = getelementptr i16, i16 addrspace(5)* %r, i32 1
+  store i16 1, i16 addrspace(5)* %r, align 2
+  store i16 2, i16 addrspace(5)* %gep.r, align 2
+  ret void
+}
+
+; ALL-LABEL: @private_store_2xi16_align1_not_alloca(
+; ALIGNED: store i16
+; ALIGNED: store i16
+; UNALIGNED: store <2 x i16>
+define void @private_store_2xi16_align1_not_alloca(i16 addrspace(5)* %p, i16 addrspace(5)* %r) #0 {
+  %gep.r = getelementptr i16, i16 addrspace(5)* %r, i32 1
+  store i16 1, i16 addrspace(5)* %r, align 1
+  store i16 2, i16 addrspace(5)* %gep.r, align 1
+  ret void
+}
+
+; ALL-LABEL: @private_load_2xi16_align2_not_alloca(
+; ALL: load i16
+; ALL: load i16
+define i32 @private_load_2xi16_align2_not_alloca(i16 addrspace(5)* %p) #0 {
+  %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(5)* %p, align 2
+  %p.1 = load i16, i16 addrspace(5)* %gep.p, align 2
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
+; ALL-LABEL: @private_load_2xi16_align1_not_alloca(
+; ALIGNED: load i16
+; ALIGNED: load i16
+; UNALIGNED: load <2 x i16>
+define i32 @private_load_2xi16_align1_not_alloca(i16 addrspace(5)* %p) #0 {
+  %gep.p = getelementptr i16, i16 addrspace(5)* %p, i64 1
+  %p.0 = load i16, i16 addrspace(5)* %p, align 1
+  %p.1 = load i16, i16 addrspace(5)* %gep.p, align 1
+  %zext.0 = zext i16 %p.0 to i32
+  %zext.1 = zext i16 %p.1 to i32
+  %shl.1 = shl i32 %zext.1, 16
+  %or = or i32 %zext.0, %shl.1
+  ret i32 %or
+}
+
 attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
index 4292cbcec850f0896e797c7f335a487cd7782762..31a1c270bd0e3d18b8ba0370009570aa1d30c241 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
@@ -57,20 +57,10 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(
 }
 
 ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32_align2(
-; ALIGNED: store i32 9, i32 addrspace(5)* %out, align 2
-; ALIGNED: store i32 1, i32 addrspace(5)* %out.gep.1, align 2
-; ALIGNED: store i32 23, i32 addrspace(5)* %out.gep.2, align 2
-; ALIGNED: store i32 19, i32 addrspace(5)* %out.gep.3, align 2
-
-; ELT16-UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 2
-
-; ELT8-UNALIGNED: store <2 x i32>
-; ELT8-UNALIGNED: store <2 x i32>
-
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
+; ALL: store i32
+; ALL: store i32
+; ALL: store i32
+; ALL: store i32
 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(i32 addrspace(5)* %out) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(5)* %out, i32 1
   %out.gep.2 = getelementptr i32, i32 addrspace(5)* %out, i32 2
@@ -127,10 +117,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(i16 add
 }
 
 ; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align2(
-; ALIGNED: store i16
-; ALIGNED: store i16
-
-; UNALIGNED: store <2 x i16> <i16 9, i16 12>, <2 x i16> addrspace(5)* %1, align 2
+; ALL: store i16
+; ALL: store i16
 define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(i16 addrspace(5)* %out) #0 {
   %out.gep.1 = getelementptr i16, i16 addrspace(5)* %out, i32 1
 
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
index 0d9a4184e718e78743f16bd4a600a4ed425e603f..8302ad9562f5276772c718f54155387019a54bb9 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
@@ -49,7 +49,8 @@ define amdgpu_kernel void @merge_global_store_2_constants_0_i16(i16 addrspace(1)
 }
 
 ; CHECK-LABEL: @merge_global_store_2_constants_i16_natural_align
-; CHECK: store <2 x i16>
+; CHECK: store i16
+; CHECK: store i16
 define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(i16 addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
 
@@ -58,8 +59,19 @@ define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(i16
   ret void
 }
 
+; CHECK-LABEL: @merge_global_store_2_constants_i16_align_1
+; CHECK: store <2 x i16>
+define amdgpu_kernel void @merge_global_store_2_constants_i16_align_1(i16 addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
+
+  store i16 123, i16 addrspace(1)* %out.gep.1, align 1
+  store i16 456, i16 addrspace(1)* %out, align 1
+  ret void
+}
+
 ; CHECK-LABEL: @merge_global_store_2_constants_half_natural_align
-; CHECK: store <2 x half>
+; CHECK: store half
+; CHECK: store half
 define amdgpu_kernel void @merge_global_store_2_constants_half_natural_align(half addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1
 
@@ -68,6 +80,16 @@ define amdgpu_kernel void @merge_global_store_2_constants_half_natural_align(hal
   ret void
 }
 
+; CHECK-LABEL: @merge_global_store_2_constants_half_align_1
+; CHECK: store <2 x half>
+define amdgpu_kernel void @merge_global_store_2_constants_half_align_1(half addrspace(1)* %out) #0 {
+  %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1
+
+  store half 2.0, half addrspace(1)* %out.gep.1, align 1
+  store half 1.0, half addrspace(1)* %out, align 1
+  ret void
+}
+
 ; CHECK-LABEL: @merge_global_store_2_constants_i32
 ; CHECK: store <2 x i32> <i32 456, i32 123>, <2 x i32> addrspace(1)* %{{[0-9]+}}, align 4
 define amdgpu_kernel void @merge_global_store_2_constants_i32(i32 addrspace(1)* %out) #0 {
diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
index 7266d8e0bb42e5e33f5571c1a1815621325555f3..bfa9c727950f35f99bf6a4307a358d7fc2b28e06 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s  -O3 -simplifycfg -keep-loops=false -mcpu=core-avx2 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix AUTO_VEC %s
 
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
 ; This test checks auto-vectorization with FP induction variable.
 ; The FP operation is not "fast" and requires "fast-math" function attribute.
 
@@ -26,29 +28,119 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    [[CAST_CRD:%.*]] = sitofp i64 [[N_VEC]] to float
 ; AUTO_VEC-NEXT:    [[TMP0:%.*]] = fmul fast float [[CAST_CRD]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[IND_END:%.*]] = fadd fast float [[TMP0]], 1.000000e+00
+; AUTO_VEC-NEXT:    [[TMP1:%.*]] = add nsw i64 [[N_VEC]], -32
+; AUTO_VEC-NEXT:    [[TMP2:%.*]] = lshr exact i64 [[TMP1]], 5
+; AUTO_VEC-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; AUTO_VEC-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP3]], 3
+; AUTO_VEC-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], 96
+; AUTO_VEC-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK_UNR_LCSSA:%.*]], label [[VECTOR_PH_NEW:%.*]]
+; AUTO_VEC:       vector.ph.new:
+; AUTO_VEC-NEXT:    [[UNROLL_ITER:%.*]] = sub nsw i64 [[TMP3]], [[XTRAITER]]
 ; AUTO_VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AUTO_VEC:       vector.body:
-; AUTO_VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; AUTO_VEC-NEXT:    [[VEC_IND:%.*]] = phi <8 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00, float 3.000000e+00, float 3.500000e+00, float 4.000000e+00, float 4.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH_NEW]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND:%.*]] = phi <8 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00, float 3.000000e+00, float 3.500000e+00, float 4.000000e+00, float 4.500000e+00>, [[VECTOR_PH_NEW]] ], [ [[VEC_IND_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[VECTOR_PH_NEW]] ], [ [[NITER_NSUB_3:%.*]], [[VECTOR_BODY]] ]
 ; AUTO_VEC-NEXT:    [[STEP_ADD:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00>
 ; AUTO_VEC-NEXT:    [[STEP_ADD5:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00>
 ; AUTO_VEC-NEXT:    [[STEP_ADD6:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01>
-; AUTO_VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDEX]]
-; AUTO_VEC-NEXT:    [[TMP2:%.*]] = bitcast float* [[TMP1]] to <8 x float>*
-; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND]], <8 x float>* [[TMP2]], align 4
-; AUTO_VEC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 8
-; AUTO_VEC-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP3]] to <8 x float>*
-; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD]], <8 x float>* [[TMP4]], align 4
-; AUTO_VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 16
+; AUTO_VEC-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[INDEX]]
 ; AUTO_VEC-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
-; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD5]], <8 x float>* [[TMP6]], align 4
-; AUTO_VEC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 24
+; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND]], <8 x float>* [[TMP6]], align 4
+; AUTO_VEC-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, float* [[TMP5]], i64 8
 ; AUTO_VEC-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP7]] to <8 x float>*
-; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD6]], <8 x float>* [[TMP8]], align 4
-; AUTO_VEC-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
-; AUTO_VEC-NEXT:    [[VEC_IND_NEXT]] = fadd fast <8 x float> [[VEC_IND]], <float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01>
-; AUTO_VEC-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AUTO_VEC-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD]], <8 x float>* [[TMP8]], align 4
+; AUTO_VEC-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP5]], i64 16
+; AUTO_VEC-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP9]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD5]], <8 x float>* [[TMP10]], align 4
+; AUTO_VEC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, float* [[TMP5]], i64 24
+; AUTO_VEC-NEXT:    [[TMP12:%.*]] = bitcast float* [[TMP11]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD6]], <8 x float>* [[TMP12]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 32
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD_1:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 2.000000e+01, float 2.000000e+01, float 2.000000e+01, float 2.000000e+01, float 2.000000e+01, float 2.000000e+01, float 2.000000e+01, float 2.000000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_1:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 2.400000e+01, float 2.400000e+01, float 2.400000e+01, float 2.400000e+01, float 2.400000e+01, float 2.400000e+01, float 2.400000e+01, float 2.400000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_1:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 2.800000e+01, float 2.800000e+01, float 2.800000e+01, float 2.800000e+01, float 2.800000e+01, float 2.800000e+01, float 2.800000e+01, float 2.800000e+01>
+; AUTO_VEC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX_NEXT]]
+; AUTO_VEC-NEXT:    [[TMP14:%.*]] = bitcast float* [[TMP13]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND_NEXT]], <8 x float>* [[TMP14]], align 4
+; AUTO_VEC-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, float* [[TMP13]], i64 8
+; AUTO_VEC-NEXT:    [[TMP16:%.*]] = bitcast float* [[TMP15]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD_1]], <8 x float>* [[TMP16]], align 4
+; AUTO_VEC-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float* [[TMP13]], i64 16
+; AUTO_VEC-NEXT:    [[TMP18:%.*]] = bitcast float* [[TMP17]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD5_1]], <8 x float>* [[TMP18]], align 4
+; AUTO_VEC-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float* [[TMP13]], i64 24
+; AUTO_VEC-NEXT:    [[TMP20:%.*]] = bitcast float* [[TMP19]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD6_1]], <8 x float>* [[TMP20]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_1:%.*]] = or i64 [[INDEX]], 64
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_1:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 3.200000e+01, float 3.200000e+01, float 3.200000e+01, float 3.200000e+01, float 3.200000e+01, float 3.200000e+01, float 3.200000e+01, float 3.200000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD_2:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 3.600000e+01, float 3.600000e+01, float 3.600000e+01, float 3.600000e+01, float 3.600000e+01, float 3.600000e+01, float 3.600000e+01, float 3.600000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_2:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 4.000000e+01, float 4.000000e+01, float 4.000000e+01, float 4.000000e+01, float 4.000000e+01, float 4.000000e+01, float 4.000000e+01, float 4.000000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_2:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 4.400000e+01, float 4.400000e+01, float 4.400000e+01, float 4.400000e+01, float 4.400000e+01, float 4.400000e+01, float 4.400000e+01, float 4.400000e+01>
+; AUTO_VEC-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX_NEXT_1]]
+; AUTO_VEC-NEXT:    [[TMP22:%.*]] = bitcast float* [[TMP21]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND_NEXT_1]], <8 x float>* [[TMP22]], align 4
+; AUTO_VEC-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, float* [[TMP21]], i64 8
+; AUTO_VEC-NEXT:    [[TMP24:%.*]] = bitcast float* [[TMP23]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD_2]], <8 x float>* [[TMP24]], align 4
+; AUTO_VEC-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, float* [[TMP21]], i64 16
+; AUTO_VEC-NEXT:    [[TMP26:%.*]] = bitcast float* [[TMP25]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD5_2]], <8 x float>* [[TMP26]], align 4
+; AUTO_VEC-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, float* [[TMP21]], i64 24
+; AUTO_VEC-NEXT:    [[TMP28:%.*]] = bitcast float* [[TMP27]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD6_2]], <8 x float>* [[TMP28]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_2:%.*]] = or i64 [[INDEX]], 96
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_2:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 4.800000e+01, float 4.800000e+01, float 4.800000e+01, float 4.800000e+01, float 4.800000e+01, float 4.800000e+01, float 4.800000e+01, float 4.800000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD_3:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 5.200000e+01, float 5.200000e+01, float 5.200000e+01, float 5.200000e+01, float 5.200000e+01, float 5.200000e+01, float 5.200000e+01, float 5.200000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_3:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 5.600000e+01, float 5.600000e+01, float 5.600000e+01, float 5.600000e+01, float 5.600000e+01, float 5.600000e+01, float 5.600000e+01, float 5.600000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_3:%.*]] = fadd fast <8 x float> [[VEC_IND]], <float 6.000000e+01, float 6.000000e+01, float 6.000000e+01, float 6.000000e+01, float 6.000000e+01, float 6.000000e+01, float 6.000000e+01, float 6.000000e+01>
+; AUTO_VEC-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX_NEXT_2]]
+; AUTO_VEC-NEXT:    [[TMP30:%.*]] = bitcast float* [[TMP29]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND_NEXT_2]], <8 x float>* [[TMP30]], align 4
+; AUTO_VEC-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, float* [[TMP29]], i64 8
+; AUTO_VEC-NEXT:    [[TMP32:%.*]] = bitcast float* [[TMP31]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD_3]], <8 x float>* [[TMP32]], align 4
+; AUTO_VEC-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, float* [[TMP29]], i64 16
+; AUTO_VEC-NEXT:    [[TMP34:%.*]] = bitcast float* [[TMP33]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD5_3]], <8 x float>* [[TMP34]], align 4
+; AUTO_VEC-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, float* [[TMP29]], i64 24
+; AUTO_VEC-NEXT:    [[TMP36:%.*]] = bitcast float* [[TMP35]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD6_3]], <8 x float>* [[TMP36]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_3]] = add i64 [[INDEX]], 128
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_3]] = fadd fast <8 x float> [[VEC_IND]], <float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01, float 6.400000e+01>
+; AUTO_VEC-NEXT:    [[NITER_NSUB_3]] = add i64 [[NITER]], -4
+; AUTO_VEC-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0
+; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop !0
+; AUTO_VEC:       middle.block.unr-lcssa:
+; AUTO_VEC-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND_UNR:%.*]] = phi <8 x float> [ <float 1.000000e+00, float 1.500000e+00, float 2.000000e+00, float 2.500000e+00, float 3.000000e+00, float 3.500000e+00, float 4.000000e+00, float 4.500000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_3]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[LCMP_MOD:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; AUTO_VEC-NEXT:    br i1 [[LCMP_MOD]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY_EPIL:%.*]]
+; AUTO_VEC:       vector.body.epil:
+; AUTO_VEC-NEXT:    [[INDEX_EPIL:%.*]] = phi i64 [ [[INDEX_NEXT_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[INDEX_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND_EPIL:%.*]] = phi <8 x float> [ [[VEC_IND_NEXT_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[VEC_IND_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AUTO_VEC-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_SUB:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[XTRAITER]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AUTO_VEC-NEXT:    [[STEP_ADD_EPIL:%.*]] = fadd fast <8 x float> [[VEC_IND_EPIL]], <float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_EPIL:%.*]] = fadd fast <8 x float> [[VEC_IND_EPIL]], <float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_EPIL:%.*]] = fadd fast <8 x float> [[VEC_IND_EPIL]], <float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01, float 1.200000e+01>
+; AUTO_VEC-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX_EPIL]]
+; AUTO_VEC-NEXT:    [[TMP38:%.*]] = bitcast float* [[TMP37]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[VEC_IND_EPIL]], <8 x float>* [[TMP38]], align 4
+; AUTO_VEC-NEXT:    [[TMP39:%.*]] = getelementptr inbounds float, float* [[TMP37]], i64 8
+; AUTO_VEC-NEXT:    [[TMP40:%.*]] = bitcast float* [[TMP39]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD_EPIL]], <8 x float>* [[TMP40]], align 4
+; AUTO_VEC-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float* [[TMP37]], i64 16
+; AUTO_VEC-NEXT:    [[TMP42:%.*]] = bitcast float* [[TMP41]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD5_EPIL]], <8 x float>* [[TMP42]], align 4
+; AUTO_VEC-NEXT:    [[TMP43:%.*]] = getelementptr inbounds float, float* [[TMP37]], i64 24
+; AUTO_VEC-NEXT:    [[TMP44:%.*]] = bitcast float* [[TMP43]] to <8 x float>*
+; AUTO_VEC-NEXT:    store <8 x float> [[STEP_ADD6_EPIL]], <8 x float>* [[TMP44]], align 4
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_EPIL]] = add i64 [[INDEX_EPIL]], 32
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_EPIL]] = fadd fast <8 x float> [[VEC_IND_EPIL]], <float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01, float 1.600000e+01>
+; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
+; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], !llvm.loop !2
 ; AUTO_VEC:       middle.block:
 ; AUTO_VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[ZEXT]]
 ; AUTO_VEC-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]]
@@ -59,8 +151,8 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 {
 ; AUTO_VEC-NEXT:    store float [[X_06]], float* [[ARRAYIDX]], align 4
 ; AUTO_VEC-NEXT:    [[CONV1]] = fadd float [[X_06]], 5.000000e-01
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; AUTO_VEC-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
-; AUTO_VEC-NEXT:    br i1 [[TMP10]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !2
+; AUTO_VEC-NEXT:    [[TMP45:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]]
+; AUTO_VEC-NEXT:    br i1 [[TMP45]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !4
 ; AUTO_VEC:       for.end:
 ; AUTO_VEC-NEXT:    ret void
 ;
@@ -167,7 +259,7 @@ define void @fp_iv_loop2(float* noalias nocapture %A, i32 %N) #1 {
 ; AUTO_VEC-NEXT:    [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
 ; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
-; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[FOR_END]], label [[FOR_BODY_EPIL]], !llvm.loop !4
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[FOR_END]], label [[FOR_BODY_EPIL]], !llvm.loop !6
 ; AUTO_VEC:       for.end:
 ; AUTO_VEC-NEXT:    ret void
 ;
@@ -207,34 +299,124 @@ define double @external_use_with_fast_math(double* %a, i64 %n) {
 ; AUTO_VEC-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
 ; AUTO_VEC-NEXT:    [[CAST_CRD:%.*]] = sitofp i64 [[N_VEC]] to double
 ; AUTO_VEC-NEXT:    [[TMP1:%.*]] = fmul fast double [[CAST_CRD]], 3.000000e+00
+; AUTO_VEC-NEXT:    [[TMP2:%.*]] = add nsw i64 [[N_VEC]], -16
+; AUTO_VEC-NEXT:    [[TMP3:%.*]] = lshr exact i64 [[TMP2]], 4
+; AUTO_VEC-NEXT:    [[TMP4:%.*]] = add nuw nsw i64 [[TMP3]], 1
+; AUTO_VEC-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP4]], 3
+; AUTO_VEC-NEXT:    [[TMP5:%.*]] = icmp ult i64 [[TMP2]], 48
+; AUTO_VEC-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK_UNR_LCSSA:%.*]], label [[VECTOR_PH_NEW:%.*]]
+; AUTO_VEC:       vector.ph.new:
+; AUTO_VEC-NEXT:    [[UNROLL_ITER:%.*]] = sub nsw i64 [[TMP4]], [[XTRAITER]]
 ; AUTO_VEC-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AUTO_VEC:       vector.body:
-; AUTO_VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; AUTO_VEC-NEXT:    [[VEC_IND:%.*]] = phi <4 x double> [ <double 0.000000e+00, double 3.000000e+00, double 6.000000e+00, double 9.000000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH_NEW]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND:%.*]] = phi <4 x double> [ <double 0.000000e+00, double 3.000000e+00, double 6.000000e+00, double 9.000000e+00>, [[VECTOR_PH_NEW]] ], [ [[VEC_IND_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[VECTOR_PH_NEW]] ], [ [[NITER_NSUB_3:%.*]], [[VECTOR_BODY]] ]
 ; AUTO_VEC-NEXT:    [[STEP_ADD:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.200000e+01, double 1.200000e+01, double 1.200000e+01, double 1.200000e+01>
 ; AUTO_VEC-NEXT:    [[STEP_ADD5:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 2.400000e+01, double 2.400000e+01, double 2.400000e+01, double 2.400000e+01>
 ; AUTO_VEC-NEXT:    [[STEP_ADD6:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 3.600000e+01, double 3.600000e+01, double 3.600000e+01, double 3.600000e+01>
-; AUTO_VEC-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[A:%.*]], i64 [[INDEX]]
-; AUTO_VEC-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <4 x double>*
-; AUTO_VEC-NEXT:    store <4 x double> [[VEC_IND]], <4 x double>* [[TMP3]], align 8
-; AUTO_VEC-NEXT:    [[TMP4:%.*]] = getelementptr double, double* [[TMP2]], i64 4
-; AUTO_VEC-NEXT:    [[TMP5:%.*]] = bitcast double* [[TMP4]] to <4 x double>*
-; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD]], <4 x double>* [[TMP5]], align 8
-; AUTO_VEC-NEXT:    [[TMP6:%.*]] = getelementptr double, double* [[TMP2]], i64 8
+; AUTO_VEC-NEXT:    [[TMP6:%.*]] = getelementptr double, double* [[A:%.*]], i64 [[INDEX]]
 ; AUTO_VEC-NEXT:    [[TMP7:%.*]] = bitcast double* [[TMP6]] to <4 x double>*
-; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD5]], <4 x double>* [[TMP7]], align 8
-; AUTO_VEC-NEXT:    [[TMP8:%.*]] = getelementptr double, double* [[TMP2]], i64 12
+; AUTO_VEC-NEXT:    store <4 x double> [[VEC_IND]], <4 x double>* [[TMP7]], align 8
+; AUTO_VEC-NEXT:    [[TMP8:%.*]] = getelementptr double, double* [[TMP6]], i64 4
 ; AUTO_VEC-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP8]] to <4 x double>*
-; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD6]], <4 x double>* [[TMP9]], align 8
-; AUTO_VEC-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
-; AUTO_VEC-NEXT:    [[VEC_IND_NEXT]] = fadd fast <4 x double> [[VEC_IND]], <double 4.800000e+01, double 4.800000e+01, double 4.800000e+01, double 4.800000e+01>
-; AUTO_VEC-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AUTO_VEC-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD]], <4 x double>* [[TMP9]], align 8
+; AUTO_VEC-NEXT:    [[TMP10:%.*]] = getelementptr double, double* [[TMP6]], i64 8
+; AUTO_VEC-NEXT:    [[TMP11:%.*]] = bitcast double* [[TMP10]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD5]], <4 x double>* [[TMP11]], align 8
+; AUTO_VEC-NEXT:    [[TMP12:%.*]] = getelementptr double, double* [[TMP6]], i64 12
+; AUTO_VEC-NEXT:    [[TMP13:%.*]] = bitcast double* [[TMP12]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD6]], <4 x double>* [[TMP13]], align 8
+; AUTO_VEC-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 16
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 4.800000e+01, double 4.800000e+01, double 4.800000e+01, double 4.800000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD_1:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 6.000000e+01, double 6.000000e+01, double 6.000000e+01, double 6.000000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_1:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 7.200000e+01, double 7.200000e+01, double 7.200000e+01, double 7.200000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_1:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 8.400000e+01, double 8.400000e+01, double 8.400000e+01, double 8.400000e+01>
+; AUTO_VEC-NEXT:    [[TMP14:%.*]] = getelementptr double, double* [[A]], i64 [[INDEX_NEXT]]
+; AUTO_VEC-NEXT:    [[TMP15:%.*]] = bitcast double* [[TMP14]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[VEC_IND_NEXT]], <4 x double>* [[TMP15]], align 8
+; AUTO_VEC-NEXT:    [[TMP16:%.*]] = getelementptr double, double* [[TMP14]], i64 4
+; AUTO_VEC-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD_1]], <4 x double>* [[TMP17]], align 8
+; AUTO_VEC-NEXT:    [[TMP18:%.*]] = getelementptr double, double* [[TMP14]], i64 8
+; AUTO_VEC-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD5_1]], <4 x double>* [[TMP19]], align 8
+; AUTO_VEC-NEXT:    [[TMP20:%.*]] = getelementptr double, double* [[TMP14]], i64 12
+; AUTO_VEC-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD6_1]], <4 x double>* [[TMP21]], align 8
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_1:%.*]] = or i64 [[INDEX]], 32
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_1:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 9.600000e+01, double 9.600000e+01, double 9.600000e+01, double 9.600000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD_2:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.080000e+02, double 1.080000e+02, double 1.080000e+02, double 1.080000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_2:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.200000e+02, double 1.200000e+02, double 1.200000e+02, double 1.200000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_2:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.320000e+02, double 1.320000e+02, double 1.320000e+02, double 1.320000e+02>
+; AUTO_VEC-NEXT:    [[TMP22:%.*]] = getelementptr double, double* [[A]], i64 [[INDEX_NEXT_1]]
+; AUTO_VEC-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[VEC_IND_NEXT_1]], <4 x double>* [[TMP23]], align 8
+; AUTO_VEC-NEXT:    [[TMP24:%.*]] = getelementptr double, double* [[TMP22]], i64 4
+; AUTO_VEC-NEXT:    [[TMP25:%.*]] = bitcast double* [[TMP24]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD_2]], <4 x double>* [[TMP25]], align 8
+; AUTO_VEC-NEXT:    [[TMP26:%.*]] = getelementptr double, double* [[TMP22]], i64 8
+; AUTO_VEC-NEXT:    [[TMP27:%.*]] = bitcast double* [[TMP26]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD5_2]], <4 x double>* [[TMP27]], align 8
+; AUTO_VEC-NEXT:    [[TMP28:%.*]] = getelementptr double, double* [[TMP22]], i64 12
+; AUTO_VEC-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD6_2]], <4 x double>* [[TMP29]], align 8
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_2:%.*]] = or i64 [[INDEX]], 48
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_2:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.440000e+02, double 1.440000e+02, double 1.440000e+02, double 1.440000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD_3:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.560000e+02, double 1.560000e+02, double 1.560000e+02, double 1.560000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_3:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.680000e+02, double 1.680000e+02, double 1.680000e+02, double 1.680000e+02>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_3:%.*]] = fadd fast <4 x double> [[VEC_IND]], <double 1.800000e+02, double 1.800000e+02, double 1.800000e+02, double 1.800000e+02>
+; AUTO_VEC-NEXT:    [[TMP30:%.*]] = getelementptr double, double* [[A]], i64 [[INDEX_NEXT_2]]
+; AUTO_VEC-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[VEC_IND_NEXT_2]], <4 x double>* [[TMP31]], align 8
+; AUTO_VEC-NEXT:    [[TMP32:%.*]] = getelementptr double, double* [[TMP30]], i64 4
+; AUTO_VEC-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD_3]], <4 x double>* [[TMP33]], align 8
+; AUTO_VEC-NEXT:    [[TMP34:%.*]] = getelementptr double, double* [[TMP30]], i64 8
+; AUTO_VEC-NEXT:    [[TMP35:%.*]] = bitcast double* [[TMP34]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD5_3]], <4 x double>* [[TMP35]], align 8
+; AUTO_VEC-NEXT:    [[TMP36:%.*]] = getelementptr double, double* [[TMP30]], i64 12
+; AUTO_VEC-NEXT:    [[TMP37:%.*]] = bitcast double* [[TMP36]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD6_3]], <4 x double>* [[TMP37]], align 8
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_3]] = add i64 [[INDEX]], 64
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_3]] = fadd fast <4 x double> [[VEC_IND]], <double 1.920000e+02, double 1.920000e+02, double 1.920000e+02, double 1.920000e+02>
+; AUTO_VEC-NEXT:    [[NITER_NSUB_3]] = add i64 [[NITER]], -4
+; AUTO_VEC-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0
+; AUTO_VEC-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop !7
+; AUTO_VEC:       middle.block.unr-lcssa:
+; AUTO_VEC-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND_UNR:%.*]] = phi <4 x double> [ <double 0.000000e+00, double 3.000000e+00, double 6.000000e+00, double 9.000000e+00>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_3]], [[VECTOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[LCMP_MOD:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; AUTO_VEC-NEXT:    br i1 [[LCMP_MOD]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY_EPIL:%.*]]
+; AUTO_VEC:       vector.body.epil:
+; AUTO_VEC-NEXT:    [[INDEX_EPIL:%.*]] = phi i64 [ [[INDEX_NEXT_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[INDEX_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AUTO_VEC-NEXT:    [[VEC_IND_EPIL:%.*]] = phi <4 x double> [ [[VEC_IND_NEXT_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[VEC_IND_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AUTO_VEC-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_SUB:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[XTRAITER]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AUTO_VEC-NEXT:    [[STEP_ADD_EPIL:%.*]] = fadd fast <4 x double> [[VEC_IND_EPIL]], <double 1.200000e+01, double 1.200000e+01, double 1.200000e+01, double 1.200000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD5_EPIL:%.*]] = fadd fast <4 x double> [[VEC_IND_EPIL]], <double 2.400000e+01, double 2.400000e+01, double 2.400000e+01, double 2.400000e+01>
+; AUTO_VEC-NEXT:    [[STEP_ADD6_EPIL:%.*]] = fadd fast <4 x double> [[VEC_IND_EPIL]], <double 3.600000e+01, double 3.600000e+01, double 3.600000e+01, double 3.600000e+01>
+; AUTO_VEC-NEXT:    [[TMP38:%.*]] = getelementptr double, double* [[A]], i64 [[INDEX_EPIL]]
+; AUTO_VEC-NEXT:    [[TMP39:%.*]] = bitcast double* [[TMP38]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[VEC_IND_EPIL]], <4 x double>* [[TMP39]], align 8
+; AUTO_VEC-NEXT:    [[TMP40:%.*]] = getelementptr double, double* [[TMP38]], i64 4
+; AUTO_VEC-NEXT:    [[TMP41:%.*]] = bitcast double* [[TMP40]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD_EPIL]], <4 x double>* [[TMP41]], align 8
+; AUTO_VEC-NEXT:    [[TMP42:%.*]] = getelementptr double, double* [[TMP38]], i64 8
+; AUTO_VEC-NEXT:    [[TMP43:%.*]] = bitcast double* [[TMP42]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD5_EPIL]], <4 x double>* [[TMP43]], align 8
+; AUTO_VEC-NEXT:    [[TMP44:%.*]] = getelementptr double, double* [[TMP38]], i64 12
+; AUTO_VEC-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <4 x double>*
+; AUTO_VEC-NEXT:    store <4 x double> [[STEP_ADD6_EPIL]], <4 x double>* [[TMP45]], align 8
+; AUTO_VEC-NEXT:    [[INDEX_NEXT_EPIL]] = add i64 [[INDEX_EPIL]], 16
+; AUTO_VEC-NEXT:    [[VEC_IND_NEXT_EPIL]] = fadd fast <4 x double> [[VEC_IND_EPIL]], <double 4.800000e+01, double 4.800000e+01, double 4.800000e+01, double 4.800000e+01>
+; AUTO_VEC-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
+; AUTO_VEC-NEXT:    [[EPIL_ITER_CMP:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
+; AUTO_VEC-NEXT:    br i1 [[EPIL_ITER_CMP]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], !llvm.loop !8
 ; AUTO_VEC:       middle.block:
 ; AUTO_VEC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; AUTO_VEC-NEXT:    [[TMP11:%.*]] = add nsw i64 [[N_VEC]], -1
-; AUTO_VEC-NEXT:    [[CAST_CMO:%.*]] = sitofp i64 [[TMP11]] to double
-; AUTO_VEC-NEXT:    [[TMP12:%.*]] = fmul fast double [[CAST_CMO]], 3.000000e+00
+; AUTO_VEC-NEXT:    [[TMP46:%.*]] = add nsw i64 [[N_VEC]], -1
+; AUTO_VEC-NEXT:    [[CAST_CMO:%.*]] = sitofp i64 [[TMP46]] to double
+; AUTO_VEC-NEXT:    [[TMP47:%.*]] = fmul fast double [[CAST_CMO]], 3.000000e+00
 ; AUTO_VEC-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; AUTO_VEC:       for.body:
 ; AUTO_VEC-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
@@ -244,9 +426,9 @@ define double @external_use_with_fast_math(double* %a, i64 %n) {
 ; AUTO_VEC-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; AUTO_VEC-NEXT:    [[J_NEXT]] = fadd fast double [[J]], 3.000000e+00
 ; AUTO_VEC-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; AUTO_VEC-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !7
+; AUTO_VEC-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !9
 ; AUTO_VEC:       for.end:
-; AUTO_VEC-NEXT:    [[J_LCSSA:%.*]] = phi double [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[J]], [[FOR_BODY]] ]
+; AUTO_VEC-NEXT:    [[J_LCSSA:%.*]] = phi double [ [[TMP47]], [[MIDDLE_BLOCK]] ], [ [[J]], [[FOR_BODY]] ]
 ; AUTO_VEC-NEXT:    ret double [[J_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
index 32d964819fee5cd8091d1abb875e5166d0c2783a..bf7c8547f271028d597b80bcf24478ec3b5d7dc1 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -10,7 +10,7 @@
 ; SUMMARY:      TypeIdMap:       
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:           
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -23,7 +23,7 @@
 ; SUMMARY-NEXT:         ResByArg:        
 ; SUMMARY-NEXT:   typeid1:
 ; SUMMARY-NEXT:     TTRes:           
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -36,7 +36,7 @@
 ; SUMMARY-NEXT:         ResByArg:        
 ; SUMMARY-NEXT:   typeid2:
 ; SUMMARY-NEXT:     TTRes:           
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl2.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl2.ll
index 63ccfb833d456016d6a72de27ebf2cb6616453e8..7c85114239cf2731ddc0c51c3e330a4820bb30a6 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl2.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl2.ll
@@ -14,7 +14,7 @@
 ; RUN:     -wholeprogramdevirt-summary-action=export -o /dev/null 2>&1 | FileCheck %s --check-prefix=MISSING-MODULE
 
 ; Check single impl devirtulation in summary
-; CHECK: typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: unsat, sizeM1BitWidth: 0), wpdResolutions: ((offset: 0, wpdRes: (kind: singleImpl, singleImplName: "_ZNK1A1fEv"))))) ; guid
+; CHECK: typeid: (name: "_ZTS1A", summary: (typeTestRes: (kind: unknown, sizeM1BitWidth: 0), wpdResolutions: ((offset: 0, wpdRes: (kind: singleImpl, singleImplName: "_ZNK1A1fEv"))))) ; guid
 
 ; MISSING-MODULE: combined summary should contain Regular LTO module
 
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
index 33ff9e1afe50f61ccf2d7f93a809085966d91cb7..861f5f6584898aa3e74961d2b15754f2ba70c09e 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-single-impl.ll
@@ -4,7 +4,7 @@
 ; SUMMARY:      TypeIdMap:
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -17,7 +17,7 @@
 ; SUMMARY-NEXT:         ResByArg:
 ; SUMMARY-NEXT:   typeid1:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -30,7 +30,7 @@
 ; SUMMARY-NEXT:         ResByArg:
 ; SUMMARY-NEXT:   typeid2:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -43,7 +43,7 @@
 ; SUMMARY-NEXT:         ResByArg:
 ; SUMMARY-NEXT:   typeid4:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll b/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll
index cb2fddd75d1d0e740caf0d58d0f0a4af4b3894b8..634eaa12196eb0f5f6a6a3dbd2d67585de96588d 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-uniform-ret-val.ll
@@ -6,7 +6,7 @@
 ; SUMMARY:      TypeIdMap:
 ; SUMMARY-NEXT:   typeid4:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll b/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll
index 0f780a3873687c0107b977038d17d205f84631fe..7b646341ece278c11b8180dbbf2c553699af6f64 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll
@@ -6,7 +6,7 @@
 ; SUMMARY:      TypeIdMap:
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -24,7 +24,7 @@
 ; SUMMARY-NEXT:             Bit:             0
 ; SUMMARY-NEXT:   typeid4:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
diff --git a/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll b/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll
index eb7b36e87dd62ba2afefdeca99d426ede65b81a1..e33abd259625a87afd09696f150fa261f4a7e09b 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/export-vcp.ll
@@ -9,7 +9,7 @@ target datalayout = "e-p:64:64"
 ; SUMMARY:      TypeIdMap:
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
@@ -29,7 +29,7 @@ target datalayout = "e-p:64:64"
 ; SUMMARY-ARM-NEXT:         Bit:             1
 ; SUMMARY-NEXT:   typeid4:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
diff --git a/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll b/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll
index 5c2be7d86296315386d2221867941d1e2697b41d..19ee68be955a0f3791f1924196e81b63cd175636 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/import-indir.ll
@@ -32,7 +32,7 @@
 ; SUMMARY-NEXT: TypeIdMap:
 ; SUMMARY-NEXT:   typeid1:
 ; SUMMARY-NEXT:     TTRes:
-; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       Kind:            Unknown
 ; SUMMARY-NEXT:       SizeM1BitWidth:  0
 ; SUMMARY-NEXT:       AlignLog2:       0
 ; SUMMARY-NEXT:       SizeM1:          0
diff --git a/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll b/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll
index 7626aba24c1ab5ac1a4f8e2a0c28810a75df5591..16f9ef822d6f3cc01b1d08d9e86cc63d156be82e 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/uniform-retval.ll
@@ -25,7 +25,7 @@ define i32 @call(i8* %obj) {
   %fptr = load i8*, i8** %fptrptr
   %fptr_casted = bitcast i8* %fptr to i32 (i8*)*
   %result = call i32 %fptr_casted(i8* %obj)
-  ; CHECK-NOT: call
+  ; CHECK-NOT: call i32 %
   ; CHECK: ret i32 123
   ret i32 %result
 }
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll.expected
index ee9582663e2aeee91c14402dcf3d21783b5de779..461a8b6a5bf6a42d8f79ee42d06b83a2af302ad8 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/argument_name_reuse.ll.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; RUN: opt < %s -S | FileCheck %s
 
 define i32 @reuse_arg_names(i32 %X, i32 %Y) {
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll.funcsig.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll.funcsig.expected
index 103b58115c19d69cf6dbc8477f2d0bfdef3e5945..d97d1b8f77bcbfe09d5c5d00d184738c815099f9 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll.funcsig.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/basic.ll.funcsig.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; Example input for update_llc_test_checks (taken from test/Transforms/InstSimplify/add.ll)
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/on_the_fly_arg_change.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/on_the_fly_arg_change.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7c54acae5cc7e44002951e4c679d1eb010d87de1
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/on_the_fly_arg_change.ll
@@ -0,0 +1,30 @@
+; RUN: opt -S < %s | FileCheck %s
+
+declare void @foo()
+
+define void @check_lines_1() {
+  ret void
+}
+
+; UTC_ARGS: --disable
+
+; A check line that would not be auto generated.
+; CHECK: define void @no_check_lines() {
+define void @no_check_lines() {
+  ret void
+}
+
+; UTC_ARGS: --enable
+
+define void @check_lines_2() {
+  ret void
+}
+
+define void @scrub() {
+  call void @foo() readnone
+  ret void
+}
+
+define i32 @signature(i32 %arg) {
+  ret i32 %arg
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/on_the_fly_arg_change.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/on_the_fly_arg_change.ll.expected
new file mode 100644
index 0000000000000000000000000000000000000000..b76f62c9cb131e8406ef2fdcc9718e955572cb36
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/on_the_fly_arg_change.ll.expected
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
+; RUN: opt -S < %s | FileCheck %s
+
+declare void @foo()
+
+define void @check_lines_1() {
+; CHECK-LABEL: define {{[^@]+}}@check_lines_1()
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+; UTC_ARGS: --disable
+
+; A check line that would not be auto generated.
+; CHECK: define void @no_check_lines() {
+define void @no_check_lines() {
+  ret void
+}
+
+; UTC_ARGS: --enable
+
+define void @check_lines_2() {
+; CHECK-LABEL: define {{[^@]+}}@check_lines_2()
+; CHECK-NEXT:    ret void
+;
+  ret void
+}
+
+define void @scrub() {
+; CHECK-LABEL: define {{[^@]+}}@scrub()
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    ret void
+;
+  call void @foo() readnone
+  ret void
+}
+
+define i32 @signature(i32 %arg) {
+; CHECK-LABEL: define {{[^@]+}}@signature
+; CHECK-SAME: (i32 [[ARG:%.*]])
+; CHECK-NEXT:    ret i32 [[ARG]]
+;
+  ret i32 %arg
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll.scrub.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll.scrub.expected
index cbea04a0971918556b6fb90d8d95d1fa267880b3..0674a8938e1f7501e7dd5b0c7faed5eb533eaf49 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll.scrub.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/scrub_attrs.ll.scrub.expected
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes
 ; RUN: opt -S < %s | FileCheck %s
 
 declare void @foo()
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/basic.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/basic.test
index 023cccc481eea047bd31eb2089e3888246e7bda3..2227ead654702fda5044ef799c62e7310f138ec7 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/basic.test
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/basic.test
@@ -8,6 +8,9 @@
 ## Also try the --function-signature flag
 # RUN: %update_test_checks %t.ll --function-signature
 # RUN: diff -u %t.ll %S/Inputs/basic.ll.funcsig.expected
-## Verify that running without the --function-signature flag removes the -SAME: lines:
+## Verify that running without the --function-signature flag does not removes
+## the -SAME: lines since the generated file will have --function-signature in
+## an UTC_ARGS: comment in the first line (from the invocation above) which is
+## added to the update invocation below.
 # RUN: %update_test_checks %t.ll
-# RUN: diff -u %t.ll %S/Inputs/basic.ll.expected
+# RUN: diff -u %t.ll %S/Inputs/basic.ll.funcsig.expected
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/on_the_fly_arg_change.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/on_the_fly_arg_change.test
new file mode 100644
index 0000000000000000000000000000000000000000..bb5f6367cd05d346a8bbf3681545aca0b48f8a0f
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/on_the_fly_arg_change.test
@@ -0,0 +1,6 @@
+# RUN: cp -f %S/Inputs/on_the_fly_arg_change.ll %t.ll
+# RUN: %update_test_checks %t.ll --function-signature --scrub-attributes
+# RUN: diff -u %t.ll %S/Inputs/on_the_fly_arg_change.ll.expected
+## Check that running the script again does not change the result:
+# RUN: %update_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/on_the_fly_arg_change.ll.expected
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/scrub_attrs.test b/llvm/test/tools/UpdateTestChecks/update_test_checks/scrub_attrs.test
index 0adfb8440b36ffdb518903920e47c9a69f729d8c..63a5c9c9aa877be4195e84f82f9406c0219e33cb 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/scrub_attrs.test
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/scrub_attrs.test
@@ -1,9 +1,9 @@
 ## scrub_attrs test checking that update_test_checks.py works correctly
 # RUN: cp -f %S/Inputs/scrub_attrs.ll %t.ll && %update_test_checks %t.ll
 # RUN: diff -u %t.ll %S/Inputs/scrub_attrs.ll.plain.expected
-## Check that running the script again does not change the result:
-# RUN: %update_test_checks %t.ll
-# RUN: diff -u %t.ll %S/Inputs/scrub_attrs.ll.plain.expected
 ## Also try the --scrub-attributes flag
 # RUN: %update_test_checks %t.ll --scrub-attributes
 # RUN: diff -u %t.ll %S/Inputs/scrub_attrs.ll.scrub.expected
+## Check that running the script again does not change the result:
+# RUN: %update_test_checks %t.ll
+# RUN: diff -u %t.ll %S/Inputs/scrub_attrs.ll.scrub.expected
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_mismatch.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_mismatch.s
index fca66237ee00d5a2aec6f1b0b8827b4242c19ae5..d5fdf5dc3d88707891eca5f029ead3634ad8497e 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_mismatch.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_mismatch.s
@@ -2,7 +2,7 @@
 # RUN: llvm-dwarfdump -debug-addr - 2> %t.warn | FileCheck %s
 # RUN: FileCheck %s -input-file %t.warn -check-prefix=WARN
 
-# WARN: .debug_addr table at offset 0x0 has address size 8 which is different from CU address size 4
+# WARN: address table at offset 0x0 has address size 8 which is different from CU address size 4
 # WARN-NOT: {{.}}
 
 # CHECK: .debug_addr contents
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_not_multiple.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_not_multiple.s
index e8835e08796270b0368814ff8ed63ae0f9311cbf..d758709c67037bb79959dc879ac9d18b4c60ef91 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_not_multiple.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_address_size_not_multiple.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_addr contents:
 # CHECK-NOT: {{.}}
-# ERR: .debug_addr table at offset 0x0 contains data of size 7 which is not a multiple of addr size 4
+# ERR: address table at offset 0x0 contains data of size 0x7 which is not a multiple of addr size 4
 # ERR-NOT: {{.}}
 
 # data size is not multiple of address_size
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf4.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf4.s
index 57e9dd3c5193cf2676d661426568b5e637cf4691..ff6364dab70c2d3e7949a9c338cd4c8332dd03e6 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf4.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf4.s
@@ -2,7 +2,6 @@
 # RUN: llvm-dwarfdump --debug-addr %t.o | FileCheck %s
 
 # CHECK: .debug_addr contents
-# CHECK-NEXT:     length = 0x00000000, version = 0x0004, addr_size = 0x04, seg_size = 0x00
 # CHECK-NEXT:     Addrs: [
 # CHECK-NEXT:     0x00000000
 # CHECK-NEXT:     0x00000001
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf64.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf64.s
index bed41952df059b0eac03a11a7019e0a504ce3bb4..6c97ae81909ded9dd0467709f71f5eff64766914 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf64.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_dwarf64.s
@@ -1,19 +1,25 @@
-# RUN: llvm-mc %s -filetype obj -triple i386-pc-linux -o - | \
-# RUN: llvm-dwarfdump -debug-addr - 2> %t.err | FileCheck %s
-# RUN: FileCheck %s -input-file %t.err -check-prefix=ERR
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o - | \
+# RUN:   llvm-dwarfdump -debug-addr - | \
+# RUN:   FileCheck %s
 
-# CHECK: .debug_addr contents:
-# CHECK-NOT: {{.}}
-# ERR: DWARF64 is not supported in .debug_addr at offset 0x0
-# ERR-NOT: {{.}}
+# CHECK:      .debug_addr contents:
+# CHECK-NEXT: Address table header:
+# CHECK-SAME: length = 0x000000000000000c,
+# CHECK-SAME: version = 0x0005,
+# CHECK-SAME: addr_size = 0x04,
+# CHECK-SAME: seg_size = 0x00
+# CHECK-NEXT: Addrs: [
+# CHECK-NEXT: 0x00000000
+# CHECK-NEXT: 0x00001000
+# CHECK-NEXT: ]
 
-# DWARF64 table
-  .section  .debug_addr,"",@progbits
-.Ldebug_addr0:
-  .long 0xffffffff # unit_length DWARF64 mark
-  .quad 12         # unit_length
-  .short 5         # version
-  .byte 3          # address_size
-  .byte 0          # segment_selector_size
-  .long 0x00000000
-  .long 0x00000001
+    .section .debug_addr,"",@progbits
+    .long 0xffffffff                    # DWARF64 mark
+    .quad .LAddr0end-.LAddr0version     # Length
+.LAddr0version:
+    .short 5                            # Version
+    .byte 4                             # Address size
+    .byte 0                             # Segment selector size
+    .long 0x00000000
+    .long 0x00001000
+.LAddr0end:
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_invalid_addr_size.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_invalid_addr_size.s
index 1ba1afebf5da8c62ac6c761467c7976f00df5655..0ad055b2753df37075cb2bf937f755ce08a5edac 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_invalid_addr_size.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_invalid_addr_size.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_addr contents:
 # CHECK-NOT: {{.}}
-# ERR: unsupported address size 3
+# ERR: unsupported address size 3 (4 and 8 are supported)
 # ERR-NOT: {{.}}
 
 # invalid addr size
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_rela.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_rela.s
new file mode 100644
index 0000000000000000000000000000000000000000..f0058b95475b2e5b01b2b3dc5c91817aa6423da7
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_rela.s
@@ -0,0 +1,28 @@
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o - | \
+# RUN:   llvm-dwarfdump -debug-addr - | \
+# RUN:   FileCheck %s
+
+## This checks that we use DWARFDataExtractor::getRelocatedAddress() to read
+## addresses of an address table. In this test, the raw data in the .debug_addr
+## section does not contain the full address, thus, it is required to resolve
+## a RELA relocation to recover the real value.
+
+# CHECK:      .debug_addr contents
+# CHECK-NEXT: length = 0x0000000c, version = 0x0005, addr_size = 0x08, seg_size = 0x00
+# CHECK-NEXT: Addrs: [
+# CHECK-NEXT: 0x000000000000002a
+# CHECK-NEXT: ]
+
+    .text
+    .space  0x2a
+.Lfoo:
+
+    .section .debug_addr,"",@progbits
+    .long   .LAddr0end-.LAddr0version   # Length
+.LAddr0version:
+    .short  5                           # Version
+    .byte   8                           # Address size
+    .byte   0                           # Segment selector size
+.LAddr0table:
+    .quad   .Lfoo
+.LAddr0end:
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_reserved_length.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_reserved_length.s
new file mode 100644
index 0000000000000000000000000000000000000000..58857808422f7e4cb10dcd0788225f035fa4f9d8
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_reserved_length.s
@@ -0,0 +1,8 @@
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o - | \
+# RUN:   llvm-dwarfdump --debug-addr - 2>&1 | \
+# RUN:   FileCheck %s --implicit-check-not=error
+
+# CHECK: error: address table at offset 0x0 has unsupported reserved unit length of value 0xfffffff0
+
+.section .debug_addr,"",@progbits
+.long 0xfffffff0
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_segment_selector.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_segment_selector.s
index 21f0322fd2ec3aaac1d48a04ef3f953ebd32560c..3200b4f2da3bdd0984403433d5a3eb8e71466548 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_segment_selector.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_segment_selector.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_addr contents:
 # CHECK-NOT: {{.}}
-# ERR: .debug_addr table at offset 0x0 has unsupported segment selector size 1
+# ERR: address table at offset 0x0 has unsupported segment selector size 1
 # ERR-NOT: {{.}}
 
 # non-zero segment_selector_size
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_small_length_field.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_small_length_field.s
index cbecb98e6becfe6a3836e3a662f9d1b9d4b3a849..3b12737ddf599e44d968040a369f591129ab7f11 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_small_length_field.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_small_length_field.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_addr contents:
 # CHECK-NOT: {{.}}
-# ERR: .debug_addr table at offset 0x0 has too small length (0x5) to contain a complete header
+# ERR: address table at offset 0x0 has a unit_length value of 0x1, which is too small to contain a complete header
 # ERR-NOT: {{.}}
 
 # too small length value
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_extended_length_field.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_extended_length_field.s
new file mode 100644
index 0000000000000000000000000000000000000000..8c2cc11de85482939e03123db7976712ed0a3af4
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_extended_length_field.s
@@ -0,0 +1,13 @@
+# RUN: llvm-mc %s -filetype obj -triple i386-pc-linux -o - | \
+# RUN: llvm-dwarfdump -debug-addr - 2> %t.err | FileCheck %s
+# RUN: FileCheck %s -input-file %t.err -check-prefix=ERR
+
+# CHECK: .debug_addr contents:
+# CHECK-NOT: {{.}}
+# ERR: section is not large enough to contain an extended length field of the address table at offset 0x0
+# ERR-NOT: {{.}}
+
+# too small section to contain an extended length field of a DWARF64 address table.
+  .section  .debug_addr,"",@progbits
+  .long 0xffffffff      # DWARF64 mark
+  .space 7              # a DWARF64 unit length field takes 8 bytes.
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_length_field.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_length_field.s
index c26bfcb02e0aee9ab84a7f025f5933250b5444d9..dcec7dad5f9e95748b2d75d3fd46ea3433162c98 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_length_field.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_length_field.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_addr contents:
 # CHECK-NOT: {{.}}
-# ERR: section is not large enough to contain a .debug_addr table length at offset 0x0
+# ERR: section is not large enough to contain an address table length at offset 0x0
 # ERR-NOT: {{.}}
 
 # too small section to contain length field
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_section.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_section.s
index facffee69fb6b2f44b38e94db54eddbda230a6cd..5bd90617e259f2f5f6f6d8b0b1b2cc76f35c4eef 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_section.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_too_small_for_section.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_addr contents:
 # CHECK-NOT: {{.}}
-# ERR: section is not large enough to contain a .debug_addr table of length 0x10 at offset 0x0
+# ERR: section is not large enough to contain an address table at offset 0x0 with a unit_length value of 0xc
 # ERR-NOT: {{.}}
 
 # too small section to contain section of given length
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_unsupported_version.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_unsupported_version.s
index f30dd8f0b979731046190c929a1d588da93ba8f7..4eeeaa7e6980273d97a664d37b6da971a9a6be0a 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_unsupported_version.s
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_unsupported_version.s
@@ -2,7 +2,8 @@
 # RUN: llvm-dwarfdump -debug-addr - 2> %t.err | FileCheck %s
 # RUN: FileCheck %s -input-file %t.err -check-prefix=ERR
 
-# ERR: version 6 of .debug_addr section at offset 0x0 is not supported
+# ERR: address table at offset 0x0 has unsupported version 6
+# ERR: address table at offset 0x20 has unsupported version 4
 # ERR-NOT: {{.}}
 
 # CHECK: .debug_addr contents
@@ -40,3 +41,13 @@
   .byte 0  # segment_selector_size
   .long 0x00000002
   .long 0x00000003
+
+	.section	.debug_addr,"",@progbits
+.Ldebug_addr2:
+  .long 12 # unit_length = .short + .byte + .byte + .long + .long
+  .short 4 # version
+  .byte 4  # address_size
+  .byte 0  # segment_selector_size
+  .long 0x00000000
+  .long 0x00000001
+
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_version_mismatch.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_version_mismatch.s
deleted file mode 100644
index e349f3386a6cdb79823265335befb2957ae99d35..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_addr_version_mismatch.s
+++ /dev/null
@@ -1,42 +0,0 @@
-# RUN: llvm-mc %s -filetype obj -triple i386-pc-linux -o - | \
-# RUN: llvm-dwarfdump -debug-addr - 2> %t.err | FileCheck %s
-# RUN: FileCheck %s -input-file %t.err -check-prefix=ERR
-
-# ERR: .debug_addr table at offset 0x0 has version 4 which is different from the version suggested by the DWARF unit header: 5
-# ERR-NOT: {{.}}
-
-# CHECK: .debug_addr contents
-# CHECK-NEXT:     length = 0x0000000c, version = 0x0005, addr_size = 0x04, seg_size = 0x00
-# CHECK-NEXT:     Addrs: [
-# CHECK-NEXT:     0x00000000
-# CHECK-NEXT:     0x00000001
-# CHECK-NEXT:     ]
-# CHECK-NOT:      {{.}}
-
-	.section	.debug_abbrev,"",@progbits
-	.byte	1                       # Abbreviation Code
-	.section	.debug_info,"",@progbits
-.Lcu_begin0:
-	.long 8	                      # Length of Unit
-	.short	5                     # DWARF version number
-  .byte 1                       # DWARF unit type
-	.byte	4                       # Address Size (in bytes)
-	.long	.debug_abbrev           # Offset Into Abbrev. Section
-
-	.section	.debug_addr,"",@progbits
-.Ldebug_addr0:
-  .long 12 # unit_length = .short + .byte + .byte + .long + .long
-  .short 4 # version
-  .byte 4  # address_size
-  .byte 0  # segment_selector_size
-  .long 0x00000000
-  .long 0x00000001
-
-	.section	.debug_addr,"",@progbits
-.Ldebug_addr1:
-  .long 12 # unit_length = .short + .byte + .byte + .long + .long
-  .short 5 # version
-  .byte 4  # address_size
-  .byte 0  # segment_selector_size
-  .long 0x00000000
-  .long 0x00000001
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test b/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test
index 73efb39cbdb3143859daa7635adbc02b2804faef..f8005d4003a66f9cec3b8a16f2d576bc91d7eba1 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_line_invalid.test
@@ -1,186 +1,186 @@
-## Test the different error cases in the debug line parsing and how they prevent
-## or don't prevent further dumping of section contents.
-
-## Show that a bad length stops parsing of the section.
-# RUN: llvm-mc -triple x86_64-pc-linux %S/Inputs/debug_line_reserved_length.s -filetype=obj -o %t-reserved.o
-# RUN: llvm-dwarfdump -debug-line %t-reserved.o 2> %t-reserved.err \
-# RUN:   | FileCheck %s --check-prefixes=FIRST,FATAL
-# RUN: FileCheck %s --input-file=%t-reserved.err --check-prefix=RESERVED
-# RUN: llvm-dwarfdump -debug-line %t-reserved.o -verbose 2> %t-reserved-verbose.err \
-# RUN:   | FileCheck %s --check-prefixes=FIRST,FATAL
-# RUN: FileCheck %s --input-file=%t-reserved-verbose.err --check-prefix=RESERVED
-
-## We only produce warnings for malformed tables after the specified unit if
-## parsing can continue.
-# RUN: llvm-dwarfdump -debug-line=0 %t-reserved.o 2> %t-reserved-off-first.err \
-# RUN:   | FileCheck %s --check-prefixes=FIRST,NOLATER
-# RUN: FileCheck %s --input-file=%t-reserved-off-first.err --check-prefix=RESERVED
-
-## Stop looking for the specified unit, if a fatally-bad prologue is detected.
-# RUN: llvm-dwarfdump -debug-line=0x4b %t-reserved.o 2> %t-reserved-off-last.err \
-# RUN:   | FileCheck %s --check-prefixes=NOFIRST,NOLATER
-# RUN: FileCheck %s --input-file=%t-reserved-off-last.err --check-prefix=RESERVED
-
-## Show that non-fatal errors do not prevent parsing the rest of the section.
-# RUN: llvm-mc -triple x86_64-pc-linux %S/Inputs/debug_line_malformed.s -filetype=obj -o %t-malformed.o
-# RUN: llvm-dwarfdump -debug-line %t-malformed.o 2> %t-malformed.err \
-# RUN:   | FileCheck %s --check-prefixes=FIRST,NONFATAL,LAST --implicit-check-not='debug_line[{{.*}}]'
-# RUN: FileCheck %s --input-file=%t-malformed.err --check-prefixes=ALL,OTHER
-# RUN: llvm-dwarfdump -debug-line %t-malformed.o -verbose 2> %t-malformed-verbose.err \
-# RUN:   | FileCheck %s --check-prefixes=FIRST,NONFATAL,LAST --implicit-check-not='debug_line[{{.*}}]'
-# RUN: FileCheck %s --input-file=%t-malformed-verbose.err --check-prefixes=ALL,OTHER
-
-## We should still produce warnings for malformed tables after the specified unit.
-# RUN: llvm-dwarfdump -debug-line=0 %t-malformed.o 2> %t-malformed-off-first.err \
-# RUN:   | FileCheck %s --check-prefixes=FIRST,NOLATER
-# RUN: FileCheck %s --input-file=%t-malformed-off-first.err --check-prefix=ALL
-
-## Don't stop looking for the later unit if non-fatal issues are found.
-# RUN: llvm-dwarfdump -debug-line=0x332 %t-malformed.o 2> %t-malformed-off-last.err \
-# RUN:   | FileCheck %s --check-prefix=LAST --implicit-check-not='debug_line[{{.*}}]'
-# RUN: FileCheck %s --input-file=%t-malformed-off-last.err --check-prefix=ALL
-
-# FIRST:       debug_line[0x00000000]
-# FIRST:       0x000000000badbeef {{.*}} end_sequence
-# NOFIRST-NOT: debug_line[0x00000000]
-# NOFIRST-NOT: 0x000000000badbeef {{.*}} end_sequence
-# NOLATER-NOT: debug_line[{{.*}}]
-# NOLATER-NOT: end_sequence
-
-## For fatal issues, the following table(s) should not be dumped.
-# FATAL:      debug_line[0x00000048]
-# FATAL-NEXT: Line table prologue
-# FATAL-NEXT: total_length: 0xfffffffe
-# FATAL-NOT:  debug_line
-
-## For non-fatal issues, the table data should be dumped.
-## Case 1: Version 0 table.
-# NONFATAL:      debug_line[0x00000048]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL-NOT:  Address
-
-## Case 2: Version 1 table.
-# NONFATAL:      debug_line[0x0000004e]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL-NOT:  Address
-
-## Case 3: Malformed directory format with no path component.
-# NONFATAL:      debug_line[0x00000054]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL-NOT:  include_directories
-# NONFATAL-NOT:  file_names
-# NONFATAL:      0x8877665544332211 {{.*}} end_sequence
-
-## Case 4: Prologue with length shorter than parsed.
-# NONFATAL:      debug_line[0x00000081]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      file_names[  2]:
-# NONFATAL-NEXT:            name: "file2"
-# NONFATAL-NEXT:       dir_index: 1
-# NONFATAL-NEXT:        mod_time: 0x00000002
-# NONFATAL-NEXT:          length: 0x00000006
-# NONFATAL:      0x1122334455667788 {{.*}} 0 end_sequence{{$}}
-
-## Case 5: Prologue with length longer than parsed.
-# NONFATAL:      debug_line[0x000000c8]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      file_names[  2]:
-# NONFATAL-NEXT:            name: "file2"
-# NONFATAL-NEXT:       dir_index: 1
-# NONFATAL-NEXT:        mod_time: 0x00000002
-# NONFATAL-NEXT:          length: 0x00000003
-# NONFATAL-NOT:  file_names
-# NONFATAL:      0x1111222233334444 {{.*}} is_stmt end_sequence
-
-## Case 6: Extended opcode with incorrect length versus expected.
-# NONFATAL:      debug_line[0x00000111]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL: 0x00000000abbadaba {{.*}} end_sequence
-# NONFATAL: 0x00000000babb1e45 {{.*}} 10 is_stmt prologue_end end_sequence{{$}}
-
-## Case 7: No end of sequence.
-# NONFATAL:      debug_line[0x0000016c]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      0x00000000deadfade {{.*}} is_stmt
-# NONFATAL-NOT:  end_sequence
-
-## Case 8: Very short prologue length for V5 (ends during parameters).
-# NONFATAL:      debug_line[0x000001b2]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      standard_opcode_lengths[DW_LNS_set_isa] = 1
-# NONFATAL-NEXT: include_directories[  0] = "/tmp"
-# NONFATAL-NEXT: file_names[  0]:
-# NONFATAL-NEXT:            name: "xyz"
-# NONFATAL:      0x0000000000000000 1 0 1 0 0 is_stmt end_sequence
-# NONFATAL:      0x0000babb1ebabb1e {{.*}} end_sequence
-
-## Case 9: V5 prologue ends during file table.
-# NONFATAL:      debug_line[0x000001ee]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      include_directories[  0] = "/tmp"
-# NONFATAL-NEXT: file_names[  0]:
-# NONFATAL-NEXT:            name: "xyz"
-# NONFATAL-NEXT:       dir_index: 1
-# NONFATAL:      0x0000000000000000 {{.*}} epilogue_begin
-# NONFATAL:      0x00000ab4acadab4a {{.*}} end_sequence
-
-## Case 10: V5 prologue ends during directory table.
-# NONFATAL:      debug_line[0x0000022f]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      include_directories[  0] = "/tmp"
-# NONFATAL-NEXT: file_names[  0]:
-# NONFATAL-NEXT:            name: "xyz"
-# NONFATAL:      0x0000000000000002 2 0 1 0 0 is_stmt{{$}}
-# NONFATAL:      0x4444333322221111 {{.*}} end_sequence
-
-## Case 11: V5 invalid MD5 hash form when there is still data to be read.
-# NONFATAL:      debug_line[0x0000026b]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      include_directories[  0] = "/tmp"
-# NONFATAL-NOT:  file_names
-# NONFATAL-NOT:  is_stmt
-# NONFATAL:      0x1234123412341234 {{.*}} end_sequence
-
-## Case 12: V5 invalid MD5 hash form when data beyond the prologue length has
-## been read before the MD5 problem is identified.
-# NONFATAL:      debug_line[0x000002ae]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      include_directories[  0] = "/tmp"
-# NONFATAL-NOT:  file_names
-# NONFATAL:      0x0000000000000000 {{.*}} epilogue_begin
-# NONFATAL:      0x4321432143214321 {{.*}} is_stmt end_sequence
-
-## Case 13: V5 invalid directory content description has unsupported form.
-# NONFATAL:      debug_line[0x000002ec]
-# NONFATAL-NEXT: Line table prologue
-# NONFATAL:      include_directories[  0] = "/foo"
-# NONFATAL-NOT:  include_directories
-# NONFATAL-NOT:  file_names
-# NONFATAL:      0xaaaabbbbccccdddd {{.*}} is_stmt end_sequence
-
-# LAST:          debug_line[0x00000332]
-# LAST:          0x00000000cafebabe {{.*}} end_sequence
-
-# RESERVED: warning: parsing line table prologue at offset 0x00000048 unsupported reserved unit length found of value 0xfffffffe
-
-# ALL-NOT:  warning:
-# ALL:      warning: parsing line table prologue at offset 0x00000048 found unsupported version 0x00
-# ALL-NEXT: warning: parsing line table prologue at offset 0x0000004e found unsupported version 0x01
-# ALL-NEXT: warning: parsing line table prologue at 0x00000054 found an invalid directory or file table description at 0x00000073
-# ALL-NEXT: warning: failed to parse entry content descriptions because no path was found
-# ALL-NEXT: warning: parsing line table prologue at 0x00000081 should have ended at 0x000000b9 but it ended at 0x000000ba
-# ALL-NEXT: warning: parsing line table prologue at 0x000000c8 should have ended at 0x00000103 but it ended at 0x00000102
-# OTHER-NEXT: warning: unexpected line op length at offset 0x00000158 expected 0x02 found 0x01
-# OTHER-NEXT: warning: unexpected line op length at offset 0x0000015c expected 0x01 found 0x02
-# OTHER-NEXT: warning: last sequence in debug line table at offset 0x0000016c is not terminated
-# ALL-NEXT: warning: parsing line table prologue at 0x000001b2 should have ended at 0x000001ce but it ended at 0x000001e1
-# ALL-NEXT: warning: parsing line table prologue at 0x000001ee should have ended at 0x00000219 but it ended at 0x00000220
-# ALL-NEXT: warning: parsing line table prologue at 0x0000022f should have ended at 0x00000251 but it ended at 0x0000025e
-# ALL-NEXT: warning: parsing line table prologue at 0x0000026b found an invalid directory or file table description at 0x0000029f
-# ALL-NEXT: warning: failed to parse file entry because the MD5 hash is invalid
-# ALL-NEXT: warning: parsing line table prologue at 0x000002ae found an invalid directory or file table description at 0x000002e0
-# ALL-NEXT: warning: failed to parse file entry because the MD5 hash is invalid
-# ALL-NEXT: warning: parsing line table prologue at 0x000002ae should have ended at 0x000002d9 but it ended at 0x000002e0
-# ALL-NEXT: warning: parsing line table prologue at 0x000002ec found an invalid directory or file table description at 0x00000315
-# ALL-NEXT: warning: failed to parse directory entry because skipping the form value failed.
-# ALL-NOT:  warning:
+## Test the different error cases in the debug line parsing and how they prevent
+## or don't prevent further dumping of section contents.
+
+## Show that a bad length stops parsing of the section.
+# RUN: llvm-mc -triple x86_64-pc-linux %S/Inputs/debug_line_reserved_length.s -filetype=obj -o %t-reserved.o
+# RUN: llvm-dwarfdump -debug-line %t-reserved.o 2> %t-reserved.err \
+# RUN:   | FileCheck %s --check-prefixes=FIRST,FATAL
+# RUN: FileCheck %s --input-file=%t-reserved.err --check-prefix=RESERVED
+# RUN: llvm-dwarfdump -debug-line %t-reserved.o -verbose 2> %t-reserved-verbose.err \
+# RUN:   | FileCheck %s --check-prefixes=FIRST,FATAL
+# RUN: FileCheck %s --input-file=%t-reserved-verbose.err --check-prefix=RESERVED
+
+## We only produce warnings for malformed tables after the specified unit if
+## parsing can continue.
+# RUN: llvm-dwarfdump -debug-line=0 %t-reserved.o 2> %t-reserved-off-first.err \
+# RUN:   | FileCheck %s --check-prefixes=FIRST,NOLATER
+# RUN: FileCheck %s --input-file=%t-reserved-off-first.err --check-prefix=RESERVED
+
+## Stop looking for the specified unit, if a fatally-bad prologue is detected.
+# RUN: llvm-dwarfdump -debug-line=0x4b %t-reserved.o 2> %t-reserved-off-last.err \
+# RUN:   | FileCheck %s --check-prefixes=NOFIRST,NOLATER
+# RUN: FileCheck %s --input-file=%t-reserved-off-last.err --check-prefix=RESERVED
+
+## Show that non-fatal errors do not prevent parsing the rest of the section.
+# RUN: llvm-mc -triple x86_64-pc-linux %S/Inputs/debug_line_malformed.s -filetype=obj -o %t-malformed.o
+# RUN: llvm-dwarfdump -debug-line %t-malformed.o 2> %t-malformed.err \
+# RUN:   | FileCheck %s --check-prefixes=FIRST,NONFATAL,LAST --implicit-check-not='debug_line[{{.*}}]'
+# RUN: FileCheck %s --input-file=%t-malformed.err --check-prefixes=ALL,OTHER
+# RUN: llvm-dwarfdump -debug-line %t-malformed.o -verbose 2> %t-malformed-verbose.err \
+# RUN:   | FileCheck %s --check-prefixes=FIRST,NONFATAL,LAST --implicit-check-not='debug_line[{{.*}}]'
+# RUN: FileCheck %s --input-file=%t-malformed-verbose.err --check-prefixes=ALL,OTHER
+
+## We should still produce warnings for malformed tables after the specified unit.
+# RUN: llvm-dwarfdump -debug-line=0 %t-malformed.o 2> %t-malformed-off-first.err \
+# RUN:   | FileCheck %s --check-prefixes=FIRST,NOLATER
+# RUN: FileCheck %s --input-file=%t-malformed-off-first.err --check-prefix=ALL
+
+## Don't stop looking for the later unit if non-fatal issues are found.
+# RUN: llvm-dwarfdump -debug-line=0x332 %t-malformed.o 2> %t-malformed-off-last.err \
+# RUN:   | FileCheck %s --check-prefix=LAST --implicit-check-not='debug_line[{{.*}}]'
+# RUN: FileCheck %s --input-file=%t-malformed-off-last.err --check-prefix=ALL
+
+# FIRST:       debug_line[0x00000000]
+# FIRST:       0x000000000badbeef {{.*}} end_sequence
+# NOFIRST-NOT: debug_line[0x00000000]
+# NOFIRST-NOT: 0x000000000badbeef {{.*}} end_sequence
+# NOLATER-NOT: debug_line[{{.*}}]
+# NOLATER-NOT: end_sequence
+
+## For fatal issues, the following table(s) should not be dumped.
+# FATAL:      debug_line[0x00000048]
+# FATAL-NEXT: Line table prologue
+# FATAL-NEXT: total_length: 0xfffffffe
+# FATAL-NOT:  debug_line
+
+## For non-fatal issues, the table data should be dumped.
+## Case 1: Version 0 table.
+# NONFATAL:      debug_line[0x00000048]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL-NOT:  Address
+
+## Case 2: Version 1 table.
+# NONFATAL:      debug_line[0x0000004e]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL-NOT:  Address
+
+## Case 3: Malformed directory format with no path component.
+# NONFATAL:      debug_line[0x00000054]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL-NOT:  include_directories
+# NONFATAL-NOT:  file_names
+# NONFATAL:      0x8877665544332211 {{.*}} end_sequence
+
+## Case 4: Prologue with length shorter than parsed.
+# NONFATAL:      debug_line[0x00000081]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      file_names[  2]:
+# NONFATAL-NEXT:            name: "file2"
+# NONFATAL-NEXT:       dir_index: 1
+# NONFATAL-NEXT:        mod_time: 0x00000002
+# NONFATAL-NEXT:          length: 0x00000006
+# NONFATAL:      0x1122334455667788 {{.*}} 0 end_sequence{{$}}
+
+## Case 5: Prologue with length longer than parsed.
+# NONFATAL:      debug_line[0x000000c8]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      file_names[  2]:
+# NONFATAL-NEXT:            name: "file2"
+# NONFATAL-NEXT:       dir_index: 1
+# NONFATAL-NEXT:        mod_time: 0x00000002
+# NONFATAL-NEXT:          length: 0x00000003
+# NONFATAL-NOT:  file_names
+# NONFATAL:      0x1111222233334444 {{.*}} is_stmt end_sequence
+
+## Case 6: Extended opcode with incorrect length versus expected.
+# NONFATAL:      debug_line[0x00000111]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL: 0x00000000abbadaba {{.*}} end_sequence
+# NONFATAL: 0x00000000babb1e45 {{.*}} 10 is_stmt prologue_end end_sequence{{$}}
+
+## Case 7: No end of sequence.
+# NONFATAL:      debug_line[0x0000016c]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      0x00000000deadfade {{.*}} is_stmt
+# NONFATAL-NOT:  end_sequence
+
+## Case 8: Very short prologue length for V5 (ends during parameters).
+# NONFATAL:      debug_line[0x000001b2]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      standard_opcode_lengths[DW_LNS_set_isa] = 1
+# NONFATAL-NEXT: include_directories[  0] = "/tmp"
+# NONFATAL-NEXT: file_names[  0]:
+# NONFATAL-NEXT:            name: "xyz"
+# NONFATAL:      0x0000000000000000 1 0 1 0 0 is_stmt end_sequence
+# NONFATAL:      0x0000babb1ebabb1e {{.*}} end_sequence
+
+## Case 9: V5 prologue ends during file table.
+# NONFATAL:      debug_line[0x000001ee]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      include_directories[  0] = "/tmp"
+# NONFATAL-NEXT: file_names[  0]:
+# NONFATAL-NEXT:            name: "xyz"
+# NONFATAL-NEXT:       dir_index: 1
+# NONFATAL:      0x0000000000000000 {{.*}} epilogue_begin
+# NONFATAL:      0x00000ab4acadab4a {{.*}} end_sequence
+
+## Case 10: V5 prologue ends during directory table.
+# NONFATAL:      debug_line[0x0000022f]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      include_directories[  0] = "/tmp"
+# NONFATAL-NEXT: file_names[  0]:
+# NONFATAL-NEXT:            name: "xyz"
+# NONFATAL:      0x0000000000000002 2 0 1 0 0 is_stmt{{$}}
+# NONFATAL:      0x4444333322221111 {{.*}} end_sequence
+
+## Case 11: V5 invalid MD5 hash form when there is still data to be read.
+# NONFATAL:      debug_line[0x0000026b]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      include_directories[  0] = "/tmp"
+# NONFATAL-NOT:  file_names
+# NONFATAL-NOT:  is_stmt
+# NONFATAL:      0x1234123412341234 {{.*}} end_sequence
+
+## Case 12: V5 invalid MD5 hash form when data beyond the prologue length has
+## been read before the MD5 problem is identified.
+# NONFATAL:      debug_line[0x000002ae]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      include_directories[  0] = "/tmp"
+# NONFATAL-NOT:  file_names
+# NONFATAL:      0x0000000000000000 {{.*}} epilogue_begin
+# NONFATAL:      0x4321432143214321 {{.*}} is_stmt end_sequence
+
+## Case 13: V5 invalid directory content description has unsupported form.
+# NONFATAL:      debug_line[0x000002ec]
+# NONFATAL-NEXT: Line table prologue
+# NONFATAL:      include_directories[  0] = "/foo"
+# NONFATAL-NOT:  include_directories
+# NONFATAL-NOT:  file_names
+# NONFATAL:      0xaaaabbbbccccdddd {{.*}} is_stmt end_sequence
+
+# LAST:          debug_line[0x00000332]
+# LAST:          0x00000000cafebabe {{.*}} end_sequence
+
+# RESERVED: warning: parsing line table prologue at offset 0x00000048 unsupported reserved unit length found of value 0xfffffffe
+
+# ALL-NOT:  warning:
+# ALL:      warning: parsing line table prologue at offset 0x00000048 found unsupported version 0x00
+# ALL-NEXT: warning: parsing line table prologue at offset 0x0000004e found unsupported version 0x01
+# ALL-NEXT: warning: parsing line table prologue at 0x00000054 found an invalid directory or file table description at 0x00000073
+# ALL-NEXT: warning: failed to parse entry content descriptions because no path was found
+# ALL-NEXT: warning: parsing line table prologue at 0x00000081 should have ended at 0x000000b9 but it ended at 0x000000ba
+# ALL-NEXT: warning: parsing line table prologue at 0x000000c8 should have ended at 0x00000103 but it ended at 0x00000102
+# OTHER-NEXT: warning: unexpected line op length at offset 0x00000158 expected 0x02 found 0x01
+# OTHER-NEXT: warning: unexpected line op length at offset 0x0000015c expected 0x01 found 0x02
+# OTHER-NEXT: warning: last sequence in debug line table at offset 0x0000016c is not terminated
+# ALL-NEXT: warning: parsing line table prologue at 0x000001b2 should have ended at 0x000001ce but it ended at 0x000001e1
+# ALL-NEXT: warning: parsing line table prologue at 0x000001ee should have ended at 0x00000219 but it ended at 0x00000220
+# ALL-NEXT: warning: parsing line table prologue at 0x0000022f should have ended at 0x00000251 but it ended at 0x0000025e
+# ALL-NEXT: warning: parsing line table prologue at 0x0000026b found an invalid directory or file table description at 0x0000029f
+# ALL-NEXT: warning: failed to parse file entry because the MD5 hash is invalid
+# ALL-NEXT: warning: parsing line table prologue at 0x000002ae found an invalid directory or file table description at 0x000002e0
+# ALL-NEXT: warning: failed to parse file entry because the MD5 hash is invalid
+# ALL-NEXT: warning: parsing line table prologue at 0x000002ae should have ended at 0x000002d9 but it ended at 0x000002e0
+# ALL-NEXT: warning: parsing line table prologue at 0x000002ec found an invalid directory or file table description at 0x00000315
+# ALL-NEXT: warning: failed to parse directory entry because skipping the form value failed.
+# ALL-NOT:  warning:
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll b/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
index 7521806c624ff84eb6fbe749752d6c716dbfcd58..ca9140f5fabd00db842073243995ab9a5e965ae0 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
@@ -1,4 +1,4 @@
-; RUN: llc -debug-entry-values %s -o - -filetype=obj \
+; RUN: llc %s -o - -filetype=obj \
 ; RUN:   | llvm-dwarfdump -statistics - | FileCheck %s
 ;
 ; CHECK: "entry value scope bytes covered":5
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll b/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll
index c304e9d768a5a86a38fdd7e663ad6b5507ebdca7..8601dc3595f702497a8ea4b46ca845beee79ca40 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/stats-dbg-callsite-info.ll
@@ -1,9 +1,6 @@
-; RUN: llc -debug-entry-values %s -o - -filetype=obj \
+; RUN: llc %s -o - -filetype=obj \
 ; RUN:   | llvm-dwarfdump -statistics - | FileCheck %s
 ;
-; The LLVM IR file was generated on this source code by using
-; option '-femit-debug-entry-values'.
-;
 ; extern void foo(int *a, int b, int c, int d, int e, int f);
 ; extern int getVal();
 ;
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll b/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll
index 935f3a2d1d35331c68feba857b454e34d4fb053e..acb875bd0f135a63f3f599927b5bd6770d4ebc9d 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/valid-call-site-GNU-extensions.ll
@@ -1,4 +1,4 @@
-; RUN: llc -debug-entry-values %s -o - -filetype=obj \
+; RUN: llc %s -o - -filetype=obj \
 ; RUN:   | llvm-dwarfdump -verify - | FileCheck %s
 ;
 ; CHECK: No errors.
diff --git a/llvm/test/tools/llvm-locstats/locstats.ll b/llvm/test/tools/llvm-locstats/locstats.ll
index f16635d2e8e42ac863f92ca7908ee0dd06d464ef..fd28679f3ec133220ce2be6caa86c2db75534431 100644
--- a/llvm/test/tools/llvm-locstats/locstats.ll
+++ b/llvm/test/tools/llvm-locstats/locstats.ll
@@ -9,9 +9,9 @@
 ; LOCSTATS: [10%,20%) 0 0%
 ; LOCSTATS: [20%,30%) 1 11%
 ; LOCSTATS: [30%,40%) 0 0%
-; LOCSTATS: [40%,50%) 1 11%
-; LOCSTATS: [50%,60%) 1 11%
-; LOCSTATS: [60%,70%) 1 11%
+; LOCSTATS: [40%,50%) 0 0%
+; LOCSTATS: [50%,60%) 0 0%
+; LOCSTATS: [60%,70%) 3 33%
 ; LOCSTATS: [70%,80%) 0 0%
 ; LOCSTATS: [80%,90%) 2 22%
 ; LOCSTATS: [90%,100%) 1 11%
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx1.s
index 62816f790cc24bcd05bc2c2bccaa246038cf2edc..210f18c132322efd3f7cbfdb0411f6e07d8058d6 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx1.s
@@ -1477,10 +1477,10 @@ vzeroupper
 # CHECK-NEXT:  2      6     1.00    *                   vpinsrq	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      2     2.00                        vpinsrw	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   vpinsrw	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddubsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddwd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmaddwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmaddwd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpmaxsb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpmaxsb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpmaxsd	%xmm0, %xmm1, %xmm2
@@ -1530,20 +1530,20 @@ vzeroupper
 # CHECK-NEXT:  2      6     1.00    *                   vpmovzxwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpmovzxwq	%xmm0, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   vpmovzxwq	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmuldq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmuldq	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhrsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmulhrsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhuw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmulhuw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmulhw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmuldq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmuldq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhrsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmulhrsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmulhuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmulhw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      10    1.00                        vpmulld	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmullw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmullw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmuludq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmuludq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmullw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmullw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmuludq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmuludq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.33                        vpor	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpor	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      3     1.00                        vpsadbw	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s
index 9f6b518da9d8da920bb5a69feb8b4d855923720f..8072a061135550ebf5eea9418f195756c08dc91c 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s
@@ -588,10 +588,10 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  4      10    2.00    *                   vphsubsw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  3      3     2.00                        vphsubw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  4      10    2.00    *                   vphsubw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddubsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmaddubsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddwd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmaddwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmaddubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmaddwd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      7     0.50    *                   vpmaskmovd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  2      8     0.50    *                   vpmaskmovd	(%rax), %ymm0, %ymm2
 # CHECK-NEXT:  2      2     1.00    *      *            vpmaskmovd	%xmm0, %xmm1, (%rax)
@@ -649,20 +649,20 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      9     1.00    *                   vpmovzxwd	(%rax), %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpmovzxwq	%xmm0, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpmovzxwq	(%rax), %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmuldq	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmuldq	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhrsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmulhrsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhuw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmulhuw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmulhw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmuldq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmuldq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhrsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmulhrsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmulhuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmulhw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00                        vpmulld	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmullw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmullw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmuludq	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmuludq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmullw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmullw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmuludq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmuludq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.33                        vpor	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpor	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpsadbw	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-mmx.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-mmx.s
index 8da7071f0b7869c7c399a33af7b31c7140d6ab31..75c49be784eba47ed58fafe709916be8a0439951 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-mmx.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-mmx.s
@@ -209,12 +209,12 @@ pxor        (%rax), %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pcmpgtd	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pcmpgtw	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pcmpgtw	(%rax), %mm2
-# CHECK-NEXT:  1      4     1.00                        pmaddwd	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmaddwd	(%rax), %mm2
-# CHECK-NEXT:  1      4     1.00                        pmulhw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmulhw	(%rax), %mm2
-# CHECK-NEXT:  1      4     1.00                        pmullw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmullw	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmaddwd	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmaddwd	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmulhw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmulhw	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmullw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmullw	(%rax), %mm2
 # CHECK-NEXT:  1      1     0.50                        por	%mm0, %mm2
 # CHECK-NEXT:  2      6     0.50    *                   por	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pslld	$1, %mm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s
index 8a2fb55ab734bbc59388c2b4091258b3bdf261e1..11557412ea515986b9e643f20cd75b4f7b176d5d 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse1.s
@@ -280,8 +280,8 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pminub	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pminub	(%rax), %mm2
 # CHECK-NEXT:  1      2     1.00                        pmovmskb	%mm0, %ecx
-# CHECK-NEXT:  1      4     1.00                        pmulhuw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmulhuw	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmulhuw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmulhuw	(%rax), %mm2
 # CHECK-NEXT:  1      5     0.50    *      *            prefetcht0	(%rax)
 # CHECK-NEXT:  1      5     0.50    *      *            prefetcht1	(%rax)
 # CHECK-NEXT:  1      5     0.50    *      *            prefetcht2	(%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
index 869a2a4a1c78e0be6b3b9d3a01e523f22c8ce1d5..7b31b045bc1010452e40ba40d4866f94a43c4b0b 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
@@ -563,8 +563,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
 # CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
 # CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
-# CHECK-NEXT:  1      4     0.50                        pmaddwd	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmaddwd	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmaddwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pmaxsw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxub	%xmm0, %xmm2
@@ -574,16 +574,16 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pminub	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pminub	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        pmovmskb	%xmm0, %ecx
-# CHECK-NEXT:  1      4     0.50                        pmulhuw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmulhuw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        pmulhw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmulhw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        pmullw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmullw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     1.00                        pmuludq	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmuludq	(%rax), %mm2
-# CHECK-NEXT:  1      4     0.50                        pmuludq	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmuludq	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmulhuw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmulhuw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmulhw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmulhw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmullw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmullw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        pmuludq	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmuludq	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmuludq	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmuludq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.33                        por	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   por	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        psadbw	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse41.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse41.s
index 5789de04c3d9d8aa1a05e0a1b840213c34482bd8..6e11bb6df8e2915c074c30b48e535629a354db6c 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse41.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-sse41.s
@@ -237,8 +237,8 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   pmovzxwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pmovzxwq	%xmm0, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   pmovzxwq	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        pmuldq	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmuldq	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmuldq	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmuldq	(%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00                        pmulld	%xmm0, %xmm2
 # CHECK-NEXT:  3      16    1.00    *                   pmulld	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        ptest	%xmm0, %xmm1
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-ssse3.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-ssse3.s
index a0e2de6e923af5f1a2f3ebafb6dbe75af4f049a7..047974f8a6c711243ac2f098e55573c111930c06 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-ssse3.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/resources-ssse3.s
@@ -146,14 +146,14 @@ psignw      (%rax), %xmm2
 # CHECK-NEXT:  4      8     2.00    *                   phsubw	(%rax), %mm2
 # CHECK-NEXT:  3      3     2.00                        phsubw	%xmm0, %xmm2
 # CHECK-NEXT:  4      9     2.00    *                   phsubw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     1.00                        pmaddubsw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmaddubsw	(%rax), %mm2
-# CHECK-NEXT:  1      4     0.50                        pmaddubsw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmaddubsw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     1.00                        pmulhrsw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmulhrsw	(%rax), %mm2
-# CHECK-NEXT:  1      4     0.50                        pmulhrsw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmulhrsw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        pmaddubsw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmaddubsw	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmaddubsw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmaddubsw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        pmulhrsw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmulhrsw	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmulhrsw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmulhrsw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pshufb	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pshufb	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pshufb	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s
index 97eb3ae44eabe51e12b9f1e66d45550356ac6adb..5a8671922f6c00b7d7acf9ad42485c1c3260480a 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx1.s
@@ -1477,10 +1477,10 @@ vzeroupper
 # CHECK-NEXT:  2      6     1.00    *                   vpinsrq	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      2     2.00                        vpinsrw	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   vpinsrw	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddubsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddwd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmaddwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmaddwd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpmaxsb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpmaxsb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpmaxsd	%xmm0, %xmm1, %xmm2
@@ -1530,20 +1530,20 @@ vzeroupper
 # CHECK-NEXT:  2      6     1.00    *                   vpmovzxwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        vpmovzxwq	%xmm0, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   vpmovzxwq	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmuldq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmuldq	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhrsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmulhrsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhuw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmulhuw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmulhw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmuldq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmuldq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhrsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmulhrsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmulhuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmulhw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      10    1.00                        vpmulld	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmullw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmullw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      4     0.50                        vpmuludq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   vpmuludq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmullw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmullw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vpmuludq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vpmuludq	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.33                        vpor	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   vpor	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      3     1.00                        vpsadbw	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s
index f9c51a18ff7d1a34bb36cf9c6baec13f1ec87b4d..546ff0c9dc67da1e3be970e13dbc28786f80dd69 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s
@@ -588,10 +588,10 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  4      10    2.00    *                   vphsubsw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  3      3     2.00                        vphsubw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  4      10    2.00    *                   vphsubw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddubsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmaddubsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmaddwd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmaddwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmaddubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmaddwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmaddwd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      7     0.50    *                   vpmaskmovd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  2      8     0.50    *                   vpmaskmovd	(%rax), %ymm0, %ymm2
 # CHECK-NEXT:  2      2     1.00    *      *            vpmaskmovd	%xmm0, %xmm1, (%rax)
@@ -649,20 +649,20 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      9     1.00    *                   vpmovzxwd	(%rax), %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpmovzxwq	%xmm0, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpmovzxwq	(%rax), %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmuldq	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmuldq	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhrsw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmulhrsw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhuw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmulhuw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmulhw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmulhw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmuldq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmuldq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhrsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmulhrsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmulhuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmulhw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmulhw	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00                        vpmulld	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmullw	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmullw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      4     0.50                        vpmuludq	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      11    0.50    *                   vpmuludq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmullw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmullw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      5     0.50                        vpmuludq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vpmuludq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.33                        vpor	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpor	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpsadbw	%ymm0, %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-mmx.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-mmx.s
index 91c550c00bf4f90e75249052b58bf1ebf965b6e9..befbe68689ff9b86a4f82da45a92070466ca2d17 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-mmx.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-mmx.s
@@ -209,12 +209,12 @@ pxor        (%rax), %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pcmpgtd	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pcmpgtw	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pcmpgtw	(%rax), %mm2
-# CHECK-NEXT:  1      4     1.00                        pmaddwd	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmaddwd	(%rax), %mm2
-# CHECK-NEXT:  1      4     1.00                        pmulhw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmulhw	(%rax), %mm2
-# CHECK-NEXT:  1      4     1.00                        pmullw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmullw	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmaddwd	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmaddwd	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmulhw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmulhw	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmullw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmullw	(%rax), %mm2
 # CHECK-NEXT:  1      1     0.50                        por	%mm0, %mm2
 # CHECK-NEXT:  2      6     0.50    *                   por	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pslld	$1, %mm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s
index 7ce873567f51b2cf1690c0592ffd84c360f732f1..9754746949ef44f3c4f7c9cf1e54f5b3ba432494 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse1.s
@@ -280,8 +280,8 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pminub	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pminub	(%rax), %mm2
 # CHECK-NEXT:  1      2     1.00                        pmovmskb	%mm0, %ecx
-# CHECK-NEXT:  1      4     1.00                        pmulhuw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmulhuw	(%rax), %mm2
+# CHECK-NEXT:  1      5     1.00                        pmulhuw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmulhuw	(%rax), %mm2
 # CHECK-NEXT:  1      5     0.50    *      *            prefetcht0	(%rax)
 # CHECK-NEXT:  1      5     0.50    *      *            prefetcht1	(%rax)
 # CHECK-NEXT:  1      5     0.50    *      *            prefetcht2	(%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
index 27ecebd9589a28eb708d6170d55648d40d473dbb..6a09e157a2ff2c7d31d7b3b492e808f4e247b5f3 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
@@ -563,8 +563,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
 # CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
 # CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
-# CHECK-NEXT:  1      4     0.50                        pmaddwd	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmaddwd	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmaddwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pmaxsw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxub	%xmm0, %xmm2
@@ -574,16 +574,16 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pminub	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pminub	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        pmovmskb	%xmm0, %ecx
-# CHECK-NEXT:  1      4     0.50                        pmulhuw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmulhuw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        pmulhw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmulhw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        pmullw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmullw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     1.00                        pmuludq	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmuludq	(%rax), %mm2
-# CHECK-NEXT:  1      4     0.50                        pmuludq	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmuludq	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmulhuw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmulhuw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmulhw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmulhw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmullw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmullw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        pmuludq	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmuludq	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmuludq	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmuludq	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.33                        por	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   por	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        psadbw	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse41.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse41.s
index 9c4ccd4987a029bdaffe641538b9fed438493196..15cd09bf7e9768eee53bc24c40a82a6fb9f2af09 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse41.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-sse41.s
@@ -237,8 +237,8 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   pmovzxwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pmovzxwq	%xmm0, %xmm2
 # CHECK-NEXT:  2      6     1.00    *                   pmovzxwq	(%rax), %xmm2
-# CHECK-NEXT:  1      4     0.50                        pmuldq	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmuldq	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmuldq	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmuldq	(%rax), %xmm2
 # CHECK-NEXT:  2      10    1.00                        pmulld	%xmm0, %xmm2
 # CHECK-NEXT:  3      16    1.00    *                   pmulld	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        ptest	%xmm0, %xmm1
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-ssse3.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-ssse3.s
index bc28ab550b098740490d7dedce81a02f1f432543..c8b5cbb95245170da9b360c6bef6e8b33fee754f 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-ssse3.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/resources-ssse3.s
@@ -146,14 +146,14 @@ psignw      (%rax), %xmm2
 # CHECK-NEXT:  4      8     2.00    *                   phsubw	(%rax), %mm2
 # CHECK-NEXT:  3      3     2.00                        phsubw	%xmm0, %xmm2
 # CHECK-NEXT:  4      9     2.00    *                   phsubw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     1.00                        pmaddubsw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmaddubsw	(%rax), %mm2
-# CHECK-NEXT:  1      4     0.50                        pmaddubsw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmaddubsw	(%rax), %xmm2
-# CHECK-NEXT:  1      4     1.00                        pmulhrsw	%mm0, %mm2
-# CHECK-NEXT:  2      9     1.00    *                   pmulhrsw	(%rax), %mm2
-# CHECK-NEXT:  1      4     0.50                        pmulhrsw	%xmm0, %xmm2
-# CHECK-NEXT:  2      10    0.50    *                   pmulhrsw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        pmaddubsw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmaddubsw	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmaddubsw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmaddubsw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        pmulhrsw	%mm0, %mm2
+# CHECK-NEXT:  2      10    1.00    *                   pmulhrsw	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmulhrsw	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   pmulhrsw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pshufb	%mm0, %mm2
 # CHECK-NEXT:  2      6     1.00    *                   pshufb	(%rax), %mm2
 # CHECK-NEXT:  1      1     1.00                        pshufb	%xmm0, %xmm2
diff --git a/llvm/test/tools/llvm-objcopy/wasm/add-section.test b/llvm/test/tools/llvm-objcopy/wasm/add-section.test
new file mode 100644
index 0000000000000000000000000000000000000000..2f32eaca0ac46225184b99cc1943eb5a0fed7376
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/wasm/add-section.test
@@ -0,0 +1,83 @@
+## Test --add-section. This test dumps and removes the section first and checks
+## that adding it back doesn't change the result.
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy --dump-section=producers=%t.sec --remove-section=producers %t %t2
+# RUN: llvm-objcopy --add-section=producers=%t.sec %t2 %t3
+# RUN: obj2yaml %t3 | FileCheck %s
+
+## Check that the producers section has been added back unchanged.
+# CHECK:      Name: producers
+# CHECK-NEXT: Tools:
+# CHECK-NEXT:  - Name:    clang
+# CHECK-NEXT:    Version: 9.0.0
+
+# Check that the section is replaced with new content in one invocation.
+# RUN: echo "123" > %t4
+# RUN: llvm-objcopy --remove-section=foo --add-section=foo=%t4 %t %t5
+# RUN: obj2yaml %t5 | FileCheck %s --check-prefix=REPLACE
+
+# REPLACE: - Type:  CUSTOM
+# REPLACE:   Name:    foo
+# REPLACE:   Payload: 3132330A
+
+--- !WASM
+FileHeader:
+  Version: 0x00000001
+Sections:
+  - Type: TYPE
+    Signatures:
+      - Index: 0
+        ParamTypes:
+          - I32
+        ReturnTypes:
+          - F32
+      - Index: 1
+        ParamTypes:
+          - I32
+          - I64
+        ReturnTypes: []
+  - Type: FUNCTION
+    FunctionTypes:
+      - 0
+      - 1
+  - Type: CODE
+    Relocations:
+      - Type: R_WASM_TABLE_INDEX_SLEB
+        Index: 0
+        Offset: 0x00000000
+      - Type: R_WASM_FUNCTION_INDEX_LEB
+        Index: 1
+        Offset: 0x0000000
+    Functions:
+      - Index: 0
+        Locals:
+         - Type: I32
+           Count: 3
+        Body: 010101010B
+      - Index: 1
+        Locals:
+         - Type:  I32
+           Count: 1
+        Body: 010101010B
+  - Type: CUSTOM
+    Name: linking
+    Version: 2
+    SymbolTable:
+      - Index:    0
+        Kind:     FUNCTION
+        Name:     func1
+        Flags:    [  ]
+        Function: 0
+      - Index:    1
+        Kind:     FUNCTION
+        Name:     func2
+        Flags:    [  ]
+        Function: 1
+  - Type: CUSTOM
+    Name: producers
+    Tools:
+      - Name:   clang
+        Version: 9.0.0
+  - Type: CUSTOM
+    Name: foo
+    Payload: ABC123
diff --git a/llvm/test/tools/llvm-objcopy/wasm/dump-section.test b/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
new file mode 100644
index 0000000000000000000000000000000000000000..19620f7a3d336ce53ad37cef753ccbb054971468
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/wasm/dump-section.test
@@ -0,0 +1,38 @@
+## Test the contents of a custom section dumped from a binary.
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy --dump-section=producers=%t.sec %t
+# RUN: od -t x1 %t.sec | FileCheck %s
+
+# RUN: not llvm-objcopy --dump-section=nonexistent=%t.sec %t 2>&1 | FileCheck --check-prefix=NONEXISTENT %s
+# RUN: not llvm-objcopy --dump-section=producers=%t.dir/bar %t 2>&1 | FileCheck --check-prefix=DIROUT %s
+
+## Raw contents of the producers section.
+# CHECK: 0000000 01 0c 70 72 6f 63 65 73 73 65 64 2d 62 79 01 05
+# CHECK: 0000020 63 6c 61 6e 67 05 39 2e 30 2e 30
+
+# NONEXISTENT: section 'nonexistent' not found
+# DIROUT: error: {{.*}}/bar': {{[nN]}}o such file or directory
+
+## Check that dumping and removing a section works in the same invocation
+# RUN: llvm-objcopy --dump-section=producers=%t.sec --remove-section=producers %t %t2
+# RUN: od -t x1 %t.sec | FileCheck %s
+# RUN: obj2yaml %t2 | FileCheck --check-prefix=REMOVED %s
+
+# REMOVED-NOT: producers
+
+--- !WASM
+FileHeader:
+  Version: 0x00000001
+Sections:
+  - Type: TYPE
+    Signatures:
+      - Index: 0
+        ParamTypes:
+          - I32
+        ReturnTypes:
+          - F32
+  - Type: CUSTOM
+    Name: producers
+    Tools:
+      - Name:   clang
+        Version: 9.0.0
diff --git a/llvm/test/tools/llvm-objcopy/wasm/remove-section.test b/llvm/test/tools/llvm-objcopy/wasm/remove-section.test
new file mode 100644
index 0000000000000000000000000000000000000000..ad4474f9c19590247cfd1e8958b04fca27d9d402
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/wasm/remove-section.test
@@ -0,0 +1,26 @@
+## Test the --remove-section flag.
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy -R producers %t %t2
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not=producers %s
+## Check that the producers section has been removed, but not the type section.
+# CHECK: TYPE
+
+## Requests to remove nonexistent sections are silently ignored.
+# RUN: llvm-objcopy --remove-section=nonexistent=%t.sec %t 2&>1 | count 0
+
+--- !WASM
+FileHeader:
+  Version: 0x00000001
+Sections:
+  - Type: TYPE
+    Signatures:
+      - Index: 0
+        ParamTypes:
+          - I32
+        ReturnTypes:
+          - F32
+  - Type: CUSTOM
+    Name: producers
+    Tools:
+      - Name:   clang
+        Version: 9.0.0
diff --git a/llvm/test/tools/yaml2obj/ELF/call-graph-profile-section.yaml b/llvm/test/tools/yaml2obj/ELF/call-graph-profile-section.yaml
index 4555bdc656d7190acb2250b04db33c14703a0e4e..a4ca37bd2aabe7a7d92e40f19785cc94c7963b67 100644
--- a/llvm/test/tools/yaml2obj/ELF/call-graph-profile-section.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/call-graph-profile-section.yaml
@@ -2,14 +2,14 @@
 
 ## Test that the content of SHT_LLVM_CALL_GRAPH_PROFILE sections
 ## for 32/64-bit little/big endian targets is correct.
-# RUN: yaml2obj --docnum=1 %s -o %t.le64
-# RUN: llvm-readobj --elf-cg-profile  --sections --section-data %t.le64 | FileCheck %s --check-prefixes=BASIC,BASIC-LE
-# RUN: yaml2obj --docnum=2 %s -o %t.be64
-# RUN: llvm-readobj --elf-cg-profile  --sections --section-data %t.be64 | FileCheck %s --check-prefixes=BASIC,BASIC-BE
-# RUN: yaml2obj --docnum=3 %s -o %t.le32
-# RUN: llvm-readobj --elf-cg-profile  --sections --section-data %t.le32 | FileCheck %s --check-prefixes=BASIC,BASIC-LE
-# RUN: yaml2obj --docnum=4 %s -o %t.be32
-# RUN: llvm-readobj --elf-cg-profile  --sections --section-data %t.be32 | FileCheck %s --check-prefixes=BASIC,BASIC-BE
+# RUN: yaml2obj --docnum=1 -D BITS=64 -D ENCODE=LSB %s -o %t.le64
+# RUN: llvm-readobj --elf-cg-profile --sections --section-data %t.le64 | FileCheck %s --check-prefixes=BASIC,BASIC-LE
+# RUN: yaml2obj --docnum=1 -D BITS=64 -D ENCODE=MSB %s -o %t.be64                      
+# RUN: llvm-readobj --elf-cg-profile --sections --section-data %t.be64 | FileCheck %s --check-prefixes=BASIC,BASIC-BE
+# RUN: yaml2obj --docnum=1 -D BITS=32 -D ENCODE=LSB %s -o %t.le32                      
+# RUN: llvm-readobj --elf-cg-profile --sections --section-data %t.le32 | FileCheck %s --check-prefixes=BASIC,BASIC-LE
+# RUN: yaml2obj --docnum=1 -D BITS=32 -D ENCODE=MSB %s -o %t.be32                      
+# RUN: llvm-readobj --elf-cg-profile --sections --section-data %t.be32 | FileCheck %s --check-prefixes=BASIC,BASIC-BE
 
 # BASIC:        Name: .llvm.call-graph-profile
 # BASIC-NEXT:   Type: SHT_LLVM_CALL_GRAPH_PROFILE
@@ -48,13 +48,10 @@
 # BASIC-NEXT:   }
 # BASIC-NEXT: ]
 
-## TODO: we should really improve yaml2obj somehow to be able to collapse
-##       the following four YAML descriptions into a single one.
-
 --- !ELF
 FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
+  Class:   ELFCLASS[[BITS]]
+  Data:    ELFDATA2[[ENCODE]]
   Type:    ET_DYN
   Machine: EM_X86_64
 Sections:
@@ -71,69 +68,9 @@ Symbols:
   - Name: foo
   - Name: bar
 
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2MSB
-  Type:    ET_DYN
-  Machine: EM_X86_64
-Sections:
-  - Name: .llvm.call-graph-profile
-    Type: SHT_LLVM_CALL_GRAPH_PROFILE
-    Entries:
-      - From:   1
-        To:     2
-        Weight: 89
-      - From:   2
-        To:     1
-        Weight: 98
-Symbols:
-  - Name: foo
-  - Name: bar
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2LSB
-  Type:    ET_DYN
-  Machine: EM_386
-Sections:
-  - Name: .llvm.call-graph-profile
-    Type: SHT_LLVM_CALL_GRAPH_PROFILE
-    Entries:
-      - From:   1
-        To:     2
-        Weight: 89
-      - From:   2
-        To:     1
-        Weight: 98
-Symbols:
-  - Name: foo
-  - Name: bar
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS32
-  Data:    ELFDATA2MSB
-  Type:    ET_DYN
-  Machine: EM_386
-Sections:
-  - Name: .llvm.call-graph-profile
-    Type: SHT_LLVM_CALL_GRAPH_PROFILE
-    Entries:
-      - From:   1
-        To:     2
-        Weight: 89
-      - From:   2
-        To:     1
-        Weight: 98
-Symbols:
-  - Name: foo
-  - Name: bar
-
 ## Check we can set arbitrary sh_link and sh_entsize values.
 ## Check we can specify neither "Content" nor "Entries" tags.
-# RUN: yaml2obj --docnum=5 %s -o %t.link
+# RUN: yaml2obj --docnum=2 %s -o %t.link
 # RUN: llvm-readelf --sections %t.link | FileCheck %s --check-prefix=LINK
 
 # LINK: [Nr] Name      Type                    Address          Off    Size   ES Flg Lk
@@ -157,7 +94,7 @@ Sections:
     EntSize: 0xFF
 
 ## Check we can't specify both "Content" and "Entries" tags.
-# RUN: not yaml2obj --docnum=6 %s 2>&1 | FileCheck %s --check-prefix=BOTH
+# RUN: not yaml2obj --docnum=3 %s 2>&1 | FileCheck %s --check-prefix=BOTH
 # BOTH: error: "Entries" and "Content" can't be used together
 
 --- !ELF
@@ -173,7 +110,7 @@ Sections:
     Entries: []
 
 ## Check we can refer to symbols by name.
-# RUN: yaml2obj --docnum=7 %s -o %t.sym
+# RUN: yaml2obj --docnum=4 %s -o %t.sym
 # RUN: llvm-readobj --elf-cg-profile %t.sym | FileCheck %s --check-prefix=SYMBOL-NAMES
 
 # SYMBOL-NAMES:      CGProfile [
@@ -213,7 +150,7 @@ Symbols:
   - Name: 'foo [1]'
 
 ## Check we can describe SHT_LLVM_CALL_GRAPH_PROFILE sections using the "Content" tag.
-# RUN: yaml2obj --docnum=8 %s -o %t.content
+# RUN: yaml2obj --docnum=5 %s -o %t.content
 # RUN: llvm-readobj --sections --section-data %t.content | FileCheck %s --check-prefix=CONTENT
 
 # CONTENT:      Name: .llvm.call-graph-profile
@@ -233,8 +170,8 @@ Sections:
     Content: "11223344"
 
 ## Check we can't reference unknown symbols by name.
-# RUN: not yaml2obj --docnum=9 %s 2>&1 | FileCheck %s --check-prefix=UNKNOWN-NAME
-# RUN: not yaml2obj --docnum=10 %s 2>&1 | FileCheck %s --check-prefix=UNKNOWN-NAME
+# RUN: not yaml2obj --docnum=6 %s 2>&1 | FileCheck %s --check-prefix=UNKNOWN-NAME
+# RUN: not yaml2obj --docnum=7 %s 2>&1 | FileCheck %s --check-prefix=UNKNOWN-NAME
 # UNKNOWN-NAME: error: unknown symbol referenced: 'bar' by YAML section '.llvm.call-graph-profile'
 
 --- !ELF
@@ -272,7 +209,7 @@ Symbols:
   - Name: foo
 
 ## Check we can specify arbitrary symbol indexes for an SHT_LLVM_CALL_GRAPH_PROFILE section entry.
-# RUN: yaml2obj --docnum=11 %s -o %t.unk
+# RUN: yaml2obj --docnum=8 %s -o %t.unk
 # RUN: llvm-readobj --sections --section-data %t.unk | FileCheck %s --check-prefix=UNKNOWN-INDEX
 
 # UNKNOWN-INDEX:      Name: .llvm.call-graph-profile
diff --git a/llvm/tools/bugpoint-passes/CMakeLists.txt b/llvm/tools/bugpoint-passes/CMakeLists.txt
index 6df49d7abd88098ac779e9609669eda1b0754c61..eea3e239b80830d1d8914d9e59829f5394962349 100644
--- a/llvm/tools/bugpoint-passes/CMakeLists.txt
+++ b/llvm/tools/bugpoint-passes/CMakeLists.txt
@@ -11,7 +11,7 @@ if( NOT LLVM_REQUIRES_RTTI )
 endif()
 
 if(WIN32 OR CYGWIN)
-  set(LLVM_LINK_COMPONENTS Core Support)
+  set(LLVM_LINK_COMPONENTS Core)
 endif()
 
 add_llvm_library( BugpointPasses MODULE BUILDTREE_ONLY
diff --git a/llvm/tools/dsymutil/CFBundle.cpp b/llvm/tools/dsymutil/CFBundle.cpp
index eb07287b63c693ac2e0348d0a724159ee079b56c..0625afb18ab6fa9a19baf55b9f5f2592b23746d7 100644
--- a/llvm/tools/dsymutil/CFBundle.cpp
+++ b/llvm/tools/dsymutil/CFBundle.cpp
@@ -34,9 +34,8 @@ template <typename T> struct CFDeleter {
 /// any valid pointer it owns unless that pointer is explicitly released using
 /// the release() member function.
 template <typename T>
-using CFReleaser =
-    std::unique_ptr<typename std::remove_pointer<T>::type,
-                    CFDeleter<typename std::remove_pointer<T>::type>>;
+using CFReleaser = std::unique_ptr<std::remove_pointer_t<T>,
+                                   CFDeleter<std::remove_pointer_t<T>>>;
 
 /// RAII wrapper around CFBundleRef.
 class CFString : public CFReleaser<CFStringRef> {
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp b/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
index 21932f416aa7dace91f347354dcc002bf50c48bf..84ef4c63598a44179ddc643eb58bc89695cf44bc 100644
--- a/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
+++ b/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
@@ -177,8 +177,7 @@ size_t randomIndex(size_t Max) {
   return Distribution(randomGenerator());
 }
 
-template <typename C>
-static auto randomElement(const C &Container) -> decltype(Container[0]) {
+template <typename C> static decltype(auto) randomElement(const C &Container) {
   assert(!Container.empty() &&
          "Can't pick a random element from an empty container)");
   return Container[randomIndex(Container.size() - 1)];
diff --git a/llvm/tools/llvm-objcopy/CMakeLists.txt b/llvm/tools/llvm-objcopy/CMakeLists.txt
index b3706bc691774024c8597106e4d5abdd1841ad06..6aa5197243dc7280aaf349c27ac79db8610ae79d 100644
--- a/llvm/tools/llvm-objcopy/CMakeLists.txt
+++ b/llvm/tools/llvm-objcopy/CMakeLists.txt
@@ -33,6 +33,7 @@ add_llvm_tool(llvm-objcopy
   MachO/MachOWriter.cpp
   MachO/MachOLayoutBuilder.cpp
   MachO/Object.cpp
+  wasm/Object.cpp
   wasm/Reader.cpp
   wasm/Writer.cpp
   wasm/WasmObjcopy.cpp
diff --git a/llvm/tools/llvm-objcopy/wasm/Object.cpp b/llvm/tools/llvm-objcopy/wasm/Object.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0c416483663f2fd82609de67033cad28e689f9bb
--- /dev/null
+++ b/llvm/tools/llvm-objcopy/wasm/Object.cpp
@@ -0,0 +1,36 @@
+//===- Object.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Object.h"
+
+#include "llvm/Support/LEB128.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace objcopy {
+namespace wasm {
+
+using namespace object;
+using namespace llvm::wasm;
+
+void Object::addSectionWithOwnedContents(
+    Section NewSection, std::unique_ptr<MemoryBuffer> &&Content) {
+  Sections.push_back(NewSection);
+  OwnedContents.emplace_back(std::move(Content));
+}
+
+void Object::removeSections(function_ref<bool(const Section &)> ToRemove) {
+  // TODO: remove reloc sections for the removed section, handle symbols, etc.
+  Sections.erase(
+      std::remove_if(std::begin(Sections), std::end(Sections), ToRemove),
+      std::end(Sections));
+}
+
+} // end namespace wasm
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/llvm/tools/llvm-objcopy/wasm/Object.h b/llvm/tools/llvm-objcopy/wasm/Object.h
index 78a43be21d479b0d73869f17b2e2fb073e3f9209..9db91c41e2e26641bdeb8d348da2543ed1a6a689 100644
--- a/llvm/tools/llvm-objcopy/wasm/Object.h
+++ b/llvm/tools/llvm-objcopy/wasm/Object.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/Wasm.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include <vector>
 
 namespace llvm {
@@ -30,6 +31,13 @@ struct Object {
   llvm::wasm::WasmObjectHeader Header;
   // For now don't discriminate between kinds of sections.
   std::vector<Section> Sections;
+
+  void addSectionWithOwnedContents(Section NewSection,
+                                   std::unique_ptr<MemoryBuffer> &&Content);
+  void removeSections(function_ref<bool(const Section &)> ToRemove);
+
+private:
+  std::vector<std::unique_ptr<MemoryBuffer>> OwnedContents;
 };
 
 } // end namespace wasm
diff --git a/llvm/tools/llvm-objcopy/wasm/WasmObjcopy.cpp b/llvm/tools/llvm-objcopy/wasm/WasmObjcopy.cpp
index 41816a0b08ad2cf94fd93c5d72bb34552c0f1e76..20781cef2d33a2ae953ab6d738c2bb1789637bb0 100644
--- a/llvm/tools/llvm-objcopy/wasm/WasmObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/wasm/WasmObjcopy.cpp
@@ -21,7 +21,58 @@ namespace wasm {
 
 using namespace object;
 
+static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
+                               Object &Obj) {
+  for (const Section &Sec : Obj.Sections) {
+    if (Sec.Name == SecName) {
+      ArrayRef<uint8_t> Contents = Sec.Contents;
+      Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
+          FileOutputBuffer::create(Filename, Contents.size());
+      if (!BufferOrErr)
+        return BufferOrErr.takeError();
+      std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
+      std::copy(Contents.begin(), Contents.end(), Buf->getBufferStart());
+      if (Error E = Buf->commit())
+        return E;
+      return Error::success();
+    }
+  }
+  return createStringError(errc::invalid_argument, "section '%s' not found",
+                           SecName.str().c_str());
+}
 static Error handleArgs(const CopyConfig &Config, Object &Obj) {
+  // Only support AddSection, DumpSection, RemoveSection for now.
+  for (StringRef Flag : Config.DumpSection) {
+    StringRef SecName;
+    StringRef FileName;
+    std::tie(SecName, FileName) = Flag.split("=");
+    if (Error E = dumpSectionToFile(SecName, FileName, Obj))
+      return createFileError(FileName, std::move(E));
+  }
+
+  Obj.removeSections([&Config](const Section &Sec) {
+    if (Config.ToRemove.matches(Sec.Name))
+      return true;
+    return false;
+  });
+
+  for (StringRef Flag : Config.AddSection) {
+    StringRef SecName, FileName;
+    std::tie(SecName, FileName) = Flag.split("=");
+    ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+        MemoryBuffer::getFile(FileName);
+    if (!BufOrErr)
+      return createFileError(FileName, errorCodeToError(BufOrErr.getError()));
+    Section Sec;
+    Sec.SectionType = llvm::wasm::WASM_SEC_CUSTOM;
+    Sec.Name = SecName;
+    std::unique_ptr<MemoryBuffer> Buf = std::move(*BufOrErr);
+    Sec.Contents = makeArrayRef<uint8_t>(
+        reinterpret_cast<const uint8_t *>(Buf->getBufferStart()),
+        Buf->getBufferSize());
+    Obj.addSectionWithOwnedContents(Sec, std::move(Buf));
+  }
+
   if (!Config.AddGnuDebugLink.empty() || !Config.BuildIdLinkDir.empty() ||
       Config.BuildIdLinkInput || Config.BuildIdLinkOutput ||
       Config.ExtractPartition || !Config.SplitDWO.empty() ||
@@ -34,12 +85,10 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj) {
       !Config.UnneededSymbolsToRemove.empty() ||
       !Config.SymbolsToWeaken.empty() || !Config.SymbolsToKeepGlobal.empty() ||
       !Config.SectionsToRename.empty() || !Config.SetSectionAlignment.empty() ||
-      !Config.SetSectionFlags.empty() || !Config.SymbolsToRename.empty() ||
-      !Config.ToRemove.empty() || !Config.DumpSection.empty() ||
-      !Config.AddSection.empty()) {
+      !Config.SetSectionFlags.empty() || !Config.SymbolsToRename.empty()) {
     return createStringError(
         llvm::errc::invalid_argument,
-        "no flags are supported yet, only basic copying is allowed");
+        "only add-section, dump-section, and remove-section are supported");
   }
   return Error::success();
 }
@@ -53,7 +102,7 @@ Error executeObjcopyOnBinary(const CopyConfig &Config,
   Object *Obj = ObjOrErr->get();
   assert(Obj && "Unable to deserialize Wasm object");
   if (Error E = handleArgs(Config, *Obj))
-    return createFileError(Config.InputFilename, std::move(E));
+    return E;
   Writer TheWriter(*Obj, Out);
   if (Error E = TheWriter.write())
     return createFileError(Config.OutputFilename, std::move(E));
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 6719aed449129d6d68618e7f00f5cdd361ce6135..a1c37bc8bff43a225956fb7a4b24914b478ff21d 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -341,16 +341,6 @@ static StringSet<> DisasmFuncsSet;
 StringSet<> FoundSectionSet;
 static StringRef ToolName;
 
-static bool operator<(const SymbolInfoTy& P1 ,const SymbolInfoTy& P2) {
-  if (P1.Addr < P2.Addr)
-    return true;
-
-  if (P1.Addr == P2.Addr)
-    return P1.Name < P2.Name;
-
-  return false;
-}
-
 namespace {
 struct FilterResult {
   // True if the section should not be skipped.
diff --git a/llvm/tools/llvm-pdbutil/FormatUtil.h b/llvm/tools/llvm-pdbutil/FormatUtil.h
index 19ce248f9a6f0b3e560655ec8344693a8c585127..1a006844e011a9a84b24ca6deb110bfbc286049c 100644
--- a/llvm/tools/llvm-pdbutil/FormatUtil.h
+++ b/llvm/tools/llvm-pdbutil/FormatUtil.h
@@ -42,8 +42,7 @@ std::string truncateQuotedNameBack(StringRef Label, StringRef Name,
     return Ret;
 
 template <typename T> std::string formatUnknownEnum(T Value) {
-  return formatv("unknown ({0})",
-                 static_cast<typename std::underlying_type<T>::type>(Value))
+  return formatv("unknown ({0})", static_cast<std::underlying_type_t<T>>(Value))
       .str();
 }
 
diff --git a/llvm/tools/llvm-xray/trie-node.h b/llvm/tools/llvm-xray/trie-node.h
index 47d4b8f1e78c84cff110bd1be291423f482f1d53..7bff81473b5df2409b026c8a20f47c9ce3508ee9 100644
--- a/llvm/tools/llvm-xray/trie-node.h
+++ b/llvm/tools/llvm-xray/trie-node.h
@@ -48,7 +48,7 @@ template <typename T, typename Callable>
 TrieNode<T> *
 mergeTrieNodes(const TrieNode<T> &Left, const TrieNode<T> &Right,
                /*Non-deduced pointer type for nullptr compatibility*/
-               typename std::remove_reference<TrieNode<T> *>::type NewParent,
+               std::remove_reference_t<TrieNode<T> *> NewParent,
                std::forward_list<TrieNode<T>> &NodeStore,
                Callable &&MergeCallable) {
   llvm::function_ref<T(const T &, const T &)> MergeFn(
diff --git a/llvm/unittests/ADT/APFloatTest.cpp b/llvm/unittests/ADT/APFloatTest.cpp
index e571e04614ae63ec3be3948c12fb230b16a63424..6822161729b2f6d0588713a3a469aea05077463c 100644
--- a/llvm/unittests/ADT/APFloatTest.cpp
+++ b/llvm/unittests/ADT/APFloatTest.cpp
@@ -3410,6 +3410,444 @@ TEST(APFloatTest, mod) {
   }
 }
 
+TEST(APFloatTest, remainder) {
+  // Test Special Cases against each other and normal values.
+
+  APFloat PInf = APFloat::getInf(APFloat::IEEEsingle(), false);
+  APFloat MInf = APFloat::getInf(APFloat::IEEEsingle(), true);
+  APFloat PZero = APFloat::getZero(APFloat::IEEEsingle(), false);
+  APFloat MZero = APFloat::getZero(APFloat::IEEEsingle(), true);
+  APFloat QNaN = APFloat::getNaN(APFloat::IEEEsingle(), false);
+  APFloat SNaN = APFloat(APFloat::IEEEsingle(), "snan123");
+  APFloat PNormalValue = APFloat(APFloat::IEEEsingle(), "0x1p+0");
+  APFloat MNormalValue = APFloat(APFloat::IEEEsingle(), "-0x1p+0");
+  APFloat PLargestValue = APFloat::getLargest(APFloat::IEEEsingle(), false);
+  APFloat MLargestValue = APFloat::getLargest(APFloat::IEEEsingle(), true);
+  APFloat PSmallestValue = APFloat::getSmallest(APFloat::IEEEsingle(), false);
+  APFloat MSmallestValue = APFloat::getSmallest(APFloat::IEEEsingle(), true);
+  APFloat PSmallestNormalized =
+      APFloat::getSmallestNormalized(APFloat::IEEEsingle(), false);
+  APFloat MSmallestNormalized =
+      APFloat::getSmallestNormalized(APFloat::IEEEsingle(), true);
+
+  APFloat PVal1(APFloat::IEEEsingle(), "0x1.fffffep+126");
+  APFloat MVal1(APFloat::IEEEsingle(), "-0x1.fffffep+126");
+  APFloat PVal2(APFloat::IEEEsingle(), "0x1.fffffep-126");
+  APFloat MVal2(APFloat::IEEEsingle(), "-0x1.fffffep-126");
+  APFloat PVal3(APFloat::IEEEsingle(), "0x1p-125");
+  APFloat MVal3(APFloat::IEEEsingle(), "-0x1p-125");
+  APFloat PVal4(APFloat::IEEEsingle(), "0x1p+127");
+  APFloat MVal4(APFloat::IEEEsingle(), "-0x1p+127");
+  APFloat PVal5(APFloat::IEEEsingle(), "1.5");
+  APFloat MVal5(APFloat::IEEEsingle(), "-1.5");
+  APFloat PVal6(APFloat::IEEEsingle(), "1");
+  APFloat MVal6(APFloat::IEEEsingle(), "-1");
+
+  struct {
+    APFloat x;
+    APFloat y;
+    const char *result;
+    int status;
+    int category;
+  } SpecialCaseTests[] = {
+    { PInf, PInf, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, MInf, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { PInf, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, PNormalValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, MNormalValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, PLargestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, MLargestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, PSmallestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, MSmallestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, PSmallestNormalized, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PInf, MSmallestNormalized, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, PInf, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, MInf, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { MInf, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, PNormalValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, MNormalValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, PLargestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, MLargestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, PSmallestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, MSmallestValue, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, PSmallestNormalized, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MInf, MSmallestNormalized, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PZero, PInf, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, MInf, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PZero, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PZero, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { PZero, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PZero, PNormalValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, MNormalValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, PLargestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, MLargestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, PSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, MSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, PSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PZero, MSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, PInf, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, MInf, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MZero, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MZero, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { MZero, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MZero, PNormalValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, MNormalValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, PLargestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, MLargestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, PSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, MSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, PSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MZero, MSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { QNaN, PInf, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, MInf, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, PZero, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, MZero, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, SNaN, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { QNaN, PNormalValue, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, MNormalValue, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, PLargestValue, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, MLargestValue, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, PSmallestValue, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, MSmallestValue, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, PSmallestNormalized, "nan", APFloat::opOK, APFloat::fcNaN },
+    { QNaN, MSmallestNormalized, "nan", APFloat::opOK, APFloat::fcNaN },
+    { SNaN, PInf, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, MInf, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, PZero, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, MZero, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, QNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, PNormalValue, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, MNormalValue, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, PLargestValue, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, MLargestValue, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, PSmallestValue, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, MSmallestValue, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, PSmallestNormalized, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { SNaN, MSmallestNormalized, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PNormalValue, PInf, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PNormalValue, MInf, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PNormalValue, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PNormalValue, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PNormalValue, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { PNormalValue, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PNormalValue, PNormalValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PNormalValue, MNormalValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PNormalValue, PLargestValue, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PNormalValue, MLargestValue, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PNormalValue, PSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PNormalValue, MSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PNormalValue, PSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PNormalValue, MSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MNormalValue, PInf, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MNormalValue, MInf, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MNormalValue, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MNormalValue, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MNormalValue, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { MNormalValue, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MNormalValue, PNormalValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MNormalValue, MNormalValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MNormalValue, PLargestValue, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MNormalValue, MLargestValue, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MNormalValue, PSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MNormalValue, MSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MNormalValue, PSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MNormalValue, MSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, PInf, "0x1.fffffep+127", APFloat::opOK, APFloat::fcNormal },
+    { PLargestValue, MInf, "0x1.fffffep+127", APFloat::opOK, APFloat::fcNormal },
+    { PLargestValue, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PLargestValue, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PLargestValue, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { PLargestValue, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PLargestValue, PNormalValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, MNormalValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, PLargestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, MLargestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, PSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, MSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, PSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PLargestValue, MSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, PInf, "-0x1.fffffep+127", APFloat::opOK, APFloat::fcNormal },
+    { MLargestValue, MInf, "-0x1.fffffep+127", APFloat::opOK, APFloat::fcNormal },
+    { MLargestValue, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MLargestValue, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MLargestValue, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { MLargestValue, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MLargestValue, PNormalValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, MNormalValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, PLargestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, MLargestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, PSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, MSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, PSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MLargestValue, MSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PSmallestValue, PInf, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, MInf, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PSmallestValue, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PSmallestValue, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { PSmallestValue, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PSmallestValue, PNormalValue, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, MNormalValue, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, PLargestValue, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, MLargestValue, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, PSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PSmallestValue, MSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PSmallestValue, PSmallestNormalized, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestValue, MSmallestNormalized, "0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, PInf, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, MInf, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MSmallestValue, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MSmallestValue, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { MSmallestValue, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MSmallestValue, PNormalValue, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, MNormalValue, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, PLargestValue, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, MLargestValue, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, PSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MSmallestValue, MSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MSmallestValue, PSmallestNormalized, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestValue, MSmallestNormalized, "-0x1p-149", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, PInf, "0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, MInf, "0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PSmallestNormalized, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PSmallestNormalized, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { PSmallestNormalized, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { PSmallestNormalized, PNormalValue, "0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, MNormalValue, "0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, PLargestValue, "0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, MLargestValue, "0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { PSmallestNormalized, PSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PSmallestNormalized, MSmallestValue, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PSmallestNormalized, PSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PSmallestNormalized, MSmallestNormalized, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MSmallestNormalized, PInf, "-0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestNormalized, MInf, "-0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestNormalized, PZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MSmallestNormalized, MZero, "nan", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MSmallestNormalized, QNaN, "nan", APFloat::opOK, APFloat::fcNaN },
+    { MSmallestNormalized, SNaN, "nan123", APFloat::opInvalidOp, APFloat::fcNaN },
+    { MSmallestNormalized, PNormalValue, "-0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestNormalized, MNormalValue, "-0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestNormalized, PLargestValue, "-0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestNormalized, MLargestValue, "-0x1p-126", APFloat::opOK, APFloat::fcNormal },
+    { MSmallestNormalized, PSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MSmallestNormalized, MSmallestValue, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MSmallestNormalized, PSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MSmallestNormalized, MSmallestNormalized, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+
+    { PVal1, PVal1, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, MVal1, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, PVal2, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, MVal2, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, PVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, MVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, PVal4, "-0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { PVal1, MVal4, "-0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { PVal1, PVal5, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, MVal5, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, PVal6, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal1, MVal6, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, PVal1, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, MVal1, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, PVal2, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, MVal2, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, PVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, MVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, PVal4, "0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { MVal1, MVal4, "0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { MVal1, PVal5, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, MVal5, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, PVal6, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal1, MVal6, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal2, PVal1, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, MVal1, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, PVal2, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal2, MVal2, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal2, PVal3, "-0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, MVal3, "-0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, PVal4, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, MVal4, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, PVal5, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, MVal5, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, PVal6, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal2, MVal6, "0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, PVal1, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, MVal1, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, PVal2, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal2, MVal2, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal2, PVal3, "0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, MVal3, "0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, PVal4, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, MVal4, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, PVal5, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, MVal5, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, PVal6, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal2, MVal6, "-0x1.fffffep-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, PVal1, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, MVal1, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, PVal2, "0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, MVal2, "0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, PVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal3, MVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal3, PVal4, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, MVal4, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, PVal5, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, MVal5, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, PVal6, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal3, MVal6, "0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, PVal1, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, MVal1, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, PVal2, "-0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, MVal2, "-0x0.000002p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, PVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal3, MVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal3, PVal4, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, MVal4, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, PVal5, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, MVal5, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, PVal6, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { MVal3, MVal6, "-0x1p-125", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, PVal1, "0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, MVal1, "0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, PVal2, "0x0.002p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, MVal2, "0x0.002p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, PVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal4, MVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal4, PVal4, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal4, MVal4, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal4, PVal5, "0.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, MVal5, "0.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal4, PVal6, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal4, MVal6, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal4, PVal1, "-0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { MVal4, MVal1, "-0x1p+103", APFloat::opOK, APFloat::fcNormal },
+    { MVal4, PVal2, "-0x0.002p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal4, MVal2, "-0x0.002p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal4, PVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal4, MVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal4, PVal4, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal4, MVal4, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal4, PVal5, "-0.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal4, MVal5, "-0.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal4, PVal6, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal4, MVal6, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal5, PVal1, "1.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, MVal1, "1.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, PVal2, "0x0.00006p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, MVal2, "0x0.00006p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, PVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal5, MVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal5, PVal4, "1.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, MVal4, "1.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, PVal5, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal5, MVal5, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal5, PVal6, "-0.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal5, MVal6, "-0.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, PVal1, "-1.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, MVal1, "-1.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, PVal2, "-0x0.00006p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, MVal2, "-0x0.00006p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, PVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal5, MVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal5, PVal4, "-1.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, MVal4, "-1.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, PVal5, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal5, MVal5, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal5, PVal6, "0.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal5, MVal6, "0.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, PVal1, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, MVal1, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, PVal2, "0x0.00004p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, MVal2, "0x0.00004p-126", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, PVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal6, MVal3, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal6, PVal4, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, MVal4, "0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, PVal5, "-0.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, MVal5, "-0.5", APFloat::opOK, APFloat::fcNormal },
+    { PVal6, PVal6, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { PVal6, MVal6, "0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal6, PVal1, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, MVal1, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, PVal2, "-0x0.00004p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, MVal2, "-0x0.00004p-126", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, PVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal6, MVal3, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal6, PVal4, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, MVal4, "-0x1p+0", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, PVal5, "0.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, MVal5, "0.5", APFloat::opOK, APFloat::fcNormal },
+    { MVal6, PVal6, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+    { MVal6, MVal6, "-0x0p+0", APFloat::opOK, APFloat::fcZero },
+  };
+
+  for (size_t i = 0; i < array_lengthof(SpecialCaseTests); ++i) {
+    APFloat x(SpecialCaseTests[i].x);
+    APFloat y(SpecialCaseTests[i].y);
+    APFloat::opStatus status = x.remainder(y);
+
+    APFloat result(x.getSemantics(), SpecialCaseTests[i].result);
+
+    EXPECT_TRUE(result.bitwiseIsEqual(x));
+    EXPECT_EQ(SpecialCaseTests[i].status, (int)status);
+    EXPECT_EQ(SpecialCaseTests[i].category, (int)x.getCategory());
+  }
+
+  {
+    APFloat f1(APFloat::IEEEdouble(), "0x1.3333333333333p-2"); // 0.3
+    APFloat f2(APFloat::IEEEdouble(), "0x1.47ae147ae147bp-7"); // 0.01
+    APFloat expected(APFloat::IEEEdouble(), "-0x1.4p-56");
+    EXPECT_EQ(APFloat::opOK, f1.remainder(f2));
+    EXPECT_TRUE(f1.bitwiseIsEqual(expected));
+  }
+  {
+    APFloat f1(APFloat::IEEEdouble(), "0x1p64"); // 1.8446744073709552e19
+    APFloat f2(APFloat::IEEEdouble(), "1.5");
+    APFloat expected(APFloat::IEEEdouble(), "-0.5");
+    EXPECT_EQ(APFloat::opOK, f1.remainder(f2));
+    EXPECT_TRUE(f1.bitwiseIsEqual(expected));
+  }
+  {
+    APFloat f1(APFloat::IEEEdouble(), "0x1p1000");
+    APFloat f2(APFloat::IEEEdouble(), "0x1p-1000");
+    APFloat expected(APFloat::IEEEdouble(), "0.0");
+    EXPECT_EQ(APFloat::opOK, f1.remainder(f2));
+    EXPECT_TRUE(f1.bitwiseIsEqual(expected));
+  }
+  {
+    APFloat f1 = APFloat::getInf(APFloat::IEEEdouble(), false);
+    APFloat f2(APFloat::IEEEdouble(), "1.0");
+    EXPECT_EQ(f1.remainder(f2), APFloat::opInvalidOp);
+    EXPECT_TRUE(f1.isNaN());
+  }
+  {
+    APFloat f1(APFloat::IEEEdouble(), "-4.0");
+    APFloat f2(APFloat::IEEEdouble(), "-2.0");
+    APFloat expected(APFloat::IEEEdouble(), "-0.0");
+    EXPECT_EQ(APFloat::opOK, f1.remainder(f2));
+    EXPECT_TRUE(f1.bitwiseIsEqual(expected));
+  }
+  {
+    APFloat f1(APFloat::IEEEdouble(), "-4.0");
+    APFloat f2(APFloat::IEEEdouble(), "2.0");
+    APFloat expected(APFloat::IEEEdouble(), "-0.0");
+    EXPECT_EQ(APFloat::opOK, f1.remainder(f2));
+    EXPECT_TRUE(f1.bitwiseIsEqual(expected));
+  }
+}
+
 TEST(APFloatTest, PPCDoubleDoubleAddSpecial) {
   using DataType = std::tuple<uint64_t, uint64_t, uint64_t, uint64_t,
                               APFloat::fltCategory, APFloat::roundingMode>;
diff --git a/llvm/unittests/ADT/DenseSetTest.cpp b/llvm/unittests/ADT/DenseSetTest.cpp
index e6fe9ca3b318f08c327250fc728a321e8ee17180..b080aa7f0c650f120dc5142a87db696494ed4b3a 100644
--- a/llvm/unittests/ADT/DenseSetTest.cpp
+++ b/llvm/unittests/ADT/DenseSetTest.cpp
@@ -52,7 +52,7 @@ protected:
 
 private:
   static T GetTestSet() {
-    typename std::remove_const<T>::type Set;
+    std::remove_const_t<T> Set;
     Set.insert(0);
     Set.insert(1);
     Set.insert(2);
diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index d0220e309fefd9732d0c229600f62fadd0a0d55b..a2bcd9e5c72cadfb0238e4ca290c377f6b3ca5e8 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -221,9 +221,7 @@ class apply_variadic {
   static StringRef apply_one(StringRef S) { return S.drop_back(); }
 
 public:
-  template <typename... Ts>
-  auto operator()(Ts &&... Items)
-      -> decltype(std::make_tuple(apply_one(Items)...)) {
+  template <typename... Ts> auto operator()(Ts &&... Items) {
     return std::make_tuple(apply_one(Items)...);
   }
 };
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
index 5d037463b636e461d8b9b44287bb672294eb32a1..b0fdabb433f0289313cefdc60e38d740311c4c73 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
@@ -917,4 +917,51 @@ TEST_F(DebugLineBasicFixture, ParserPrintsStandardOpcodesWhenRequested) {
   EXPECT_TRUE(InOutput("0x0000003f: 0c DW_LNS_set_isa (66)\n")) << Output;
 }
 
+TEST_F(DebugLineBasicFixture, PrintPathsProperly) {
+  if (!setupGenerator(5))
+    return;
+
+  LineTable &LT = Gen->addLineTable();
+  DWARFDebugLine::Prologue P = LT.createBasicPrologue();
+  P.IncludeDirectories.push_back(
+      DWARFFormValue::createFromPValue(DW_FORM_string, "b dir"));
+  P.FileNames.push_back(DWARFDebugLine::FileNameEntry());
+  P.FileNames.back().Name =
+      DWARFFormValue::createFromPValue(DW_FORM_string, "b file");
+  P.FileNames.back().DirIdx = 1;
+  P.PrologueLength += 14;
+  LT.setPrologue(P);
+  generate();
+
+  auto ExpectedLineTable = Line.getOrParseLineTable(LineData, 0, *Context,
+                                                    nullptr, RecordRecoverable);
+  EXPECT_THAT_EXPECTED(ExpectedLineTable, Succeeded());
+  std::string Result;
+  // DWARF 5 stores the compilation directory in two places: the Compilation
+  // Unit and the directory table entry 0, and implementations are free to use
+  // one or the other. This copy serves as the one stored in the CU.
+  StringRef CompDir = "a dir";
+  EXPECT_FALSE(
+      (*ExpectedLineTable)
+          ->Prologue.getFileNameByIndex(
+              1, CompDir, DILineInfoSpecifier::FileLineInfoKind::None, Result));
+  EXPECT_TRUE((*ExpectedLineTable)
+                  ->Prologue.getFileNameByIndex(
+                      1, CompDir,
+                      DILineInfoSpecifier::FileLineInfoKind::Default, Result));
+  EXPECT_STREQ(Result.c_str(), "b file");
+  EXPECT_TRUE((*ExpectedLineTable)
+                  ->Prologue.getFileNameByIndex(
+                      1, CompDir,
+                      DILineInfoSpecifier::FileLineInfoKind::RelativeFilePath,
+                      Result));
+  EXPECT_THAT(Result.c_str(), MatchesRegex("b dir.b file"));
+  EXPECT_TRUE((*ExpectedLineTable)
+                  ->Prologue.getFileNameByIndex(
+                      1, CompDir,
+                      DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath,
+                      Result));
+  EXPECT_THAT(Result.c_str(), MatchesRegex("a dir.b dir.b file"));
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index c67e087887913ba64bf56882afce2c3eb8f63480..4cc72c8eeedbbf4100a766f6bf9ec4b69932bcc6 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -1319,8 +1319,8 @@ TYPED_TEST_CASE(MutableConstTest, MutableConstTestTypes);
 TYPED_TEST(MutableConstTest, ICmp) {
   auto &IRB = PatternMatchTest::IRB;
 
-  typedef typename std::tuple_element<0, TypeParam>::type ValueType;
-  typedef typename std::tuple_element<1, TypeParam>::type InstructionType;
+  typedef std::tuple_element_t<0, TypeParam> ValueType;
+  typedef std::tuple_element_t<1, TypeParam> InstructionType;
 
   Value *L = IRB.getInt32(1);
   Value *R = IRB.getInt32(2);
diff --git a/llvm/unittests/Support/FormatVariadicTest.cpp b/llvm/unittests/Support/FormatVariadicTest.cpp
index a4e8826ecf4ef954e12db7a242f3d574c4337799..e3754fbd1bf02b9c245cb42c4c3bd8aea67289ed 100644
--- a/llvm/unittests/Support/FormatVariadicTest.cpp
+++ b/llvm/unittests/Support/FormatVariadicTest.cpp
@@ -487,9 +487,7 @@ struct format_tuple {
   const char *Fmt;
   explicit format_tuple(const char *Fmt) : Fmt(Fmt) {}
 
-  template <typename... Ts>
-  auto operator()(Ts &&... Values) const
-      -> decltype(formatv(Fmt, std::forward<Ts>(Values)...)) {
+  template <typename... Ts> auto operator()(Ts &&... Values) const {
     return formatv(Fmt, std::forward<Ts>(Values)...);
   }
 };
diff --git a/llvm/unittests/TextAPI/TextStubV3Tests.cpp b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
index 0180989ff331eca14e72e5f81a7f0f24bbe16c29..a9e54807cc85fe942cebd9090267fcaf94b2a2ac 100644
--- a/llvm/unittests/TextAPI/TextStubV3Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
@@ -364,6 +364,78 @@ TEST(TBDv3, Platform_zippered) {
             stripWhitespace(Buffer.c_str()));
 }
 
+TEST(TBDv3, Platform_iOSSim) {
+  static const char tbd_v3_platform_iossim[] = "--- !tapi-tbd-v3\n"
+                                               "archs: [ x86_64 ]\n"
+                                               "platform: ios\n"
+                                               "install-name: Test.dylib\n"
+                                               "...\n";
+
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(tbd_v3_platform_iossim, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::iOSSimulator;
+  auto File = std::move(Result.get());
+  EXPECT_EQ(FileType::TBD_V3, File->getFileType());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
+
+  SmallString<4096> Buffer;
+  raw_svector_ostream OS(Buffer);
+  auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
+  EXPECT_TRUE(!WriteResult);
+  EXPECT_EQ(stripWhitespace(tbd_v3_platform_iossim),
+            stripWhitespace(Buffer.c_str()));
+}
+
+TEST(TBDv3, Platform_watchOSSim) {
+  static const char tbd_v3_platform_watchossim[] = "--- !tapi-tbd-v3\n"
+                                                   "archs: [ x86_64 ]\n"
+                                                   "platform: watchos\n"
+                                                   "install-name: Test.dylib\n"
+                                                   "...\n";
+
+  auto Result = TextAPIReader::get(
+      MemoryBufferRef(tbd_v3_platform_watchossim, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::watchOSSimulator;
+  auto File = std::move(Result.get());
+  EXPECT_EQ(FileType::TBD_V3, File->getFileType());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
+
+  SmallString<4096> Buffer;
+  raw_svector_ostream OS(Buffer);
+  auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
+  EXPECT_TRUE(!WriteResult);
+  EXPECT_EQ(stripWhitespace(tbd_v3_platform_watchossim),
+            stripWhitespace(Buffer.c_str()));
+}
+
+TEST(TBDv3, Platform_tvOSSim) {
+  static const char tbd_v3_platform_tvossim[] = "--- !tapi-tbd-v3\n"
+                                                "archs: [ x86_64 ]\n"
+                                                "platform: tvos\n"
+                                                "install-name: Test.dylib\n"
+                                                "...\n";
+
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(tbd_v3_platform_tvossim, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto File = std::move(Result.get());
+  auto Platform = PlatformKind::tvOSSimulator;
+  EXPECT_EQ(FileType::TBD_V3, File->getFileType());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
+
+  SmallString<4096> Buffer;
+  raw_svector_ostream OS(Buffer);
+  auto WriteResult = TextAPIWriter::writeToStream(OS, *File);
+  EXPECT_TRUE(!WriteResult);
+  EXPECT_EQ(stripWhitespace(tbd_v3_platform_tvossim),
+            stripWhitespace(Buffer.c_str()));
+}
+
 TEST(TBDv3, Swift_1_0) {
   static const char tbd_v3_swift_1_0[] = "--- !tapi-tbd-v3\n"
                                          "archs: [ arm64 ]\n"
diff --git a/llvm/unittests/XRay/GraphTest.cpp b/llvm/unittests/XRay/GraphTest.cpp
index 3c82d53492e8d2aa2c57759c5356e02fae6dfaa1..52580b3068266eb5067c7ab1f1098434b0d303c0 100644
--- a/llvm/unittests/XRay/GraphTest.cpp
+++ b/llvm/unittests/XRay/GraphTest.cpp
@@ -34,7 +34,7 @@ protected:
 private:
   static T getTestGraph() {
     using std::make_pair;
-    typename std::remove_const<T>::type G;
+    std::remove_const_t<T> G;
     G.insert(make_pair(1u, VAttr({3u})));
     G.insert(make_pair(2u, VAttr({5u})));
     G.insert(make_pair(3u, VAttr({7u})));
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index ae4621c76978e02f4a81f70befb82db64b5d3692..f3a87a07037cf8ef157be79c8d29632c5cd62113 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -63,6 +63,9 @@ CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')
 PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')
 CHECK_RE = re.compile(r'^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:')
 
+UTC_ARGS_KEY = 'UTC_ARGS:'
+UTC_ARGS_CMD = re.compile(r'.*' + UTC_ARGS_KEY + '\s*(?P<cmd>.*)\s*$')
+
 OPT_FUNCTION_RE = re.compile(
     r'^\s*define\s+(?:internal\s+)?[^@]*@(?P<func>[\w-]+?)\s*'
     r'(?P<args_and_sig>\((\)|(.*?[\w\.\-]+?)\))[^{]*)\{\n(?P<body>.*?)^\}$',
@@ -396,3 +399,26 @@ def verify_filecheck_prefixes(fc_cmd):
         check_prefix(prefix)
         if prefixes.count(prefix) > 1:
           warn("Supplied prefix '%s' is not unique in the prefix list." % (prefix,))
+
+def get_autogennote_suffix(parser, args):
+    autogenerated_note_args = ''
+    for k, v in args._get_kwargs():
+        if parser.get_default(k) == v or k == 'tests' or k == 'update_only' or k == 'opt_binary':
+            continue
+        k = k.replace('_', '-')
+        if type(v) is bool:
+            autogenerated_note_args += '--%s ' % (k)
+        else:
+            autogenerated_note_args += '--%s %s ' % (k, v)
+    if autogenerated_note_args:
+        autogenerated_note_args = ' %s %s' % (UTC_ARGS_KEY, autogenerated_note_args[:-1])
+    return autogenerated_note_args
+
+
+def check_for_command(line, parser, args, argv):
+    cmd_m = UTC_ARGS_CMD.match(line)
+    if cmd_m:
+        cmd = cmd_m.group('cmd').strip().split(' ')
+        argv = argv + cmd
+        args = parser.parse_args(filter(lambda arg: arg not in args.tests, argv))
+    return args, argv
diff --git a/llvm/utils/benchmark/include/benchmark/benchmark.h b/llvm/utils/benchmark/include/benchmark/benchmark.h
index af935a6ef3e144e86ed7847cacbd7287faf3c98a..ab61c46e938647412d796bc75471509dfb4c36ca 100644
--- a/llvm/utils/benchmark/include/benchmark/benchmark.h
+++ b/llvm/utils/benchmark/include/benchmark/benchmark.h
@@ -990,8 +990,7 @@ inline internal::Benchmark* RegisterBenchmark(const char* name,
 #ifdef BENCHMARK_HAS_CXX11
 template <class Lambda>
 internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) {
-  using BenchType =
-      internal::LambdaBenchmark<typename std::decay<Lambda>::type>;
+  using BenchType = internal::LambdaBenchmark<std::decay_t<Lambda>>;
   return internal::RegisterBenchmarkInternal(
       ::new BenchType(name, std::forward<Lambda>(fn)));
 }
diff --git a/llvm/utils/benchmark/src/sysinfo.cc b/llvm/utils/benchmark/src/sysinfo.cc
index 01dd8a0317b05cbeaf8a174789510548caeaf247..04254bbed365a6fe78ac7c56b46a5a12174c285d 100644
--- a/llvm/utils/benchmark/src/sysinfo.cc
+++ b/llvm/utils/benchmark/src/sysinfo.cc
@@ -176,9 +176,8 @@ bool GetSysctl(std::string const& Name, std::string* Out) {
   return true;
 }
 
-template <class Tp,
-          class = typename std::enable_if<std::is_integral<Tp>::value>::type>
-bool GetSysctl(std::string const& Name, Tp* Out) {
+template <class Tp, class = std::enable_if_t<std::is_integral<Tp>::value>>
+bool GetSysctl(std::string const &Name, Tp *Out) {
   *Out = 0;
   auto Buff = GetSysctlImp(Name);
   if (!Buff) return false;
diff --git a/llvm/utils/gn/get.py b/llvm/utils/gn/get.py
index c39649df78a42bf7ca5160a4e00e8a8a90723862..adb8eb7d571516cf9a412666281b5bfd6c08e4c0 100755
--- a/llvm/utils/gn/get.py
+++ b/llvm/utils/gn/get.py
@@ -5,7 +5,12 @@ from __future__ import print_function
 
 import io
 import os
-import urllib2
+try:
+    # In Python 3, we need the module urllib.reqest. In Python 2, this
+    # functionality was in the urllib2 module.
+    from urllib import request as urllib_request
+except ImportError:
+    import urllib2 as urllib_request
 import sys
 import zipfile
 
@@ -14,7 +19,7 @@ def download_and_unpack(url, output_dir, gn):
     """Download an archive from url and extract gn from it into output_dir."""
     print('downloading %s ...' % url, end='')
     sys.stdout.flush()
-    data = urllib2.urlopen(url).read()
+    data = urllib_request.urlopen(url).read()
     print(' done')
     zipfile.ZipFile(io.BytesIO(data)).extract(gn, path=output_dir)
 
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
index 14bff487fff5a7b50842b31a99a8a33b268415c8..bcf28662f7511cec0fc35faf7a0fa6dc7150606e 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/BUILD.gn
@@ -1,8 +1,14 @@
 import("//compiler-rt/target.gni")
 
+scudo_cflags = [
+  "-Werror=conversion",
+  "-nostdinc++",
+]
+
 source_set("sources") {
   configs -= [ "//llvm/utils/gn/build:llvm_code" ]
   configs += [ "//llvm/utils/gn/build:crt_code" ]
+  cflags = scudo_cflags
   sources = [
     "allocator_config.h",
     "atomic_helpers.h",
@@ -47,10 +53,10 @@ source_set("sources") {
   ]
 
   if (current_cpu == "arm" || current_cpu == "arm64") {
-    cflags = [ "-mcrc" ]
+    cflags += [ "-mcrc" ]
   }
   if (current_cpu == "x64") {
-    cflags = [ "-msse4.2" ]
+    cflags += [ "-msse4.2" ]
   }
 
   public_configs = [ ":scudo_config" ]
@@ -59,6 +65,7 @@ source_set("sources") {
 source_set("c_wrapper_sources") {
   configs -= [ "//llvm/utils/gn/build:llvm_code" ]
   configs += [ "//llvm/utils/gn/build:crt_code" ]
+  cflags = scudo_cflags
   sources = [
     # Make `gn format` not collapse this, for sync_source_lists_from_cmake.py.
     "wrappers_c.cpp",
@@ -70,6 +77,7 @@ source_set("c_wrapper_sources") {
 source_set("cxx_wrapper_sources") {
   configs -= [ "//llvm/utils/gn/build:llvm_code" ]
   configs += [ "//llvm/utils/gn/build:crt_code" ]
+  cflags = scudo_cflags
   sources = [
     # Make `gn format` not collapse this, for sync_source_lists_from_cmake.py.
     "wrappers_cpp.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn
index 3c8ed7353e1a45656d33278c311e88b1befec447..4574278677aa2c506e1fc96fa761654bdae5863a 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/scudo/standalone/tests/BUILD.gn
@@ -1,9 +1,14 @@
 import("//llvm/utils/gn/build/toolchain/compiler.gni")
 import("//llvm/utils/unittest/unittest.gni")
 
+test_cflags = [ "-DSCUDO_DEBUG=1" ]
+
 unittest("ScudoUnitTest") {
   configs += [ "//llvm/utils/gn/build:crt_code" ]
-  deps = [ "//compiler-rt/lib/scudo/standalone:sources" ]
+  cflags = test_cflags
+  deps = [
+    "//compiler-rt/lib/scudo/standalone:sources",
+  ]
   sources = [
     "atomic_test.cpp",
     "bytemap_test.cpp",
@@ -31,6 +36,7 @@ unittest("ScudoUnitTest") {
 
 unittest("ScudoCUnitTest") {
   configs += [ "//llvm/utils/gn/build:crt_code" ]
+  cflags = test_cflags
   deps = [
     "//compiler-rt/lib/scudo/standalone:c_wrapper_sources",
     "//compiler-rt/lib/scudo/standalone:sources",
@@ -44,6 +50,7 @@ unittest("ScudoCUnitTest") {
 
 unittest("ScudoCxxUnitTest") {
   configs += [ "//llvm/utils/gn/build:crt_code" ]
+  cflags = test_cflags
   deps = [
     "//compiler-rt/lib/scudo/standalone:c_wrapper_sources",
     "//compiler-rt/lib/scudo/standalone:cxx_wrapper_sources",
@@ -54,7 +61,7 @@ unittest("ScudoCxxUnitTest") {
     "wrappers_cpp_test.cpp",
   ]
   if (is_clang) {
-    cflags = [ "-Wno-mismatched-new-delete" ]
+    cflags += [ "-Wno-mismatched-new-delete" ]
   }
   has_custom_main = true
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 37d08f813a256ddeeab8e06a0d6f2a98ddcf2316..d33e4e1e4aeedc20220dae41acfb2c86221a6927 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -130,7 +130,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUPromoteAlloca.cpp",
     "AMDGPUPropagateAttributes.cpp",
     "AMDGPURegisterBankInfo.cpp",
-    "AMDGPURegisterInfo.cpp",
     "AMDGPURewriteOutArguments.cpp",
     "AMDGPUSubtarget.cpp",
     "AMDGPUTargetMachine.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
index a186ab94131012e9c1085ca8e488f9617dd9597a..c9bc3947456ee899e2ef9afa663cc02d5e12bcb1 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
@@ -72,6 +72,7 @@ executable("llvm-objcopy") {
     "MachO/MachOWriter.cpp",
     "MachO/Object.cpp",
     "llvm-objcopy.cpp",
+    "wasm/Object.cpp",
     "wasm/Reader.cpp",
     "wasm/WasmObjcopy.cpp",
     "wasm/Writer.cpp",
diff --git a/llvm/utils/unittest/googlemock/include/gmock/gmock-matchers.h b/llvm/utils/unittest/googlemock/include/gmock/gmock-matchers.h
index 865cdc7da108e7cf438b977a223bc93ad4cbefbc..755dbb9cdedb4f8fcae6c4f78c421af938c8fa24 100644
--- a/llvm/utils/unittest/googlemock/include/gmock/gmock-matchers.h
+++ b/llvm/utils/unittest/googlemock/include/gmock/gmock-matchers.h
@@ -3118,8 +3118,9 @@ class ElementsAreMatcherImpl : public MatcherInterface<Container> {
   typedef typename View::const_reference StlContainerReference;
   typedef decltype(std::begin(
       std::declval<StlContainerReference>())) StlContainerConstIterator;
-  typedef typename std::remove_reference<decltype(
-      *std::declval<StlContainerConstIterator &>())>::type Element;
+  typedef std::remove_reference_t<decltype(
+      *std::declval<StlContainerConstIterator &>())>
+      Element;
 
   // Constructs the matcher from a sequence of element values or
   // element matchers.
@@ -3360,8 +3361,9 @@ class UnorderedElementsAreMatcherImpl
   typedef typename View::const_reference StlContainerReference;
   typedef decltype(std::begin(
       std::declval<StlContainerReference>())) StlContainerConstIterator;
-  typedef typename std::remove_reference<decltype(
-      *std::declval<StlContainerConstIterator &>())>::type Element;
+  typedef std::remove_reference_t<decltype(
+      *std::declval<StlContainerConstIterator &>())>
+      Element;
 
   // Constructs the matcher from a sequence of element values or
   // element matchers.
@@ -3470,8 +3472,9 @@ class UnorderedElementsAreMatcher {
     typedef typename View::const_reference StlContainerReference;
     typedef decltype(std::begin(
         std::declval<StlContainerReference>())) StlContainerConstIterator;
-    typedef typename std::remove_reference<decltype(
-        *std::declval<StlContainerConstIterator &>())>::type Element;
+    typedef std::remove_reference_t<decltype(
+        *std::declval<StlContainerConstIterator &>())>
+        Element;
     typedef ::std::vector<Matcher<const Element&> > MatcherVec;
     MatcherVec matchers;
     matchers.reserve(::testing::tuple_size<MatcherTuple>::value);
@@ -3499,8 +3502,9 @@ class ElementsAreMatcher {
     typedef typename View::const_reference StlContainerReference;
     typedef decltype(std::begin(
         std::declval<StlContainerReference>())) StlContainerConstIterator;
-    typedef typename std::remove_reference<decltype(
-        *std::declval<StlContainerConstIterator &>())>::type Element;
+    typedef std::remove_reference_t<decltype(
+        *std::declval<StlContainerConstIterator &>())>
+        Element;
     typedef ::std::vector<Matcher<const Element&> > MatcherVec;
     MatcherVec matchers;
     matchers.reserve(::testing::tuple_size<MatcherTuple>::value);
diff --git a/llvm/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h b/llvm/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h
index 72f23184abd596bb25d883a3441ea6723b6737dd..cff78f5c33f6f167a6f7cdb5f3c6bcaae496bff3 100644
--- a/llvm/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h
+++ b/llvm/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h
@@ -31,8 +31,7 @@ template <typename T, typename Enable = void> struct StreamSwitch {
 
 // printable() returns a version of its argument that can be streamed into a
 // std::ostream. This may be the argument itself, or some other representation.
-template <typename T>
-auto printable(const T &V) -> decltype(StreamSwitch<T>::printable(V)) {
+template <typename T> decltype(auto) printable(const T &V) {
   // We delegate to the trait, to allow partial specialization.
   return StreamSwitch<T>::printable(V);
 }
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index 94e3fef1fcadeddd26da7cc09141a17361b95302..f6fee1437f5d40fd191fd61e9afd7c126726bcbf 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -66,6 +66,10 @@ def main():
                       help='Keep function signature information around for the check line')
   parser.add_argument('--scrub-attributes', action='store_true',
                       help='Remove attribute annotations (#0) from the end of check line')
+  parser.add_argument('--enable', action='store_true', dest='enabled', default=True,
+                      help='Activate CHECK line generation from this point forward')
+  parser.add_argument('--disable', action='store_false', dest='enabled',
+                      help='Deactivate CHECK line generation from this point forward')
   parser.add_argument('tests', nargs='+')
   args = common.parse_commandline_args(parser)
 
@@ -86,6 +90,8 @@ def main():
   # On Windows we must expand the patterns ourselves.
   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
   for test in test_paths:
+    argv = sys.argv[:]
+    args = parser.parse_args()
     with open(test) as f:
       input_lines = [l.rstrip() for l in f]
 
@@ -93,6 +99,9 @@ def main():
     if 'autogenerated' in first_line and script_name not in first_line:
       common.warn("Skipping test which wasn't autogenerated by " + script_name, test)
       continue
+    if first_line and 'autogenerated' in first_line:
+      args, argv = common.check_for_command(first_line, parser, args, argv)
+    test_autogenerated_note = autogenerated_note + common.get_autogennote_suffix(parser, args)
 
     if args.update_only:
       if not first_line or 'autogenerated' not in first_line:
@@ -154,9 +163,17 @@ def main():
     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
     common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
     output_lines = []
-    output_lines.append(autogenerated_note)
+    output_lines.append(test_autogenerated_note)
 
     for input_line in input_lines:
+      # Discard any previous script advertising.
+      if input_line.startswith(ADVERT):
+        continue
+
+      args, argv = common.check_for_command(input_line, parser, args, argv)
+      if not args.enabled:
+          output_lines.append(input_line)
+          continue
       if is_in_function_start:
         if input_line == '':
           continue
@@ -183,10 +200,6 @@ def main():
           is_in_function = False
         continue
 
-      # Discard any previous script advertising.
-      if input_line.startswith(ADVERT):
-        continue
-
       # If it's outside a function, it just gets copied to the output.
       output_lines.append(input_line)
 
diff --git a/mlir/examples/toy/Ch5/CMakeLists.txt b/mlir/examples/toy/Ch5/CMakeLists.txt
index de868daa9b7a54ae789b6c7ab174a034353dc948..df71c35a7a7799c5040935d30746b6a8388e0c16 100644
--- a/mlir/examples/toy/Ch5/CMakeLists.txt
+++ b/mlir/examples/toy/Ch5/CMakeLists.txt
@@ -27,15 +27,10 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 target_link_libraries(toyc-ch5
   PRIVATE
-    MLIRAffineOps
+    MLIRAllDialects
     MLIRAnalysis
     MLIRIR
     MLIRParser
     MLIRPass
-    MLIRStandardOps
     MLIRTransforms)
 
-whole_archive_link(toyc-ch5
-  MLIRAffineOps
-  MLIRStandardOps
-  )
diff --git a/mlir/examples/toy/Ch5/toyc.cpp b/mlir/examples/toy/Ch5/toyc.cpp
index d3a83a971587c1ae992bd43921f86ac428f53e1d..c9a52c606b213fc5faa30df20a254eafab6ed5e7 100644
--- a/mlir/examples/toy/Ch5/toyc.cpp
+++ b/mlir/examples/toy/Ch5/toyc.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Analysis/Verifier.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Module.h"
+#include "mlir/InitAllDialects.h"
 #include "mlir/Parser.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
@@ -171,6 +172,7 @@ int dumpAST() {
 }
 
 int main(int argc, char **argv) {
+  mlir::registerAllDialects();
   mlir::registerPassManagerCLOptions();
   cl::ParseCommandLineOptions(argc, argv, "toy compiler\n");
 
diff --git a/mlir/examples/toy/Ch6/CMakeLists.txt b/mlir/examples/toy/Ch6/CMakeLists.txt
index a539d591b90c33053db638fd752bf79d1dfc6cb4..1497cc349868f5c30cdf92fbe6b61c047e38da8d 100644
--- a/mlir/examples/toy/Ch6/CMakeLists.txt
+++ b/mlir/examples/toy/Ch6/CMakeLists.txt
@@ -29,7 +29,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 target_link_libraries(toyc-ch6
   PRIVATE
-    MLIRAffineOps
+    MLIRAllDialects
     MLIRAffineToStandard
     MLIRAnalysis
     MLIRExecutionEngine
@@ -38,15 +38,8 @@ target_link_libraries(toyc-ch6
     MLIRLoopToStandard
     MLIRParser
     MLIRPass
-    MLIRStandardOps
     MLIRStandardToLLVM
     MLIRTargetLLVMIR
     MLIRTransforms
     )
 
-whole_archive_link(toyc-ch6
-  MLIRAffineToStandard
-  MLIRAffineOps
-  MLIRLLVMIR
-  MLIRStandardOps
-  )
diff --git a/mlir/examples/toy/Ch6/toyc.cpp b/mlir/examples/toy/Ch6/toyc.cpp
index f33f7390344a44bc998c12fc631d84aa2478e4aa..3c54f731ff42971434047f6e9f4891bf0aa50843 100644
--- a/mlir/examples/toy/Ch6/toyc.cpp
+++ b/mlir/examples/toy/Ch6/toyc.cpp
@@ -20,6 +20,7 @@
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Module.h"
+#include "mlir/InitAllDialects.h"
 #include "mlir/Parser.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
@@ -238,6 +239,7 @@ int runJit(mlir::ModuleOp module) {
 }
 
 int main(int argc, char **argv) {
+  mlir::registerAllDialects();
   mlir::registerPassManagerCLOptions();
   cl::ParseCommandLineOptions(argc, argv, "toy compiler\n");
 
diff --git a/mlir/examples/toy/Ch7/CMakeLists.txt b/mlir/examples/toy/Ch7/CMakeLists.txt
index 66d5439877525a933cd4d7da20533316486d07ae..ece8c873786e91c6d6265aa69dfcc750d97bd64a 100644
--- a/mlir/examples/toy/Ch7/CMakeLists.txt
+++ b/mlir/examples/toy/Ch7/CMakeLists.txt
@@ -29,24 +29,15 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/)
 target_link_libraries(toyc-ch7
   PRIVATE
-    MLIRAffineOps
+    MLIRAllDialects
     MLIRAffineToStandard
     MLIRAnalysis
     MLIRExecutionEngine
     MLIRIR
-    MLIRLLVMIR
     MLIRLoopToStandard
     MLIRParser
     MLIRPass
-    MLIRStandardOps
     MLIRStandardToLLVM
     MLIRTargetLLVMIR
     MLIRTransforms
     )
-
-whole_archive_link(toyc-ch7
-  MLIRAffineToStandard
-  MLIRAffineOps
-  MLIRLLVMIR
-  MLIRStandardOps
-  )
diff --git a/mlir/examples/toy/Ch7/toyc.cpp b/mlir/examples/toy/Ch7/toyc.cpp
index 199fff96d3b3f9d4ef41bb65d66ac1bdf1e33547..1f5f988caca3c49d6ac13faad68a75254834674c 100644
--- a/mlir/examples/toy/Ch7/toyc.cpp
+++ b/mlir/examples/toy/Ch7/toyc.cpp
@@ -20,6 +20,7 @@
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Module.h"
+#include "mlir/InitAllDialects.h"
 #include "mlir/Parser.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
@@ -239,6 +240,7 @@ int runJit(mlir::ModuleOp module) {
 }
 
 int main(int argc, char **argv) {
+  mlir::registerAllDialects();
   mlir::registerPassManagerCLOptions();
   cl::ParseCommandLineOptions(argc, argv, "toy compiler\n");
 
diff --git a/mlir/include/mlir/Analysis/Passes.h b/mlir/include/mlir/Analysis/Passes.h
index db53f60b495459f582e7b10bd201f335b40c42ad..296b3b9838c5b11967f3e073d0f22a5c1226bd05 100644
--- a/mlir/include/mlir/Analysis/Passes.h
+++ b/mlir/include/mlir/Analysis/Passes.h
@@ -28,9 +28,6 @@ std::unique_ptr<OpPassBase<FuncOp>> createTestMemRefBoundCheckPass();
 /// Creates a pass to check memref access dependences in a Function.
 std::unique_ptr<OpPassBase<FuncOp>> createTestMemRefDependenceCheckPass();
 
-/// Creates a pass to test parallelism detection; emits note for parallel loops.
-std::unique_ptr<OpPassBase<FuncOp>> createParallelismDetectionTestPass();
-
 } // end namespace mlir
 
 #endif // MLIR_ANALYSIS_PASSES_H
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
index 8f319029f7b50dfdb9c7443b18bf89038c95865d..7d2a076aeb25bef36c385bc73502debb163017a4 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h
@@ -58,7 +58,8 @@ void populateStdToLLVMBarePtrConversionPatterns(
 /// Specifying `useAlloca-true` emits stack allocations instead. In the future
 /// this may become an enum when we have concrete uses for other options.
 std::unique_ptr<OpPassBase<ModuleOp>>
-createLowerToLLVMPass(bool useAlloca = false, bool emitCWrappers = false);
+createLowerToLLVMPass(bool useAlloca = false, bool useBarePtrCallConv = false,
+                      bool emitCWrappers = false);
 
 namespace LLVM {
 /// Make argument-taking successors of each block distinct.  PHI nodes in LLVM
diff --git a/mlir/include/mlir/Dialect/CMakeLists.txt b/mlir/include/mlir/Dialect/CMakeLists.txt
index 8b7eaeef397474e2c2bd32adfc408e3d08039305..db9de7b7ea3ab27f9c7808853293166541d19f74 100644
--- a/mlir/include/mlir/Dialect/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/CMakeLists.txt
@@ -6,6 +6,7 @@ add_subdirectory(LLVMIR)
 add_subdirectory(LoopOps)
 add_subdirectory(OpenMP)
 add_subdirectory(QuantOps)
+add_subdirectory(Shape)
 add_subdirectory(SPIRV)
 add_subdirectory(StandardOps)
 add_subdirectory(VectorOps)
diff --git a/mlir/include/mlir/Dialect/Shape/CMakeLists.txt b/mlir/include/mlir/Dialect/Shape/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f33061b2d87cffc48b072763eb55d349e9070aba
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Shape/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(IR)
diff --git a/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3d1adca9c2be55d3bd89cdff7b392ee238c8585f
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Shape/IR/CMakeLists.txt
@@ -0,0 +1 @@
+add_mlir_dialect(ShapeOps ShapeOps)
diff --git a/mlir/include/mlir/Dialect/Shape/IR/Shape.h b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
new file mode 100644
index 0000000000000000000000000000000000000000..f2cb12d99f909a0c31bdb457904143ebf172e97a
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
@@ -0,0 +1,117 @@
+//===- Shape.h - MLIR Shape dialect -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the shape dialect that is used to describe and solve shape
+// relations of MLIR operations using ShapedType.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_SHAPE_IR_SHAPE_H
+#define MLIR_SHAPE_IR_SHAPE_H
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+
+namespace mlir {
+namespace shape {
+
+/// This dialect contains shape inference related operations and facilities.
+class ShapeDialect : public Dialect {
+public:
+  /// Create the dialect in the given `context`.
+  explicit ShapeDialect(MLIRContext *context);
+};
+
+namespace ShapeTypes {
+enum Kind {
+  Component = Type::FIRST_SHAPE_TYPE,
+  Element,
+  Shape,
+  Size,
+  ValueShape
+};
+} // namespace ShapeTypes
+
+/// The component type corresponding to shape, element type and attribute.
+class ComponentType : public Type::TypeBase<ComponentType, Type> {
+public:
+  using Base::Base;
+
+  static ComponentType get(MLIRContext *context) {
+    return Base::get(context, ShapeTypes::Kind::Component);
+  }
+
+  /// Support method to enable LLVM-style type casting.
+  static bool kindof(unsigned kind) {
+    return kind == ShapeTypes::Kind::Component;
+  }
+};
+
+/// The element type of the shaped type.
+class ElementType : public Type::TypeBase<ElementType, Type> {
+public:
+  using Base::Base;
+
+  static ElementType get(MLIRContext *context) {
+    return Base::get(context, ShapeTypes::Kind::Element);
+  }
+
+  /// Support method to enable LLVM-style type casting.
+  static bool kindof(unsigned kind) {
+    return kind == ShapeTypes::Kind::Element;
+  }
+};
+
+/// The shape descriptor type represents rank and dimension sizes.
+class ShapeType : public Type::TypeBase<ShapeType, Type> {
+public:
+  using Base::Base;
+
+  static ShapeType get(MLIRContext *context) {
+    return Base::get(context, ShapeTypes::Kind::Shape);
+  }
+
+  /// Support method to enable LLVM-style type casting.
+  static bool kindof(unsigned kind) { return kind == ShapeTypes::Kind::Shape; }
+};
+
+/// The type of a single dimension.
+class SizeType : public Type::TypeBase<SizeType, Type> {
+public:
+  using Base::Base;
+
+  static SizeType get(MLIRContext *context) {
+    return Base::get(context, ShapeTypes::Kind::Size);
+  }
+
+  /// Support method to enable LLVM-style type casting.
+  static bool kindof(unsigned kind) { return kind == ShapeTypes::Kind::Size; }
+};
+
+/// The ValueShape represents a (potentially unknown) runtime value and shape.
+class ValueShapeType : public Type::TypeBase<ValueShapeType, Type> {
+public:
+  using Base::Base;
+
+  static ValueShapeType get(MLIRContext *context) {
+    return Base::get(context, ShapeTypes::Kind::ValueShape);
+  }
+
+  /// Support method to enable LLVM-style type casting.
+  static bool kindof(unsigned kind) {
+    return kind == ShapeTypes::Kind::ValueShape;
+  }
+};
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Shape/IR/ShapeOps.h.inc"
+
+} // namespace shape
+} // namespace mlir
+
+#endif // MLIR_SHAPE_IR_SHAPE_H
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
new file mode 100644
index 0000000000000000000000000000000000000000..105677fd08e1ba1a45a6b6a8826d5791410e60a6
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -0,0 +1,284 @@
+//===- Shape.td - Shape operations definition --------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the operation definition file for Shape dialect operations.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SHAPE_OPS
+#define SHAPE_OPS
+
+include "mlir/IR/OpBase.td"
+
+// TODO(jpienaar): Move to base.
+def AnyShaped: ShapedContainerType<[AnyType], IsShapedTypePred, "shaped">;
+
+//===----------------------------------------------------------------------===//
+// Shape Inference dialect definitions
+//===----------------------------------------------------------------------===//
+
+def ShapeDialect : Dialect {
+  let name = "shape";
+
+  let summary = "Types and operations for shape dialect";
+  let description = [{
+    This dialect contains operations for shape inference.
+
+    Note: Unless explicitly stated, all functions that return a shape and take
+    shapes as input, return the invalid shape if one of its operands is an
+    invalid shape. This avoids flagging multiple errors for one verification
+    failure. The dialect itself does not specify how errors should be combined
+    (there are multiple different options, from always chosing first operand,
+    concatting etc. on how to combine them).
+  }];
+
+  let cppNamespace = "shape";
+}
+
+def Shape_SizeType : DialectType<ShapeDialect,
+    CPred<"$_self.isa<DimType>()">, "dim"> {
+  let typeDescription = [{
+    `shape.size` represents a non-negative integer with support for being
+    unknown and invalid.
+
+    Operations on `shape.size` types are specialized to handle unknown/dynamic
+    value. So, for example, `<unknown> + x == <unknown>` for all non-error `x :
+    !shape.size` (e.g., an unknown value does not become known due to addition).
+  }];
+}
+
+def Shape_ShapeType : DialectType<ShapeDialect,
+    CPred<"$_self.isa<ShapeType>()">, "shape"> {
+  let typeDescription = [{
+    `shape.type` represents either an unranked shape, a ranked shape with
+    possibly unknown dimensions or an invalid shape. The rank is of type
+    `shape.size` and, if rank is known, the extent is a 1D tensor of type
+    `shape.size`.
+
+    Shape is printed:
+    * `[*]` if it is an unranked shape
+    * `[?, 2]` if a rank 2 tensor with one unknown dimension
+    * `[3, 4]` is a rank 2 static tensor
+    * `[]` is a scalar
+    * `[1]` is a rank 1 tensor with 1 element
+    * `[invalid]` for an invalid shape
+  }];
+}
+
+def Shape_ElementType : DialectType<ShapeDialect,
+    CPred<"$_self.isa<ElementType>()">, "element type"> {
+  let typeDescription = [{
+    `shape.element_type` represents the element type of the ShapedType. It may
+    be unknown, error or regular element type supported by ShapedType.
+  }];
+}
+
+def Shape_ComponentType : DialectType<ShapeDialect,
+    CPred<"$_self.isa<ComponentType>()">, "component type"> {
+  let typeDescription = [{
+    `shape.element_type` represents the element type of the ShapedType. It may
+    be unknown, error or regular element type supported by ShapedType.
+  }];
+}
+
+def Shape_ValueShapeType : DialectType<ShapeDialect,
+    CPred<"$_self.isa<ValueShapeType>()">, "value shape"> {
+  let typeDescription = [{
+    `shape.value_shape` represents the value produced by an operation (this
+    corresponds to `Value` in the compiler) and a shape. Conceptually this is a
+    tuple of a value (potentially unknown) and `shape.type`. The value and shape
+    can either or both be unknown. If both the `value` and `shape` are known,
+    then the shape of `value` is conformant with `shape`.
+  }];
+}
+
+def Shape_ShapeOrSizeType: AnyTypeOf<[Shape_SizeType, Shape_ShapeType],
+  "shape or size">;
+
+//===----------------------------------------------------------------------===//
+// Shape op definitions
+//===----------------------------------------------------------------------===//
+
+// Base class for the operation in this dialect
+class Shape_Op<string mnemonic, list<OpTrait> traits = []> :
+    Op<ShapeDialect, mnemonic, traits>;
+
+def Shape_AddOp : Shape_Op<"add", [SameOperandsAndResultType]> {
+  let summary = "Addition of sizes";
+  let description = [{
+    Adds two valid sizes as follows:
+    * lhs + rhs = unknown if either lhs or rhs unknown;
+    * lhs + rhs = (int)lhs + (int)rhs if known;
+  }];
+
+  let arguments = (ins Shape_ShapeType:$lhs, Shape_ShapeType:$rhs);
+  let results = (outs Shape_ShapeType:$result);
+}
+
+def Shape_BroadcastOp : Shape_Op<"broadcast", []> {
+  let summary = "Returns the broadcasted output shape of two inputs";
+  let description = [{
+    Computes the broadcasted output shape following:
+    1. If any inputs are unranked, output is unranked;
+    2. Else the input array with number of dimensions smaller than the max
+       input dimension, has 1’s prepended to its shapes and the output shape is
+       calculated as follows:
+
+           output[i] = lhs[i] if lhs[i] == rhs[i] or rhs[i] is unknown/undefined
+                     = rhs[i] if lhs[i] is unknown/undefined
+                     = lhs[i] if rhs[i] == 1
+                     = rhs[i] if lhs[i] == 1
+                     = error  if lhs[i] != rhs[i]
+
+    Op has an optional string attribute for the error case where there is no
+    broadcastable output shape possible for the given inputs.
+  }];
+
+  let arguments = (ins Shape_ShapeType:$lhs, Shape_ShapeType:$rhs,
+                   OptionalAttr<StrAttr>:$error);
+  let results = (outs Shape_ShapeType:$result);
+}
+
+def Shape_ConstantOp : Shape_Op<"constant", []> {
+  let summary = "Creates a shape constant";
+  let description = [{
+    An operation that builds a size or shape from integer or array attribute.
+    It allows for creating dynamically valued shapes by using `?` for unknown
+    values. A constant shape specified with `*` will return an unranked shape.
+
+    ```mlir
+    %x = shape.constant 10 : !shape.size
+    ```
+  }];
+
+  // TODO(jpienaar): Change to a more specialized attribute that would
+  // encapsulate the unknown parsing while using denser packing.
+  let arguments = (ins ArrayAttr:$value);
+  let results = (outs Shape_ShapeOrSizeType:$result);
+}
+
+def Shape_CreateShapeOp : Shape_Op<"create_shape", []> {
+  let summary = "Creates a shape descriptor from a tensor";
+  let description = [{
+    Creates a shape from a 1D integral tensor. The rank equals the number of
+    elements in the tensor, and extent matches the values of the elements.
+  }];
+
+  let arguments = (ins I32Tensor:$input);
+  let results = (outs Shape_ShapeType:$result);
+}
+
+def Shape_JoinOp : Shape_Op<"join", []> {
+  let summary = "Returns the least general shape.size of its operands";
+  let description = [{
+    An operation that computes the least general shape of input operands. This
+    effectively asserts that corresponding static dimensions are equal. The
+    behavior is to match each element of the `shape.type` and propagate the most
+    restrictive information, returning an invalid shape if there are
+    contradictory requirements. E.g., using pseudo code
+
+    ```
+    shape.join([*], [*]) -> [*]
+    shape.join([*], [1, ?]) -> [1, ?]
+    shape.join([1, 2], [1, ?]) -> [1, 2]
+    shape.join([*], [1, 2]) -> [1, 2]
+    shape.join([], []) -> []
+    shape.join([], [*]) -> []
+    shape.join([], [?, ?]) -> [invalid]
+    shape.join([1, ?], [2, ?, ?]) -> [invalid]
+    ```
+
+    `shape.join` also allows specifying an optional error string, that may be
+    used to return an error to the user upon mismatch of dimensions.
+
+    ```mlir
+    %c = shape.join %a, %b, error="<reason>" : !shape.type
+    ```
+  }];
+
+  let arguments = (ins Shape_ShapeOrSizeType:$arg0, Shape_ShapeOrSizeType:$arg1,
+                   OptionalAttr<StrAttr>:$error);
+  let results = (outs Shape_ShapeOrSizeType:$result);
+}
+
+def Shape_MulOp : Shape_Op<"mul", [SameOperandsAndResultType]> {
+  let summary = "Multiplication of sizes";
+  let description = [{
+    Multiplies two valid sizes as follows:
+    - lhs * rhs = unknown if either lhs or rhs unknown;
+    - lhs * rhs = (int)lhs * (int)rhs if both known;
+  }];
+
+  let arguments = (ins Shape_ShapeType:$lhs, Shape_ShapeType:$rhs);
+  let results = (outs Shape_ShapeType:$result);
+}
+
+def Shape_ReduceOp : Shape_Op<"reduce", []> {
+  let summary = "Returns an expression reduced over a shape";
+  let description = [{
+    An operation that takes as input a shape, number of initial values and has a
+    region/function that is applied repeatedly for every dimension of the shape.
+
+    Conceptually this op performs the following reduction:
+
+    ```
+    res[] = init;
+    for (int i = 0, e = shape.rank(); i != e; ++i) {
+      res = fn(i, shape[i], res[0], ..., res[n]);
+    }
+    ```
+
+    Where fn is provided by the user and the result of the reduce op is the
+    last computed output of the reduce function. As an example, computing the
+    number of elements
+
+    ```mlir
+    func @shape_num_elements(%shape : !shape.type) -> !shape.size {
+      %0 = "shape.constant_dim"() {value = 1 : i32} : () -> !shape.size
+      %1 = "shape.reduce"(%shape, %0) ( {
+        ^bb0(%index: i32, %dim: !shape.size, %lci: !shape.size):
+          %acc = "shape.mul"(%lci, %dim) :
+            (!shape.size, !shape.size) -> !shape.size
+          "shape.return"(%acc) : (!shape.size) -> ()
+        }) : (!shape.type, !shape.size) -> (!shape.size)
+      return %1 : !shape.size
+    }
+    ```
+
+    If the shape is unranked, then the results of the op is also unranked.
+  }];
+
+  let arguments = (ins Shape_ShapeType:$shape, Variadic<AnyType>:$args);
+  let results = (outs Variadic<AnyType>:$result);
+
+  let regions = (region SizedRegion<1>:$body);
+}
+
+def Shape_ShapeOfOp : Shape_Op<"shape_of", []> {
+  let summary = "Returns shape of a value or shaped type operand";
+
+  let arguments = (ins AnyTypeOf<[AnyShaped, Shape_ValueShapeType]>:$arg);
+  let results = (outs Shape_ShapeType:$result);
+}
+
+// TODO: Add Ops: if_static, if_ranked
+
+// For testing usage.
+def Shape_DebugPrintOp : Shape_Op<"debug_print", []> {
+  let summary = "Prints the input shape or size";
+  let description = [{
+    Prints the input dim or shape and passes through input.
+
+    Note: This is intended for testing and debugging only.
+  }];
+
+  let arguments = (ins Shape_ShapeOrSizeType:$input);
+  let results =  (outs Shape_ShapeOrSizeType:$output);
+}
+
+#endif // SHAPE_OPS
diff --git a/mlir/include/mlir/Dialect/VectorOps/VectorOps.td b/mlir/include/mlir/Dialect/VectorOps/VectorOps.td
index de7007e3b509e896286863d3ecc34c21796182dc..074a6d0053762233b072c0ebccf7a1a288a39f02 100644
--- a/mlir/include/mlir/Dialect/VectorOps/VectorOps.td
+++ b/mlir/include/mlir/Dialect/VectorOps/VectorOps.td
@@ -183,6 +183,39 @@ def Vector_ContractionOp :
   }];
 }
 
+def Vector_ReductionOp :
+  Vector_Op<"reduction", [NoSideEffect,
+     PredOpTrait<"source operand and result have same element type",
+                 TCresVTEtIsSameAsOpBase<0, 0>>]>,
+    Arguments<(ins StrAttr:$kind, AnyVector:$vector)>,
+    Results<(outs AnyType:$dest)> {
+  let summary = "reduction operation";
+  let description = [{
+    Reduces an 1-D vector "horizontally" into a scalar using the given
+    operation (add/mul/min/max for int/fp and and/or/xor for int only).
+    Note that these operations are restricted to 1-D vectors to remain
+    close to the corresponding LLVM intrinsics:
+
+    http://llvm.org/docs/LangRef.html#experimental-vector-reduction-intrinsics
+
+    Examples:
+    ```
+      %1 = vector.reduction "add", %0 : vector<16xf32> into f32
+
+      %3 = vector.reduction "xor", %2 : vector<4xi32> into i32
+    ```
+  }];
+  let verifier = [{ return ::verify(*this); }];
+  let assemblyFormat = [{
+    $kind `,` $vector attr-dict `:` type($vector) `into` type($dest)
+  }];
+  let extraClassDeclaration = [{
+    VectorType getVectorType() {
+      return vector().getType().cast<VectorType>();
+    }
+  }];
+}
+
 def Vector_BroadcastOp :
   Vector_Op<"broadcast", [NoSideEffect,
      PredOpTrait<"source operand and result have same element type",
diff --git a/mlir/include/mlir/IR/DialectSymbolRegistry.def b/mlir/include/mlir/IR/DialectSymbolRegistry.def
index 361ff89e86bb8e1cc978bfdfb6ffc9b8fa1cd562..d082e16026b10772a5ade160208e7f24dec8aead 100644
--- a/mlir/include/mlir/IR/DialectSymbolRegistry.def
+++ b/mlir/include/mlir/IR/DialectSymbolRegistry.def
@@ -1,6 +1,6 @@
 //===- DialectSymbolRegistry.def - MLIR Dialect Symbol Registry -*- C++ -*-===//
 //
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
@@ -24,6 +24,7 @@ DEFINE_SYM_KIND_RANGE(OPENMP) // OpenMP IR Dialect
 DEFINE_SYM_KIND_RANGE(TOY) // Toy language (tutorial) Dialect
 DEFINE_SYM_KIND_RANGE(SPIRV) // SPIR-V dialect
 DEFINE_SYM_KIND_RANGE(XLA_HLO) // XLA HLO dialect
+DEFINE_SYM_KIND_RANGE(SHAPE) // Shape dialect
 
 // The following ranges are reserved for experimenting with MLIR dialects in a
 // private context without having to register them here.
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index c610e0b4e91170505083550f94e46ec5f0ad024f..d73b7ba9bb716c92ab36460a75ee9357e378a870 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -1040,6 +1040,26 @@ class IntElementsAttr<int width> : ElementsAttrBase<
 def I32ElementsAttr : IntElementsAttr<32>;
 def I64ElementsAttr : IntElementsAttr<64>;
 
+// A `width`-bit integer elements attribute. The attribute should be ranked and
+// has a shape as specified in `dims`.
+class RankedIntElementsAttr<int width, list<int> dims> : IntElementsAttr<width> {
+  // Check that this has the specified shape.
+  let predicate = And<[
+    IntElementsAttr<width>.predicate,
+    CPred<"$_self.cast<DenseIntElementsAttr>().getType().getShape() == "
+        "ArrayRef<int64_t>({" # StrJoinInt<dims>.result # "})">]>;
+
+  let description = width # "-bit int elements attribute of shape [" #
+                    StrJoinInt<dims>.result # "]";
+
+  let constBuilderCall = "DenseIntElementsAttr::get("
+    "RankedTensorType::get({" # StrJoinInt<dims>.result #
+    "}, $_builder.getIntegerType(" # width # ")), makeArrayRef($0))";
+}
+
+class RankedI32ElementsAttr<list<int> dims> : RankedIntElementsAttr<32, dims>;
+class RankedI64ElementsAttr<list<int> dims> : RankedIntElementsAttr<64, dims>;
+
 class FloatElementsAttr<int width> : ElementsAttrBase<
   CPred<"$_self.isa<DenseFPElementsAttr>() &&"
       "$_self.cast<DenseElementsAttr>().getType()."
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
new file mode 100644
index 0000000000000000000000000000000000000000..eeac773cff94a2f4732d34def91a2129b5a7b190
--- /dev/null
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -0,0 +1,59 @@
+//===- InitAllDialects.h - MLIR Dialects Registration -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a helper to trigger the registration of all dialects and
+// passes to the system.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INITALLDIALECTS_H_
+#define MLIR_INITALLDIALECTS_H_
+
+#include "mlir/Dialect/AffineOps/AffineOps.h"
+#include "mlir/Dialect/FxpMathOps/FxpMathOps.h"
+#include "mlir/Dialect/GPU/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/LLVMIR/NVVMDialect.h"
+#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
+#include "mlir/Dialect/Linalg/IR/LinalgOps.h"
+#include "mlir/Dialect/LoopOps/LoopOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Dialect/QuantOps/QuantOps.h"
+#include "mlir/Dialect/SDBM/SDBMDialect.h"
+#include "mlir/Dialect/SPIRV/SPIRVDialect.h"
+#include "mlir/Dialect/StandardOps/Ops.h"
+#include "mlir/Dialect/VectorOps/VectorOps.h"
+#include "mlir/IR/Dialect.h"
+
+namespace mlir {
+
+// This function should be called before creating any MLIRContext if one expect
+// all the possible dialects to be made available to the context automatically.
+inline void registerAllDialects() {
+  static bool init_once = []() {
+    registerDialect<AffineOpsDialect>();
+    registerDialect<fxpmath::FxpMathOpsDialect>();
+    registerDialect<gpu::GPUDialect>();
+    registerDialect<LLVM::LLVMDialect>();
+    registerDialect<linalg::LinalgDialect>();
+    registerDialect<loop::LoopOpsDialect>();
+    registerDialect<omp::OpenMPDialect>();
+    registerDialect<quant::QuantizationDialect>();
+    registerDialect<spirv::SPIRVDialect>();
+    registerDialect<StandardOpsDialect>();
+    registerDialect<vector::VectorOpsDialect>();
+    registerDialect<NVVM::NVVMDialect>();
+    registerDialect<ROCDL::ROCDLDialect>();
+    registerDialect<SDBMDialect>();
+    return true;
+  }();
+  (void)init_once;
+}
+} // namespace mlir
+
+#endif // MLIR_INITALLDIALECTS_H_
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
new file mode 100644
index 0000000000000000000000000000000000000000..e48af6f5ff9edb5cd2bbcde34f48f51f05588878
--- /dev/null
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -0,0 +1,124 @@
+//===- LinkAllPassesAndDialects.h - MLIR Registration -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a helper to trigger the registration of all dialects and
+// passes to the system.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_INITALLPASSES_H_
+#define MLIR_INITALLPASSES_H_
+
+#include "mlir/Analysis/Passes.h"
+#include "mlir/Conversion/GPUToCUDA/GPUToCUDAPass.h"
+#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
+#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
+#include "mlir/Conversion/GPUToSPIRV/ConvertGPUToSPIRVPass.h"
+#include "mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h"
+#include "mlir/Conversion/LinalgToSPIRV/LinalgToSPIRVPass.h"
+#include "mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h"
+#include "mlir/Conversion/StandardToSPIRV/ConvertStandardToSPIRVPass.h"
+#include "mlir/Dialect/FxpMathOps/Passes.h"
+#include "mlir/Dialect/GPU/Passes.h"
+#include "mlir/Dialect/Linalg/Passes.h"
+#include "mlir/Dialect/QuantOps/Passes.h"
+#include "mlir/Dialect/SPIRV/Passes.h"
+#include "mlir/Quantizer/Transforms/Passes.h"
+#include "mlir/Transforms/LocationSnapshot.h"
+#include "mlir/Transforms/Passes.h"
+
+#include <cstdlib>
+
+namespace mlir {
+
+// This function may be called to register the MLIR passes with the
+// global registry.
+// If you're building a compiler, you likely don't need this: you would build a
+// pipeline programmatically without the need to register with the global
+// registry, since it would already be calling the creation routine of the
+// individual passes.
+// The global registry is interesting to interact with the command-line tools.
+inline void registerAllPasses() {
+  // At the moment we still rely on global initializers for registering passes,
+  // but we may not do it in the future.
+  // We must reference the passes in such a way that compilers will not
+  // delete it all as dead code, even with whole program optimization,
+  // yet is effectively a NO-OP. As the compiler isn't smart enough
+  // to know that getenv() never returns -1, this will do the job.
+  if (std::getenv("bar") != (char *)-1)
+    return;
+
+  // Init general passes
+  createCanonicalizerPass();
+  createCSEPass();
+  createVectorizePass({});
+  createLoopUnrollPass();
+  createLoopUnrollAndJamPass();
+  createSimplifyAffineStructuresPass();
+  createLoopFusionPass();
+  createLoopInvariantCodeMotionPass();
+  createAffineLoopInvariantCodeMotionPass();
+  createPipelineDataTransferPass();
+  createLowerAffinePass();
+  createLoopTilingPass(0);
+  createLoopCoalescingPass();
+  createAffineDataCopyGenerationPass(0, 0);
+  createMemRefDataFlowOptPass();
+  createStripDebugInfoPass();
+  createPrintOpStatsPass();
+  createInlinerPass();
+  createSymbolDCEPass();
+  createLocationSnapshotPass({});
+
+  // GPUtoRODCLPass
+  createLowerGpuOpsToROCDLOpsPass();
+
+  // FxpOpsDialect passes
+  fxpmath::createLowerUniformRealMathPass();
+  fxpmath::createLowerUniformCastsPass();
+
+  // GPU
+  createGpuKernelOutliningPass();
+  createSimpleLoopsToGPUPass(0, 0);
+  createLoopToGPUPass({}, {});
+
+  // CUDA
+  createConvertGpuLaunchFuncToCudaCallsPass();
+  createConvertGPUKernelToCubinPass(
+      [](const std::string &, Location, StringRef) { return nullptr; });
+  createLowerGpuOpsToNVVMOpsPass();
+
+  // Linalg
+  createLinalgFusionPass();
+  createLinalgTilingPass();
+  createLinalgTilingToParallelLoopsPass();
+  createLinalgPromotionPass(0);
+  createConvertLinalgToLoopsPass();
+  createConvertLinalgToParallelLoopsPass();
+  createConvertLinalgToAffineLoopsPass();
+  createConvertLinalgToLLVMPass();
+
+  // QuantOps
+  quant::createConvertSimulatedQuantPass();
+  quant::createConvertConstPass();
+  quantizer::createAddDefaultStatsPass();
+  quantizer::createRemoveInstrumentationPass();
+  quantizer::registerInferQuantizedTypesPass();
+
+  // SPIR-V
+  spirv::createDecorateSPIRVCompositeTypeLayoutPass();
+  spirv::createLowerABIAttributesPass();
+  createConvertGPUToSPIRVPass();
+  createConvertStandardToSPIRVPass();
+  createLegalizeStdOpsForSPIRVLoweringPass();
+  createLinalgToSPIRVPass();
+}
+
+} // namespace mlir
+
+#endif // MLIR_INITALLPASSES_H_
diff --git a/mlir/include/mlir/Quantizer/Transforms/Passes.h b/mlir/include/mlir/Quantizer/Transforms/Passes.h
index a8a53b95fae8d7a7c1584aeda2b8cd705280c470..3cd0475d2f8ee998647ca110219912513b13b4d4 100644
--- a/mlir/include/mlir/Quantizer/Transforms/Passes.h
+++ b/mlir/include/mlir/Quantizer/Transforms/Passes.h
@@ -28,6 +28,9 @@ std::unique_ptr<OpPassBase<ModuleOp>>
 createInferQuantizedTypesPass(SolverContext &solverContext,
                               const TargetConfiguration &config);
 
+/// Registers the InferQuantizedTypes pass with the global registry.
+void registerInferQuantizedTypesPass();
+
 /// Creates a pass which removes any instrumentation and hint ops which have
 /// no effect on final runtime.
 std::unique_ptr<OpPassBase<FuncOp>> createRemoveInstrumentationPass();
diff --git a/mlir/include/mlir/Transforms/LoopFusionUtils.h b/mlir/include/mlir/Transforms/LoopFusionUtils.h
index f79571f523f86e51845acdb94c8cfcdd20c8e0d8..2832f050fc55b8ee24668fe7e65190aa4d6f4220 100644
--- a/mlir/include/mlir/Transforms/LoopFusionUtils.h
+++ b/mlir/include/mlir/Transforms/LoopFusionUtils.h
@@ -51,6 +51,11 @@ FusionResult canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
                           unsigned dstLoopDepth,
                           ComputationSliceState *srcSlice);
 
+/// Fuses 'srcForOp' into 'dstForOp' with destination loop block insertion point
+/// and source slice loop bounds specified in 'srcSlice'.
+void fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
+               ComputationSliceState *srcSlice);
+
 /// LoopNestStats aggregates various per-loop statistics (eg. loop trip count
 /// and operation count) for a loop nest up until (and including) the innermost
 /// loop body.
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 673f3db468d38f062e7a1f09ca73cb47011fca4e..8b7495ec0e581faf69c27fcd84721d8ee880ea0f 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -37,14 +37,6 @@ std::unique_ptr<Pass> createCSEPass();
 std::unique_ptr<OpPassBase<FuncOp>>
 createVectorizePass(ArrayRef<int64_t> virtualVectorSize);
 
-/// Creates a pass to allow independent testing of vectorizer functionality with
-/// FileCheck.
-std::unique_ptr<OpPassBase<FuncOp>> createVectorizerTestPass();
-
-/// Creates a pass to lower super-vectors to target-dependent HW vectors.
-std::unique_ptr<OpPassBase<FuncOp>>
-createMaterializeVectorsPass(ArrayRef<int64_t> vectorSize);
-
 /// Creates a loop unrolling pass with the provided parameters.
 /// 'getUnrollFactor' is a function callback for clients to supply a function
 /// that computes an unroll factor - the callback takes precedence over unroll
@@ -95,12 +87,6 @@ std::unique_ptr<OpPassBase<FuncOp>> createLowerAffinePass();
 std::unique_ptr<OpPassBase<FuncOp>>
 createLoopTilingPass(uint64_t cacheSizeBytes);
 
-/// Creates a pass that performs parametric tiling so that the outermost loops
-/// have the given fixed number of iterations.  Assumes outermost loop nests
-/// are permutable.
-std::unique_ptr<OpPassBase<FuncOp>>
-createSimpleParametricTilingPass(ArrayRef<int64_t> outerLoopSizes);
-
 /// Creates a pass that transforms perfectly nested loops with independent
 /// bounds into a single loop.
 std::unique_ptr<OpPassBase<FuncOp>> createLoopCoalescingPass();
@@ -120,11 +106,12 @@ std::unique_ptr<OpPassBase<FuncOp>> createMemRefDataFlowOptPass();
 /// Creates a pass to strip debug information from a function.
 std::unique_ptr<Pass> createStripDebugInfoPass();
 
-/// Creates a pass which tests loop fusion utilities.
-std::unique_ptr<OpPassBase<FuncOp>> createTestLoopFusionPass();
+/// Creates a pass which prints the list of ops and the number of occurences in
+/// the module.
+std::unique_ptr<OpPassBase<ModuleOp>> createPrintOpStatsPass();
 
-/// Creates a pass which inlines calls and callable operations as defined by the
-/// CallGraph.
+/// Creates a pass which inlines calls and callable operations as defined by
+/// the CallGraph.
 std::unique_ptr<Pass> createInlinerPass();
 
 /// Creates a pass which delete symbol operations that are unreachable. This
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index 570f956ee4e93209cce7af4da279f70f76786fe5..059bdfb1661fd69bf59dc94ebf6c998629ecb9a5 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -95,6 +95,24 @@ private:
   const std::string f64Func;
 };
 
+namespace gpu {
+/// Returns a predicate to be used with addDynamicallyLegalOp. The predicate
+/// returns false for calls to the provided intrinsics and true otherwise.
+inline std::function<bool(Operation *)>
+filterIllegalLLVMIntrinsics(ArrayRef<StringRef> intrinsics, MLIRContext *ctx) {
+  SmallVector<StringRef, 4> illegalIds(intrinsics.begin(), intrinsics.end());
+  return [illegalIds](Operation *op) -> bool {
+    LLVM::CallOp callOp = dyn_cast<LLVM::CallOp>(op);
+    if (!callOp || !callOp.callee())
+      return true;
+    StringRef callee = callOp.callee().getValue();
+    return !llvm::any_of(illegalIds, [callee](StringRef intrinsic) {
+      return callee.equals(intrinsic);
+    });
+  };
+}
+} // namespace gpu
+
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_GPUCOMMON_OPTOFUNCCALLLOWERING_H_
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 09fbd309516852748dca057946ad43b142b54328..d9f37dbf0d8c773af5fd9dcbbb62fc1c7a58860c 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -695,6 +695,8 @@ public:
     target.addIllegalOp<FuncOp>();
     target.addLegalDialect<LLVM::LLVMDialect>();
     target.addLegalDialect<NVVM::NVVMDialect>();
+    target.addDynamicallyLegalOp<mlir::LLVM::CallOp>(
+        gpu::filterIllegalLLVMIntrinsics({"tanh", "tanhf"}, m.getContext()));
     // TODO(csigg): Remove once we support replacing non-root ops.
     target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
     if (failed(applyPartialConversion(m, target, patterns, &converter)))
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index d74fcdbf05071112f4a88096f16af5c276e96c62..0fd8be01f15a12c358c6b09bf4a5604560a39b3e 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -65,8 +65,9 @@ public:
     target.addLegalDialect<LLVM::LLVMDialect, ROCDL::ROCDLDialect>();
     target.addIllegalOp<LLVM::FAbsOp, LLVM::FCeilOp, LLVM::CosOp,
                         LLVM::ExpOp>();
-    target.addDynamicallyLegalOp<FuncOp>(
-        [&](FuncOp op) { return converter.isSignatureLegal(op.getType()); });
+    target.addDynamicallyLegalOp<LLVM::CallOp>(
+        gpu::filterIllegalLLVMIntrinsics({"tanh", "tanhf"}, m.getContext()));
+    target.addIllegalOp<FuncOp>();
     if (failed(applyPartialConversion(m, target, patterns, &converter)))
       signalPassFailure();
   }
diff --git a/mlir/lib/Conversion/StandardToLLVM/ConvertStandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/ConvertStandardToLLVM.cpp
index 36c98d0e85b156c39f366631a249f1aa6f81d425..17209c72e05450e981c546c5d4d0cbb7fff87cee 100644
--- a/mlir/lib/Conversion/StandardToLLVM/ConvertStandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/ConvertStandardToLLVM.cpp
@@ -36,25 +36,6 @@ using namespace mlir;
 
 #define PASS_NAME "convert-std-to-llvm"
 
-static llvm::cl::OptionCategory
-    clOptionsCategory("Standard to LLVM lowering options");
-
-static llvm::cl::opt<bool>
-    clUseAlloca(PASS_NAME "-use-alloca",
-                llvm::cl::desc("Replace emission of malloc/free by alloca"),
-                llvm::cl::init(false));
-
-static llvm::cl::opt<bool>
-    clEmitCWrappers(PASS_NAME "-emit-c-wrappers",
-                    llvm::cl::desc("Emit C-compatible wrapper functions"),
-                    llvm::cl::init(false));
-
-static llvm::cl::opt<bool> clUseBarePtrCallConv(
-    PASS_NAME "-use-bare-ptr-memref-call-conv",
-    llvm::cl::desc("Replace FuncOp's MemRef arguments with "
-                   "bare pointers to the MemRef element types"),
-    llvm::cl::init(false));
-
 // Extract an LLVM IR type from the LLVM IR dialect type.
 static LLVM::LLVMType unwrap(Type type) {
   if (!type)
@@ -2730,11 +2711,14 @@ namespace {
 /// A pass converting MLIR operations into the LLVM IR dialect.
 struct LLVMLoweringPass : public ModulePass<LLVMLoweringPass> {
   /// Creates an LLVM lowering pass.
-  explicit LLVMLoweringPass(bool useAlloca = false,
-                            bool useBarePtrCallConv = false,
-                            bool emitCWrappers = false)
-      : useAlloca(useAlloca), useBarePtrCallConv(useBarePtrCallConv),
-        emitCWrappers(emitCWrappers) {}
+  explicit LLVMLoweringPass(bool useAlloca, bool useBarePtrCallConv,
+                            bool emitCWrappers) {
+    this->useAlloca = useAlloca;
+    this->useBarePtrCallConv = useBarePtrCallConv;
+    this->emitCWrappers = emitCWrappers;
+  }
+  explicit LLVMLoweringPass() {}
+  LLVMLoweringPass(const LLVMLoweringPass &pass) {}
 
   /// Run the dialect converter on the module.
   void runOnModule() override {
@@ -2769,27 +2753,33 @@ struct LLVMLoweringPass : public ModulePass<LLVMLoweringPass> {
   }
 
   /// Use `alloca` instead of `call @malloc` for converting std.alloc.
-  bool useAlloca;
+  Option<bool> useAlloca{
+      *this, "use-alloca",
+      llvm::cl::desc("Replace emission of malloc/free by alloca"),
+      llvm::cl::init(false)};
 
   /// Convert memrefs to bare pointers in function signatures.
-  bool useBarePtrCallConv;
+  Option<bool> useBarePtrCallConv{
+      *this, "use-bare-ptr-memref-call-conv",
+      llvm::cl::desc("Replace FuncOp's MemRef arguments with "
+                     "bare pointers to the MemRef element types"),
+      llvm::cl::init(false)};
 
   /// Emit wrappers for C-compatible pointer-to-struct memref descriptors.
-  bool emitCWrappers;
+  Option<bool> emitCWrappers{
+      *this, "emit-c-wrappers",
+      llvm::cl::desc("Emit C-compatible wrapper functions"),
+      llvm::cl::init(false)};
 };
 } // end namespace
 
 std::unique_ptr<OpPassBase<ModuleOp>>
-mlir::createLowerToLLVMPass(bool useAlloca, bool emitCWrappers) {
-  return std::make_unique<LLVMLoweringPass>(useAlloca, emitCWrappers);
+mlir::createLowerToLLVMPass(bool useAlloca, bool useBarePtrCallConv,
+                            bool emitCWrappers) {
+  return std::make_unique<LLVMLoweringPass>(useAlloca, useBarePtrCallConv,
+                                            emitCWrappers);
 }
 
 static PassRegistration<LLVMLoweringPass>
-    pass(PASS_NAME,
-         "Convert scalar and vector operations from the "
-         "Standard to the LLVM dialect",
-         [] {
-           return std::make_unique<LLVMLoweringPass>(
-               clUseAlloca.getValue(), clUseBarePtrCallConv.getValue(),
-               clEmitCWrappers.getValue());
-         });
+    pass(PASS_NAME, "Convert scalar and vector operations from the "
+                    "Standard to the LLVM dialect");
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index a3d724bea83430dfb3c661357f2e8c5ca2439097..9fcad2f5063a4cb8d0ae4ee30abf877f882d80b8 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -124,6 +124,7 @@ static SmallVector<int64_t, 4> getI64SubArray(ArrayAttr arrayAttr,
 }
 
 namespace {
+
 class VectorBroadcastOpConversion : public LLVMOpLowering {
 public:
   explicit VectorBroadcastOpConversion(MLIRContext *context,
@@ -272,6 +273,73 @@ private:
   }
 };
 
+class VectorReductionOpConversion : public LLVMOpLowering {
+public:
+  explicit VectorReductionOpConversion(MLIRContext *context,
+                                       LLVMTypeConverter &typeConverter)
+      : LLVMOpLowering(vector::ReductionOp::getOperationName(), context,
+                       typeConverter) {}
+
+  PatternMatchResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto reductionOp = cast<vector::ReductionOp>(op);
+    auto kind = reductionOp.kind();
+    Type eltType = reductionOp.dest().getType();
+    Type llvmType = lowering.convertType(eltType);
+    if (eltType.isInteger(32) || eltType.isInteger(64)) {
+      // Integer reductions: add/mul/min/max/and/or/xor.
+      if (kind == "add")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_add>(
+            op, llvmType, operands[0]);
+      else if (kind == "mul")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_mul>(
+            op, llvmType, operands[0]);
+      else if (kind == "min")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_smin>(
+            op, llvmType, operands[0]);
+      else if (kind == "max")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_smax>(
+            op, llvmType, operands[0]);
+      else if (kind == "and")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_and>(
+            op, llvmType, operands[0]);
+      else if (kind == "or")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_or>(
+            op, llvmType, operands[0]);
+      else if (kind == "xor")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_xor>(
+            op, llvmType, operands[0]);
+      else
+        return matchFailure();
+      return matchSuccess();
+
+    } else if (eltType.isF32() || eltType.isF64()) {
+      // Floating-point reductions: add/mul/min/max
+      if (kind == "add") {
+        Value zero = rewriter.create<LLVM::ConstantOp>(
+            op->getLoc(), llvmType, rewriter.getZeroAttr(eltType));
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_v2_fadd>(
+            op, llvmType, zero, operands[0]);
+      } else if (kind == "mul") {
+        Value one = rewriter.create<LLVM::ConstantOp>(
+            op->getLoc(), llvmType, rewriter.getFloatAttr(eltType, 1.0));
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_v2_fmul>(
+            op, llvmType, one, operands[0]);
+      } else if (kind == "min")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_fmin>(
+            op, llvmType, operands[0]);
+      else if (kind == "max")
+        rewriter.replaceOpWithNewOp<LLVM::experimental_vector_reduce_fmax>(
+            op, llvmType, operands[0]);
+      else
+        return matchFailure();
+      return matchSuccess();
+    }
+    return matchFailure();
+  }
+};
+
 class VectorShuffleOpConversion : public LLVMOpLowering {
 public:
   explicit VectorShuffleOpConversion(MLIRContext *context,
@@ -1056,12 +1124,12 @@ void mlir::populateVectorToLLVMConversionPatterns(
                   VectorInsertStridedSliceOpDifferentRankRewritePattern,
                   VectorInsertStridedSliceOpSameRankRewritePattern,
                   VectorStridedSliceOpConversion>(ctx);
-  patterns.insert<VectorBroadcastOpConversion, VectorShuffleOpConversion,
-                  VectorExtractElementOpConversion, VectorExtractOpConversion,
-                  VectorFMAOp1DConversion, VectorInsertElementOpConversion,
-                  VectorInsertOpConversion, VectorOuterProductOpConversion,
-                  VectorTypeCastOpConversion, VectorPrintOpConversion>(
-      ctx, converter);
+  patterns.insert<VectorBroadcastOpConversion, VectorReductionOpConversion,
+                  VectorShuffleOpConversion, VectorExtractElementOpConversion,
+                  VectorExtractOpConversion, VectorFMAOp1DConversion,
+                  VectorInsertElementOpConversion, VectorInsertOpConversion,
+                  VectorOuterProductOpConversion, VectorTypeCastOpConversion,
+                  VectorPrintOpConversion>(ctx, converter);
 }
 
 namespace {
diff --git a/mlir/lib/Dialect/AffineOps/CMakeLists.txt b/mlir/lib/Dialect/AffineOps/CMakeLists.txt
index 9648f27f1067ca8f65bec8104b0ab95ea8c95e9b..3d07cde5d6e6138a014eb98b34bbca7b13479980 100644
--- a/mlir/lib/Dialect/AffineOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/AffineOps/CMakeLists.txt
@@ -1,9 +1,8 @@
 add_llvm_library(MLIRAffineOps
   AffineOps.cpp
   AffineValueMap.cpp
-  DialectRegistration.cpp
   EDSC/Builders.cpp
-  
+
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AffineOps
   )
diff --git a/mlir/lib/Dialect/AffineOps/DialectRegistration.cpp b/mlir/lib/Dialect/AffineOps/DialectRegistration.cpp
deleted file mode 100644
index b9535b5f0d2a5caec35e6d064691593e947206a9..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/AffineOps/DialectRegistration.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===- DialectRegistration.cpp - Register Affine Op dialect ---------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/AffineOps/AffineOps.h"
-using namespace mlir;
-
-// Static initialization for Affine op dialect registration.
-static DialectRegistration<AffineOpsDialect> StandardOps;
diff --git a/mlir/lib/Dialect/CMakeLists.txt b/mlir/lib/Dialect/CMakeLists.txt
index 77f5296336588ff1c3906e692f9db5b24e9e83cb..0542830c1660ddf7c165700faaec3fe77a4cf992 100644
--- a/mlir/lib/Dialect/CMakeLists.txt
+++ b/mlir/lib/Dialect/CMakeLists.txt
@@ -7,10 +7,16 @@ add_subdirectory(LoopOps)
 add_subdirectory(OpenMP)
 add_subdirectory(QuantOps)
 add_subdirectory(SDBM)
+add_subdirectory(Shape)
 add_subdirectory(SPIRV)
 add_subdirectory(StandardOps)
 add_subdirectory(VectorOps)
 
+
+set(LLVM_OPTIONAL_SOURCES
+  Traits.cpp
+)
+
 add_llvm_library(MLIRDialect
   Traits.cpp
 
@@ -18,3 +24,29 @@ add_llvm_library(MLIRDialect
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect
   )
 target_link_libraries(MLIRDialect MLIRIR)
+
+# Create a dummy MLIRAllDialects library for the purpose
+# of having an easy tracking of all dialects when linking
+# them in tools.
+# Empty libraries aren't possible with CMake, create a dummy file.
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/all_dialects.c "typedef int make_iso_compilers_happy;\n")
+add_llvm_library(MLIRAllDialects
+  ${CMAKE_CURRENT_BINARY_DIR}/all_dialects.c
+)
+target_link_libraries(MLIRAllDialects
+  MLIRAffineOps
+  MLIRFxpMathOps
+  MLIRGPU
+  MLIRLLVMIR
+  MLIRNVVMIR
+  MLIRROCDLIR
+  MLIRLinalgOps
+  MLIRLoopOps
+  MLIROpenMP
+  MLIRQuantOps
+  MLIRSDBM
+  MLIRShape
+  MLIRSPIRV
+  MLIRStandardOps
+  MLIRVectorOps
+)
diff --git a/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt b/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt
index 61806e0d8089a90269e8ecd8b7160049455c0477..ba9a3001154ee96b84c99363413084254e496611 100644
--- a/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/FxpMathOps/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_llvm_library(MLIRFxpMathOps
   IR/FxpMathOps.cpp
-  IR/DialectRegistration.cpp
   Transforms/LowerUniformRealMath.cpp
 
   ADDITIONAL_HEADER_DIRS
diff --git a/mlir/lib/Dialect/FxpMathOps/IR/DialectRegistration.cpp b/mlir/lib/Dialect/FxpMathOps/IR/DialectRegistration.cpp
deleted file mode 100644
index d732ff944b6de9e85325ae2a424d6db0dc308366..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/FxpMathOps/IR/DialectRegistration.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-//===- DialectRegistration.cpp - Register FxpMathOps dialect --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/FxpMathOps/FxpMathOps.h"
-
-using namespace mlir;
-using namespace mlir::fxpmath;
-
-// Static initialization for the fxpmath ops dialect registration.
-static mlir::DialectRegistration<FxpMathOpsDialect> FxpMathOps;
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index 063aa1ea5ce69a6da7b8d46c130c0abf5483d49e..17342f85f9d2501e85a726527106351a8f4305b3 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_llvm_library(MLIRGPU
   IR/GPUDialect.cpp
-  IR/DialectRegistration.cpp
   Transforms/AllReduceLowering.cpp
   Transforms/KernelOutlining.cpp
   Transforms/MemoryPromotion.cpp
diff --git a/mlir/lib/Dialect/GPU/IR/DialectRegistration.cpp b/mlir/lib/Dialect/GPU/IR/DialectRegistration.cpp
deleted file mode 100644
index 2a1b68779dcdb42f578645c4f05e10052c952fa0..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/GPU/IR/DialectRegistration.cpp
+++ /dev/null
@@ -1,12 +0,0 @@
-//===- DialectRegistration.cpp - MLIR GPU dialect registration ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/GPU/GPUDialect.h"
-
-// Static initialization for GPU dialect registration.
-static mlir::DialectRegistration<mlir::gpu::GPUDialect> kernelDialect;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 8e2e3c5ff0658a631d1e062a6ae9ed4f4b9ec8cd..a63a593dfc1e182c4fc37fda1abb94235defa1d4 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -1658,8 +1658,6 @@ LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op,
   return success();
 }
 
-static DialectRegistration<LLVMDialect> llvmDialect;
-
 //===----------------------------------------------------------------------===//
 // LLVMType.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index b7b32df12ca61ade98b9bbb371fc035e5147e2ca..210507a61fc3ce9e18de305fe8cf4c7318994555 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -159,4 +159,3 @@ namespace NVVM {
 } // namespace NVVM
 } // namespace mlir
 
-static DialectRegistration<NVVMDialect> nvvmDialect;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index 28ee28f71e868322f379cc4287bec936f874701a..83d3ffb1d761e9fe488037ed87a530075847f05a 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -52,4 +52,3 @@ namespace ROCDL {
 } // namespace ROCDL
 } // namespace mlir
 
-static DialectRegistration<ROCDLDialect> rocdlDialect;
diff --git a/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt b/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt
index df71c831a7f7eecf60ad8e0c52d83d2fbaaa8b23..40ce1724235a6a4c77a636588fcd3dc7a07f8987 100644
--- a/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/IR/CMakeLists.txt
@@ -7,7 +7,6 @@ set(LIBS
 add_llvm_library(MLIRLinalgOps
   LinalgOps.cpp
   LinalgTypes.cpp
-  LinalgRegistration.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Linalg
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgRegistration.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgRegistration.cpp
deleted file mode 100644
index f9c2c9b4b637e45120c1bf0bd2848a33fce118e9..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/Linalg/IR/LinalgRegistration.cpp
+++ /dev/null
@@ -1,16 +0,0 @@
-//===- LinalgRegistration.cpp - Register the linalg dialect statically ----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/Linalg/IR/LinalgOps.h"
-#include "mlir/Dialect/Linalg/IR/LinalgTypes.h"
-
-using namespace mlir;
-using namespace mlir::linalg;
-
-// Static initialization for LinalgOps dialect registration.
-static DialectRegistration<LinalgDialect> LinalgOps;
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 445323b42e469b472f9d3c2db29cf5256551ab36..edb1a4eb5eb4f647a5b95aec1add47e7047c5551 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -30,5 +30,3 @@ namespace omp {
 #include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
 } // namespace omp
 } // namespace mlir
-
-static DialectRegistration<OpenMPDialect> ompDialect;
diff --git a/mlir/lib/Dialect/QuantOps/CMakeLists.txt b/mlir/lib/Dialect/QuantOps/CMakeLists.txt
index bfbcbdbd3a6e180713c8619bec55c69c0cdd6c46..483056675ef444f08053bc25bedc9d85fb9a8e84 100644
--- a/mlir/lib/Dialect/QuantOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/QuantOps/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_llvm_library(MLIRQuantOps
-  IR/DialectRegistration.cpp
   IR/QuantOps.cpp
   IR/QuantTypes.cpp
   IR/TypeDetail.h
diff --git a/mlir/lib/Dialect/QuantOps/IR/DialectRegistration.cpp b/mlir/lib/Dialect/QuantOps/IR/DialectRegistration.cpp
deleted file mode 100644
index 8321e88b0b6c15d3c0e192d33e2678baa074b7a9..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/QuantOps/IR/DialectRegistration.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-//===- DialectRegistration.cpp - Register Quantization dialect ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/QuantOps/QuantOps.h"
-
-using namespace mlir;
-using namespace mlir::quant;
-
-// Static initialization for Quantization dialect registration.
-static mlir::DialectRegistration<QuantizationDialect> QuantizationOps;
diff --git a/mlir/lib/Dialect/SDBM/CMakeLists.txt b/mlir/lib/Dialect/SDBM/CMakeLists.txt
index e36308e0edafd4cccc90d1d95dd080bc1feac017..36be9466c573f267d812e71f680a075739a0dc69 100644
--- a/mlir/lib/Dialect/SDBM/CMakeLists.txt
+++ b/mlir/lib/Dialect/SDBM/CMakeLists.txt
@@ -1,7 +1,6 @@
 add_llvm_library(MLIRSDBM
   SDBM.cpp
   SDBMExpr.cpp
-  SDBMDialect.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SDBM
diff --git a/mlir/lib/Dialect/SDBM/SDBMDialect.cpp b/mlir/lib/Dialect/SDBM/SDBMDialect.cpp
deleted file mode 100644
index 9b18a126958bc3d813f8327344b9c65958660738..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/SDBM/SDBMDialect.cpp
+++ /dev/null
@@ -1,11 +0,0 @@
-//===- SDBMDialect.cpp - Dialect for striped difference-bound matrices ----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/SDBM/SDBMDialect.h"
-
-static mlir::DialectRegistration<mlir::SDBMDialect> SDBMDialect;
diff --git a/mlir/lib/Dialect/SPIRV/CMakeLists.txt b/mlir/lib/Dialect/SPIRV/CMakeLists.txt
index da7fd1ae75411d227743eee3818e36fde7d80f44..d0ff25ef68f019d89a7de32db36b85c387290bb1 100644
--- a/mlir/lib/Dialect/SPIRV/CMakeLists.txt
+++ b/mlir/lib/Dialect/SPIRV/CMakeLists.txt
@@ -3,7 +3,6 @@ mlir_tablegen(SPIRVCanonicalization.inc -gen-rewriters)
 add_public_tablegen_target(MLIRSPIRVCanonicalizationIncGen)
 
 add_llvm_library(MLIRSPIRV
-  DialectRegistration.cpp
   LayoutUtils.cpp
   SPIRVDialect.cpp
   SPIRVOps.cpp
diff --git a/mlir/lib/Dialect/SPIRV/DialectRegistration.cpp b/mlir/lib/Dialect/SPIRV/DialectRegistration.cpp
deleted file mode 100644
index 2af7593487c0f167681cb41a08e0ae31769f4f0e..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/SPIRV/DialectRegistration.cpp
+++ /dev/null
@@ -1,12 +0,0 @@
-//===- DialectRegistration.cpp - MLIR SPIR-V dialect registration ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/SPIRV/SPIRVDialect.h"
-
-// Static initialization for SPIR-V dialect registration.
-static mlir::DialectRegistration<mlir::spirv::SPIRVDialect> spirvDialect;
diff --git a/mlir/lib/Dialect/Shape/CMakeLists.txt b/mlir/lib/Dialect/Shape/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..219a78b563df23ef2e5e3ea792a205664c134406
--- /dev/null
+++ b/mlir/lib/Dialect/Shape/CMakeLists.txt
@@ -0,0 +1,9 @@
+file(GLOB globbed *.c *.cpp)
+add_llvm_library(MLIRShape
+  ${globbed}
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Shape
+  )
+add_dependencies(MLIRShape MLIRShapeOpsIncGen LLVMSupport)
+target_link_libraries(MLIRShape LLVMSupport)
diff --git a/mlir/lib/Dialect/LoopOps/DialectRegistration.cpp b/mlir/lib/Dialect/Shape/DialectRegistration.cpp
similarity index 59%
rename from mlir/lib/Dialect/LoopOps/DialectRegistration.cpp
rename to mlir/lib/Dialect/Shape/DialectRegistration.cpp
index c0ca419d506cdd02edc4eeb2bc2923ffa1d4c599..818ac149453cc312f756bca42c7ae1e94ea34565 100644
--- a/mlir/lib/Dialect/LoopOps/DialectRegistration.cpp
+++ b/mlir/lib/Dialect/Shape/DialectRegistration.cpp
@@ -1,4 +1,4 @@
-//===- DialectRegistration.cpp - Register loop dialect --------------------===//
+//===- DialectRegistration.cpp - Register shape dialect -------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/LoopOps/LoopOps.h"
+#include "mlir/Dialect/Shape/IR/Shape.h"
 using namespace mlir;
 
-// Static initialization for loop dialect registration.
-static DialectRegistration<loop::LoopOpsDialect> LoopOps;
+// Static initialization for shape dialect registration.
+static DialectRegistration<shape::ShapeDialect> ShapeOps;
diff --git a/mlir/lib/Dialect/StandardOps/DialectRegistration.cpp b/mlir/lib/Dialect/StandardOps/DialectRegistration.cpp
deleted file mode 100644
index 54d35f0bf8a7210f31a4b6a5e3d88e2f55538d85..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/StandardOps/DialectRegistration.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===- DialectRegistration.cpp - Register standard Op dialect -------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/StandardOps/Ops.h"
-using namespace mlir;
-
-// Static initialization for standard op dialect registration.
-static DialectRegistration<StandardOpsDialect> StandardOps;
diff --git a/mlir/lib/Dialect/VectorOps/CMakeLists.txt b/mlir/lib/Dialect/VectorOps/CMakeLists.txt
index 2a9071036331e612feea1af1d2ffed0e32058d2e..27cb10f53c20df5bc2ea91201192c3d4b7cd672e 100644
--- a/mlir/lib/Dialect/VectorOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/VectorOps/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_llvm_library(MLIRVectorOps
-  DialectRegistration.cpp
   VectorOps.cpp
   VectorTransforms.cpp
   VectorUtils.cpp
diff --git a/mlir/lib/Dialect/VectorOps/DialectRegistration.cpp b/mlir/lib/Dialect/VectorOps/DialectRegistration.cpp
deleted file mode 100644
index 848504f8c52eb0d9abea1a18052da5dbe5022f03..0000000000000000000000000000000000000000
--- a/mlir/lib/Dialect/VectorOps/DialectRegistration.cpp
+++ /dev/null
@@ -1,13 +0,0 @@
-//===- DialectRegistration.cpp - Register super vectorization dialect -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/VectorOps/VectorOps.h"
-using namespace mlir;
-
-// Static initialization for VectorOps dialect registration.
-static DialectRegistration<vector::VectorOpsDialect> VectorOps;
diff --git a/mlir/lib/Dialect/VectorOps/VectorOps.cpp b/mlir/lib/Dialect/VectorOps/VectorOps.cpp
index a987a54f5ea17b30936620c854a2873faf387a27..b4d7aee70b17385f522150f74d10fa10a5b7ed8e 100644
--- a/mlir/lib/Dialect/VectorOps/VectorOps.cpp
+++ b/mlir/lib/Dialect/VectorOps/VectorOps.cpp
@@ -60,6 +60,33 @@ ArrayAttr vector::getVectorSubscriptAttr(Builder &builder,
   return builder.getI64ArrayAttr(values);
 }
 
+//===----------------------------------------------------------------------===//
+// ReductionOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(ReductionOp op) {
+  // Verify for 1-D vector.
+  int64_t rank = op.getVectorType().getRank();
+  if (rank != 1)
+    return op.emitOpError("unsupported reduction rank: ") << rank;
+
+  // Verify supported reduction kind.
+  auto kind = op.kind();
+  Type eltType = op.dest().getType();
+  if (kind == "add" || kind == "mul" || kind == "min" || kind == "max") {
+    if (eltType.isF32() || eltType.isF64() || eltType.isInteger(32) ||
+        eltType.isInteger(64))
+      return success();
+    return op.emitOpError("unsupported reduction type");
+  }
+  if (kind == "and" || kind == "or" || kind == "xor") {
+    if (eltType.isInteger(32) || eltType.isInteger(64))
+      return success();
+    return op.emitOpError("unsupported reduction type");
+  }
+  return op.emitOpError("unknown reduction kind: ") << kind;
+}
+
 //===----------------------------------------------------------------------===//
 // ContractionOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp b/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp
index 440c7707ce75fd80b85bf3b3318bd1faac900cc6..8bdeb92afe5e3ddc8196a7f824704daf5887c56e 100644
--- a/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/VectorOps/VectorTransforms.cpp
@@ -646,6 +646,90 @@ struct SplitTransferWriteOp : public OpRewritePattern<vector::TransferWriteOp> {
   }
 };
 
+/// Decomposes ShapeCastOp on tuple-of-vectors to multiple ShapeCastOps, each
+/// on vector types.
+struct ShapeCastOpDecomposer : public OpRewritePattern<vector::ShapeCastOp> {
+  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
+
+  PatternMatchResult matchAndRewrite(vector::ShapeCastOp shapeCastOp,
+                                     PatternRewriter &rewriter) const override {
+    // Check if 'shapeCastOp' has tuple source/result type.
+    auto sourceTupleType =
+        shapeCastOp.source().getType().dyn_cast_or_null<TupleType>();
+    auto resultTupleType =
+        shapeCastOp.result().getType().dyn_cast_or_null<TupleType>();
+    if (!sourceTupleType || !resultTupleType)
+      return matchFailure();
+    assert(sourceTupleType.size() == resultTupleType.size());
+
+    // Create single-vector ShapeCastOp for each source tuple element.
+    Location loc = shapeCastOp.getLoc();
+    SmallVector<Value, 8> resultElements;
+    resultElements.reserve(resultTupleType.size());
+    for (unsigned i = 0, e = sourceTupleType.size(); i < e; ++i) {
+      auto sourceElement = rewriter.create<vector::TupleGetOp>(
+          loc, sourceTupleType.getType(i), shapeCastOp.source(),
+          rewriter.getI64IntegerAttr(i));
+      resultElements.push_back(rewriter.create<vector::ShapeCastOp>(
+          loc, resultTupleType.getType(i), sourceElement));
+    }
+
+    // Replace 'shapeCastOp' with tuple of 'resultElements'.
+    rewriter.replaceOpWithNewOp<vector::TupleOp>(shapeCastOp, resultTupleType,
+                                                 resultElements);
+    return matchSuccess();
+  }
+};
+
+/// ShapeCastOpFolder folds cancelling ShapeCastOps away.
+//
+// Example:
+//
+//  The following MLIR with cancelling ShapeCastOps:
+//
+//   %0 = source : vector<5x4x2xf32>
+//   %1 = shape_cast %0 : vector<5x4x2xf32> to vector<20x2xf32>
+//   %2 = shape_cast %1 : vector<20x2xf32> to vector<5x4x2xf32>
+//   %3 = user %2 : vector<5x4x2xf32>
+//
+//  Should canonicalize to the following:
+//
+//   %0 = source : vector<5x4x2xf32>
+//   %1 = user %0 : vector<5x4x2xf32>
+//
+struct ShapeCastOpFolder : public OpRewritePattern<vector::ShapeCastOp> {
+  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
+
+  PatternMatchResult matchAndRewrite(vector::ShapeCastOp shapeCastOp,
+                                     PatternRewriter &rewriter) const override {
+    // Check if 'shapeCastOp' has vector source/result type.
+    auto sourceVectorType =
+        shapeCastOp.source().getType().dyn_cast_or_null<VectorType>();
+    auto resultVectorType =
+        shapeCastOp.result().getType().dyn_cast_or_null<VectorType>();
+    if (!sourceVectorType || !resultVectorType)
+      return matchFailure();
+
+    // Check if shape cast op source operand is also a shape cast op.
+    auto sourceShapeCastOp = dyn_cast_or_null<vector::ShapeCastOp>(
+        shapeCastOp.source().getDefiningOp());
+    if (!sourceShapeCastOp)
+      return matchFailure();
+    auto operandSourceVectorType =
+        sourceShapeCastOp.source().getType().cast<VectorType>();
+    auto operandResultVectorType =
+        sourceShapeCastOp.result().getType().cast<VectorType>();
+
+    // Check if shape cast operations invert each other.
+    if (operandSourceVectorType != resultVectorType ||
+        operandResultVectorType != sourceVectorType)
+      return matchFailure();
+
+    rewriter.replaceOp(shapeCastOp, sourceShapeCastOp.source());
+    return matchSuccess();
+  }
+};
+
 // Patter rewrite which forward tuple elements to their users.
 // User(TupleGetOp(ExtractSlicesOp(InsertSlicesOp(TupleOp(Producer)))))
 //   -> User(Producer)
@@ -784,8 +868,8 @@ public:
 // TODO(andydavis) Add this as DRR pattern.
 void mlir::vector::populateVectorToVectorTransformationPatterns(
     OwningRewritePatternList &patterns, MLIRContext *context) {
-  patterns.insert<SplitTransferReadOp, SplitTransferWriteOp, TupleGetFolderOp>(
-      context);
+  patterns.insert<ShapeCastOpDecomposer, ShapeCastOpFolder, SplitTransferReadOp,
+                  SplitTransferWriteOp, TupleGetFolderOp>(context);
 }
 
 void mlir::vector::populateVectorSlicesLoweringPatterns(
diff --git a/mlir/lib/IR/StandardTypes.cpp b/mlir/lib/IR/StandardTypes.cpp
index bd12cff65f7e292392d3cafb0cd19b1b269ab4c1..c4269af9030e3c51c3d10f4537c8c2950c96fdb9 100644
--- a/mlir/lib/IR/StandardTypes.cpp
+++ b/mlir/lib/IR/StandardTypes.cpp
@@ -333,7 +333,8 @@ MemRefType MemRefType::getImpl(ArrayRef<int64_t> shape, Type elementType,
   auto *context = elementType.getContext();
 
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>())
+  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>() &&
+      !elementType.isa<ComplexType>())
     return emitOptionalError(location, "invalid memref element type"),
            MemRefType();
 
@@ -411,7 +412,8 @@ LogicalResult UnrankedMemRefType::verifyConstructionInvariants(
     Optional<Location> loc, MLIRContext *context, Type elementType,
     unsigned memorySpace) {
   // Check that memref is formed from allowed types.
-  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>())
+  if (!elementType.isIntOrFloat() && !elementType.isa<VectorType>() &&
+      !elementType.isa<ComplexType>())
     return emitOptionalError(*loc, "invalid memref element type");
   return success();
 }
diff --git a/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp b/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp
index dac67fb7ffae5c7ced65bf4bd6467309a7ec7a89..48c2ca2c075e5e71586b22072889bf4dc429a206 100644
--- a/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp
+++ b/mlir/lib/Quantizer/Transforms/InferQuantizedTypesPass.cpp
@@ -282,6 +282,11 @@ mlir::quantizer::createInferQuantizedTypesPass(
     SolverContext &solverContext, const TargetConfiguration &config) {
   return std::make_unique<InferQuantizedTypesPass>(solverContext, config);
 }
+void mlir::quantizer::registerInferQuantizedTypesPass() {
+  // Do nothing, this will be enough to force link this file and the static
+  // registration will kick-in. This is temporary while we're refactoring pass
+  // registration to move away from static constructors.
+}
 
 static PassRegistration<InferQuantizedTypesPass>
     pass("quantizer-infer-quantized-types",
diff --git a/mlir/lib/Support/CMakeLists.txt b/mlir/lib/Support/CMakeLists.txt
index 84fecd05fbfd35b146a60bd1c2ce115d4310c61d..b8c0b17a68f89992b79ceaeffbd1ee74f4716d03 100644
--- a/mlir/lib/Support/CMakeLists.txt
+++ b/mlir/lib/Support/CMakeLists.txt
@@ -17,13 +17,13 @@ add_llvm_library(MLIRSupport
   )
 target_link_libraries(MLIRSupport LLVMSupport ${LLVM_PTHREAD_LIB})
 
-add_llvm_library(MLIROptMain
+add_llvm_library(MLIROptLib
   MlirOptMain.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Support
   )
-target_link_libraries(MLIROptMain
+target_link_libraries(MLIROptLib
   MLIRPass
   MLIRParser
   LLVMSupport
diff --git a/mlir/lib/Support/JitRunner.cpp b/mlir/lib/Support/JitRunner.cpp
index ab7f0bb9ca039dd17a348a5de5a66cd10d38ed05..dd8f0832afb5ea6d42e207aec5e4a037732ce6b0 100644
--- a/mlir/lib/Support/JitRunner.cpp
+++ b/mlir/lib/Support/JitRunner.cpp
@@ -22,6 +22,7 @@
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Module.h"
 #include "mlir/IR/StandardTypes.h"
+#include "mlir/InitAllDialects.h"
 #include "mlir/Parser.h"
 #include "mlir/Support/FileUtilities.h"
 
@@ -209,6 +210,7 @@ static Error compileAndExecuteSingleFloatReturnFunction(
 int mlir::JitRunnerMain(
     int argc, char **argv,
     function_ref<LogicalResult(mlir::ModuleOp)> mlirTransformer) {
+  registerAllDialects();
   llvm::InitLLVM y(argc, argv);
 
   initializeLLVM();
diff --git a/mlir/lib/Transforms/OpStats.cpp b/mlir/lib/Transforms/OpStats.cpp
index dd684d373562bf7c0602c2fa54ab291bc9969a33..0377b719f6d26b5c82580396d1c4a69538da3395 100644
--- a/mlir/lib/Transforms/OpStats.cpp
+++ b/mlir/lib/Transforms/OpStats.cpp
@@ -10,6 +10,7 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
@@ -80,5 +81,9 @@ void PrintOpStatsPass::printSummary() {
   }
 }
 
+std::unique_ptr<OpPassBase<ModuleOp>> mlir::createPrintOpStatsPass() {
+  return std::make_unique<PrintOpStatsPass>();
+}
+
 static PassRegistration<PrintOpStatsPass>
     pass("print-op-stats", "Print statistics of operations");
diff --git a/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp b/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
index c5011a79c74d012a875c5f9fc8b57a80edc14334..2860a61f18d99b20bed2ba8a3ae282b8d92ad575 100644
--- a/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
@@ -24,6 +24,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Function.h"
 #include "mlir/IR/Operation.h"
+#include "mlir/Transforms/LoopUtils.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
@@ -246,6 +247,34 @@ FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
   return FusionResult::Success;
 }
 
+/// Fuses 'srcForOp' into 'dstForOp' with destination loop block insertion point
+/// and source slice loop bounds specified in 'srcSlice'.
+void mlir::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
+                     ComputationSliceState *srcSlice) {
+  // Clone 'srcForOp' into 'dstForOp' at 'srcSlice->insertPoint'.
+  OpBuilder b(srcSlice->insertPoint->getBlock(), srcSlice->insertPoint);
+  BlockAndValueMapping mapper;
+  b.clone(*srcForOp, mapper);
+
+  // Update 'sliceLoopNest' upper and lower bounds from computed 'srcSlice'.
+  SmallVector<AffineForOp, 4> sliceLoops;
+  for (unsigned i = 0, e = srcSlice->ivs.size(); i < e; ++i) {
+    auto loopIV = mapper.lookupOrNull(srcSlice->ivs[i]);
+    if (!loopIV)
+      continue;
+    auto forOp = getForInductionVarOwner(loopIV);
+    sliceLoops.push_back(forOp);
+    if (AffineMap lbMap = srcSlice->lbs[i])
+      forOp.setLowerBound(srcSlice->lbOperands[i], lbMap);
+    if (AffineMap ubMap = srcSlice->ubs[i])
+      forOp.setUpperBound(srcSlice->ubOperands[i], ubMap);
+  }
+
+  // Promote any single iteration slice loops.
+  for (AffineForOp forOp : sliceLoops)
+    promoteIfSingleIteration(forOp);
+}
+
 /// Collect loop nest statistics (eg. loop trip count and operation count)
 /// in 'stats' for loop nest rooted at 'forOp'. Returns true on success,
 /// returns false otherwise.
diff --git a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
index d9b5b9474d4899cf7b9d69c58d62eb69802fa340..348afff339673622f258efab317d306e2c10e41e 100644
--- a/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/calling-convention.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -convert-std-to-llvm -convert-std-to-llvm-emit-c-wrappers %s | FileCheck %s
+// RUN: mlir-opt -convert-std-to-llvm='emit-c-wrappers=1' %s | FileCheck %s
 
 // This tests the default memref calling convention and the emission of C
 // wrappers. We don't need to separate runs because the wrapper-emission
diff --git a/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir b/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
index c25d8a235701e98a839b136cbd6c5dcc758ec126..cc8cfc3b691705cd0b1779f383e0be66df36d9e3 100644
--- a/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
+++ b/mlir/test/Conversion/StandardToLLVM/convert-static-memref-ops.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt -convert-std-to-llvm %s | FileCheck %s
-// RUN: mlir-opt -convert-std-to-llvm -convert-std-to-llvm-use-alloca=1 %s | FileCheck %s --check-prefix=ALLOCA
-// RUN: mlir-opt -convert-std-to-llvm -split-input-file -convert-std-to-llvm-use-bare-ptr-memref-call-conv=1 %s | FileCheck %s --check-prefix=BAREPTR
+// RUN: mlir-opt -convert-std-to-llvm='use-alloca=1' %s | FileCheck %s --check-prefix=ALLOCA
+// RUN: mlir-opt -convert-std-to-llvm='use-bare-ptr-memref-call-conv=1' -split-input-file %s | FileCheck %s --check-prefix=BAREPTR
 
 // BAREPTR-LABEL: func @check_noalias
 // BAREPTR-SAME: %{{.*}}: !llvm<"float*"> {llvm.noalias = true}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index d1535d59593c336479f7a380a8d2e4f6ff3ab5be..5159031339aae75c33456122ff832111c4723208 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -645,7 +645,7 @@ func @vector_fma(%a: vector<8xf32>, %b: vector<2x4xf32>) -> (vector<8xf32>, vect
   //         CHECK: "llvm.intr.fma"(%[[A]], %[[A]], %[[A]]) :
   //    CHECK-SAME:   (!llvm<"<8 x float>">, !llvm<"<8 x float>">, !llvm<"<8 x float>">) -> !llvm<"<8 x float>">
   %0 = vector.fma %a, %a, %a : vector<8xf32>
-  
+
   //       CHECK: %[[b00:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
   //       CHECK: %[[b01:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
   //       CHECK: %[[b02:.*]] = llvm.extractvalue %[[B]][0] : !llvm<"[2 x <4 x float>]">
@@ -659,7 +659,45 @@ func @vector_fma(%a: vector<8xf32>, %b: vector<2x4xf32>) -> (vector<8xf32>, vect
   //  CHECK-SAME: (!llvm<"<4 x float>">, !llvm<"<4 x float>">, !llvm<"<4 x float>">) -> !llvm<"<4 x float>">
   //       CHECK: llvm.insertvalue %[[B1]], {{.*}}[1] : !llvm<"[2 x <4 x float>]">
   %1 = vector.fma %b, %b, %b : vector<2x4xf32>
-  
+
   return %0, %1: vector<8xf32>, vector<2x4xf32>
 }
-        
+
+func @reduce_f32(%arg0: vector<16xf32>) -> f32 {
+  %0 = vector.reduction "add", %arg0 : vector<16xf32> into f32
+  return %0 : f32
+}
+// CHECK-LABEL: llvm.func @reduce_f32
+// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x float>">
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f32) : !llvm.float
+//      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
+//      CHECK: llvm.return %[[V]] : !llvm.float
+
+func @reduce_f64(%arg0: vector<16xf64>) -> f64 {
+  %0 = vector.reduction "add", %arg0 : vector<16xf64> into f64
+  return %0 : f64
+}
+// CHECK-LABEL: llvm.func @reduce_f64
+// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x double>">
+//      CHECK: %[[C:.*]] = llvm.mlir.constant(0.000000e+00 : f64) : !llvm.double
+//      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.v2.fadd"(%[[C]], %[[A]])
+//      CHECK: llvm.return %[[V]] : !llvm.double
+
+func @reduce_i32(%arg0: vector<16xi32>) -> i32 {
+  %0 = vector.reduction "add", %arg0 : vector<16xi32> into i32
+  return %0 : i32
+}
+// CHECK-LABEL: llvm.func @reduce_i32
+// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x i32>">
+//      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.add"(%[[A]])
+//      CHECK: llvm.return %[[V]] : !llvm.i32
+
+func @reduce_i64(%arg0: vector<16xi64>) -> i64 {
+  %0 = vector.reduction "add", %arg0 : vector<16xi64> into i64
+  return %0 : i64
+}
+// CHECK-LABEL: llvm.func @reduce_i64
+// CHECK-SAME: %[[A:.*]]: !llvm<"<16 x i64>">
+//      CHECK: %[[V:.*]] = "llvm.intr.experimental.vector.reduce.add"(%[[A]])
+//      CHECK: llvm.return %[[V]] : !llvm.i64
+
diff --git a/mlir/test/Dialect/SPIRV/TestAvailability.cpp b/mlir/test/Dialect/SPIRV/TestAvailability.cpp
index 373397bdc2cd936e181bb002f3c1f7d3094e6dc5..0055085c1661c2f29a850d95cb5a0a6e9dce4e0b 100644
--- a/mlir/test/Dialect/SPIRV/TestAvailability.cpp
+++ b/mlir/test/Dialect/SPIRV/TestAvailability.cpp
@@ -213,6 +213,9 @@ ConvertToSubgroupBallot::matchAndRewrite(Operation *op,
   return matchSuccess();
 }
 
-static PassRegistration<ConvertToTargetEnv>
-    convertToTargetEnvPass("test-spirv-target-env",
-                           "Test SPIR-V target environment");
+namespace mlir {
+void registerConvertToTargetEnvPass() {
+  PassRegistration<ConvertToTargetEnv> convertToTargetEnvPass(
+      "test-spirv-target-env", "Test SPIR-V target environment");
+}
+} // namespace mlir
diff --git a/mlir/test/Dialect/VectorOps/invalid.mlir b/mlir/test/Dialect/VectorOps/invalid.mlir
index a1fee2cf9a62fdd1709d4c9f3add5a29f3605ad2..2a45820be7b04deba71d40c5a910b60e648bf7d4 100644
--- a/mlir/test/Dialect/VectorOps/invalid.mlir
+++ b/mlir/test/Dialect/VectorOps/invalid.mlir
@@ -990,3 +990,31 @@ func @shape_cast_different_tuple_sizes(
   %1 = vector.shape_cast %arg1 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>> to
                                  tuple<vector<20x2xf32>>
 }
+
+// -----
+
+func @reduce_unknown_kind(%arg0: vector<16xf32>) -> f32 {
+  // expected-error@+1 {{'vector.reduction' op unknown reduction kind: joho}}
+  %0 = vector.reduction "joho", %arg0 : vector<16xf32> into f32
+}
+
+// -----
+
+func @reduce_elt_type_mismatch(%arg0: vector<16xf32>) -> i32 {
+  // expected-error@+1 {{'vector.reduction' op failed to verify that source operand and result have same element type}}
+  %0 = vector.reduction "add", %arg0 : vector<16xf32> into i32
+}
+
+// -----
+
+func @reduce_unsupported_type(%arg0: vector<16xf32>) -> f32 {
+  // expected-error@+1 {{'vector.reduction' op unsupported reduction type}}
+  %0 = vector.reduction "xor", %arg0 : vector<16xf32> into f32
+}
+
+// -----
+
+func @reduce_unsupported_rank(%arg0: vector<4x16xf32>) -> f32 {
+  // expected-error@+1 {{'vector.reduction' op unsupported reduction rank: 2}}
+  %0 = vector.reduction "add", %arg0 : vector<4x16xf32> into f32
+}
diff --git a/mlir/test/Dialect/VectorOps/ops.mlir b/mlir/test/Dialect/VectorOps/ops.mlir
index ff0078310af20bf2c8d65bf18a85f043b1379551..bb5ca6eb8538cb63e887f80a57ad15704cab49d9 100644
--- a/mlir/test/Dialect/VectorOps/ops.mlir
+++ b/mlir/test/Dialect/VectorOps/ops.mlir
@@ -277,3 +277,37 @@ func @vector_fma(%a: vector<8xf32>, %b: vector<8x4xf32>) {
   vector.fma %b, %b, %b : vector<8x4xf32>
   return
 }
+
+// CHECK-LABEL: reduce_fp
+func @reduce_fp(%arg0: vector<16xf32>) -> f32 {
+  // CHECK:    vector.reduction "add", %{{.*}} : vector<16xf32> into f32
+  vector.reduction "add", %arg0 : vector<16xf32> into f32
+  // CHECK:    vector.reduction "mul", %{{.*}} : vector<16xf32> into f32
+  vector.reduction "mul", %arg0 : vector<16xf32> into f32
+  // CHECK:    vector.reduction "min", %{{.*}} : vector<16xf32> into f32
+  vector.reduction "min", %arg0 : vector<16xf32> into f32
+  // CHECK:    %[[X:.*]] = vector.reduction "max", %{{.*}} : vector<16xf32> into f32
+  %0 = vector.reduction "max", %arg0 : vector<16xf32> into f32
+  // CHECK:    return %[[X]] : f32
+  return %0 : f32
+}
+
+// CHECK-LABEL: reduce_int
+func @reduce_int(%arg0: vector<16xi32>) -> i32 {
+  // CHECK:    vector.reduction "add", %{{.*}} : vector<16xi32> into i32
+  vector.reduction "add", %arg0 : vector<16xi32> into i32
+  // CHECK:    vector.reduction "mul", %{{.*}} : vector<16xi32> into i32
+  vector.reduction "mul", %arg0 : vector<16xi32> into i32
+  // CHECK:    vector.reduction "min", %{{.*}} : vector<16xi32> into i32
+  vector.reduction "min", %arg0 : vector<16xi32> into i32
+  // CHECK:    vector.reduction "max", %{{.*}} : vector<16xi32> into i32
+  vector.reduction "max", %arg0 : vector<16xi32> into i32
+  // CHECK:    vector.reduction "and", %{{.*}} : vector<16xi32> into i32
+  vector.reduction "and", %arg0 : vector<16xi32> into i32
+  // CHECK:    vector.reduction "or", %{{.*}} : vector<16xi32> into i32
+  vector.reduction "or", %arg0 : vector<16xi32> into i32
+  // CHECK:    %[[X:.*]] = vector.reduction "xor", %{{.*}} : vector<16xi32> into i32
+  %0 = vector.reduction "xor", %arg0 : vector<16xi32> into i32
+  // CHECK:    return %[[X]] : i32
+  return %0 : i32
+}
diff --git a/mlir/test/Dialect/VectorOps/vector-transforms.mlir b/mlir/test/Dialect/VectorOps/vector-transforms.mlir
index 1153ffb2999f58a6b1f2a8813a077c359d3b8d50..7582758384ce55bac86bdca089fc64ba52213e33 100644
--- a/mlir/test/Dialect/VectorOps/vector-transforms.mlir
+++ b/mlir/test/Dialect/VectorOps/vector-transforms.mlir
@@ -346,3 +346,62 @@ func @vector_transfers_vector_element_type() {
 
   return
 }
+
+// Test that ShapeCastOp on tuple of vectors, decomposes to multiple
+// ShapeCastOps on vectors.
+// CHECK-LABEL: func @shape_cast_decomposition
+//       CHECK: %[[V0:.*]] = vector.shape_cast %{{.*}} : vector<5x4x2xf32> to vector<20x2xf32>
+//  CHECK-NEXT: %[[V1:.*]] = vector.shape_cast %{{.*}} : vector<3x4x2xf32> to vector<12x2xf32>
+//  CHECK-NEXT: return %[[V0]], %[[V1]] : vector<20x2xf32>, vector<12x2xf32>
+
+func @shape_cast_decomposition(%arg0 : vector<5x4x2xf32>,
+                               %arg1 : vector<3x4x2xf32>)
+  -> (vector<20x2xf32>, vector<12x2xf32>) {
+  %0 = vector.tuple %arg0, %arg1 : vector<5x4x2xf32>, vector<3x4x2xf32>
+  %1 = vector.shape_cast %0 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>> to
+                              tuple<vector<20x2xf32>, vector<12x2xf32>>
+  %2 = vector.tuple_get %1, 0 : tuple<vector<20x2xf32>, vector<12x2xf32>>
+  %3 = vector.tuple_get %1, 1 : tuple<vector<20x2xf32>, vector<12x2xf32>>
+  return %2, %3 : vector<20x2xf32>, vector<12x2xf32>
+}
+
+// Test that cancelling ShapeCastOps are canonicalized away.
+// EX:
+//
+//  The following MLIR with cancelling ShapeCastOps:
+//
+//   %0 = source : vector<5x4x2xf32>
+//   %1 = shape_cast %0 : vector<5x4x2xf32> to vector<20x2xf32>
+//   %2 = shape_cast %1 : vector<20x2xf32> to vector<5x4x2xf32>
+//   %3 = user %2 : vector<5x4x2xf32>
+//
+//  Should canonicalize to the following:
+//
+//
+//   %0 = source : vector<5x4x2xf32>
+//   %1 = user %0 : vector<5x4x2xf32>
+//
+
+// ShapeCastOps on vectors.
+// CHECK-LABEL: func @shape_cast_fold
+//       CHECK: return %{{.*}},  %{{.*}} : vector<5x4x2xf32>, vector<3x4x2xf32>
+
+func @shape_cast_fold(%arg0 : vector<5x4x2xf32>, %arg1 : vector<3x4x2xf32>)
+  -> (vector<5x4x2xf32>, vector<3x4x2xf32>) {
+  %0 = vector.tuple %arg0, %arg1 : vector<5x4x2xf32>, vector<3x4x2xf32>
+
+  %1 = vector.shape_cast %0 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>> to
+                              tuple<vector<20x2xf32>, vector<12x2xf32>>
+
+  %2 = vector.tuple_get %1, 0 : tuple<vector<20x2xf32>, vector<12x2xf32>>
+  %3 = vector.tuple_get %1, 1 : tuple<vector<20x2xf32>, vector<12x2xf32>>
+
+  %4 = vector.tuple %2, %3 : vector<20x2xf32>, vector<12x2xf32>
+  %5 = vector.shape_cast %4 : tuple<vector<20x2xf32>, vector<12x2xf32>> to
+                              tuple<vector<5x4x2xf32>, vector<3x4x2xf32>>
+
+  %6 = vector.tuple_get %5, 0 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>>
+  %7 = vector.tuple_get %5, 1 : tuple<vector<5x4x2xf32>, vector<3x4x2xf32>>
+
+  return %6, %7 : vector<5x4x2xf32>, vector<3x4x2xf32>
+}
diff --git a/mlir/test/EDSC/builder-api-test.cpp b/mlir/test/EDSC/builder-api-test.cpp
index 92f05680e43290224291b13b3d06b845ad0721f9..2cf808642905570607306bc7dbb4ca0eed69957c 100644
--- a/mlir/test/EDSC/builder-api-test.cpp
+++ b/mlir/test/EDSC/builder-api-test.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// RUN: mlir-edsc-builder-api-test | FileCheck %s
+// RUN: mlir-edsc-builder-api-test | FileCheck %s -dump-input-on-failure
 
 #include "mlir/Dialect/AffineOps/EDSC/Intrinsics.h"
 #include "mlir/Dialect/Linalg/EDSC/Intrinsics.h"
@@ -37,6 +37,15 @@ using namespace mlir::edsc;
 using namespace mlir::edsc::intrinsics;
 
 static MLIRContext &globalContext() {
+  static bool init_once = []() {
+    registerDialect<AffineOpsDialect>();
+    registerDialect<linalg::LinalgDialect>();
+    registerDialect<loop::LoopOpsDialect>();
+    registerDialect<StandardOpsDialect>();
+    registerDialect<vector::VectorOpsDialect>();
+    return true;
+  }();
+  (void)init_once;
   static thread_local MLIRContext context;
   return context;
 }
diff --git a/mlir/test/IR/attribute.mlir b/mlir/test/IR/attribute.mlir
index 318837d30cc9d6d7b18c2ef65f74a69b371e9bde..c0ee566738e304f5eef21830379b865b40af9263 100644
--- a/mlir/test/IR/attribute.mlir
+++ b/mlir/test/IR/attribute.mlir
@@ -243,3 +243,53 @@ func @fn() { return }
 
 // expected-error @+1 {{referencing to a 'FuncOp' symbol}}
 "test.symbol_ref_attr"() {symbol = @foo} : () -> ()
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// Test IntElementsAttr
+//===----------------------------------------------------------------------===//
+
+func @correct_type_pass() {
+  "test.int_elements_attr"() {
+    // CHECK: matrix_i64_attr = dense<6> : tensor<4x8xi64>
+    // CHECK: vector_i32_attr = dense<5> : tensor<2xi32>
+    matrix_i64_attr = dense<6> : tensor<4x8xi64>,
+    vector_i32_attr = dense<5> : tensor<2xi32>
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @wrong_element_type_fail() {
+  // expected-error @+1 {{failed to satisfy constraint: 32-bit int elements attribute of shape [2]}}
+  "test.int_elements_attr"() {
+    matrix_i64_attr = dense<6> : tensor<4x8xi64>,
+    vector_i32_attr = dense<5> : tensor<2xi64>
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @wrong_shape_fail() {
+  // expected-error @+1 {{failed to satisfy constraint: 64-bit int elements attribute of shape [4, 8]}}
+  "test.int_elements_attr"() {
+    matrix_i64_attr = dense<6> : tensor<4xi64>,
+    vector_i32_attr = dense<5> : tensor<2xi32>
+  } : () -> ()
+  return
+}
+
+// -----
+
+func @wrong_shape_fail() {
+  // expected-error @+1 {{failed to satisfy constraint: 32-bit int elements attribute of shape [2]}}
+  "test.int_elements_attr"() {
+    matrix_i64_attr = dense<6> : tensor<4x8xi64>,
+    vector_i32_attr = dense<5> : tensor<i32>
+  } : () -> ()
+  return
+}
+
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index b2c93ce152915bc096ac990ece458331eed2b285..71e82b0fe0906810529646e0edc156f11ab0d137 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -133,6 +133,16 @@ func @memrefs_compose_with_id(memref<2x2xi8, affine_map<(d0, d1) -> (d0, d1)>,
 // CHECK: func @complex_types(complex<i1>) -> complex<f32>
 func @complex_types(complex<i1>) -> complex<f32>
 
+
+// CHECK: func @memref_with_complex_elems(memref<1x?xcomplex<f32>>)
+func @memref_with_complex_elems(memref<1x?xcomplex<f32>>)
+
+// CHECK: func @memref_with_vector_elems(memref<1x?xvector<10xf32>>)
+func @memref_with_vector_elems(memref<1x?xvector<10xf32>>)
+
+// CHECK: func @unranked_memref_with_complex_elems(memref<*xcomplex<f32>>)
+func @unranked_memref_with_complex_elems(memref<*xcomplex<f32>>)
+
 // CHECK: func @functions((memref<1x?x4x?x?xi32, #map0>, memref<8xi8>) -> (), () -> ())
 func @functions((memref<1x?x4x?x?xi32, #map0, 0>, memref<8xi8, #map1, 0>) -> (), ()->())
 
diff --git a/mlir/test/SDBM/sdbm-api-test.cpp b/mlir/test/SDBM/sdbm-api-test.cpp
index 4606df9a85e1ddca06c01164d83e54660762ca3d..0b58e2948145cc95a8428e68ba60cbcce795b0f1 100644
--- a/mlir/test/SDBM/sdbm-api-test.cpp
+++ b/mlir/test/SDBM/sdbm-api-test.cpp
@@ -19,6 +19,9 @@
 
 using namespace mlir;
 
+// Load the SDBM dialect
+static DialectRegistration<SDBMDialect> SDBMRegistration;
+
 static MLIRContext *ctx() {
   static thread_local MLIRContext context;
   return &context;
diff --git a/mlir/test/Transforms/loop-fusion-transformation.mlir b/mlir/test/Transforms/loop-fusion-transformation.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..1a0106301627218a0af043c24838ed4a5034ba6e
--- /dev/null
+++ b/mlir/test/Transforms/loop-fusion-transformation.mlir
@@ -0,0 +1,105 @@
+// RUN: mlir-opt %s -test-loop-fusion -test-loop-fusion-transformation -split-input-file -canonicalize | FileCheck %s
+
+// CHECK-LABEL: func @slice_depth1_loop_nest() {
+func @slice_depth1_loop_nest() {
+  %0 = alloc() : memref<100xf32>
+  %cst = constant 7.000000e+00 : f32
+  affine.for %i0 = 0 to 16 {
+    affine.store %cst, %0[%i0] : memref<100xf32>
+  }
+  affine.for %i1 = 0 to 5 {
+    %1 = affine.load %0[%i1] : memref<100xf32>
+  }
+  // CHECK:      affine.for %[[IV0:.*]] = 0 to 5 {
+  // CHECK-NEXT:   affine.store %{{.*}}, %{{.*}}[%[[IV0]]] : memref<100xf32>
+  // CHECK-NEXT:   affine.load %{{.*}}[%[[IV0]]] : memref<100xf32>
+  // CHECK-NEXT: }
+  // CHECK-NEXT: return
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @should_fuse_reduction_to_pointwise() {
+func @should_fuse_reduction_to_pointwise() {
+  %a = alloc() : memref<10x10xf32>
+  %b = alloc() : memref<10xf32>
+  %c = alloc() : memref<10xf32>
+
+  %cf7 = constant 7.0 : f32
+
+  affine.for %i0 = 0 to 10 {
+    affine.for %i1 = 0 to 10 {
+      %v0 = affine.load %b[%i0] : memref<10xf32>
+      %v1 = affine.load %a[%i0, %i1] : memref<10x10xf32>
+      %v3 = addf %v0, %v1 : f32
+      affine.store %v3, %b[%i0] : memref<10xf32>
+    }
+  }
+  affine.for %i2 = 0 to 10 {
+    %v4 = affine.load %b[%i2] : memref<10xf32>
+    affine.store %v4, %c[%i2] : memref<10xf32>
+  }
+
+  // Match on the fused loop nest.
+  // Should fuse in entire inner loop on %i1 from source loop nest, as %i1
+  // is not used in the access function of the store/load on %b.
+  // CHECK:       affine.for %{{.*}} = 0 to 10 {
+  // CHECK-NEXT:    affine.for %{{.*}} = 0 to 10 {
+  // CHECK-NEXT:      affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:      affine.load %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x10xf32>
+  // CHECK-NEXT:      addf %{{.*}}, %{{.*}} : f32
+  // CHECK-NEXT:      affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:    }
+  // CHECK-NEXT:    affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:    affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:  }
+  // CHECK-NEXT:  return
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func @should_fuse_avoiding_dependence_cycle() {
+func @should_fuse_avoiding_dependence_cycle() {
+  %a = alloc() : memref<10xf32>
+  %b = alloc() : memref<10xf32>
+  %c = alloc() : memref<10xf32>
+
+  %cf7 = constant 7.0 : f32
+
+  // Set up the following dependences:
+  // 1) loop0 -> loop1 on memref '%{{.*}}'
+  // 2) loop0 -> loop2 on memref '%{{.*}}'
+  // 3) loop1 -> loop2 on memref '%{{.*}}'
+  affine.for %i0 = 0 to 10 {
+    %v0 = affine.load %a[%i0] : memref<10xf32>
+    affine.store %cf7, %b[%i0] : memref<10xf32>
+  }
+  affine.for %i1 = 0 to 10 {
+    affine.store %cf7, %a[%i1] : memref<10xf32>
+    %v1 = affine.load %c[%i1] : memref<10xf32>
+  }
+  affine.for %i2 = 0 to 10 {
+    %v2 = affine.load %b[%i2] : memref<10xf32>
+    affine.store %cf7, %c[%i2] : memref<10xf32>
+  }
+  // Fusing loop first loop into last would create a cycle:
+  //   {1} <--> {0, 2}
+  // However, we can avoid the dependence cycle if we first fuse loop0 into
+  // loop1:
+  //   {0, 1) --> {2}
+  // Then fuse this loop nest with loop2:
+  //   {0, 1, 2}
+  //
+  // CHECK:      affine.for %{{.*}} = 0 to 10 {
+  // CHECK-NEXT:   affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:   affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:   affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:   affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:   affine.load %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT:   affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>
+  // CHECK-NEXT: }
+  // CHECK-NEXT: return
+  return
+}
diff --git a/mlir/test/lib/IR/TestFunc.cpp b/mlir/test/lib/IR/TestFunc.cpp
index 7003600984f9763ad6ecc08c7dd59220b93a0787..0e885c555e38a534ddb97fd9b5a529abfadb8ef9 100644
--- a/mlir/test/lib/IR/TestFunc.cpp
+++ b/mlir/test/lib/IR/TestFunc.cpp
@@ -51,8 +51,12 @@ struct TestFuncSetType : public ModulePass<TestFuncSetType> {
 };
 } // end anonymous namespace
 
-static PassRegistration<TestFuncEraseArg> pass("test-func-erase-arg",
-                                               "Test erasing func args.");
-
-static PassRegistration<TestFuncSetType> pass2("test-func-set-type",
-                                               "Test FuncOp::setType.");
+namespace mlir {
+void registerTestFunc() {
+  PassRegistration<TestFuncEraseArg> pass("test-func-erase-arg",
+                                          "Test erasing func args.");
+
+  PassRegistration<TestFuncSetType> pass2("test-func-set-type",
+                                          "Test FuncOp::setType.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/IR/TestMatchers.cpp b/mlir/test/lib/IR/TestMatchers.cpp
index 5620df3d6ed8ccda27b9221ac5ee07e195212671..c6c144ef1fd0bc52057c6d14a29fd7d6f59d35a3 100644
--- a/mlir/test/lib/IR/TestMatchers.cpp
+++ b/mlir/test/lib/IR/TestMatchers.cpp
@@ -146,5 +146,8 @@ void TestMatchers::runOnFunction() {
     test2(f);
 }
 
-static PassRegistration<TestMatchers> pass("test-matchers",
-                                           "Test C++ pattern matchers.");
+namespace mlir {
+void registerTestMatchers() {
+  PassRegistration<TestMatchers>("test-matchers", "Test C++ pattern matchers.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/IR/TestSymbolUses.cpp b/mlir/test/lib/IR/TestSymbolUses.cpp
index 7757da5c968a8a1bfc62aed2fbae466874726acb..6082cdcbe72b256dc7e7984e77e63853308040f5 100644
--- a/mlir/test/lib/IR/TestSymbolUses.cpp
+++ b/mlir/test/lib/IR/TestSymbolUses.cpp
@@ -103,8 +103,12 @@ struct SymbolReplacementPass : public ModulePass<SymbolReplacementPass> {
 };
 } // end anonymous namespace
 
-static PassRegistration<SymbolUsesPass> pass("test-symbol-uses",
-                                             "Test detection of symbol uses");
+namespace mlir {
+void registerSymbolTestPasses() {
+  PassRegistration<SymbolUsesPass>("test-symbol-uses",
+                                   "Test detection of symbol uses");
 
-static PassRegistration<SymbolReplacementPass>
-    rauwPass("test-symbol-rauw", "Test replacement of symbol uses");
+  PassRegistration<SymbolReplacementPass>("test-symbol-rauw",
+                                          "Test replacement of symbol uses");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Pass/TestPassManager.cpp b/mlir/test/lib/Pass/TestPassManager.cpp
index 4097fd0675e94dd42fbf68a5ec73b6d3dc2ee966..95bef9b878e2251979076c51ae8e1467d16f9381 100644
--- a/mlir/test/lib/Pass/TestPassManager.cpp
+++ b/mlir/test/lib/Pass/TestPassManager.cpp
@@ -88,40 +88,43 @@ static void testNestedPipelineTextual(OpPassManager &pm) {
   (void)parsePassPipeline("test-pm-nested-pipeline", pm);
 }
 
-static PassRegistration<TestOptionsPass>
-    reg("test-options-pass", "Test options parsing capabilities");
-
-static PassRegistration<TestModulePass>
-    unusedMP("test-module-pass", "Test a module pass in the pass manager");
-static PassRegistration<TestFunctionPass>
-    unusedFP("test-function-pass", "Test a function pass in the pass manager");
-
-static PassRegistration<TestCrashRecoveryPass>
-    unusedCrashP("test-pass-crash",
-                 "Test a pass in the pass manager that always crashes");
-
-static PassRegistration<TestStatisticPass> unusedStatP("test-stats-pass",
-                                                       "Test pass statistics");
-
-static PassPipelineRegistration<>
-    unused("test-pm-nested-pipeline",
-           "Test a nested pipeline in the pass manager", testNestedPipeline);
-static PassPipelineRegistration<>
-    unusedTextual("test-textual-pm-nested-pipeline",
-                  "Test a nested pipeline in the pass manager",
-                  testNestedPipelineTextual);
-static PassPipelineRegistration<>
-    unusedDump("test-dump-pipeline",
-               "Dumps the pipeline build so far for debugging purposes",
-               [](OpPassManager &pm) {
-                 pm.printAsTextualPipeline(llvm::errs());
-                 llvm::errs() << "\n";
-               });
-
-static PassPipelineRegistration<TestOptionsPass::Options>
-    registerOptionsPassPipeline(
-        "test-options-pass-pipeline",
-        "Parses options using pass pipeline registration",
-        [](OpPassManager &pm, const TestOptionsPass::Options &options) {
-          pm.addPass(std::make_unique<TestOptionsPass>(options));
-        });
+namespace mlir {
+void registerPassManagerTestPass() {
+  PassRegistration<TestOptionsPass>("test-options-pass",
+                                    "Test options parsing capabilities");
+
+  PassRegistration<TestModulePass>("test-module-pass",
+                                   "Test a module pass in the pass manager");
+
+  PassRegistration<TestFunctionPass>(
+      "test-function-pass", "Test a function pass in the pass manager");
+
+  PassRegistration<TestCrashRecoveryPass>(
+      "test-pass-crash", "Test a pass in the pass manager that always crashes");
+
+  PassRegistration<TestStatisticPass> unusedStatP("test-stats-pass",
+                                                  "Test pass statistics");
+
+  PassPipelineRegistration<>("test-pm-nested-pipeline",
+                             "Test a nested pipeline in the pass manager",
+                             testNestedPipeline);
+  PassPipelineRegistration<>("test-textual-pm-nested-pipeline",
+                             "Test a nested pipeline in the pass manager",
+                             testNestedPipelineTextual);
+  PassPipelineRegistration<>(
+      "test-dump-pipeline",
+      "Dumps the pipeline build so far for debugging purposes",
+      [](OpPassManager &pm) {
+        pm.printAsTextualPipeline(llvm::errs());
+        llvm::errs() << "\n";
+      });
+
+  PassPipelineRegistration<TestOptionsPass::Options>
+      registerOptionsPassPipeline(
+          "test-options-pass-pipeline",
+          "Parses options using pass pipeline registration",
+          [](OpPassManager &pm, const TestOptionsPass::Options &options) {
+            pm.addPass(std::make_unique<TestOptionsPass>(options));
+          });
+}
+} // namespace mlir
diff --git a/mlir/test/lib/TestDialect/TestOps.td b/mlir/test/lib/TestDialect/TestOps.td
index de7f1875ef057f79671db45d9c4439e37f13fc46..48adf2dd1585bf890d37d1dd1bc92c1964787c1b 100644
--- a/mlir/test/lib/TestDialect/TestOps.td
+++ b/mlir/test/lib/TestDialect/TestOps.td
@@ -204,6 +204,13 @@ def UpdateFloatElementsAttr : Pat<
     ConstantAttr<RankedF32ElementsAttr<[2]>, "{5.0f, 6.0f}">:$f32attr,
     $f64attr)>;
 
+def IntElementsAttrOp : TEST_Op<"int_elements_attr"> {
+  let arguments = (ins
+      RankedI32ElementsAttr<[2]>:$vector_i32_attr,
+      RankedI64ElementsAttr<[4, 8]>:$matrix_i64_attr
+  );
+}
+
 //===----------------------------------------------------------------------===//
 // Test Attribute Constraints
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/TestDialect/TestPatterns.cpp b/mlir/test/lib/TestDialect/TestPatterns.cpp
index 30330052d5e558681f8d9f21726a03f1948be697..d975d67191706671c99cf9353ff66dc24cc13c54 100644
--- a/mlir/test/lib/TestDialect/TestPatterns.cpp
+++ b/mlir/test/lib/TestDialect/TestPatterns.cpp
@@ -50,9 +50,6 @@ struct TestPatternDriver : public FunctionPass<TestPatternDriver> {
 };
 } // end anonymous namespace
 
-static mlir::PassRegistration<TestPatternDriver>
-    pass("test-patterns", "Run test dialect patterns");
-
 //===----------------------------------------------------------------------===//
 // ReturnType Driver.
 //===----------------------------------------------------------------------===//
@@ -107,9 +104,6 @@ struct TestReturnTypeDriver : public FunctionPass<TestReturnTypeDriver> {
 };
 } // end anonymous namespace
 
-static mlir::PassRegistration<TestReturnTypeDriver>
-    rt_pass("test-return-type", "Run return type functions");
-
 //===----------------------------------------------------------------------===//
 // Legalization Driver.
 //===----------------------------------------------------------------------===//
@@ -440,13 +434,6 @@ static llvm::cl::opt<TestLegalizePatternDriver::ConversionMode>
             clEnumValN(TestLegalizePatternDriver::ConversionMode::Partial,
                        "partial", "Perform a partial conversion")));
 
-static mlir::PassRegistration<TestLegalizePatternDriver>
-    legalizer_pass("test-legalize-patterns",
-                   "Run test dialect legalization patterns", [] {
-                     return std::make_unique<TestLegalizePatternDriver>(
-                         legalizerConversionMode);
-                   });
-
 //===----------------------------------------------------------------------===//
 // ConversionPatternRewriter::getRemappedValue testing. This method is used
 // to get the remapped value of a original value that was replaced using
@@ -505,6 +492,22 @@ struct TestRemappedValue : public mlir::FunctionPass<TestRemappedValue> {
 };
 } // end anonymous namespace
 
-static PassRegistration<TestRemappedValue> remapped_value_pass(
-    "test-remapped-value",
-    "Test public remapped value mechanism in ConversionPatternRewriter");
+namespace mlir {
+void registerPatternsTestPass() {
+  mlir::PassRegistration<TestReturnTypeDriver>("test-return-type",
+                                               "Run return type functions");
+
+  mlir::PassRegistration<TestPatternDriver>("test-patterns",
+                                            "Run test dialect patterns");
+
+  mlir::PassRegistration<TestLegalizePatternDriver>(
+      "test-legalize-patterns", "Run test dialect legalization patterns", [] {
+        return std::make_unique<TestLegalizePatternDriver>(
+            legalizerConversionMode);
+      });
+
+  PassRegistration<TestRemappedValue>(
+      "test-remapped-value",
+      "Test public remapped value mechanism in ConversionPatternRewriter");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestAllReduceLowering.cpp b/mlir/test/lib/Transforms/TestAllReduceLowering.cpp
index fef08050d2f5175bedb9ea1b26341ce5cd7d2d4f..508f70887350281efdae2a0a4d4abddf34032538 100644
--- a/mlir/test/lib/Transforms/TestAllReduceLowering.cpp
+++ b/mlir/test/lib/Transforms/TestAllReduceLowering.cpp
@@ -27,6 +27,10 @@ struct TestAllReduceLoweringPass
 };
 } // namespace
 
-static PassRegistration<TestAllReduceLoweringPass>
-    pass("test-all-reduce-lowering",
-         "Lowers gpu.all-reduce ops within the GPU dialect.");
+namespace mlir {
+void registerTestAllReduceLoweringPass() {
+  PassRegistration<TestAllReduceLoweringPass> pass(
+      "test-all-reduce-lowering",
+      "Lowers gpu.all-reduce ops within the GPU dialect.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestCallGraph.cpp b/mlir/test/lib/Transforms/TestCallGraph.cpp
index 13115c9cc5782a1c412f0783a5f514529d5b4a56..89c25da9e8edd881b6498d85da8ffceb4bf002aa 100644
--- a/mlir/test/lib/Transforms/TestCallGraph.cpp
+++ b/mlir/test/lib/Transforms/TestCallGraph.cpp
@@ -25,6 +25,9 @@ struct TestCallGraphPass : public ModulePass<TestCallGraphPass> {
 };
 } // end anonymous namespace
 
-static PassRegistration<TestCallGraphPass>
-    pass("test-print-callgraph",
-         "Print the contents of a constructed callgraph.");
+namespace mlir {
+void registerTestCallGraphPass() {
+  PassRegistration<TestCallGraphPass> pass(
+      "test-print-callgraph", "Print the contents of a constructed callgraph.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestConstantFold.cpp b/mlir/test/lib/Transforms/TestConstantFold.cpp
index 4c19f7932da0e6e39a949e9fcbe88f395c0d565b..269f5f613c181ba4f6a63d2dcbbb580f7d90e36e 100644
--- a/mlir/test/lib/Transforms/TestConstantFold.cpp
+++ b/mlir/test/lib/Transforms/TestConstantFold.cpp
@@ -64,5 +64,9 @@ void TestConstantFold::runOnFunction() {
   }
 }
 
-static PassRegistration<TestConstantFold>
-    pass("test-constant-fold", "Test operation constant folding");
+namespace mlir {
+void registerTestConstantFold() {
+  PassRegistration<TestConstantFold>("test-constant-fold",
+                                     "Test operation constant folding");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestGpuMemoryPromotion.cpp b/mlir/test/lib/Transforms/TestGpuMemoryPromotion.cpp
index 24ea0d95cf413f2893b70df80db334427254a550..72304244d8fc484e3faf243420bae47ab7a9646f 100644
--- a/mlir/test/lib/Transforms/TestGpuMemoryPromotion.cpp
+++ b/mlir/test/lib/Transforms/TestGpuMemoryPromotion.cpp
@@ -35,6 +35,10 @@ class TestGpuMemoryPromotionPass
 };
 } // end namespace
 
-static PassRegistration<TestGpuMemoryPromotionPass> registration(
-    "test-gpu-memory-promotion",
-    "Promotes the annotated arguments of gpu.func to workgroup memory.");
+namespace mlir {
+void registerTestGpuMemoryPromotionPass() {
+  PassRegistration<TestGpuMemoryPromotionPass>(
+      "test-gpu-memory-promotion",
+      "Promotes the annotated arguments of gpu.func to workgroup memory.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestInlining.cpp b/mlir/test/lib/Transforms/TestInlining.cpp
index 3d64b55c46ac127f6a5511ff18a5e100ae45efe5..513594f655ccbcf49809003e4ba56e9c15185734 100644
--- a/mlir/test/lib/Transforms/TestInlining.cpp
+++ b/mlir/test/lib/Transforms/TestInlining.cpp
@@ -60,5 +60,8 @@ struct Inliner : public FunctionPass<Inliner> {
 };
 } // end anonymous namespace
 
-static PassRegistration<Inliner> pass("test-inline",
-                                      "Test inlining region calls");
+namespace mlir {
+void registerInliner() {
+  PassRegistration<Inliner>("test-inline", "Test inlining region calls");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
index 94e4a887a4e3cd90753edee6ea1f9dc6d5c4f59f..645ce887135d7e0ce347c91bddbcd9ec32d35e79 100644
--- a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
@@ -47,6 +47,10 @@ void TestLinalgTransforms::runOnFunction() {
   });
 }
 
-static PassRegistration<TestLinalgTransforms>
-    pass("test-linalg-transform-patterns",
-         "Test Linalg transformation patterns by applying them greedily.");
+namespace mlir {
+void registerTestLinalgTransforms() {
+  PassRegistration<TestLinalgTransforms>(
+      "test-linalg-transform-patterns",
+      "Test Linalg transformation patterns by applying them greedily.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestLiveness.cpp b/mlir/test/lib/Transforms/TestLiveness.cpp
index 64276157ced9aac526d4db9c47c83b7e10ea2cbc..9b98ebd49a9c571102ebc0c251f9f0dfe1e9ccd9 100644
--- a/mlir/test/lib/Transforms/TestLiveness.cpp
+++ b/mlir/test/lib/Transforms/TestLiveness.cpp
@@ -28,6 +28,10 @@ struct TestLivenessPass : public FunctionPass<TestLivenessPass> {
 
 } // end anonymous namespace
 
-static PassRegistration<TestLivenessPass>
-    pass("test-print-liveness",
-         "Print the contents of a constructed liveness information.");
+namespace mlir {
+void registerTestLivenessPass() {
+  PassRegistration<TestLivenessPass>(
+      "test-print-liveness",
+      "Print the contents of a constructed liveness information.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestLoopFusion.cpp b/mlir/test/lib/Transforms/TestLoopFusion.cpp
index 4a4e5f9c05f61d44b9e168cd478fca7e97f1d2c4..9ffa347173f65ebfad523b97d7b65b9589dff5ef 100644
--- a/mlir/test/lib/Transforms/TestLoopFusion.cpp
+++ b/mlir/test/lib/Transforms/TestLoopFusion.cpp
@@ -41,6 +41,11 @@ static llvm::cl::opt<bool> clTestSliceComputation(
     llvm::cl::desc("Enable testing of loop fusion slice computation"),
     llvm::cl::cat(clOptionsCategory));
 
+static llvm::cl::opt<bool> clTestLoopFusionTransformation(
+    "test-loop-fusion-transformation",
+    llvm::cl::desc("Enable testing of loop fusion transformation"),
+    llvm::cl::cat(clOptionsCategory));
+
 namespace {
 
 struct TestLoopFusion : public FunctionPass<TestLoopFusion> {
@@ -49,10 +54,6 @@ struct TestLoopFusion : public FunctionPass<TestLoopFusion> {
 
 } // end anonymous namespace
 
-std::unique_ptr<OpPassBase<FuncOp>> mlir::createTestLoopFusionPass() {
-  return std::make_unique<TestLoopFusion>();
-}
-
 // Gathers all AffineForOps in 'block' at 'currLoopDepth' in 'depthToLoops'.
 static void
 gatherLoops(Block *block, unsigned currLoopDepth,
@@ -69,11 +70,10 @@ gatherLoops(Block *block, unsigned currLoopDepth,
 // Run fusion dependence check on 'loops[i]' and 'loops[j]' at loop depths
 // in range ['loopDepth' + 1, 'maxLoopDepth'].
 // Emits a remark on 'loops[i]' if a fusion-preventing dependence exists.
-static void testDependenceCheck(SmallVector<AffineForOp, 2> &loops, unsigned i,
-                                unsigned j, unsigned loopDepth,
+// Returns false as IR is not transformed.
+static bool testDependenceCheck(AffineForOp srcForOp, AffineForOp dstForOp,
+                                unsigned i, unsigned j, unsigned loopDepth,
                                 unsigned maxLoopDepth) {
-  AffineForOp srcForOp = loops[i];
-  AffineForOp dstForOp = loops[j];
   mlir::ComputationSliceState sliceUnion;
   for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
     FusionResult result =
@@ -84,6 +84,7 @@ static void testDependenceCheck(SmallVector<AffineForOp, 2> &loops, unsigned i,
           << i << " into loop nest " << j << " at depth " << loopDepth;
     }
   }
+  return false;
 }
 
 // Returns the index of 'op' in its block.
@@ -121,11 +122,10 @@ static std::string getSliceStr(const mlir::ComputationSliceState &sliceUnion) {
 // Computes fusion slice union on 'loops[i]' and 'loops[j]' at loop depths
 // in range ['loopDepth' + 1, 'maxLoopDepth'].
 // Emits a string representation of the slice union as a remark on 'loops[j]'.
-static void testSliceComputation(SmallVector<AffineForOp, 2> &loops, unsigned i,
-                                 unsigned j, unsigned loopDepth,
+// Returns false as IR is not transformed.
+static bool testSliceComputation(AffineForOp forOpA, AffineForOp forOpB,
+                                 unsigned i, unsigned j, unsigned loopDepth,
                                  unsigned maxLoopDepth) {
-  AffineForOp forOpA = loops[i];
-  AffineForOp forOpB = loops[j];
   for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
     mlir::ComputationSliceState sliceUnion;
     FusionResult result = mlir::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
@@ -135,32 +135,88 @@ static void testSliceComputation(SmallVector<AffineForOp, 2> &loops, unsigned i,
           << " : " << getSliceStr(sliceUnion) << ")";
     }
   }
+  return false;
 }
 
-void TestLoopFusion::runOnFunction() {
-  // Gather all AffineForOps by loop depth.
-  DenseMap<unsigned, SmallVector<AffineForOp, 2>> depthToLoops;
-  for (auto &block : getFunction()) {
-    gatherLoops(&block, /*currLoopDepth=*/0, depthToLoops);
+// Attempts to fuse 'forOpA' into 'forOpB' at loop depths in range
+// ['loopDepth' + 1, 'maxLoopDepth'].
+// Returns true if loops were successfully fused, false otherwise.
+static bool testLoopFusionTransformation(AffineForOp forOpA, AffineForOp forOpB,
+                                         unsigned i, unsigned j,
+                                         unsigned loopDepth,
+                                         unsigned maxLoopDepth) {
+  for (unsigned d = loopDepth + 1; d <= maxLoopDepth; ++d) {
+    mlir::ComputationSliceState sliceUnion;
+    FusionResult result = mlir::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
+    if (result.value == FusionResult::Success) {
+      mlir::fuseLoops(forOpA, forOpB, &sliceUnion);
+      // Note: 'forOpA' is removed to simplify test output. A proper loop
+      // fusion pass should check the data dependence graph and run memref
+      // region analysis to ensure removing 'forOpA' is safe.
+      forOpA.erase();
+      return true;
+    }
   }
+  return false;
+}
 
-  // Run tests on all combinations of src/dst loop nests in 'depthToLoops'.
+using LoopFunc = function_ref<bool(AffineForOp, AffineForOp, unsigned, unsigned,
+                                   unsigned, unsigned)>;
+
+// Run tests on all combinations of src/dst loop nests in 'depthToLoops'.
+// If 'return_on_change' is true, returns on first invocation of 'fn' which
+// returns true.
+static bool
+iterateLoops(DenseMap<unsigned, SmallVector<AffineForOp, 2>> &depthToLoops,
+             LoopFunc fn, bool return_on_change = false) {
+  bool changed = false;
   for (auto &depthAndLoops : depthToLoops) {
     unsigned loopDepth = depthAndLoops.first;
     auto &loops = depthAndLoops.second;
     unsigned numLoops = loops.size();
     for (unsigned j = 0; j < numLoops; ++j) {
       for (unsigned k = 0; k < numLoops; ++k) {
-        if (j == k)
-          continue;
-        if (clTestDependenceCheck)
-          testDependenceCheck(loops, j, k, loopDepth, depthToLoops.size());
-        if (clTestSliceComputation)
-          testSliceComputation(loops, j, k, loopDepth, depthToLoops.size());
+        if (j != k)
+          changed |=
+              fn(loops[j], loops[k], j, k, loopDepth, depthToLoops.size());
+        if (changed && return_on_change)
+          return true;
       }
     }
   }
+  return changed;
 }
 
-static PassRegistration<TestLoopFusion>
-    pass("test-loop-fusion", "Tests loop fusion utility functions.");
+void TestLoopFusion::runOnFunction() {
+  DenseMap<unsigned, SmallVector<AffineForOp, 2>> depthToLoops;
+  if (clTestLoopFusionTransformation) {
+    // Run loop fusion until a fixed point is reached.
+    do {
+      depthToLoops.clear();
+      // Gather all AffineForOps by loop depth.
+      for (auto &block : getFunction())
+        gatherLoops(&block, /*currLoopDepth=*/0, depthToLoops);
+
+      // Try to fuse all combinations of src/dst loop nests in 'depthToLoops'.
+    } while (iterateLoops(depthToLoops, testLoopFusionTransformation,
+                          /*return_on_change=*/true));
+    return;
+  }
+
+  // Gather all AffineForOps by loop depth.
+  for (Block &block : getFunction())
+    gatherLoops(&block, /*currLoopDepth=*/0, depthToLoops);
+
+  // Run tests on all combinations of src/dst loop nests in 'depthToLoops'.
+  if (clTestDependenceCheck)
+    iterateLoops(depthToLoops, testDependenceCheck);
+  if (clTestSliceComputation)
+    iterateLoops(depthToLoops, testSliceComputation);
+}
+
+namespace mlir {
+void registerTestLoopFusion() {
+  PassRegistration<TestLoopFusion>("test-loop-fusion",
+                                   "Tests loop fusion utility functions.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestLoopMapping.cpp b/mlir/test/lib/Transforms/TestLoopMapping.cpp
index f20a0ba19c334479b7d645997fd1bbd493c37d1f..ee96d630ae0e8e3ca3214c6a3cc53c7341a1a9cb 100644
--- a/mlir/test/lib/Transforms/TestLoopMapping.cpp
+++ b/mlir/test/lib/Transforms/TestLoopMapping.cpp
@@ -50,7 +50,10 @@ public:
 };
 } // end namespace
 
-static PassRegistration<TestLoopMappingPass>
-    reg("test-mapping-to-processing-elements",
-        "test mapping a single loop on a virtual processor grid",
-        [] { return std::make_unique<TestLoopMappingPass>(); });
+namespace mlir {
+void registerTestLoopMappingPass() {
+  PassRegistration<TestLoopMappingPass>(
+      "test-mapping-to-processing-elements",
+      "test mapping a single loop on a virtual processor grid");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp b/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp
index 0ef602a746f36c86c09f06b02113c0f5e23e9955..f82ea3aed284a8089b7ab08e23c375cb913983bf 100644
--- a/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp
+++ b/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp
@@ -48,12 +48,11 @@ public:
 };
 } // end namespace
 
-std::unique_ptr<OpPassBase<FuncOp>>
-mlir::createSimpleParametricTilingPass(ArrayRef<int64_t> outerLoopSizes) {
-  return std::make_unique<SimpleParametricLoopTilingPass>(outerLoopSizes);
+namespace mlir {
+void registerSimpleParametricTilingPass() {
+  PassRegistration<SimpleParametricLoopTilingPass>(
+      "test-extract-fixed-outer-loops",
+      "test application of parametric tiling to the outer loops so that the "
+      "ranges of outer loops become static");
 }
-
-static PassRegistration<SimpleParametricLoopTilingPass>
-    reg("test-extract-fixed-outer-loops",
-        "test application of parametric tiling to the outer loops so that the "
-        "ranges of outer loops become static");
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp b/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp
index bd53956ee1f461ff1ed0cc2de2e76407cf1e75cf..224d9c34e73d823bcc4a05217295453483151e51 100644
--- a/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp
+++ b/mlir/test/lib/Transforms/TestMemRefBoundCheck.cpp
@@ -48,6 +48,9 @@ void TestMemRefBoundCheck::runOnFunction() {
   });
 }
 
-static PassRegistration<TestMemRefBoundCheck>
-    memRefBoundCheck("test-memref-bound-check",
-                     "Check memref access bounds in a Function");
+namespace mlir {
+void registerMemRefBoundCheck() {
+  PassRegistration<TestMemRefBoundCheck>(
+      "test-memref-bound-check", "Check memref access bounds in a Function");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp b/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp
index f9c45c5cb3380f244dac8efed41010524a98a807..289c9a75f5b19413f44b6bdc7f50d66afe1c7ab1 100644
--- a/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp
+++ b/mlir/test/lib/Transforms/TestMemRefDependenceCheck.cpp
@@ -116,6 +116,10 @@ void TestMemRefDependenceCheck::runOnFunction() {
   checkDependences(loadsAndStores);
 }
 
-static PassRegistration<TestMemRefDependenceCheck>
-    pass("test-memref-dependence-check",
-         "Checks dependences between all pairs of memref accesses.");
+namespace mlir {
+void registerTestMemRefDependenceCheck() {
+  PassRegistration<TestMemRefDependenceCheck> pass(
+      "test-memref-dependence-check",
+      "Checks dependences between all pairs of memref accesses.");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp b/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp
index 3c80741c1d7d2a4c28483f31358e81d69e1afde8..52af86e39682539823803ed9669654bf2c32de2c 100644
--- a/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp
+++ b/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp
@@ -50,5 +50,9 @@ void TestMemRefStrideCalculation::runOnFunction() {
   llvm::outs().flush();
 }
 
-static PassRegistration<TestMemRefStrideCalculation>
-    pass("test-memref-stride-calculation", "Test operation constant folding");
+namespace mlir {
+void registerTestMemRefStrideCalculation() {
+  PassRegistration<TestMemRefStrideCalculation> pass(
+      "test-memref-stride-calculation", "Test operation constant folding");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestOpaqueLoc.cpp b/mlir/test/lib/Transforms/TestOpaqueLoc.cpp
index 1b21b0013d9782187ec2d90c44f2577a244c0867..5f2dab20ea093efdc56db682c42ef04ab0f9b830 100644
--- a/mlir/test/lib/Transforms/TestOpaqueLoc.cpp
+++ b/mlir/test/lib/Transforms/TestOpaqueLoc.cpp
@@ -80,5 +80,9 @@ struct TestOpaqueLoc : public ModulePass<TestOpaqueLoc> {
 
 } // end anonymous namespace
 
-static PassRegistration<TestOpaqueLoc>
-    pass("test-opaque-loc", "Changes all leaf locations to opaque locations");
+namespace mlir {
+void registerTestOpaqueLoc() {
+  PassRegistration<TestOpaqueLoc> pass(
+      "test-opaque-loc", "Changes all leaf locations to opaque locations");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestParallelismDetection.cpp b/mlir/test/lib/Transforms/TestParallelismDetection.cpp
index eca8dda4aaa702f5637c53925d0ec3c1d2c569f3..7c16a259723fdef6c97c051a903c02789d704ff3 100644
--- a/mlir/test/lib/Transforms/TestParallelismDetection.cpp
+++ b/mlir/test/lib/Transforms/TestParallelismDetection.cpp
@@ -27,9 +27,6 @@ struct TestParallelismDetection
 
 } // end anonymous namespace
 
-std::unique_ptr<OpPassBase<FuncOp>> mlir::createParallelismDetectionTestPass() {
-  return std::make_unique<TestParallelismDetection>();
-}
 
 // Walks the function and emits a note for all 'affine.for' ops detected as
 // parallel.
@@ -44,5 +41,9 @@ void TestParallelismDetection::runOnFunction() {
   });
 }
 
-static PassRegistration<TestParallelismDetection>
-    pass("test-detect-parallel", "Test parallelism detection ");
+namespace mlir {
+void registerTestParallelismDetection() {
+  PassRegistration<TestParallelismDetection> pass(
+      "test-detect-parallel", "Test parallelism detection ");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestVectorToLoopsConversion.cpp b/mlir/test/lib/Transforms/TestVectorToLoopsConversion.cpp
index 2d4329db3aa2fead608f0cc49887bc4490b7d671..88d08ce631194f52d42c05406be6a36048b61c85 100644
--- a/mlir/test/lib/Transforms/TestVectorToLoopsConversion.cpp
+++ b/mlir/test/lib/Transforms/TestVectorToLoopsConversion.cpp
@@ -29,6 +29,10 @@ struct TestVectorToLoopsPass
 
 } // end anonymous namespace
 
-static PassRegistration<TestVectorToLoopsPass>
-    pass("test-convert-vector-to-loops",
-         "Converts vector transfer ops to loops over scalars and vector casts");
+namespace mlir {
+void registerTestVectorToLoopsPass() {
+  PassRegistration<TestVectorToLoopsPass> pass(
+      "test-convert-vector-to-loops",
+      "Converts vector transfer ops to loops over scalars and vector casts");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 09561cfb18618bfccd396196f7fb59b1e04f1bf5..d0ac7189a4d935554561d03331915794931210f6 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -44,10 +44,14 @@ struct TestVectorSlicesConversion
 
 } // end anonymous namespace
 
-static PassRegistration<TestVectorToVectorConversion>
-    pass("test-vector-to-vector-conversion",
-         "Test conversion patterns between ops in the vector dialect");
-
-static PassRegistration<TestVectorSlicesConversion> slices_pass(
-    "test-vector-slices-conversion",
-    "Test conversion patterns that lower slices ops in the vector dialect");
+namespace mlir {
+void registerTestVectorConversions() {
+  PassRegistration<TestVectorToVectorConversion> pass(
+      "test-vector-to-vector-conversion",
+      "Test conversion patterns between ops in the vector dialect");
+
+  PassRegistration<TestVectorSlicesConversion> slices_pass(
+      "test-vector-slices-conversion",
+      "Test conversion patterns that lower slices ops in the vector dialect");
+}
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestVectorizationUtils.cpp b/mlir/test/lib/Transforms/TestVectorizationUtils.cpp
index 05705ae5f72a878c4b117512facd97d53ef7492c..50086e5c333a7c4342c819674099d206075d2af8 100644
--- a/mlir/test/lib/Transforms/TestVectorizationUtils.cpp
+++ b/mlir/test/lib/Transforms/TestVectorizationUtils.cpp
@@ -281,12 +281,9 @@ void VectorizerTestPass::runOnFunction() {
   }
 }
 
-std::unique_ptr<OpPassBase<FuncOp>> mlir::createVectorizerTestPass() {
-  return std::make_unique<VectorizerTestPass>();
+namespace mlir {
+void registerVectorizerTestPass() {
+  PassRegistration<VectorizerTestPass> pass(
+      "affine-vectorizer-test", "Tests vectorizer standalone functionality.");
 }
-
-static PassRegistration<VectorizerTestPass>
-    pass("affine-vectorizer-test",
-         "Tests vectorizer standalone functionality.");
-
-#undef DEBUG_TYPE
+} // namespace mlir
diff --git a/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir b/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir
index 59fd969661e316bdd1932032b32f3f6019b2bdcd..a0022afac7e7d47ea58e98afcf403601acbd1cc4 100644
--- a/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir
+++ b/mlir/test/mlir-cpu-runner/bare_ptr_call_conv.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -convert-loop-to-std -convert-std-to-llvm -convert-std-to-llvm-use-bare-ptr-memref-call-conv | mlir-cpu-runner -shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext -entry-point-result=void | FileCheck %s
+// RUN: mlir-opt %s -convert-loop-to-std -convert-std-to-llvm='use-bare-ptr-memref-call-conv=1' | mlir-cpu-runner -shared-libs=%linalg_test_lib_dir/libmlir_runner_utils%shlibext -entry-point-result=void | FileCheck %s
 
 // Verify bare pointer memref calling convention. `simple_add1_add2_test`
 // gets two 2xf32 memrefs, adds 1.0f to the first one and 2.0f to the second
diff --git a/mlir/tools/mlir-cpu-runner/CMakeLists.txt b/mlir/tools/mlir-cpu-runner/CMakeLists.txt
index 21da9052d05cfb9c7bd79dc5a057519f64e12fc6..b0676ef7de71601731b25ca9c6277307efea4b85 100644
--- a/mlir/tools/mlir-cpu-runner/CMakeLists.txt
+++ b/mlir/tools/mlir-cpu-runner/CMakeLists.txt
@@ -2,13 +2,8 @@ add_llvm_tool(mlir-cpu-runner
   mlir-cpu-runner.cpp
 )
 llvm_update_compile_flags(mlir-cpu-runner)
-whole_archive_link(mlir-cpu-runner
-  MLIRAffineOps
-  MLIRLLVMIR
-  MLIRTargetLLVMIR
-  MLIRTranslation
-)
 target_link_libraries(mlir-cpu-runner PRIVATE
+  MLIRAllDialects
   MLIRAnalysis
   MLIREDSC
   MLIRExecutionEngine
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index de18159f18f09012a373e49441ac4102c67c6f2b..b7e1b4442e214b61fc5ce5a1ef446246572d6920 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -5,20 +5,20 @@ set(LLVM_OPTIONAL_SOURCES
 set(LIB_LIBS
   MLIRAnalysis
   MLIRLLVMIR
-  MLIROptMain
+  MLIROptLib
   MLIRParser
   MLIRPass
   MLIRTransforms
   MLIRSupport
 )
-add_llvm_library(MLIRMlirOptLib
+add_llvm_library(MLIRMlirOptMain
   mlir-opt.cpp
 )
-target_link_libraries(MLIRMlirOptLib
+target_link_libraries(MLIRMlirOptMain
   ${LIB_LIBS}
 )
 
-set(FULL_LINK_LIBS
+set(LIBS
   MLIRLoopAnalysis
   MLIRAnalysis
   MLIRAffineOps
@@ -43,7 +43,7 @@ set(FULL_LINK_LIBS
   MLIRLoopOps
   MLIRNVVMIR
   MLIROpenMP
-  MLIROptMain
+  MLIROptLib
   MLIRParser
   MLIRPass
   MLIRQuantizerFxpMathConfig
@@ -68,10 +68,8 @@ set(FULL_LINK_LIBS
   MLIRVectorOps
   MLIRVectorToLLVM
   MLIRVectorToLoops
-)
-set(LIBS
   MLIRIR
-  MLIROptMain
+  MLIROptLib
   LLVMSupport
   LLVMCore
   LLVMAsmParser
@@ -82,19 +80,12 @@ if(MLIR_CUDA_CONVERSIONS_ENABLED)
     LLVMNVPTXCodeGen
     LLVMNVPTXDesc
     LLVMNVPTXInfo
-    MLIRTargetNVVMIR
-  )
-  # Order matters here, so insert at front.
-  list(INSERT FULL_LINK_LIBS 0
     MLIRGPUtoCUDATransforms
+    MLIRTargetNVVMIR
   )
 endif()
 add_llvm_tool(mlir-opt
  mlir-opt.cpp
 )
 llvm_update_compile_flags(mlir-opt)
-# It is necessary to use whole_archive_link to ensure that all static
-# initializers are called. However, whole_archive_link libraries cannot
-# also be target_link_libraries.
-whole_archive_link(mlir-opt ${FULL_LINK_LIBS})
 target_link_libraries(mlir-opt PRIVATE ${LIBS})
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index f637a2feea95390aafdce220669cd8df43c1e1ae..bf6b57c2b62479499185684e204140f10641a531 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -11,6 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Analysis/Passes.h"
+#include "mlir/InitAllDialects.h"
+#include "mlir/InitAllPasses.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Support/FileUtilities.h"
@@ -23,6 +25,34 @@
 using namespace llvm;
 using namespace mlir;
 
+namespace mlir {
+// Defined in the test directory, no public header.
+void registerConvertToTargetEnvPass();
+void registerInliner();
+void registerMemRefBoundCheck();
+void registerPassManagerTestPass();
+void registerPatternsTestPass();
+void registerSimpleParametricTilingPass();
+void registerSymbolTestPasses();
+void registerTestAllReduceLoweringPass();
+void registerTestCallGraphPass();
+void registerTestConstantFold();
+void registerTestFunc();
+void registerTestGpuMemoryPromotionPass();
+void registerTestLinalgTransforms();
+void registerTestLivenessPass();
+void registerTestLoopFusion();
+void registerTestLoopMappingPass();
+void registerTestMatchers();
+void registerTestMemRefDependenceCheck();
+void registerTestMemRefStrideCalculation();
+void registerTestOpaqueLoc();
+void registerTestParallelismDetection();
+void registerTestVectorConversions();
+void registerTestVectorToLoopsPass();
+void registerVectorizerTestPass();
+} // namespace mlir
+
 static cl::opt<std::string>
     inputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
 
@@ -47,7 +77,45 @@ static cl::opt<bool>
                  cl::desc("Run the verifier after each transformation pass"),
                  cl::init(true));
 
+void registerTestPasses() {
+  registerConvertToTargetEnvPass();
+  registerInliner();
+  registerMemRefBoundCheck();
+  registerPassManagerTestPass();
+  registerPatternsTestPass();
+  registerSimpleParametricTilingPass();
+  registerSymbolTestPasses();
+  registerTestAllReduceLoweringPass();
+  registerTestCallGraphPass();
+  registerTestConstantFold();
+  registerTestFunc();
+  registerTestGpuMemoryPromotionPass();
+  registerTestLinalgTransforms();
+  registerTestLivenessPass();
+  registerTestLoopFusion();
+  registerTestLoopMappingPass();
+  registerTestMatchers();
+  registerTestMemRefDependenceCheck();
+  registerTestMemRefStrideCalculation();
+  registerTestOpaqueLoc();
+  registerTestParallelismDetection();
+  registerTestVectorConversions();
+  registerTestVectorToLoopsPass();
+  registerVectorizerTestPass();
+
+  // The following passes are using global initializers, just link them in.
+  if (std::getenv("bar") != (char *)-1)
+    return;
+
+  // TODO: move these to the test folder.
+  createTestMemRefBoundCheckPass();
+  createTestMemRefDependenceCheckPass();
+}
+
 int main(int argc, char **argv) {
+  registerAllDialects();
+  registerAllPasses();
+  registerTestPasses();
   InitLLVM y(argc, argv);
 
   // Register any pass manager command line options.
diff --git a/mlir/tools/mlir-translate/CMakeLists.txt b/mlir/tools/mlir-translate/CMakeLists.txt
index 9d585579060d7d3b763ab4d466b5c29ee91e19a3..4b95a953e86718aab34a3d2ed1285a3660047071 100644
--- a/mlir/tools/mlir-translate/CMakeLists.txt
+++ b/mlir/tools/mlir-translate/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LIBS
+  MLIRAllDialects
   MLIRParser
   MLIRPass
   MLIRSPIRV
@@ -9,9 +10,15 @@ set(LIBS
   MLIRTranslation
   MLIRSupport
 )
+set(FULL_LIBS
+  MLIRSPIRVSerialization
+  MLIRTargetLLVMIR
+  MLIRTargetNVVMIR
+  MLIRTargetROCDLIR
+)
 add_llvm_tool(mlir-translate
   mlir-translate.cpp
 )
 llvm_update_compile_flags(mlir-translate)
-whole_archive_link(mlir-translate ${LIBS})
+whole_archive_link(mlir-translate ${FULL_LIBS})
 target_link_libraries(mlir-translate PRIVATE MLIRIR MLIRTranslateClParser ${LIBS} LLVMSupport)
diff --git a/mlir/tools/mlir-translate/mlir-translate.cpp b/mlir/tools/mlir-translate/mlir-translate.cpp
index 5181436d7c8da5bfefd8d2ddc4e804f576408212..cd1b226deff7d68c2dbd95743789e0144bf760cd 100644
--- a/mlir/tools/mlir-translate/mlir-translate.cpp
+++ b/mlir/tools/mlir-translate/mlir-translate.cpp
@@ -13,6 +13,7 @@
 
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/MLIRContext.h"
+#include "mlir/InitAllDialects.h"
 #include "mlir/Support/FileUtilities.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Support/ToolUtilities.h"
@@ -45,6 +46,7 @@ static llvm::cl::opt<bool> verifyDiagnostics(
     llvm::cl::init(false));
 
 int main(int argc, char **argv) {
+  registerAllDialects();
   llvm::InitLLVM y(argc, argv);
 
   // Add flags for all the registered translations.
diff --git a/mlir/unittests/Dialect/QuantOps/QuantizationUtilsTest.cpp b/mlir/unittests/Dialect/QuantOps/QuantizationUtilsTest.cpp
index 46dc57fb4da34f04f510dd2fde7e3d2df76a18fe..e6188ea0de7ed5c3eab86453ad25e01c6c462c9e 100644
--- a/mlir/unittests/Dialect/QuantOps/QuantizationUtilsTest.cpp
+++ b/mlir/unittests/Dialect/QuantOps/QuantizationUtilsTest.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/QuantOps/QuantOps.h"
 #include "mlir/Dialect/QuantOps/QuantizeUtils.h"
 #include "mlir/Dialect/QuantOps/UniformSupport.h"
 #include "mlir/IR/Attributes.h"
@@ -16,6 +17,9 @@
 using namespace mlir;
 using namespace mlir::quant;
 
+// Load the quant dialect
+static DialectRegistration<QuantizationDialect> QuantOpsRegistration;
+
 namespace {
 
 // Test UniformQuantizedValueConverter converts all APFloat to a magic number 5.
diff --git a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
index bcc3d1cb29aef2cb4877dcc08edbf30804d031b8..bee6a2d434b33d9c96edbe042f5678422a696b74 100644
--- a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
+++ b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/SPIRV/SPIRVBinaryUtils.h"
+#include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Serialization.h"
 #include "mlir/IR/Diagnostics.h"
@@ -23,6 +24,9 @@
 
 using namespace mlir;
 
+// Load the SPIRV dialect
+static DialectRegistration<spirv::SPIRVDialect> SPIRVRegistration;
+
 using ::testing::StrEq;
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/unittests/SDBM/SDBMTest.cpp b/mlir/unittests/SDBM/SDBMTest.cpp
index eadb3597f1dd9cf18c0bae601344f94a1d5b479d..e599094ee5929ab762e1b7bfed69fedfbe8bdd7c 100644
--- a/mlir/unittests/SDBM/SDBMTest.cpp
+++ b/mlir/unittests/SDBM/SDBMTest.cpp
@@ -17,6 +17,9 @@
 
 using namespace mlir;
 
+// Load the SDBM dialect
+static DialectRegistration<SDBMDialect> SDBMRegistration;
+
 static MLIRContext *ctx() {
   static thread_local MLIRContext context;
   return &context;
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index b705694a4584173afbafbdbf6fe0a7598def9b16..2beafca9c6693c66e241e0e80bd63062b9a4a449 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -63,7 +63,7 @@ set(ENABLE_LIBOMPTARGET ON)
 # Currently libomptarget cannot be compiled on Windows or MacOS X.
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
-if (APPLE OR WIN32 OR NOT OPENMP_HAVE_STD_CPP11_FLAG)
+if (APPLE OR WIN32 OR NOT OPENMP_HAVE_STD_CPP14_FLAG)
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
 
@@ -73,8 +73,8 @@ if (OPENMP_ENABLE_LIBOMPTARGET)
   # Check that the library can actually be built.
   if (APPLE OR WIN32)
     message(FATAL_ERROR "libomptarget cannot be built on Windows and MacOS X!")
-  elseif (NOT OPENMP_HAVE_STD_CPP11_FLAG)
-    message(FATAL_ERROR "Host compiler must support C++11 to build libomptarget!")
+  elseif (NOT OPENMP_HAVE_STD_CPP14_FLAG)
+    message(FATAL_ERROR "Host compiler must support C++14 to build libomptarget!")
   endif()
 
   add_subdirectory(libomptarget)
diff --git a/openmp/cmake/HandleOpenMPOptions.cmake b/openmp/cmake/HandleOpenMPOptions.cmake
index eb7b286b3383bb95d42e474715a98df6471fe66e..15382bcf12de6ccc31e64de1541ef44acdd7fc52 100644
--- a/openmp/cmake/HandleOpenMPOptions.cmake
+++ b/openmp/cmake/HandleOpenMPOptions.cmake
@@ -29,7 +29,7 @@ append_if(OPENMP_HAVE_WNO_EXTRA_FLAG "-Wno-extra" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
 append_if(OPENMP_HAVE_WNO_PEDANTIC_FLAG "-Wno-pedantic" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
 append_if(OPENMP_HAVE_WNO_MAYBE_UNINITIALIZED_FLAG "-Wno-maybe-uninitialized" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
 
-append_if(OPENMP_HAVE_STD_GNUPP11_FLAG "-std=gnu++11" CMAKE_CXX_FLAGS)
-if (NOT OPENMP_HAVE_STD_GNUPP11_FLAG)
-  append_if(OPENMP_HAVE_STD_CPP11_FLAG "-std=c++11" CMAKE_CXX_FLAGS)
+append_if(OPENMP_HAVE_STD_GNUPP14_FLAG "-std=gnu++14" CMAKE_CXX_FLAGS)
+if (NOT OPENMP_HAVE_STD_GNUPP14_FLAG)
+  append_if(OPENMP_HAVE_STD_CPP14_FLAG "-std=c++14" CMAKE_CXX_FLAGS)
 endif()
diff --git a/openmp/cmake/config-ix.cmake b/openmp/cmake/config-ix.cmake
index ebb2530ae54137468b2dff1a826d53fcd283c796..d9ea3bbb05749f07b7b33a959a74dc41a04220de 100644
--- a/openmp/cmake/config-ix.cmake
+++ b/openmp/cmake/config-ix.cmake
@@ -14,5 +14,5 @@ check_cxx_compiler_flag(-Wno-extra OPENMP_HAVE_WNO_EXTRA_FLAG)
 check_cxx_compiler_flag(-Wno-pedantic OPENMP_HAVE_WNO_PEDANTIC_FLAG)
 check_cxx_compiler_flag(-Wno-maybe-uninitialized OPENMP_HAVE_WNO_MAYBE_UNINITIALIZED_FLAG)
 
-check_cxx_compiler_flag(-std=gnu++11 OPENMP_HAVE_STD_GNUPP11_FLAG)
-check_cxx_compiler_flag(-std=c++11 OPENMP_HAVE_STD_CPP11_FLAG)
+check_cxx_compiler_flag(-std=gnu++14 OPENMP_HAVE_STD_GNUPP14_FLAG)
+check_cxx_compiler_flag(-std=c++14 OPENMP_HAVE_STD_CPP14_FLAG)
diff --git a/openmp/libomptarget/cmake/Modules/LibomptargetNVPTXBitcodeLibrary.cmake b/openmp/libomptarget/cmake/Modules/LibomptargetNVPTXBitcodeLibrary.cmake
index 538bf9277495744f81a75fdca7cfb660af8db793..6ec0cc2b61bc0993d11f3aeb1b4aa31ea6b20e9d 100644
--- a/openmp/libomptarget/cmake/Modules/LibomptargetNVPTXBitcodeLibrary.cmake
+++ b/openmp/libomptarget/cmake/Modules/LibomptargetNVPTXBitcodeLibrary.cmake
@@ -78,7 +78,7 @@ endfunction()
 
 # These flags are required to emit LLVM Bitcode. We check them together because
 # if any of them are not supported, there is no point in finding out which are.
-set(compiler_flags_required -emit-llvm -O1 --cuda-device-only -std=c++11 --cuda-path=${CUDA_TOOLKIT_ROOT_DIR})
+set(compiler_flags_required -emit-llvm -O1 --cuda-device-only -std=c++14 --cuda-path=${CUDA_TOOLKIT_ROOT_DIR})
 set(compiler_flags_required_src "extern \"C\" __device__ int thread() { return threadIdx.x; }")
 check_bitcode_compilation(LIBOMPTARGET_NVPTX_CUDA_COMPILER_SUPPORTS_FLAGS_REQUIRED "${compiler_flags_required_src}" ${compiler_flags_required})
 
diff --git a/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt b/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt
index aac7ddc28adc21b908010fb2fc150e29a5008e22..ec08bd912663a316bc28a361db165a611940be47 100644
--- a/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt
+++ b/openmp/libomptarget/deviceRTLs/amdgcn/CMakeLists.txt
@@ -98,7 +98,7 @@ endif()
 
 macro(add_cuda_bc_library)
   set(cu_cmd ${AOMP_BINDIR}/clang++
-    -std=c++11
+    -std=c++14
     -fcuda-rdc
     -fvisibility=default
     --cuda-device-only
diff --git a/openmp/libomptarget/plugins/cuda/src/rtl.cpp b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
index 49acb6cacbfe985c5089f84eb012c27d404d7bb6..54248daa7f19cab0463c1edb6d14aaed3a9cdb44 100644
--- a/openmp/libomptarget/plugins/cuda/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/cuda/src/rtl.cpp
@@ -10,10 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <atomic>
 #include <cassert>
 #include <cstddef>
 #include <cuda.h>
 #include <list>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -90,11 +92,13 @@ std::list<KernelTy> KernelsList;
 /// Class containing all the device information.
 class RTLDeviceInfoTy {
   std::vector<std::list<FuncOrGblEntryTy>> FuncGblEntries;
+  std::vector<std::unique_ptr<std::atomic_uint>> NextStreamId;
 
 public:
   int NumberOfDevices;
   std::vector<CUmodule> Modules;
   std::vector<CUcontext> Contexts;
+  std::vector<std::vector<CUstream>> Streams;
 
   // Device properties
   std::vector<int> ThreadsPerBlock;
@@ -108,6 +112,7 @@ public:
   // OpenMP Environment properties
   int EnvNumTeams;
   int EnvTeamLimit;
+  int EnvNumStreams;
 
   // OpenMP Requires Flags
   int64_t RequiresFlags;
@@ -173,6 +178,15 @@ public:
     E.Table.EntriesBegin = E.Table.EntriesEnd = 0;
   }
 
+  // Get the next stream on a given device in a round robin manner
+  CUstream &getNextStream(const int DeviceId) {
+    assert(DeviceId >= 0 &&
+           static_cast<size_t>(DeviceId) < NextStreamId.size() &&
+           "Unexpected device id!");
+    const unsigned int Id = NextStreamId[DeviceId]->fetch_add(1);
+    return Streams[DeviceId][Id % EnvNumStreams];
+  }
+
   RTLDeviceInfoTy() {
 #ifdef OMPTARGET_DEBUG
     if (char *envStr = getenv("LIBOMPTARGET_DEBUG")) {
@@ -205,6 +219,8 @@ public:
 
     FuncGblEntries.resize(NumberOfDevices);
     Contexts.resize(NumberOfDevices);
+    Streams.resize(NumberOfDevices);
+    NextStreamId.resize(NumberOfDevices);
     ThreadsPerBlock.resize(NumberOfDevices);
     BlocksPerGrid.resize(NumberOfDevices);
     WarpSize.resize(NumberOfDevices);
@@ -229,6 +245,23 @@ public:
       EnvNumTeams = -1;
     }
 
+    // By default let's create 256 streams per device
+    EnvNumStreams = 256;
+    envStr = getenv("LIBOMPTARGET_NUM_STREAMS");
+    if (envStr) {
+      EnvNumStreams = std::stoi(envStr);
+    }
+
+    // Initialize streams for each device
+    for (std::vector<CUstream> &S : Streams) {
+      S.resize(EnvNumStreams);
+    }
+
+    // Initialize the next stream id
+    for (std::unique_ptr<std::atomic_uint> &Ptr : NextStreamId) {
+      Ptr = std::make_unique<std::atomic_uint>(0);
+    }
+
     // Default state.
     RequiresFlags = OMP_REQ_UNDEFINED;
   }
@@ -244,6 +277,24 @@ public:
         }
       }
 
+    // Destroy streams before contexts
+    for (int I = 0; I < NumberOfDevices; ++I) {
+      CUresult err = cuCtxSetCurrent(Contexts[I]);
+      if (err != CUDA_SUCCESS) {
+        DP("Error when setting current CUDA context\n");
+        CUDA_ERR_STRING(err);
+      }
+
+      for (auto &S : Streams[I])
+        if (S) {
+          err = cuStreamDestroy(S);
+          if (err != CUDA_SUCCESS) {
+            DP("Error when destroying CUDA stream\n");
+            CUDA_ERR_STRING(err);
+          }
+        }
+    }
+
     // Destroy contexts
     for (auto &ctx : Contexts)
       if (ctx) {
@@ -294,6 +345,20 @@ int32_t __tgt_rtl_init_device(int32_t device_id) {
     return OFFLOAD_FAIL;
   }
 
+  err = cuCtxSetCurrent(DeviceInfo.Contexts[device_id]);
+  if (err != CUDA_SUCCESS) {
+    DP("Error when setting current CUDA context\n");
+    CUDA_ERR_STRING(err);
+  }
+
+  for (CUstream &Stream : DeviceInfo.Streams[device_id]) {
+    err = cuStreamCreate(&Stream, CU_STREAM_NON_BLOCKING);
+    if (err != CUDA_SUCCESS) {
+      DP("Error when creating CUDA stream\n");
+      CUDA_ERR_STRING(err);
+    }
+  }
+
   // Query attributes to determine number of threads/block and blocks/grid.
   int maxGridDimX;
   err = cuDeviceGetAttribute(&maxGridDimX, CU_DEVICE_ATTRIBUTE_MAX_GRID_DIM_X,
@@ -607,14 +672,26 @@ int32_t __tgt_rtl_data_submit(int32_t device_id, void *tgt_ptr, void *hst_ptr,
     return OFFLOAD_FAIL;
   }
 
-  err = cuMemcpyHtoD((CUdeviceptr)tgt_ptr, hst_ptr, size);
+  CUstream &Stream = DeviceInfo.getNextStream(device_id);
+
+  err = cuMemcpyHtoDAsync((CUdeviceptr)tgt_ptr, hst_ptr, size, Stream);
   if (err != CUDA_SUCCESS) {
     DP("Error when copying data from host to device. Pointers: host = " DPxMOD
-       ", device = " DPxMOD ", size = %" PRId64 "\n", DPxPTR(hst_ptr),
-       DPxPTR(tgt_ptr), size);
+       ", device = " DPxMOD ", size = %" PRId64 "\n",
+       DPxPTR(hst_ptr), DPxPTR(tgt_ptr), size);
+    CUDA_ERR_STRING(err);
+    return OFFLOAD_FAIL;
+  }
+
+  err = cuStreamSynchronize(Stream);
+  if (err != CUDA_SUCCESS) {
+    DP("Error when synchronizing async data transfer from host to device. "
+       "Pointers: host = " DPxMOD ", device = " DPxMOD ", size = %" PRId64 "\n",
+       DPxPTR(hst_ptr), DPxPTR(tgt_ptr), size);
     CUDA_ERR_STRING(err);
     return OFFLOAD_FAIL;
   }
+
   return OFFLOAD_SUCCESS;
 }
 
@@ -628,14 +705,26 @@ int32_t __tgt_rtl_data_retrieve(int32_t device_id, void *hst_ptr, void *tgt_ptr,
     return OFFLOAD_FAIL;
   }
 
-  err = cuMemcpyDtoH(hst_ptr, (CUdeviceptr)tgt_ptr, size);
+  CUstream &Stream = DeviceInfo.getNextStream(device_id);
+
+  err = cuMemcpyDtoHAsync(hst_ptr, (CUdeviceptr)tgt_ptr, size, Stream);
   if (err != CUDA_SUCCESS) {
     DP("Error when copying data from device to host. Pointers: host = " DPxMOD
-        ", device = " DPxMOD ", size = %" PRId64 "\n", DPxPTR(hst_ptr),
-        DPxPTR(tgt_ptr), size);
+       ", device = " DPxMOD ", size = %" PRId64 "\n",
+       DPxPTR(hst_ptr), DPxPTR(tgt_ptr), size);
+    CUDA_ERR_STRING(err);
+    return OFFLOAD_FAIL;
+  }
+
+  err = cuStreamSynchronize(Stream);
+  if (err != CUDA_SUCCESS) {
+    DP("Error when synchronizing async data transfer from device to host. "
+       "Pointers: host = " DPxMOD ", device = " DPxMOD ", size = %" PRId64 "\n",
+       DPxPTR(hst_ptr), DPxPTR(tgt_ptr), size);
     CUDA_ERR_STRING(err);
     return OFFLOAD_FAIL;
   }
+
   return OFFLOAD_SUCCESS;
 }
 
@@ -755,8 +844,11 @@ int32_t __tgt_rtl_run_target_team_region(int32_t device_id, void *tgt_entry_ptr,
   DP("Launch kernel with %d blocks and %d threads\n", cudaBlocksPerGrid,
      cudaThreadsPerBlock);
 
+  CUstream &Stream = DeviceInfo.getNextStream(device_id);
+
   err = cuLaunchKernel(KernelInfo->Func, cudaBlocksPerGrid, 1, 1,
-      cudaThreadsPerBlock, 1, 1, 0 /*bytes of shared memory*/, 0, &args[0], 0);
+                       cudaThreadsPerBlock, 1, 1, 0 /*bytes of shared memory*/,
+                       Stream, &args[0], 0);
   if (err != CUDA_SUCCESS) {
     DP("Device kernel launch failed!\n");
     CUDA_ERR_STRING(err);
@@ -766,7 +858,7 @@ int32_t __tgt_rtl_run_target_team_region(int32_t device_id, void *tgt_entry_ptr,
   DP("Launch of entry point at " DPxMOD " successful!\n",
       DPxPTR(tgt_entry_ptr));
 
-  CUresult sync_err = cuCtxSynchronize();
+  CUresult sync_err = cuStreamSynchronize(Stream);
   if (sync_err != CUDA_SUCCESS) {
     DP("Kernel execution error at " DPxMOD "!\n", DPxPTR(tgt_entry_ptr));
     CUDA_ERR_STRING(sync_err);
diff --git a/openmp/libomptarget/test/offloading/parallel_offloading_map.c b/openmp/libomptarget/test/offloading/parallel_offloading_map.c
new file mode 100644
index 0000000000000000000000000000000000000000..bcdc6f96f4d004c9779a7e7229e24085ac856f86
--- /dev/null
+++ b/openmp/libomptarget/test/offloading/parallel_offloading_map.c
@@ -0,0 +1,41 @@
+// RUN: %libomptarget-compilexx-run-and-check-aarch64-unknown-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-powerpc64-ibm-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-powerpc64le-ibm-linux-gnu
+// RUN: %libomptarget-compilexx-run-and-check-x86_64-pc-linux-gnu
+#include <assert.h>
+#include <stdio.h>
+
+int main(int argc, char *argv[]) {
+  const int num_threads = 64, N = 128;
+  int array[num_threads] = {0};
+
+#pragma omp parallel for
+  for (int i = 0; i < num_threads; ++i) {
+    int tmp[N];
+
+    for (int j = 0; j < N; ++j) {
+      tmp[j] = i;
+    }
+
+#pragma omp target teams distribute parallel for map(tofrom : tmp)
+    for (int j = 0; j < N; ++j) {
+      tmp[j] += j;
+    }
+
+    for (int j = 0; j < N; ++j) {
+      array[i] += tmp[j];
+    }
+  }
+
+  // Verify
+  for (int i = 0; i < num_threads; ++i) {
+    const int ref = (0 + N - 1) * N / 2 + i * N;
+    assert(array[i] == ref);
+  }
+
+  printf("PASS\n");
+
+  return 0;
+}
+
+// CHECK: PASS
diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg
index ff1dd2ac700de3e1034bc6521eaeea7d6e01da92..974e354c00135b645a619ebd04eeeb1302266895 100644
--- a/openmp/runtime/test/lit.cfg
+++ b/openmp/runtime/test/lit.cfg
@@ -111,7 +111,7 @@ config.substitutions.append(("%libomp-compile-and-run", \
 config.substitutions.append(("%libomp-cxx-compile-and-run", \
     "%libomp-cxx-compile && %libomp-run"))
 config.substitutions.append(("%libomp-cxx-compile", \
-    "%clangXX %openmp_flags %flags -std=c++11 %s -o %t" + libs))
+    "%clangXX %openmp_flags %flags -std=c++14 %s -o %t" + libs))
 config.substitutions.append(("%libomp-compile", \
     "%clang %openmp_flags %flags %s -o %t" + libs))
 config.substitutions.append(("%libomp-run", config.runner))
diff --git a/openmp/tools/archer/tests/lit.cfg b/openmp/tools/archer/tests/lit.cfg
index 3014c020bb99a7342c8dddaf02d8370526631e94..7ec88d3ca4e45d80dd7854253d30fd249bb5e281 100644
--- a/openmp/tools/archer/tests/lit.cfg
+++ b/openmp/tools/archer/tests/lit.cfg
@@ -97,7 +97,7 @@ config.substitutions.append(("%libarcher-compile-and-run", \
 config.substitutions.append(("%libarcher-cxx-compile-and-run", \
     "%libarcher-cxx-compile && %libarcher-run"))
 config.substitutions.append(("%libarcher-cxx-compile", \
-    "%clang-archerXX %openmp_flags %archer_flags %flags -std=c++11 %s -o %t" + libs))
+    "%clang-archerXX %openmp_flags %archer_flags %flags -std=c++14 %s -o %t" + libs))
 config.substitutions.append(("%libarcher-compile", \
                              "%clang-archer %openmp_flags %archer_flags %flags %s -o %t" + libs))
 config.substitutions.append(("%libarcher-run-race", "%suppression %deflake %t 2>&1 | tee %t.log"))
diff --git a/polly/lib/CodeGen/LoopGeneratorsKMP.cpp b/polly/lib/CodeGen/LoopGeneratorsKMP.cpp
index e57205107f9aa5b1055ede13a8414daeaef00f8d..db753b1f4606895aed5d18a76ad43df7144d7bf0 100644
--- a/polly/lib/CodeGen/LoopGeneratorsKMP.cpp
+++ b/polly/lib/CodeGen/LoopGeneratorsKMP.cpp
@@ -103,28 +103,31 @@ Function *ParallelLoopGeneratorKMP::prepareSubFnDefinition(Function *F) const {
 
 // Create a subfunction of the following (preliminary) structure:
 //
-//    PrevBB
-//       |
-//       v
-//    HeaderBB
-//       |   _____
-//       v  v    |
-//   CheckNextBB  PreHeaderBB
-//       |\       |
-//       | \______/
-//       |
-//       v
-//     ExitBB
+//        PrevBB
+//           |
+//           v
+//        HeaderBB
+//       /   |    _____
+//      /    v   v     |
+//     / PreHeaderBB   |
+//    |      |         |
+//    |      v         |
+//    |  CheckNextBB   |
+//     \     |   \_____/
+//      \    |
+//       v   v
+//       ExitBB
 //
 // HeaderBB will hold allocations, loading of variables and kmp-init calls.
-// CheckNextBB will check for more work (dynamic) or will be "empty" (static).
+// CheckNextBB will check for more work (dynamic / static chunked) or will be
+// empty (static non chunked).
 // If there is more work to do: go to PreHeaderBB, otherwise go to ExitBB.
 // PreHeaderBB loads the new boundaries (& will lead to the loop body later on).
-// Just like CheckNextBB: PreHeaderBB is empty in the static scheduling case.
-// ExitBB marks the end of the parallel execution.
+// Just like CheckNextBB: PreHeaderBB is (preliminary) empty in the static non
+// chunked scheduling case. ExitBB marks the end of the parallel execution.
 // The possibly empty BasicBlocks will automatically be removed.
 std::tuple<Value *, Function *>
-ParallelLoopGeneratorKMP::createSubFn(Value *StrideNotUsed,
+ParallelLoopGeneratorKMP::createSubFn(Value *SequentialLoopStride,
                                       AllocaInst *StructData,
                                       SetVector<Value *> Data, ValueMapT &Map) {
   Function *SubFn = createSubFnDefinition();
@@ -193,7 +196,10 @@ ParallelLoopGeneratorKMP::createSubFn(Value *StrideNotUsed,
   Value *ChunkSize =
       ConstantInt::get(LongType, std::max<int>(PollyChunkSize, 1));
 
-  switch (PollyScheduling) {
+  OMPGeneralSchedulingType Scheduling =
+      getSchedType(PollyChunkSize, PollyScheduling);
+
+  switch (Scheduling) {
   case OMPGeneralSchedulingType::Dynamic:
   case OMPGeneralSchedulingType::Guided:
   case OMPGeneralSchedulingType::Runtime:
@@ -224,24 +230,56 @@ ParallelLoopGeneratorKMP::createSubFn(Value *StrideNotUsed,
   case OMPGeneralSchedulingType::StaticNonChunked:
     // "STATIC" scheduling types are handled below
     {
+      Builder.CreateAlignedStore(AdjustedUB, UBPtr, Alignment);
       createCallStaticInit(ID, IsLastPtr, LBPtr, UBPtr, StridePtr, ChunkSize);
 
-      LB = Builder.CreateAlignedLoad(LBPtr, Alignment, "polly.indvar.LB");
-      UB = Builder.CreateAlignedLoad(UBPtr, Alignment, "polly.indvar.UB");
+      Value *ChunkedStride =
+          Builder.CreateAlignedLoad(StridePtr, Alignment, "polly.kmpc.stride");
 
-      Value *AdjUBOutOfBounds =
-          Builder.CreateICmp(llvm::CmpInst::Predicate::ICMP_SLT, UB, AdjustedUB,
-                             "polly.adjustedUBOutOfBounds");
+      LB = Builder.CreateAlignedLoad(LBPtr, Alignment, "polly.indvar.LB");
+      UB = Builder.CreateAlignedLoad(UBPtr, Alignment, "polly.indvar.UB.temp");
 
-      UB = Builder.CreateSelect(AdjUBOutOfBounds, UB, AdjustedUB);
+      Value *UBInRange =
+          Builder.CreateICmp(llvm::CmpInst::Predicate::ICMP_SLE, UB, AdjustedUB,
+                             "polly.indvar.UB.inRange");
+      UB = Builder.CreateSelect(UBInRange, UB, AdjustedUB, "polly.indvar.UB");
       Builder.CreateAlignedStore(UB, UBPtr, Alignment);
 
       Value *HasIteration = Builder.CreateICmp(
           llvm::CmpInst::Predicate::ICMP_SLE, LB, UB, "polly.hasIteration");
       Builder.CreateCondBr(HasIteration, PreHeaderBB, ExitBB);
 
+      if (Scheduling == OMPGeneralSchedulingType::StaticChunked) {
+        Builder.SetInsertPoint(PreHeaderBB);
+        LB = Builder.CreateAlignedLoad(LBPtr, Alignment,
+                                       "polly.indvar.LB.entry");
+        UB = Builder.CreateAlignedLoad(UBPtr, Alignment,
+                                       "polly.indvar.UB.entry");
+      }
+
       Builder.SetInsertPoint(CheckNextBB);
-      Builder.CreateBr(ExitBB);
+
+      if (Scheduling == OMPGeneralSchedulingType::StaticChunked) {
+        Value *NextLB =
+            Builder.CreateAdd(LB, ChunkedStride, "polly.indvar.nextLB");
+        Value *NextUB = Builder.CreateAdd(UB, ChunkedStride);
+
+        Value *NextUBOutOfBounds =
+            Builder.CreateICmp(llvm::CmpInst::Predicate::ICMP_SGT, NextUB,
+                               AdjustedUB, "polly.indvar.nextUB.outOfBounds");
+        NextUB = Builder.CreateSelect(NextUBOutOfBounds, AdjustedUB, NextUB,
+                                      "polly.indvar.nextUB");
+
+        Builder.CreateAlignedStore(NextLB, LBPtr, Alignment);
+        Builder.CreateAlignedStore(NextUB, UBPtr, Alignment);
+
+        Value *HasWork =
+            Builder.CreateICmp(llvm::CmpInst::Predicate::ICMP_SLE, NextLB,
+                               AdjustedUB, "polly.hasWork");
+        Builder.CreateCondBr(HasWork, PreHeaderBB, ExitBB);
+      } else {
+        Builder.CreateBr(ExitBB);
+      }
 
       Builder.SetInsertPoint(PreHeaderBB);
     }
@@ -251,7 +289,7 @@ ParallelLoopGeneratorKMP::createSubFn(Value *StrideNotUsed,
   Builder.CreateBr(CheckNextBB);
   Builder.SetInsertPoint(&*--Builder.GetInsertPoint());
   BasicBlock *AfterBB;
-  Value *IV = createLoop(LB, UB, Stride, Builder, LI, DT, AfterBB,
+  Value *IV = createLoop(LB, UB, SequentialLoopStride, Builder, LI, DT, AfterBB,
                          ICmpInst::ICMP_SLE, nullptr, true,
                          /* UseGuard */ false);
 
@@ -260,7 +298,8 @@ ParallelLoopGeneratorKMP::createSubFn(Value *StrideNotUsed,
   // Add code to terminate this subfunction.
   Builder.SetInsertPoint(ExitBB);
   // Static (i.e. non-dynamic) scheduling types, are terminated with a fini-call
-  if (PollyScheduling == OMPGeneralSchedulingType::StaticChunked) {
+  if (Scheduling == OMPGeneralSchedulingType::StaticChunked ||
+      Scheduling == OMPGeneralSchedulingType::StaticNonChunked) {
     createCallStaticFini(ID);
   }
   Builder.CreateRetVoid();
diff --git a/polly/lib/Transform/ScheduleOptimizer.cpp b/polly/lib/Transform/ScheduleOptimizer.cpp
index c54bc08445b73c86142286539c7e12427c2cb2d3..d9dffb8f1f11d287eff9a7bc066645b23a96d6b1 100644
--- a/polly/lib/Transform/ScheduleOptimizer.cpp
+++ b/polly/lib/Transform/ScheduleOptimizer.cpp
@@ -809,7 +809,7 @@ static isl::schedule_node permuteBandNodeDimensions(isl::schedule_node Node,
                                                     unsigned FirstDim,
                                                     unsigned SecondDim) {
   assert(isl_schedule_node_get_type(Node.get()) == isl_schedule_node_band &&
-         isl_schedule_node_band_n_member(Node.get()) >
+         (unsigned)isl_schedule_node_band_n_member(Node.get()) >
              std::max(FirstDim, SecondDim));
   auto PartialSchedule =
       isl::manage(isl_schedule_node_band_get_partial_schedule(Node.get()));
diff --git a/polly/test/Isl/CodeGen/OpenMP/single_loop.ll b/polly/test/Isl/CodeGen/OpenMP/single_loop.ll
index 6de65bd8ad98bc2eca4fe0600d49db8aa7820922..38e336a564d28297677aa776cd483e75b6dce33c 100644
--- a/polly/test/Isl/CodeGen/OpenMP/single_loop.ll
+++ b/polly/test/Isl/CodeGen/OpenMP/single_loop.ll
@@ -4,7 +4,8 @@
 ; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-ast -analyze < %s | FileCheck %s -check-prefix=AST-STRIDE4
 ; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-codegen -S < %s | FileCheck %s -check-prefix=IR-STRIDE4
 
-; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -polly-scheduling-chunksize=43 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -polly-scheduling-chunksize=43 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC-CHUNKED
+; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=static -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-STATIC
 ; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC
 ; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-codegen -polly-omp-backend=LLVM -polly-scheduling=dynamic -polly-scheduling-chunksize=4 -S -verify-dom-info < %s | FileCheck %s -check-prefix=LIBOMP-IR-DYNAMIC-FOUR
 ; RUN: opt %loadPolly -polly-parallel -polly-parallel-force -polly-import-jscop -polly-codegen -polly-omp-backend=LLVM -S < %s | FileCheck %s -check-prefix=LIBOMP-IR-STRIDE4
@@ -88,67 +89,110 @@
 ; IR-STRIDE4:  %polly.indvar_next = add nsw i64 %polly.indvar, 4
 ; IR-STRIDE4   %polly.adjust_ub = sub i64 %polly.par.UBAdjusted, 4
 
-; LIBOMP-IR: %struct.ident_t = type { i32, i32, i32, i32, i8* }
-
-; LIBOMP-IR-LABEL: single_parallel_loop()
-; LIBOMP-IR-NEXT: entry
-; LIBOMP-IR-NEXT:   %polly.par.userContext = alloca
-
-; LIBOMP-IR-LABEL: polly.parallel.for:
-; LIBOMP-IR-NEXT:   %polly.par.userContext1 = bitcast {}* %polly.par.userContext to i8*
-; LIBOMP-IR-NEXT:   call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @.loc.dummy, i32 4, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i64, i64, i64, i8*)* @single_parallel_loop_polly_subfn to void (i32*, i32*, ...)*), i64 0, i64 1024, i64 1, i8* %polly.par.userContext1)
-; LIBOMP-IR-NEXT:   br label %polly.exiting
-
-; LIBOMP-IR: define internal void @single_parallel_loop_polly_subfn(i32* %polly.kmpc.global_tid, i32* %polly.kmpc.bound_tid, i64 %polly.kmpc.lb, i64 %polly.kmpc.ub, i64 %polly.kmpc.inc, i8* %polly.kmpc.shared)
-; LIBOMP-IR-LABEL: polly.par.setup:
-; LIBOMP-IR-NEXT:   %polly.par.LBPtr = alloca i64
-; LIBOMP-IR-NEXT:   %polly.par.UBPtr = alloca i64
-; LIBOMP-IR-NEXT:   %polly.par.lastIterPtr = alloca i32
-; LIBOMP-IR-NEXT:   %polly.par.StridePtr = alloca i64
-; LIBOMP-IR-NEXT:   %polly.par.userContext = bitcast i8* %polly.kmpc.shared
-; LIBOMP-IR-NEXT:   %polly.par.global_tid = load i32, i32* %polly.kmpc.global_tid
-; LIBOMP-IR-NEXT:   store i64 %polly.kmpc.lb, i64* %polly.par.LBPtr
-; LIBOMP-IR-NEXT:   store i64 %polly.kmpc.ub, i64* %polly.par.UBPtr
-; LIBOMP-IR-NEXT:   store i32 0, i32* %polly.par.lastIterPtr
-; LIBOMP-IR-NEXT:   store i64 %polly.kmpc.inc, i64* %polly.par.StridePtr
-; LIBOMP-IR-NEXT:   %polly.indvar.UBAdjusted = add i64 %polly.kmpc.ub, -1
-; LIBOMP-IR-NEXT:   call void @__kmpc_for_static_init_{{[4|8]}}(%struct.ident_t* @.loc.dummy{{[.0-9]*}}, i32 %polly.par.global_tid, i32 33, i32* %polly.par.lastIterPtr, i64* %polly.par.LBPtr, i64* %polly.par.UBPtr, i64* %polly.par.StridePtr, i64 1, i64 43)
-; LIBOMP-IR-NEXT:   %polly.indvar.LB = load i64, i64* %polly.par.LBPtr
-; LIBOMP-IR-NEXT:   %polly.indvar.UB = load i64, i64* %polly.par.UBPtr
-; LIBOMP-IR-NEXT:   %polly.adjustedUBOutOfBounds = icmp slt i64 %polly.indvar.UB, %polly.indvar.UBAdjusted
-; LIBOMP-IR-NEXT:   %{{[0-9]+}} = select i1 %polly.adjustedUBOutOfBounds, i64 %polly.indvar.UB, i64 %polly.indvar.UBAdjusted
-; LIBOMP-IR-NEXT:   store i64 %{{[0-9]+}}, i64* %polly.par.UBPtr
-; LIBOMP-IR-NEXT:   %polly.hasIteration = icmp sle i64 %polly.indvar.LB, %{{[0-9]+}}
-; LIBOMP-IR:   br i1 %polly.hasIteration, label %polly.par.loadIVBounds, label %polly.par.exit
-
-; LIBOMP-IR-LABEL: polly.par.exit:
-; LIBOMP-IR-NEXT:   call void @__kmpc_for_static_fini(%struct.ident_t* @.loc.dummy, i32 %polly.par.global_tid)
-; LIBOMP-IR-NEXT:   ret void
-
-; LIBOMP-IR-LABEL: polly.par.checkNext:
-; LIBOMP-IR-NEXT:   br label %polly.par.exit
-
-; LIBOMP-IR-LABEL: polly.par.loadIVBounds:
-; LIBOMP-IR-NEXT:   br label %polly.loop_preheader
-
-; LIBOMP-IR-LABEL: polly.loop_exit:
-; LIBOMP-IR-NEXT:   br label %polly.par.checkNext
-
-; LIBOMP-IR-LABEL: polly.loop_header:
-; LIBOMP-IR-NEXT:   %polly.indvar = phi i64 [ %polly.indvar.LB, %polly.loop_preheader ], [ %polly.indvar_next, %polly.stmt.S ]
-; LIBOMP-IR-NEXT:   br label %polly.stmt.S
-
-; LIBOMP-IR-LABEL: polly.stmt.S:
-; LIBOMP-IR-NEXT:   %[[gep:[._a-zA-Z0-9]*]] = getelementptr [1024 x float], [1024 x float]* {{.*}}, i64 0, i64 %polly.indvar
-; LIBOMP-IR-NEXT:   store float 1.000000e+00, float* %[[gep]]
-; LIBOMP-IR-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, %polly.kmpc.inc
-; LIBOMP-IR-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, %{{[0-9]+}}
-; LIBOMP-IR-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
-
-; LIBOMP-IR-LABEL: polly.loop_preheader:
-; LIBOMP-IR-NEXT:   br label %polly.loop_header
-
-; LIBOMP-IR: attributes #1 = { "polly.skip.fn" }
+; LIBOMP-IR-STATIC-CHUNKED: %struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: single_parallel_loop()
+; LIBOMP-IR-STATIC-CHUNKED-NEXT: entry
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.userContext = alloca
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.parallel.for:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.userContext1 = bitcast {}* %polly.par.userContext to i8*
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @.loc.dummy, i32 4, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i64, i64, i64, i8*)* @single_parallel_loop_polly_subfn to void (i32*, i32*, ...)*), i64 0, i64 1024, i64 1, i8* %polly.par.userContext1)
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br label %polly.exiting
+
+; LIBOMP-IR-STATIC-CHUNKED: define internal void @single_parallel_loop_polly_subfn(i32* %polly.kmpc.global_tid, i32* %polly.kmpc.bound_tid, i64 %polly.kmpc.lb, i64 %polly.kmpc.ub, i64 %polly.kmpc.inc, i8* %polly.kmpc.shared)
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.par.setup:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.LBPtr = alloca i64
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.UBPtr = alloca i64
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.lastIterPtr = alloca i32
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.StridePtr = alloca i64
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.userContext = bitcast i8* %polly.kmpc.shared
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.par.global_tid = load i32, i32* %polly.kmpc.global_tid
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.kmpc.lb, i64* %polly.par.LBPtr
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.kmpc.ub, i64* %polly.par.UBPtr
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i32 0, i32* %polly.par.lastIterPtr
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.kmpc.inc, i64* %polly.par.StridePtr
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.UBAdjusted = add i64 %polly.kmpc.ub, -1
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.indvar.UBAdjusted, i64* %polly.par.UBPtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   call void @__kmpc_for_static_init_{{[4|8]}}(%struct.ident_t* @.loc.dummy{{[.0-9]*}}, i32 %polly.par.global_tid, i32 33, i32* %polly.par.lastIterPtr, i64* %polly.par.LBPtr, i64* %polly.par.UBPtr, i64* %polly.par.StridePtr, i64 1, i64 43)
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.kmpc.stride = load i64, i64* %polly.par.StridePtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.LB = load i64, i64* %polly.par.LBPtr
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.UB.temp = load i64, i64* %polly.par.UBPtr
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.UB.inRange = icmp sle i64 %polly.indvar.UB.temp, %polly.indvar.UBAdjusted
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.UB = select i1 %polly.indvar.UB.inRange, i64 %polly.indvar.UB.temp, i64 %polly.indvar.UBAdjusted
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.indvar.UB, i64* %polly.par.UBPtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.hasIteration = icmp sle i64 %polly.indvar.LB, %polly.indvar.UB
+; LIBOMP-IR-STATIC-CHUNKED:   br i1 %polly.hasIteration, label %polly.par.loadIVBounds, label %polly.par.exit
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.par.exit:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   call void @__kmpc_for_static_fini(%struct.ident_t* @.loc.dummy, i32 %polly.par.global_tid)
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   ret void
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.par.checkNext:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.nextLB = add i64 %polly.indvar.LB.entry, %polly.kmpc.stride
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %{{[0-9]+}} = add i64 %polly.indvar.UB.entry, %polly.kmpc.stride
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.nextUB.outOfBounds = icmp sgt i64 %{{[0-9]+}}, %polly.indvar.UBAdjusted
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.nextUB = select i1 %polly.indvar.nextUB.outOfBounds, i64 %polly.indvar.UBAdjusted, i64 %{{[0-9]+}}
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.indvar.nextLB, i64* %polly.par.LBPtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store i64 %polly.indvar.nextUB, i64* %polly.par.UBPtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.hasWork = icmp sle i64 %polly.indvar.nextLB, %polly.indvar.UBAdjusted
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br i1 %polly.hasWork, label %polly.par.loadIVBounds, label %polly.par.exit
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.par.loadIVBounds:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.LB.entry = load i64, i64* %polly.par.LBPtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar.UB.entry = load i64, i64* %polly.par.UBPtr, align 8
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br label %polly.loop_preheader
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.loop_exit:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br label %polly.par.checkNext
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.loop_header:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar = phi i64 [ %polly.indvar.LB.entry, %polly.loop_preheader ], [ %polly.indvar_next, %polly.stmt.S ]
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br label %polly.stmt.S
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.stmt.S:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %[[gep:[._a-zA-Z0-9]*]] = getelementptr [1024 x float], [1024 x float]* {{.*}}, i64 0, i64 %polly.indvar
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   store float 1.000000e+00, float* %[[gep]]
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, %polly.indvar.UB.entry
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
+
+; LIBOMP-IR-STATIC-CHUNKED-LABEL: polly.loop_preheader:
+; LIBOMP-IR-STATIC-CHUNKED-NEXT:   br label %polly.loop_header
+
+; LIBOMP-IR-STATIC-CHUNKED: attributes #1 = { "polly.skip.fn" }
+
+; LIBOMP-IR-STATIC: define internal void @single_parallel_loop_polly_subfn(i32* %polly.kmpc.global_tid, i32* %polly.kmpc.bound_tid, i64 %polly.kmpc.lb, i64 %polly.kmpc.ub, i64 %polly.kmpc.inc, i8* %polly.kmpc.shared)
+; LIBOMP-IR-STATIC-LABEL: polly.par.setup:
+; LIBOMP-IR-STATIC:   call void @__kmpc_for_static_init_{{[4|8]}}(%struct.ident_t* @.loc.dummy{{[.0-9]*}}, i32 %polly.par.global_tid, i32 34, i32* %polly.par.lastIterPtr, i64* %polly.par.LBPtr, i64* %polly.par.UBPtr, i64* %polly.par.StridePtr, i64 1, i64 1)
+; LIBOMP-IR-STATIC:   br i1 %polly.hasIteration, label %polly.par.loadIVBounds, label %polly.par.exit
+
+; LIBOMP-IR-STATIC-LABEL: polly.par.exit:
+; LIBOMP-IR-STATIC-NEXT:   call void @__kmpc_for_static_fini(%struct.ident_t* @.loc.dummy, i32 %polly.par.global_tid)
+; LIBOMP-IR-STATIC-NEXT:   ret void
+
+; LIBOMP-IR-STATIC-LABEL: polly.par.checkNext:
+; LIBOMP-IR-STATIC-NEXT:   br label %polly.par.exit
+
+; LIBOMP-IR-STATIC-LABEL: polly.par.loadIVBounds:
+; LIBOMP-IR-STATIC-NEXT:   br label %polly.loop_preheader
+
+; LIBOMP-IR-STATIC-LABEL: polly.loop_exit:
+; LIBOMP-IR-STATIC-NEXT:   br label %polly.par.checkNext
+
+; LIBOMP-IR-STATIC-LABEL: polly.loop_header:
+; LIBOMP-IR-STATIC-NEXT:   %polly.indvar = phi i64 [ %polly.indvar.LB, %polly.loop_preheader ], [ %polly.indvar_next, %polly.stmt.S ]
+; LIBOMP-IR-STATIC-NEXT:   br label %polly.stmt.S
+
+; LIBOMP-IR-STATIC-LABEL: polly.stmt.S:
+; LIBOMP-IR-STATIC-NEXT:   %[[gep:[._a-zA-Z0-9]*]] = getelementptr [1024 x float], [1024 x float]* {{.*}}, i64 0, i64 %polly.indvar
+; LIBOMP-IR-STATIC-NEXT:   store float 1.000000e+00, float* %[[gep]]
+; LIBOMP-IR-STATIC-NEXT:   %polly.indvar_next = add nsw i64 %polly.indvar, 1
+; LIBOMP-IR-STATIC-NEXT:   %polly.loop_cond = icmp sle i64 %polly.indvar_next, %polly.indvar.UB
+; LIBOMP-IR-STATIC-NEXT:   br i1 %polly.loop_cond, label %polly.loop_header, label %polly.loop_exit
+
+; LIBOMP-IR-STATIC-LABEL: polly.loop_preheader:
+; LIBOMP-IR-STATIC-NEXT:   br label %polly.loop_header
 
 ; LIBOMP-IR-DYNAMIC:   call void @__kmpc_dispatch_init_{{[4|8]}}(%struct.ident_t* @.loc.dummy, i32 %polly.par.global_tid, i32 35, i64 %polly.kmpc.lb, i64 %polly.indvar.UBAdjusted, i64 %polly.kmpc.inc, i64 1)
 ; LIBOMP-IR-DYNAMIC-NEXT:   %{{[0-9]+}} = call i32 @__kmpc_dispatch_next_{{[4|8]}}(%struct.ident_t* @.loc.dummy, i32 %polly.par.global_tid, i32* %polly.par.lastIterPtr, i64* %polly.par.LBPtr, i64* %polly.par.UBPtr, i64* %polly.par.StridePtr)