From 9575c48b8959dae3c3e39e0227435ae6ebd71443 Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Mon, 15 Mar 2021 23:27:10 -0700 Subject: [PATCH 0001/1206] [AArch64][GlobalISel] Fix crash on lowering <1 x half> types. --- .../Target/AArch64/GISel/AArch64CallLowering.cpp | 11 ++++++++--- .../AArch64/GlobalISel/call-lowering-vectors.ll | 13 +++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index d16e2fd13475..dbe5f5635048 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -357,9 +357,14 @@ bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, return false; } } else { - // A scalar extend. - CurVReg = - MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}).getReg(0); + // If the split EVT was a <1 x T> vector, and NewVT is T, then we + // don't have to do anything since we don't distinguish between the + // two. + if (NewLLT != MRI.getType(CurVReg)) { + // A scalar extend. + CurVReg = MIRBuilder.buildInstr(ExtendOp, {NewLLT}, {CurVReg}) + .getReg(0); + } } } } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll index ee73e58798c7..f34f0981c211 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-lowering-vectors.ll @@ -31,3 +31,16 @@ define i24 @test_v3i8(<3 x i8> %a) { ret i24 %res } + +define <1 x half> @test_v1s16(<1 x float> %x) { + ; CHECK-LABEL: name: test_v1s16 + ; CHECK: bb.1 (%ir-block.0): + ; CHECK: liveins: $d0 + ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>) + ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV]](s32) + ; CHECK: $h0 = COPY [[FPTRUNC]](s16) + ; CHECK: RET_ReallyLR implicit $h0 + %tmp = fptrunc <1 x float> %x to <1 x half> + ret <1 x half> %tmp +} -- GitLab From 678241795c957b18bc473045e48abe3f2a61ff5c Mon Sep 17 00:00:00 2001 From: Jim Lin Date: Tue, 16 Mar 2021 14:57:45 +0800 Subject: [PATCH 0002/1206] [RISCV] Don't emit #undef BUILTIN from RISCVVEmitter.cpp In BuiltinsRISCV.def, other extension 's intrinsics need to be defined by using macro BUILTIN. So, it shouldn't undefine macro BUILTIN in the end of declaration for V intrinsics. Reviewed By: craig.topper Differential Revision: https://reviews.llvm.org/D98682 --- clang/include/clang/Basic/BuiltinsRISCV.def | 2 ++ clang/utils/TableGen/RISCVVEmitter.cpp | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/include/clang/Basic/BuiltinsRISCV.def b/clang/include/clang/Basic/BuiltinsRISCV.def index e76c853787c9..c91b3d1b1f5c 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.def +++ b/clang/include/clang/Basic/BuiltinsRISCV.def @@ -17,3 +17,5 @@ #include "clang/Basic/riscv_vector_builtins.inc" +#undef BUILTIN +#undef TARGET_BUILTIN diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp index ba96396c780d..f2b555a8b05c 100644 --- a/clang/utils/TableGen/RISCVVEmitter.cpp +++ b/clang/utils/TableGen/RISCVVEmitter.cpp @@ -881,7 +881,6 @@ void RVVEmitter::createBuiltins(raw_ostream &OS) { else OS << "\"\")\n"; } - OS << "\n#undef BUILTIN\n"; OS << "#undef RISCVV_BUILTIN\n"; } -- GitLab From fd7eee64c570e5e14e511045c64d4d8cf98dde25 Mon Sep 17 00:00:00 2001 From: Lorenzo Chelini Date: Tue, 16 Mar 2021 06:46:10 +0000 Subject: [PATCH 0003/1206] scf::ForOp: Fold away iterator arguments with no use and for which the corresponding input is yielded Enhance 'ForOpIterArgsFolder' to remove unused iteration arguments in a scf::ForOp. If the block argument corresponding to the given iterator has no use and the yielded value equals the input, we fold it away. Reviewed By: nicolasvasilache Differential Revision: https://reviews.llvm.org/D98503 --- mlir/lib/Dialect/SCF/SCF.cpp | 22 ++++++++++++++++------ mlir/test/Dialect/SCF/canonicalize.mlir | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/mlir/lib/Dialect/SCF/SCF.cpp b/mlir/lib/Dialect/SCF/SCF.cpp index 9c0df1b47c35..c66d0ea497a3 100644 --- a/mlir/lib/Dialect/SCF/SCF.cpp +++ b/mlir/lib/Dialect/SCF/SCF.cpp @@ -408,9 +408,14 @@ static void replaceOpWithRegion(PatternRewriter &rewriter, Operation *op, } namespace { -// Fold away ForOp iter arguments that are also yielded by the op. -// These arguments must be defined outside of the ForOp region and can just be -// forwarded after simplifying the op inits, yields and returns. +// Fold away ForOp iter arguments when: +// 1) The op yields the iter arguments. +// 2) The iter arguments have no use and the corresponding outer region +// iterators (inputs) are yielded. +// +// These arguments must be defined outside of +// the ForOp region and can just be forwarded after simplifying the op inits, +// yields and returns. // // The implementation uses `mergeBlockBefore` to steal the content of the // original ForOp and avoid cloning. @@ -441,8 +446,13 @@ struct ForOpIterArgsFolder : public OpRewritePattern { forOp.getRegionIterArgs(), // iter inside region yieldOp.getOperands() // iter yield )) { - // Forwarded is `true` when the region `iter` argument is yielded. - bool forwarded = (std::get<1>(it) == std::get<2>(it)); + // Forwarded is `true` when: + // 1) The region `iter` argument is yielded. + // 2) The region `iter` argument has zero use, and the corresponding iter + // operand (input) is yielded. + bool forwarded = + ((std::get<1>(it) == std::get<2>(it)) || + (std::get<1>(it).use_empty() && std::get<0>(it) == std::get<2>(it))); keepMask.push_back(!forwarded); canonicalize |= forwarded; if (forwarded) { @@ -483,7 +493,7 @@ struct ForOpIterArgsFolder : public OpRewritePattern { "unexpected argument size mismatch"); // No results case: the scf::ForOp builder already created a zero - // reult terminator. Merge before this terminator and just get rid of the + // result terminator. Merge before this terminator and just get rid of the // original terminator that has been merged in. if (newIterArgs.empty()) { auto newYieldOp = cast(newBlock.getTerminator()); diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir index 8f76926bdff0..6f75532b9bc7 100644 --- a/mlir/test/Dialect/SCF/canonicalize.mlir +++ b/mlir/test/Dialect/SCF/canonicalize.mlir @@ -335,6 +335,7 @@ func @remove_empty_parallel_loop(%lb: index, %ub: index, %s: index) { } // ----- + func private @process(%0 : memref<128x128xf32>) func private @process_tensor(%0 : tensor<128x128xf32>) -> memref<128x128xf32> @@ -382,3 +383,22 @@ func @last_value(%t0: tensor<128x128xf32>, %t1: tensor<128x128xf32>, // CHECK-NEXT: return %[[R0]], %[[R1]], %[[FOR_RES]] : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> return %0#0, %0#1, %0#2 : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> } + +// ----- + +// CHECK-LABEL: fold_away_iter_with_no_use_and_yielded_input +// CHECK-SAME: %[[A0:[0-9a-z]*]]: i32 +func @fold_away_iter_with_no_use_and_yielded_input(%arg0 : i32, + %ub : index, %lb : index, %step : index) -> (i32, i32) { + // CHECK-NEXT: %[[C32:.*]] = constant 32 : i32 + %cst = constant 32 : i32 + // CHECK-NEXT: %[[FOR_RES:.*]] = scf.for {{.*}} iter_args({{.*}} = %[[A0]]) -> (i32) { + %0:2 = scf.for %arg1 = %lb to %ub step %step iter_args(%arg2 = %arg0, %arg3 = %cst) + -> (i32, i32) { + %1 = addi %arg2, %cst : i32 + scf.yield %1, %cst : i32, i32 + } + + // CHECK: return %[[FOR_RES]], %[[C32]] : i32, i32 + return %0#0, %0#1 : i32, i32 +} -- GitLab From 3c03635d530066028aa3e041bc9e68743281e56b Mon Sep 17 00:00:00 2001 From: Caroline Concatto Date: Tue, 19 Jan 2021 11:30:50 +0000 Subject: [PATCH 0004/1206] [SVE][LoopVectorize] Add support for scalable vectorization of loops with vector reverse This patch adds support for reverse loop vectorization. It is possible to vectorize the following loop: ``` for (int i = n-1; i >= 0; --i) a[i] = b[i] + 1.0; ``` with fixed or scalable vector. The loop-vectorizer will use 'reverse' on the loads/stores to make sure the lanes themselves are also handled in the right order. This patch adds support for scalable vector on IRBuilder interface to create a reverse vector. The IR function CreateVectorReverse lowers to experimental.vector.reverse for scalable vector and keedp the original behavior for fixed vector using shuffle reverse. Differential Revision: https://reviews.llvm.org/D95363 --- llvm/include/llvm/IR/IRBuilder.h | 3 + llvm/lib/IR/IRBuilder.cpp | 16 +++ .../Transforms/Vectorize/LoopVectorize.cpp | 26 ++--- .../AArch64/sve-vector-reverse-mask4.ll | 68 +++++++++++ .../AArch64/sve-vector-reverse.ll | 108 ++++++++++++++++++ .../AArch64/vector-reverse-mask4.ll | 65 +++++++++++ .../LoopVectorize/AArch64/vector-reverse.ll | 91 +++++++++++++++ 7 files changed, 363 insertions(+), 14 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index 477d4815bc9d..f2da98a98b70 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2504,6 +2504,9 @@ public: /// address space before call and casted back to Ptr type after call. Value *CreateStripInvariantGroup(Value *Ptr); + /// Return a vector value that contains the vector V reversed + Value *CreateVectorReverse(Value *V, const Twine &Name = ""); + /// Return a vector value that contains \arg V broadcasted to \p /// NumElts elements. Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = ""); diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp index 81fa2098c485..d4292b3cfc1b 100644 --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -992,6 +992,22 @@ Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) { return Fn; } +Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) { + auto *Ty = cast(V->getType()); + if (isa(Ty)) { + Module *M = BB->getParent()->getParent(); + Function *F = Intrinsic::getDeclaration( + M, Intrinsic::experimental_vector_reverse, Ty); + return Insert(CallInst::Create(F, V), Name); + } + // Keep the original behaviour for fixed vector + SmallVector ShuffleMask; + int NumElts = Ty->getElementCount().getKnownMinValue(); + for (int i = 0; i < NumElts; ++i) + ShuffleMask.push_back(NumElts - i - 1); + return CreateShuffleVector(V, ShuffleMask, Name); +} + Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name) { auto EC = ElementCount::getFixed(NumElts); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 1fb99e9e7857..c92b00078c7e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2568,12 +2568,7 @@ void InnerLoopVectorizer::packScalarIntoVectorValue(VPValue *Def, Value *InnerLoopVectorizer::reverseVector(Value *Vec) { assert(Vec->getType()->isVectorTy() && "Invalid type"); - assert(!VF.isScalable() && "Cannot reverse scalable vectors"); - SmallVector ShuffleMask; - for (unsigned i = 0; i < VF.getKnownMinValue(); ++i) - ShuffleMask.push_back(VF.getKnownMinValue() - i - 1); - - return Builder.CreateShuffleVector(Vec, ShuffleMask, "reverse"); + return Builder.CreateVectorReverse(Vec, "reverse"); } // Return whether we allow using masked interleave-groups (for dealing with @@ -2854,18 +2849,21 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction( bool InBounds = false; if (auto *gep = dyn_cast(Ptr->stripPointerCasts())) InBounds = gep->isInBounds(); - if (Reverse) { - assert(!VF.isScalable() && - "Reversing vectors is not yet supported for scalable vectors."); - // If the address is consecutive but reversed, then the // wide store needs to start at the last vector element. - PartPtr = cast(Builder.CreateGEP( - ScalarDataTy, Ptr, Builder.getInt32(-Part * VF.getKnownMinValue()))); + // RunTimeVF = VScale * VF.getKnownMinValue() + // For fixed-width VScale is 1, then RunTimeVF = VF.getKnownMinValue() + Value *RunTimeVF = getRuntimeVF(Builder, Builder.getInt32Ty(), VF); + // NumElt = -Part * RunTimeVF + Value *NumElt = Builder.CreateMul(Builder.getInt32(-Part), RunTimeVF); + // LastLane = 1 - RunTimeVF + Value *LastLane = Builder.CreateSub(Builder.getInt32(1), RunTimeVF); + PartPtr = + cast(Builder.CreateGEP(ScalarDataTy, Ptr, NumElt)); PartPtr->setIsInBounds(InBounds); - PartPtr = cast(Builder.CreateGEP( - ScalarDataTy, PartPtr, Builder.getInt32(1 - VF.getKnownMinValue()))); + PartPtr = cast( + Builder.CreateGEP(ScalarDataTy, PartPtr, LastLane)); PartPtr->setIsInBounds(InBounds); if (isMaskRequired) // Reverse of a null all-one mask is a null mask. BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll new file mode 100644 index 000000000000..d803f6b75ed9 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll @@ -0,0 +1,68 @@ +; This is the loop in c++ being vectorize in this file with +; experimental.vector.reverse + +;#pragma clang loop vectorize_width(4, scalable) +; for (long int i = N - 1; i >= 0; i--) +; { +; if (cond[i]) +; a[i] += 1; +; } + +; The test checks if the mask is being correctly created, reverted and used + +; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64-unknown-linux-gnu" + +define void @vector_reverse_mask_nxv4i1(double* %a, double* %cond, i64 %N) #0 { +; CHECK-LABEL: vector.body: +; CHECK: %[[REVERSE6:.*]] = call @llvm.experimental.vector.reverse.nxv4i1( %{{.*}}) +; CHECK: %[[WIDEMSKLOAD:.*]] = call @llvm.masked.load.nxv4f64.p0nxv4f64(* nonnull %{{.*}}, i32 8, %[[REVERSE6]], poison) +; CHECK-NEXT: %[[REVERSE7:.*]] = call @llvm.experimental.vector.reverse.nxv4f64( %[[WIDEMSKLOAD]]) +; CHECK-NEXT: %[[FADD:.*]] = fadd %[[REVERSE7]] +; CHECK-NEXT: %[[REVERSE8:.*]] = call @llvm.experimental.vector.reverse.nxv4f64( %[[FADD]]) +; CHECK: %[[REVERSE9:.*]] = call @llvm.experimental.vector.reverse.nxv4i1( %{{.*}}) +; CHECK: call void @llvm.masked.store.nxv4f64.p0nxv4f64( %[[REVERSE8]], * %{{.*}}, i32 8, %[[REVERSE9]] + +entry: + %cmp7 = icmp sgt i64 %N, 0 + br i1 %cmp7, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond.cleanup, %entry + ret void + +for.body: ; preds = %for.body, %entry + %i.08.in = phi i64 [ %i.08, %for.inc ], [ %N, %entry ] + %i.08 = add nsw i64 %i.08.in, -1 + %arrayidx = getelementptr inbounds double, double* %cond, i64 %i.08 + %0 = load double, double* %arrayidx, align 8 + %tobool = fcmp une double %0, 0.000000e+00 + br i1 %tobool, label %if.then, label %for.inc + +if.then: ; preds = %for.body + %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08 + %1 = load double, double* %arrayidx1, align 8 + %add = fadd double %1, 1.000000e+00 + store double %add, double* %arrayidx1, align 8 + br label %for.inc + +for.inc: ; preds = %for.body, %if.then + %cmp = icmp sgt i64 %i.08.in, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0 +} + +attributes #0 = {"target-cpu"="generic" "target-features"="+neon,+sve"} + + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 4} +!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} +!4 = !{!"llvm.loop.vectorize.enable", i1 true} diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll new file mode 100644 index 000000000000..aef5efe030f5 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll @@ -0,0 +1,108 @@ +; This is the loop in c++ being vectorize in this file with +;experimental.vector.reverse +; #pragma clang loop vectorize_width(8, scalable) +; for (int i = N-1; i >= 0; --i) +; a[i] = b[i] + 1.0; + +; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + +define void @vector_reverse_f64(i64 %N, double* %a, double* %b) #0{ +; CHECK-LABEL: @vector_reverse_f64 +; CHECK-LABEL: vector.body: +; CHECK: %[[ADD:.*]] = add i64 %{{.*}}, %N +; CHECK-NEXT: %[[GEP:.*]] = getelementptr inbounds double, double* %b, i64 %[[ADD]] +; CHECK-NEXT: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: %[[MUL:.*]] = mul i32 %[[VSCALE]], -8 +; CHECK-NEXT: %[[OR:.*]] = or i32 %[[MUL]], 1 +; CHECK-NEXT: %[[SEXT:.*]] = sext i32 %[[OR]] to i64 +; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds double, double* %[[GEP]], i64 %[[SEXT]] +; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP1]] to * +; CHECK-NEXT: %[[WIDE:.*]] = load , * %[[CAST]], align 8 +; CHECK-NEXT: %[[REVERSE:.*]] = call @llvm.experimental.vector.reverse.nxv8f64( %[[WIDE]]) +; CHECK-NEXT: %[[FADD:.*]] = fadd %[[REVERSE]], shufflevector +; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds double, double* %a, i64 %[[ADD]] +; CHECK-NEXT: %[[REVERSE6:.*]] = call @llvm.experimental.vector.reverse.nxv8f64( %[[FADD]]) +; CHECK-NEXT: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], -8 +; CHECK-NEXT: %[[OR1:.*]] = or i32 %[[MUL1]], 1 +; CHECK-NEXT: %[[SEXT1:.*]] = sext i32 %[[OR1]] to i64 +; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds double, double* %[[GEP2]], i64 %[[SEXT1]] +; CHECK-NEXT: %[[CAST1:.*]] = bitcast double* %[[GEP3]] to * +; CHECK-NEXT: store %[[REVERSE6]], * %[[CAST1]], align 8 + +entry: + %cmp7 = icmp sgt i64 %N, 0 + br i1 %cmp7, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.body + ret void + +for.body: ; preds = %entry, %for.body + %i.08.in = phi i64 [ %i.08, %for.body ], [ %N, %entry ] + %i.08 = add nsw i64 %i.08.in, -1 + %arrayidx = getelementptr inbounds double, double* %b, i64 %i.08 + %0 = load double, double* %arrayidx, align 8 + %add = fadd double %0, 1.000000e+00 + %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08 + store double %add, double* %arrayidx1, align 8 + %cmp = icmp sgt i64 %i.08.in, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0 +} + + +define void @vector_reverse_i64(i64 %N, i64* %a, i64* %b) #0 { +; CHECK-LABEL: vector_reverse_i64 +; CHECK-LABEL: vector.body: +; CHECK: %[[ADD:.*]] = add i64 %{{.*}}, %N +; CHECK-NEXT: %[[GEP:.*]] = getelementptr inbounds i64, i64* %b, i64 %[[ADD]] +; CHECK-NEXT: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: %[[MUL:.*]] = mul i32 %[[VSCALE]], -8 +; CHECK-NEXT: %[[OR:.*]] = or i32 %[[MUL]], 1 +; CHECK-NEXT: %[[SEXT:.*]] = sext i32 %[[OR]] to i64 +; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds i64, i64* %[[GEP]], i64 %[[SEXT]] +; CHECK-NEXT: %[[CAST:.*]] = bitcast i64* %[[GEP1]] to * +; CHECK-NEXT: %[[WIDE:.*]] = load , * %[[CAST]], align 8 +; CHECK-NEXT: %[[REVERSE:.*]] = call @llvm.experimental.vector.reverse.nxv8i64( %[[WIDE]]) +; CHECK-NEXT: %[[ADD1:.*]] = add %[[REVERSE]] +; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds i64, i64* %a, i64 %[[ADD]] +; CHECK-NEXT: %[[REVERSE6]] = call @llvm.experimental.vector.reverse.nxv8i64( %[[ADD1]]) +; CHECK-NEXT: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: %[[MUL1:.*]] = mul i32 %[[VSCALE]], -8 +; CHECK-NEXT: %[[OR1:.*]] = or i32 %[[MUL1]], 1 +; CHECK-NEXT: %[[SEXT1:.*]] = sext i32 %[[OR1]] to i64 +; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds i64, i64* %[[GEP2]], i64 %[[SEXT1]] +; CHECK-NEXT: %[[CAST1:.*]] = bitcast i64* %[[GEP3]] to * +; CHECK-NEXT: store %[[REVERSE6]], * %[[CAST1]], align 8 + +entry: + %cmp8 = icmp sgt i64 %N, 0 + br i1 %cmp8, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.body + ret void + +for.body: ; preds = %entry, %for.body + %i.09.in = phi i64 [ %i.09, %for.body ], [ %N, %entry ] + %i.09 = add nsw i64 %i.09.in, -1 + %arrayidx = getelementptr inbounds i64, i64* %b, i64 %i.09 + %0 = load i64, i64* %arrayidx, align 8 + %add = add i64 %0, 1 + %arrayidx2 = getelementptr inbounds i64, i64* %a, i64 %i.09 + store i64 %add, i64* %arrayidx2, align 8 + %cmp = icmp sgt i64 %i.09.in, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0 +} + +attributes #0 = { "target-cpu"="generic" "target-features"="+neon,+sve" } + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 8} +!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true} +!4 = !{!"llvm.loop.vectorize.enable", i1 true} + diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll new file mode 100644 index 000000000000..22de7c1e1ca8 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll @@ -0,0 +1,65 @@ +; This is the loop in c++ being vectorize in this file with +; shuffle reverse + +;#pragma clang loop vectorize_width(4, fixed) +; for (long int i = N - 1; i >= 0; i--) +; { +; if (cond[i]) +; a[i] += 1; +; } + +; The test checks if the mask is being correctly created, reverted and used + +; RUN: opt -loop-vectorize -dce -instcombine -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + + +target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" +target triple = "aarch64-unknown-linux-gnu" + +define void @vector_reverse_mask_v4i1(double* %a, double* %cond, i64 %N) #0 { +; CHECK-LABEL: vector.body: +; CHECK: %[[REVERSE6:.*]] = shufflevector <4 x i1> %{{.*}}, <4 x i1> poison, <4 x i32> +; CHECK: %[[WIDEMSKLOAD:.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull %{{.*}}, i32 8, <4 x i1> %[[REVERSE6]], <4 x double> poison) +; CHECK-NEXT: %[[FADD:.*]] = fadd <4 x double> %[[WIDEMSKLOAD]] +; CHECK: call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> %[[FADD]], <4 x double>* %{{.*}}, i32 8, <4 x i1> %[[REVERSE6]]) + +entry: + %cmp7 = icmp sgt i64 %N, 0 + br i1 %cmp7, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond.cleanup, %entry + ret void + +for.body: ; preds = %for.body, %entry + %i.08.in = phi i64 [ %i.08, %for.inc ], [ %N, %entry ] + %i.08 = add nsw i64 %i.08.in, -1 + %arrayidx = getelementptr inbounds double, double* %cond, i64 %i.08 + %0 = load double, double* %arrayidx, align 8 + %tobool = fcmp une double %0, 0.000000e+00 + br i1 %tobool, label %if.then, label %for.inc + +if.then: ; preds = %for.body + %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08 + %1 = load double, double* %arrayidx1, align 8 + %add = fadd double %1, 1.000000e+00 + store double %add, double* %arrayidx1, align 8 + br label %for.inc + +for.inc: ; preds = %for.body, %if.then + %cmp = icmp sgt i64 %i.08.in, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0 +} + +attributes #0 = {"target-cpu"="generic" "target-features"="+neon,+sve"} + + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 4} +!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +!4 = !{!"llvm.loop.vectorize.enable", i1 true} diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll new file mode 100644 index 000000000000..ae3aad1e75a8 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll @@ -0,0 +1,91 @@ +; Test VLA for reverse with fixed size vector +; This is the loop in c++ being vectorize in this file with +; shuffle reverse +; #pragma clang loop vectorize_width(8, fixed) +; for (int i = N-1; i >= 0; --i) +; a[i] = b[i] + 1.0; + +; RUN: opt -loop-vectorize -dce -mtriple aarch64-linux-gnu -S < %s 2>%t | FileCheck %s + +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.$ +; WARN-NOT: warning + +define void @vector_reverse_f64(i64 %N, double* %a, double* %b) #0 { +; CHECK-LABEL: vector_reverse_f64 +; CHECK-LABEL: vector.body +; CHECK: %[[GEP:.*]] = getelementptr inbounds double, double* %{{.*}}, i32 0 +; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds double, double* %[[GEP]], i32 -7 +; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP1]] to <8 x double>* +; CHECK-NEXT: %[[WIDE:.*]] = load <8 x double>, <8 x double>* %[[CAST]], align 8 +; CHECK-NEXT: %[[REVERSE:.*]] = shufflevector <8 x double> %[[WIDE]], <8 x double> poison, <8 x i32> +; CHECK-NEXT: %[[FADD:.*]] = fadd <8 x double> %[[REVERSE]] +; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds double, double* {{.*}}, i64 {{.*}} +; CHECK-NEXT: %[[REVERSE6:.*]] = shufflevector <8 x double> %[[FADD]], <8 x double> poison, <8 x i32> +; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds double, double* %[[GEP2]], i32 0 +; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds double, double* %[[GEP3]], i32 -7 +; CHECK-NEXT: %[[CAST:.*]] = bitcast double* %[[GEP4]] to <8 x double>* +; CHECK-NEXT: store <8 x double> %[[REVERSE6]], <8 x double>* %[[CAST]], align 8 + +entry: + %cmp7 = icmp sgt i64 %N, 0 + br i1 %cmp7, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond.cleanup, %entry + ret void + +for.body: ; preds = %entry, %for.body + %i.08.in = phi i64 [ %i.08, %for.body ], [ %N, %entry ] + %i.08 = add nsw i64 %i.08.in, -1 + %arrayidx = getelementptr inbounds double, double* %b, i64 %i.08 + %0 = load double, double* %arrayidx, align 8 + %add = fadd double %0, 1.000000e+00 + %arrayidx1 = getelementptr inbounds double, double* %a, i64 %i.08 + store double %add, double* %arrayidx1, align 8 + %cmp = icmp sgt i64 %i.08.in, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0 +} + +define void @vector_reverse_i64(i64 %N, i64* %a, i64* %b) #0 { +; CHECK-LABEL: vector_reverse_i64 +; CHECK-LABEL: vector.body +; CHECK: %[[GEP:.*]] = getelementptr inbounds i64, i64* %{{.*}}, i32 0 +; CHECK-NEXT: %[[GEP1:.*]] = getelementptr inbounds i64, i64* %[[GEP]], i32 -7 +; CHECK-NEXT: %[[CAST:.*]] = bitcast i64* %[[GEP1]] to <8 x i64>* +; CHECK-NEXT: %[[WIDE:.*]] = load <8 x i64>, <8 x i64>* %[[CAST]], align 8 +; CHECK-NEXT: %[[REVERSE:.*]] = shufflevector <8 x i64> %[[WIDE]], <8 x i64> poison, <8 x i32> +; CHECK-NEXT: %[[FADD:.*]] = add <8 x i64> %[[REVERSE]] +; CHECK-NEXT: %[[GEP2:.*]] = getelementptr inbounds i64, i64* {{.*}}, i64 {{.*}} +; CHECK-NEXT: %[[REVERSE6:.*]] = shufflevector <8 x i64> %[[FADD]], <8 x i64> poison, <8 x i32> +; CHECK-NEXT: %[[GEP3:.*]] = getelementptr inbounds i64, i64* %[[GEP2]], i32 0 +; CHECK-NEXT: %[[GEP4:.*]] = getelementptr inbounds i64, i64* %[[GEP3]], i32 -7 +; CHECK-NEXT: %[[CAST1:.*]] = bitcast i64* %[[GEP4]] to <8 x i64>* +; CHECK-NEXT: store <8 x i64> %[[REVERSE6]], <8 x i64>* %[[CAST1]], align 8 + +entry: + %cmp8 = icmp sgt i64 %N, 0 + br i1 %cmp8, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond.cleanup, %entry + ret void + +for.body: ; preds = %entry, %for.body + %i.09.in = phi i64 [ %i.09, %for.body ], [ %N, %entry ] + %i.09 = add nsw i64 %i.09.in, -1 + %arrayidx = getelementptr inbounds i64, i64* %b, i64 %i.09 + %0 = load i64, i64* %arrayidx, align 8 + %add = add i64 %0, 1 + %arrayidx2 = getelementptr inbounds i64, i64* %a, i64 %i.09 + store i64 %add, i64* %arrayidx2, align 8 + %cmp = icmp sgt i64 %i.09.in, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !0 +} + +attributes #0 = { "target-cpu"="generic" "target-features"="+neon,+sve" } + +!0 = distinct !{!0, !1, !2, !3, !4} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.vectorize.width", i32 8} +!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +!4 = !{!"llvm.loop.vectorize.enable", i1 true} -- GitLab From 6e040a19dbb20b28ba97374f7eb50e1ff266b15e Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Tue, 16 Mar 2021 10:11:57 +0100 Subject: [PATCH 0005/1206] [NFC] Wisely nest dyn_cast in FunctionLoweringInfo Take advantage of the inheritance tree to avoid a few comparison. --- .../SelectionDAG/FunctionLoweringInfo.cpp | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index cc931df5c75c..85c6eca5775e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -192,10 +192,8 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, MF->getFrameInfo().CreateVariableSizedObject( Alignment <= StackAlign ? Align(1) : Alignment, AI); } - } - - // Look for inline asm that clobbers the SP register. - if (auto *Call = dyn_cast(&I)) { + } else if (auto *Call = dyn_cast(&I)) { + // Look for inline asm that clobbers the SP register. if (Call->isInlineAsm()) { Register SP = TLI->getStackPointerRegisterToSaveRestore(); const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); @@ -214,21 +212,20 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, } } } - } - - // Look for calls to the @llvm.va_start intrinsic. We can omit some - // prologue boilerplate for variadic functions that don't examine their - // arguments. - if (const auto *II = dyn_cast(&I)) { - if (II->getIntrinsicID() == Intrinsic::vastart) - MF->getFrameInfo().setHasVAStart(true); - } + // Look for calls to the @llvm.va_start intrinsic. We can omit some + // prologue boilerplate for variadic functions that don't examine their + // arguments. + if (const auto *II = dyn_cast(&I)) { + if (II->getIntrinsicID() == Intrinsic::vastart) + MF->getFrameInfo().setHasVAStart(true); + } - // If we have a musttail call in a variadic function, we need to ensure we - // forward implicit register parameters. - if (const auto *CI = dyn_cast(&I)) { - if (CI->isMustTailCall() && Fn->isVarArg()) - MF->getFrameInfo().setHasMustTailInVarArgFunc(true); + // If we have a musttail call in a variadic function, we need to ensure + // we forward implicit register parameters. + if (const auto *CI = dyn_cast(&I)) { + if (CI->isMustTailCall() && Fn->isVarArg()) + MF->getFrameInfo().setHasMustTailInVarArgFunc(true); + } } // Mark values used outside their block as exported, by allocating -- GitLab From 1d297f90649dd63187590548e20de0eced61750c Mon Sep 17 00:00:00 2001 From: David Zarzycki Date: Wed, 10 Mar 2021 10:19:15 -0500 Subject: [PATCH 0006/1206] [lit] Sort test start times based on prior test timing data Lit as it exists today has three hacks that allow users to run tests earlier: 1) An entire test suite can set the `is_early` boolean. 2) A very recently introduced "early_tests" feature. 3) The `--incremental` flag forces failing tests to run first. All of these approaches have problems. 1) The `is_early` feature was until very recently undocumented. Nevertheless it still lacks testing and is a imprecise way of optimizing test starting times. 2) The `early_tests` feature requires manual updates and doesn't scale. 3) `--incremental` is undocumented, untested, and it requires modifying the *source* file system by "touching" the file. This "touch" based approach is arguably a hack because it confuses editors (because it looks like the test was modified behind the back of the editor) and "touching" the test source file doesn't work if the test suite is read only from the perspective of `lit` (via advanced filesystem/build tricks). This patch attempts to simplify and address all of the above problems. This patch formalizes, documents, tests, and defaults lit to recording the execution time of tests and then reordering all tests during the next execution. By reordering the tests, high core count machines run faster, sometimes significantly so. This patch also always runs failing tests first, which is a positive user experience win for those that didn't know about the hidden `--incremental` flag. Finally, if users want, they can _optionally_ commit the test timing data (or a subset thereof) back to the repository to accelerate bots and first-time runs of the test suite. Reviewed By: jhenderson, yln Differential Revision: https://reviews.llvm.org/D98179 --- llvm/docs/CommandGuide/lit.rst | 27 ++++++----- llvm/test/Unit/lit.cfg.py | 3 -- llvm/utils/lit/lit/Test.py | 34 +++++++++----- llvm/utils/lit/lit/TestingConfig.py | 4 -- llvm/utils/lit/lit/cl_arguments.py | 12 ++--- llvm/utils/lit/lit/discovery.py | 5 ++ llvm/utils/lit/lit/main.py | 46 +++++++++++-------- .../tests/Inputs/reorder/.lit_test_times.txt | 3 ++ .../Inputs/{early-tests => reorder}/aaa.txt | 0 .../Inputs/{early-tests => reorder}/bbb.txt | 0 .../Inputs/{early-tests => reorder}/lit.cfg | 3 +- .../{early-tests => reorder}/subdir/ccc.txt | 0 llvm/utils/lit/tests/early-tests.py | 9 ---- llvm/utils/lit/tests/ignore-fail.py | 8 ++-- llvm/utils/lit/tests/reorder.py | 12 +++++ llvm/utils/lit/tests/shtest-shell.py | 2 + mlir/test/Unit/lit.cfg.py | 3 -- 17 files changed, 96 insertions(+), 75 deletions(-) create mode 100644 llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt rename llvm/utils/lit/tests/Inputs/{early-tests => reorder}/aaa.txt (100%) rename llvm/utils/lit/tests/Inputs/{early-tests => reorder}/bbb.txt (100%) rename llvm/utils/lit/tests/Inputs/{early-tests => reorder}/lit.cfg (67%) rename llvm/utils/lit/tests/Inputs/{early-tests => reorder}/subdir/ccc.txt (100%) delete mode 100644 llvm/utils/lit/tests/early-tests.py create mode 100644 llvm/utils/lit/tests/reorder.py diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst index 7e61a276765b..413b64e95007 100644 --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -20,7 +20,7 @@ user interface as possible. command line. Tests can be either individual test files or directories to search for tests (see :ref:`test-discovery`). -Each specified test will be executed (potentially in parallel) and once all +Each specified test will be executed (potentially concurrently) and once all tests have been run :program:`lit` will print summary information on the number of tests which passed or failed (see :ref:`test-status-results`). The :program:`lit` program will execute with a non-zero exit code if any tests @@ -151,8 +151,7 @@ EXECUTION OPTIONS Track the wall time individual tests take to execute and includes the results in the summary output. This is useful for determining which tests in a test - suite take the most time to execute. Note that this option is most useful - with ``-j 1``. + suite take the most time to execute. .. option:: --ignore-fail @@ -168,6 +167,17 @@ EXECUTION OPTIONS SELECTION OPTIONS ----------------- +By default, `lit` will run failing tests first, then run tests in descending +execution time order to optimize concurrency. + +The timing data is stored in the `test_exec_root` in a file named +`.lit_test_times.txt`. If this file does not exist, then `lit` checks the +`test_source_root` for the file to optionally accelerate clean builds. + +.. option:: --shuffle + + Run the tests in a random order, not failing/slowest first. + .. option:: --max-failures N Stop execution after the given number ``N`` of failures. @@ -201,10 +211,6 @@ SELECTION OPTIONS must be in the range ``1..M``. The environment variable ``LIT_RUN_SHARD`` can also be used in place of this option. -.. option:: --shuffle - - Run the tests in a random order. - .. option:: --timeout=N Spend at most ``N`` seconds (approximately) running each individual test. @@ -416,13 +422,6 @@ executed, two important global variables are predefined: **root** The root configuration. This is the top-most :program:`lit` configuration in the project. - **is_early** Whether the test suite as a whole should be given a head start - before other test suites run. - - **early_tests** An explicit set of '/' separated test paths that should be - given a head start before other tests run. For example, the top five or so - slowest tests. See also: `--time-tests` - **pipefail** Normally a test using a shell pipe fails if any of the commands on the pipe fail. If this is not desired, setting this variable to false makes the test fail only if the last command in the pipe fails. diff --git a/llvm/test/Unit/lit.cfg.py b/llvm/test/Unit/lit.cfg.py index 3198ab2c9539..3a5c40dc14da 100644 --- a/llvm/test/Unit/lit.cfg.py +++ b/llvm/test/Unit/lit.cfg.py @@ -13,9 +13,6 @@ config.name = 'LLVM-Unit' # suffixes: A list of file extensions to treat as test files. config.suffixes = [] -# is_early; Request to run this suite early. -config.is_early = True - # test_source_root: The root path where tests are located. # test_exec_root: The root path where tests should be run. config.test_exec_root = os.path.join(config.llvm_obj_root, 'unittests') diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py index ce87cfa8abb5..ad42ef183ede 100644 --- a/llvm/utils/lit/lit/Test.py +++ b/llvm/utils/lit/lit/Test.py @@ -207,6 +207,16 @@ class TestSuite: # The test suite configuration. self.config = config + self.test_times = {} + test_times_file = os.path.join(exec_root, '.lit_test_times.txt') + if not os.path.exists(test_times_file): + test_times_file = os.path.join(source_root, '.lit_test_times.txt') + if os.path.exists(test_times_file): + with open(test_times_file, 'r') as time_file: + for line in time_file: + time, path = line.split(maxsplit=1) + self.test_times[path.strip('\n')] = float(time) + def getSourcePath(self, components): return os.path.join(self.source_root, *components) @@ -246,6 +256,18 @@ class Test: # The test result, once complete. self.result = None + # The previous test failure state, if applicable. + self.previous_failure = False + + # The previous test elapsed time, if applicable. + self.previous_elapsed = 0.0 + + if os.sep.join(path_in_suite) in suite.test_times: + time = suite.test_times[os.sep.join(path_in_suite)] + self.previous_elapsed = abs(time) + self.previous_failure = time < 0 + + def setResult(self, result): assert self.result is None, "result already set" assert isinstance(result, Result), "unexpected result type" @@ -395,15 +417,3 @@ class Test: ) identifiers = set(filter(BooleanExpression.isIdentifier, tokens)) return identifiers - - def isEarlyTest(self): - """ - isEarlyTest() -> bool - - Check whether this test should be executed early in a particular run. - This can be used for test suites with long running tests to maximize - parallelism or where it is desirable to surface their failures early. - """ - if '/'.join(self.path_in_suite) in self.suite.config.early_tests: - return True - return self.suite.config.is_early diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py index fafc754c1bc1..612db574677e 100644 --- a/llvm/utils/lit/lit/TestingConfig.py +++ b/llvm/utils/lit/lit/TestingConfig.py @@ -125,10 +125,6 @@ class TestingConfig(object): # require one of the features in this list if this list is non-empty. # Configurations can set this list to restrict the set of tests to run. self.limit_to_features = set(limit_to_features) - # Whether the suite should be tested early in a given run. - self.is_early = bool(is_early) - # List of tests to run early. - self.early_tests = {} self.parallelism_group = parallelism_group self._recursiveExpansionLimit = None diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py index 4d829659ea18..3eb1870bf16d 100644 --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -9,8 +9,7 @@ import lit.util class TestOrder(enum.Enum): - EARLY_TESTS_THEN_BY_NAME = enum.auto() - FAILING_FIRST = enum.auto() + DEFAULT = enum.auto() RANDOM = enum.auto() @@ -155,7 +154,7 @@ def parse_args(): help="Run tests in random order", action="store_true") selection_group.add_argument("-i", "--incremental", - help="Run modified and failing tests first (updates mtimes)", + help="Run failed tests first (DEPRECATED: now always enabled)", action="store_true") selection_group.add_argument("--filter", metavar="REGEX", @@ -208,12 +207,13 @@ def parse_args(): if opts.echoAllCommands: opts.showOutput = True + if opts.incremental: + print('WARNING: --incremental is deprecated. Failing tests now always run first.') + if opts.shuffle: opts.order = TestOrder.RANDOM - elif opts.incremental: - opts.order = TestOrder.FAILING_FIRST else: - opts.order = TestOrder.EARLY_TESTS_THEN_BY_NAME + opts.order = TestOrder.DEFAULT if opts.numShards or opts.runShard: if not opts.numShards or not opts.runShard: diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py index a185ae676d14..43481d8bd3b3 100644 --- a/llvm/utils/lit/lit/discovery.py +++ b/llvm/utils/lit/lit/discovery.py @@ -281,6 +281,11 @@ def find_tests_for_inputs(lit_config, inputs, indirectlyRunCheck): if prev == len(tests): lit_config.warning('input %r contained no tests' % input) + # This data is no longer needed but keeping it around causes awful + # performance problems while the test suites run. + for k, suite in test_suite_cache.items(): + suite[0].test_times = None + # If there were any errors during test discovery, exit now. if lit_config.numErrors: sys.stderr.write('%d errors, exiting.\n' % lit_config.numErrors) diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py index 3f265446be2e..cfc12661a785 100755 --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -105,6 +105,8 @@ def main(builtin_params={}): run_tests(selected_tests, lit_config, opts, len(discovered_tests)) elapsed = time.time() - start + record_test_times(selected_tests, lit_config) + if opts.time_tests: print_histogram(discovered_tests) @@ -163,20 +165,12 @@ def print_discovered(tests, show_suites, show_tests): def determine_order(tests, order): from lit.cl_arguments import TestOrder - if order == TestOrder.EARLY_TESTS_THEN_BY_NAME: - tests.sort(key=lambda t: (not t.isEarlyTest(), t.getFullName())) - elif order == TestOrder.FAILING_FIRST: - def by_mtime(test): - return os.path.getmtime(test.getFilePath()) - tests.sort(key=by_mtime, reverse=True) - elif order == TestOrder.RANDOM: + if order == TestOrder.RANDOM: import random random.shuffle(tests) - - -def touch_file(test): - if test.isFailure(): - os.utime(test.getFilePath(), None) + else: + assert order == TestOrder.DEFAULT, 'Unknown TestOrder value' + tests.sort(key=lambda t: (not t.previous_failure, -t.previous_elapsed, t.getFullName())) def filter_by_shard(tests, run, shards, lit_config): @@ -213,12 +207,7 @@ def run_tests(tests, lit_config, opts, discovered_tests): display = lit.display.create_display(opts, len(tests), discovered_tests, workers) - def progress_callback(test): - display.update(test) - if opts.order == lit.cl_arguments.TestOrder.FAILING_FIRST: - touch_file(test) - - run = lit.run.Run(tests, lit_config, workers, progress_callback, + run = lit.run.Run(tests, lit_config, workers, display.update, opts.max_failures, opts.timeout) display.print_header() @@ -267,6 +256,27 @@ def execute_in_tmp_dir(run, lit_config): lit_config.warning("Failed to delete temp directory '%s', try upgrading your version of Python to fix this" % tmp_dir) +def record_test_times(tests, lit_config): + times_by_suite = {} + for t in tests: + if not t.result.elapsed: + continue + if not t.suite.exec_root in times_by_suite: + times_by_suite[t.suite.exec_root] = [] + time = -t.result.elapsed if t.isFailure() else t.result.elapsed + times_by_suite[t.suite.exec_root].append((os.sep.join(t.path_in_suite), t.result.elapsed)) + + for s, value in times_by_suite.items(): + try: + path = os.path.join(s, '.lit_test_times.txt') + with open(path, 'w') as time_file: + for name, time in value: + time_file.write(("%e" % time) + ' ' + name + '\n') + except: + lit_config.warning('Could not save test time: ' + path) + continue + + def print_histogram(tests): test_times = [(t.getFullName(), t.result.elapsed) for t in tests if t.result.elapsed] diff --git a/llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt b/llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt new file mode 100644 index 000000000000..00aecc968ed3 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/reorder/.lit_test_times.txt @@ -0,0 +1,3 @@ +3.0 subdir/ccc.txt +2.0 bbb.txt +0.1 aaa.txt diff --git a/llvm/utils/lit/tests/Inputs/early-tests/aaa.txt b/llvm/utils/lit/tests/Inputs/reorder/aaa.txt similarity index 100% rename from llvm/utils/lit/tests/Inputs/early-tests/aaa.txt rename to llvm/utils/lit/tests/Inputs/reorder/aaa.txt diff --git a/llvm/utils/lit/tests/Inputs/early-tests/bbb.txt b/llvm/utils/lit/tests/Inputs/reorder/bbb.txt similarity index 100% rename from llvm/utils/lit/tests/Inputs/early-tests/bbb.txt rename to llvm/utils/lit/tests/Inputs/reorder/bbb.txt diff --git a/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg b/llvm/utils/lit/tests/Inputs/reorder/lit.cfg similarity index 67% rename from llvm/utils/lit/tests/Inputs/early-tests/lit.cfg rename to llvm/utils/lit/tests/Inputs/reorder/lit.cfg index db030510c249..6320609a1e6c 100644 --- a/llvm/utils/lit/tests/Inputs/early-tests/lit.cfg +++ b/llvm/utils/lit/tests/Inputs/reorder/lit.cfg @@ -1,7 +1,6 @@ import lit.formats -config.name = 'early-tests' +config.name = 'reorder' config.suffixes = ['.txt'] config.test_format = lit.formats.ShTest() config.test_source_root = None config.test_exec_root = None -config.early_tests = { "subdir/ccc.txt" } diff --git a/llvm/utils/lit/tests/Inputs/early-tests/subdir/ccc.txt b/llvm/utils/lit/tests/Inputs/reorder/subdir/ccc.txt similarity index 100% rename from llvm/utils/lit/tests/Inputs/early-tests/subdir/ccc.txt rename to llvm/utils/lit/tests/Inputs/reorder/subdir/ccc.txt diff --git a/llvm/utils/lit/tests/early-tests.py b/llvm/utils/lit/tests/early-tests.py deleted file mode 100644 index b2ca9ac0a97d..000000000000 --- a/llvm/utils/lit/tests/early-tests.py +++ /dev/null @@ -1,9 +0,0 @@ -## Check that we can run tests early. - -# RUN: %{lit} -j1 %{inputs}/early-tests | FileCheck %s - -# CHECK: -- Testing: 3 tests, 1 workers -- -# CHECK-NEXT: PASS: early-tests :: subdir/ccc.txt -# CHECK-NEXT: PASS: early-tests :: aaa.txt -# CHECK-NEXT: PASS: early-tests :: bbb.txt -# CHECK: Passed: 3 diff --git a/llvm/utils/lit/tests/ignore-fail.py b/llvm/utils/lit/tests/ignore-fail.py index 135e29baa5a6..63c34516226d 100644 --- a/llvm/utils/lit/tests/ignore-fail.py +++ b/llvm/utils/lit/tests/ignore-fail.py @@ -6,10 +6,10 @@ # END. -# CHECK: FAIL: ignore-fail :: fail.txt -# CHECK: UNRESOLVED: ignore-fail :: unresolved.txt -# CHECK: XFAIL: ignore-fail :: xfail.txt -# CHECK: XPASS: ignore-fail :: xpass.txt +# CHECK-DAG: FAIL: ignore-fail :: fail.txt +# CHECK-DAG: UNRESOLVED: ignore-fail :: unresolved.txt +# CHECK-DAG: XFAIL: ignore-fail :: xfail.txt +# CHECK-DAG: XPASS: ignore-fail :: xpass.txt # CHECK: Testing Time: # CHECK-NEXT: Expectedly Failed : 1 diff --git a/llvm/utils/lit/tests/reorder.py b/llvm/utils/lit/tests/reorder.py new file mode 100644 index 000000000000..7c9dc8d21fe3 --- /dev/null +++ b/llvm/utils/lit/tests/reorder.py @@ -0,0 +1,12 @@ +## Check that we can reorder test runs. + +# RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig +# RUN: %{lit} -j1 %{inputs}/reorder | FileCheck %s +# RUN: not diff %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig +# END. + +# CHECK: -- Testing: 3 tests, 1 workers -- +# CHECK-NEXT: PASS: reorder :: subdir/ccc.txt +# CHECK-NEXT: PASS: reorder :: bbb.txt +# CHECK-NEXT: PASS: reorder :: aaa.txt +# CHECK: Passed: 3 diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py index 4c247de15ddd..3f1ead3b297a 100644 --- a/llvm/utils/lit/tests/shtest-shell.py +++ b/llvm/utils/lit/tests/shtest-shell.py @@ -8,6 +8,8 @@ # # Test again in non-UTF shell to catch potential errors with python 2 seen # on stdout-encoding.txt +# FIXME: lit's testing sets source_root == exec_root which complicates running lit more than once per test. +# RUN: rm -f %{inputs}/shtest-shell/.lit_test_times.txt # RUN: env PYTHONIOENCODING=ascii not %{lit} -j 1 -a %{inputs}/shtest-shell > %t.ascii.out # FIXME: Temporarily dump test output so we can debug failing tests on # buildbots. diff --git a/mlir/test/Unit/lit.cfg.py b/mlir/test/Unit/lit.cfg.py index ea14853e71d6..d645971074f5 100644 --- a/mlir/test/Unit/lit.cfg.py +++ b/mlir/test/Unit/lit.cfg.py @@ -13,9 +13,6 @@ config.name = 'MLIR-Unit' # suffixes: A list of file extensions to treat as test files. config.suffixes = [] -# is_early; Request to run this suite early. -config.is_early = True - # test_source_root: The root path where tests are located. # test_exec_root: The root path where tests should be run. config.test_exec_root = os.path.join(config.mlir_obj_root, 'unittests') -- GitLab From 92d27b969ae16bab23d2ccb1be2c350a26739bd0 Mon Sep 17 00:00:00 2001 From: Jean Perier Date: Tue, 16 Mar 2021 09:47:35 +0100 Subject: [PATCH 0007/1206] [flang] Save AllocateObject and PointerObject analyzed expression `parser::AllocateObject` and `parser::PointerObject` can be represented as typed expressions once analyzed. This simplifies the work for parse-tree consumers that work with typed expressions to deal with allocatable and pointer objects such as lowering. This change also makes it easier to add typedExpr in the future by automatically handling nodes that have this member when possible. Changes: - Add a `mutable TypedExpr typedExpr` field to `parser::PointerObject` and `parser::AllocateObject`. - Add a `parser::HasTypedExpr` helper to better share code relating to typedExpr in the parse tree. - Add hooks in `semantics::ExprChecker` for AllocateObject and PointerObject nodes, and use ExprOrVariable on it to analyze and set the tyedExpr field during expression analysis. This required adding overloads for `AssumedTypeDummy`. - Update check-nullify.cpp and check-deallocate.cpp to not re-analyze the StructureComponent but to use the typedExpr field instead. - Update dump/unparse to use HasTypedExpr and use the typedExpr when there is one. Differential Revision: https://reviews.llvm.org/D98256 --- flang/include/flang/Parser/dump-parse-tree.h | 2 +- flang/include/flang/Parser/parse-tree.h | 2 + flang/include/flang/Parser/tools.h | 5 ++ flang/include/flang/Semantics/expression.h | 17 ++++-- flang/include/flang/Semantics/tools.h | 6 +++ flang/lib/Parser/unparse.cpp | 18 +++---- flang/lib/Semantics/check-deallocate.cpp | 5 +- flang/lib/Semantics/check-nullify.cpp | 5 +- flang/lib/Semantics/expression.cpp | 56 +++++++++++++++++--- flang/lib/Semantics/tools.cpp | 17 ++++-- 10 files changed, 102 insertions(+), 31 deletions(-) diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h index bc0fd388b11c..150b011ad8ba 100644 --- a/flang/include/flang/Parser/dump-parse-tree.h +++ b/flang/include/flang/Parser/dump-parse-tree.h @@ -793,7 +793,7 @@ protected: template std::string AsFortran(const T &x) { std::string buf; llvm::raw_string_ostream ss{buf}; - if constexpr (std::is_same_v) { + if constexpr (HasTypedExpr::value) { if (asFortran_ && x.typedExpr) { asFortran_->expr(ss, *x.typedExpr); } diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h index dcc38090a3a1..152c2c8c9076 100644 --- a/flang/include/flang/Parser/parse-tree.h +++ b/flang/include/flang/Parser/parse-tree.h @@ -1836,6 +1836,7 @@ struct ArrayElement { // R933 allocate-object -> variable-name | structure-component struct AllocateObject { UNION_CLASS_BOILERPLATE(AllocateObject); + mutable TypedExpr typedExpr; std::variant u; }; @@ -1907,6 +1908,7 @@ struct AllocateStmt { // variable-name | structure-component | proc-pointer-name struct PointerObject { UNION_CLASS_BOILERPLATE(PointerObject); + mutable TypedExpr typedExpr; std::variant u; }; diff --git a/flang/include/flang/Parser/tools.h b/flang/include/flang/Parser/tools.h index 66c8793399c9..ccd49d2a790e 100644 --- a/flang/include/flang/Parser/tools.h +++ b/flang/include/flang/Parser/tools.h @@ -117,5 +117,10 @@ template struct HasSource(A::source), 0)> : std::true_type {}; +// Detects parse tree nodes with "typedExpr" members. +template struct HasTypedExpr : std::false_type {}; +template +struct HasTypedExpr(A::typedExpr), 0)> + : std::true_type {}; } // namespace Fortran::parser #endif // FORTRAN_PARSER_TOOLS_H_ diff --git a/flang/include/flang/Semantics/expression.h b/flang/include/flang/Semantics/expression.h index f81d5199dc20..2f89820f4b0b 100644 --- a/flang/include/flang/Semantics/expression.h +++ b/flang/include/flang/Semantics/expression.h @@ -74,14 +74,13 @@ struct SetExprHelper { x.Reset(new GenericExprWrapper{std::move(expr_)}, evaluate::GenericExprWrapper::Deleter); } - void Set(const parser::Expr &x) { Set(x.typedExpr); } - void Set(const parser::Variable &x) { Set(x.typedExpr); } - void Set(const parser::DataStmtConstant &x) { Set(x.typedExpr); } template void Set(const common::Indirection &x) { Set(x.value()); } template void Set(const T &x) { - if constexpr (ConstraintTrait) { + if constexpr (parser::HasTypedExpr::value) { + Set(x.typedExpr); + } else if constexpr (ConstraintTrait) { Set(x.thing); } else if constexpr (WrapperTrait) { Set(x.v); @@ -157,6 +156,8 @@ public: MaybeExpr Analyze(const parser::Variable &); MaybeExpr Analyze(const parser::Designator &); MaybeExpr Analyze(const parser::DataStmtValue &); + MaybeExpr Analyze(const parser::AllocateObject &); + MaybeExpr Analyze(const parser::PointerObject &); template MaybeExpr Analyze(const common::Indirection &x) { return Analyze(x.value()); @@ -451,6 +452,14 @@ public: exprAnalyzer_.Analyze(x); return false; } + bool Pre(const parser::AllocateObject &x) { + exprAnalyzer_.Analyze(x); + return false; + } + bool Pre(const parser::PointerObject &x) { + exprAnalyzer_.Analyze(x); + return false; + } bool Pre(const parser::DataImpliedDo &); bool Pre(const parser::CallStmt &x) { diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h index 3e8d1993f9a0..550cc99f85ef 100644 --- a/flang/include/flang/Semantics/tools.h +++ b/flang/include/flang/Semantics/tools.h @@ -257,9 +257,13 @@ bool ExprTypeKindIsDefault( const SomeExpr &expr, const SemanticsContext &context); struct GetExprHelper { + // Specializations for parse tree nodes that have a typedExpr member. static const SomeExpr *Get(const parser::Expr &); static const SomeExpr *Get(const parser::Variable &); static const SomeExpr *Get(const parser::DataStmtConstant &); + static const SomeExpr *Get(const parser::AllocateObject &); + static const SomeExpr *Get(const parser::PointerObject &); + template static const SomeExpr *Get(const common::Indirection &x) { return Get(x.value()); @@ -268,6 +272,8 @@ struct GetExprHelper { return x ? Get(*x) : nullptr; } template static const SomeExpr *Get(const T &x) { + static_assert( + !parser::HasTypedExpr::value, "explicit Get overload must be added"); if constexpr (ConstraintTrait) { return Get(x.thing); } else if constexpr (WrapperTrait) { diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp index 8adcc32b87af..eaa4c926068c 100644 --- a/flang/lib/Parser/unparse.cpp +++ b/flang/lib/Parser/unparse.cpp @@ -16,6 +16,7 @@ #include "flang/Parser/characters.h" #include "flang/Parser/parse-tree-visitor.h" #include "flang/Parser/parse-tree.h" +#include "flang/Parser/tools.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -48,6 +49,14 @@ public: Unparse(x); Post(x); return false; // Walk() does not visit descendents + } else if constexpr (HasTypedExpr::value) { + // Format the expression representation from semantics + if (asFortran_ && x.typedExpr) { + asFortran_->expr(out_, *x.typedExpr); + return false; + } else { + return true; + } } else { Before(x); return true; // there's no Unparse() defined here, Walk() the descendents @@ -816,15 +825,6 @@ public: } // R1001 - R1022 - bool Pre(const Expr &x) { - if (asFortran_ && x.typedExpr) { - // Format the expression representation from semantics - asFortran_->expr(out_, *x.typedExpr); - return false; - } else { - return true; - } - } void Unparse(const Expr::Parentheses &x) { Put('('), Walk(x.v), Put(')'); } void Before(const Expr::UnaryPlus &) { Put("+"); } void Before(const Expr::Negate &) { Put("-"); } diff --git a/flang/lib/Semantics/check-deallocate.cpp b/flang/lib/Semantics/check-deallocate.cpp index 92e197bbb322..03c2d6ebddda 100644 --- a/flang/lib/Semantics/check-deallocate.cpp +++ b/flang/lib/Semantics/check-deallocate.cpp @@ -34,8 +34,9 @@ void DeallocateChecker::Leave(const parser::DeallocateStmt &deallocateStmt) { } }, [&](const parser::StructureComponent &structureComponent) { - evaluate::ExpressionAnalyzer analyzer{context_}; - if (MaybeExpr checked{analyzer.Analyze(structureComponent)}) { + // Only perform structureComponent checks it was successfully + // analyzed in expression analysis. + if (GetExpr(allocateObject)) { if (!IsAllocatableOrPointer( *structureComponent.component.symbol)) { // C932 context_.Say(structureComponent.component.source, diff --git a/flang/lib/Semantics/check-nullify.cpp b/flang/lib/Semantics/check-nullify.cpp index ff49a661e206..4c6e78e7f7e3 100644 --- a/flang/lib/Semantics/check-nullify.cpp +++ b/flang/lib/Semantics/check-nullify.cpp @@ -40,13 +40,12 @@ void NullifyChecker::Leave(const parser::NullifyStmt &nullifyStmt) { } }, [&](const parser::StructureComponent &structureComponent) { - evaluate::ExpressionAnalyzer analyzer{context_}; - if (MaybeExpr checked{analyzer.Analyze(structureComponent)}) { + if (const auto *checkedExpr{GetExpr(pointerObject)}) { if (!IsPointer(*structureComponent.component.symbol)) { // C951 messages.Say(structureComponent.component.source, "component in NULLIFY statement must have the POINTER attribute"_err_en_US); } else if (pure) { - if (const Symbol * symbol{GetFirstSymbol(checked)}) { + if (const Symbol * symbol{GetFirstSymbol(*checkedExpr)}) { CheckDefinabilityInPureScope( messages, *symbol, scope, *pure); } diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp index 3413a7531759..0b36de464129 100644 --- a/flang/lib/Semantics/expression.cpp +++ b/flang/lib/Semantics/expression.cpp @@ -2139,18 +2139,48 @@ template static const Symbol *AssumedTypeDummy(const A &x) { if (const auto *dataRef{ std::get_if(&designator->value().u)}) { if (const auto *name{std::get_if(&dataRef->u)}) { - if (const Symbol * symbol{name->symbol}) { - if (const auto *type{symbol->GetType()}) { - if (type->category() == semantics::DeclTypeSpec::TypeStar) { - return symbol; - } - } - } + return AssumedTypeDummy(*name); } } } return nullptr; } +template <> +const Symbol *AssumedTypeDummy(const parser::Name &name) { + if (const Symbol * symbol{name.symbol}) { + if (const auto *type{symbol->GetType()}) { + if (type->category() == semantics::DeclTypeSpec::TypeStar) { + return symbol; + } + } + } + return nullptr; +} +template +static const Symbol *AssumedTypePointerOrAllocatableDummy(const A &object) { + // It is illegal for allocatable of pointer objects to be TYPE(*), but at that + // point it is is not guaranteed that it has been checked the object has + // POINTER or ALLOCATABLE attribute, so do not assume nullptr can be directly + // returned. + return std::visit( + common::visitors{ + [&](const parser::StructureComponent &x) { + return AssumedTypeDummy(x.component); + }, + [&](const parser::Name &x) { return AssumedTypeDummy(x); }, + }, + object.u); +} +template <> +const Symbol *AssumedTypeDummy( + const parser::AllocateObject &x) { + return AssumedTypePointerOrAllocatableDummy(x); +} +template <> +const Symbol *AssumedTypeDummy( + const parser::PointerObject &x) { + return AssumedTypePointerOrAllocatableDummy(x); +} MaybeExpr ExpressionAnalyzer::Analyze(const parser::FunctionReference &funcRef, std::optional *structureConstructor) { @@ -2737,6 +2767,18 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::DataStmtConstant &x) { return ExprOrVariable(x, x.source); } +MaybeExpr ExpressionAnalyzer::Analyze(const parser::AllocateObject &x) { + parser::CharBlock source{parser::FindSourceLocation(x)}; + auto restorer{GetContextualMessages().SetLocation(source)}; + return ExprOrVariable(x, source); +} + +MaybeExpr ExpressionAnalyzer::Analyze(const parser::PointerObject &x) { + parser::CharBlock source{parser::FindSourceLocation(x)}; + auto restorer{GetContextualMessages().SetLocation(source)}; + return ExprOrVariable(x, source); +} + Expr ExpressionAnalyzer::AnalyzeKindSelector( TypeCategory category, const std::optional &selector) { diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp index 2d7fa9de9392..256a5cc1d317 100644 --- a/flang/lib/Semantics/tools.cpp +++ b/flang/lib/Semantics/tools.cpp @@ -374,17 +374,24 @@ static void CheckMissingAnalysis(bool absent, const T &x) { } } -const SomeExpr *GetExprHelper::Get(const parser::Expr &x) { +template static const SomeExpr *GetTypedExpr(const T &x) { CheckMissingAnalysis(!x.typedExpr, x); return common::GetPtrFromOptional(x.typedExpr->v); } +const SomeExpr *GetExprHelper::Get(const parser::Expr &x) { + return GetTypedExpr(x); +} const SomeExpr *GetExprHelper::Get(const parser::Variable &x) { - CheckMissingAnalysis(!x.typedExpr, x); - return common::GetPtrFromOptional(x.typedExpr->v); + return GetTypedExpr(x); } const SomeExpr *GetExprHelper::Get(const parser::DataStmtConstant &x) { - CheckMissingAnalysis(!x.typedExpr, x); - return common::GetPtrFromOptional(x.typedExpr->v); + return GetTypedExpr(x); +} +const SomeExpr *GetExprHelper::Get(const parser::AllocateObject &x) { + return GetTypedExpr(x); +} +const SomeExpr *GetExprHelper::Get(const parser::PointerObject &x) { + return GetTypedExpr(x); } const evaluate::Assignment *GetAssignment(const parser::AssignmentStmt &x) { -- GitLab From 2995e161b05f0787dd40273062bc387ecbb3dfd8 Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Wed, 3 Mar 2021 09:47:24 +0100 Subject: [PATCH 0008/1206] [mlir]: Add canonicalization for dim of 1D alloc of size rank. Differential Revision: https://reviews.llvm.org/D97542 --- mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp | 4 +++ mlir/test/Dialect/Standard/canonicalize.mlir | 28 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp index db8a96fa2bed..fddab63e3e98 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp @@ -617,6 +617,10 @@ OpFoldResult DimOp::fold(ArrayRef operands) { return *(alloc.getDynamicSizes().begin() + memrefType.getDynamicDimIndex(unsignedIndex)); + if (auto alloca = dyn_cast_or_null(definingOp)) + return *(alloca.getDynamicSizes().begin() + + memrefType.getDynamicDimIndex(unsignedIndex)); + if (auto view = dyn_cast_or_null(definingOp)) return *(view.getDynamicSizes().begin() + memrefType.getDynamicDimIndex(unsignedIndex)); diff --git a/mlir/test/Dialect/Standard/canonicalize.mlir b/mlir/test/Dialect/Standard/canonicalize.mlir index 41ae2248b299..a6bf0c78321a 100644 --- a/mlir/test/Dialect/Standard/canonicalize.mlir +++ b/mlir/test/Dialect/Standard/canonicalize.mlir @@ -134,6 +134,34 @@ func @cmpi_equal_operands(%arg0: i64) // ----- +// Test case: Folding of memref.dim(memref.alloca(%size), %idx) -> %size +// CHECK-LABEL: func @dim_of_alloca( +// CHECK-SAME: %[[SIZE:[0-9a-z]+]]: index +// CHECK-NEXT: return %[[SIZE]] : index +func @dim_of_alloca(%size: index) -> index { + %0 = memref.alloca(%size) : memref + %c0 = constant 0 : index + %1 = memref.dim %0, %c0 : memref + return %1 : index +} + +// ----- + +// Test case: Folding of memref.dim(memref.alloca(rank(%v)), %idx) -> rank(%v) +// CHECK-LABEL: func @dim_of_alloca_with_dynamic_size( +// CHECK-SAME: %[[MEM:[0-9a-z]+]]: memref<*xf32> +// CHECK-NEXT: %[[RANK:.*]] = rank %[[MEM]] : memref<*xf32> +// CHECK-NEXT: return %[[RANK]] : index +func @dim_of_alloca_with_dynamic_size(%arg0: memref<*xf32>) -> index { + %0 = rank %arg0 : memref<*xf32> + %1 = memref.alloca(%0) : memref + %c0 = constant 0 : index + %2 = memref.dim %1, %c0 : memref + return %2 : index +} + +// ----- + // Test case: Folding of memref.dim(memref.reshape %v %shp, %idx) -> memref.load %shp[%idx] // CHECK-LABEL: func @dim_of_memref_reshape( // CHECK-SAME: %[[MEM:[0-9a-z]+]]: memref<*xf32>, -- GitLab From 4a17ac0387f078529da02e355a24df99f645d364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Tue, 16 Mar 2021 11:05:48 +0100 Subject: [PATCH 0009/1206] [test][NFC] Minor formatting and comment adjustments in GetErrcMessages.cmake These changes address post-commit review comments discussed in https://reviews.llvm.org/D98278 --- llvm/cmake/modules/GetErrcMessages.cmake | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/cmake/modules/GetErrcMessages.cmake b/llvm/cmake/modules/GetErrcMessages.cmake index 79aa6456cc7e..908b1f538b95 100644 --- a/llvm/cmake/modules/GetErrcMessages.cmake +++ b/llvm/cmake/modules/GetErrcMessages.cmake @@ -1,9 +1,8 @@ - # This function returns the messages of various POSIX error codes as they are returned by std::error_code. -# The purpose of this function is to supply those error messages to llvm-lit using the errc_messages config -# Currently supplied and needed error codes: ENOENT, EISDIR, EINVAL and EACCES -# Messages are semi colon separated -# Keep amount, order and tested error codes in sync with llvm/utils/lit/lit/llvm/config.py +# The purpose of this function is to supply those error messages to llvm-lit using the errc_messages config. +# Currently supplied and needed error codes: ENOENT, EISDIR, EINVAL and EACCES. +# Messages are semi colon separated. +# Keep amount, order and tested error codes in sync with llvm/utils/lit/lit/llvm/config.py. function(get_errc_messages outvar) set(errc_test_code ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/getErrc.cpp) -- GitLab From 596db9934b91703d0a9b97d194ae82f110388330 Mon Sep 17 00:00:00 2001 From: Dmitry Preobrazhensky Date: Tue, 16 Mar 2021 13:51:03 +0300 Subject: [PATCH 0010/1206] [AMDGPU][MC] Disabled lds_direct for GFX90a Fixed bug 49382. Differential Revision: https://reviews.llvm.org/D98626 --- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 47 +++++++++---------- llvm/test/MC/AMDGPU/gfx10_err_pos.s | 37 ++++++++++++--- llvm/test/MC/AMDGPU/gfx90a_err.s | 12 +++++ llvm/test/MC/AMDGPU/lds_direct-err.s | 32 ++++++------- llvm/test/MC/AMDGPU/lds_direct-gfx10.s | 28 +++++------ 5 files changed, 93 insertions(+), 63 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 5547cd6c1c32..8a8831f22ff1 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -1539,7 +1539,6 @@ private: bool validateMIMGD16(const MCInst &Inst); bool validateMIMGDim(const MCInst &Inst); bool validateMIMGMSAA(const MCInst &Inst); - bool validateLdsDirect(const MCInst &Inst); bool validateOpSel(const MCInst &Inst); bool validateVccOperand(unsigned Reg) const; bool validateVOP3Literal(const MCInst &Inst, const OperandVector &Operands); @@ -1549,6 +1548,7 @@ private: bool validateDivScale(const MCInst &Inst); bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands, const SMLoc &IDLoc); + Optional validateLdsDirect(const MCInst &Inst); unsigned getConstantBusLimit(unsigned Opcode) const; bool usesConstantBus(const MCInst &Inst, unsigned OpIdx); bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const; @@ -3768,7 +3768,7 @@ static bool IsRevOpcode(const unsigned Opcode) } } -bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) { +Optional AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) { using namespace SIInstrFlags; const unsigned Opcode = Inst.getOpcode(); @@ -3776,33 +3776,29 @@ bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) { // lds_direct register is defined so that it can be used // with 9-bit operands only. Ignore encodings which do not accept these. - if ((Desc.TSFlags & (VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA)) == 0) - return true; + const auto Enc = VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA; + if ((Desc.TSFlags & Enc) == 0) + return None; - const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); - const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); - const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); + for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) { + auto SrcIdx = getNamedOperandIdx(Opcode, SrcName); + if (SrcIdx == -1) + break; + const auto &Src = Inst.getOperand(SrcIdx); + if (Src.isReg() && Src.getReg() == LDS_DIRECT) { - const int SrcIndices[] = { Src1Idx, Src2Idx }; + if (isGFX90A()) + return StringRef("lds_direct is not supported on this GPU"); - // lds_direct cannot be specified as either src1 or src2. - for (int SrcIdx : SrcIndices) { - if (SrcIdx == -1) break; - const MCOperand &Src = Inst.getOperand(SrcIdx); - if (Src.isReg() && Src.getReg() == LDS_DIRECT) { - return false; + if (IsRevOpcode(Opcode) || (Desc.TSFlags & SIInstrFlags::SDWA)) + return StringRef("lds_direct cannot be used with this instruction"); + + if (SrcName != OpName::src0) + return StringRef("lds_direct may be used as src0 only"); } } - if (Src0Idx == -1) - return true; - - const MCOperand &Src = Inst.getOperand(Src0Idx); - if (!Src.isReg() || Src.getReg() != LDS_DIRECT) - return true; - - // lds_direct is specified as src0. Check additional limitations. - return (Desc.TSFlags & SIInstrFlags::SDWA) == 0 && !IsRevOpcode(Opcode); + return None; } SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const { @@ -4133,9 +4129,8 @@ bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst, bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands) { - if (!validateLdsDirect(Inst)) { - Error(getRegLoc(AMDGPU::LDS_DIRECT, Operands), - "invalid use of lds_direct"); + if (auto ErrMsg = validateLdsDirect(Inst)) { + Error(getRegLoc(LDS_DIRECT, Operands), *ErrMsg); return false; } if (!validateSOPLiteral(Inst)) { diff --git a/llvm/test/MC/AMDGPU/gfx10_err_pos.s b/llvm/test/MC/AMDGPU/gfx10_err_pos.s index e9e2add95d88..ebd314455c1a 100644 --- a/llvm/test/MC/AMDGPU/gfx10_err_pos.s +++ b/llvm/test/MC/AMDGPU/gfx10_err_pos.s @@ -935,20 +935,43 @@ v_ceil_f32 v0, --1 // CHECK-NEXT:{{^}} ^ //============================================================================== -// invalid use of lds_direct +// lane id must be in the interval [0,group size - 1] + +ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1) +// CHECK: error: lane id must be in the interval [0,group size - 1] +// CHECK-NEXT:{{^}}ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1) +// CHECK-NEXT:{{^}} ^ + +//============================================================================== +// lds_direct cannot be used with this instruction v_ashrrev_i16 v0, lds_direct, v0 -// CHECK: error: invalid use of lds_direct +// CHECK: error: lds_direct cannot be used with this instruction // CHECK-NEXT:{{^}}v_ashrrev_i16 v0, lds_direct, v0 // CHECK-NEXT:{{^}} ^ +v_ashrrev_i16 v0, v1, lds_direct +// CHECK: error: lds_direct cannot be used with this instruction +// CHECK-NEXT:{{^}}v_ashrrev_i16 v0, v1, lds_direct +// CHECK-NEXT:{{^}} ^ + +v_mov_b32_sdwa v1, src_lds_direct dst_sel:DWORD +// CHECK: error: lds_direct cannot be used with this instruction +// CHECK-NEXT:{{^}}v_mov_b32_sdwa v1, src_lds_direct dst_sel:DWORD +// CHECK-NEXT:{{^}} ^ + +v_add_f32_sdwa v5, v1, lds_direct dst_sel:DWORD +// CHECK: error: lds_direct cannot be used with this instruction +// CHECK-NEXT:{{^}}v_add_f32_sdwa v5, v1, lds_direct dst_sel:DWORD +// CHECK-NEXT:{{^}} ^ + //============================================================================== -// lane id must be in the interval [0,group size - 1] +// lds_direct may be used as src0 only -ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1) -// CHECK: error: lane id must be in the interval [0,group size - 1] -// CHECK-NEXT:{{^}}ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,2,-1) -// CHECK-NEXT:{{^}} ^ +v_add_f32 v5, v1, lds_direct +// CHECK: error: lds_direct may be used as src0 only +// CHECK-NEXT:{{^}}v_add_f32 v5, v1, lds_direct +// CHECK-NEXT:{{^}} ^ //============================================================================== // message does not support operations diff --git a/llvm/test/MC/AMDGPU/gfx90a_err.s b/llvm/test/MC/AMDGPU/gfx90a_err.s index 246291961599..15df69b05a17 100644 --- a/llvm/test/MC/AMDGPU/gfx90a_err.s +++ b/llvm/test/MC/AMDGPU/gfx90a_err.s @@ -230,3 +230,15 @@ global_atomic_min_f64 v[0:1], v[2:3], off scc global_atomic_max_f64 v[0:1], v[2:3], off scc // GFX90A: error: instruction must not use scc + +v_mov_b32_sdwa v1, src_lds_direct dst_sel:DWORD +// GFX90A: error: lds_direct is not supported on this GPU + +v_add_f32_sdwa v5, v1, lds_direct dst_sel:DWORD +// GFX90A: error: lds_direct is not supported on this GPU + +v_ashrrev_i16 v0, lds_direct, v0 +// GFX90A: error: lds_direct is not supported on this GPU + +v_add_f32 v5, v1, lds_direct +// GFX90A: error: lds_direct is not supported on this GPU diff --git a/llvm/test/MC/AMDGPU/lds_direct-err.s b/llvm/test/MC/AMDGPU/lds_direct-err.s index 48314613b040..854efad84ec0 100644 --- a/llvm/test/MC/AMDGPU/lds_direct-err.s +++ b/llvm/test/MC/AMDGPU/lds_direct-err.s @@ -12,46 +12,46 @@ s_and_b32 s2, lds_direct, s1 //---------------------------------------------------------------------------// v_ashrrev_i16 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_ashrrev_i32 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_lshlrev_b16 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_lshlrev_b32 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_lshrrev_b16 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_lshrrev_b32 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_pk_ashrrev_i16 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_pk_lshlrev_b16 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_pk_lshrrev_b16 v0, lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_subbrev_co_u32 v0, vcc, src_lds_direct, v0, vcc -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_subrev_co_u32 v0, vcc, src_lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_subrev_f16 v0, src_lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_subrev_u16 v0, src_lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction v_subrev_u32 v0, src_lds_direct, v0 -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct cannot be used with this instruction //---------------------------------------------------------------------------// // lds_direct may not be used with v_writelane_b32 for VI/GFX9 @@ -72,10 +72,10 @@ v_add_f64 v[0:1], lds_direct, v[0:1] //---------------------------------------------------------------------------// v_add_i32 v0, v0, lds_direct -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct may be used as src0 only v_add_i32 lds_direct, v0, v0 // NOGFX9: error: invalid operand for instruction v_fma_f32 v0, v0, v0, lds_direct -// NOGFX9: error: invalid use of lds_direct +// NOGFX9: error: lds_direct may be used as src0 only diff --git a/llvm/test/MC/AMDGPU/lds_direct-gfx10.s b/llvm/test/MC/AMDGPU/lds_direct-gfx10.s index 61e4de3e4691..df83471ed6da 100644 --- a/llvm/test/MC/AMDGPU/lds_direct-gfx10.s +++ b/llvm/test/MC/AMDGPU/lds_direct-gfx10.s @@ -17,43 +17,43 @@ v_permlanex16_b32 v0, lds_direct, s0, s0 // NOGFX10: error: invalid operand for instruction v_ashrrev_i16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_ashrrev_i32 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_lshlrev_b16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_lshlrev_b32 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_lshrrev_b16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_lshrrev_b32 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_pk_ashrrev_i16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_pk_lshlrev_b16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_pk_lshrrev_b16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_subrev_co_ci_u32 v0, vcc_lo, src_lds_direct, v0, vcc_lo -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_subrev_co_u32 v0, s0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_subrev_f16 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_subrev_f32 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction v_subrev_nc_u32 v0, src_lds_direct, v0 -// NOGFX10: error: invalid use of lds_direct +// NOGFX10: error: lds_direct cannot be used with this instruction -- GitLab From 1310c686c25e237844c927d7bf777aa26b0bac1f Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Tue, 16 Mar 2021 03:55:29 -0700 Subject: [PATCH 0011/1206] [sanitizer][NFC] Don't inherit InternalMmapVector --- .../lib/sanitizer_common/sanitizer_common.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h index 2fecc3b4bf7c..e8a15556d161 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -592,20 +592,26 @@ class InternalMmapVector : public InternalMmapVectorNoCtor { InternalMmapVector &operator=(InternalMmapVector &&) = delete; }; -class InternalScopedString : public InternalMmapVector { +class InternalScopedString { public: explicit InternalScopedString(uptr max_length) - : InternalMmapVector(max_length), length_(0) { - (*this)[0] = '\0'; + : buffer_(max_length), length_(0) { + buffer_[0] = '\0'; } - uptr length() { return length_; } + uptr size() const { return buffer_.size(); } + uptr length() const { return length_; } void clear() { (*this)[0] = '\0'; length_ = 0; } void append(const char *format, ...); + char *data() { return buffer_.data(); } + const char *data() const { return buffer_.data(); } + char &operator[](uptr i) { return buffer_[i]; } + const char &operator[](uptr i) const { return buffer_[i]; } private: + InternalMmapVector buffer_; uptr length_; }; -- GitLab From a92693dac4592e7bfbd9caf09939d46756de3821 Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Fri, 12 Mar 2021 00:01:25 +0100 Subject: [PATCH 0012/1206] [CodeCompletion] Don't track preferred types if code completion is disabled. Some of this work isn't quite trivial. (As requested in D96058) Differential Revision: https://reviews.llvm.org/D98459 --- clang/include/clang/Parse/Parser.h | 4 ++-- clang/include/clang/Sema/Sema.h | 14 +++++++------- clang/lib/Parse/ParseInit.cpp | 3 --- clang/lib/Parse/Parser.cpp | 8 ++++---- clang/lib/Sema/SemaCodeComplete.cpp | 22 +++++++++++++++++++++- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index 09a0dd2cf233..e1bd3531be8e 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -941,8 +941,8 @@ private: bool isActive; public: - explicit TentativeParsingAction(Parser& p) : P(p) { - PrevPreferredType = P.PreferredType; + explicit TentativeParsingAction(Parser &p) + : P(p), PrevPreferredType(P.PreferredType) { PrevTok = P.Tok; PrevTentativelyDeclaredIdentifierCount = P.TentativelyDeclaredIdentifiers.size(); diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index a919740aa662..9e3eb4f07472 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -286,14 +286,13 @@ public: } }; -/// Keeps track of expected type during expression parsing. The type is tied to -/// a particular token, all functions that update or consume the type take a -/// start location of the token they are looking at as a parameter. This allows -/// to avoid updating the type on hot paths in the parser. +/// Tracks expected type during expression parsing, for use in code completion. +/// The type is tied to a particular token, all functions that update or consume +/// the type take a start location of the token they are looking at as a +/// parameter. This avoids updating the type on hot paths in the parser. class PreferredTypeBuilder { public: - PreferredTypeBuilder() = default; - explicit PreferredTypeBuilder(QualType Type) : Type(Type) {} + PreferredTypeBuilder(bool Enabled) : Enabled(Enabled) {} void enterCondition(Sema &S, SourceLocation Tok); void enterReturn(Sema &S, SourceLocation Tok); @@ -320,7 +319,7 @@ public: void enterTypeCast(SourceLocation Tok, QualType CastType); QualType get(SourceLocation Tok) const { - if (Tok != ExpectedLoc) + if (!Enabled || Tok != ExpectedLoc) return QualType(); if (!Type.isNull()) return Type; @@ -330,6 +329,7 @@ public: } private: + bool Enabled; /// Start position of a token for which we store expected type. SourceLocation ExpectedLoc; /// Expected type for a token starting at ExpectedLoc. diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp index 50e1f1eaba4d..97bd7d8fc51a 100644 --- a/clang/lib/Parse/ParseInit.cpp +++ b/clang/lib/Parse/ParseInit.cpp @@ -160,9 +160,6 @@ static void CheckArrayDesignatorSyntax(Parser &P, SourceLocation Loc, /// \p CodeCompleteCB is called with Designation parsed so far. ExprResult Parser::ParseInitializerWithPotentialDesignator( DesignatorCompletionInfo DesignatorCompletion) { - if (!getPreprocessor().isCodeCompletionEnabled()) - DesignatorCompletion.PreferredBaseType = QualType(); // skip field lookup - // If this is the old-style GNU extension: // designation ::= identifier ':' // Handle it as a field designator. Otherwise, this must be the start of a diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 9b0f921b4269..fb182883b88a 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -49,10 +49,10 @@ IdentifierInfo *Parser::getSEHExceptKeyword() { } Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies) - : PP(pp), Actions(actions), Diags(PP.getDiagnostics()), - GreaterThanIsOperator(true), ColonIsSacred(false), - InMessageExpression(false), TemplateParameterDepth(0), - ParsingInObjCContainer(false) { + : PP(pp), PreferredType(pp.isCodeCompletionEnabled()), Actions(actions), + Diags(PP.getDiagnostics()), GreaterThanIsOperator(true), + ColonIsSacred(false), InMessageExpression(false), + TemplateParameterDepth(0), ParsingInObjCContainer(false) { SkipFunctionBodies = pp.isCodeCompletionEnabled() || skipFunctionBodies; Tok.startToken(); Tok.setKind(tok::eof); diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index 2feb02bbe4ed..18605b321c70 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -381,6 +381,8 @@ public: } // namespace void PreferredTypeBuilder::enterReturn(Sema &S, SourceLocation Tok) { + if (!Enabled) + return; if (isa(S.CurContext)) { if (sema::BlockScopeInfo *BSI = S.getCurBlock()) { ComputeType = nullptr; @@ -399,6 +401,8 @@ void PreferredTypeBuilder::enterReturn(Sema &S, SourceLocation Tok) { } void PreferredTypeBuilder::enterVariableInit(SourceLocation Tok, Decl *D) { + if (!Enabled) + return; auto *VD = llvm::dyn_cast_or_null(D); ComputeType = nullptr; Type = VD ? VD->getType() : QualType(); @@ -410,6 +414,8 @@ static QualType getDesignatedType(QualType BaseType, const Designation &Desig); void PreferredTypeBuilder::enterDesignatedInitializer(SourceLocation Tok, QualType BaseType, const Designation &D) { + if (!Enabled) + return; ComputeType = nullptr; Type = getDesignatedType(BaseType, D); ExpectedLoc = Tok; @@ -417,6 +423,8 @@ void PreferredTypeBuilder::enterDesignatedInitializer(SourceLocation Tok, void PreferredTypeBuilder::enterFunctionArgument( SourceLocation Tok, llvm::function_ref ComputeType) { + if (!Enabled) + return; this->ComputeType = ComputeType; Type = QualType(); ExpectedLoc = Tok; @@ -424,6 +432,8 @@ void PreferredTypeBuilder::enterFunctionArgument( void PreferredTypeBuilder::enterParenExpr(SourceLocation Tok, SourceLocation LParLoc) { + if (!Enabled) + return; // expected type for parenthesized expression does not change. if (ExpectedLoc == LParLoc) ExpectedLoc = Tok; @@ -541,6 +551,8 @@ static QualType getPreferredTypeOfUnaryArg(Sema &S, QualType ContextType, void PreferredTypeBuilder::enterBinary(Sema &S, SourceLocation Tok, Expr *LHS, tok::TokenKind Op) { + if (!Enabled) + return; ComputeType = nullptr; Type = getPreferredTypeOfBinaryRHS(S, LHS, Op); ExpectedLoc = Tok; @@ -548,7 +560,7 @@ void PreferredTypeBuilder::enterBinary(Sema &S, SourceLocation Tok, Expr *LHS, void PreferredTypeBuilder::enterMemAccess(Sema &S, SourceLocation Tok, Expr *Base) { - if (!Base) + if (!Enabled || !Base) return; // Do we have expected type for Base? if (ExpectedLoc != Base->getBeginLoc()) @@ -561,6 +573,8 @@ void PreferredTypeBuilder::enterMemAccess(Sema &S, SourceLocation Tok, void PreferredTypeBuilder::enterUnary(Sema &S, SourceLocation Tok, tok::TokenKind OpKind, SourceLocation OpLoc) { + if (!Enabled) + return; ComputeType = nullptr; Type = getPreferredTypeOfUnaryArg(S, this->get(OpLoc), OpKind); ExpectedLoc = Tok; @@ -568,6 +582,8 @@ void PreferredTypeBuilder::enterUnary(Sema &S, SourceLocation Tok, void PreferredTypeBuilder::enterSubscript(Sema &S, SourceLocation Tok, Expr *LHS) { + if (!Enabled) + return; ComputeType = nullptr; Type = S.getASTContext().IntTy; ExpectedLoc = Tok; @@ -575,12 +591,16 @@ void PreferredTypeBuilder::enterSubscript(Sema &S, SourceLocation Tok, void PreferredTypeBuilder::enterTypeCast(SourceLocation Tok, QualType CastType) { + if (!Enabled) + return; ComputeType = nullptr; Type = !CastType.isNull() ? CastType.getCanonicalType() : QualType(); ExpectedLoc = Tok; } void PreferredTypeBuilder::enterCondition(Sema &S, SourceLocation Tok) { + if (!Enabled) + return; ComputeType = nullptr; Type = S.getASTContext().BoolTy; ExpectedLoc = Tok; -- GitLab From 43d0b1c9c16c7b435ae301d0a856fc48123e08c7 Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Thu, 11 Mar 2021 13:56:24 +0100 Subject: [PATCH 0013/1206] [clangd] Reject renames to non-identifier characters Differential Revision: https://reviews.llvm.org/D98424 --- clang-tools-extra/clangd/refactor/Rename.cpp | 32 ++++++++++++++++--- .../clangd/unittests/RenameTests.cpp | 15 +++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp index 853fc57bb906..5431046836ca 100644 --- a/clang-tools-extra/clangd/refactor/Rename.cpp +++ b/clang-tools-extra/clangd/refactor/Rename.cpp @@ -22,14 +22,17 @@ #include "clang/AST/DeclTemplate.h" #include "clang/AST/ParentMapContext.h" #include "clang/AST/Stmt.h" +#include "clang/Basic/CharInfo.h" #include "clang/Basic/LLVM.h" #include "clang/Basic/SourceLocation.h" #include "clang/Tooling/Syntax/Tokens.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Error.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/JSON.h" #include namespace clang { @@ -178,8 +181,7 @@ enum class ReasonToReject { UnsupportedSymbol, AmbiguousSymbol, - // name validation. - RenameToKeywords, + // name validation. FIXME: reconcile with InvalidName SameName, }; @@ -241,8 +243,6 @@ llvm::Error makeError(ReasonToReject Reason) { return "symbol is not a supported kind (e.g. namespace, macro)"; case ReasonToReject::AmbiguousSymbol: return "there are multiple symbols at the given location"; - case ReasonToReject::RenameToKeywords: - return "the chosen name is a keyword"; case ReasonToReject::SameName: return "new name is the same as the old name"; } @@ -437,6 +437,7 @@ struct InvalidName { enum Kind { Keywords, Conflict, + BadIdentifier, }; Kind K; std::string Details; @@ -447,6 +448,8 @@ std::string toString(InvalidName::Kind K) { return "Keywords"; case InvalidName::Conflict: return "Conflict"; + case InvalidName::BadIdentifier: + return "BadIdentifier"; } llvm_unreachable("unhandled InvalidName kind"); } @@ -459,12 +462,31 @@ llvm::Error makeError(InvalidName Reason) { Reason.Details); case InvalidName::Conflict: return llvm::formatv("conflict with the symbol in {0}", Reason.Details); + case InvalidName::BadIdentifier: + return llvm::formatv("the chosen name \"{0}\" is not a valid identifier", + Reason.Details); } llvm_unreachable("unhandled InvalidName kind"); }; return error("invalid name: {0}", Message(Reason)); } +static bool mayBeValidIdentifier(llvm::StringRef Ident) { + assert(llvm::json::isUTF8(Ident)); + if (Ident.empty()) + return false; + // We don't check all the rules for non-ascii characters (most are allowed). + bool AllowDollar = true; // lenient + if (llvm::isASCII(Ident.front()) && + !isIdentifierHead(Ident.front(), AllowDollar)) + return false; + for (char C : Ident) { + if (llvm::isASCII(C) && !isIdentifierBody(C, AllowDollar)) + return false; + } + return true; +} + // Check if we can rename the given RenameDecl into NewName. // Return details if the rename would produce a conflict. llvm::Optional checkName(const NamedDecl &RenameDecl, @@ -476,6 +498,8 @@ llvm::Optional checkName(const NamedDecl &RenameDecl, llvm::Optional Result; if (isKeyword(NewName, ASTCtx.getLangOpts())) Result = InvalidName{InvalidName::Keywords, NewName.str()}; + else if (!mayBeValidIdentifier(NewName)) + Result = InvalidName{InvalidName::BadIdentifier, NewName.str()}; else { // Name conflict detection. // Function conflicts are subtle (overloading), so ignore them. diff --git a/clang-tools-extra/clangd/unittests/RenameTests.cpp b/clang-tools-extra/clangd/unittests/RenameTests.cpp index ca0e7ff24306..5b35ac00d888 100644 --- a/clang-tools-extra/clangd/unittests/RenameTests.cpp +++ b/clang-tools-extra/clangd/unittests/RenameTests.cpp @@ -1240,6 +1240,21 @@ TEST(RenameTest, PrepareRename) { testing::HasSubstr("keyword")); EXPECT_THAT(Tracer.takeMetric("rename_name_invalid", "Keywords"), ElementsAre(1)); + + for (std::string BadIdent : {"foo!bar", "123foo", "😀@"}) { + Results = runPrepareRename(Server, FooCCPath, FooCC.point(), + /*NewName=*/BadIdent, {}); + EXPECT_FALSE(Results); + EXPECT_THAT(llvm::toString(Results.takeError()), + testing::HasSubstr("identifier")); + EXPECT_THAT(Tracer.takeMetric("rename_name_invalid", "BadIdentifier"), + ElementsAre(1)); + } + for (std::string GoodIdent : {"fooBar", "__foo$", "😀"}) { + Results = runPrepareRename(Server, FooCCPath, FooCC.point(), + /*NewName=*/GoodIdent, {}); + EXPECT_TRUE(bool(Results)); + } } TEST(CrossFileRenameTests, DirtyBuffer) { -- GitLab From 953bb5e5c8f60dc769942a3615d800fe166ffd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Tue, 16 Mar 2021 12:13:31 +0100 Subject: [PATCH 0014/1206] [test] Make sure the test program in GetErrcMessages.cmake exits normally. If for some reason the test program does not exit normally it'd currently lead to a false positive and it's stdout output being assigned to the output variable. Instead, check the test program exited normally before assigning the process output to the out variable. Follow up on rGaf2796c76d2ff4b73165ed47959afd35a769beee Fixes an issue discovered post commit in https://reviews.llvm.org/D98278 --- llvm/cmake/modules/GetErrcMessages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/cmake/modules/GetErrcMessages.cmake b/llvm/cmake/modules/GetErrcMessages.cmake index 908b1f538b95..2db1e0304ba0 100644 --- a/llvm/cmake/modules/GetErrcMessages.cmake +++ b/llvm/cmake/modules/GetErrcMessages.cmake @@ -29,7 +29,7 @@ function(get_errc_messages outvar) ${errc_test_code} RUN_OUTPUT_VARIABLE errc_result COMPILE_OUTPUT_VARIABLE errc_compile_errors) - if (errc_compiled) + if (errc_compiled AND "${errc_exit_code}" STREQUAL "0") set(${outvar} ${errc_result} PARENT_SCOPE) else() set(${outvar} "" PARENT_SCOPE) -- GitLab From 3b99731c4e7bb844699eda6640bd99344f800c79 Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Thu, 11 Mar 2021 13:43:59 +0100 Subject: [PATCH 0015/1206] [clangd] Turn off implicit cancellation based on client capabilities Capability is in upcoming 3.17: https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/ (This is also useful for C++ embedders) Differential Revision: https://reviews.llvm.org/D98414 --- clang-tools-extra/clangd/ClangdLSPServer.cpp | 1 + clang-tools-extra/clangd/ClangdServer.cpp | 18 +++++++++--------- clang-tools-extra/clangd/ClangdServer.h | 8 ++++++++ clang-tools-extra/clangd/Protocol.cpp | 6 ++++++ clang-tools-extra/clangd/Protocol.h | 4 ++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index cd13e013aa50..b4a5cf337296 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -518,6 +518,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params, if (Params.capabilities.WorkDoneProgress) BackgroundIndexProgressState = BackgroundIndexProgress::Empty; BackgroundIndexSkipCreate = Params.capabilities.ImplicitProgressCreation; + Opts.ImplicitCancellation = !Params.capabilities.CancelsStaleRequests; llvm::json::Object ServerCaps{ {"textDocumentSync", diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 164e387bd454..e9724e7516aa 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -150,6 +150,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, DynamicIdx(Opts.BuildDynamicSymbolIndex ? new FileIndex() : nullptr), ClangTidyProvider(Opts.ClangTidyProvider), WorkspaceRoot(Opts.WorkspaceRoot), + Transient(Opts.ImplicitCancellation ? TUScheduler::InvalidateOnUpdate + : TUScheduler::NoInvalidation), DirtyFS(std::make_unique(TFS, DraftMgr)) { // Pass a callback into `WorkScheduler` to extract symbols from a newly // parsed file and rebuild the file index synchronously each time an AST @@ -593,7 +595,7 @@ void ClangdServer::enumerateTweaks( }; WorkScheduler->runWithAST("EnumerateTweaks", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + Transient); } void ClangdServer::applyTweak(PathRef File, Range Sel, StringRef TweakID, @@ -683,8 +685,7 @@ void ClangdServer::findDocumentHighlights( CB(clangd::findDocumentHighlights(InpAST->AST, Pos)); }; - WorkScheduler->runWithAST("Highlights", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + WorkScheduler->runWithAST("Highlights", File, std::move(Action), Transient); } void ClangdServer::findHover(PathRef File, Position Pos, @@ -698,8 +699,7 @@ void ClangdServer::findHover(PathRef File, Position Pos, CB(clangd::getHover(InpAST->AST, Pos, std::move(Style), Index)); }; - WorkScheduler->runWithAST("Hover", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + WorkScheduler->runWithAST("Hover", File, std::move(Action), Transient); } void ClangdServer::typeHierarchy(PathRef File, Position Pos, int Resolve, @@ -771,7 +771,7 @@ void ClangdServer::documentSymbols(llvm::StringRef File, CB(clangd::getDocumentSymbols(InpAST->AST)); }; WorkScheduler->runWithAST("DocumentSymbols", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + Transient); } void ClangdServer::foldingRanges(llvm::StringRef File, @@ -783,7 +783,7 @@ void ClangdServer::foldingRanges(llvm::StringRef File, CB(clangd::getFoldingRanges(InpAST->AST)); }; WorkScheduler->runWithAST("FoldingRanges", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + Transient); } void ClangdServer::findImplementations( @@ -850,7 +850,7 @@ void ClangdServer::documentLinks(PathRef File, CB(clangd::getDocumentLinks(InpAST->AST)); }; WorkScheduler->runWithAST("DocumentLinks", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + Transient); } void ClangdServer::semanticHighlights( @@ -862,7 +862,7 @@ void ClangdServer::semanticHighlights( CB(clangd::getSemanticHighlightings(InpAST->AST)); }; WorkScheduler->runWithAST("SemanticHighlights", File, std::move(Action), - TUScheduler::InvalidateOnUpdate); + Transient); } void ClangdServer::getAST(PathRef File, Range R, diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h index e76ef65922ee..b633d3139683 100644 --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -146,6 +146,12 @@ public: /*RebuildRatio=*/1, }; + /// Cancel certain requests if the file changes before they begin running. + /// This is useful for "transient" actions like enumerateTweaks that were + /// likely implicitly generated, and avoids redundant work if clients forget + /// to cancel. Clients that always cancel stale requests should clear this. + bool ImplicitCancellation = true; + /// Clangd will execute compiler drivers matching one of these globs to /// fetch system include path. std::vector QueryDriverGlobs; @@ -391,6 +397,8 @@ private: llvm::Optional WorkspaceRoot; llvm::Optional WorkScheduler; + // Invalidation policy used for actions that we assume are "transient". + TUScheduler::ASTActionInvalidation Transient; // Store of the current versions of the open documents. // Only written from the main thread (despite being threadsafe). diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp index b3ff124df4de..42ca721ebcbb 100644 --- a/clang-tools-extra/clangd/Protocol.cpp +++ b/clang-tools-extra/clangd/Protocol.cpp @@ -414,6 +414,12 @@ bool fromJSON(const llvm::json::Value &Params, ClientCapabilities &R, if (auto Implicit = Window->getBoolean("implicitWorkDoneProgressCreate")) R.ImplicitProgressCreation = *Implicit; } + if (auto *General = O->getObject("general")) { + if (auto *StaleRequestSupport = General->getObject("staleRequestSupport")) { + if (auto Cancel = StaleRequestSupport->getBoolean("cancel")) + R.CancelsStaleRequests = *Cancel; + } + } if (auto *OffsetEncoding = O->get("offsetEncoding")) { R.offsetEncoding.emplace(); if (!fromJSON(*OffsetEncoding, *R.offsetEncoding, diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h index c6074abcb04e..1334ddf4b5ce 100644 --- a/clang-tools-extra/clangd/Protocol.h +++ b/clang-tools-extra/clangd/Protocol.h @@ -475,6 +475,10 @@ struct ClientCapabilities { /// window.implicitWorkDoneProgressCreate bool ImplicitProgressCreation = false; + /// Whether the client claims to cancel stale requests. + /// general.staleRequestSupport.cancel + bool CancelsStaleRequests = false; + /// Whether the client implementation supports a refresh request sent from the /// server to the client. bool SemanticTokenRefreshSupport = false; -- GitLab From ca13f5595ae8dc7326f29c8658de70bbc1854db0 Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Tue, 2 Mar 2021 22:16:29 +0100 Subject: [PATCH 0016/1206] [clangd] Add `limit` extension on completion and workspace-symbols This overrides the --limit-results command-line flag, and is not constrained by it. See https://github.com/clangd/clangd/issues/707 Differential Revision: https://reviews.llvm.org/D97801 --- clang-tools-extra/clangd/ClangdLSPServer.cpp | 35 +++++++++++--------- clang-tools-extra/clangd/Protocol.cpp | 6 ++-- clang-tools-extra/clangd/Protocol.h | 11 +++++- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index b4a5cf337296..aef849d8d8d9 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -780,7 +780,7 @@ void ClangdLSPServer::onWorkspaceSymbol( const WorkspaceSymbolParams &Params, Callback> Reply) { Server->workspaceSymbols( - Params.query, Opts.CodeComplete.Limit, + Params.query, Params.limit.getValueOr(Opts.CodeComplete.Limit), [Reply = std::move(Reply), this](llvm::Expected> Items) mutable { if (!Items) @@ -1031,21 +1031,24 @@ void ClangdLSPServer::onCompletion(const CompletionParams &Params, vlog("ignored auto-triggered completion, preceding char did not match"); return Reply(CompletionList()); } - Server->codeComplete( - Params.textDocument.uri.file(), Params.position, Opts.CodeComplete, - [Reply = std::move(Reply), - this](llvm::Expected List) mutable { - if (!List) - return Reply(List.takeError()); - CompletionList LSPList; - LSPList.isIncomplete = List->HasMore; - for (const auto &R : List->Completions) { - CompletionItem C = R.render(Opts.CodeComplete); - C.kind = adjustKindToCapability(C.kind, SupportedCompletionItemKinds); - LSPList.items.push_back(std::move(C)); - } - return Reply(std::move(LSPList)); - }); + auto Opts = this->Opts.CodeComplete; + if (Params.limit && *Params.limit >= 0) + Opts.Limit = *Params.limit; + Server->codeComplete(Params.textDocument.uri.file(), Params.position, Opts, + [Reply = std::move(Reply), Opts, + this](llvm::Expected List) mutable { + if (!List) + return Reply(List.takeError()); + CompletionList LSPList; + LSPList.isIncomplete = List->HasMore; + for (const auto &R : List->Completions) { + CompletionItem C = R.render(Opts); + C.kind = adjustKindToCapability( + C.kind, SupportedCompletionItemKinds); + LSPList.items.push_back(std::move(C)); + } + return Reply(std::move(LSPList)); + }); } void ClangdLSPServer::onSignatureHelp(const TextDocumentPositionParams &Params, diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp index 42ca721ebcbb..099c8531d341 100644 --- a/clang-tools-extra/clangd/Protocol.cpp +++ b/clang-tools-extra/clangd/Protocol.cpp @@ -750,7 +750,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &O, const SymbolDetails &S) { bool fromJSON(const llvm::json::Value &Params, WorkspaceSymbolParams &R, llvm::json::Path P) { llvm::json::ObjectMapper O(Params, P); - return O && O.map("query", R.query); + return O && O.map("query", R.query) && + mapOptOrNull(Params, "limit", R.limit, P); } llvm::json::Value toJSON(const Command &C) { @@ -851,7 +852,8 @@ bool fromJSON(const llvm::json::Value &Params, CompletionContext &R, bool fromJSON(const llvm::json::Value &Params, CompletionParams &R, llvm::json::Path P) { - if (!fromJSON(Params, static_cast(R), P)) + if (!fromJSON(Params, static_cast(R), P) || + !mapOptOrNull(Params, "limit", R.limit, P)) return false; if (auto *Context = Params.getAsObject()->get("context")) return fromJSON(*Context, R.context, P.field("context")); diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h index 1334ddf4b5ce..8e90f1f47831 100644 --- a/clang-tools-extra/clangd/Protocol.h +++ b/clang-tools-extra/clangd/Protocol.h @@ -1056,8 +1056,13 @@ bool operator==(const SymbolDetails &, const SymbolDetails &); /// The parameters of a Workspace Symbol Request. struct WorkspaceSymbolParams { - /// A non-empty query string + /// A query string to filter symbols by. + /// Clients may send an empty string here to request all the symbols. std::string query; + + /// Max results to return, overriding global default. 0 means no limit. + /// Clangd extension. + llvm::Optional limit; }; bool fromJSON(const llvm::json::Value &, WorkspaceSymbolParams &, llvm::json::Path); @@ -1106,6 +1111,10 @@ bool fromJSON(const llvm::json::Value &, CompletionContext &, llvm::json::Path); struct CompletionParams : TextDocumentPositionParams { CompletionContext context; + + /// Max results to return, overriding global default. 0 means no limit. + /// Clangd extension. + llvm::Optional limit; }; bool fromJSON(const llvm::json::Value &, CompletionParams &, llvm::json::Path); -- GitLab From 40fdb43d300ceb8609f9e6a513cbaaf5924080a2 Mon Sep 17 00:00:00 2001 From: Sanjay Patel Date: Tue, 16 Mar 2021 06:53:08 -0400 Subject: [PATCH 0017/1206] [SLP] improve readability in reduction logic; NFC We had 2 different and ambiguously-named 'I' variables. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index c9f33edfb644..02d93fa4260d 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -6692,6 +6692,7 @@ class HorizontalReduction { /// Expected number of uses for reduction operations/reduced values. static bool hasRequiredNumberOfUses(RecurKind Kind, Instruction *I, bool IsReductionOp) { + assert(Kind != RecurKind::None && "Reduction type not set"); // SelectInst must be used twice while the condition op must have single // use only. if (isCmpSel(Kind)) @@ -6795,8 +6796,8 @@ public: if (IsReducedValue) ReducedVals.push_back(TreeN); else { - auto I = ExtraArgs.find(TreeN); - if (I != ExtraArgs.end() && !I->second) { + auto ExtraArgsIter = ExtraArgs.find(TreeN); + if (ExtraArgsIter != ExtraArgs.end() && !ExtraArgsIter->second) { // Check if TreeN is an extra argument of its parent operation. if (Stack.size() <= 1) { // TreeN can't be an extra argument as it is a root reduction @@ -6818,14 +6819,14 @@ public: // Visit left or right. Value *EdgeVal = TreeN->getOperand(EdgeToVisit); - auto *I = dyn_cast(EdgeVal); - if (!I) { + auto *EdgeInst = dyn_cast(EdgeVal); + if (!EdgeInst) { // Edge value is not a reduction instruction or a leaf instruction. // (It may be a constant, function argument, or something else.) markExtraArg(Stack.back(), EdgeVal); continue; } - RecurKind EdgeRdxKind = getRdxKind(I); + RecurKind EdgeRdxKind = getRdxKind(EdgeInst); // Continue analysis if the next operand is a reduction operation or // (possibly) a leaf value. If the leaf value opcode is not set, // the first met operation != reduction operation is considered as the @@ -6834,25 +6835,26 @@ public: // Each tree node needs to have minimal number of users except for the // ultimate reduction. const bool IsRdxInst = EdgeRdxKind == RdxKind; - if (I != Phi && I != B && - hasSameParent(RdxKind, I, B->getParent(), IsRdxInst) && - hasRequiredNumberOfUses(RdxKind, I, IsRdxInst) && - (!LeafOpcode || LeafOpcode == I->getOpcode() || IsRdxInst)) { + if (EdgeInst != Phi && EdgeInst != B && + hasSameParent(RdxKind, EdgeInst, B->getParent(), IsRdxInst) && + hasRequiredNumberOfUses(RdxKind, EdgeInst, IsRdxInst) && + (!LeafOpcode || LeafOpcode == EdgeInst->getOpcode() || IsRdxInst)) { if (IsRdxInst) { // We need to be able to reassociate the reduction operations. - if (!isVectorizable(EdgeRdxKind, I)) { + if (!isVectorizable(EdgeRdxKind, EdgeInst)) { // I is an extra argument for TreeN (its parent operation). - markExtraArg(Stack.back(), I); + markExtraArg(Stack.back(), EdgeInst); continue; } } else if (!LeafOpcode) { - LeafOpcode = I->getOpcode(); + LeafOpcode = EdgeInst->getOpcode(); } - Stack.push_back(std::make_pair(I, getFirstOperandIndex(EdgeRdxKind))); + Stack.push_back( + std::make_pair(EdgeInst, getFirstOperandIndex(EdgeRdxKind))); continue; } // I is an extra argument for TreeN (its parent operation). - markExtraArg(Stack.back(), I); + markExtraArg(Stack.back(), EdgeInst); } return true; } -- GitLab From 5ac3b37599d3da80887033df66ecea4aea4dc347 Mon Sep 17 00:00:00 2001 From: Bjorn Pettersson Date: Fri, 12 Mar 2021 12:24:43 +0100 Subject: [PATCH 0018/1206] [TableGen/GlobalISel] Emit MI_predicate custom code for PatFrags (not only PatFrag) When GlobalISelEmitter::emitCxxPredicateFns emitted code for MI predicates it used "PatFrag" when searching for definitions. With this patch it will search for all "PatFrags" instead. Since PatFrag derives from PatFrags the difference is that we now include all definitions using PatFrags directly as well. Thus making it possible to use GISelPredicateCode together with a PatFrags definition. It might be noted that the matcher code was emitted also for PatFrags in the past. But then one ended up with errors since the custom code in testMIPredicate_MI was missing. Differential Revision: https://reviews.llvm.org/D98486 --- .../GlobalISelEmitterCustomPredicate.td | 75 ++++++++++++++++--- llvm/utils/TableGen/GlobalISelEmitter.cpp | 2 +- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td index 6f6320f6389d..408055da34c9 100644 --- a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td +++ b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td @@ -1,5 +1,27 @@ // RUN: llvm-tblgen %s -gen-global-isel -optimize-match-table=false -I %p/../../include -I %p/Common -o - | FileCheck %s +// Verify that all MI predicates are enumerated. +// +// CHECK: // PatFrag predicates. +// CHECK-NEXT: enum { +// CHECK-NEXT: GIPFP_MI_Predicate_and_or_pat = GIPFP_MI_Invalid + 1, +// CHECK-NEXT: GIPFP_MI_Predicate_or_oneuse, +// CHECK-NEXT: GIPFP_MI_Predicate_patfrags_test_pat, +// CHECK-NEXT: GIPFP_MI_Predicate_sub3_pat, +// CHECK-NEXT: }; + +// Verify that we emit cases for all MI predicates. +// +// CHECK: bool MyTargetInstructionSelector::testMIPredicate_MI( +// CHECK: case GIPFP_MI_Predicate_and_or_pat: { +// CHECK: llvm_unreachable("GISelPredicateCode should have returned"); +// CHECK: case GIPFP_MI_Predicate_or_oneuse: { +// CHECK: llvm_unreachable("GISelPredicateCode should have returned"); +// CHECK: case GIPFP_MI_Predicate_patfrags_test_pat: { +// CHECK: llvm_unreachable("GISelPredicateCode should have returned"); +// CHECK: case GIPFP_MI_Predicate_sub3_pat: { +// CHECK: llvm_unreachable("GISelPredicateCode should have returned"); + include "llvm/Target/Target.td" include "GlobalISelEmitterCommon.td" @@ -28,7 +50,6 @@ def DRegs : MyClass<32, [i32], (sequence "D%u", 0, 0)>; def DOP : RegisterOperand; def AND_OR : I<(outs DRegs:$dst), (ins DOP:$src0, DOP:$src1, DOP:$src2), []>; - def or_oneuse : PatFrag< (ops node:$x, node:$y), (or node:$x, node:$y), [{ return foo(); }]> { @@ -48,7 +69,7 @@ def and_or_pat : PatFrag< let PredicateCodeUsesOperands = 1; } -// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 2 // +// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 6 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, // CHECK-NEXT: // MIs[0] dst @@ -56,7 +77,7 @@ def and_or_pat : PatFrag< // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:2:z +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:3:z // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, @@ -67,18 +88,18 @@ def and_or_pat : PatFrag< // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[1] src0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:2:x +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[1] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:2:y +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:2:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:2:x, DOP:{ *:[i32] }:$src1:$pred:2:y))<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) +// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:3:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y))<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR, -// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 1 // +// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 3 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, // CHECK-NEXT: // MIs[0] dst @@ -93,19 +114,19 @@ def and_or_pat : PatFrag< // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[1] src0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:2:x +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[1] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:2:y +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:2:z +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:3:z // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:2:x, DOP:{ *:[i32] }:$src1:$pred:2:y), DOP:{ *:[i32] }:$src2:$pred:2:z)<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) +// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y), DOP:{ *:[i32] }:$src2:$pred:3:z)<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR, // Test commutative, standalone pattern. @@ -157,3 +178,35 @@ def SUB3 : I<(outs DRegs:$dst), (ins DOP:$src0, DOP:$src1, DOP:$src2), [(set DRegs:$dst, (sub3_pat i32:$src0, i32:$src1, i32:$src2))] >; + + +def patfrags_test_pat : PatFrags< + (ops node:$x, node:$y, node:$z), + [ (xor (add node:$x, node:$y), node:$z), + (xor (sub node:$x, node:$y), node:$z) + ], [{ return foo(); }]> { + let GISelPredicateCode = [{ + return doesComplexCheck(MI); + }]; + + let PredicateCodeUsesOperands = 1; +} + +// CHECK: GIM_Try, /*On fail goto*//*Label 3*/ 372, // Rule ID 1 // +// CHECK: // (xor:{ *:[i32] } (add:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y), i32:{ *:[i32] }:$src2:$pred:2:z)<> => (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) + +// CHECK: GIM_Try, /*On fail goto*//*Label 4*/ 459, // Rule ID 2 // +// CHECK: // (xor:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y), i32:{ *:[i32] }:$src2:$pred:2:z)<> => (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) + +// CHECK: GIM_Try, /*On fail goto*//*Label 5*/ 546, // Rule ID 4 // +// CHECK: // (xor:{ *:[i32] } i32:{ *:[i32] }:$src2:$pred:2:z, (add:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y))<> => (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) + +// CHECK: GIM_Try, /*On fail goto*//*Label 6*/ 633, // Rule ID 5 // +// CHECK: // (xor:{ *:[i32] } i32:{ *:[i32] }:$src2:$pred:2:z, (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y))<> => (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) + + +// Test a commutative pattern using multiple patterns using PatFrags. +def PATFRAGS : I<(outs DRegs:$dst), + (ins DOP:$src0, DOP:$src1, DOP:$src2), + [(set DRegs:$dst, (patfrags_test_pat i32:$src0, i32:$src1, i32:$src2))] +>; diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 3113198a65e7..8b0d8a663892 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -5351,7 +5351,7 @@ void GlobalISelEmitter::emitCxxPredicateFns( StringRef AdditionalDeclarations, std::function Filter) { std::vector MatchedRecords; - const auto &Defs = RK.getAllDerivedDefinitions("PatFrag"); + const auto &Defs = RK.getAllDerivedDefinitions("PatFrags"); std::copy_if(Defs.begin(), Defs.end(), std::back_inserter(MatchedRecords), [&](Record *Record) { return !Record->getValueAsString(CodeFieldName).empty() && -- GitLab From 128ce70eef9948b81e725fd0e2ed46a7c004a118 Mon Sep 17 00:00:00 2001 From: Sam McCall Date: Fri, 12 Mar 2021 12:00:54 +0100 Subject: [PATCH 0019/1206] [CodeCompletion] Avoid spurious signature help for init-list args Somewhat surprisingly, signature help is emitted as a side-effect of computing the expected type of a function argument. The reason is that both actions require enumerating the possible function signatures and running partial overload resolution, and doing this twice would be wasteful and complicated. Change #1: document this, it's subtle :-) However, sometimes we need to compute the expected type without having reached the code completion cursor yet - in particular to allow completion of designators. eb4ab3358cd4dc834a761191b5531b38114f7b13 did this but introduced a regression - it emits signature help in the wrong location as a side-effect. Change #2: only emit signature help if the code completion cursor was reached. Currently there is PP.isCodeCompletionReached(), but we can't use it because it's set *after* running code completion. It'd be nice to set this implicitly when the completion token is lexed, but ConsumeCodeCompletionToken() makes this complicated. Change #3: call cutOffParsing() *first* when seeing a completion token. After this, the fact that the Sema::Produce*SignatureHelp() functions are even more confusing, as they only sometimes do that. I don't want to rename them in this patch as it's another large mechanical change, but we should soon. Change #4: prepare to rename ProduceSignatureHelp() to GuessArgumentType() etc. Differential Revision: https://reviews.llvm.org/D98488 --- .../clangd/unittests/CodeCompleteTests.cpp | 13 ++++ clang/include/clang/Sema/Sema.h | 19 ++++- clang/lib/Lex/PPDirectives.cpp | 4 +- clang/lib/Lex/Preprocessor.cpp | 4 +- clang/lib/Parse/ParseDecl.cpp | 19 +++-- clang/lib/Parse/ParseDeclCXX.cpp | 16 ++-- clang/lib/Parse/ParseExpr.cpp | 17 +++-- clang/lib/Parse/ParseExprCXX.cpp | 14 ++-- clang/lib/Parse/ParseInit.cpp | 2 +- clang/lib/Parse/ParseObjc.cpp | 74 ++++++++++--------- clang/lib/Parse/ParseOpenMP.cpp | 2 +- clang/lib/Parse/ParseStmt.cpp | 14 ++-- clang/lib/Parse/Parser.cpp | 10 +-- clang/lib/Sema/SemaCodeComplete.cpp | 5 +- clang/test/CodeCompletion/desig-init.cpp | 15 ++++ 15 files changed, 144 insertions(+), 84 deletions(-) diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp index 0ff1e83b7613..a57ae49f9159 100644 --- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp +++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp @@ -1253,6 +1253,19 @@ TEST(SignatureHelpTest, Overloads) { EXPECT_EQ(0, Results.activeParameter); } +TEST(SignatureHelpTest, OverloadInitListRegression) { + auto Results = signatures(R"cpp( + struct A {int x;}; + struct B {B(A);}; + void f(); + int main() { + B b({1}); + f(^); + } + )cpp"); + EXPECT_THAT(Results.signatures, UnorderedElementsAre(Sig("f() -> void"))); +} + TEST(SignatureHelpTest, DefaultArgs) { auto Results = signatures(R"cpp( void bar(int x, int y = 0); diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 9e3eb4f07472..79e2471fdabe 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -306,6 +306,9 @@ public: /// Clients should be very careful when using this funciton, as it stores a /// function_ref, clients should make sure all calls to get() with the same /// location happen while function_ref is alive. + /// + /// The callback should also emit signature help as a side-effect, but only + /// if the completion point has been reached. void enterFunctionArgument(SourceLocation Tok, llvm::function_ref ComputeType); @@ -318,6 +321,12 @@ public: /// Handles all type casts, including C-style cast, C++ casts, etc. void enterTypeCast(SourceLocation Tok, QualType CastType); + /// Get the expected type associated with this location, if any. + /// + /// If the location is a function argument, determining the expected type + /// involves considering all function overloads and the arguments so far. + /// In this case, signature help for these function overloads will be reported + /// as a side-effect (only if the completion point has been reached). QualType get(SourceLocation Tok) const { if (!Enabled || Tok != ExpectedLoc) return QualType(); @@ -12216,8 +12225,14 @@ public: const VirtSpecifiers *VS = nullptr); void CodeCompleteBracketDeclarator(Scope *S); void CodeCompleteCase(Scope *S); - /// Reports signatures for a call to CodeCompleteConsumer and returns the - /// preferred type for the current argument. Returned type can be null. + /// Determines the preferred type of the current function argument, by + /// examining the signatures of all possible overloads. + /// Returns null if unknown or ambiguous, or if code completion is off. + /// + /// If the code completion point has been reached, also reports the function + /// signatures that were considered. + /// + /// FIXME: rename to GuessCallArgumentType to reduce confusion. QualType ProduceCallSignatureHelp(Scope *S, Expr *Fn, ArrayRef Args, SourceLocation OpenParLoc); QualType ProduceConstructorSignatureHelp(Scope *S, QualType Type, diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index c854d3e9c02b..f04d896247c9 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -441,9 +441,9 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, CurLexer->Lex(Tok); if (Tok.is(tok::code_completion)) { + setCodeCompletionReached(); if (CodeComplete) CodeComplete->CodeCompleteInConditionalExclusion(); - setCodeCompletionReached(); continue; } @@ -966,10 +966,10 @@ void Preprocessor::HandleDirective(Token &Result) { case tok::eod: return; // null directive. case tok::code_completion: + setCodeCompletionReached(); if (CodeComplete) CodeComplete->CodeCompleteDirective( CurPPLexer->getConditionalStackDepth() > 0); - setCodeCompletionReached(); return; case tok::numeric_constant: // # 7 GNU line marker directive. if (getLangOpts().AsmPreprocessor) diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp index 177786d90390..e39b78d5ffec 100644 --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -442,15 +442,15 @@ bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled) { + setCodeCompletionReached(); if (CodeComplete) CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled); - setCodeCompletionReached(); } void Preprocessor::CodeCompleteNaturalLanguage() { + setCodeCompletionReached(); if (CodeComplete) CodeComplete->CodeCompleteNaturalLanguage(); - setCodeCompletionReached(); } /// getSpelling - This method is used to get the spelling of a token into a diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 9edf4d3d614a..a044fbc3039c 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -1970,8 +1970,8 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS, // Check to see if we have a function *definition* which must have a body. if (D.isFunctionDeclarator()) { if (Tok.is(tok::equal) && NextToken().is(tok::code_completion)) { - Actions.CodeCompleteAfterFunctionEquals(D); cutOffParsing(); + Actions.CodeCompleteAfterFunctionEquals(D); return nullptr; } // Look at the next token to make sure that this isn't a function @@ -2310,9 +2310,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes( InitializerScopeRAII InitScope(*this, D, ThisDecl); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteInitializer(getCurScope(), ThisDecl); Actions.FinalizeDeclaration(ThisDecl); - cutOffParsing(); return nullptr; } @@ -3090,10 +3090,11 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, = DSContext == DeclSpecContext::DSC_top_level || (DSContext == DeclSpecContext::DSC_class && DS.isFriendSpecified()); + cutOffParsing(); Actions.CodeCompleteDeclSpec(getCurScope(), DS, AllowNonIdentifiers, AllowNestedNameSpecifiers); - return cutOffParsing(); + return; } if (getCurScope()->getFnParent() || getCurScope()->getBlockParent()) @@ -3106,8 +3107,9 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS, else if (CurParsedObjCImpl) CCC = Sema::PCC_ObjCImplementation; + cutOffParsing(); Actions.CodeCompleteOrdinaryName(getCurScope(), CCC); - return cutOffParsing(); + return; } case tok::coloncolon: // ::foo::bar @@ -4362,8 +4364,9 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS, // Parse the tag portion of this. if (Tok.is(tok::code_completion)) { // Code completion for an enum name. + cutOffParsing(); Actions.CodeCompleteTag(getCurScope(), DeclSpec::TST_enum); - return cutOffParsing(); + return; } // If attributes exist after tag, parse them. @@ -5457,11 +5460,12 @@ void Parser::ParseTypeQualifierListOpt( switch (Tok.getKind()) { case tok::code_completion: + cutOffParsing(); if (CodeCompletionHandler) (*CodeCompletionHandler)(); else Actions.CodeCompleteTypeQualifiers(DS); - return cutOffParsing(); + return; case tok::kw_const: isInvalid = DS.SetTypeQual(DeclSpec::TQ_const , Loc, PrevSpec, DiagID, @@ -6998,8 +7002,9 @@ void Parser::ParseBracketDeclarator(Declarator &D) { std::move(attrs), T.getCloseLocation()); return; } else if (Tok.getKind() == tok::code_completion) { + cutOffParsing(); Actions.CodeCompleteBracketDeclarator(getCurScope()); - return cutOffParsing(); + return; } // If valid, this location is the position where we read the 'static' keyword. diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index dd1cccf72668..0e9bc42bfcb8 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -63,8 +63,8 @@ Parser::DeclGroupPtrTy Parser::ParseNamespace(DeclaratorContext Context, ObjCDeclContextSwitch ObjCDC(*this); if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteNamespaceDecl(getCurScope()); cutOffParsing(); + Actions.CodeCompleteNamespaceDecl(getCurScope()); return nullptr; } @@ -283,8 +283,8 @@ Decl *Parser::ParseNamespaceAlias(SourceLocation NamespaceLoc, ConsumeToken(); // eat the '='. if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteNamespaceAliasDecl(getCurScope()); cutOffParsing(); + Actions.CodeCompleteNamespaceAliasDecl(getCurScope()); return nullptr; } @@ -471,8 +471,8 @@ Parser::ParseUsingDirectiveOrDeclaration(DeclaratorContext Context, SourceLocation UsingLoc = ConsumeToken(); if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteUsing(getCurScope()); cutOffParsing(); + Actions.CodeCompleteUsing(getCurScope()); return nullptr; } @@ -525,8 +525,8 @@ Decl *Parser::ParseUsingDirective(DeclaratorContext Context, SourceLocation NamespcLoc = ConsumeToken(); if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteUsingDirective(getCurScope()); cutOffParsing(); + Actions.CodeCompleteUsingDirective(getCurScope()); return nullptr; } @@ -1433,8 +1433,9 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind, if (Tok.is(tok::code_completion)) { // Code completion for a struct, class, or union name. + cutOffParsing(); Actions.CodeCompleteTag(getCurScope(), TagType); - return cutOffParsing(); + return; } // C++03 [temp.explicit] 14.7.2/8: @@ -2749,8 +2750,8 @@ Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS, else if (KW.is(tok::kw_delete)) DefinitionKind = FunctionDefinitionKind::Deleted; else if (KW.is(tok::code_completion)) { - Actions.CodeCompleteAfterFunctionEquals(DeclaratorInfo); cutOffParsing(); + Actions.CodeCompleteAfterFunctionEquals(DeclaratorInfo); return nullptr; } } @@ -3498,9 +3499,10 @@ void Parser::ParseConstructorInitializer(Decl *ConstructorDecl) { do { if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteConstructorInitializer(ConstructorDecl, MemInitializers); - return cutOffParsing(); + return; } MemInitResult MemInit = ParseMemInitializer(ConstructorDecl); diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index c417985cbe34..c2b47f6375b8 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -159,9 +159,9 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) { /// Parse an expr that doesn't include (top-level) commas. ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) { if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteExpression(getCurScope(), PreferredType.get(Tok.getLocation())); - cutOffParsing(); return ExprError(); } @@ -1156,9 +1156,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, ConsumeToken(); if (Tok.is(tok::code_completion) && &II != Ident_super) { + cutOffParsing(); Actions.CodeCompleteObjCClassPropertyRefExpr( getCurScope(), II, ILoc, ExprStatementTokLoc == ILoc); - cutOffParsing(); return ExprError(); } // Allow either an identifier or the keyword 'class' (in C++). @@ -1724,9 +1724,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, Res = ParseBlockLiteralExpression(); break; case tok::code_completion: { + cutOffParsing(); Actions.CodeCompleteExpression(getCurScope(), PreferredType.get(Tok.getLocation())); - cutOffParsing(); return ExprError(); } case tok::l_square: @@ -1856,9 +1856,9 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) { if (InMessageExpression) return LHS; + cutOffParsing(); Actions.CodeCompletePostfixExpression( getCurScope(), LHS, PreferredType.get(Tok.getLocation())); - cutOffParsing(); return ExprError(); case tok::identifier: @@ -2140,12 +2140,12 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) { CorrectedBase = Base; // Code completion for a member access expression. + cutOffParsing(); Actions.CodeCompleteMemberReferenceExpr( getCurScope(), Base, CorrectedBase, OpLoc, OpKind == tok::arrow, Base && ExprStatementTokLoc == Base->getBeginLoc(), PreferredType.get(Tok.getLocation())); - cutOffParsing(); return ExprError(); } @@ -2778,10 +2778,10 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr, CastTy = nullptr; if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteExpression( getCurScope(), PreferredType.get(Tok.getLocation()), /*IsParenthesized=*/ExprType >= CompoundLiteral); - cutOffParsing(); return ExprError(); } @@ -3412,8 +3412,9 @@ Parser::ParseSimpleExpressionList(SmallVectorImpl &Exprs, /// \endverbatim void Parser::ParseBlockId(SourceLocation CaretLoc) { if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type); - return cutOffParsing(); + return; } // Parse the specifier-qualifier-list piece. @@ -3598,8 +3599,8 @@ Optional Parser::ParseAvailabilitySpec() { } else { // Parse the platform name. if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteAvailabilityPlatformName(); cutOffParsing(); + Actions.CodeCompleteAvailabilityPlatformName(); return None; } if (Tok.isNot(tok::identifier)) { diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 9292541d7ede..8052795c0c1e 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -235,6 +235,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier( while (true) { if (HasScopeSpecifier) { if (Tok.is(tok::code_completion)) { + cutOffParsing(); // Code completion for a nested-name-specifier, where the code // completion token follows the '::'. Actions.CodeCompleteQualifiedId(getCurScope(), SS, EnteringContext, @@ -245,7 +246,6 @@ bool Parser::ParseOptionalCXXScopeSpecifier( // token will cause assertion in // Preprocessor::AnnotatePreviousCachedTokens. SS.setEndLoc(Tok.getLocation()); - cutOffParsing(); return true; } @@ -877,9 +877,9 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro, // expression parser perform the completion. if (Tok.is(tok::code_completion) && !(getLangOpts().ObjC && Tentative)) { + cutOffParsing(); Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro, /*AfterAmpersand=*/false); - cutOffParsing(); break; } @@ -891,6 +891,7 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro, } if (Tok.is(tok::code_completion)) { + cutOffParsing(); // If we're in Objective-C++ and we have a bare '[', then this is more // likely to be a message receiver. if (getLangOpts().ObjC && Tentative && First) @@ -898,7 +899,6 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro, else Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro, /*AfterAmpersand=*/false); - cutOffParsing(); break; } @@ -943,9 +943,9 @@ bool Parser::ParseLambdaIntroducer(LambdaIntroducer &Intro, ConsumeToken(); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteLambdaIntroducer(getCurScope(), Intro, /*AfterAmpersand=*/true); - cutOffParsing(); break; } } @@ -1996,8 +1996,8 @@ Sema::ConditionResult Parser::ParseCXXCondition(StmtResult *InitStmt, PreferredType.enterCondition(Actions, Tok.getLocation()); if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition); cutOffParsing(); + Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Condition); return Sema::ConditionError(); } @@ -2608,10 +2608,10 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext, } case tok::code_completion: { + // Don't try to parse any further. + cutOffParsing(); // Code completion for the operator name. Actions.CodeCompleteOperatorName(getCurScope()); - cutOffParsing(); - // Don't try to parse any further. return true; } diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp index 97bd7d8fc51a..9d9c03d28a97 100644 --- a/clang/lib/Parse/ParseInit.cpp +++ b/clang/lib/Parse/ParseInit.cpp @@ -200,9 +200,9 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator( SourceLocation DotLoc = ConsumeToken(); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteDesignator(DesignatorCompletion.PreferredBaseType, DesignatorCompletion.InitExprs, Desig); - cutOffParsing(); return ExprError(); } if (Tok.isNot(tok::identifier)) { diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp index 223b36d7a0e6..9e145f57d61f 100644 --- a/clang/lib/Parse/ParseObjc.cpp +++ b/clang/lib/Parse/ParseObjc.cpp @@ -50,8 +50,8 @@ Parser::ParseObjCAtDirectives(ParsedAttributesWithRange &Attrs) { SourceLocation AtLoc = ConsumeToken(); // the "@" if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCAtDirective(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCAtDirective(getCurScope()); return nullptr; } @@ -219,8 +219,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc, // Code completion after '@interface'. if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCInterfaceDecl(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCInterfaceDecl(getCurScope()); return nullptr; } @@ -253,8 +253,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc, SourceLocation categoryLoc; IdentifierInfo *categoryId = nullptr; if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc); cutOffParsing(); + Actions.CodeCompleteObjCInterfaceCategory(getCurScope(), nameId, nameLoc); return nullptr; } @@ -308,8 +308,8 @@ Decl *Parser::ParseObjCAtInterfaceDeclaration(SourceLocation AtLoc, // Code completion of superclass names. if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc); cutOffParsing(); + Actions.CodeCompleteObjCSuperclass(getCurScope(), nameId, nameLoc); return nullptr; } @@ -472,8 +472,8 @@ ObjCTypeParamList *Parser::parseObjCTypeParamListOrProtocolRefs( if (Tok.is(tok::code_completion)) { // FIXME: If these aren't protocol references, we'll need different // completions. - Actions.CodeCompleteObjCProtocolReferences(protocolIdents); cutOffParsing(); + Actions.CodeCompleteObjCProtocolReferences(protocolIdents); // FIXME: Better recovery here?. return nullptr; @@ -635,10 +635,11 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey, // Code completion within an Objective-C interface. if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteOrdinaryName(getCurScope(), CurParsedObjCImpl? Sema::PCC_ObjCImplementation : Sema::PCC_ObjCInterface); - return cutOffParsing(); + return; } // If we don't have an @ directive, parse it as a function definition. @@ -668,8 +669,9 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey, // Otherwise, we have an @ directive, eat the @. SourceLocation AtLoc = ConsumeToken(); // the "@" if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCAtDirective(getCurScope()); - return cutOffParsing(); + return; } tok::ObjCKeywordKind DirectiveKind = Tok.getObjCKeywordID(); @@ -778,8 +780,9 @@ void Parser::ParseObjCInterfaceDeclList(tok::ObjCKeywordKind contextKey, // We break out of the big loop in two cases: when we see @end or when we see // EOF. In the former case, eat the @end. In the later case, emit an error. if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCAtDirective(getCurScope()); - return cutOffParsing(); + return; } else if (Tok.isObjCAtKeyword(tok::objc_end)) { ConsumeToken(); // the "end" identifier } else { @@ -847,8 +850,9 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) { while (1) { if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCPropertyFlags(getCurScope(), DS); - return cutOffParsing(); + return; } const IdentifierInfo *II = Tok.getIdentifierInfo(); @@ -893,11 +897,12 @@ void Parser::ParseObjCPropertyAttribute(ObjCDeclSpec &DS) { } if (Tok.is(tok::code_completion)) { + cutOffParsing(); if (IsSetter) Actions.CodeCompleteObjCPropertySetter(getCurScope()); else Actions.CodeCompleteObjCPropertyGetter(getCurScope()); - return cutOffParsing(); + return; } SourceLocation SelLoc; @@ -1146,9 +1151,10 @@ void Parser::ParseObjCTypeQualifierList(ObjCDeclSpec &DS, while (1) { if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCPassingType( getCurScope(), DS, Context == DeclaratorContext::ObjCParameter); - return cutOffParsing(); + return; } if (Tok.isNot(tok::identifier)) @@ -1335,9 +1341,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc, ParsingDeclRAIIObject PD(*this, ParsingDeclRAIIObject::NoParent); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, /*ReturnType=*/nullptr); - cutOffParsing(); return nullptr; } @@ -1354,9 +1360,9 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc, methodAttrs); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCMethodDecl(getCurScope(), mType == tok::minus, ReturnType); - cutOffParsing(); return nullptr; } @@ -1416,12 +1422,12 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc, // Code completion for the next piece of the selector. if (Tok.is(tok::code_completion)) { + cutOffParsing(); KeyIdents.push_back(SelIdent); Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(), mType == tok::minus, /*AtParameterName=*/true, ReturnType, KeyIdents); - cutOffParsing(); return nullptr; } @@ -1441,11 +1447,11 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc, // Code completion for the next piece of the selector. if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCMethodDeclSelector(getCurScope(), mType == tok::minus, /*AtParameterName=*/false, ReturnType, KeyIdents); - cutOffParsing(); return nullptr; } @@ -1527,8 +1533,8 @@ ParseObjCProtocolReferences(SmallVectorImpl &Protocols, while (1) { if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents); cutOffParsing(); + Actions.CodeCompleteObjCProtocolReferences(ProtocolIdents); return true; } @@ -1626,12 +1632,12 @@ void Parser::parseObjCTypeArgsOrProtocolQualifiers( } QualType BaseT = Actions.GetTypeFromParser(baseType); + cutOffParsing(); if (!BaseT.isNull() && BaseT->acceptsObjCTypeParams()) { Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Type); } else { Actions.CodeCompleteObjCProtocolReferences(identifierLocPairs); } - cutOffParsing(); return; } @@ -1920,8 +1926,9 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl, // Set the default visibility to private. if (TryConsumeToken(tok::at)) { // parse objc-visibility-spec if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteObjCAtVisibility(getCurScope()); - return cutOffParsing(); + return; } switch (Tok.getObjCKeywordID()) { @@ -1950,9 +1957,10 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl, } if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_ObjCInstanceVariableList); - return cutOffParsing(); + return; } // This needs to duplicate a small amount of code from @@ -2017,8 +2025,8 @@ Parser::ParseObjCAtProtocolDeclaration(SourceLocation AtLoc, ConsumeToken(); // the "protocol" identifier if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCProtocolDecl(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCProtocolDecl(getCurScope()); return nullptr; } @@ -2101,8 +2109,8 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc, // Code completion after '@implementation'. if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCImplementationDecl(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCImplementationDecl(getCurScope()); return nullptr; } @@ -2139,8 +2147,8 @@ Parser::ParseObjCAtImplementationDeclaration(SourceLocation AtLoc, IdentifierInfo *categoryId = nullptr; if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc); cutOffParsing(); + Actions.CodeCompleteObjCImplementationCategory(getCurScope(), nameId, nameLoc); return nullptr; } @@ -2309,8 +2317,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) { while (true) { if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCPropertyDefinition(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCPropertyDefinition(getCurScope()); return nullptr; } @@ -2327,8 +2335,8 @@ Decl *Parser::ParseObjCPropertySynthesize(SourceLocation atLoc) { if (TryConsumeToken(tok::equal)) { // property '=' ivar-name if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId); cutOffParsing(); + Actions.CodeCompleteObjCPropertySynthesizeIvar(getCurScope(), propertyId); return nullptr; } @@ -2387,8 +2395,8 @@ Decl *Parser::ParseObjCPropertyDynamic(SourceLocation atLoc) { while (true) { if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCPropertyDefinition(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCPropertyDefinition(getCurScope()); return nullptr; } @@ -2724,8 +2732,8 @@ Decl *Parser::ParseObjCMethodDefinition() { StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc, ParsedStmtContext StmtCtx) { if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCAtStatement(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCAtStatement(getCurScope()); return StmtError(); } @@ -2765,8 +2773,8 @@ StmtResult Parser::ParseObjCAtStatement(SourceLocation AtLoc, ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) { switch (Tok.getKind()) { case tok::code_completion: - Actions.CodeCompleteObjCAtExpression(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCAtExpression(getCurScope()); return ExprError(); case tok::minus: @@ -3012,8 +3020,8 @@ ExprResult Parser::ParseObjCMessageExpression() { SourceLocation LBracLoc = ConsumeBracket(); // consume '[' if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCMessageReceiver(getCurScope()); cutOffParsing(); + Actions.CodeCompleteObjCMessageReceiver(getCurScope()); return ExprError(); } @@ -3149,6 +3157,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc, InMessageExpressionRAIIObject InMessage(*this, true); if (Tok.is(tok::code_completion)) { + cutOffParsing(); if (SuperLoc.isValid()) Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, None, false); @@ -3158,7 +3167,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc, else Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr, None, false); - cutOffParsing(); return ExprError(); } @@ -3187,6 +3195,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc, /// Parse the expression after ':' if (Tok.is(tok::code_completion)) { + cutOffParsing(); if (SuperLoc.isValid()) Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, KeyIdents, @@ -3200,7 +3209,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc, KeyIdents, /*AtArgumentExpression=*/true); - cutOffParsing(); return ExprError(); } @@ -3225,6 +3233,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc, // Code completion after each argument. if (Tok.is(tok::code_completion)) { + cutOffParsing(); if (SuperLoc.isValid()) Actions.CodeCompleteObjCSuperMessage(getCurScope(), SuperLoc, KeyIdents, @@ -3237,7 +3246,6 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc, Actions.CodeCompleteObjCInstanceMessage(getCurScope(), ReceiverExpr, KeyIdents, /*AtArgumentExpression=*/false); - cutOffParsing(); return ExprError(); } @@ -3577,8 +3585,8 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) { ConsumeParen(); if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents); cutOffParsing(); + Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents); return ExprError(); } @@ -3603,8 +3611,8 @@ ExprResult Parser::ParseObjCSelectorExpression(SourceLocation AtLoc) { break; if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents); cutOffParsing(); + Actions.CodeCompleteObjCSelector(getCurScope(), KeyIdents); return ExprError(); } diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp index 3de01be0db87..54c05aea0e33 100644 --- a/clang/lib/Parse/ParseOpenMP.cpp +++ b/clang/lib/Parse/ParseOpenMP.cpp @@ -441,9 +441,9 @@ void Parser::ParseOpenMPReductionInitializerForDecl(VarDecl *OmpPrivParm) { ConsumeToken(); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteInitializer(getCurScope(), OmpPrivParm); Actions.FinalizeDeclaration(OmpPrivParm); - cutOffParsing(); return; } diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp index f59271c45848..54655863e3ab 100644 --- a/clang/lib/Parse/ParseStmt.cpp +++ b/clang/lib/Parse/ParseStmt.cpp @@ -178,8 +178,8 @@ Retry: } case tok::code_completion: - Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement); cutOffParsing(); + Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Statement); return StmtError(); case tok::identifier: { @@ -726,8 +726,8 @@ StmtResult Parser::ParseCaseStatement(ParsedStmtContext StmtCtx, ColonLoc = SourceLocation(); if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteCase(getCurScope()); cutOffParsing(); + Actions.CodeCompleteCase(getCurScope()); return StmtError(); } @@ -1472,8 +1472,8 @@ StmtResult Parser::ParseIfStatement(SourceLocation *TrailingElseLoc) { // Pop the 'else' scope if needed. InnerScope.Exit(); } else if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteAfterIf(getCurScope(), IsBracedThen); cutOffParsing(); + Actions.CodeCompleteAfterIf(getCurScope(), IsBracedThen); return StmtError(); } else if (InnerStatementTrailingElseLoc.isValid()) { Diag(InnerStatementTrailingElseLoc, diag::warn_dangling_else); @@ -1827,10 +1827,10 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) { FullExprArg ThirdPart(Actions); if (Tok.is(tok::code_completion)) { + cutOffParsing(); Actions.CodeCompleteOrdinaryName(getCurScope(), C99orCXXorObjC? Sema::PCC_ForInit : Sema::PCC_Expression); - cutOffParsing(); return StmtError(); } @@ -1898,8 +1898,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) { ConsumeToken(); // consume 'in' if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCForCollection(getCurScope(), DG); cutOffParsing(); + Actions.CodeCompleteObjCForCollection(getCurScope(), DG); return StmtError(); } Collection = ParseExpression(); @@ -1934,8 +1934,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) { ConsumeToken(); // consume 'in' if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteObjCForCollection(getCurScope(), nullptr); cutOffParsing(); + Actions.CodeCompleteObjCForCollection(getCurScope(), nullptr); return StmtError(); } Collection = ParseExpression(); @@ -2188,9 +2188,9 @@ StmtResult Parser::ParseReturnStatement() { PreferredType.enterReturn(Actions, Tok.getLocation()); // FIXME: Code completion for co_return. if (Tok.is(tok::code_completion) && !IsCoreturn) { + cutOffParsing(); Actions.CodeCompleteExpression(getCurScope(), PreferredType.get(Tok.getLocation())); - cutOffParsing(); return StmtError(); } diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index fb182883b88a..b178b56e967c 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -870,6 +870,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs, SingleDecl = ParseObjCMethodDefinition(); break; case tok::code_completion: + cutOffParsing(); if (CurParsedObjCImpl) { // Code-complete Objective-C methods even without leading '-'/'+' prefix. Actions.CodeCompleteObjCMethodDecl(getCurScope(), @@ -879,7 +880,6 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs, Actions.CodeCompleteOrdinaryName( getCurScope(), CurParsedObjCImpl ? Sema::PCC_ObjCImplementation : Sema::PCC_Namespace); - cutOffParsing(); return nullptr; case tok::kw_import: SingleDecl = ParseModuleImport(SourceLocation()); @@ -2114,21 +2114,21 @@ SourceLocation Parser::handleUnexpectedCodeCompletionToken() { for (Scope *S = getCurScope(); S; S = S->getParent()) { if (S->getFlags() & Scope::FnScope) { + cutOffParsing(); Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_RecoveryInFunction); - cutOffParsing(); return PrevTokLocation; } if (S->getFlags() & Scope::ClassScope) { - Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class); cutOffParsing(); + Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Class); return PrevTokLocation; } } - Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace); cutOffParsing(); + Actions.CodeCompleteOrdinaryName(getCurScope(), Sema::PCC_Namespace); return PrevTokLocation; } @@ -2452,8 +2452,8 @@ bool Parser::ParseModuleName( while (true) { if (!Tok.is(tok::identifier)) { if (Tok.is(tok::code_completion)) { - Actions.CodeCompleteModuleImport(UseLoc, Path); cutOffParsing(); + Actions.CodeCompleteModuleImport(UseLoc, Path); return true; } diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index 18605b321c70..dc7a67e92827 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -5711,8 +5711,9 @@ ProduceSignatureHelp(Sema &SemaRef, Scope *S, unsigned CurrentArg, SourceLocation OpenParLoc) { if (Candidates.empty()) return QualType(); - SemaRef.CodeCompleter->ProcessOverloadCandidates( - SemaRef, CurrentArg, Candidates.data(), Candidates.size(), OpenParLoc); + if (SemaRef.getPreprocessor().isCodeCompletionReached()) + SemaRef.CodeCompleter->ProcessOverloadCandidates( + SemaRef, CurrentArg, Candidates.data(), Candidates.size(), OpenParLoc); return getParamType(SemaRef, Candidates, CurrentArg); } diff --git a/clang/test/CodeCompletion/desig-init.cpp b/clang/test/CodeCompletion/desig-init.cpp index 8a66f4554217..999f368ba563 100644 --- a/clang/test/CodeCompletion/desig-init.cpp +++ b/clang/test/CodeCompletion/desig-init.cpp @@ -62,3 +62,18 @@ void aux() { Test X{.x = T(2)}; // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:62:14 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC3 %s } + +namespace signature_regression { + // Verify that an old bug is gone: passing an init-list as a constructor arg + // would emit overloads as a side-effect. + struct S{int x;}; + int wrongFunction(S); + int rightFunction(); + int dummy = wrongFunction({1}); + int x = rightFunction(); + // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:73:25 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-SIGNATURE-REGRESSION %s + // CHECK-SIGNATURE-REGRESSION-NOT: OVERLOAD: [#int#]wrongFunction + // CHECK-SIGNATURE-REGRESSION: OVERLOAD: [#int#]rightFunction + // CHECK-SIGNATURE-REGRESSION-NOT: OVERLOAD: [#int#]wrongFunction +} + -- GitLab From 35368bbdbb6f87543c3ea4c7f70c113954ce1bef Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Tue, 16 Mar 2021 12:48:25 +0100 Subject: [PATCH 0020/1206] [NFC] Replace loop by idiomatic llvm::find_if --- llvm/lib/CodeGen/TwoAddressInstructionPass.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp index a011b03d747c..bd20f32ee253 100644 --- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -1357,11 +1357,9 @@ void TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI, TiedPairList &TiedPairs, unsigned &Dist) { - bool IsEarlyClobber = false; - for (unsigned tpi = 0, tpe = TiedPairs.size(); tpi != tpe; ++tpi) { - const MachineOperand &DstMO = MI->getOperand(TiedPairs[tpi].second); - IsEarlyClobber |= DstMO.isEarlyClobber(); - } + bool IsEarlyClobber = llvm::find_if(TiedPairs, [MI](auto const &TP) { + return MI->getOperand(TP.second).isEarlyClobber(); + }) != TiedPairs.end(); bool RemovedKillFlag = false; bool AllUsesCopied = true; @@ -1369,9 +1367,9 @@ TwoAddressInstructionPass::processTiedPairs(MachineInstr *MI, SlotIndex LastCopyIdx; Register RegB = 0; unsigned SubRegB = 0; - for (unsigned tpi = 0, tpe = TiedPairs.size(); tpi != tpe; ++tpi) { - unsigned SrcIdx = TiedPairs[tpi].first; - unsigned DstIdx = TiedPairs[tpi].second; + for (auto &TP : TiedPairs) { + unsigned SrcIdx = TP.first; + unsigned DstIdx = TP.second; const MachineOperand &DstMO = MI->getOperand(DstIdx); Register RegA = DstMO.getReg(); -- GitLab From 2772c3a9752289ffec473b62f84855262a22de0b Mon Sep 17 00:00:00 2001 From: Kadir Cetinkaya Date: Mon, 15 Mar 2021 10:18:12 +0100 Subject: [PATCH 0021/1206] [clangd] Introduce pullDiags endpoint Implement initial support for pull-based diagnostics in ClangdServer. This is planned for LSP 3.17, and initial proposal is in https://github.com/microsoft/vscode-languageserver-node/blob/d15eb0671e0947d14e3548d13754104ee13aa56d/protocol/src/common/proposed.diagnostic.ts#L111. We chose to serve the requests only when clangd has a fresh preamble available. In case of a stale preamble we just drop the request on the floor. This patch doesn't plumb this to LSP layer yet, as pullDiags is still a proposal with only an implementation in vscode. Differential Revision: https://reviews.llvm.org/D98623 --- clang-tools-extra/clangd/ClangdServer.cpp | 20 +++- clang-tools-extra/clangd/ClangdServer.h | 12 +++ clang-tools-extra/clangd/ParsedAST.cpp | 34 +++--- clang-tools-extra/clangd/ParsedAST.h | 10 +- clang-tools-extra/clangd/Preamble.h | 3 + clang-tools-extra/clangd/tool/Check.cpp | 2 +- .../clangd/unittests/DiagnosticsTests.cpp | 100 +++++++++--------- .../clangd/unittests/ModulesTests.cpp | 2 +- .../clangd/unittests/ParsedASTTests.cpp | 6 +- .../clangd/unittests/PreambleTests.cpp | 19 +++- .../clangd/unittests/SelectionTests.cpp | 2 +- .../clangd/unittests/TUSchedulerTests.cpp | 2 +- clang-tools-extra/clangd/unittests/TestTU.cpp | 8 +- clang-tools-extra/clangd/unittests/TestTU.h | 1 + .../clangd/unittests/TypeHierarchyTests.cpp | 2 +- 15 files changed, 141 insertions(+), 82 deletions(-) diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index e9724e7516aa..f9516a1537a0 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -9,6 +9,7 @@ #include "ClangdServer.h" #include "CodeComplete.h" #include "Config.h" +#include "Diagnostics.h" #include "DumpAST.h" #include "FindSymbols.h" #include "Format.h" @@ -81,7 +82,9 @@ struct UpdateIndexCallbacks : public ParsingCallbacks { if (FIndex) FIndex->updateMain(Path, AST); - std::vector Diagnostics = AST.getDiagnostics(); + assert(AST.getDiagnostics().hasValue() && + "We issue callback only with fresh preambles"); + std::vector Diagnostics = *AST.getDiagnostics(); if (ServerCallbacks) Publish([&]() { ServerCallbacks->onDiagnosticsReady(Path, AST.version(), @@ -902,6 +905,21 @@ void ClangdServer::customAction(PathRef File, llvm::StringRef Name, WorkScheduler->runWithAST(Name, File, std::move(Action)); } +void ClangdServer::diagnostics(PathRef File, Callback> CB) { + auto Action = + [CB = std::move(CB)](llvm::Expected InpAST) mutable { + if (!InpAST) + return CB(InpAST.takeError()); + if (auto Diags = InpAST->AST.getDiagnostics()) + return CB(*Diags); + // FIXME: Use ServerCancelled error once it is settled in LSP-3.17. + return CB(llvm::make_error("server is busy parsing includes", + ErrorCode::InternalError)); + }; + + WorkScheduler->runWithAST("Diagnostics", File, std::move(Action)); +} + llvm::StringMap ClangdServer::fileStats() const { return WorkScheduler->fileStats(); } diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h index b633d3139683..37ac30f70cb4 100644 --- a/clang-tools-extra/clangd/ClangdServer.h +++ b/clang-tools-extra/clangd/ClangdServer.h @@ -12,6 +12,7 @@ #include "../clang-tidy/ClangTidyOptions.h" #include "CodeComplete.h" #include "ConfigProvider.h" +#include "Diagnostics.h" #include "DraftStore.h" #include "FeatureModule.h" #include "GlobalCompilationDatabase.h" @@ -40,6 +41,7 @@ #include #include #include +#include namespace clang { namespace clangd { @@ -64,6 +66,8 @@ public: virtual ~Callbacks() = default; /// Called by ClangdServer when \p Diagnostics for \p File are ready. + /// These pushed diagnostics might correspond to an older version of the + /// file, they do not interfere with "pull-based" ClangdServer::diagnostics. /// May be called concurrently for separate files, not for a single file. virtual void onDiagnosticsReady(PathRef File, llvm::StringRef Version, std::vector Diagnostics) {} @@ -345,6 +349,14 @@ public: void customAction(PathRef File, llvm::StringRef Name, Callback Action); + /// Fetches diagnostics for current version of the \p File. This might fail if + /// server is busy (building a preamble) and would require a long time to + /// prepare diagnostics. If it fails, clients should wait for + /// onSemanticsMaybeChanged and then retry. + /// These 'pulled' diagnostics do not interfere with the diagnostics 'pushed' + /// to Callbacks::onDiagnosticsReady, and clients may use either or both. + void diagnostics(PathRef File, Callback> CB); + /// Returns estimated memory usage and other statistics for each of the /// currently open files. /// Overall memory usage of clangd may be significantly more than reported diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp index 24038f0ec35f..119263f0a891 100644 --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -264,9 +264,11 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, StoreDiags ASTDiags; llvm::Optional Patch; + bool PreserveDiags = true; if (Preamble) { Patch = PreamblePatch::create(Filename, Inputs, *Preamble); Patch->apply(*CI); + PreserveDiags = Patch->preserveDiagnostics(); } auto Clang = prepareCompilerInstance( std::move(CI), PreamblePCH, @@ -441,14 +443,20 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, // CompilerInstance won't run this callback, do it directly. ASTDiags.EndSourceFile(); - std::vector Diags = CompilerInvocationDiags; - // Add diagnostics from the preamble, if any. - if (Preamble) - Diags.insert(Diags.end(), Preamble->Diags.begin(), Preamble->Diags.end()); - // Finally, add diagnostics coming from the AST. - { - std::vector D = ASTDiags.take(CTContext.getPointer()); - Diags.insert(Diags.end(), D.begin(), D.end()); + llvm::Optional> Diags; + // FIXME: Also skip generation of diagnostics alltogether to speed up ast + // builds when we are patching a stale preamble. + if (PreserveDiags) { + Diags = CompilerInvocationDiags; + // Add diagnostics from the preamble, if any. + if (Preamble) + Diags->insert(Diags->end(), Preamble->Diags.begin(), + Preamble->Diags.end()); + // Finally, add diagnostics coming from the AST. + { + std::vector D = ASTDiags.take(CTContext.getPointer()); + Diags->insert(Diags->end(), D.begin(), D.end()); + } } return ParsedAST(Inputs.Version, std::move(Preamble), std::move(Clang), std::move(Action), std::move(Tokens), std::move(Macros), @@ -493,14 +501,12 @@ llvm::ArrayRef ParsedAST::getLocalTopLevelDecls() { const MainFileMacros &ParsedAST::getMacros() const { return Macros; } -const std::vector &ParsedAST::getDiagnostics() const { return Diags; } - std::size_t ParsedAST::getUsedBytes() const { auto &AST = getASTContext(); // FIXME(ibiryukov): we do not account for the dynamically allocated part of // Message and Fixes inside each diagnostic. - std::size_t Total = - clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags); + std::size_t Total = clangd::getUsedBytes(LocalTopLevelDecls) + + (Diags ? clangd::getUsedBytes(*Diags) : 0); // FIXME: the rest of the function is almost a direct copy-paste from // libclang's clang_getCXTUResourceUsage. We could share the implementation. @@ -541,8 +547,8 @@ ParsedAST::ParsedAST(llvm::StringRef Version, std::unique_ptr Action, syntax::TokenBuffer Tokens, MainFileMacros Macros, std::vector LocalTopLevelDecls, - std::vector Diags, IncludeStructure Includes, - CanonicalIncludes CanonIncludes) + llvm::Optional> Diags, + IncludeStructure Includes, CanonicalIncludes CanonIncludes) : Version(Version), Preamble(std::move(Preamble)), Clang(std::move(Clang)), Action(std::move(Action)), Tokens(std::move(Tokens)), Macros(std::move(Macros)), Diags(std::move(Diags)), diff --git a/clang-tools-extra/clangd/ParsedAST.h b/clang-tools-extra/clangd/ParsedAST.h index 263a097b14cb..c1ce6fce7029 100644 --- a/clang-tools-extra/clangd/ParsedAST.h +++ b/clang-tools-extra/clangd/ParsedAST.h @@ -88,7 +88,9 @@ public: /// (These should be const, but RecursiveASTVisitor requires Decl*). ArrayRef getLocalTopLevelDecls(); - const std::vector &getDiagnostics() const; + const llvm::Optional> &getDiagnostics() const { + return Diags; + } /// Returns the estimated size of the AST and the accessory structures, in /// bytes. Does not include the size of the preamble. @@ -120,7 +122,7 @@ private: std::unique_ptr Clang, std::unique_ptr Action, syntax::TokenBuffer Tokens, MainFileMacros Macros, std::vector LocalTopLevelDecls, - std::vector Diags, IncludeStructure Includes, + llvm::Optional> Diags, IncludeStructure Includes, CanonicalIncludes CanonIncludes); std::string Version; @@ -142,8 +144,8 @@ private: /// All macro definitions and expansions in the main file. MainFileMacros Macros; - // Data, stored after parsing. - std::vector Diags; + // Data, stored after parsing. None if AST was built with a stale preamble. + llvm::Optional> Diags; // Top-level decls inside the current file. Not that this does not include // top-level decls from the preamble. std::vector LocalTopLevelDecls; diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h index 1de1d6cfe5fa..5b9d17840214 100644 --- a/clang-tools-extra/clangd/Preamble.h +++ b/clang-tools-extra/clangd/Preamble.h @@ -126,6 +126,9 @@ public: /// Returns textual patch contents. llvm::StringRef text() const { return PatchContents; } + /// Whether diagnostics generated using this patch are trustable. + bool preserveDiagnostics() const { return PatchContents.empty(); } + private: PreamblePatch() = default; std::string PatchContents; diff --git a/clang-tools-extra/clangd/tool/Check.cpp b/clang-tools-extra/clangd/tool/Check.cpp index 00a4609e5134..20b86daff8af 100644 --- a/clang-tools-extra/clangd/tool/Check.cpp +++ b/clang-tools-extra/clangd/tool/Check.cpp @@ -181,7 +181,7 @@ public: elog("Failed to build AST"); return false; } - ErrCount += showErrors(llvm::makeArrayRef(AST->getDiagnostics()) + ErrCount += showErrors(llvm::makeArrayRef(*AST->getDiagnostics()) .drop_front(Preamble->Diags.size())); if (Opts.BuildDynamicSymbolIndex) { diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp index aebb231f39f9..d5b4a08a4229 100644 --- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp +++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp @@ -135,7 +135,7 @@ o]](); auto TU = TestTU::withCode(Test.code()); TU.ClangTidyProvider = addTidyChecks("google-explicit-constructor"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre( // This range spans lines. AllOf(Diag(Test.range("typo"), @@ -173,14 +173,14 @@ o]](); TEST(DiagnosticsTest, FlagsMatter) { Annotations Test("[[void]] main() {} // error-ok"); auto TU = TestTU::withCode(Test.code()); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(AllOf(Diag(Test.range(), "'main' must return 'int'"), WithFix(Fix(Test.range(), "int", "change 'void' to 'int'"))))); // Same code built as C gets different diagnostics. TU.Filename = "Plain.c"; EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre(AllOf( Diag(Test.range(), "return type of 'main' is not 'int'"), WithFix(Fix(Test.range(), "int", "change return type to 'int'"))))); @@ -192,7 +192,7 @@ TEST(DiagnosticsTest, DiagnosticPreamble) { )cpp"); auto TU = TestTU::withCode(Test.code()); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(::testing::AllOf( Diag(Test.range(), "'not-found.h' file not found"), DiagSource(Diag::Clang), DiagName("pp_file_not_found")))); @@ -209,7 +209,7 @@ TEST(DiagnosticsTest, DeduplicatedClangTidyDiagnostics) { "hicpp-uppercase-literal-suffix"); // Verify that we filter out the duplicated diagnostic message. EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(::testing::AllOf( Diag(Test.range(), "floating point literal has suffix 'f', which is not uppercase"), @@ -229,7 +229,7 @@ TEST(DiagnosticsTest, DeduplicatedClangTidyDiagnostics) { // The check doesn't handle template instantiations which ends up emitting // duplicated messages, verify that we deduplicate them. EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(::testing::AllOf( Diag(Test.range(), "floating point literal has suffix 'f', which is not uppercase"), @@ -254,7 +254,7 @@ TEST(DiagnosticsTest, ClangTidy) { "modernize-deprecated-headers," "modernize-use-trailing-return-type"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre( AllOf(Diag(Test.range("deprecated"), "inclusion of deprecated C++ header 'assert.h'; consider " @@ -296,7 +296,7 @@ TEST(DiagnosticsTest, ClangTidyEOF) { TU.AdditionalFiles["a.h"] = TU.AdditionalFiles["b.h"] = ""; TU.ClangTidyProvider = addTidyChecks("llvm-include-order"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), Contains(AllOf(Diag(Test.range(), "#includes are not sorted properly"), DiagSource(Diag::ClangTidy), DiagName("llvm-include-order")))); @@ -314,7 +314,7 @@ TEST(DiagnosticTest, TemplatesInHeaders) { TestTU TU = TestTU::withCode(Main.code()); TU.HeaderCode = Header.code().str(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre(AllOf( Diag(Main.range(), "in template: base specifier must name a class"), WithNote(Diag(Header.range(), "error occurred here"), @@ -340,7 +340,7 @@ TEST(DiagnosticTest, MakeUnique) { } } )cpp"; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range(), "in template: " @@ -368,7 +368,7 @@ TEST(DiagnosticTest, NoMultipleDiagnosticInFlight) { TestTU TU = TestTU::withCode(Main.code()); TU.ClangTidyProvider = addTidyChecks("modernize-loop-convert"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(::testing::AllOf( Diag(Main.range(), "use range-based for loop instead"), DiagSource(Diag::ClangTidy), DiagName("modernize-loop-convert")))); @@ -384,14 +384,14 @@ TEST(DiagnosticTest, RespectsDiagnosticConfig) { )cpp"); auto TU = TestTU::withCode(Main.code()); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre(Diag(Main.range(), "use of undeclared identifier 'unknown'"), Diag(Main.range("ret"), "void function 'x' should not return a value"))); Config Cfg; Cfg.Diagnostics.Suppress.insert("return-type"); WithContextValue WithCfg(Config::Key, std::move(Cfg)); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(Diag(Main.range(), "use of undeclared identifier 'unknown'"))); } @@ -413,7 +413,7 @@ TEST(DiagnosticTest, ClangTidySuppressionComment) { TestTU TU = TestTU::withCode(Main.code()); TU.ClangTidyProvider = addTidyChecks("bugprone-integer-division"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(::testing::AllOf( Diag(Main.range(), "result of integer division used in a floating " "point context; possible loss of precision"), @@ -431,7 +431,7 @@ TEST(DiagnosticTest, ClangTidyWarningAsError) { TU.ClangTidyProvider = addTidyChecks("bugprone-integer-division", "bugprone-integer-division"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(::testing::AllOf( Diag(Main.range(), "result of integer division used in a floating " "point context; possible loss of precision"), @@ -450,7 +450,7 @@ TEST(DiagnosticTest, LongFixMessages) { )cpp"); TestTU TU = TestTU::withCode(Source.code()); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre(WithFix(Fix( Source.range(), "somereallyreallyreallyreallyreallyreallyreallyreallylongidentifier", @@ -466,7 +466,7 @@ n]] = 10; // error-ok } )cpp"); TU.Code = std::string(Source.code()); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(WithFix( Fix(Source.range(), "ident", "change 'ide\\…' to 'ident'")))); } @@ -481,7 +481,7 @@ TEST(DiagnosticTest, ClangTidySuppressionCommentTrumpsWarningAsError) { TestTU TU = TestTU::withCode(Main.code()); TU.ClangTidyProvider = addTidyChecks("bugprone-integer-division", "bugprone-integer-division"); - EXPECT_THAT(TU.build().getDiagnostics(), UnorderedElementsAre()); + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre()); } TEST(DiagnosticTest, ClangTidyNoLiteralDataInMacroToken) { @@ -496,7 +496,7 @@ TEST(DiagnosticTest, ClangTidyNoLiteralDataInMacroToken) { )cpp"); TestTU TU = TestTU::withCode(Main.code()); TU.ClangTidyProvider = addTidyChecks("bugprone-bad-signal-to-kill-thread"); - EXPECT_THAT(TU.build().getDiagnostics(), UnorderedElementsAre()); // no-crash + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre()); // no-crash } TEST(DiagnosticTest, ElseAfterReturnRange) { @@ -513,7 +513,7 @@ TEST(DiagnosticTest, ElseAfterReturnRange) { TestTU TU = TestTU::withCode(Main.code()); TU.ClangTidyProvider = addTidyChecks("llvm-else-after-return"); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre(Diag(Main.range(), "do not use 'else' after 'return'"))); } @@ -532,7 +532,7 @@ TEST(DiagnosticsTest, Preprocessor) { #endif )cpp"); EXPECT_THAT( - TestTU::withCode(Test.code()).build().getDiagnostics(), + *TestTU::withCode(Test.code()).build().getDiagnostics(), ElementsAre(Diag(Test.range(), "use of undeclared identifier 'b'"))); } @@ -542,7 +542,7 @@ TEST(DiagnosticsTest, IgnoreVerify) { )cpp"); TU.ExtraArgs.push_back("-Xclang"); TU.ExtraArgs.push_back("-verify"); - EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty()); + EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty()); } // Recursive main-file include is diagnosed, and doesn't crash. @@ -552,7 +552,7 @@ TEST(DiagnosticsTest, RecursivePreamble) { int symbol; )cpp"); TU.Filename = "foo.h"; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(DiagName("pp_including_mainfile_in_preamble"))); EXPECT_THAT(TU.build().getLocalTopLevelDecls(), SizeIs(1)); } @@ -565,7 +565,7 @@ TEST(DiagnosticsTest, RecursivePreamblePragmaOnce) { int symbol; )cpp"); TU.Filename = "foo.h"; - EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty()); + EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty()); EXPECT_THAT(TU.build().getLocalTopLevelDecls(), SizeIs(1)); } @@ -581,7 +581,7 @@ TEST(DiagnosticsTest, RecursivePreambleIfndefGuard) { )cpp"); TU.Filename = "foo.h"; // FIXME: should be no errors here. - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(DiagName("pp_including_mainfile_in_preamble"))); EXPECT_THAT(TU.build().getLocalTopLevelDecls(), SizeIs(1)); } @@ -598,7 +598,7 @@ TEST(DiagnosticsTest, InsideMacros) { return $bar[[TEN]]; } )cpp"); - EXPECT_THAT(TestTU::withCode(Test.code()).build().getDiagnostics(), + EXPECT_THAT(*TestTU::withCode(Test.code()).build().getDiagnostics(), ElementsAre(Diag(Test.range("foo"), "cannot initialize return object of type " "'int *' with an rvalue of type 'int'"), @@ -614,7 +614,7 @@ TEST(DiagnosticsTest, NoFixItInMacro) { [[Define]](main) // error-ok )cpp"); auto TU = TestTU::withCode(Test.code()); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), ElementsAre(AllOf(Diag(Test.range(), "'main' must return 'int'"), Not(WithFix(_))))); } @@ -625,7 +625,7 @@ TEST(ClangdTest, MSAsm) { llvm::InitializeAllTargetInfos(); // As in ClangdMain auto TU = TestTU::withCode("void fn() { __asm { cmp cl,64 } }"); TU.ExtraArgs = {"-fms-extensions"}; - EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty()); + EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty()); } TEST(DiagnosticsTest, ToLSP) { @@ -783,7 +783,7 @@ class T { TU.ExternalIndex = Index.get(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAreArray( {AllOf(Diag(Test.range("nested"), "incomplete type 'ns::X' named in nested name specifier"), @@ -868,7 +868,7 @@ int main() { MemIndex::build(std::move(Slab).build(), RefSlab(), RelationSlab()); TU.ExternalIndex = Index.get(); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Test.range("base"), "base class has incomplete type"), Diag(Test.range("access"), @@ -901,7 +901,7 @@ using Type = ns::$template[[Foo]]; TU.ExternalIndex = Index.get(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre( AllOf(Diag(Test.range("unqualified1"), "unknown type name 'X'"), DiagName("unknown_typename"), @@ -946,7 +946,7 @@ void foo() { SymbolWithHeader{"na::nb::X", "unittest:///b.h", "\"b.h\""}}); TU.ExternalIndex = Index.get(); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre(AllOf( Diag(Test.range("unqualified"), "unknown type name 'X'"), DiagName("unknown_typename"), @@ -967,7 +967,7 @@ TEST(IncludeFixerTest, NoCrashMemebrAccess) { TU.ExternalIndex = Index.get(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(Diag(Test.range(), "no member named 'xy' in 'X'"))); } @@ -1002,7 +1002,7 @@ void bar(X *x) { TU.ExternalIndex = Index.get(); auto Parsed = TU.build(); - for (const auto &D : Parsed.getDiagnostics()) { + for (const auto &D : *Parsed.getDiagnostics()) { if (D.Fixes.size() != 1) { ADD_FAILURE() << "D.Fixes.size() != 1"; continue; @@ -1027,7 +1027,7 @@ void g() { ns::$[[scope]]::X_Y(); } TU.ExternalIndex = Index.get(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre(AllOf( Diag(Test.range(), "no member named 'scope' in namespace 'ns'"), DiagName("no_member"), @@ -1055,7 +1055,7 @@ void f() { TU.ExternalIndex = Index.get(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), UnorderedElementsAre( AllOf( Diag(Test.range("q1"), "use of undeclared identifier 'clangd'; " @@ -1098,7 +1098,7 @@ namespace c { SymbolWithHeader{"a::X", "unittest:///x.h", "\"x.h\""}); TU.ExternalIndex = Index.get(); - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre(AllOf( Diag(Test.range(), "no type named 'X' in namespace 'a'"), DiagName("typename_nested_not_found"), @@ -1124,7 +1124,7 @@ TEST(IncludeFixerTest, NoCrashOnTemplateInstantiations) { TU.ExternalIndex = Index.get(); EXPECT_THAT( - TU.build().getDiagnostics(), + *TU.build().getDiagnostics(), ElementsAre(Diag(Test.range(), "use of undeclared identifier 'a'"))); } @@ -1135,7 +1135,7 @@ TEST(DiagsInHeaders, DiagInsideHeader) { Annotations Header("[[no_type_spec]]; // error-ok"); TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", std::string(Header.code())}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre(AllOf( Diag(Main.range(), "in included file: C++ requires a " "type specifier for all declarations"), @@ -1149,7 +1149,7 @@ TEST(DiagsInHeaders, DiagInTransitiveInclude) { TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", "#include \"b.h\""}, {"b.h", "no_type_spec; // error-ok"}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range(), "in included file: C++ requires a " "type specifier for all declarations"))); @@ -1163,7 +1163,7 @@ TEST(DiagsInHeaders, DiagInMultipleHeaders) { TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", "no_type_spec; // error-ok"}, {"b.h", "no_type_spec; // error-ok"}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range("a"), "in included file: C++ requires a type " "specifier for all declarations"), @@ -1180,7 +1180,7 @@ TEST(DiagsInHeaders, PreferExpansionLocation) { TU.AdditionalFiles = { {"a.h", "#include \"b.h\"\n"}, {"b.h", "#ifndef X\n#define X\nno_type_spec; // error-ok\n#endif"}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre(Diag(Main.range(), "in included file: C++ requires a type " "specifier for all declarations"))); @@ -1198,7 +1198,7 @@ TEST(DiagsInHeaders, PreferExpansionLocationMacros) { {"a.h", "#include \"c.h\"\n"}, {"b.h", "#include \"c.h\"\n"}, {"c.h", "#ifndef X\n#define X\nno_type_spec; // error-ok\n#endif"}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range(), "in included file: C++ requires a " "type specifier for all declarations"))); @@ -1227,7 +1227,7 @@ TEST(DiagsInHeaders, LimitDiagsOutsideMainFile) { no_type_spec_9; no_type_spec_10; #endif)cpp"}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range(), "in included file: C++ requires a " "type specifier for all declarations"))); @@ -1242,7 +1242,7 @@ TEST(DiagsInHeaders, OnlyErrorOrFatal) { int x = 5/0;)cpp"); TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", std::string(Header.code())}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre(AllOf( Diag(Main.range(), "in included file: C++ requires " "a type specifier for all declarations"), @@ -1260,7 +1260,7 @@ TEST(DiagsInHeaders, OnlyDefaultErrorOrFatal) { TU.AdditionalFiles = {{"a.h", std::string(Header.code())}}; // promote warnings to errors. TU.ExtraArgs = {"-Werror", "-Wunused"}; - EXPECT_THAT(TU.build().getDiagnostics(), IsEmpty()); + EXPECT_THAT(*TU.build().getDiagnostics(), IsEmpty()); } TEST(DiagsInHeaders, FromNonWrittenSources) { @@ -1273,7 +1273,7 @@ TEST(DiagsInHeaders, FromNonWrittenSources) { TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", std::string(Header.code())}}; TU.ExtraArgs = {"-DFOO=NOOO"}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre(AllOf( Diag(Main.range(), "in included file: use of undeclared identifier 'NOOO'"), @@ -1291,7 +1291,7 @@ TEST(DiagsInHeaders, ErrorFromMacroExpansion) { X;)cpp"); TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", std::string(Header.code())}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range(), "in included file: use of undeclared " "identifier 'foo'; did you mean 'fo'?"))); @@ -1308,7 +1308,7 @@ TEST(DiagsInHeaders, ErrorFromMacroArgument) { X(foo);)cpp"); TestTU TU = TestTU::withCode(Main.code()); TU.AdditionalFiles = {{"a.h", std::string(Header.code())}}; - EXPECT_THAT(TU.build().getDiagnostics(), + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre( Diag(Main.range(), "in included file: use of undeclared " "identifier 'foo'; did you mean 'fo'?"))); @@ -1320,7 +1320,7 @@ TEST(IgnoreDiags, FromNonWrittenInclude) { TU.AdditionalFiles = {{"a.h", "void main();"}}; // The diagnostic "main must return int" is from the header, we don't attempt // to render it in the main file as there is no written location there. - EXPECT_THAT(TU.build().getDiagnostics(), UnorderedElementsAre()); + EXPECT_THAT(*TU.build().getDiagnostics(), UnorderedElementsAre()); } TEST(ToLSPDiag, RangeIsInMain) { diff --git a/clang-tools-extra/clangd/unittests/ModulesTests.cpp b/clang-tools-extra/clangd/unittests/ModulesTests.cpp index 83d6b28d6dfc..b56b91836508 100644 --- a/clang-tools-extra/clangd/unittests/ModulesTests.cpp +++ b/clang-tools-extra/clangd/unittests/ModulesTests.cpp @@ -61,7 +61,7 @@ TEST(Modules, PreambleBuildVisibility) { header "module.h" } )modulemap"; - EXPECT_TRUE(TU.build().getDiagnostics().empty()); + EXPECT_TRUE(TU.build().getDiagnostics()->empty()); } TEST(Modules, Diagnostic) { diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp index b96d1243d243..5435648cd9be 100644 --- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp +++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp @@ -511,7 +511,7 @@ TEST(ParsedASTTest, ReplayPreambleForTidyCheckers) { auto PatchedAST = ParsedAST::build(testPath(TU.Filename), TU.inputs(FS), std::move(CI), {}, BaselinePreamble); ASSERT_TRUE(PatchedAST); - EXPECT_TRUE(PatchedAST->getDiagnostics().empty()); + EXPECT_FALSE(PatchedAST->getDiagnostics()); } // Then ensure correctness by making sure includes were seen only once. @@ -526,7 +526,7 @@ TEST(ParsedASTTest, ReplayPreambleForTidyCheckers) { auto PatchedAST = ParsedAST::build(testPath(TU.Filename), TU.inputs(FS), std::move(CI), {}, BaselinePreamble); ASSERT_TRUE(PatchedAST); - EXPECT_TRUE(PatchedAST->getDiagnostics().empty()); + EXPECT_FALSE(PatchedAST->getDiagnostics()); EXPECT_THAT(Includes, ElementsAre(WithFileName(testPath("__preamble_patch__.h")), WithFileName("b.h"), WithFileName("a.h"))); @@ -569,7 +569,7 @@ TEST(ParsedASTTest, PatchesAdditionalIncludes) { auto PatchedAST = ParsedAST::build(testPath("foo.cpp"), Inputs, std::move(CI), {}, EmptyPreamble); ASSERT_TRUE(PatchedAST); - ASSERT_TRUE(PatchedAST->getDiagnostics().empty()); + ASSERT_FALSE(PatchedAST->getDiagnostics()); // Ensure source location information is correct, including resolved paths. EXPECT_THAT(PatchedAST->getIncludeStructure().MainFileIncludes, diff --git a/clang-tools-extra/clangd/unittests/PreambleTests.cpp b/clang-tools-extra/clangd/unittests/PreambleTests.cpp index 4eee9effb824..70a14241a8ac 100644 --- a/clang-tools-extra/clangd/unittests/PreambleTests.cpp +++ b/clang-tools-extra/clangd/unittests/PreambleTests.cpp @@ -274,8 +274,12 @@ TEST(PreamblePatchTest, Define) { auto AST = createPatchedAST("", Modified.code()); ASSERT_TRUE(AST); - EXPECT_THAT(AST->getDiagnostics(), - Not(Contains(Field(&Diag::Range, Modified.range())))); + std::vector MacroRefRanges; + for (auto &M : AST->getMacros().MacroRefs) { + for (auto &O : M.getSecond()) + MacroRefRanges.push_back(O.Rng); + } + EXPECT_THAT(MacroRefRanges, Contains(Modified.range())); } } @@ -298,8 +302,6 @@ TEST(PreamblePatchTest, OrderingPreserved) { auto AST = createPatchedAST(Baseline, Modified.code()); ASSERT_TRUE(AST); - EXPECT_THAT(AST->getDiagnostics(), - Not(Contains(Field(&Diag::Range, Modified.range())))); } TEST(PreamblePatchTest, LocateMacroAtWorks) { @@ -535,6 +537,15 @@ TEST(PreamblePatch, ModifiedBounds) { ExpectedBounds.PreambleEndsAtStartOfLine); } } + +TEST(PreamblePatch, DropsDiagnostics) { + llvm::StringLiteral Code = "#define FOO\nx;/* error-ok */"; + // First check that this code generates diagnostics. + EXPECT_THAT(*TestTU::withCode(Code).build().getDiagnostics(), + testing::Not(testing::IsEmpty())); + // Ensure they are dropeed when a patched preamble is used. + EXPECT_FALSE(createPatchedAST("", Code)->getDiagnostics()); +} } // namespace } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp index e9c689f329ab..a063c84a6a4c 100644 --- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp +++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp @@ -581,7 +581,7 @@ TEST(SelectionTest, PathologicalPreprocessor) { auto TU = TestTU::withCode(Test.code()); TU.AdditionalFiles["Expand.inc"] = "MACRO\n"; auto AST = TU.build(); - EXPECT_THAT(AST.getDiagnostics(), ::testing::IsEmpty()); + EXPECT_THAT(*AST.getDiagnostics(), ::testing::IsEmpty()); auto T = makeSelectionTree(Case, AST); EXPECT_EQ("BreakStmt", T.commonAncestor()->kind()); diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp index 22b6ea2296d2..5f8faf78df3c 100644 --- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp +++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp @@ -121,7 +121,7 @@ protected: class CaptureDiags : public ParsingCallbacks { public: void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override { - reportDiagnostics(File, AST.getDiagnostics(), Publish); + reportDiagnostics(File, *AST.getDiagnostics(), Publish); } void onFailedAST(PathRef File, llvm::StringRef Version, diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp index 8d336b3f4e19..a36f2508cd31 100644 --- a/clang-tools-extra/clangd/unittests/TestTU.cpp +++ b/clang-tools-extra/clangd/unittests/TestTU.cpp @@ -113,6 +113,11 @@ ParsedAST TestTU::build() const { ADD_FAILURE() << "Failed to build code:\n" << Code; llvm_unreachable("Failed to build TestTU!"); } + if (!AST->getDiagnostics()) { + ADD_FAILURE() << "TestTU should always build an AST with a fresh Preamble" + << Code; + return std::move(*AST); + } // Check for error diagnostics and report gtest failures (unless expected). // This guards against accidental syntax errors silently subverting tests. // error-ok is awfully primitive - using clang -verify would be nicer. @@ -128,7 +133,8 @@ ParsedAST TestTU::build() const { return false; }(); if (!ErrorOk) { - for (const auto &D : AST->getDiagnostics()) + // We always build AST with a fresh preamble in TestTU. + for (const auto &D : *AST->getDiagnostics()) if (D.Severity >= DiagnosticsEngine::Error) { ADD_FAILURE() << "TestTU failed to build (suppress with /*error-ok*/): \n" diff --git a/clang-tools-extra/clangd/unittests/TestTU.h b/clang-tools-extra/clangd/unittests/TestTU.h index 18b490332b1a..169cab045ea1 100644 --- a/clang-tools-extra/clangd/unittests/TestTU.h +++ b/clang-tools-extra/clangd/unittests/TestTU.h @@ -78,6 +78,7 @@ struct TestTU { // By default, build() will report Error diagnostics as GTest errors. // Suppress this behavior by adding an 'error-ok' comment to the code. + // The result will always have getDiagnostics() populated. ParsedAST build() const; std::shared_ptr preamble(PreambleParsedCallback PreambleCallback = nullptr) const; diff --git a/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp b/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp index 08f936ce8b55..09f90fd6e6b5 100644 --- a/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp +++ b/clang-tools-extra/clangd/unittests/TypeHierarchyTests.cpp @@ -398,7 +398,7 @@ TEST(TypeHierarchy, RecursiveHierarchyUnbounded) { // The compiler should produce a diagnostic for hitting the // template instantiation depth. - ASSERT_TRUE(!AST.getDiagnostics().empty()); + ASSERT_TRUE(!AST.getDiagnostics()->empty()); // Make sure getTypeHierarchy() doesn't get into an infinite recursion. // The parent is reported as "S" because "S<0>" is an invalid instantiation. -- GitLab From 0fda5e844128615308e6772f02f2bce55805244c Mon Sep 17 00:00:00 2001 From: David Zarzycki Date: Tue, 16 Mar 2021 07:57:31 -0400 Subject: [PATCH 0022/1206] [llvm-exegesis testing] Workaround unreliable test Picking an instruction at random is not perfectly reliable. --- llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s b/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s index fcdaf6a40341..c57b61a55d5f 100644 --- a/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s +++ b/llvm/test/tools/llvm-exegesis/X86/latency-IN16rr.s @@ -1,5 +1,8 @@ # RUN: llvm-exegesis -mode=latency -opcode-name=IN16rr -repetition-mode=duplicate | FileCheck %s +# FIXME: Sometimes fails with: 'unimplemented operand type' +# ALLOW_RETRIES: 2 + CHECK: --- CHECK-NEXT: mode: latency CHECK-NEXT: key: -- GitLab From b661788b77e570dc82fe2f89a355713c144407f1 Mon Sep 17 00:00:00 2001 From: Nicolas Vasilache Date: Tue, 16 Mar 2021 12:05:12 +0000 Subject: [PATCH 0023/1206] [mlir] NFC - Expose GlobalCreator so it can be reused. --- mlir/include/mlir/Transforms/BufferUtils.h | 18 +++++ .../Transforms/TensorConstantBufferize.cpp | 77 ++++++++----------- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/mlir/include/mlir/Transforms/BufferUtils.h b/mlir/include/mlir/Transforms/BufferUtils.h index 70da6a025343..33edffa372a3 100644 --- a/mlir/include/mlir/Transforms/BufferUtils.h +++ b/mlir/include/mlir/Transforms/BufferUtils.h @@ -120,6 +120,24 @@ protected: Liveness liveness; }; +namespace memref { +class GlobalOp; +} // namespace memref + +// Support class to create global ops for tensor-valued constants in the +// program. Globals are created lazily at the top of the `moduleOp` with pretty +// names. Duplicates are avoided. +class GlobalCreator { +public: + explicit GlobalCreator(ModuleOp module) : moduleOp(module) {} + memref::GlobalOp getGlobalFor(ConstantOp constantOp); + +private: + ModuleOp moduleOp; + // This could use memref::GlobalOp key but we avoid introducing a new + // dependence to the memref dialect for this. + DenseMap globals; +}; } // end namespace mlir #endif // MLIR_TRANSFORMS_BUFFERUTILS_H diff --git a/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp b/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp index 18c3be94685b..55d34059e033 100644 --- a/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp +++ b/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp @@ -15,64 +15,47 @@ #include "mlir/Dialect/StandardOps/IR/Ops.h" #include "mlir/Dialect/StandardOps/Transforms/Passes.h" #include "mlir/IR/BlockAndValueMapping.h" +#include "mlir/Transforms/BufferUtils.h" #include "mlir/Transforms/Bufferize.h" #include "mlir/Transforms/DialectConversion.h" using namespace mlir; -namespace { -// This class creates global ops for all tensor-valued constants in the program. -// It creates them with pretty names and makes sure that duplicate globals -// aren't created. -class GlobalCreator { -public: - explicit GlobalCreator(ModuleOp module); - memref::GlobalOp getGlobalFor(Attribute attr) { - assert(globals.find(attr) != globals.end() && "unknown constant attr"); - return globals[attr]; - } - -private: - DenseMap globals; -}; +memref::GlobalOp GlobalCreator::getGlobalFor(ConstantOp constantOp) { + auto type = constantOp.getType().cast(); -GlobalCreator::GlobalCreator(ModuleOp module) { BufferizeTypeConverter typeConverter; + + // If we already have a global for this constant value, no need to do + // anything else. + auto it = globals.find(constantOp.getValue()); + if (it != globals.end()) + return cast(it->second); + // Create a builder without an insertion point. We will insert using the // symbol table to guarantee unique names. - OpBuilder globalBuilder(module.getContext()); - SymbolTable symbolTable(module); - module.walk([&](ConstantOp op) { - // We only want tensor constants for now. - auto type = op.getType().dyn_cast(); - if (!type) - return; - // If we already have a global for this constant value, no need to do - // anything else. - auto it = globals.find(op.getValue()); - if (it != globals.end()) - return; + OpBuilder globalBuilder(moduleOp.getContext()); + SymbolTable symbolTable(moduleOp); - // Create a pretty name. - SmallString<64> buf; - llvm::raw_svector_ostream os(buf); - interleave(type.getShape(), os, "x"); - os << "x" << type.getElementType(); + // Create a pretty name. + SmallString<64> buf; + llvm::raw_svector_ostream os(buf); + interleave(type.getShape(), os, "x"); + os << "x" << type.getElementType(); - auto global = globalBuilder.create( - op.getLoc(), (Twine("__constant_") + os.str()).str(), - /*sym_visibility=*/globalBuilder.getStringAttr("private"), - /*type=*/typeConverter.convertType(type), - /*initial_value=*/op.getValue().cast(), - /*constant=*/true); - symbolTable.insert(global); - // The symbol table inserts at the end of the module, but globals are a bit - // nicer if they are at the beginning. - global->moveBefore(&module.front()); - globals[op.getValue()] = global; - }); + auto global = globalBuilder.create( + constantOp.getLoc(), (Twine("__constant_") + os.str()).str(), + /*sym_visibility=*/globalBuilder.getStringAttr("private"), + /*type=*/typeConverter.convertType(type), + /*initial_value=*/constantOp.getValue().cast(), + /*constant=*/true); + symbolTable.insert(global); + // The symbol table inserts at the end of the module, but globals are a bit + // nicer if they are at the beginning. + global->moveBefore(&moduleOp.front()); + globals[constantOp.getValue()] = global; + return global; } -} // namespace namespace { class BufferizeTensorConstantOp : public OpConversionPattern { @@ -89,7 +72,7 @@ public: if (!type) return failure(); - auto globalMemref = globals.getGlobalFor(op.value()); + auto globalMemref = globals.getGlobalFor(op); rewriter.replaceOpWithNewOp(op, globalMemref.type(), globalMemref.getName()); return success(); -- GitLab From b2e78a061c06546c42a977071047cd9da2194a32 Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Mon, 8 Mar 2021 17:46:35 +0100 Subject: [PATCH 0024/1206] [NFC] Use SmallString instead of std::string for the AttrBuilder This avoids a few unnecessary conversion from StringRef to std::string, and a bunch of extra allocation thanks to the SmallString. Differential Revision: https://reviews.llvm.org/D98190 --- llvm/include/llvm/IR/Attributes.h | 5 +++-- llvm/lib/IR/Attributes.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/IR/Attributes.h b/llvm/include/llvm/IR/Attributes.h index b4056540663f..20f5cf1b1917 100644 --- a/llvm/include/llvm/IR/Attributes.h +++ b/llvm/include/llvm/IR/Attributes.h @@ -18,6 +18,7 @@ #include "llvm-c/Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Config/llvm-config.h" @@ -756,7 +757,7 @@ template <> struct DenseMapInfo { /// equality, presence of attributes, etc. class AttrBuilder { std::bitset Attrs; - std::map> TargetDepAttrs; + std::map, SmallString<32>, std::less<>> TargetDepAttrs; MaybeAlign Alignment; MaybeAlign StackAlignment; uint64_t DerefBytes = 0; @@ -921,7 +922,7 @@ public: bool empty() const { return Attrs.none(); } // Iterators for target-dependent attributes. - using td_type = std::pair; + using td_type = decltype(TargetDepAttrs)::value_type; using td_iterator = decltype(TargetDepAttrs)::iterator; using td_const_iterator = decltype(TargetDepAttrs)::const_iterator; using td_range = iterator_range; diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index c98e5c36a518..18c2f3aad5f0 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -1624,7 +1624,7 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) { } AttrBuilder &AttrBuilder::addAttribute(StringRef A, StringRef V) { - TargetDepAttrs[std::string(A)] = std::string(V); + TargetDepAttrs[A] = V; return *this; } -- GitLab From 524fe515091d31e1c054fc521113a3bf2088d159 Mon Sep 17 00:00:00 2001 From: Kirill Bobyrev Date: Tue, 16 Mar 2021 13:37:48 +0100 Subject: [PATCH 0025/1206] [clangd] Add basic monitoring info request for remote index server This allows requesting information about the server uptime and start time. This is the first patch in a series of monitoring changes, hence it's not immediately useful. Next step is propagating the index freshness information and then probably loading metadata into the index server. The way to test new behaviour through command line: ``` $ grpc_cli call localhost:50051 Monitor/MonitoringInfo '' connecting to localhost:50051 uptime_seconds: 42 index_age_seconds: 609568 Rpc succeeded with OK status ``` Reviewed By: kadircet Differential Revision: https://reviews.llvm.org/D98246 --- .../clangd/index/remote/CMakeLists.txt | 3 ++ .../index/remote/MonitoringService.proto | 27 ++++++++++ .../clangd/index/remote/Service.proto | 1 - .../clangd/index/remote/server/Server.cpp | 52 +++++++++++++++++-- 4 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 clang-tools-extra/clangd/index/remote/MonitoringService.proto diff --git a/clang-tools-extra/clangd/index/remote/CMakeLists.txt b/clang-tools-extra/clangd/index/remote/CMakeLists.txt index eaa000b745e5..ded3f9274f86 100644 --- a/clang-tools-extra/clangd/index/remote/CMakeLists.txt +++ b/clang-tools-extra/clangd/index/remote/CMakeLists.txt @@ -1,5 +1,7 @@ if (CLANGD_ENABLE_REMOTE) generate_protos(RemoteIndexProto "Index.proto") + generate_protos(MonitoringServiceProto "MonitoringService.proto" + GRPC) generate_protos(RemoteIndexServiceProto "Service.proto" DEPENDS "Index.proto" GRPC) @@ -8,6 +10,7 @@ if (CLANGD_ENABLE_REMOTE) target_link_libraries(RemoteIndexServiceProto PRIVATE RemoteIndexProto + MonitoringServiceProto ) include_directories(${CMAKE_CURRENT_BINARY_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../) diff --git a/clang-tools-extra/clangd/index/remote/MonitoringService.proto b/clang-tools-extra/clangd/index/remote/MonitoringService.proto new file mode 100644 index 000000000000..75d807c19005 --- /dev/null +++ b/clang-tools-extra/clangd/index/remote/MonitoringService.proto @@ -0,0 +1,27 @@ +//===--- MonitoringService.proto - CLangd Remote index monitoring service -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +syntax = "proto2"; + +package clang.clangd.remote.v1; + +message MonitoringInfoRequest {} +message MonitoringInfoReply { + // Time since the server started (in seconds). + optional uint64 uptime_seconds = 1; + // Time since the index was built on the indexing machine. + optional uint64 index_age_seconds = 2; + // ID of the indexed commit in Version Control System. + optional string index_commit_hash = 3; + // URL to the index file. + optional string index_link = 4; +} + +service Monitor { + rpc MonitoringInfo(MonitoringInfoRequest) returns (MonitoringInfoReply) {} +} diff --git a/clang-tools-extra/clangd/index/remote/Service.proto b/clang-tools-extra/clangd/index/remote/Service.proto index 4e39ff9ec666..7c7efa530200 100644 --- a/clang-tools-extra/clangd/index/remote/Service.proto +++ b/clang-tools-extra/clangd/index/remote/Service.proto @@ -23,4 +23,3 @@ service SymbolIndex { rpc Relations(RelationsRequest) returns (stream RelationsReply) {} } - diff --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp b/clang-tools-extra/clangd/index/remote/server/Server.cpp index be0e844a1f80..f3cf131bb8a5 100644 --- a/clang-tools-extra/clangd/index/remote/server/Server.cpp +++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp @@ -8,7 +8,10 @@ #include "Features.inc" #include "Index.pb.h" +#include "MonitoringService.grpc.pb.h" +#include "MonitoringService.pb.h" #include "Service.grpc.pb.h" +#include "Service.pb.h" #include "index/Index.h" #include "index/Serialization.h" #include "index/Symbol.h" @@ -288,11 +291,46 @@ private: clangd::SymbolIndex &Index; }; +class Monitor final : public v1::Monitor::Service { +public: + Monitor(llvm::sys::TimePoint<> IndexAge) + : StartTime(std::chrono::system_clock::now()), IndexBuildTime(IndexAge) {} + + void updateIndex(llvm::sys::TimePoint<> UpdateTime) { + IndexBuildTime.exchange(UpdateTime); + } + +private: + // FIXME(kirillbobyrev): Most fields should be populated when the index + // reloads (probably in adjacent metadata.txt file next to loaded .idx) but + // they aren't right now. + grpc::Status MonitoringInfo(grpc::ServerContext *Context, + const v1::MonitoringInfoRequest *Request, + v1::MonitoringInfoReply *Reply) override { + Reply->set_uptime_seconds(std::chrono::duration_cast( + std::chrono::system_clock::now() - StartTime) + .count()); + // FIXME(kirillbobyrev): We are currently making use of the last + // modification time of the index artifact to deduce its age. This is wrong + // as it doesn't account for the indexing delay. Propagate some metadata + // with the index artifacts to indicate time of the commit we indexed. + Reply->set_index_age_seconds( + std::chrono::duration_cast( + std::chrono::system_clock::now() - IndexBuildTime.load()) + .count()); + return grpc::Status::OK; + } + + const llvm::sys::TimePoint<> StartTime; + std::atomic> IndexBuildTime; +}; + // Detect changes in \p IndexPath file and load new versions of the index // whenever they become available. void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath, llvm::vfs::Status &LastStatus, - llvm::IntrusiveRefCntPtr &FS) { + llvm::IntrusiveRefCntPtr &FS, + Monitor &Monitor) { auto Status = FS->status(IndexPath); // Requested file is same as loaded index: no reload is needed. if (!Status || (Status->getLastModificationTime() == @@ -309,12 +347,13 @@ void hotReload(clangd::SwapIndex &Index, llvm::StringRef IndexPath, return; } Index.reset(std::move(NewIndex)); + Monitor.updateIndex(Status->getLastModificationTime()); log("New index version loaded. Last modification time: {0}, size: {1} bytes.", Status->getLastModificationTime(), Status->getSize()); } void runServerAndWait(clangd::SymbolIndex &Index, llvm::StringRef ServerAddress, - llvm::StringRef IndexPath) { + llvm::StringRef IndexPath, Monitor &Monitor) { RemoteIndexServer Service(Index, IndexRoot); grpc::EnableDefaultHealthCheckService(true); @@ -327,6 +366,7 @@ void runServerAndWait(clangd::SymbolIndex &Index, llvm::StringRef ServerAddress, Builder.AddChannelArgument(GRPC_ARG_MAX_CONNECTION_IDLE_MS, IdleTimeoutSeconds * 1000); Builder.RegisterService(&Service); + Builder.RegisterService(&Monitor); std::unique_ptr Server(Builder.BuildAndStart()); log("Server listening on {0}", ServerAddress); @@ -425,16 +465,18 @@ int main(int argc, char *argv[]) { } clang::clangd::SwapIndex Index(std::move(SymIndex)); - std::thread HotReloadThread([&Index, &Status, &FS]() { + Monitor Monitor(Status->getLastModificationTime()); + + std::thread HotReloadThread([&Index, &Status, &FS, &Monitor]() { llvm::vfs::Status LastStatus = *Status; static constexpr auto RefreshFrequency = std::chrono::seconds(30); while (!clang::clangd::shutdownRequested()) { - hotReload(Index, llvm::StringRef(IndexPath), LastStatus, FS); + hotReload(Index, llvm::StringRef(IndexPath), LastStatus, FS, Monitor); std::this_thread::sleep_for(RefreshFrequency); } }); - runServerAndWait(Index, ServerAddress, IndexPath); + runServerAndWait(Index, ServerAddress, IndexPath, Monitor); HotReloadThread.join(); } -- GitLab From 534a1f4b05c267543be4521bbab43f4cc104cdeb Mon Sep 17 00:00:00 2001 From: Max Kazantsev Date: Tue, 16 Mar 2021 19:38:57 +0700 Subject: [PATCH 0026/1206] [Test] Update auto-generated checks --- .../IndVarSimplify/eliminate-comparison.ll | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll index 0bbcd3fa3ae0..3eb7b12dce2a 100644 --- a/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll +++ b/llvm/test/Transforms/IndVarSimplify/eliminate-comparison.ll @@ -380,7 +380,7 @@ declare void @side_effect() define void @func_13(i32* %len.ptr) { ; CHECK-LABEL: @func_13( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0:!range !.*]] +; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0:![0-9]+]] ; CHECK-NEXT: [[LEN_IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0 ; CHECK-NEXT: br i1 [[LEN_IS_ZERO]], label [[LEAVE:%.*]], label [[LOOP_PREHEADER:%.*]] ; CHECK: loop.preheader: @@ -424,7 +424,7 @@ leave: define void @func_14(i32* %len.ptr) { ; CHECK-LABEL: @func_14( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[LEN_IS_ZERO:%.*]] = icmp eq i32 [[LEN]], 0 ; CHECK-NEXT: [[LEN_IS_INT_MIN:%.*]] = icmp eq i32 [[LEN]], -2147483648 ; CHECK-NEXT: [[NO_ENTRY:%.*]] = or i1 [[LEN_IS_ZERO]], [[LEN_IS_INT_MIN]] @@ -472,7 +472,7 @@ leave: define void @func_15(i32* %len.ptr) { ; CHECK-LABEL: @func_15( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[LEN_ADD_1:%.*]] = add i32 [[LEN]], 1 ; CHECK-NEXT: [[LEN_ADD_1_IS_ZERO:%.*]] = icmp eq i32 [[LEN_ADD_1]], 0 ; CHECK-NEXT: br i1 [[LEN_ADD_1_IS_ZERO]], label [[LEAVE:%.*]], label [[LOOP_PREHEADER:%.*]] @@ -517,7 +517,7 @@ leave: define void @func_16(i32* %len.ptr) { ; CHECK-LABEL: @func_16( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[LEN_ADD_5:%.*]] = add i32 [[LEN]], 5 ; CHECK-NEXT: [[ENTRY_COND_0:%.*]] = icmp slt i32 [[LEN]], 2147483643 ; CHECK-NEXT: [[ENTRY_COND_1:%.*]] = icmp slt i32 4, [[LEN_ADD_5]] @@ -624,7 +624,7 @@ leave: define i1 @func_18(i16* %tmp20, i32* %len.addr) { ; CHECK-LABEL: @func_18( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_ADDR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LEN:%.*]] = load i32, i32* [[LEN_ADDR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i32 [[LEN]], 0 ; CHECK-NEXT: br i1 [[TMP18]], label [[BB2:%.*]], label [[BB0_PREHEADER:%.*]] ; CHECK: bb0.preheader: @@ -686,7 +686,7 @@ bb3: define void @func_19(i32* %length.ptr) { ; CHECK-LABEL: @func_19( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[LENGTH_IS_NONZERO:%.*]] = icmp ne i32 [[LENGTH]], 0 ; CHECK-NEXT: br i1 [[LENGTH_IS_NONZERO]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]] ; CHECK: loop.preheader: @@ -774,7 +774,7 @@ leave: define void @func_21(i32* %length.ptr) { ; CHECK-LABEL: @func_21( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[LIM:%.*]] = sub i32 [[LENGTH]], 1 ; CHECK-NEXT: [[ENTRY_COND:%.*]] = icmp sgt i32 [[LENGTH]], 1 ; CHECK-NEXT: br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]] @@ -819,7 +819,7 @@ leave: define void @func_22(i32* %length.ptr) { ; CHECK-LABEL: @func_22( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[ENTRY_COND:%.*]] = icmp sgt i32 [[LENGTH]], 1 ; CHECK-NEXT: br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]] ; CHECK: loop.preheader: @@ -861,7 +861,7 @@ leave: define void @func_23(i32* %length.ptr) { ; CHECK-LABEL: @func_23( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[ENTRY_COND:%.*]] = icmp ult i32 4, [[LENGTH]] ; CHECK-NEXT: br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]] ; CHECK: loop.preheader: @@ -902,7 +902,7 @@ leave: define void @func_24(i32* %init.ptr) { ; CHECK-LABEL: @func_24( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[INIT:%.*]] = load i32, i32* [[INIT_PTR:%.*]], align 4, [[RNG0]] +; CHECK-NEXT: [[INIT:%.*]] = load i32, i32* [[INIT_PTR:%.*]], align 4, !range [[RNG0]] ; CHECK-NEXT: [[ENTRY_COND:%.*]] = icmp ugt i32 [[INIT]], 4 ; CHECK-NEXT: br i1 [[ENTRY_COND]], label [[LOOP_PREHEADER:%.*]], label [[LEAVE:%.*]] ; CHECK: loop.preheader: -- GitLab From b044f76bc8d678eb4916abd3842c533351d2962e Mon Sep 17 00:00:00 2001 From: Max Kazantsev Date: Tue, 16 Mar 2021 19:46:36 +0700 Subject: [PATCH 0027/1206] [Test] Add test with loops guarded by trivial conditions --- .../IndVarSimplify/trivial-guard.ll | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 llvm/test/Transforms/IndVarSimplify/trivial-guard.ll diff --git a/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll b/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll new file mode 100644 index 000000000000..7506259aa7a3 --- /dev/null +++ b/llvm/test/Transforms/IndVarSimplify/trivial-guard.ll @@ -0,0 +1,139 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -indvars -S < %s | FileCheck %s +; RUN: opt -passes=indvars -S < %s | FileCheck %s + +declare i1 @cond() + +define void @test_01(i32 %x) { +; CHECK-LABEL: @test_01( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 true, label [[LOOP_1_PREHEADER:%.*]], label [[LOOP_2_PREHEADER:%.*]] +; CHECK: loop.2.preheader: +; CHECK-NEXT: br label [[LOOP_2:%.*]] +; CHECK: loop.1.preheader: +; CHECK-NEXT: br label [[LOOP_1:%.*]] +; CHECK: loop.1: +; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ [[IV_NEXT_1:%.*]], [[GUARDED_1:%.*]] ], [ 0, [[LOOP_1_PREHEADER]] ] +; CHECK-NEXT: [[CHECK_1:%.*]] = icmp slt i32 [[IV_1]], [[X:%.*]] +; CHECK-NEXT: br i1 [[CHECK_1]], label [[GUARDED_1]], label [[FAIL_LOOPEXIT:%.*]] +; CHECK: guarded.1: +; CHECK-NEXT: [[IV_NEXT_1]] = add nuw i32 [[IV_1]], 1 +; CHECK-NEXT: [[LOOP_COND_1:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[LOOP_COND_1]], label [[LOOP_1]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK: loop.2: +; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[IV_NEXT_2:%.*]], [[GUARDED_2:%.*]] ], [ 0, [[LOOP_2_PREHEADER]] ] +; CHECK-NEXT: [[CHECK_2:%.*]] = icmp slt i32 [[IV_2]], [[X]] +; CHECK-NEXT: br i1 [[CHECK_2]], label [[GUARDED_2]], label [[FAIL_LOOPEXIT1:%.*]] +; CHECK: guarded.2: +; CHECK-NEXT: [[IV_NEXT_2]] = add nuw i32 [[IV_2]], 1 +; CHECK-NEXT: [[LOOP_COND_2:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[LOOP_COND_2]], label [[LOOP_2]], label [[EXIT_LOOPEXIT2:%.*]] +; CHECK: exit.loopexit: +; CHECK-NEXT: br label [[EXIT:%.*]] +; CHECK: exit.loopexit2: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret void +; CHECK: fail.loopexit: +; CHECK-NEXT: br label [[FAIL:%.*]] +; CHECK: fail.loopexit1: +; CHECK-NEXT: br label [[FAIL]] +; CHECK: fail: +; CHECK-NEXT: unreachable +; +entry: + br i1 true, label %loop.1, label %loop.2 + +loop.1: + %iv.1 = phi i32 [0, %entry], [%iv.next.1, %guarded.1] + %check.1 = icmp slt i32 %iv.1, %x + br i1 %check.1, label %guarded.1, label %fail + +guarded.1: + %iv.next.1 = add i32 %iv.1, 1 + %loop.cond.1 = call i1 @cond() + br i1 %loop.cond.1, label %loop.1, label %exit + +loop.2: + %iv.2 = phi i32 [0, %entry], [%iv.next.2, %guarded.2] + %check.2 = icmp slt i32 %iv.2, %x + br i1 %check.2, label %guarded.2, label %fail + +guarded.2: + %iv.next.2 = add i32 %iv.2, 1 + %loop.cond.2 = call i1 @cond() + br i1 %loop.cond.2, label %loop.2, label %exit + +exit: + ret void + +fail: + unreachable +} + +define void @test_02(i32 %x) { +; CHECK-LABEL: @test_02( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 false, label [[LOOP_1_PREHEADER:%.*]], label [[LOOP_2_PREHEADER:%.*]] +; CHECK: loop.2.preheader: +; CHECK-NEXT: br label [[LOOP_2:%.*]] +; CHECK: loop.1.preheader: +; CHECK-NEXT: br label [[LOOP_1:%.*]] +; CHECK: loop.1: +; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ [[IV_NEXT_1:%.*]], [[GUARDED_1:%.*]] ], [ 0, [[LOOP_1_PREHEADER]] ] +; CHECK-NEXT: [[CHECK_1:%.*]] = icmp slt i32 [[IV_1]], [[X:%.*]] +; CHECK-NEXT: br i1 [[CHECK_1]], label [[GUARDED_1]], label [[FAIL_LOOPEXIT:%.*]] +; CHECK: guarded.1: +; CHECK-NEXT: [[IV_NEXT_1]] = add nuw i32 [[IV_1]], 1 +; CHECK-NEXT: [[LOOP_COND_1:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[LOOP_COND_1]], label [[LOOP_1]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK: loop.2: +; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[IV_NEXT_2:%.*]], [[GUARDED_2:%.*]] ], [ 0, [[LOOP_2_PREHEADER]] ] +; CHECK-NEXT: [[CHECK_2:%.*]] = icmp slt i32 [[IV_2]], [[X]] +; CHECK-NEXT: br i1 [[CHECK_2]], label [[GUARDED_2]], label [[FAIL_LOOPEXIT1:%.*]] +; CHECK: guarded.2: +; CHECK-NEXT: [[IV_NEXT_2]] = add nuw i32 [[IV_2]], 1 +; CHECK-NEXT: [[LOOP_COND_2:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[LOOP_COND_2]], label [[LOOP_2]], label [[EXIT_LOOPEXIT2:%.*]] +; CHECK: exit.loopexit: +; CHECK-NEXT: br label [[EXIT:%.*]] +; CHECK: exit.loopexit2: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret void +; CHECK: fail.loopexit: +; CHECK-NEXT: br label [[FAIL:%.*]] +; CHECK: fail.loopexit1: +; CHECK-NEXT: br label [[FAIL]] +; CHECK: fail: +; CHECK-NEXT: unreachable +; +entry: + br i1 false, label %loop.1, label %loop.2 + +loop.1: + %iv.1 = phi i32 [0, %entry], [%iv.next.1, %guarded.1] + %check.1 = icmp slt i32 %iv.1, %x + br i1 %check.1, label %guarded.1, label %fail + +guarded.1: + %iv.next.1 = add i32 %iv.1, 1 + %loop.cond.1 = call i1 @cond() + br i1 %loop.cond.1, label %loop.1, label %exit + +loop.2: + %iv.2 = phi i32 [0, %entry], [%iv.next.2, %guarded.2] + %check.2 = icmp slt i32 %iv.2, %x + br i1 %check.2, label %guarded.2, label %fail + +guarded.2: + %iv.next.2 = add i32 %iv.2, 1 + %loop.cond.2 = call i1 @cond() + br i1 %loop.cond.2, label %loop.2, label %exit + +exit: + ret void + +fail: + unreachable +} -- GitLab From 49d0e115d5df40aa89339f4ace7a8dee378c03bb Mon Sep 17 00:00:00 2001 From: David Zarzycki Date: Tue, 16 Mar 2021 09:11:01 -0400 Subject: [PATCH 0028/1206] [lit testing] Fix Windows reliability? --- llvm/utils/lit/tests/reorder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/lit/tests/reorder.py b/llvm/utils/lit/tests/reorder.py index 7c9dc8d21fe3..8e5ecda22219 100644 --- a/llvm/utils/lit/tests/reorder.py +++ b/llvm/utils/lit/tests/reorder.py @@ -3,6 +3,7 @@ # RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig # RUN: %{lit} -j1 %{inputs}/reorder | FileCheck %s # RUN: not diff %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig +# RUN: cp %{inputs}/reorder/.lit_test_times.txt.orig %{inputs}/reorder/.lit_test_times.txt # END. # CHECK: -- Testing: 3 tests, 1 workers -- -- GitLab From a9773248001229ed67f239c7ebb2043f7e9ddb94 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Sun, 14 Mar 2021 22:54:18 +0200 Subject: [PATCH 0029/1206] [InstSimplify] Match PtrToInt more directly in a GEP transform (NFC) In preparation for D98611, the upcoming change will need to apply additional checks to `P` and `V`, and so this refactor paves the way for adding additional checks in a less awkward way. Reviewed By: lebedev.ri Differential Revision: https://reviews.llvm.org/D98672 --- llvm/lib/Analysis/InstructionSimplify.cpp | 56 ++++++++++------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 5e05cb03d831..95a4e8d82c76 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -4349,40 +4349,34 @@ static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef Ops, // doesn't truncate the pointers. if (Ops[1]->getType()->getScalarSizeInBits() == Q.DL.getPointerSizeInBits(AS)) { - auto PtrToInt = [GEPTy](Value *P) -> Value * { - Value *Temp; - if (match(P, m_PtrToInt(m_Value(Temp)))) - if (Temp->getType() == GEPTy) - return Temp; - return nullptr; + auto CanSimplify = [GEPTy, &P]() -> bool { + // FIXME: The following transforms are only legal if P and V have the + // same provenance (PR44403). Check whether getUnderlyingObject() is + // the same? + return P->getType() == GEPTy; }; - - // FIXME: The following transforms are only legal if P and V have the - // same provenance (PR44403). Check whether getUnderlyingObject() is - // the same? - // getelementptr V, (sub P, V) -> P if P points to a type of size 1. if (TyAllocSize == 1 && - match(Ops[1], m_Sub(m_Value(P), m_PtrToInt(m_Specific(Ops[0]))))) - if (Value *R = PtrToInt(P)) - return R; - - // getelementptr V, (ashr (sub P, V), C) -> Q - // if P points to a type of size 1 << C. - if (match(Ops[1], - m_AShr(m_Sub(m_Value(P), m_PtrToInt(m_Specific(Ops[0]))), - m_ConstantInt(C))) && - TyAllocSize == 1ULL << C) - if (Value *R = PtrToInt(P)) - return R; - - // getelementptr V, (sdiv (sub P, V), C) -> Q - // if P points to a type of size C. - if (match(Ops[1], - m_SDiv(m_Sub(m_Value(P), m_PtrToInt(m_Specific(Ops[0]))), - m_SpecificInt(TyAllocSize)))) - if (Value *R = PtrToInt(P)) - return R; + match(Ops[1], m_Sub(m_PtrToInt(m_Value(P)), + m_PtrToInt(m_Specific(Ops[0])))) && + CanSimplify()) + return P; + + // getelementptr V, (ashr (sub P, V), C) -> P if P points to a type of + // size 1 << C. + if (match(Ops[1], m_AShr(m_Sub(m_PtrToInt(m_Value(P)), + m_PtrToInt(m_Specific(Ops[0]))), + m_ConstantInt(C))) && + TyAllocSize == 1ULL << C && CanSimplify()) + return P; + + // getelementptr V, (sdiv (sub P, V), C) -> P if P points to a type of + // size C. + if (match(Ops[1], m_SDiv(m_Sub(m_PtrToInt(m_Value(P)), + m_PtrToInt(m_Specific(Ops[0]))), + m_SpecificInt(TyAllocSize))) && + CanSimplify()) + return P; } } } -- GitLab From 43f2d269b3830e643472c6a9993b2d007bfaad02 Mon Sep 17 00:00:00 2001 From: RamNalamothu Date: Tue, 16 Mar 2021 16:36:24 +0530 Subject: [PATCH 0030/1206] [AMDGPU, NFC] Refactor FP/BP spill index code in emitPrologue/emitEpilogue Reviewed By: scott.linder Differential Revision: https://reviews.llvm.org/D98617 --- llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 91 +++++++++------------- 1 file changed, 36 insertions(+), 55 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp index c9490da5efbd..e7588b716150 100644 --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -838,6 +838,13 @@ static Register buildScratchExecCopy(LivePhysRegs &LiveRegs, return ScratchExecCopy; } +// A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR. +// Otherwise we are spilling to memory. +static bool spilledToMemory(const MachineFunction &MF, int SaveIndex) { + const MachineFrameInfo &MFI = MF.getFrameInfo(); + return MFI.getStackID(SaveIndex) != TargetStackID::SGPRSpill; +} + void SIFrameLowering::emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const { SIMachineFunctionInfo *FuncInfo = MF.getInfo(); @@ -869,23 +876,8 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, // turn on all lanes before doing the spill to memory. Register ScratchExecCopy; - bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue(); - bool SpillFPToMemory = false; - // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR. - // Otherwise we are spilling the FP to memory. - if (HasFPSaveIndex) { - SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) != - TargetStackID::SGPRSpill; - } - - bool HasBPSaveIndex = FuncInfo->BasePointerSaveIndex.hasValue(); - bool SpillBPToMemory = false; - // A StackID of SGPRSpill implies that this is a spill from SGPR to VGPR. - // Otherwise we are spilling the BP to memory. - if (HasBPSaveIndex) { - SpillBPToMemory = MFI.getStackID(*FuncInfo->BasePointerSaveIndex) != - TargetStackID::SGPRSpill; - } + Optional FPSaveIndex = FuncInfo->FramePointerSaveIndex; + Optional BPSaveIndex = FuncInfo->BasePointerSaveIndex; for (const SIMachineFunctionInfo::SGPRSpillVGPRCSR &Reg : FuncInfo->getSGPRSpillVGPRs()) { @@ -901,8 +893,9 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, Reg.FI.getValue()); } - if (HasFPSaveIndex && SpillFPToMemory) { - assert(!MFI.isDeadObjectIndex(FuncInfo->FramePointerSaveIndex.getValue())); + if (FPSaveIndex && spilledToMemory(MF, *FPSaveIndex)) { + const int FramePtrFI = *FPSaveIndex; + assert(!MFI.isDeadObjectIndex(FramePtrFI)); if (!ScratchExecCopy) ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true); @@ -916,12 +909,12 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, .addReg(FramePtrReg); buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, TmpVGPR, - FuncInfo->getScratchRSrcReg(), StackPtrReg, - FuncInfo->FramePointerSaveIndex.getValue()); + FuncInfo->getScratchRSrcReg(), StackPtrReg, FramePtrFI); } - if (HasBPSaveIndex && SpillBPToMemory) { - assert(!MFI.isDeadObjectIndex(*FuncInfo->BasePointerSaveIndex)); + if (BPSaveIndex && spilledToMemory(MF, *BPSaveIndex)) { + const int BasePtrFI = *BPSaveIndex; + assert(!MFI.isDeadObjectIndex(BasePtrFI)); if (!ScratchExecCopy) ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, true); @@ -935,8 +928,7 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, .addReg(BasePtrReg); buildPrologSpill(ST, LiveRegs, MBB, MBBI, TII, TmpVGPR, - FuncInfo->getScratchRSrcReg(), StackPtrReg, - *FuncInfo->BasePointerSaveIndex); + FuncInfo->getScratchRSrcReg(), StackPtrReg, BasePtrFI); } if (ScratchExecCopy) { @@ -949,13 +941,13 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, } // In this case, spill the FP to a reserved VGPR. - if (HasFPSaveIndex && !SpillFPToMemory) { - const int FI = FuncInfo->FramePointerSaveIndex.getValue(); - assert(!MFI.isDeadObjectIndex(FI)); + if (FPSaveIndex && !spilledToMemory(MF, *FPSaveIndex)) { + const int FramePtrFI = *FPSaveIndex; + assert(!MFI.isDeadObjectIndex(FramePtrFI)); - assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill); + assert(MFI.getStackID(FramePtrFI) == TargetStackID::SGPRSpill); ArrayRef Spill = - FuncInfo->getSGPRToVGPRSpills(FI); + FuncInfo->getSGPRToVGPRSpills(FramePtrFI); assert(Spill.size() == 1); // Save FP before setting it up. @@ -967,8 +959,8 @@ void SIFrameLowering::emitPrologue(MachineFunction &MF, } // In this case, spill the BP to a reserved VGPR. - if (HasBPSaveIndex && !SpillBPToMemory) { - const int BasePtrFI = *FuncInfo->BasePointerSaveIndex; + if (BPSaveIndex && !spilledToMemory(MF, *BPSaveIndex)) { + const int BasePtrFI = *BPSaveIndex; assert(!MFI.isDeadObjectIndex(BasePtrFI)); assert(MFI.getStackID(BasePtrFI) == TargetStackID::SGPRSpill); @@ -1107,19 +1099,8 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF, const Register BasePtrReg = TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register(); - bool HasFPSaveIndex = FuncInfo->FramePointerSaveIndex.hasValue(); - bool SpillFPToMemory = false; - if (HasFPSaveIndex) { - SpillFPToMemory = MFI.getStackID(*FuncInfo->FramePointerSaveIndex) != - TargetStackID::SGPRSpill; - } - - bool HasBPSaveIndex = FuncInfo->BasePointerSaveIndex.hasValue(); - bool SpillBPToMemory = false; - if (HasBPSaveIndex) { - SpillBPToMemory = MFI.getStackID(*FuncInfo->BasePointerSaveIndex) != - TargetStackID::SGPRSpill; - } + Optional FPSaveIndex = FuncInfo->FramePointerSaveIndex; + Optional BPSaveIndex = FuncInfo->BasePointerSaveIndex; if (RoundedSize != 0 && hasFP(MF)) { BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::S_SUB_U32), StackPtrReg) @@ -1141,10 +1122,10 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF, } Register ScratchExecCopy; - if (HasFPSaveIndex) { - const int FI = FuncInfo->FramePointerSaveIndex.getValue(); - assert(!MFI.isDeadObjectIndex(FI)); - if (SpillFPToMemory) { + if (FPSaveIndex) { + const int FramePtrFI = *FPSaveIndex; + assert(!MFI.isDeadObjectIndex(FramePtrFI)); + if (spilledToMemory(MF, FramePtrFI)) { if (!ScratchExecCopy) ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false); @@ -1153,14 +1134,14 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF, if (!TempVGPR) report_fatal_error("failed to find free scratch register"); buildEpilogReload(ST, LiveRegs, MBB, MBBI, TII, TempVGPR, - FuncInfo->getScratchRSrcReg(), StackPtrReg, FI); + FuncInfo->getScratchRSrcReg(), StackPtrReg, FramePtrFI); BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), FramePtrReg) .addReg(TempVGPR, RegState::Kill); } else { // Reload from VGPR spill. - assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill); + assert(MFI.getStackID(FramePtrFI) == TargetStackID::SGPRSpill); ArrayRef Spill = - FuncInfo->getSGPRToVGPRSpills(FI); + FuncInfo->getSGPRToVGPRSpills(FramePtrFI); assert(Spill.size() == 1); BuildMI(MBB, MBBI, DL, TII->get(AMDGPU::V_READLANE_B32), FramePtrReg) .addReg(Spill[0].VGPR) @@ -1168,10 +1149,10 @@ void SIFrameLowering::emitEpilogue(MachineFunction &MF, } } - if (HasBPSaveIndex) { - const int BasePtrFI = *FuncInfo->BasePointerSaveIndex; + if (BPSaveIndex) { + const int BasePtrFI = *BPSaveIndex; assert(!MFI.isDeadObjectIndex(BasePtrFI)); - if (SpillBPToMemory) { + if (spilledToMemory(MF, BasePtrFI)) { if (!ScratchExecCopy) ScratchExecCopy = buildScratchExecCopy(LiveRegs, MF, MBB, MBBI, false); -- GitLab From 64687f2cc3f743a3e9073a4d7633d3691caaf18e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Tue, 16 Mar 2021 12:31:39 +0000 Subject: [PATCH 0031/1206] [X86][SSE] canonicalizeShuffleWithBinOps - add PERMILPS/PERMILPD + PERMPD/PERMQ + INSERTPS handling. Bail if the INSERTPS would introduce zeros across the binop. --- llvm/lib/Target/X86/X86ISelLowering.cpp | 12 ++++- llvm/test/CodeGen/X86/haddsub-undef.ll | 20 +++---- llvm/test/CodeGen/X86/horizontal-sum.ll | 52 ++++++++----------- llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 6 +-- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 0cd08b4c52aa..ea61af073d93 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -36852,7 +36852,9 @@ static SDValue canonicalizeShuffleWithBinOps(SDValue N, SelectionDAG &DAG, } case X86ISD::VBROADCAST: case X86ISD::MOVDDUP: - case X86ISD::PSHUFD: { + case X86ISD::PSHUFD: + case X86ISD::VPERMI: + case X86ISD::VPERMILPI: { if (N.getOperand(0).getValueType() == ShuffleVT && N->isOnlyUserOf(N.getOperand(0).getNode())) { SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0)); @@ -36882,6 +36884,14 @@ static SDValue canonicalizeShuffleWithBinOps(SDValue N, SelectionDAG &DAG, break; } // Binary and Binary+Permute Shuffles. + case X86ISD::INSERTPS: { + // Don't merge INSERTPS if it contains zero'd elements. + unsigned InsertPSMask = N.getConstantOperandVal(2); + unsigned ZeroMask = InsertPSMask & 0xF; + if (ZeroMask != 0) + break; + LLVM_FALLTHROUGH; + } case X86ISD::BLENDI: case X86ISD::SHUFP: case X86ISD::UNPCKH: diff --git a/llvm/test/CodeGen/X86/haddsub-undef.ll b/llvm/test/CodeGen/X86/haddsub-undef.ll index 1be27fa3846d..1c06749440ee 100644 --- a/llvm/test/CodeGen/X86/haddsub-undef.ll +++ b/llvm/test/CodeGen/X86/haddsub-undef.ll @@ -475,8 +475,8 @@ define <2 x double> @add_pd_010(<2 x double> %x) { ; AVX-SLOW-LABEL: add_pd_010: ; AVX-SLOW: # %bb.0: ; AVX-SLOW-NEXT: vmovddup {{.*#+}} xmm1 = xmm0[0,0] -; AVX-SLOW-NEXT: vaddpd %xmm0, %xmm1, %xmm0 ; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] +; AVX-SLOW-NEXT: vaddpd %xmm0, %xmm1, %xmm0 ; AVX-SLOW-NEXT: retq ; ; AVX-FAST-LABEL: add_pd_010: @@ -607,9 +607,9 @@ define <4 x float> @add_ps_017(<4 x float> %x) { ; ; AVX-SLOW-LABEL: add_ps_017: ; AVX-SLOW: # %bb.0: -; AVX-SLOW-NEXT: vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] -; AVX-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 -; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[3,3,3,3] +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[2,2,2,2] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 ; AVX-SLOW-NEXT: retq ; ; AVX-FAST-LABEL: add_ps_017: @@ -931,9 +931,9 @@ define <4 x float> @PR45747_1(<4 x float> %a, <4 x float> %b) nounwind { ; ; AVX-SLOW-LABEL: PR45747_1: ; AVX-SLOW: # %bb.0: -; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] -; AVX-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 -; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[2,2,2,2] +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm0[2,2,2,2] +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 ; AVX-SLOW-NEXT: retq ; ; AVX-FAST-LABEL: PR45747_1: @@ -963,9 +963,9 @@ define <4 x float> @PR45747_2(<4 x float> %a, <4 x float> %b) nounwind { ; ; AVX-SLOW-LABEL: PR45747_2: ; AVX-SLOW: # %bb.0: -; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] -; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 -; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm0 = xmm0[1,0] +; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm0 = xmm1[1,0] +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,3,1,1] +; AVX-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 ; AVX-SLOW-NEXT: retq ; ; AVX-FAST-LABEL: PR45747_2: diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll index 156a423970bc..f9ed90e34872 100644 --- a/llvm/test/CodeGen/X86/horizontal-sum.ll +++ b/llvm/test/CodeGen/X86/horizontal-sum.ll @@ -46,13 +46,12 @@ define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl ; AVX1-SLOW: # %bb.0: ; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0 ; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1 -; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[1,3],xmm1[1,1] -; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1] -; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm0, %xmm0 +; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,1] +; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1] ; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1 -; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] -; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 -; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[1] +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0] +; AVX1-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 ; AVX1-SLOW-NEXT: retq ; ; AVX-FAST-LABEL: pair_sum_v4f32_v4f32: @@ -66,13 +65,12 @@ define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x fl ; AVX2-SLOW: # %bb.0: ; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0 ; AVX2-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1 -; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[1,3],xmm1[1,1] -; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3] -; AVX2-SLOW-NEXT: vaddps %xmm2, %xmm0, %xmm0 +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,3] +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1] ; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1 -; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] -; AVX2-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 -; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[1] +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0] +; AVX2-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 ; AVX2-SLOW-NEXT: retq %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> @@ -648,17 +646,15 @@ define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, < ; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] ; AVX-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 ; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1] -; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm2[2,3] -; AVX-SLOW-NEXT: vaddps %xmm1, %xmm4, %xmm1 ; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3] -; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 -; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] -; AVX-SLOW-NEXT: vaddps %xmm3, %xmm1, %xmm1 -; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] +; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3] +; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3] +; AVX-SLOW-NEXT: vaddps %xmm3, %xmm4, %xmm4 +; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0] +; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2] ; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1 -; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] -; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1 -; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0 ; AVX-SLOW-NEXT: retq ; ; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32: @@ -669,16 +665,14 @@ define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, < ; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero ; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1 ; AVX-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1] -; AVX-FAST-NEXT: vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm2[2,3] -; AVX-FAST-NEXT: vaddps %xmm1, %xmm4, %xmm1 ; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3] -; AVX-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0 -; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1 -; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] +; AVX-FAST-NEXT: vblendps {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3] +; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm4 +; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0] +; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2] ; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 -; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] -; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 -; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX-FAST-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3] +; AVX-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0 ; AVX-FAST-NEXT: retq %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll index a2a2f1a43881..123fba437141 100644 --- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll +++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll @@ -1277,10 +1277,8 @@ define <16 x i8> @negative(<32 x i8> %v, <32 x i8> %w) nounwind { ; AVX512VL-LABEL: negative: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm0[2,4,6,8,10,12,14,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,18,20,22,24,26,28,30] -; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255] -; AVX512VL-NEXT: # ymm2 = mem[0,1,0,1] -; AVX512VL-NEXT: vpternlogq $206, %ymm1, %ymm0, %ymm2 -; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm2[0,3,2,3] +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,3,2,3] +; AVX512VL-NEXT: vpternlogq $248, {{.*}}(%rip), %xmm1, %xmm0 ; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 ; AVX512VL-NEXT: vzeroupper ; AVX512VL-NEXT: retq -- GitLab From a6f9cb6adc591d19a6c43234245de1e2048ed373 Mon Sep 17 00:00:00 2001 From: Hansang Bae Date: Thu, 11 Mar 2021 11:24:29 -0600 Subject: [PATCH 0032/1206] [OpenMP] Add runtime interface for OpenMP 5.1 error directive The proposed new interface is for supporting `at(execution)` clause in the error directive. Differential Revision: https://reviews.llvm.org/D98448 --- openmp/runtime/src/dllexports | 3 +- openmp/runtime/src/i18n/en_US.txt | 2 ++ openmp/runtime/src/include/omp-tools.h.var | 7 ++++ openmp/runtime/src/kmp.h | 7 ++++ openmp/runtime/src/kmp_csupport.cpp | 32 +++++++++++++++++ openmp/runtime/src/ompt-event-specific.h | 2 +- openmp/runtime/test/ompt/callback.h | 10 ++++++ openmp/runtime/test/ompt/misc/runtime_error.c | 35 +++++++++++++++++++ 8 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 openmp/runtime/test/ompt/misc/runtime_error.c diff --git a/openmp/runtime/src/dllexports b/openmp/runtime/src/dllexports index 79bca795d91e..c6be679494ce 100644 --- a/openmp/runtime/src/dllexports +++ b/openmp/runtime/src/dllexports @@ -390,8 +390,9 @@ kmpc_set_disp_num_buffers 267 __kmpc_taskred_init 277 __kmpc_taskred_modifier_init 278 __kmpc_omp_target_task_alloc 279 + __kmpc_error 281 __kmpc_masked 282 - __kmpc_end_masked 283 + __kmpc_end_masked 283 %endif # User API entry points that have both lower- and upper- case versions for Fortran. diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt index dc33fdbc7ff3..5aa3115dc5a4 100644 --- a/openmp/runtime/src/i18n/en_US.txt +++ b/openmp/runtime/src/i18n/en_US.txt @@ -455,6 +455,8 @@ AffHWSubsetManyDies "KMP_HW_SUBSET ignored: too many Dies requested." AffUseGlobCpuidL "%1$s: Affinity capable, using global cpuid leaf %2$d info" AffNotCapableUseLocCpuidL "%1$s: Affinity not capable, using local cpuid leaf %2$d info" AffNotUsingHwloc "%1$s: Affinity not capable, using hwloc." +UserDirectedError "%1$s: Encountered user-directed error: %2$s." +UserDirectedWarning "%1$s: Encountered user-directed warning: %2$s." FailedToCreateTeam "Failed to create teams between lower bound (%1$d) and upper bound (%2$d)." # -------------------------------------------------------------------------------------------------- diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var index 961e767c63c9..8e822750b53e 100644 --- a/openmp/runtime/src/include/omp-tools.h.var +++ b/openmp/runtime/src/include/omp-tools.h.var @@ -1099,6 +1099,13 @@ typedef void (*ompt_callback_error_t) ( const void *codeptr_ra ); +typedef struct ompt_record_error_t { + ompt_severity_t severity; + const char *message; + size_t length; + const void *codeptr_ra; +} ompt_record_error_t; + typedef struct ompd_address_t { ompd_seg_t segment; ompd_addr_t address; diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h index bf4c812a8875..c37e1d9feb57 100644 --- a/openmp/runtime/src/kmp.h +++ b/openmp/runtime/src/kmp.h @@ -4087,6 +4087,13 @@ extern void __kmp_hidden_helper_main_thread_release(); #define KMP_GTID_TO_SHADOW_GTID(gtid) \ ((gtid) % (__kmp_hidden_helper_threads_num - 1) + 2) +// Support for error directive +typedef enum kmp_severity_t { + severity_warning = 1, + severity_fatal = 2 +} kmp_severity_t; +extern void __kmpc_error(ident_t *loc, int severity, const char *message); + #ifdef __cplusplus } #endif diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp index 4f34f3ac87b0..59d0dec50534 100644 --- a/openmp/runtime/src/kmp_csupport.cpp +++ b/openmp/runtime/src/kmp_csupport.cpp @@ -4357,3 +4357,35 @@ int __kmpc_pause_resource(kmp_pause_status_t level) { } return __kmp_pause_resource(level); } + +void __kmpc_error(ident_t *loc, int severity, const char *message) { + if (!__kmp_init_serial) + __kmp_serial_initialize(); + + KMP_ASSERT(severity == severity_warning || severity == severity_fatal); + +#if OMPT_SUPPORT + if (ompt_enabled.enabled && ompt_enabled.ompt_callback_error) { + ompt_callbacks.ompt_callback(ompt_callback_error)( + (ompt_severity_t)severity, message, KMP_STRLEN(message), + OMPT_GET_RETURN_ADDRESS(0)); + } +#endif // OMPT_SUPPORT + + char *src_loc; + if (loc && loc->psource) { + kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, false); + src_loc = + __kmp_str_format("%s:%s:%s", str_loc.file, str_loc.line, str_loc.col); + __kmp_str_loc_free(&str_loc); + } else { + src_loc = __kmp_str_format("unknown"); + } + + if (severity == severity_warning) + KMP_WARNING(UserDirectedWarning, src_loc, message); + else + KMP_FATAL(UserDirectedError, src_loc, message); + + __kmp_str_free(&src_loc); +} diff --git a/openmp/runtime/src/ompt-event-specific.h b/openmp/runtime/src/ompt-event-specific.h index 799fa0d578ea..875d6921b7b7 100644 --- a/openmp/runtime/src/ompt-event-specific.h +++ b/openmp/runtime/src/ompt-event-specific.h @@ -106,6 +106,6 @@ #define ompt_callback_dispatch_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_error_implemented ompt_event_UNIMPLEMENTED +#define ompt_callback_error_implemented ompt_event_MAY_ALWAYS_OPTIONAL #endif diff --git a/openmp/runtime/test/ompt/callback.h b/openmp/runtime/test/ompt/callback.h index e426558bf869..c21b16741d33 100644 --- a/openmp/runtime/test/ompt/callback.h +++ b/openmp/runtime/test/ompt/callback.h @@ -1124,6 +1124,15 @@ on_ompt_callback_control_tool( return 0; //success } +static void on_ompt_callback_error(ompt_severity_t severity, + const char *message, size_t length, + const void *codeptr_ra) { + printf("%" PRIu64 ": ompt_event_runtime_error: severity=%" PRIu32 + ", message=%s, length=%" PRIu64 ", codeptr_ra=%p\n", + ompt_get_thread_data()->value, severity, message, (uint64_t)length, + codeptr_ra); +} + int ompt_initialize( ompt_function_lookup_t lookup, int initial_device_num, @@ -1173,6 +1182,7 @@ int ompt_initialize( register_ompt_callback(ompt_callback_task_dependence); register_ompt_callback(ompt_callback_thread_begin); register_ompt_callback(ompt_callback_thread_end); + register_ompt_callback(ompt_callback_error); printf("0: NULL_POINTER=%p\n", (void*)NULL); return 1; //success } diff --git a/openmp/runtime/test/ompt/misc/runtime_error.c b/openmp/runtime/test/ompt/misc/runtime_error.c new file mode 100644 index 000000000000..ee9e2e832bfa --- /dev/null +++ b/openmp/runtime/test/ompt/misc/runtime_error.c @@ -0,0 +1,35 @@ +// RUN: %libomp-compile-and-run 2>&1 | sort | FileCheck %s +// REQUIRES: ompt + +#include +#include +#include "callback.h" + +// TODO: use error directive when compiler suppors +typedef void ident_t; +extern void __kmpc_error(ident_t *, int, const char *); + +int main() { +#pragma omp parallel num_threads(2) + { + if (omp_get_thread_num() == 0) { + const char *msg = "User message goes here"; + printf("0: Message length=%" PRIu64 "\n", (uint64_t)strlen(msg)); + __kmpc_error(NULL, ompt_warning, msg); + } + } + return 0; +} + +// CHECK: {{^}}0: Message length=[[LENGTH:[0-9]+]] +// CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]] + +// CHECK: {{^}}[[PRIMARY_ID:[0-9]+]]: ompt_event_implicit_task_begin +// CHECK: {{^}}[[PRIMARY_ID]]: ompt_event_runtime_error +// CHECK-SAME: severity=1 +// CHECK-SAME: message=User message goes here +// CHECK-SAME: length=[[LENGTH]] +// CHECK-SAME: codeptr_ra={{0x[0-f]+}} + +// Message from runtime +// CHECK: {{^}}OMP: Warning{{.*}}User message goes here -- GitLab From f51427afb5333e5dd2eb04ea4630037667c64553 Mon Sep 17 00:00:00 2001 From: Michael Kruse Date: Tue, 16 Mar 2021 08:50:37 -0500 Subject: [PATCH 0033/1206] [Polly][Unroll] Fix unroll_double test. We enumerated the cross product Domain x Scatter, but sorted only be the scatter key. In case there are are multiple statement instances per scatter value, the order between statement instances of the same loop iteration was undefined. Propertly enumerate and sort only by the scatter value, and group the domains using the scatter dimension again. Thanks to Leonard Chan for the report. --- polly/lib/Transform/ScheduleTreeTransform.cpp | 18 ++++++------ .../ManualOptimization/unroll_double.ll | 28 +++++++++++++------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/polly/lib/Transform/ScheduleTreeTransform.cpp b/polly/lib/Transform/ScheduleTreeTransform.cpp index c6f9c32172be..32cef0ff959d 100644 --- a/polly/lib/Transform/ScheduleTreeTransform.cpp +++ b/polly/lib/Transform/ScheduleTreeTransform.cpp @@ -533,13 +533,13 @@ isl::schedule polly::applyFullUnroll(isl::schedule_node BandToUnroll) { PartialSchedUAff = PartialSchedUAff.intersect_domain(Domain); isl::union_map PartialSchedUMap = isl::union_map(PartialSchedUAff); - // Make consumable for the following code. - // Schedule at the beginning so it is at coordinate 0. - isl::union_set PartialSchedUSet = PartialSchedUMap.reverse().wrap(); + // Enumerator only the scatter elements. + isl::union_set ScatterList = PartialSchedUMap.range(); - SmallVector Elts; + // Enumerate all loop iterations. // TODO: Diagnose if not enumerable or depends on a parameter. - PartialSchedUSet.foreach_point([&Elts](isl::point P) -> isl::stat { + SmallVector Elts; + ScatterList.foreach_point([&Elts](isl::point P) -> isl::stat { Elts.push_back(P); return isl::stat::ok(); }); @@ -554,12 +554,10 @@ isl::schedule polly::applyFullUnroll(isl::schedule_node BandToUnroll) { // Convert the points to a sequence of filters. isl::union_set_list List = isl::union_set_list::alloc(Ctx, Elts.size()); for (isl::point P : Elts) { - isl::basic_set AsSet{P}; - - // Throw away the scatter dimension. - AsSet = AsSet.unwrap().range(); + // Determine the domains that map this scatter element. + isl::union_set DomainFilter = PartialSchedUMap.intersect_range(P).domain(); - List = List.add(AsSet); + List = List.add(DomainFilter); } // Replace original band with unrolled sequence. diff --git a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll index a9577271119a..dcd65b357d97 100644 --- a/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll +++ b/polly/test/ScheduleOptimizer/ManualOptimization/unroll_double.ll @@ -38,15 +38,25 @@ return: ; CHECK-LABEL: Printing analysis 'Polly - Optimize schedule of SCoP' for region: 'for => return' in function 'func': +; CHECK: domain: "{ Stmt_body[i0] : 0 <= i0 <= 11 }" +; CHECK sequence: +; CHECK: - filter: "{ Stmt_body[i0] : 0 <= i0 <= 3 }" +; CHECK sequence: ; CHECK: - filter: "{ Stmt_body[0] }" -; CHECK: - filter: "{ Stmt_body[1] }" -; CHECK: - filter: "{ Stmt_body[2] }" -; CHECK: - filter: "{ Stmt_body[3] }" +; CHECK: - filter: "{ Stmt_body[i0] : (-1 + i0) mod 4 = 0 }" +; CHECK: - filter: "{ Stmt_body[i0] : (2 + i0) mod 4 = 0 }" +; CHECK: - filter: "{ Stmt_body[i0] : (1 + i0) mod 4 = 0 }" +; CHECK sequence: +; CHECK: - filter: "{ Stmt_body[i0] : 4 <= i0 <= 7 }" +; CHECK sequence: ; CHECK: - filter: "{ Stmt_body[4] }" -; CHECK: - filter: "{ Stmt_body[5] }" -; CHECK: - filter: "{ Stmt_body[6] }" -; CHECK: - filter: "{ Stmt_body[7] }" +; CHECK: - filter: "{ Stmt_body[i0] : (-1 + i0) mod 4 = 0 }" +; CHECK: - filter: "{ Stmt_body[i0] : (2 + i0) mod 4 = 0 }" +; CHECK: - filter: "{ Stmt_body[i0] : (1 + i0) mod 4 = 0 }" +; CHECK sequence: +; CHECK: - filter: "{ Stmt_body[i0] : 8 <= i0 <= 11 }" +; CHECK sequence: ; CHECK: - filter: "{ Stmt_body[8] }" -; CHECK: - filter: "{ Stmt_body[9] }" -; CHECK: - filter: "{ Stmt_body[10] }" -; CHECK: - filter: "{ Stmt_body[11] }" +; CHECK: - filter: "{ Stmt_body[i0] : (-1 + i0) mod 4 = 0 }" +; CHECK: - filter: "{ Stmt_body[i0] : (2 + i0) mod 4 = 0 }" +; CHECK: - filter: "{ Stmt_body[i0] : (1 + i0) mod 4 = 0 }" -- GitLab From 9a5af541ee058b85a92113ecf9d38a06ef2b313d Mon Sep 17 00:00:00 2001 From: Nathan James Date: Tue, 16 Mar 2021 14:03:29 +0000 Subject: [PATCH 0034/1206] [clang-tidy] Remove readability-deleted-default The deprecation notice was cherrypicked to the release branch in https://github.com/llvm/llvm-project/commit/f8b32989241cca87a8690c8cc404f06ce1f90e4c so its safe to remove this for the 13.X release cycle. Reviewed By: aaron.ballman Differential Revision: https://reviews.llvm.org/D98612 --- .../clang-tidy/readability/CMakeLists.txt | 1 - .../readability/DeletedDefaultCheck.cpp | 68 ---------- .../readability/DeletedDefaultCheck.h | 35 ----- .../readability/ReadabilityTidyModule.cpp | 3 - clang-tools-extra/docs/ReleaseNotes.rst | 7 +- .../docs/clang-tidy/checks/list.rst | 1 - .../checks/readability-deleted-default.rst | 8 -- .../checkers/readability-deleted-default.cpp | 127 ------------------ 8 files changed, 3 insertions(+), 247 deletions(-) delete mode 100644 clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp delete mode 100644 clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h delete mode 100644 clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst delete mode 100644 clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp diff --git a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt index ecf37b5b9157..78a3851f66be 100644 --- a/clang-tools-extra/clang-tidy/readability/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/readability/CMakeLists.txt @@ -10,7 +10,6 @@ add_clang_library(clangTidyReadabilityModule ContainerSizeEmptyCheck.cpp ConvertMemberFunctionsToStatic.cpp DeleteNullPointerCheck.cpp - DeletedDefaultCheck.cpp ElseAfterReturnCheck.cpp FunctionCognitiveComplexityCheck.cpp FunctionSizeCheck.cpp diff --git a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp b/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp deleted file mode 100644 index ff2f00b94e36..000000000000 --- a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.cpp +++ /dev/null @@ -1,68 +0,0 @@ -//===--- DeletedDefaultCheck.cpp - clang-tidy------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "DeletedDefaultCheck.h" -#include "clang/AST/ASTContext.h" -#include "clang/ASTMatchers/ASTMatchFinder.h" - -using namespace clang::ast_matchers; - -namespace clang { -namespace tidy { -namespace readability { - -void DeletedDefaultCheck::registerMatchers(MatchFinder *Finder) { - // We match constructors/assignment operators that are: - // - explicitly marked '= default' - // - actually deleted - // - not in template instantiation. - // We bind the declaration to "method-decl" and also to "constructor" when - // it is a constructor. - - Finder->addMatcher( - cxxMethodDecl(anyOf(cxxConstructorDecl().bind("constructor"), - isCopyAssignmentOperator(), - isMoveAssignmentOperator()), - isDefaulted(), unless(isImplicit()), isDeleted(), - unless(isInstantiated())) - .bind("method-decl"), - this); -} - -void DeletedDefaultCheck::check(const MatchFinder::MatchResult &Result) { - const StringRef Message = "%0 is explicitly defaulted but implicitly " - "deleted, probably because %1; definition can " - "either be removed or explicitly deleted"; - if (const auto *Constructor = - Result.Nodes.getNodeAs("constructor")) { - auto Diag = diag(Constructor->getBeginLoc(), Message); - if (Constructor->isDefaultConstructor()) { - Diag << "default constructor" - << "a non-static data member or a base class is lacking a default " - "constructor"; - } else if (Constructor->isCopyConstructor()) { - Diag << "copy constructor" - << "a non-static data member or a base class is not copyable"; - } else if (Constructor->isMoveConstructor()) { - Diag << "move constructor" - << "a non-static data member or a base class is neither copyable " - "nor movable"; - } - } else if (const auto *Assignment = - Result.Nodes.getNodeAs("method-decl")) { - diag(Assignment->getBeginLoc(), Message) - << (Assignment->isCopyAssignmentOperator() ? "copy assignment operator" - : "move assignment operator") - << "a base class or a non-static data member is not assignable, e.g. " - "because the latter is marked 'const'"; - } -} - -} // namespace readability -} // namespace tidy -} // namespace clang diff --git a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h b/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h deleted file mode 100644 index ab7f141417d0..000000000000 --- a/clang-tools-extra/clang-tidy/readability/DeletedDefaultCheck.h +++ /dev/null @@ -1,35 +0,0 @@ -//===--- DeletedDefaultCheck.h - clang-tidy----------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_DELETED_DEFAULT_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_DELETED_DEFAULT_H - -#include "../ClangTidyCheck.h" - -namespace clang { -namespace tidy { -namespace readability { - -/// Checks when a constructor or an assignment operator is marked as '= default' -/// but is actually deleted by the compiler. -/// -/// For the user-facing documentation see: -/// http://clang.llvm.org/extra/clang-tidy/checks/readability-deleted-default.html -class DeletedDefaultCheck : public ClangTidyCheck { -public: - DeletedDefaultCheck(StringRef Name, ClangTidyContext *Context) - : ClangTidyCheck(Name, Context) {} - void registerMatchers(ast_matchers::MatchFinder *Finder) override; - void check(const ast_matchers::MatchFinder::MatchResult &Result) override; -}; - -} // namespace readability -} // namespace tidy -} // namespace clang - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_READABILITY_DELETED_DEFAULT_H diff --git a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp index bbd2e24e503b..088b9f09082e 100644 --- a/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/readability/ReadabilityTidyModule.cpp @@ -15,7 +15,6 @@ #include "ContainerSizeEmptyCheck.h" #include "ConvertMemberFunctionsToStatic.h" #include "DeleteNullPointerCheck.h" -#include "DeletedDefaultCheck.h" #include "ElseAfterReturnCheck.h" #include "FunctionCognitiveComplexityCheck.h" #include "FunctionSizeCheck.h" @@ -67,8 +66,6 @@ public: "readability-convert-member-functions-to-static"); CheckFactories.registerCheck( "readability-delete-null-pointer"); - CheckFactories.registerCheck( - "readability-deleted-default"); CheckFactories.registerCheck( "readability-else-after-return"); CheckFactories.registerCheck( diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 91207090902d..d9625db3f99e 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -112,11 +112,10 @@ Changes in existing checks function or assignment to ``nullptr``. Added support for pointers to ``std::unique_ptr``. -Deprecated checks -^^^^^^^^^^^^^^^^^ +Removed checks +^^^^^^^^^^^^^^ -- The :doc:`readability-deleted-default - ` check has been deprecated. +- The readability-deleted-default check has been removed. The clang warning `Wdefaulted-function-deleted `_ diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index e53c0e704963..bda9cc1aa015 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -280,7 +280,6 @@ Clang-Tidy Checks `readability-container-size-empty `_, "Yes" `readability-convert-member-functions-to-static `_, `readability-delete-null-pointer `_, "Yes" - `readability-deleted-default `_, `readability-else-after-return `_, "Yes" `readability-function-cognitive-complexity `_, `readability-function-size `_, diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst deleted file mode 100644 index 5f2083e00061..000000000000 --- a/clang-tools-extra/docs/clang-tidy/checks/readability-deleted-default.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. title:: clang-tidy - readability-deleted-default - -readability-deleted-default -=========================== - -This check has been deprecated prefer to make use of the `Wdefaulted-function-deleted -`_ -flag. diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp deleted file mode 100644 index 232f224128a6..000000000000 --- a/clang-tools-extra/test/clang-tidy/checkers/readability-deleted-default.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// RUN: %check_clang_tidy %s readability-deleted-default %t -- -- -fno-ms-compatibility - -class NoDefault { -public: - NoDefault() = delete; - NoDefault(NoDefault &&Other) = delete; - NoDefault(const NoDefault &Other) = delete; -}; - -class MissingEverything { -public: - MissingEverything() = default; - // CHECK-MESSAGES: warning: default constructor is explicitly defaulted but implicitly deleted, probably because a non-static data member or a base class is lacking a default constructor; definition can either be removed or explicitly deleted [readability-deleted-default] - MissingEverything(MissingEverything &&Other) = default; - // CHECK-MESSAGES: warning: move constructor is explicitly defaulted but implicitly deleted, probably because a non-static data member or a base class is neither copyable nor movable; definition can either be removed or explicitly deleted [readability-deleted-default] - MissingEverything(const MissingEverything &Other) = default; - // CHECK-MESSAGES: warning: copy constructor is explicitly defaulted but implicitly deleted, probably because a non-static data member or a base class is not copyable; definition can either be removed or explicitly deleted [readability-deleted-default] - MissingEverything &operator=(MissingEverything &&Other) = default; - // CHECK-MESSAGES: warning: move assignment operator is explicitly defaulted but implicitly deleted, probably because a base class or a non-static data member is not assignable, e.g. because the latter is marked 'const'; definition can either be removed or explicitly deleted [readability-deleted-default] - MissingEverything &operator=(const MissingEverything &Other) = default; - // CHECK-MESSAGES: warning: copy assignment operator is explicitly defaulted but implicitly deleted, probably because a base class or a non-static data member is not assignable, e.g. because the latter is marked 'const'; definition can either be removed or explicitly deleted [readability-deleted-default] - -private: - NoDefault ND; -}; - -class NotAssignable { -public: - NotAssignable(NotAssignable &&Other) = default; - NotAssignable(const NotAssignable &Other) = default; - NotAssignable &operator=(NotAssignable &&Other) = default; - // CHECK-MESSAGES: warning: move assignment operator is explicitly defaulted but implicitly deleted - NotAssignable &operator=(const NotAssignable &Other) = default; - // CHECK-MESSAGES: warning: copy assignment operator is explicitly defaulted but implicitly deleted - -private: - const int I = 0; -}; - -class Movable { -public: - Movable() = default; - Movable(Movable &&Other) = default; - Movable(const Movable &Other) = delete; - Movable &operator=(Movable &&Other) = default; - Movable &operator=(const Movable &Other) = delete; -}; - -class NotCopyable { -public: - NotCopyable(NotCopyable &&Other) = default; - NotCopyable(const NotCopyable &Other) = default; - // CHECK-MESSAGES: warning: copy constructor is explicitly defaulted but implicitly deleted - NotCopyable &operator=(NotCopyable &&Other) = default; - NotCopyable &operator=(const NotCopyable &Other) = default; - // CHECK-MESSAGES: warning: copy assignment operator is explicitly defaulted but implicitly deleted -private: - Movable M; -}; - -template class Templated { -public: - // No warning here, it is a templated class. - Templated() = default; - Templated(Templated &&Other) = default; - Templated(const Templated &Other) = default; - Templated &operator=(Templated &&Other) = default; - Templated &operator=(const Templated &Other) = default; - - class InnerTemplated { - public: - // This class is not in itself templated, but we still don't have warning. - InnerTemplated() = default; - InnerTemplated(InnerTemplated &&Other) = default; - InnerTemplated(const InnerTemplated &Other) = default; - InnerTemplated &operator=(InnerTemplated &&Other) = default; - InnerTemplated &operator=(const InnerTemplated &Other) = default; - - private: - T TVar; - }; - - class InnerNotTemplated { - public: - // This one could technically have warnings, but currently doesn't. - InnerNotTemplated() = default; - InnerNotTemplated(InnerNotTemplated &&Other) = default; - InnerNotTemplated(const InnerNotTemplated &Other) = default; - InnerNotTemplated &operator=(InnerNotTemplated &&Other) = default; - InnerNotTemplated &operator=(const InnerNotTemplated &Other) = default; - - private: - int I; - }; - -private: - const T TVar{}; -}; - -int FunctionWithInnerClass() { - class InnerNotAssignable { - public: - InnerNotAssignable &operator=(InnerNotAssignable &&Other) = default; - // CHECK-MESSAGES: warning: move assignment operator is explicitly defaulted but implicitly deleted - private: - const int I = 0; - }; - return 1; -}; - -template -int TemplateFunctionWithInnerClass() { - class InnerNotAssignable { - public: - InnerNotAssignable &operator=(InnerNotAssignable &&Other) = default; - private: - const T TVar{}; - }; - return 1; -}; - -void Foo() { - Templated V1; - Templated::InnerTemplated V2; - Templated::InnerNotTemplated V3; - TemplateFunctionWithInnerClass(); -} -- GitLab From 70aa319ee729227d036806fbfd00860db4565aec Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Tue, 16 Mar 2021 14:03:53 +0000 Subject: [PATCH 0035/1206] [gn build] Port 9a5af541ee05 --- .../secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn index fa1e24d32312..2c8d8cb5c0fc 100644 --- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn +++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn @@ -18,7 +18,6 @@ static_library("readability") { "ContainerSizeEmptyCheck.cpp", "ConvertMemberFunctionsToStatic.cpp", "DeleteNullPointerCheck.cpp", - "DeletedDefaultCheck.cpp", "ElseAfterReturnCheck.cpp", "FunctionCognitiveComplexityCheck.cpp", "FunctionSizeCheck.cpp", -- GitLab From 1cb15b10ea370178871769929ff9690f461191fc Mon Sep 17 00:00:00 2001 From: Aaron Puchert Date: Tue, 16 Mar 2021 15:17:43 +0100 Subject: [PATCH 0036/1206] Correct Doxygen syntax for inline code There is no syntax like {@code ...} in Doxygen, @code is a block command that ends with @endcode, and generally these are not enclosed in braces. The correct syntax for inline code snippets is @c . Reviewed By: aaron.ballman Differential Revision: https://reviews.llvm.org/D98665 --- clang/include/clang/Analysis/AnyCall.h | 10 +++++----- clang/include/clang/Analysis/RetainSummaryManager.h | 8 ++++---- clang/lib/Analysis/RetainSummaryManager.cpp | 4 ++-- clang/lib/Sema/SemaDeclAttr.cpp | 8 ++++---- .../Checkers/NonnullGlobalConstantsChecker.cpp | 4 ++-- .../Checkers/ObjCAutoreleaseWriteChecker.cpp | 2 +- .../RetainCountChecker/RetainCountDiagnostics.cpp | 6 +++--- .../Checkers/RunLoopAutoreleaseLeakChecker.cpp | 4 ++-- clang/lib/StaticAnalyzer/Core/BugReporter.cpp | 6 +++--- clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp | 2 +- clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 4 ++-- clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp | 8 ++++---- llvm/include/llvm/Support/GraphWriter.h | 4 ++-- 13 files changed, 35 insertions(+), 35 deletions(-) diff --git a/clang/include/clang/Analysis/AnyCall.h b/clang/include/clang/Analysis/AnyCall.h index 16371eb1da18..846ff7719ce1 100644 --- a/clang/include/clang/Analysis/AnyCall.h +++ b/clang/include/clang/Analysis/AnyCall.h @@ -107,8 +107,8 @@ public: } - /// If {@code E} is a generic call (to ObjC method /function/block/etc), - /// return a constructed {@code AnyCall} object. Return None otherwise. + /// If @c E is a generic call (to ObjC method /function/block/etc), + /// return a constructed @c AnyCall object. Return None otherwise. static Optional forExpr(const Expr *E) { if (const auto *ME = dyn_cast(E)) { return AnyCall(ME); @@ -127,8 +127,8 @@ public: } } - /// If {@code D} is a callable (Objective-C method or a function), return - /// a constructed {@code AnyCall} object. Return None otherwise. + /// If @c D is a callable (Objective-C method or a function), return + /// a constructed @c AnyCall object. Return None otherwise. // FIXME: block support. static Optional forDecl(const Decl *D) { if (const auto *FD = dyn_cast(D)) { @@ -186,7 +186,7 @@ public: } /// \returns Function identifier if it is a named declaration, - /// {@code nullptr} otherwise. + /// @c nullptr otherwise. const IdentifierInfo *getIdentifier() const { if (const auto *ND = dyn_cast_or_null(D)) return ND->getIdentifier(); diff --git a/clang/include/clang/Analysis/RetainSummaryManager.h b/clang/include/clang/Analysis/RetainSummaryManager.h index 6acefb563d8c..b7ccb0317830 100644 --- a/clang/include/clang/Analysis/RetainSummaryManager.h +++ b/clang/include/clang/Analysis/RetainSummaryManager.h @@ -613,8 +613,8 @@ class RetainSummaryManager { const FunctionType *FT, bool &AllowAnnotations); - /// Apply the annotation of {@code pd} in function {@code FD} - /// to the resulting summary stored in out-parameter {@code Template}. + /// Apply the annotation of @c pd in function @c FD + /// to the resulting summary stored in out-parameter @c Template. /// \return whether an annotation was applied. bool applyParamAnnotationEffect(const ParmVarDecl *pd, unsigned parm_idx, const NamedDecl *FD, @@ -715,8 +715,8 @@ private: /// Set argument types for arguments which are not doing anything. void updateSummaryForArgumentTypes(const AnyCall &C, const RetainSummary *&RS); - /// Determine whether a declaration {@code D} of correspondent type (return - /// type for functions/methods) {@code QT} has any of the given attributes, + /// Determine whether a declaration @c D of correspondent type (return + /// type for functions/methods) @c QT has any of the given attributes, /// provided they pass necessary validation checks AND tracking the given /// attribute is enabled. /// Returns the object kind corresponding to the present attribute, or None, diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp index 00bc854a8804..ecda47a67c1d 100644 --- a/clang/lib/Analysis/RetainSummaryManager.cpp +++ b/clang/lib/Analysis/RetainSummaryManager.cpp @@ -881,8 +881,8 @@ RetainSummaryManager::getRetEffectFromAnnotations(QualType RetTy, return None; } -/// \return Whether the chain of typedefs starting from {@code QT} -/// has a typedef with a given name {@code Name}. +/// \return Whether the chain of typedefs starting from @c QT +/// has a typedef with a given name @c Name. static bool hasTypedefNamed(QualType QT, StringRef Name) { while (auto *T = dyn_cast(QT)) { diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 459343637318..d713c1ff1016 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -390,10 +390,10 @@ appendDiagnostics(const Sema::SemaDiagnosticBuilder &Bldr, T &&ExtraArg, std::forward(ExtraArgs)...); } -/// Add an attribute {@code AttrType} to declaration {@code D}, provided that -/// {@code PassesCheck} is true. -/// Otherwise, emit diagnostic {@code DiagID}, passing in all parameters -/// specified in {@code ExtraArgs}. +/// Add an attribute @c AttrType to declaration @c D, provided that +/// @c PassesCheck is true. +/// Otherwise, emit diagnostic @c DiagID, passing in all parameters +/// specified in @c ExtraArgs. template static void handleSimpleAttributeOrDiagnose(Sema &S, Decl *D, const AttributeCommonInfo &CI, diff --git a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp index 80b705fb7392..c5437b16c688 100644 --- a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp @@ -89,7 +89,7 @@ void NonnullGlobalConstantsChecker::checkLocation(SVal location, bool isLoad, } /// \param V loaded lvalue. -/// \return whether {@code val} is a string-like const global. +/// \return whether @c val is a string-like const global. bool NonnullGlobalConstantsChecker::isGlobalConstString(SVal V) const { Optional RegionVal = V.getAs(); if (!RegionVal) @@ -127,7 +127,7 @@ bool NonnullGlobalConstantsChecker::isGlobalConstString(SVal V) const { return false; } -/// \return whether {@code type} is extremely unlikely to be null +/// \return whether @c type is extremely unlikely to be null bool NonnullGlobalConstantsChecker::isNonnullType(QualType Ty) const { if (Ty->isPointerType() && Ty->getPointeeType()->isCharType()) diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp index 7fd6e2abef4c..c8eab3288094 100644 --- a/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/ObjCAutoreleaseWriteChecker.cpp @@ -8,7 +8,7 @@ // // This file defines ObjCAutoreleaseWriteChecker which warns against writes // into autoreleased out parameters which cause crashes. -// An example of a problematic write is a write to {@code error} in the example +// An example of a problematic write is a write to @c error in the example // below: // // - (BOOL) mymethod:(NSError *__autoreleasing *)error list:(NSArray*) list { diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp index 1d903530201f..1fc3ee03d2e1 100644 --- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp @@ -89,7 +89,7 @@ static std::string getPrettyTypeName(QualType QT) { return QT.getAsString(); } -/// Write information about the type state change to {@code os}, +/// Write information about the type state change to @c os, /// return whether the note should be generated. static bool shouldGenerateNote(llvm::raw_string_ostream &os, const RefVal *PrevT, @@ -164,8 +164,8 @@ static bool shouldGenerateNote(llvm::raw_string_ostream &os, return true; } -/// Finds argument index of the out paramter in the call {@code S} -/// corresponding to the symbol {@code Sym}. +/// Finds argument index of the out paramter in the call @c S +/// corresponding to the symbol @c Sym. /// If none found, returns None. static Optional findArgIdxOfSymbol(ProgramStateRef CurrSt, const LocationContext *LCtx, diff --git a/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp index d9dc72ddaa21..2cf6c6ff47f1 100644 --- a/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/RunLoopAutoreleaseLeakChecker.cpp @@ -57,8 +57,8 @@ public: } // end anonymous namespace -/// \return Whether {@code A} occurs before {@code B} in traversal of -/// {@code Parent}. +/// \return Whether @c A occurs before @c B in traversal of +/// @c Parent. /// Conceptually a very incomplete/unsound approximation of happens-before /// relationship (A is likely to be evaluated before B), /// but useful enough in this case. diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp index bf38891b370a..b64c0798d7e2 100644 --- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp +++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp @@ -2738,8 +2738,8 @@ static void CompactMacroExpandedPieces(PathPieces &path, } /// Generate notes from all visitors. -/// Notes associated with {@code ErrorNode} are generated using -/// {@code getEndPath}, and the rest are generated with {@code VisitNode}. +/// Notes associated with @c ErrorNode are generated using +/// @c getEndPath, and the rest are generated with @c VisitNode. static std::unique_ptr generateVisitorsDiagnostics(PathSensitiveBugReport *R, const ExplodedNode *ErrorNode, @@ -2749,7 +2749,7 @@ generateVisitorsDiagnostics(PathSensitiveBugReport *R, PathSensitiveBugReport::VisitorList visitors; // Run visitors on all nodes starting from the node *before* the last one. - // The last node is reserved for notes generated with {@code getEndPath}. + // The last node is reserved for notes generated with @c getEndPath. const ExplodedNode *NextNode = ErrorNode->getFirstPred(); while (NextNode) { diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp index bc72f4f8c1e3..a12a78af7a9e 100644 --- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp +++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp @@ -1846,7 +1846,7 @@ static const MemRegion *getLocationRegionIfReference(const Expr *E, return nullptr; } -/// \return A subexpression of {@code Ex} which represents the +/// \return A subexpression of @c Ex which represents the /// expression-of-interest. static const Expr *peelOffOuterExpr(const Expr *Ex, const ExplodedNode *N) { diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp index a388fc9e6e26..37885ed0b7b9 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp @@ -3139,8 +3139,8 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { /// \p PreCallback: callback before break. /// \p PostCallback: callback after break. - /// \p Stop: stop iteration if returns {@code true} - /// \return Whether {@code Stop} ever returned {@code true}. + /// \p Stop: stop iteration if returns @c true + /// \return Whether @c Stop ever returned @c true. static bool traverseHiddenNodes( const ExplodedNode *N, llvm::function_ref PreCallback, diff --git a/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp index 9842f3ace484..7e7fe75082bb 100644 --- a/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp +++ b/clang/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp @@ -157,8 +157,8 @@ private: } // end of anonymous namespace -/// Print coverage information to output stream {@code o}. -/// May modify the used list of files {@code Fids} by inserting new ones. +/// Print coverage information to output stream @c o. +/// May modify the used list of files @c Fids by inserting new ones. static void printCoverage(const PathDiagnostic *D, unsigned InputIndentLevel, SmallVectorImpl &Fids, @@ -484,8 +484,8 @@ void PlistPrinter::ReportPopUp(raw_ostream &o, // Static function definitions. //===----------------------------------------------------------------------===// -/// Print coverage information to output stream {@code o}. -/// May modify the used list of files {@code Fids} by inserting new ones. +/// Print coverage information to output stream @c o. +/// May modify the used list of files @c Fids by inserting new ones. static void printCoverage(const PathDiagnostic *D, unsigned InputIndentLevel, SmallVectorImpl &Fids, diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h index 1f60fbc35126..4bb410d99f3f 100644 --- a/llvm/include/llvm/Support/GraphWriter.h +++ b/llvm/include/llvm/Support/GraphWriter.h @@ -318,8 +318,8 @@ raw_ostream &WriteGraph(raw_ostream &O, const GraphType &G, std::string createGraphFilename(const Twine &Name, int &FD); -/// Writes graph into a provided {@code Filename}. -/// If {@code Filename} is empty, generates a random one. +/// Writes graph into a provided @c Filename. +/// If @c Filename is empty, generates a random one. /// \return The resulting filename, or an empty string if writing /// failed. template -- GitLab From a33ce06cf59a31c96a484a11b526392d9f8c9548 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 16 Mar 2021 00:29:42 -0700 Subject: [PATCH 0037/1206] [RISCV] Improve i32 UADDSAT/USUBSAT on RV64. The default promotion uses zero extends that become shifts. We cam use sign extend instead which is better for RISCV. I've used two different implementations based on whether we have minu/maxu instructions. Differential Revision: https://reviews.llvm.org/D98683 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 25 +++++++++++++++++++ llvm/test/CodeGen/RISCV/uadd_sat.ll | 27 ++++++--------------- llvm/test/CodeGen/RISCV/uadd_sat_plus.ll | 26 ++++++-------------- llvm/test/CodeGen/RISCV/usub_sat.ll | 20 +++++---------- llvm/test/CodeGen/RISCV/usub_sat_plus.ll | 18 +++++--------- 5 files changed, 52 insertions(+), 64 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 9e45307f42c0..ade1bc20cad7 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -207,6 +207,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::UADDO, MVT::i32, Custom); setOperationAction(ISD::USUBO, MVT::i32, Custom); + setOperationAction(ISD::UADDSAT, MVT::i32, Custom); + setOperationAction(ISD::USUBSAT, MVT::i32, Custom); } if (!Subtarget.hasStdExtM()) { @@ -3521,6 +3523,29 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(Overflow); return; } + case ISD::UADDSAT: + case ISD::USUBSAT: { + assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && + "Unexpected custom legalisation"); + SDLoc DL(N); + if (Subtarget.hasStdExtZbb()) { + // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using + // sign extend allows overflow of the lower 32 bits to be detected on + // the promoted size. + SDValue LHS = + DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); + SDValue RHS = + DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1)); + SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS); + Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); + return; + } + + // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom + // promotion for UADDO/USUBO. + Results.push_back(expandAddSubSat(N, DAG)); + return; + } case ISD::BITCAST: { assert(((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF()) || diff --git a/llvm/test/CodeGen/RISCV/uadd_sat.ll b/llvm/test/CodeGen/RISCV/uadd_sat.ll index bac2a1915344..8f817b3e4972 100644 --- a/llvm/test/CodeGen/RISCV/uadd_sat.ll +++ b/llvm/test/CodeGen/RISCV/uadd_sat.ll @@ -24,19 +24,13 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind { ; ; RV64I-LABEL: func: ; RV64I: # %bb.0: -; RV64I-NEXT: slli a1, a1, 32 -; RV64I-NEXT: srli a1, a1, 32 -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a0, a0, 32 -; RV64I-NEXT: add a0, a0, a1 -; RV64I-NEXT: addi a1, zero, 1 -; RV64I-NEXT: slli a1, a1, 32 -; RV64I-NEXT: addi a1, a1, -1 -; RV64I-NEXT: bltu a0, a1, .LBB0_2 +; RV64I-NEXT: mv a2, a0 +; RV64I-NEXT: addw a1, a0, a1 +; RV64I-NEXT: addi a0, zero, -1 +; RV64I-NEXT: bltu a1, a2, .LBB0_2 ; RV64I-NEXT: # %bb.1: ; RV64I-NEXT: mv a0, a1 ; RV64I-NEXT: .LBB0_2: -; RV64I-NEXT: sext.w a0, a0 ; RV64I-NEXT: ret ; ; RV32IZbb-LABEL: func: @@ -48,16 +42,9 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind { ; ; RV64IZbb-LABEL: func: ; RV64IZbb: # %bb.0: -; RV64IZbb-NEXT: slli a1, a1, 32 -; RV64IZbb-NEXT: srli a1, a1, 32 -; RV64IZbb-NEXT: slli a0, a0, 32 -; RV64IZbb-NEXT: srli a0, a0, 32 -; RV64IZbb-NEXT: add a0, a0, a1 -; RV64IZbb-NEXT: addi a1, zero, 1 -; RV64IZbb-NEXT: slli a1, a1, 32 -; RV64IZbb-NEXT: addi a1, a1, -1 -; RV64IZbb-NEXT: minu a0, a0, a1 -; RV64IZbb-NEXT: sext.w a0, a0 +; RV64IZbb-NEXT: not a2, a1 +; RV64IZbb-NEXT: minu a0, a0, a2 +; RV64IZbb-NEXT: addw a0, a0, a1 ; RV64IZbb-NEXT: ret %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y); ret i32 %tmp; diff --git a/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll b/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll index 20bb7a24d754..589374493d71 100644 --- a/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll +++ b/llvm/test/CodeGen/RISCV/uadd_sat_plus.ll @@ -25,16 +25,11 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind { ; ; RV64I-LABEL: func32: ; RV64I: # %bb.0: -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a0, a0, 32 ; RV64I-NEXT: mul a1, a1, a2 -; RV64I-NEXT: slli a1, a1, 32 -; RV64I-NEXT: srli a1, a1, 32 -; RV64I-NEXT: add a0, a0, a1 -; RV64I-NEXT: addi a1, zero, 1 -; RV64I-NEXT: slli a1, a1, 32 -; RV64I-NEXT: addi a1, a1, -1 -; RV64I-NEXT: bltu a0, a1, .LBB0_2 +; RV64I-NEXT: addw a1, a0, a1 +; RV64I-NEXT: sext.w a2, a0 +; RV64I-NEXT: addi a0, zero, -1 +; RV64I-NEXT: bltu a1, a2, .LBB0_2 ; RV64I-NEXT: # %bb.1: ; RV64I-NEXT: mv a0, a1 ; RV64I-NEXT: .LBB0_2: @@ -50,16 +45,11 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind { ; ; RV64IZbb-LABEL: func32: ; RV64IZbb: # %bb.0: -; RV64IZbb-NEXT: slli a0, a0, 32 -; RV64IZbb-NEXT: srli a0, a0, 32 -; RV64IZbb-NEXT: mul a1, a1, a2 -; RV64IZbb-NEXT: slli a1, a1, 32 -; RV64IZbb-NEXT: srli a1, a1, 32 +; RV64IZbb-NEXT: mulw a1, a1, a2 +; RV64IZbb-NEXT: not a2, a1 +; RV64IZbb-NEXT: sext.w a0, a0 +; RV64IZbb-NEXT: minu a0, a0, a2 ; RV64IZbb-NEXT: add a0, a0, a1 -; RV64IZbb-NEXT: addi a1, zero, 1 -; RV64IZbb-NEXT: slli a1, a1, 32 -; RV64IZbb-NEXT: addi a1, a1, -1 -; RV64IZbb-NEXT: minu a0, a0, a1 ; RV64IZbb-NEXT: ret %a = mul i32 %y, %z %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %a) diff --git a/llvm/test/CodeGen/RISCV/usub_sat.ll b/llvm/test/CodeGen/RISCV/usub_sat.ll index 52657effaa97..353e8eaf838e 100644 --- a/llvm/test/CodeGen/RISCV/usub_sat.ll +++ b/llvm/test/CodeGen/RISCV/usub_sat.ll @@ -24,17 +24,13 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind { ; ; RV64I-LABEL: func: ; RV64I: # %bb.0: -; RV64I-NEXT: slli a1, a1, 32 -; RV64I-NEXT: srli a1, a1, 32 -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a2, a0, 32 -; RV64I-NEXT: sub a0, a2, a1 -; RV64I-NEXT: mv a1, zero -; RV64I-NEXT: bltu a2, a0, .LBB0_2 +; RV64I-NEXT: mv a2, a0 +; RV64I-NEXT: subw a1, a0, a1 +; RV64I-NEXT: mv a0, zero +; RV64I-NEXT: bltu a2, a1, .LBB0_2 ; RV64I-NEXT: # %bb.1: -; RV64I-NEXT: mv a1, a0 +; RV64I-NEXT: mv a0, a1 ; RV64I-NEXT: .LBB0_2: -; RV64I-NEXT: sext.w a0, a1 ; RV64I-NEXT: ret ; ; RV32IZbb-LABEL: func: @@ -45,11 +41,7 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind { ; ; RV64IZbb-LABEL: func: ; RV64IZbb: # %bb.0: -; RV64IZbb-NEXT: slli a2, a1, 32 -; RV64IZbb-NEXT: srli a2, a2, 32 -; RV64IZbb-NEXT: slli a0, a0, 32 -; RV64IZbb-NEXT: srli a0, a0, 32 -; RV64IZbb-NEXT: maxu a0, a0, a2 +; RV64IZbb-NEXT: maxu a0, a0, a1 ; RV64IZbb-NEXT: subw a0, a0, a1 ; RV64IZbb-NEXT: ret %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y); diff --git a/llvm/test/CodeGen/RISCV/usub_sat_plus.ll b/llvm/test/CodeGen/RISCV/usub_sat_plus.ll index 590df5e65a72..beeaf54e4238 100644 --- a/llvm/test/CodeGen/RISCV/usub_sat_plus.ll +++ b/llvm/test/CodeGen/RISCV/usub_sat_plus.ll @@ -25,14 +25,11 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind { ; ; RV64I-LABEL: func32: ; RV64I: # %bb.0: -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a3, a0, 32 -; RV64I-NEXT: mul a0, a1, a2 -; RV64I-NEXT: slli a0, a0, 32 -; RV64I-NEXT: srli a0, a0, 32 -; RV64I-NEXT: sub a1, a3, a0 +; RV64I-NEXT: mul a1, a1, a2 +; RV64I-NEXT: subw a1, a0, a1 +; RV64I-NEXT: sext.w a2, a0 ; RV64I-NEXT: mv a0, zero -; RV64I-NEXT: bltu a3, a1, .LBB0_2 +; RV64I-NEXT: bltu a2, a1, .LBB0_2 ; RV64I-NEXT: # %bb.1: ; RV64I-NEXT: mv a0, a1 ; RV64I-NEXT: .LBB0_2: @@ -47,11 +44,8 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind { ; ; RV64IZbb-LABEL: func32: ; RV64IZbb: # %bb.0: -; RV64IZbb-NEXT: slli a0, a0, 32 -; RV64IZbb-NEXT: srli a0, a0, 32 -; RV64IZbb-NEXT: mul a1, a1, a2 -; RV64IZbb-NEXT: slli a1, a1, 32 -; RV64IZbb-NEXT: srli a1, a1, 32 +; RV64IZbb-NEXT: mulw a1, a1, a2 +; RV64IZbb-NEXT: sext.w a0, a0 ; RV64IZbb-NEXT: maxu a0, a0, a1 ; RV64IZbb-NEXT: sub a0, a0, a1 ; RV64IZbb-NEXT: ret -- GitLab From 14bd44edc6afbb2bf7c823750b3d0f4e15fb02c8 Mon Sep 17 00:00:00 2001 From: Joe Ellis Date: Mon, 8 Mar 2021 12:52:40 +0000 Subject: [PATCH 0038/1206] [AArch64][SVEIntrinsicOpts] Factor out redundant SVE mul/fmul intrinsics This commit implements an IR-level optimization to eliminate idempotent SVE mul/fmul intrinsic calls. Currently, the following patterns are captured: fmul pg (dup_x 1.0) V => V mul pg (dup_x 1) V => V fmul pg V (dup_x 1.0) => V mul pg V (dup_x 1) => V fmul pg V (dup v pg 1.0) => V mul pg V (dup v pg 1) => V The result of this commit is that code such as: 1 #include 2 3 svfloat64_t foo(svfloat64_t a) { 4 svbool_t t = svptrue_b64(); 5 svfloat64_t b = svdup_f64(1.0); 6 return svmul_m(t, a, b); 7 } will lower to a nop. This commit does not capture all possibilities; only the simple cases described above. There is still room for further optimisation. Differential Revision: https://reviews.llvm.org/D98033 --- llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp | 76 +++++++++++ .../CodeGen/AArch64/sve-fmul-idempotency.ll | 123 ++++++++++++++++++ .../CodeGen/AArch64/sve-mul-idempotency.ll | 123 ++++++++++++++++++ 3 files changed, 322 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll create mode 100644 llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp index 3d9080f7997d..6b8cb786bb6c 100644 --- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp +++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp @@ -77,6 +77,7 @@ private: static bool optimizeConvertFromSVBool(IntrinsicInst *I); static bool optimizePTest(IntrinsicInst *I); + static bool optimizeVectorMul(IntrinsicInst *I); static bool processPhiNode(IntrinsicInst *I); }; @@ -366,6 +367,76 @@ bool SVEIntrinsicOpts::optimizePTest(IntrinsicInst *I) { return false; } +bool SVEIntrinsicOpts::optimizeVectorMul(IntrinsicInst *I) { + assert((I->getIntrinsicID() == Intrinsic::aarch64_sve_mul || + I->getIntrinsicID() == Intrinsic::aarch64_sve_fmul) && + "Unexpected opcode"); + + auto *OpPredicate = I->getOperand(0); + auto *OpMultiplicand = I->getOperand(1); + auto *OpMultiplier = I->getOperand(2); + + // Return true if a given instruction is an aarch64_sve_dup_x intrinsic call + // with a unit splat value, false otherwise. + auto IsUnitDupX = [](auto *I) { + auto *IntrI = dyn_cast(I); + if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup_x) + return false; + + auto *SplatValue = IntrI->getOperand(0); + return match(SplatValue, m_FPOne()) || match(SplatValue, m_One()); + }; + + // Return true if a given instruction is an aarch64_sve_dup intrinsic call + // with a unit splat value, false otherwise. + auto IsUnitDup = [](auto *I) { + auto *IntrI = dyn_cast(I); + if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup) + return false; + + auto *SplatValue = IntrI->getOperand(2); + return match(SplatValue, m_FPOne()) || match(SplatValue, m_One()); + }; + + bool Changed = true; + + // The OpMultiplier variable should always point to the dup (if any), so + // swap if necessary. + if (IsUnitDup(OpMultiplicand) || IsUnitDupX(OpMultiplicand)) + std::swap(OpMultiplier, OpMultiplicand); + + if (IsUnitDupX(OpMultiplier)) { + // [f]mul pg (dupx 1) %n => %n + I->replaceAllUsesWith(OpMultiplicand); + I->eraseFromParent(); + Changed = true; + } else if (IsUnitDup(OpMultiplier)) { + // [f]mul pg (dup pg 1) %n => %n + auto *DupInst = cast(OpMultiplier); + auto *DupPg = DupInst->getOperand(1); + // TODO: this is naive. The optimization is still valid if DupPg + // 'encompasses' OpPredicate, not only if they're the same predicate. + if (OpPredicate == DupPg) { + I->replaceAllUsesWith(OpMultiplicand); + I->eraseFromParent(); + Changed = true; + } + } + + // If an instruction was optimized out then it is possible that some dangling + // instructions are left. + if (Changed) { + auto *OpPredicateInst = dyn_cast(OpPredicate); + auto *OpMultiplierInst = dyn_cast(OpMultiplier); + if (OpMultiplierInst && OpMultiplierInst->use_empty()) + OpMultiplierInst->eraseFromParent(); + if (OpPredicateInst && OpPredicateInst->use_empty()) + OpPredicateInst->eraseFromParent(); + } + + return Changed; +} + bool SVEIntrinsicOpts::optimizeConvertFromSVBool(IntrinsicInst *I) { assert(I->getIntrinsicID() == Intrinsic::aarch64_sve_convert_from_svbool && "Unexpected opcode"); @@ -429,6 +500,9 @@ bool SVEIntrinsicOpts::optimizeIntrinsic(Instruction *I) { switch (IntrI->getIntrinsicID()) { case Intrinsic::aarch64_sve_convert_from_svbool: return optimizeConvertFromSVBool(IntrI); + case Intrinsic::aarch64_sve_fmul: + case Intrinsic::aarch64_sve_mul: + return optimizeVectorMul(IntrI); case Intrinsic::aarch64_sve_ptest_any: case Intrinsic::aarch64_sve_ptest_first: case Intrinsic::aarch64_sve_ptest_last: @@ -484,6 +558,8 @@ bool SVEIntrinsicOpts::runOnModule(Module &M) { case Intrinsic::aarch64_sve_ptest_first: case Intrinsic::aarch64_sve_ptest_last: case Intrinsic::aarch64_sve_ptrue: + case Intrinsic::aarch64_sve_mul: + case Intrinsic::aarch64_sve_fmul: for (User *U : F.users()) Functions.insert(cast(U)->getFunction()); break; diff --git a/llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll b/llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll new file mode 100644 index 000000000000..e716aa091c61 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-fmul-idempotency.ll @@ -0,0 +1,123 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -aarch64-sve-intrinsic-opts < %s 2>%t | FileCheck %s +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + +; Idempotent fmuls -- should compile to just a ret. +define @idempotent_fmul_f16( %pg, %a) { +; CHECK-LABEL: @idempotent_fmul_f16( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv8f16(half 1.0) + %2 = call @llvm.aarch64.sve.fmul.nxv8f16( %pg, %a, %1) + ret %2 +} + +define @idempotent_fmul_f32( %pg, %a) { +; CHECK-LABEL: @idempotent_fmul_f32( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv4f32(float 1.0) + %2 = call @llvm.aarch64.sve.fmul.nxv4f32( %pg, %a, %1) + ret %2 +} + +define @idempotent_fmul_f64( %pg, %a) { +; CHECK-LABEL: @idempotent_fmul_f64( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv2f64(double 1.0) + %2 = call @llvm.aarch64.sve.fmul.nxv2f64( %pg, %a, %1) + ret %2 +} + +define @idempotent_fmul_different_argument_order( %pg, %a) { +; CHECK-LABEL: @idempotent_fmul_different_argument_order( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv2f64(double 1.0) + ; Different argument order to the above tests. + %2 = call @llvm.aarch64.sve.fmul.nxv2f64( %pg, %1, %a) + ret %2 +} + +define @idempotent_fmul_with_predicated_dup( %pg, %a) { +; CHECK-LABEL: @idempotent_fmul_with_predicated_dup( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.nxv8f16( undef, %pg, half 1.0) + %2 = call @llvm.aarch64.sve.fmul.nxv8f16( %pg, %a, %1) + ret %2 +} + +define @idempotent_fmul_two_dups( %pg, %a) { + ; Edge case -- make sure that the case where we're fmultiplying two dups + ; together is sane. +; CHECK-LABEL: @idempotent_fmul_two_dups( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv8f16(half 0xH3C00) +; CHECK-NEXT: ret [[TMP1]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv8f16(half 1.0) + %2 = call @llvm.aarch64.sve.dup.x.nxv8f16(half 1.0) + %3 = call @llvm.aarch64.sve.fmul.nxv8f16( %pg, %1, %2) + ret %3 +} + +; Non-idempotent fmuls -- we don't expect these to be optimised out. +define @non_idempotent_fmul_f16( %pg, %a) { +; CHECK-LABEL: @non_idempotent_fmul_f16( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv8f16(half 0xH4000) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.fmul.nxv8f16( [[PG:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv8f16(half 2.0) + %2 = call @llvm.aarch64.sve.fmul.nxv8f16( %pg, %a, %1) + ret %2 +} + +define @non_idempotent_fmul_f32( %pg, %a) { +; CHECK-LABEL: @non_idempotent_fmul_f32( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv4f32(float 2.000000e+00) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.fmul.nxv4f32( [[PG:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv4f32(float 2.0) + %2 = call @llvm.aarch64.sve.fmul.nxv4f32( %pg, %a, %1) + ret %2 +} + +define @non_idempotent_fmul_f64( %pg, %a) { +; CHECK-LABEL: @non_idempotent_fmul_f64( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv2f64(double 2.000000e+00) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.fmul.nxv2f64( [[PG:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv2f64(double 2.0) + %2 = call @llvm.aarch64.sve.fmul.nxv2f64( %pg, %a, %1) + ret %2 +} + +define @non_idempotent_fmul_with_predicated_dup( %pg1, %pg2, %a) { + ; Different predicates +; CHECK-LABEL: @non_idempotent_fmul_with_predicated_dup( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.nxv2f64( undef, [[PG1:%.*]], double 1.000000e+00) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.fmul.nxv2f64( [[PG2:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.nxv2f64( undef, %pg1, double 1.0) + %2 = call @llvm.aarch64.sve.fmul.nxv2f64( %pg2, %a, %1) + ret %2 +} + +declare @llvm.aarch64.sve.dup.x.nxv8f16(half) +declare @llvm.aarch64.sve.dup.x.nxv4f32(float) +declare @llvm.aarch64.sve.dup.x.nxv2f64(double) + +declare @llvm.aarch64.sve.dup.nxv2f64(, , double) +declare @llvm.aarch64.sve.dup.nxv8f16(, , half) + +declare @llvm.aarch64.sve.fmul.nxv8f16(, , ) +declare @llvm.aarch64.sve.fmul.nxv4f32(, , ) +declare @llvm.aarch64.sve.fmul.nxv2f64(, , ) diff --git a/llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll b/llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll new file mode 100644 index 000000000000..d07e100f9d57 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-mul-idempotency.ll @@ -0,0 +1,123 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -aarch64-sve-intrinsic-opts < %s 2>%t | FileCheck %s +; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t + +; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it. +; WARN-NOT: warning + +; Idempotent muls -- should compile to just a ret. +define @idempotent_mul_i16( %pg, %a) { +; CHECK-LABEL: @idempotent_mul_i16( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv8i16(i16 1) + %2 = call @llvm.aarch64.sve.mul.nxv8i16( %pg, %a, %1) + ret %2 +} + +define @idempotent_mul_i32( %pg, %a) { +; CHECK-LABEL: @idempotent_mul_i32( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv4i32(i32 1) + %2 = call @llvm.aarch64.sve.mul.nxv4i32( %pg, %a, %1) + ret %2 +} + +define @idempotent_mul_i64( %pg, %a) { +; CHECK-LABEL: @idempotent_mul_i64( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv2i64(i64 1) + %2 = call @llvm.aarch64.sve.mul.nxv2i64( %pg, %a, %1) + ret %2 +} + +define @idempotent_mul_different_argument_order( %pg, %a) { +; CHECK-LABEL: @idempotent_mul_different_argument_order( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv2i64(i64 1) + ; Different argument order to the above tests. + %2 = call @llvm.aarch64.sve.mul.nxv2i64( %pg, %1, %a) + ret %2 +} + +define @idempotent_mul_with_predicated_dup( %pg, %a) { +; CHECK-LABEL: @idempotent_mul_with_predicated_dup( +; CHECK-NEXT: ret [[A:%.*]] +; + %1 = call @llvm.aarch64.sve.dup.nxv8i16( undef, %pg, i16 1) + %2 = call @llvm.aarch64.sve.mul.nxv8i16( %pg, %a, %1) + ret %2 +} + +define @idempotent_mul_two_dups( %pg, %a) { + ; Edge case -- make sure that the case where we're multiplying two dups + ; together is sane. +; CHECK-LABEL: @idempotent_mul_two_dups( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv8i16(i16 1) +; CHECK-NEXT: ret [[TMP1]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv8i16(i16 1) + %2 = call @llvm.aarch64.sve.dup.x.nxv8i16(i16 1) + %3 = call @llvm.aarch64.sve.mul.nxv8i16( %pg, %1, %2) + ret %3 +} + +; Non-idempotent muls -- we don't expect these to be optimised out. +define @non_idempotent_mul_i16( %pg, %a) { +; CHECK-LABEL: @non_idempotent_mul_i16( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv8i16(i16 2) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.mul.nxv8i16( [[PG:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv8i16(i16 2) + %2 = call @llvm.aarch64.sve.mul.nxv8i16( %pg, %a, %1) + ret %2 +} + +define @non_idempotent_mul_i32( %pg, %a) { +; CHECK-LABEL: @non_idempotent_mul_i32( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv4i32(i32 2) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.mul.nxv4i32( [[PG:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv4i32(i32 2) + %2 = call @llvm.aarch64.sve.mul.nxv4i32( %pg, %a, %1) + ret %2 +} + +define @non_idempotent_mul_i64( %pg, %a) { +; CHECK-LABEL: @non_idempotent_mul_i64( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.x.nxv2i64(i64 2) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.mul.nxv2i64( [[PG:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.x.nxv2i64(i64 2) + %2 = call @llvm.aarch64.sve.mul.nxv2i64( %pg, %a, %1) + ret %2 +} + +define @non_idempotent_mul_with_predicated_dup( %pg1, %pg2, %a) { + ; Different predicates +; CHECK-LABEL: @non_idempotent_mul_with_predicated_dup( +; CHECK-NEXT: [[TMP1:%.*]] = call @llvm.aarch64.sve.dup.nxv2i64( undef, [[PG1:%.*]], i64 1) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.aarch64.sve.mul.nxv2i64( [[PG2:%.*]], [[A:%.*]], [[TMP1]]) +; CHECK-NEXT: ret [[TMP2]] +; + %1 = call @llvm.aarch64.sve.dup.nxv2i64( undef, %pg1, i64 1) + %2 = call @llvm.aarch64.sve.mul.nxv2i64( %pg2, %a, %1) + ret %2 +} + +declare @llvm.aarch64.sve.dup.x.nxv8i16(i16) +declare @llvm.aarch64.sve.dup.x.nxv4i32(i32) +declare @llvm.aarch64.sve.dup.x.nxv2i64(i64) + +declare @llvm.aarch64.sve.dup.nxv2i64(, , i64) +declare @llvm.aarch64.sve.dup.nxv8i16(, , i16) + +declare @llvm.aarch64.sve.mul.nxv8i16(, , ) +declare @llvm.aarch64.sve.mul.nxv4i32(, , ) +declare @llvm.aarch64.sve.mul.nxv2i64(, , ) -- GitLab From 61ca706461c5e1edc18526c9ddc3250fe074ed94 Mon Sep 17 00:00:00 2001 From: David Zarzycki Date: Tue, 16 Mar 2021 10:52:11 -0400 Subject: [PATCH 0039/1206] [lit testing] Mark reorder.py as unavailable on Windows The test file has embedded slashes. This is fine for normal users that are just recording and reordering paths, but not great when the trace data is committed back to a repository that should work on both Unix and Windows. --- llvm/utils/lit/tests/reorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/utils/lit/tests/reorder.py b/llvm/utils/lit/tests/reorder.py index 8e5ecda22219..fb1c4bc41249 100644 --- a/llvm/utils/lit/tests/reorder.py +++ b/llvm/utils/lit/tests/reorder.py @@ -3,7 +3,7 @@ # RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig # RUN: %{lit} -j1 %{inputs}/reorder | FileCheck %s # RUN: not diff %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig -# RUN: cp %{inputs}/reorder/.lit_test_times.txt.orig %{inputs}/reorder/.lit_test_times.txt +# UNSUPPORTED: windows # END. # CHECK: -- Testing: 3 tests, 1 workers -- -- GitLab From 229eeb187d42fab9ef73be7ce82a50ed63914819 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 16 Mar 2021 07:49:24 -0700 Subject: [PATCH 0040/1206] [RISCV] Look through copies when trying to find an implicit def in addVSetVL. The InstrEmitter can sometimes insert a copy after an IMPLICIT_DEF before connecting it to the vector instruction. This occurs when constrainRegClass reduces to a class with less than 4 registers. I believe LMUL8 on masked instructions triggers this since the result can only use the v8, v16, or v24 register group as the mask is using v0. Reviewed By: frasercrmck Differential Revision: https://reviews.llvm.org/D98567 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 20 +++++- llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll | 6 +- .../test/CodeGen/RISCV/rvv/masked-load-int.ll | 8 +-- .../RISCV/rvv/tail-agnostic-impdef-copy.mir | 68 +++++++++++++++++++ 4 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index ade1bc20cad7..9bf9143b0558 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -4845,6 +4845,19 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI, return TailMBB; } +static MachineInstr *elideCopies(MachineInstr *MI, + const MachineRegisterInfo &MRI) { + while (true) { + if (!MI->isFullCopy()) + return MI; + if (!Register::isVirtualRegister(MI->getOperand(1).getReg())) + return nullptr; + MI = MRI.getVRegDef(MI->getOperand(1).getReg()); + if (!MI) + return nullptr; + } +} + static MachineBasicBlock *addVSetVL(MachineInstr &MI, MachineBasicBlock *BB, int VLIndex, unsigned SEWIndex, RISCVVLMUL VLMul, bool ForceTailAgnostic) { @@ -4905,8 +4918,11 @@ static MachineBasicBlock *addVSetVL(MachineInstr &MI, MachineBasicBlock *BB, // If the tied operand is an IMPLICIT_DEF we can keep TailAgnostic. const MachineOperand &UseMO = MI.getOperand(UseOpIdx); MachineInstr *UseMI = MRI.getVRegDef(UseMO.getReg()); - if (UseMI && UseMI->isImplicitDef()) - TailAgnostic = true; + if (UseMI) { + UseMI = elideCopies(UseMI, MRI); + if (UseMI && UseMI->isImplicitDef()) + TailAgnostic = true; + } } // For simplicity we reuse the vtype representation here. diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll b/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll index 85a7cd023f2d..93d2faef24a8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/masked-load-fp.ll @@ -126,7 +126,7 @@ declare @llvm.masked.load.nxv8f32(*, i3 define @masked_load_nxv8f64(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv8f64: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e64,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e64,m8,ta,mu ; CHECK-NEXT: vle64.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv8f64(* %a, i32 8, %mask, undef) @@ -148,7 +148,7 @@ declare @llvm.masked.load.nxv16f16(*, i define @masked_load_nxv16f32(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv16f32: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e32,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e32,m8,ta,mu ; CHECK-NEXT: vle32.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv16f32(* %a, i32 4, %mask, undef) @@ -159,7 +159,7 @@ declare @llvm.masked.load.nxv16f32(*, define @masked_load_nxv32f16(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv32f16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e16,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e16,m8,ta,mu ; CHECK-NEXT: vle16.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv32f16(* %a, i32 2, %mask, undef) diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll b/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll index c7133f6d7900..50136e98f64a 100644 --- a/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll +++ b/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll @@ -170,7 +170,7 @@ declare @llvm.masked.load.nxv8i32(*, i32, < define @masked_load_nxv8i64(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv8i64: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e64,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e64,m8,ta,mu ; CHECK-NEXT: vle64.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv8i64(* %a, i32 8, %mask, undef) @@ -203,7 +203,7 @@ declare @llvm.masked.load.nxv16i16(*, i32 define @masked_load_nxv16i32(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv16i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e32,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e32,m8,ta,mu ; CHECK-NEXT: vle32.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv16i32(* %a, i32 4, %mask, undef) @@ -225,7 +225,7 @@ declare @llvm.masked.load.nxv32i8(*, i32, < define @masked_load_nxv32i16(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv32i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e16,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e16,m8,ta,mu ; CHECK-NEXT: vle16.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv32i16(* %a, i32 2, %mask, undef) @@ -236,7 +236,7 @@ declare @llvm.masked.load.nxv32i16(*, i32 define @masked_load_nxv64i8(* %a, %mask) nounwind { ; CHECK-LABEL: masked_load_nxv64i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a1, zero, e8,m8,tu,mu +; CHECK-NEXT: vsetvli a1, zero, e8,m8,ta,mu ; CHECK-NEXT: vle8.v v8, (a0), v0.t ; CHECK-NEXT: ret %load = call @llvm.masked.load.nxv64i8(* %a, i32 1, %mask, undef) diff --git a/llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir b/llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir new file mode 100644 index 000000000000..5e34d25826b2 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/tail-agnostic-impdef-copy.mir @@ -0,0 +1,68 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc %s -mtriple=riscv64 -mattr=experimental-v -riscv-v-vector-bits-min=128 -run-pass=finalize-isel -o - | FileCheck %s + +# This test makes sure we peak through the COPY instruction between the +# IMPLICIT_DEF and PseudoVLE64_V_M8_MASK in order to select the tail agnostic +# policy. The test is working if the second argument to PseudoVSETVLI has bit 6 +# set. + +--- | + ; ModuleID = 'test.ll' + source_filename = "test.ll" + target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128" + target triple = "riscv64" + + ; Function Attrs: nounwind + define @masked_load_nxv8i64(* %a, %mask) #0 { + %load = call @llvm.masked.load.nxv8i64.p0nxv8i64(* %a, i32 8, %mask, undef) + ret %load + } + + ; Function Attrs: argmemonly nofree nosync nounwind readonly willreturn + declare @llvm.masked.load.nxv8i64.p0nxv8i64(*, i32 immarg, , ) #1 + + attributes #0 = { nounwind "target-features"="+experimental-v" } + attributes #1 = { argmemonly nofree nosync nounwind readonly willreturn "target-features"="+experimental-v" } + +... +--- +name: masked_load_nxv8i64 +alignment: 4 +tracksRegLiveness: true +registers: + - { id: 0, class: gpr } + - { id: 1, class: vr } + - { id: 2, class: vrm8nov0 } + - { id: 3, class: vrm8 } + - { id: 4, class: vrm8nov0 } +liveins: + - { reg: '$x10', virtual-reg: '%0' } + - { reg: '$v0', virtual-reg: '%1' } +frameInfo: + maxAlignment: 1 +machineFunctionInfo: {} +body: | + bb.0 (%ir-block.0): + liveins: $x10, $v0 + + ; CHECK-LABEL: name: masked_load_nxv8i64 + ; CHECK: liveins: $x10, $v0 + ; CHECK: [[COPY:%[0-9]+]]:vr = COPY $v0 + ; CHECK: [[COPY1:%[0-9]+]]:gpr = COPY $x10 + ; CHECK: $v0 = COPY [[COPY]] + ; CHECK: [[DEF:%[0-9]+]]:vrm8 = IMPLICIT_DEF + ; CHECK: [[COPY2:%[0-9]+]]:vrm8nov0 = COPY [[DEF]] + ; CHECK: dead %5:gpr = PseudoVSETVLI $x0, 91, implicit-def $vl, implicit-def $vtype + ; CHECK: [[PseudoVLE64_V_M8_MASK:%[0-9]+]]:vrm8nov0 = PseudoVLE64_V_M8_MASK [[COPY2]], [[COPY1]], $v0, $noreg, 64, implicit $vl, implicit $vtype :: (load 64 from %ir.a, align 8) + ; CHECK: $v8m8 = COPY [[PseudoVLE64_V_M8_MASK]] + ; CHECK: PseudoRET implicit $v8m8 + %1:vr = COPY $v0 + %0:gpr = COPY $x10 + $v0 = COPY %1 + %3:vrm8 = IMPLICIT_DEF + %4:vrm8nov0 = COPY %3 + %2:vrm8nov0 = PseudoVLE64_V_M8_MASK %4, %0, $v0, $x0, 64, implicit $vl, implicit $vtype :: (load 64 from %ir.a, align 8) + $v8m8 = COPY %2 + PseudoRET implicit $v8m8 + +... -- GitLab From 5097143f0e7124d73646daa5de5d205579b9f7d2 Mon Sep 17 00:00:00 2001 From: Max Kazantsev Date: Tue, 16 Mar 2021 21:27:25 +0700 Subject: [PATCH 0041/1206] [SCEV][NFC] Move check up the stack One of (and primary) callers of isBasicBlockEntryGuardedByCond is isKnownPredicateAt, which makes isKnownPredicate check before it. It already makes non-recursive check inside. So, on this execution path this check is made twice. The only other caller is isLoopEntryGuardedByCond. Moving the check there should save some compile time. --- llvm/lib/Analysis/ScalarEvolution.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index c94aca576282..ddb56562799e 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -9991,9 +9991,6 @@ bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB, assert(!verifyFunction(*BB->getParent(), &dbgs()) && "This cannot be done on broken IR!"); - if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS)) - return true; - // If we cannot prove strict comparison (e.g. a > b), maybe we can prove // the facts (a >= b && a != b) separately. A typical situation is when the // non-strict comparison is known from ranges and non-equality is known from @@ -10102,6 +10099,10 @@ bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, "LHS is not available at Loop Entry"); assert(isAvailableAtLoopEntry(RHS, L) && "RHS is not available at Loop Entry"); + + if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS)) + return true; + return isBasicBlockEntryGuardedByCond(L->getHeader(), Pred, LHS, RHS); } -- GitLab From ff2dd8a21251ba0e6d284c9823ff1118a23b59ae Mon Sep 17 00:00:00 2001 From: Joe Ellis Date: Thu, 4 Mar 2021 09:06:49 +0000 Subject: [PATCH 0042/1206] [AArch64][SVE] Fold vector ZExt/SExt into gather loads where possible This commit folds sxtw'd or uxtw'd offsets into gather loads where possible with a DAGCombine optimization. As an example, the following code: 1 #include 2 3 svuint64_t func(svbool_t pred, const int32_t *base, svint64_t offsets) { 4 return svld1sw_gather_s64offset_u64( 5 pred, base, svextw_s64_x(pred, offsets) 6 ); 7 } would previously lower to the following assembly: sxtw z0.d, p0/m, z0.d ld1sw { z0.d }, p0/z, [x0, z0.d] ret but now lowers to: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw] ret Differential Revision: https://reviews.llvm.org/D97858 --- .../Target/AArch64/AArch64ISelLowering.cpp | 72 ++++++ ...insics-gather-loads-64bit-scaled-offset.ll | 187 ++++++++++++++ ...sics-gather-loads-64bit-unscaled-offset.ll | 243 ++++++++++++++++++ 3 files changed, 502 insertions(+) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 613895dd3625..e61a6edac34c 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -14400,6 +14400,63 @@ static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG) { return SDValue(); } +static SDValue performGLD1Combine(SDNode *N, SelectionDAG &DAG) { + unsigned Opc = N->getOpcode(); + + assert(((Opc >= AArch64ISD::GLD1_MERGE_ZERO && // unsigned gather loads + Opc <= AArch64ISD::GLD1_IMM_MERGE_ZERO) || + (Opc >= AArch64ISD::GLD1S_MERGE_ZERO && // signed gather loads + Opc <= AArch64ISD::GLD1S_IMM_MERGE_ZERO)) && + "Invalid opcode."); + + const bool Scaled = Opc == AArch64ISD::GLD1_SCALED_MERGE_ZERO || + Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO; + const bool Signed = Opc == AArch64ISD::GLD1S_MERGE_ZERO || + Opc == AArch64ISD::GLD1S_SCALED_MERGE_ZERO; + const bool Extended = Opc == AArch64ISD::GLD1_SXTW_MERGE_ZERO || + Opc == AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO || + Opc == AArch64ISD::GLD1_UXTW_MERGE_ZERO || + Opc == AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO; + + SDLoc DL(N); + SDValue Chain = N->getOperand(0); + SDValue Pg = N->getOperand(1); + SDValue Base = N->getOperand(2); + SDValue Offset = N->getOperand(3); + SDValue Ty = N->getOperand(4); + + EVT ResVT = N->getValueType(0); + + const auto OffsetOpc = Offset.getOpcode(); + const bool OffsetIsZExt = + OffsetOpc == AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU; + const bool OffsetIsSExt = + OffsetOpc == AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU; + + // Fold sign/zero extensions of vector offsets into GLD1 nodes where possible. + if (!Extended && (OffsetIsSExt || OffsetIsZExt)) { + SDValue ExtPg = Offset.getOperand(0); + VTSDNode *ExtFrom = cast(Offset.getOperand(2).getNode()); + EVT ExtFromEVT = ExtFrom->getVT().getVectorElementType(); + + // If the predicate for the sign- or zero-extended offset is the + // same as the predicate used for this load and the sign-/zero-extension + // was from a 32-bits... + if (ExtPg == Pg && ExtFromEVT == MVT::i32) { + SDValue UnextendedOffset = Offset.getOperand(1); + + unsigned NewOpc = getGatherVecOpcode(Scaled, OffsetIsSExt, true); + if (Signed) + NewOpc = getSignExtendedGatherOpcode(NewOpc); + + return DAG.getNode(NewOpc, DL, {ResVT, MVT::Other}, + {Chain, Pg, Base, UnextendedOffset, Ty}); + } + } + + return SDValue(); +} + /// Target-specific DAG combine function for post-increment LD1 (lane) and /// post-increment LD1R. static SDValue performPostLD1Combine(SDNode *N, @@ -15777,6 +15834,21 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, return performNVCASTCombine(N); case AArch64ISD::UZP1: return performUzpCombine(N, DAG); + case AArch64ISD::GLD1_MERGE_ZERO: + case AArch64ISD::GLD1_SCALED_MERGE_ZERO: + case AArch64ISD::GLD1_UXTW_MERGE_ZERO: + case AArch64ISD::GLD1_SXTW_MERGE_ZERO: + case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO: + case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO: + case AArch64ISD::GLD1_IMM_MERGE_ZERO: + case AArch64ISD::GLD1S_MERGE_ZERO: + case AArch64ISD::GLD1S_SCALED_MERGE_ZERO: + case AArch64ISD::GLD1S_UXTW_MERGE_ZERO: + case AArch64ISD::GLD1S_SXTW_MERGE_ZERO: + case AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO: + case AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO: + case AArch64ISD::GLD1S_IMM_MERGE_ZERO: + return performGLD1Combine(N, DAG); case ISD::INSERT_VECTOR_ELT: return performPostLD1Combine(N, DCI, true); case ISD::EXTRACT_VECTOR_ELT: diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll index 64cb89edd679..57778847b545 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-scaled-offset.ll @@ -78,7 +78,194 @@ define @gld1sw_index( %pg, i32* %base, %res } +; +; LD1H, LD1W, LD1D: base + 64-bit sxtw'd scaled offset +; e.g. ld1h z0.d, p0/z, [x0, z0.d, sxtw #1] +; + +define @gld1h_index_sxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1h_index_sxtw +; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i16( %pg, + i16* %base, + %sxtw) + %res = zext %load to + ret %res +} + +define @gld1w_index_sxtw( %pg, i32* %base, %b) { +; CHECK-LABEL: gld1w_index_sxtw +; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i32( %pg, + i32* %base, + %sxtw) + %res = zext %load to + ret %res +} + +define @gld1d_index_sxtw( %pg, i64* %base, %b) { +; CHECK-LABEL: gld1d_index_sxtw +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i64( %pg, + i64* %base, + %sxtw) + ret %load +} + +define @gld1d_index_double_sxtw( %pg, double* %base, %b) { +; CHECK-LABEL: gld1d_index_double_sxtw +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2f64( %pg, + double* %base, + %sxtw) + ret %load +} + +; +; LD1SH, LD1SW: base + 64-bit sxtw'd scaled offset +; e.g. ld1sh z0.d, p0/z, [x0, z0.d, sxtw #1] +; + +define @gld1sh_index_sxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1sh_index_sxtw +; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i16( %pg, + i16* %base, + %sxtw) + %res = sext %load to + ret %res +} + +define @gld1sw_index_sxtw( %pg, i32* %base, %b) { +; CHECK-LABEL: gld1sw_index_sxtw +; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i32( %pg, + i32* %base, + %sxtw) + %res = sext %load to + ret %res +} + +; +; LD1H, LD1W, LD1D: base + 64-bit sxtw'd scaled offset +; e.g. ld1h z0.d, p0/z, [x0, z0.d, uxtw #1] +; + +define @gld1h_index_uxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1h_index_uxtw +; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i16( %pg, + i16* %base, + %uxtw) + %res = zext %load to + ret %res +} + +define @gld1w_index_uxtw( %pg, i32* %base, %b) { +; CHECK-LABEL: gld1w_index_uxtw +; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i32( %pg, + i32* %base, + %uxtw) + %res = zext %load to + ret %res +} + +define @gld1d_index_uxtw( %pg, i64* %base, %b) { +; CHECK-LABEL: gld1d_index_uxtw +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i64( %pg, + i64* %base, + %uxtw) + ret %load +} + +define @gld1d_index_double_uxtw( %pg, double* %base, %b) { +; CHECK-LABEL: gld1d_index_double_uxtw +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2f64( %pg, + double* %base, + %uxtw) + ret %load +} + +; +; LD1SH, LD1SW: base + 64-bit uxtw'd scaled offset +; e.g. ld1sh z0.d, p0/z, [x0, z0.d, uxtw #1] +; + +define @gld1sh_index_uxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1sh_index_uxtw +; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i16( %pg, + i16* %base, + %uxtw) + %res = sext %load to + ret %res +} + +define @gld1sw_index_uxtw( %pg, i32* %base, %b) { +; CHECK-LABEL: gld1sw_index_uxtw +; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.index.nxv2i32( %pg, + i32* %base, + %uxtw) + %res = sext %load to + ret %res +} + declare @llvm.aarch64.sve.ld1.gather.index.nxv2i16(, i16*, ) declare @llvm.aarch64.sve.ld1.gather.index.nxv2i32(, i32*, ) declare @llvm.aarch64.sve.ld1.gather.index.nxv2i64(, i64*, ) declare @llvm.aarch64.sve.ld1.gather.index.nxv2f64(, double*, ) + +declare @llvm.aarch64.sve.sxtw.nxv2i64(, , ) +declare @llvm.aarch64.sve.uxtw.nxv2i64(, , ) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll index 7cf641a26427..21c08d152ef3 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-gather-loads-64bit-unscaled-offset.ll @@ -100,8 +100,251 @@ define @gld1sw_d( %pg, i32* %base, %res } +; +; LD1B, LD1W, LD1H, LD1D: base + 64-bit sxtw'd unscaled offset +; e.g. ld1h { z0.d }, p0/z, [x0, z0.d, sxtw] +; + +define @gld1b_d_sxtw( %pg, i8* %base, %b) { +; CHECK-LABEL: gld1b_d_sxtw: +; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i8( %pg, + i8* %base, + %sxtw) + %res = zext %load to + ret %res +} + +define @gld1h_d_sxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1h_d_sxtw: +; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i16( %pg, + i16* %base, + %sxtw) + %res = zext %load to + ret %res +} + +define @gld1w_d_sxtw( %pg, i32* %base, %offsets) { +; CHECK-LABEL: gld1w_d_sxtw: +; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %offsets) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i32( %pg, + i32* %base, + %sxtw) + %res = zext %load to + ret %res +} + +define @gld1d_d_sxtw( %pg, i64* %base, %b) { +; CHECK-LABEL: gld1d_d_sxtw: +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i64( %pg, + i64* %base, + %sxtw) + ret %load +} + +define @gld1d_d_double_sxtw( %pg, double* %base, %b) { +; CHECK-LABEL: gld1d_d_double_sxtw: +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2f64( %pg, + double* %base, + %sxtw) + ret %load +} + +; +; LD1SB, LD1SW, LD1SH: base + 64-bit sxtw'd unscaled offset +; e.g. ld1sh { z0.d }, p0/z, [x0, z0.d] +; + +define @gld1sb_d_sxtw( %pg, i8* %base, %b) { +; CHECK-LABEL: gld1sb_d_sxtw: +; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i8( %pg, + i8* %base, + %sxtw) + %res = sext %load to + ret %res +} + +define @gld1sh_d_sxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1sh_d_sxtw: +; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i16( %pg, + i16* %base, + %sxtw) + %res = sext %load to + ret %res +} + +define @gld1sw_d_sxtw( %pg, i32* %base, %offsets) { +; CHECK-LABEL: gld1sw_d_sxtw: +; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw] +; CHECK-NEXT: ret + %sxtw = call @llvm.aarch64.sve.sxtw.nxv2i64( undef, + %pg, + %offsets) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i32( %pg, + i32* %base, + %sxtw) + %res = sext %load to + ret %res +} + +; +; LD1B, LD1W, LD1H, LD1D: base + 64-bit uxtw'd unscaled offset +; e.g. ld1h { z0.d }, p0/z, [x0, z0.d, uxtw] +; + +define @gld1b_d_uxtw( %pg, i8* %base, %b) { +; CHECK-LABEL: gld1b_d_uxtw: +; CHECK: ld1b { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i8( %pg, + i8* %base, + %uxtw) + %res = zext %load to + ret %res +} + +define @gld1h_d_uxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1h_d_uxtw: +; CHECK: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i16( %pg, + i16* %base, + %uxtw) + %res = zext %load to + ret %res +} + +define @gld1w_d_uxtw( %pg, i32* %base, %offsets) { +; CHECK-LABEL: gld1w_d_uxtw: +; CHECK: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %offsets) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i32( %pg, + i32* %base, + %uxtw) + %res = zext %load to + ret %res +} + +define @gld1d_d_uxtw( %pg, i64* %base, %b) { +; CHECK-LABEL: gld1d_d_uxtw: +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i64( %pg, + i64* %base, + %uxtw) + ret %load +} + +define @gld1d_d_double_uxtw( %pg, double* %base, %b) { +; CHECK-LABEL: gld1d_d_double_uxtw: +; CHECK: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2f64( %pg, + double* %base, + %uxtw) + ret %load +} + +; +; LD1SB, LD1SW, LD1SH: base + 64-bit uxtw'd unscaled offset +; e.g. ld1sh { z0.d }, p0/z, [x0, z0.d] +; + +define @gld1sb_d_uxtw( %pg, i8* %base, %b) { +; CHECK-LABEL: gld1sb_d_uxtw: +; CHECK: ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i8( %pg, + i8* %base, + %uxtw) + %res = sext %load to + ret %res +} + +define @gld1sh_d_uxtw( %pg, i16* %base, %b) { +; CHECK-LABEL: gld1sh_d_uxtw: +; CHECK: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %b) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i16( %pg, + i16* %base, + %uxtw) + %res = sext %load to + ret %res +} + +define @gld1sw_d_uxtw( %pg, i32* %base, %offsets) { +; CHECK-LABEL: gld1sw_d_uxtw: +; CHECK: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw] +; CHECK-NEXT: ret + %uxtw = call @llvm.aarch64.sve.uxtw.nxv2i64( undef, + %pg, + %offsets) + %load = call @llvm.aarch64.sve.ld1.gather.nxv2i32( %pg, + i32* %base, + %uxtw) + %res = sext %load to + ret %res +} + declare @llvm.aarch64.sve.ld1.gather.nxv2i8(, i8*, ) declare @llvm.aarch64.sve.ld1.gather.nxv2i16(, i16*, ) declare @llvm.aarch64.sve.ld1.gather.nxv2i32(, i32*, ) declare @llvm.aarch64.sve.ld1.gather.nxv2i64(, i64*, ) declare @llvm.aarch64.sve.ld1.gather.nxv2f64(, double*, ) + +declare @llvm.aarch64.sve.sxtw.nxv2i64(, , ) +declare @llvm.aarch64.sve.uxtw.nxv2i64(, , ) -- GitLab From ece6d8e72eaab1ce6b37c4f658d75ed787181174 Mon Sep 17 00:00:00 2001 From: Josh Berdine Date: Fri, 12 Mar 2021 22:50:21 +0000 Subject: [PATCH 0043/1206] [OCaml] Add missing TypeKinds, Opcode, and AtomicRMWBinOps There are several enum values that have been added to LLVM-C that are missing from the OCaml bindings. The types defined in bindings/ocaml/llvm/llvm.ml should be in sync with the corresponding enum definitions in include/llvm-c/Core.h. The enum values are passed from C to OCaml unmodified, and clients of the OCaml bindings interpret them as tags of the corresponding OCaml types. So the only changes needed are to add the missing constructors to the type definitions, and to change the name of the maximum opcode in an assertion. Differential Revision: https://reviews.llvm.org/D98578 --- llvm/bindings/ocaml/llvm/llvm.ml | 6 ++++++ llvm/bindings/ocaml/llvm/llvm.mli | 6 ++++++ llvm/bindings/ocaml/llvm/llvm_ocaml.c | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/llvm/bindings/ocaml/llvm/llvm.ml b/llvm/bindings/ocaml/llvm/llvm.ml index 723ac66ffd05..b1065d770867 100644 --- a/llvm/bindings/ocaml/llvm/llvm.ml +++ b/llvm/bindings/ocaml/llvm/llvm.ml @@ -42,6 +42,9 @@ module TypeKind = struct | Metadata | X86_mmx | Token + | ScalableVector + | BFloat + | X86_amx end module Linkage = struct @@ -246,6 +249,7 @@ module Opcode = struct | CatchSwitch | FNeg | CallBr + | Freeze end module LandingPadClauseTy = struct @@ -288,6 +292,8 @@ module AtomicRMWBinOp = struct | Min | UMax | UMin + | FAdd + | FSub end module ValueKind = struct diff --git a/llvm/bindings/ocaml/llvm/llvm.mli b/llvm/bindings/ocaml/llvm/llvm.mli index ba9e6c1f2120..19ff22ef33e8 100644 --- a/llvm/bindings/ocaml/llvm/llvm.mli +++ b/llvm/bindings/ocaml/llvm/llvm.mli @@ -77,6 +77,9 @@ module TypeKind : sig | Metadata | X86_mmx | Token + | ScalableVector + | BFloat + | X86_amx end (** The linkage of a global value, accessed with {!linkage} and @@ -268,6 +271,7 @@ module Opcode : sig | CatchSwitch | FNeg | CallBr + | Freeze end (** The type of a clause of a [landingpad] instruction. @@ -319,6 +323,8 @@ module AtomicRMWBinOp : sig | Min | UMax | UMin + | FAdd + | FSub end (** The kind of an [llvalue], the result of [classify_value v]. diff --git a/llvm/bindings/ocaml/llvm/llvm_ocaml.c b/llvm/bindings/ocaml/llvm/llvm_ocaml.c index 5845783278d9..1d68eb5e6d42 100644 --- a/llvm/bindings/ocaml/llvm/llvm_ocaml.c +++ b/llvm/bindings/ocaml/llvm/llvm_ocaml.c @@ -1541,7 +1541,7 @@ CAMLprim value llvm_instr_get_opcode(LLVMValueRef Inst) { if (!LLVMIsAInstruction(Inst)) failwith("Not an instruction"); o = LLVMGetInstructionOpcode(Inst); - assert (o <= LLVMCallBr); + assert(o <= LLVMFreeze); return Val_int(o); } -- GitLab From b388bbd3f9d0d394e3b85b53e1d944510f84023c Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Mon, 15 Mar 2021 21:18:52 -0700 Subject: [PATCH 0044/1206] [mlir][amx] blocked tilezero integration test This adds a new integration test. However, it also adapts to a recent memref.XXX change for existing tests Reviewed By: ftynse Differential Revision: https://reviews.llvm.org/D98680 --- .../Dialect/Vector/CPU/AMX/test-mulf.mlir | 12 +-- .../Dialect/Vector/CPU/AMX/test-muli.mlir | 12 +-- .../Vector/CPU/AMX/test-tilezero-block.mlir | 81 +++++++++++++++++++ .../Dialect/Vector/CPU/AMX/test-tilezero.mlir | 6 +- 4 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir index 73d866af972c..d188b86b2eda 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-mulf.mlir @@ -38,9 +38,9 @@ func @entry() { %c2 = constant 2: index // Set up memory. - %a = alloc() : memref<2x4xbf16> - %b = alloc() : memref<2x4xbf16> - %c = alloc() : memref<2x2xf32> + %a = memref.alloc() : memref<2x4xbf16> + %b = memref.alloc() : memref<2x4xbf16> + %c = memref.alloc() : memref<2x2xf32> %0 = std.constant dense<[[1.0, 2.0, 3.0, 4.0 ], [5.0, 6.0, 7.0, 8.0 ]]> : vector<2x4xbf16> @@ -75,9 +75,9 @@ func @entry() { } // Release resources. - dealloc %a : memref<2x4xbf16> - dealloc %b : memref<2x4xbf16> - dealloc %c : memref<2x2xf32> + memref.dealloc %a : memref<2x4xbf16> + memref.dealloc %b : memref<2x4xbf16> + memref.dealloc %c : memref<2x2xf32> return } diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir index 59eff35d33cf..a52f66c640f8 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-muli.mlir @@ -38,9 +38,9 @@ func @entry() { %c2 = constant 2: index // Set up memory. - %a = alloc() : memref<2x8xi8> - %b = alloc() : memref<2x8xi8> - %c = alloc() : memref<2x2xi32> + %a = memref.alloc() : memref<2x8xi8> + %b = memref.alloc() : memref<2x8xi8> + %c = memref.alloc() : memref<2x2xi32> %0 = std.constant dense<[[1 , 2, 3 , 4 , 5, 6, 7, 8], [9, 10, 11, 12, 13, 14, 15, 16]]> : vector<2x8xi8> @@ -75,9 +75,9 @@ func @entry() { } // Release resources. - dealloc %a : memref<2x8xi8> - dealloc %b : memref<2x8xi8> - dealloc %c : memref<2x2xi32> + memref.dealloc %a : memref<2x8xi8> + memref.dealloc %b : memref<2x8xi8> + memref.dealloc %c : memref<2x2xi32> return } diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir new file mode 100644 index 000000000000..64cf39d47b80 --- /dev/null +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero-block.mlir @@ -0,0 +1,81 @@ +// RUN: mlir-opt %s -convert-vector-to-scf -lower-affine -convert-scf-to-std -convert-vector-to-llvm="enable-amx" -convert-std-to-llvm | \ +// RUN: mlir-translate -mlir-to-llvmir | \ +// RUN: %lli --entry-function=entry --mattr="+amx-tile,+amx-int8,+amx-bf16" --dlopen=%mlir_integration_test_dir/libmlir_c_runner_utils%shlibext | \ +// RUN: FileCheck %s + +// Note: To run this test, your CPU must support AMX. + +func @print(%arg0: memref<4x32xf32>) { + %fu = constant -1.0: f32 + %c0 = constant 0: index + %c1 = constant 1: index + %c4 = constant 4: index + scf.for %i = %c0 to %c4 step %c1 { + %0 = vector.transfer_read %arg0[%i, %c0], %fu: memref<4x32xf32>, vector<32xf32> + vector.print %0 : vector<32xf32> + } + return +} + +func @kernel(%arg0: memref<4x32xf32>) { + %c0 = constant 0: index + %c2 = constant 2 : index + %c4 = constant 4 : index + %c16 = constant 16 : index + %c32 = constant 32 : index + scf.for %i = %c0 to %c4 step %c2 { + scf.for %j = %c0 to %c32 step %c16 { + %0 = amx.tile_zero : vector<2x16xf32> + amx.tile_store %arg0[%i, %j], %0 : memref<4x32xf32>, vector<2x16xf32> + call @print(%arg0) : (memref<4x32xf32>) -> () + } + } + return +} + +func @entry() { + %f1 = constant 1.0: f32 + %c0 = constant 0: index + %c1 = constant 1: index + %c4 = constant 4 : index + %c32 = constant 32 : index + + // Set up memory. + %a = memref.alloc() : memref<4x32xf32> + scf.for %i = %c0 to %c4 step %c1 { + scf.for %j = %c0 to %c32 step %c1 { + memref.store %f1, %a[%i, %j] : memref<4x32xf32> + } + } + + // Call kernel. + call @kernel(%a) : (memref<4x32xf32>) -> () + + // Verify progress of blocked tilezero. + // + // CHECK: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // CHECK-NEXT: ( 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + // + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // CHECK-NEXT: ( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ) + // + + // Release resources. + memref.dealloc %a : memref<4x32xf32> + + return +} diff --git a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir index f49c66e4ce4b..1a833f9ef172 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/AMX/test-tilezero.mlir @@ -20,10 +20,10 @@ func @entry() { %c19 = constant 19: index // Set up memory. - %a = alloc(%c19, %c19) : memref + %a = memref.alloc(%c19, %c19) : memref scf.for %i = %c0 to %c19 step %c1 { scf.for %j = %c0 to %c19 step %c1 { - store %i1, %a[%i, %j] : memref + memref.store %i1, %a[%i, %j] : memref } } @@ -90,7 +90,7 @@ func @entry() { } // Release resources. - dealloc %a : memref + memref.dealloc %a : memref return } -- GitLab From f12433f127150054fdb0ed7a735b0e4ab4ae1cd9 Mon Sep 17 00:00:00 2001 From: Thomas Preud'homme Date: Tue, 16 Mar 2021 15:49:16 +0000 Subject: [PATCH 0045/1206] [MemDepAnalysis] Remove redundant comment. Exact same comment is found 2 lines above. --- llvm/lib/Analysis/MemoryDependenceAnalysis.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp index 886b5bf4acd3..3131da2f8b0a 100644 --- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp @@ -1135,9 +1135,6 @@ bool MemoryDependenceResults::getNonLocalPointerDepFromBB( // investigating, just return it with no recomputation. // Don't use cached information for invariant loads since it is valid for // non-invariant loads only. - // - // Don't use cached information for invariant loads since it is valid for - // non-invariant loads only. if (!IsIncomplete && !isInvariantLoad && CacheInfo->Pair == BBSkipFirstBlockPair(StartBB, SkipFirstBlock)) { // We have a fully cached result for this query then we can just return the -- GitLab From d2eae990a1bd0efcd2838187627d6e02ea23d998 Mon Sep 17 00:00:00 2001 From: Sanjay Patel Date: Tue, 16 Mar 2021 09:42:25 -0400 Subject: [PATCH 0046/1206] [LoopVectorize] add FP induction test with minimal FMF; NFC --- .../LoopVectorize/X86/float-induction-x86.ll | 138 +++++++++++++++++- 1 file changed, 130 insertions(+), 8 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll index 4f3fd288d710..9db01e701010 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/float-induction-x86.ll @@ -111,7 +111,7 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 { ; AUTO_VEC-NEXT: [[VEC_IND_NEXT_3]] = fadd fast <8 x float> [[VEC_IND]], ; AUTO_VEC-NEXT: [[NITER_NSUB_3]] = add i64 [[NITER]], -4 ; AUTO_VEC-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0 -; AUTO_VEC-NEXT: br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; AUTO_VEC: middle.block.unr-lcssa: ; AUTO_VEC-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ] ; AUTO_VEC-NEXT: [[VEC_IND_UNR:%.*]] = phi <8 x float> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_3]], [[VECTOR_BODY]] ] @@ -140,7 +140,7 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 { ; AUTO_VEC-NEXT: [[VEC_IND_NEXT_EPIL]] = fadd fast <8 x float> [[VEC_IND_EPIL]], ; AUTO_VEC-NEXT: [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1 ; AUTO_VEC-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0 -; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], [[LOOP2:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], !llvm.loop [[LOOP2:![0-9]+]] ; AUTO_VEC: middle.block: ; AUTO_VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[ZEXT]] ; AUTO_VEC-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY]] @@ -152,7 +152,7 @@ define void @fp_iv_loop1(float* noalias nocapture %A, i32 %N) #0 { ; AUTO_VEC-NEXT: [[CONV1]] = fadd fast float [[X_06]], 5.000000e-01 ; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; AUTO_VEC-NEXT: [[TMP45:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[ZEXT]] -; AUTO_VEC-NEXT: br i1 [[TMP45]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP4:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[TMP45]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; AUTO_VEC: for.end: ; AUTO_VEC-NEXT: ret void ; @@ -259,7 +259,7 @@ define void @fp_iv_loop2(float* noalias nocapture %A, i32 %N) { ; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1 ; AUTO_VEC-NEXT: [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1 ; AUTO_VEC-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0 -; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], [[LOOP6:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], !llvm.loop [[LOOP6:![0-9]+]] ; AUTO_VEC: for.end: ; AUTO_VEC-NEXT: ret void ; @@ -381,7 +381,7 @@ define double @external_use_with_fast_math(double* %a, i64 %n) { ; AUTO_VEC-NEXT: [[VEC_IND_NEXT_3]] = fadd fast <4 x double> [[VEC_IND]], ; AUTO_VEC-NEXT: [[NITER_NSUB_3]] = add i64 [[NITER]], -4 ; AUTO_VEC-NEXT: [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0 -; AUTO_VEC-NEXT: br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], [[LOOP7:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; AUTO_VEC: middle.block.unr-lcssa: ; AUTO_VEC-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ] ; AUTO_VEC-NEXT: [[VEC_IND_UNR:%.*]] = phi <4 x double> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT_3]], [[VECTOR_BODY]] ] @@ -410,7 +410,7 @@ define double @external_use_with_fast_math(double* %a, i64 %n) { ; AUTO_VEC-NEXT: [[VEC_IND_NEXT_EPIL]] = fadd fast <4 x double> [[VEC_IND_EPIL]], ; AUTO_VEC-NEXT: [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1 ; AUTO_VEC-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0 -; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], [[LOOP8:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], !llvm.loop [[LOOP8:![0-9]+]] ; AUTO_VEC: middle.block: ; AUTO_VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]] ; AUTO_VEC-NEXT: [[TMP45:%.*]] = add nsw i64 [[N_VEC]], -1 @@ -425,7 +425,7 @@ define double @external_use_with_fast_math(double* %a, i64 %n) { ; AUTO_VEC-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 ; AUTO_VEC-NEXT: [[J_NEXT]] = fadd fast double [[J]], 3.000000e+00 ; AUTO_VEC-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[I_NEXT]], [[SMAX]] -; AUTO_VEC-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP9:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; AUTO_VEC: for.end: ; AUTO_VEC-NEXT: [[J_LCSSA:%.*]] = phi double [ [[TMP46]], [[MIDDLE_BLOCK]] ], [ [[J]], [[FOR_BODY]] ] ; AUTO_VEC-NEXT: ret double [[J_LCSSA]] @@ -514,7 +514,7 @@ define double @external_use_without_fast_math(double* %a, i64 %n) { ; AUTO_VEC-NEXT: [[J_NEXT_EPIL]] = fadd double [[J_EPIL]], 3.000000e+00 ; AUTO_VEC-NEXT: [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1 ; AUTO_VEC-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0 -; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], [[LOOP10:!llvm.loop !.*]] +; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_END]], label [[FOR_BODY_EPIL]], !llvm.loop [[LOOP10:![0-9]+]] ; AUTO_VEC: for.end: ; AUTO_VEC-NEXT: [[J_LCSSA:%.*]] = phi double [ [[J_LCSSA_PH]], [[FOR_END_UNR_LCSSA]] ], [ [[J_EPIL]], [[FOR_BODY_EPIL]] ] ; AUTO_VEC-NEXT: ret double [[J_LCSSA]] @@ -536,3 +536,125 @@ for.end: %t1 = phi double [ %j, %for.body ] ret double %t1 } + +;; void fadd_induction(float *p, unsigned N) { +;; float x = 1.0f; +;; for (unsigned i=0; i!=N; ++i) { +;; p[i] = p[i] + x; +;; x += 42.0f; +;; } +;; } + +define void @fadd_reassoc_FMF(float* nocapture %p, i32 %N) { +; AUTO_VEC-LABEL: @fadd_reassoc_FMF( +; AUTO_VEC-NEXT: entry: +; AUTO_VEC-NEXT: [[CMP_NOT11:%.*]] = icmp eq i32 [[N:%.*]], 0 +; AUTO_VEC-NEXT: br i1 [[CMP_NOT11]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY_PREHEADER:%.*]] +; AUTO_VEC: for.body.preheader: +; AUTO_VEC-NEXT: [[TMP0:%.*]] = zext i32 [[N]] to i64 +; AUTO_VEC-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], -1 +; AUTO_VEC-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP0]], 7 +; AUTO_VEC-NEXT: [[TMP2:%.*]] = icmp ult i64 [[TMP1]], 7 +; AUTO_VEC-NEXT: br i1 [[TMP2]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]] +; AUTO_VEC: for.body.preheader.new: +; AUTO_VEC-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[TMP0]], 4294967288 +; AUTO_VEC-NEXT: br label [[FOR_BODY:%.*]] +; AUTO_VEC: for.cond.cleanup.loopexit.unr-lcssa: +; AUTO_VEC-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[X_012_UNR:%.*]] = phi float [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ], [ [[ADD3_7:%.*]], [[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 +; AUTO_VEC-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_EPIL:%.*]] +; AUTO_VEC: for.body.epil: +; AUTO_VEC-NEXT: [[INDVARS_IV_EPIL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[INDVARS_IV_UNR]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; AUTO_VEC-NEXT: [[X_012_EPIL:%.*]] = phi float [ [[ADD3_EPIL:%.*]], [[FOR_BODY_EPIL]] ], [ [[X_012_UNR]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; AUTO_VEC-NEXT: [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_SUB:%.*]], [[FOR_BODY_EPIL]] ], [ [[XTRAITER]], [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]] ] +; AUTO_VEC-NEXT: [[ARRAYIDX_EPIL:%.*]] = getelementptr inbounds float, float* [[P:%.*]], i64 [[INDVARS_IV_EPIL]] +; AUTO_VEC-NEXT: [[TMP3:%.*]] = load float, float* [[ARRAYIDX_EPIL]], align 4 +; AUTO_VEC-NEXT: [[ADD_EPIL:%.*]] = fadd reassoc float [[X_012_EPIL]], [[TMP3]] +; AUTO_VEC-NEXT: store float [[ADD_EPIL]], float* [[ARRAYIDX_EPIL]], align 4 +; AUTO_VEC-NEXT: [[ADD3_EPIL]] = fadd reassoc float [[X_012_EPIL]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_EPIL]] = add nuw nsw i64 [[INDVARS_IV_EPIL]], 1 +; AUTO_VEC-NEXT: [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1 +; AUTO_VEC-NEXT: [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0 +; AUTO_VEC-NEXT: br i1 [[EPIL_ITER_CMP_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_EPIL]], !llvm.loop [[LOOP11:![0-9]+]] +; AUTO_VEC: for.cond.cleanup: +; AUTO_VEC-NEXT: ret void +; AUTO_VEC: for.body: +; AUTO_VEC-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER_NEW]] ], [ [[INDVARS_IV_NEXT_7]], [[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[X_012:%.*]] = phi float [ 1.000000e+00, [[FOR_BODY_PREHEADER_NEW]] ], [ [[ADD3_7]], [[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[FOR_BODY_PREHEADER_NEW]] ], [ [[NITER_NSUB_7:%.*]], [[FOR_BODY]] ] +; AUTO_VEC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV]] +; AUTO_VEC-NEXT: [[TMP4:%.*]] = load float, float* [[ARRAYIDX]], align 4 +; AUTO_VEC-NEXT: [[ADD:%.*]] = fadd reassoc float [[X_012]], [[TMP4]] +; AUTO_VEC-NEXT: store float [[ADD]], float* [[ARRAYIDX]], align 4 +; AUTO_VEC-NEXT: [[ADD3:%.*]] = fadd reassoc float [[X_012]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT:%.*]] = or i64 [[INDVARS_IV]], 1 +; AUTO_VEC-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT]] +; AUTO_VEC-NEXT: [[TMP5:%.*]] = load float, float* [[ARRAYIDX_1]], align 4 +; AUTO_VEC-NEXT: [[ADD_1:%.*]] = fadd reassoc float [[ADD3]], [[TMP5]] +; AUTO_VEC-NEXT: store float [[ADD_1]], float* [[ARRAYIDX_1]], align 4 +; AUTO_VEC-NEXT: [[ADD3_1:%.*]] = fadd reassoc float [[ADD3]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = or i64 [[INDVARS_IV]], 2 +; AUTO_VEC-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT_1]] +; AUTO_VEC-NEXT: [[TMP6:%.*]] = load float, float* [[ARRAYIDX_2]], align 4 +; AUTO_VEC-NEXT: [[ADD_2:%.*]] = fadd reassoc float [[ADD3_1]], [[TMP6]] +; AUTO_VEC-NEXT: store float [[ADD_2]], float* [[ARRAYIDX_2]], align 4 +; AUTO_VEC-NEXT: [[ADD3_2:%.*]] = fadd reassoc float [[ADD3_1]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = or i64 [[INDVARS_IV]], 3 +; AUTO_VEC-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT_2]] +; AUTO_VEC-NEXT: [[TMP7:%.*]] = load float, float* [[ARRAYIDX_3]], align 4 +; AUTO_VEC-NEXT: [[ADD_3:%.*]] = fadd reassoc float [[ADD3_2]], [[TMP7]] +; AUTO_VEC-NEXT: store float [[ADD_3]], float* [[ARRAYIDX_3]], align 4 +; AUTO_VEC-NEXT: [[ADD3_3:%.*]] = fadd reassoc float [[ADD3_2]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_3:%.*]] = or i64 [[INDVARS_IV]], 4 +; AUTO_VEC-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT_3]] +; AUTO_VEC-NEXT: [[TMP8:%.*]] = load float, float* [[ARRAYIDX_4]], align 4 +; AUTO_VEC-NEXT: [[ADD_4:%.*]] = fadd reassoc float [[ADD3_3]], [[TMP8]] +; AUTO_VEC-NEXT: store float [[ADD_4]], float* [[ARRAYIDX_4]], align 4 +; AUTO_VEC-NEXT: [[ADD3_4:%.*]] = fadd reassoc float [[ADD3_3]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_4:%.*]] = or i64 [[INDVARS_IV]], 5 +; AUTO_VEC-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT_4]] +; AUTO_VEC-NEXT: [[TMP9:%.*]] = load float, float* [[ARRAYIDX_5]], align 4 +; AUTO_VEC-NEXT: [[ADD_5:%.*]] = fadd reassoc float [[ADD3_4]], [[TMP9]] +; AUTO_VEC-NEXT: store float [[ADD_5]], float* [[ARRAYIDX_5]], align 4 +; AUTO_VEC-NEXT: [[ADD3_5:%.*]] = fadd reassoc float [[ADD3_4]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_5:%.*]] = or i64 [[INDVARS_IV]], 6 +; AUTO_VEC-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT_5]] +; AUTO_VEC-NEXT: [[TMP10:%.*]] = load float, float* [[ARRAYIDX_6]], align 4 +; AUTO_VEC-NEXT: [[ADD_6:%.*]] = fadd reassoc float [[ADD3_5]], [[TMP10]] +; AUTO_VEC-NEXT: store float [[ADD_6]], float* [[ARRAYIDX_6]], align 4 +; AUTO_VEC-NEXT: [[ADD3_6:%.*]] = fadd reassoc float [[ADD3_5]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_6:%.*]] = or i64 [[INDVARS_IV]], 7 +; AUTO_VEC-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[P]], i64 [[INDVARS_IV_NEXT_6]] +; AUTO_VEC-NEXT: [[TMP11:%.*]] = load float, float* [[ARRAYIDX_7]], align 4 +; AUTO_VEC-NEXT: [[ADD_7:%.*]] = fadd reassoc float [[ADD3_6]], [[TMP11]] +; AUTO_VEC-NEXT: store float [[ADD_7]], float* [[ARRAYIDX_7]], align 4 +; AUTO_VEC-NEXT: [[ADD3_7]] = fadd reassoc float [[ADD3_6]], 4.200000e+01 +; AUTO_VEC-NEXT: [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8 +; AUTO_VEC-NEXT: [[NITER_NSUB_7]] = add i64 [[NITER]], -8 +; AUTO_VEC-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NSUB_7]], 0 +; AUTO_VEC-NEXT: br i1 [[NITER_NCMP_7]], label [[FOR_COND_CLEANUP_LOOPEXIT_UNR_LCSSA]], label [[FOR_BODY]] +; +entry: + %cmp.not11 = icmp eq i32 %N, 0 + br i1 %cmp.not11, label %for.cond.cleanup, label %for.body.preheader + +for.body.preheader: + %0 = zext i32 %N to i64 + br label %for.body + +for.cond.cleanup: + ret void + +for.body: + %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ] + %x.012 = phi float [ 1.000000e+00, %for.body.preheader ], [ %add3, %for.body ] + %arrayidx = getelementptr inbounds float, float* %p, i64 %indvars.iv + %1 = load float, float* %arrayidx, align 4 + %add = fadd reassoc float %x.012, %1 + store float %add, float* %arrayidx, align 4 + %add3 = fadd reassoc float %x.012, 4.200000e+01 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %cmp.not = icmp eq i64 %indvars.iv.next, %0 + br i1 %cmp.not, label %for.cond.cleanup, label %for.body +} -- GitLab From 440f6bdf34f4ce3ac3435d650f5296dcc0102488 Mon Sep 17 00:00:00 2001 From: Luke Drummond Date: Tue, 16 Mar 2021 13:28:06 +0000 Subject: [PATCH 0047/1206] [OpenCL][NFCI] Prefer CodeGenFunction::EmitRuntimeCall `CodeGenFunction::EmitRuntimeCall` automatically sets the right calling convention for the callee so we can avoid setting it ourselves. As requested in https://reviews.llvm.org/D98411 Reviewed by: anastasia Differential Revision: https://reviews.llvm.org/D98705 --- clang/lib/CodeGen/CodeGenModule.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 75854f69b110..f3a73f8783dc 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -6265,9 +6265,8 @@ CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E, llvm::Constant *C = ConstantEmitter(CGF).emitAbstract(E, E->getType()); auto *SamplerT = getOpenCLRuntime().getSamplerType(E->getType().getTypePtr()); auto *FTy = llvm::FunctionType::get(SamplerT, {C->getType()}, false); - auto *Call = CGF.Builder.CreateCall( + auto *Call = CGF.EmitRuntimeCall( CreateRuntimeFunction(FTy, "__translate_sampler_initializer"), {C}); - Call->setCallingConv(Call->getCalledFunction()->getCallingConv()); return Call; } -- GitLab From 64595f9b84fa77a130085260b1b4e26a2756dce8 Mon Sep 17 00:00:00 2001 From: Tomas Matheson Date: Tue, 23 Feb 2021 14:05:55 +0000 Subject: [PATCH 0048/1206] [libcxx][type_traits] add tests for is_signed and is_unsigned In previous versions of clang, __is_signed and __is_unsigned builtins did not correspond to is_signed and is_unsigned behaviour for enums. The builtins were fixed in D67897 and D98104. * Disable the fast path of is_unsigned for clang versions < 13 * Add more tests for is_signed, is_unsigned and is_arithmetic Differential Revision: https://reviews.llvm.org/D97283 --- libcxx/include/type_traits | 7 +- .../meta.unary.comp/is_arithmetic.pass.cpp | 9 +++ .../meta.unary.prop/is_signed.pass.cpp | 78 +++++++++++++++---- .../meta.unary.prop/is_unsigned.pass.cpp | 78 +++++++++++++++---- 4 files changed, 143 insertions(+), 29 deletions(-) diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits index efc5c41c5f9b..7477e6d143de 100644 --- a/libcxx/include/type_traits +++ b/libcxx/include/type_traits @@ -1451,7 +1451,8 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_signed_v // is_unsigned -#if __has_keyword(__is_unsigned) +// Before clang 13, __is_unsigned returned true for enums with signed underlying type +#if __has_keyword(__is_unsigned) && _LIBCPP_CLANG_VER >= 1300 template struct _LIBCPP_TEMPLATE_VIS is_unsigned : _BoolConstant<__is_unsigned(_Tp)> { }; @@ -1461,7 +1462,7 @@ template _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_unsigned_v = __is_unsigned(_Tp); #endif -#else // __has_keyword(__is_unsigned) +#else // __has_keyword(__is_unsigned) && _LIBCPP_CLANG_VER >= 1300 template ::value> struct __libcpp_is_unsigned_impl : public _LIBCPP_BOOL_CONSTANT(_Tp(0) < _Tp(-1)) {}; @@ -1482,7 +1483,7 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_unsigned_v = is_unsigned<_Tp>::value; #endif -#endif // __has_keyword(__is_unsigned) +#endif // __has_keyword(__is_unsigned) && _LIBCPP_CLANG_VER >= 1300 // rank diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp index 683e885e288d..a6fc44384c3b 100644 --- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp +++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.comp/is_arithmetic.pass.cpp @@ -69,6 +69,12 @@ class Abstract enum Enum {zero, one}; +enum EnumSigned : int { two }; + +enum EnumUnsigned : unsigned { three }; + +enum class EnumClass { zero, one }; + typedef void (*FunctionPtr)(); @@ -97,6 +103,9 @@ int main(int, char**) test_is_not_arithmetic(); test_is_not_arithmetic(); test_is_not_arithmetic(); + test_is_not_arithmetic(); + test_is_not_arithmetic(); + test_is_not_arithmetic(); test_is_not_arithmetic(); test_is_not_arithmetic(); test_is_not_arithmetic(); diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp index 4936cc788e25..29a259fb2588 100644 --- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp +++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_signed.pass.cpp @@ -51,21 +51,73 @@ public: struct A; // incomplete +class incomplete_type; + +class Empty {}; + +class NotEmpty { + virtual ~NotEmpty(); +}; + +union Union {}; + +struct bit_zero { + int : 0; +}; + +class Abstract { + virtual ~Abstract() = 0; +}; + +enum Enum { zero, one }; + +enum EnumSigned : int { two }; + +enum EnumUnsigned : unsigned { three }; + +enum class EnumClass { zero, one }; + +typedef void (*FunctionPtr)(); + int main(int, char**) { - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - test_is_not_signed(); - - test_is_signed(); - test_is_signed(); + // Cases where !is_arithmetic implies !is_signed + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + + test_is_signed(); + test_is_signed(); + test_is_signed(); + test_is_signed(); + test_is_signed(); + test_is_signed(); + + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + test_is_not_signed(); + + test_is_not_signed(); + test_is_not_signed(); #ifndef _LIBCPP_HAS_NO_INT128 test_is_signed<__int128_t>(); diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp index bc70a43b9bd5..3c200b8f3905 100644 --- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp +++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_unsigned.pass.cpp @@ -51,21 +51,73 @@ public: struct A; // incomplete +class incomplete_type; + +class Empty {}; + +class NotEmpty { + virtual ~NotEmpty(); +}; + +union Union {}; + +struct bit_zero { + int : 0; +}; + +class Abstract { + virtual ~Abstract() = 0; +}; + +enum Enum { zero, one }; + +enum EnumSigned : int { two }; + +enum EnumUnsigned : unsigned { three }; + +enum class EnumClass { zero, one }; + +typedef void (*FunctionPtr)(); + int main(int, char**) { - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - test_is_not_unsigned(); - - test_is_unsigned(); - test_is_unsigned(); + // Cases where !is_arithmetic implies !is_unsigned + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + test_is_not_unsigned(); + + test_is_unsigned(); + test_is_unsigned(); + test_is_unsigned(); + test_is_unsigned(); + + test_is_unsigned(); + test_is_unsigned(); #ifndef _LIBCPP_HAS_NO_INT128 test_is_unsigned<__uint128_t>(); -- GitLab From 264f101ae6b4e35a0d377f3fb8bc35dbe9ba173e Mon Sep 17 00:00:00 2001 From: Jeremy Morse Date: Tue, 16 Mar 2021 16:50:49 +0000 Subject: [PATCH 0049/1206] Tweak spelling of system-windows UNSUPPORTED line --- llvm/utils/lit/tests/reorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/utils/lit/tests/reorder.py b/llvm/utils/lit/tests/reorder.py index fb1c4bc41249..2b699067bbc3 100644 --- a/llvm/utils/lit/tests/reorder.py +++ b/llvm/utils/lit/tests/reorder.py @@ -3,7 +3,7 @@ # RUN: cp %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig # RUN: %{lit} -j1 %{inputs}/reorder | FileCheck %s # RUN: not diff %{inputs}/reorder/.lit_test_times.txt %{inputs}/reorder/.lit_test_times.txt.orig -# UNSUPPORTED: windows +# UNSUPPORTED: system-windows # END. # CHECK: -- Testing: 3 tests, 1 workers -- -- GitLab From 6513995be37b73cb168bec5f7fa66015893659bf Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Tue, 16 Mar 2021 02:04:42 +0200 Subject: [PATCH 0050/1206] [InstSimplify] Restrict a GEP transform to avoid provenance changes This is a follow-up to D98588, and fixes the inline `FIXME` about a GEP-related simplification not preserving the provenance. https://alive2.llvm.org/ce/z/qbQoAY Additional tests were added in {rGf125f28afdb59eba29d2491dac0dfc0a7bf1b60b} Depends on D98672 Reviewed By: lebedev.ri Differential Revision: https://reviews.llvm.org/D98611 --- llvm/lib/Analysis/InstructionSimplify.cpp | 8 +++----- llvm/test/Transforms/InstSimplify/gep.ll | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp index 95a4e8d82c76..7790255e22c4 100644 --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -4349,11 +4349,9 @@ static Value *SimplifyGEPInst(Type *SrcTy, ArrayRef Ops, // doesn't truncate the pointers. if (Ops[1]->getType()->getScalarSizeInBits() == Q.DL.getPointerSizeInBits(AS)) { - auto CanSimplify = [GEPTy, &P]() -> bool { - // FIXME: The following transforms are only legal if P and V have the - // same provenance (PR44403). Check whether getUnderlyingObject() is - // the same? - return P->getType() == GEPTy; + auto CanSimplify = [GEPTy, &P, V = Ops[0]]() -> bool { + return P->getType() == GEPTy && + getUnderlyingObject(P) == getUnderlyingObject(V); }; // getelementptr V, (sub P, V) -> P if P points to a type of size 1. if (TyAllocSize == 1 && diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll index 3c460ecc4a67..e1da60ee5668 100644 --- a/llvm/test/Transforms/InstSimplify/gep.ll +++ b/llvm/test/Transforms/InstSimplify/gep.ll @@ -7,7 +7,12 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" define %struct.A* @test1(%struct.A* %b, %struct.A* %e) { ; CHECK-LABEL: @test1( -; CHECK-NEXT: ret %struct.A* [[E:%.*]] +; CHECK-NEXT: [[E_PTR:%.*]] = ptrtoint %struct.A* [[E:%.*]] to i64 +; CHECK-NEXT: [[B_PTR:%.*]] = ptrtoint %struct.A* [[B:%.*]] to i64 +; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]] +; CHECK-NEXT: [[SDIV:%.*]] = sdiv exact i64 [[SUB]], 7 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds [[STRUCT_A:%.*]], %struct.A* [[B]], i64 [[SDIV]] +; CHECK-NEXT: ret %struct.A* [[GEP]] ; %e_ptr = ptrtoint %struct.A* %e to i64 %b_ptr = ptrtoint %struct.A* %b to i64 @@ -19,7 +24,11 @@ define %struct.A* @test1(%struct.A* %b, %struct.A* %e) { define i8* @test2(i8* %b, i8* %e) { ; CHECK-LABEL: @test2( -; CHECK-NEXT: ret i8* [[E:%.*]] +; CHECK-NEXT: [[E_PTR:%.*]] = ptrtoint i8* [[E:%.*]] to i64 +; CHECK-NEXT: [[B_PTR:%.*]] = ptrtoint i8* [[B:%.*]] to i64 +; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, i8* [[B]], i64 [[SUB]] +; CHECK-NEXT: ret i8* [[GEP]] ; %e_ptr = ptrtoint i8* %e to i64 %b_ptr = ptrtoint i8* %b to i64 @@ -30,7 +39,12 @@ define i8* @test2(i8* %b, i8* %e) { define i64* @test3(i64* %b, i64* %e) { ; CHECK-LABEL: @test3( -; CHECK-NEXT: ret i64* [[E:%.*]] +; CHECK-NEXT: [[E_PTR:%.*]] = ptrtoint i64* [[E:%.*]] to i64 +; CHECK-NEXT: [[B_PTR:%.*]] = ptrtoint i64* [[B:%.*]] to i64 +; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[E_PTR]], [[B_PTR]] +; CHECK-NEXT: [[ASHR:%.*]] = ashr exact i64 [[SUB]], 3 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 [[ASHR]] +; CHECK-NEXT: ret i64* [[GEP]] ; %e_ptr = ptrtoint i64* %e to i64 %b_ptr = ptrtoint i64* %b to i64 -- GitLab From 6ab8927931851bb42b2c93a00801dc499d7d9b1e Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 16 Mar 2021 10:02:35 -0700 Subject: [PATCH 0051/1206] [RISCV] Support clang -fpatchable-function-entry && GNU function attribute 'patchable_function_entry' Similar to D72215 (AArch64) and D72220 (x86). ``` % clang -target riscv32 -march=rv64g -c -fpatchable-function-entry=2 a.c && llvm-objdump -dr a.o ... 0000000000000000
: 0: 13 00 00 00 nop 4: 13 00 00 00 nop % clang -target riscv32 -march=rv64gc -c -fpatchable-function-entry=2 a.c && llvm-objdump -dr a.o ... 00000002
: 2: 01 00 nop 4: 01 00 nop ``` Recently the mainline kernel started to use -fpatchable-function-entry=8 for riscv (https://git.kernel.org/linus/afc76b8b80112189b6f11e67e19cf58301944814). Differential Revision: https://reviews.llvm.org/D98610 --- clang/include/clang/Basic/Attr.td | 3 +- clang/include/clang/Basic/AttrDocs.td | 3 + clang/lib/Driver/ToolChains/Clang.cpp | 3 +- clang/test/Driver/fpatchable-function-entry.c | 2 + .../Sema/patchable-function-entry-attr.cpp | 2 + llvm/lib/Target/RISCV/RISCV.h | 4 +- llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp | 4 +- llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 10 +++ llvm/lib/Target/RISCV/RISCVInstrInfo.h | 2 + llvm/lib/Target/RISCV/RISCVMCInstLower.cpp | 35 ++++++--- .../CodeGen/RISCV/patchable-function-entry.ll | 71 +++++++++++++++++++ 11 files changed, 121 insertions(+), 18 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/patchable-function-entry.ll diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 664eb566a703..6b50894512cd 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -730,7 +730,8 @@ def XRayLogArgs : InheritableAttr { def PatchableFunctionEntry : InheritableAttr, - TargetSpecificAttr> { + TargetSpecificAttr> { let Spellings = [GCC<"patchable_function_entry">]; let Subjects = SubjectList<[Function, ObjCMethod]>; let Args = [UnsignedArgument<"Count">, DefaultIntArgument<"Offset", 0>]; diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index 77d3bd1fdcd6..f73fbd08e3bf 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -4810,6 +4810,9 @@ def PatchableFunctionEntryDocs : Documentation { before the function entry and N-M NOPs after the function entry. This attribute takes precedence over the command line option ``-fpatchable-function-entry=N,M``. ``M`` defaults to 0 if omitted. + +This attribute is only supported on +aarch64/aarch64-be/riscv32/riscv64/i386/x86-64 targets. }]; } diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 246bdf42a66a..2a3dde9ea9ac 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5614,8 +5614,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, if (Arg *A = Args.getLastArg(options::OPT_fpatchable_function_entry_EQ)) { StringRef S0 = A->getValue(), S = S0; unsigned Size, Offset = 0; - if (!Triple.isAArch64() && Triple.getArch() != llvm::Triple::x86 && - Triple.getArch() != llvm::Triple::x86_64) + if (!Triple.isAArch64() && !Triple.isRISCV() && !Triple.isX86()) D.Diag(diag::err_drv_unsupported_opt_for_target) << A->getAsString(Args) << TripleStr; else if (S.consumeInteger(10, Size) || diff --git a/clang/test/Driver/fpatchable-function-entry.c b/clang/test/Driver/fpatchable-function-entry.c index 5ac262c1a46d..da7370a4d87a 100644 --- a/clang/test/Driver/fpatchable-function-entry.c +++ b/clang/test/Driver/fpatchable-function-entry.c @@ -2,6 +2,8 @@ // RUN: %clang -target x86_64 %s -fpatchable-function-entry=1 -c -### 2>&1 | FileCheck %s // RUN: %clang -target aarch64 %s -fpatchable-function-entry=1 -c -### 2>&1 | FileCheck %s // RUN: %clang -target aarch64 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s +// RUN: %clang -target riscv32 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s +// RUN: %clang -target riscv64 %s -fpatchable-function-entry=1,0 -c -### 2>&1 | FileCheck %s // CHECK: "-fpatchable-function-entry=1" // RUN: %clang -target aarch64 -fsyntax-only %s -fpatchable-function-entry=1,1 -c -### 2>&1 | FileCheck --check-prefix=11 %s diff --git a/clang/test/Sema/patchable-function-entry-attr.cpp b/clang/test/Sema/patchable-function-entry-attr.cpp index 63de5a2abf70..3dd050498730 100644 --- a/clang/test/Sema/patchable-function-entry-attr.cpp +++ b/clang/test/Sema/patchable-function-entry-attr.cpp @@ -2,6 +2,8 @@ // RUN: %clang_cc1 -triple aarch64_be -fsyntax-only -verify=silence %s // RUN: %clang_cc1 -triple i386 -fsyntax-only -verify=silence %s // RUN: %clang_cc1 -triple x86_64 -fsyntax-only -verify=silence %s +// RUN: %clang_cc1 -triple riscv32 -fsyntax-only -verify=silence %s +// RUN: %clang_cc1 -triple riscv64 -fsyntax-only -verify=silence %s // RUN: %clang_cc1 -triple ppc64le -fsyntax-only -verify %s // silence-no-diagnostics diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h index 2538d9992de7..ef386fe16920 100644 --- a/llvm/lib/Target/RISCV/RISCV.h +++ b/llvm/lib/Target/RISCV/RISCV.h @@ -30,8 +30,8 @@ class MachineInstr; class MachineOperand; class PassRegistry; -void LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, - const AsmPrinter &AP); +bool lowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, + AsmPrinter &AP); bool LowerRISCVMachineOperandToMCOperand(const MachineOperand &MO, MCOperand &MCOp, const AsmPrinter &AP); diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp index 0a915cbcc1af..1b7a923e23b1 100644 --- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp +++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp @@ -93,8 +93,8 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) { return; MCInst TmpInst; - LowerRISCVMachineInstrToMCInst(MI, TmpInst, *this); - EmitToStreamer(*OutStreamer, TmpInst); + if (!lowerRISCVMachineInstrToMCInst(MI, TmpInst, *this)) + EmitToStreamer(*OutStreamer, TmpInst); } bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 5e1be68b4835..a2ce3597be8f 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -23,6 +23,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" @@ -49,6 +50,15 @@ RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI) : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP), STI(STI) {} +MCInst RISCVInstrInfo::getNop() const { + if (STI.getFeatureBits()[RISCV::FeatureStdExtC]) + return MCInstBuilder(RISCV::C_NOP); + return MCInstBuilder(RISCV::ADDI) + .addReg(RISCV::X0) + .addReg(RISCV::X0) + .addImm(0); +} + unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const { switch (MI.getOpcode()) { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 64f6c6236453..f15d61ede037 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -29,6 +29,8 @@ class RISCVInstrInfo : public RISCVGenInstrInfo { public: explicit RISCVInstrInfo(RISCVSubtarget &STI); + MCInst getNop() const override; + unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override; unsigned isStoreToStackSlot(const MachineInstr &MI, diff --git a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp index 3c38dd1bf64d..1841e8a0a432 100644 --- a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp +++ b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp @@ -204,10 +204,10 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI, return true; } -void llvm::LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, - const AsmPrinter &AP) { +bool llvm::lowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, + AsmPrinter &AP) { if (lowerRISCVVMachineInstrToMCInst(MI, OutMI)) - return; + return false; OutMI.setOpcode(MI->getOpcode()); @@ -217,19 +217,32 @@ void llvm::LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI, OutMI.addOperand(MCOp); } - if (OutMI.getOpcode() == RISCV::PseudoReadVLENB) { + switch (OutMI.getOpcode()) { + case TargetOpcode::PATCHABLE_FUNCTION_ENTER: { + const Function &F = MI->getParent()->getParent()->getFunction(); + if (F.hasFnAttribute("patchable-function-entry")) { + unsigned Num; + if (F.getFnAttribute("patchable-function-entry") + .getValueAsString() + .getAsInteger(10, Num)) + return false; + AP.emitNops(Num); + return true; + } + break; + } + case RISCV::PseudoReadVLENB: OutMI.setOpcode(RISCV::CSRRS); OutMI.addOperand(MCOperand::createImm( RISCVSysReg::lookupSysRegByName("VLENB")->Encoding)); OutMI.addOperand(MCOperand::createReg(RISCV::X0)); - return; - } - - if (OutMI.getOpcode() == RISCV::PseudoReadVL) { + break; + case RISCV::PseudoReadVL: OutMI.setOpcode(RISCV::CSRRS); - OutMI.addOperand(MCOperand::createImm( - RISCVSysReg::lookupSysRegByName("VL")->Encoding)); + OutMI.addOperand( + MCOperand::createImm(RISCVSysReg::lookupSysRegByName("VL")->Encoding)); OutMI.addOperand(MCOperand::createReg(RISCV::X0)); - return; + break; } + return false; } diff --git a/llvm/test/CodeGen/RISCV/patchable-function-entry.ll b/llvm/test/CodeGen/RISCV/patchable-function-entry.ll new file mode 100644 index 000000000000..9c4eb2c414df --- /dev/null +++ b/llvm/test/CodeGen/RISCV/patchable-function-entry.ll @@ -0,0 +1,71 @@ +;; Test the function attribute "patchable-function-entry". +; RUN: llc -mtriple=riscv32 --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV32,NORVC +; RUN: llc -mtriple=riscv64 --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV64,NORVC +; RUN: llc -mtriple=riscv32 -mattr=+c --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV32,RVC +; RUN: llc -mtriple=riscv64 -mattr=+c --riscv-no-aliases < %s | FileCheck %s --check-prefixes=CHECK,RV64,RVC + +define void @f0() "patchable-function-entry"="0" { +; CHECK-LABEL: f0: +; CHECK-NEXT: .Lfunc_begin0: +; CHECK-NOT: {{addi|c.nop}} +; NORVC: jalr zero, 0(ra) +; RVC: c.jr ra +; CHECK-NOT: .section __patchable_function_entries + ret void +} + +define void @f1() "patchable-function-entry"="1" { +; CHECK-LABEL: f1: +; CHECK-NEXT: .Lfunc_begin1: +; NORVC: addi zero, zero, 0 +; NORVC-NEXT: jalr zero, 0(ra) +; RVC: c.nop +; RVC-NEXT: c.jr ra +; CHECK: .section __patchable_function_entries,"awo",@progbits,f1{{$}} +; 32: .p2align 2 +; 32-NEXT: .word .Lfunc_begin1 +; 64: .p2align 3 +; 64-NEXT: .quad .Lfunc_begin1 + ret void +} + +$f5 = comdat any +define void @f5() "patchable-function-entry"="5" comdat { +; CHECK-LABEL: f5: +; CHECK-NEXT: .Lfunc_begin2: +; NORVC-COUNT-5: addi zero, zero, 0 +; NORVC-NEXT: jalr zero, 0(ra) +; RVC-COUNT-5: c.nop +; RVC-NEXT: c.jr ra +; CHECK: .section __patchable_function_entries,"aGwo",@progbits,f5,comdat,f5{{$}} +; RV32: .p2align 2 +; RV32-NEXT: .word .Lfunc_begin2 +; RV64: .p2align 3 +; RV64-NEXT: .quad .Lfunc_begin2 + ret void +} + +;; -fpatchable-function-entry=3,2 +;; "patchable-function-prefix" emits data before the function entry label. +define void @f3_2() "patchable-function-entry"="1" "patchable-function-prefix"="2" { +; CHECK-LABEL: .type f3_2,@function +; CHECK-NEXT: .Ltmp0: # @f3_2 +; NORVC-COUNT-2: addi zero, zero, 0 +; RVC-COUNT-2: c.nop +; CHECK-NEXT: f3_2: +; CHECK: # %bb.0: +; NORVC-NEXT: addi zero, zero, 0 +; NORVC-NEXT: addi sp, sp, -16 +; RVC-NEXT: c.nop +; RVC-NEXT: c.addi sp, -16 +;; .size does not include the prefix. +; CHECK: .Lfunc_end3: +; CHECK-NEXT: .size f3_2, .Lfunc_end3-f3_2 +; CHECK: .section __patchable_function_entries,"awo",@progbits,f3_2{{$}} +; RV32: .p2align 2 +; RV32-NEXT: .word .Ltmp0 +; RV64: .p2align 3 +; RV64-NEXT: .quad .Ltmp0 + %frame = alloca i8, i32 16 + ret void +} -- GitLab From b04c87e0537672d1377d26a7bfe141021119f618 Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 15 Mar 2021 16:16:36 -0700 Subject: [PATCH 0052/1206] Support !heapallocsite attachments in stripNonLineTableDebugInfo(). They point into the DIType type system, so they need to be stripped as well. rdar://75341300 Differential Revision: https://reviews.llvm.org/D98667 --- llvm/lib/IR/DebugInfo.cpp | 4 ++ ...ip-nonlinetable-debuginfo-heapallocsite.ll | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index fec98f5a36c0..381f13396dab 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -654,6 +654,10 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) { updateLoopMetadataDebugLocations(I, [&](const DILocation &Loc) { return remapDebugLoc(&Loc).get(); }); + + // Strip heapallocsite attachments, they point into the DIType system. + if (I.hasMetadataOtherThanDebugLoc()) + I.setMetadata("heapallocsite", nullptr); } } } diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll new file mode 100644 index 000000000000..f99fb320a1fc --- /dev/null +++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll @@ -0,0 +1,46 @@ +; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - | FileCheck %s +; int *get() { return new int[256]; } +; ModuleID = '/tmp/heapallocsite.cpp' +source_filename = "/tmp/heapallocsite.cpp" +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx11.0.0" + +; Function Attrs: noinline optnone ssp uwtable mustprogress +define dso_local i32* @_Z3getv() #0 !dbg !8 { +entry: +; CHECK-LABEL: entry: +; CHECK-NOT: !heapallocsite + %call = call noalias nonnull i8* @_Znam(i64 1024) #2, !dbg !14, !heapallocsite !13 + %0 = bitcast i8* %call to i32*, !dbg !14 + ret i32* %0, !dbg !15 +} + +; Function Attrs: nobuiltin allocsize(0) +declare nonnull i8* @_Znam(i64) #1 + +attributes #0 = { noinline optnone ssp uwtable mustprogress } +attributes #1 = { nobuiltin allocsize(0) "frame-pointer"="all" } +attributes #2 = { builtin allocsize(0) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5, !6} +!llvm.ident = !{!7} + +; CHECK-LABEL: !0 = +; CHECK-NOT: !DIBasicType(name: "int" +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (git@github.com:llvm/llvm-project 6d4ce49dae17715de502acbd50ab4c9b3c18215b)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None, sysroot: "/") +!1 = !DIFile(filename: "/tmp/heapallocsite.cpp", directory: "/Volumes/Data/llvm-project") +!2 = !{} +!3 = !{i32 7, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{i32 7, !"PIC Level", i32 2} +!7 = !{!"clang version 13.0.0 (git@github.com:llvm/llvm-project 6d4ce49dae17715de502acbd50ab4c9b3c18215b)"} +!8 = distinct !DISubprogram(name: "get", linkageName: "_Z3getv", scope: !9, file: !9, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2) +!9 = !DIFile(filename: "/tmp/heapallocsite.cpp", directory: "") +!10 = !DISubroutineType(types: !11) +!11 = !{!12} +!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64) +!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!14 = !DILocation(line: 1, column: 21, scope: !8) +!15 = !DILocation(line: 1, column: 14, scope: !8) -- GitLab From c3a18bb1e83149f38f0064f11a2ad97245a5848b Mon Sep 17 00:00:00 2001 From: Adrian Prantl Date: Mon, 15 Mar 2021 16:23:31 -0700 Subject: [PATCH 0053/1206] Support !heapallocsite attachments in StripDebugInfo(). They point into the DIType type system, so they need to be stripped as well. rdar://75341300 Differential Revision: https://reviews.llvm.org/D98668 --- llvm/lib/IR/DebugInfo.cpp | 3 +++ .../Util/strip-nonlinetable-debuginfo-heapallocsite.ll | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 381f13396dab..99349a304e30 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -345,6 +345,9 @@ bool llvm::stripDebugInfo(Function &F) { if (NewLoopID != LoopID) I.setMetadata(LLVMContext::MD_loop, NewLoopID); } + // Strip heapallocsite attachments, they point into the DIType system. + if (I.hasMetadataOtherThanDebugLoc()) + I.setMetadata("heapallocsite", nullptr); } } return Changed; diff --git a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll index f99fb320a1fc..ceb18addace4 100644 --- a/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll +++ b/llvm/test/Transforms/Util/strip-nonlinetable-debuginfo-heapallocsite.ll @@ -1,4 +1,5 @@ ; RUN: opt -S -strip-nonlinetable-debuginfo %s -o - | FileCheck %s +; RUN: opt -S -strip-debug %s -o - | FileCheck %s ; int *get() { return new int[256]; } ; ModuleID = '/tmp/heapallocsite.cpp' source_filename = "/tmp/heapallocsite.cpp" @@ -26,7 +27,7 @@ attributes #2 = { builtin allocsize(0) } !llvm.module.flags = !{!3, !4, !5, !6} !llvm.ident = !{!7} -; CHECK-LABEL: !0 = +; CHECK-LABEL: !llvm.ident ; CHECK-NOT: !DIBasicType(name: "int" !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 13.0.0 (git@github.com:llvm/llvm-project 6d4ce49dae17715de502acbd50ab4c9b3c18215b)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None, sysroot: "/") !1 = !DIFile(filename: "/tmp/heapallocsite.cpp", directory: "/Volumes/Data/llvm-project") -- GitLab From 8fbedb6b908f20b9972cb064dfe7b6d7ea3782ef Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Tue, 16 Mar 2021 10:07:01 -0700 Subject: [PATCH 0054/1206] [llvm-nm] Add --format=just-symbols and make --just-symbol-name its alias https://sourceware.org/bugzilla/show_bug.cgi?id=27487 binutils will have --format=just-symbols/-j as well. Arbitrarily prefer `-j` to `--format=sysv`. Previously `--format=sysv -j` prints in the sysv format while `-j` takes precedence over other formats. Differential Revision: https://reviews.llvm.org/D98569 --- llvm/docs/CommandGuide/llvm-nm.rst | 5 +- llvm/test/tools/llvm-nm/just-symbol-name.test | 38 ------------- llvm/test/tools/llvm-nm/just-symbols.test | 53 +++++++++++++++++++ llvm/tools/llvm-nm/llvm-nm.cpp | 20 ++++--- 4 files changed, 68 insertions(+), 48 deletions(-) delete mode 100644 llvm/test/tools/llvm-nm/just-symbol-name.test create mode 100644 llvm/test/tools/llvm-nm/just-symbols.test diff --git a/llvm/docs/CommandGuide/llvm-nm.rst b/llvm/docs/CommandGuide/llvm-nm.rst index 747192a9d924..20779b17fdde 100644 --- a/llvm/docs/CommandGuide/llvm-nm.rst +++ b/llvm/docs/CommandGuide/llvm-nm.rst @@ -149,7 +149,8 @@ OPTIONS .. option:: --format=, -f - Select an output format; *format* may be *sysv*, *posix*, *darwin*, or *bsd*. + Select an output format; *format* may be *sysv*, *posix*, *darwin*, *bsd* or + *just-symbols*. The default is *bsd*. .. option:: --help, -h @@ -162,7 +163,7 @@ OPTIONS .. option:: --just-symbol-name, -j - Print just the symbol names. + Print just the symbol names. Alias for `--format=just-symbols``. .. option:: -m diff --git a/llvm/test/tools/llvm-nm/just-symbol-name.test b/llvm/test/tools/llvm-nm/just-symbol-name.test deleted file mode 100644 index 85be79a6a240..000000000000 --- a/llvm/test/tools/llvm-nm/just-symbol-name.test +++ /dev/null @@ -1,38 +0,0 @@ -## Show that the -j/--just-symbol-name prints only the the symbol name (except -## in posix output). - -# RUN: yaml2obj %s -o %t.o - -# RUN: llvm-nm --just-symbol-name %t.o > %t.bsd.txt -# RUN: llvm-nm -j %t.o > %t.j.txt -# RUN: cmp %t.bsd.txt %t.j.txt - -# RUN: FileCheck %s --input-file=%t.bsd.txt --implicit-check-not={{.}} --check-prefix=COMMON -# RUN: llvm-nm -j %t.o --format=sysv | \ -# RUN: FileCheck %s --implicit-check-not={{.}} --check-prefixes=COMMON,SYSV -DFILE=%t.o -# RUN: llvm-nm -j %t.o --format=posix | FileCheck %s --implicit-check-not={{.}} --check-prefix=POSIX - -# SYSV: Symbols from [[FILE]]: -# SYSV-EMPTY: -# SYSV-NEXT: Name Value Class Type Size Line Section -# COMMON: {{^}}defined{{$}} -# COMMON-NEXT: {{^}}undefined{{$}} - -# POSIX: defined T 0 0 -# POSIX-NEXT: undefined U 0 0 - ---- !ELF -FileHeader: - Class: ELFCLASS64 - Data: ELFDATA2LSB - Type: ET_REL -Sections: - - Name: .text - Type: SHT_PROGBITS - Flags: [ SHF_ALLOC, SHF_EXECINSTR ] -Symbols: - - Name: defined - Section: .text - Binding: STB_GLOBAL - - Name: undefined - Binding: STB_GLOBAL diff --git a/llvm/test/tools/llvm-nm/just-symbols.test b/llvm/test/tools/llvm-nm/just-symbols.test new file mode 100644 index 000000000000..53bd2e25a62e --- /dev/null +++ b/llvm/test/tools/llvm-nm/just-symbols.test @@ -0,0 +1,53 @@ +## Show that the -j/--just-symbol-name/--format=just-symbols prints only the the +## symbol name. + +# RUN: yaml2obj %s -o %t.o + +# RUN: llvm-nm -j %t.o > %t.txt +# RUN: llvm-nm --just-symbol-name %t.o | diff %t.txt - +# RUN: llvm-nm --format=just-symbols %t.o | diff %t.txt - +# RUN: llvm-nm --format=sysv -j %t.o | diff %t.txt - +# RUN: llvm-nm -j --format=posix %t.o | diff %t.txt - + +# RUN: FileCheck %s --input-file=%t.txt --implicit-check-not={{.}} --check-prefix=COMMON + +# COMMON: {{^}}defined{{$}} +# COMMON-NEXT: {{^}}undefined{{$}} + +# RUN: llvm-nm -j %t.o %t.o | FileCheck %s --check-prefix=MULTI1 -DFILE=%t.o + +# MULTI1-NOT: {{.}} +# MULTI1: {{^$}} +# MULTI1-NEXT: [[FILE]]: +# MULTI1-NEXT: defined +# MULTI1-NEXT: undefined +# MULTI1-EMPTY: +# MULTI1-NEXT: [[FILE]]: +# MULTI1-NEXT: defined +# MULTI1-NEXT: undefined +# MULTI1-NOT: {{.}} + +# RUN: llvm-nm -j --print-file-name %t.o %t.o | FileCheck %s --check-prefix=MULTI2 -DFILE=%t.o + +# MULTI2-NOT: {{.}} +# MULTI2: [[FILE]]: defined +# MULTI2-NEXT: [[FILE]]: undefined +# MULTI2-NEXT: [[FILE]]: defined +# MULTI2-NEXT: [[FILE]]: undefined +# MULTI2-NOT: {{.}} + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL +Sections: + - Name: .text + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] +Symbols: + - Name: defined + Section: .text + Binding: STB_GLOBAL + - Name: undefined + Binding: STB_GLOBAL diff --git a/llvm/tools/llvm-nm/llvm-nm.cpp b/llvm/tools/llvm-nm/llvm-nm.cpp index b978eafcbda7..6438b7867dbd 100644 --- a/llvm/tools/llvm-nm/llvm-nm.cpp +++ b/llvm/tools/llvm-nm/llvm-nm.cpp @@ -47,7 +47,7 @@ using namespace llvm; using namespace object; namespace { -enum OutputFormatTy { bsd, sysv, posix, darwin }; +enum OutputFormatTy { bsd, sysv, posix, darwin, just_symbols }; cl::OptionCategory NMCat("llvm-nm Options"); @@ -55,7 +55,9 @@ cl::opt OutputFormat( "format", cl::desc("Specify output format"), cl::values(clEnumVal(bsd, "BSD format"), clEnumVal(sysv, "System V format"), clEnumVal(posix, "POSIX.2 format"), - clEnumVal(darwin, "Darwin -m format")), + clEnumVal(darwin, "Darwin -m format"), + cl::OptionEnumValue{"just-symbols", int(just_symbols), + "just symbol names"}), cl::init(bsd), cl::cat(NMCat)); cl::alias OutputFormat2("f", cl::desc("Alias for --format"), cl::aliasopt(OutputFormat)); @@ -180,9 +182,9 @@ cl::alias RadixAlias("t", cl::desc("Alias for --radix"), cl::aliasopt(AddressRadix)); cl::opt JustSymbolName("just-symbol-name", - cl::desc("Print just the symbol's name"), + cl::desc("Alias for --format=just-symbols"), cl::cat(NMCat)); -cl::alias JustSymbolNames("j", cl::desc("Alias for --just-symbol-name"), +cl::alias JustSymbolNames("j", cl::desc("Alias for --format-just-symbols"), cl::aliasopt(JustSymbolName), cl::Grouping); cl::opt @@ -772,10 +774,10 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName, } if (!PrintFileName) { - if (OutputFormat == posix && MultipleFiles && printName) { + if ((OutputFormat == bsd || OutputFormat == posix || + OutputFormat == just_symbols) && + MultipleFiles && printName) { outs() << '\n' << CurrentFilename << ":\n"; - } else if (OutputFormat == bsd && MultipleFiles && printName) { - outs() << "\n" << CurrentFilename << ":\n"; } else if (OutputFormat == sysv) { outs() << "\n\nSymbols from " << CurrentFilename << ":\n\n"; if (isSymbolList64Bit(Obj)) @@ -844,7 +846,7 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName, continue; if (PrintFileName) writeFileName(outs(), ArchiveName, ArchitectureName); - if ((JustSymbolName || + if ((OutputFormat == just_symbols || (UndefinedOnly && MachO && OutputFormat != darwin)) && OutputFormat != posix) { outs() << Name << "\n"; @@ -2251,6 +2253,8 @@ int main(int argc, char **argv) { OutputFormat = posix; if (DarwinFormat) OutputFormat = darwin; + if (JustSymbolName) + OutputFormat = just_symbols; // The relative order of these is important. If you pass --size-sort it should // only print out the size. However, if you pass -S --size-sort, it should -- GitLab From b85d3e27ad775a372435981302cb5d2c73811d56 Mon Sep 17 00:00:00 2001 From: Aart Bik Date: Tue, 16 Mar 2021 09:43:42 -0700 Subject: [PATCH 0055/1206] [mlir][amx] reformatted examples Examples were missing the underscore of the actual ops format. Reviewed By: ftynse Differential Revision: https://reviews.llvm.org/D98723 --- mlir/include/mlir/Dialect/AMX/AMX.td | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mlir/include/mlir/Dialect/AMX/AMX.td b/mlir/include/mlir/Dialect/AMX/AMX.td index 710387e70b55..45c63a99e670 100644 --- a/mlir/include/mlir/Dialect/AMX/AMX.td +++ b/mlir/include/mlir/Dialect/AMX/AMX.td @@ -71,7 +71,7 @@ def TileZeroOp : AMX_Op<"tile_zero", [NoSideEffect]> { Example: ```mlir - %0 = amx.tilezero : vector<16x16xbf16> + %0 = amx.tile_zero : vector<16x16xbf16> ``` }]; let verifier = [{ return ::verify(*this); }]; @@ -100,7 +100,7 @@ def TileLoadOp : AMX_Op<"tile_load", [NoSideEffect]> { Example: ```mlir - %0 = amx.tileload %arg0[%c0, %c0] : memref into vector<16x64xi8> + %0 = amx.tile_load %arg0[%c0, %c0] : memref into vector<16x64xi8> ``` }]; let verifier = [{ return ::verify(*this); }]; @@ -131,7 +131,7 @@ def TileStoreOp : AMX_Op<"tile_store"> { Example: ```mlir - amx.tilestore %arg1[%c0, %c0], %0 : memref, vector<16x64xi8> + amx.tile_store %arg1[%c0, %c0], %0 : memref, vector<16x64xi8> ``` }]; let verifier = [{ return ::verify(*this); }]; @@ -165,7 +165,7 @@ def TileMulFOp : AMX_Op<"tile_mulf", [NoSideEffect, AllTypesMatch<["acc", "res"] Example: ```mlir - %0 = amx.tilemulf %a, %b, %c + %0 = amx.tile_mulf %a, %b, %c : vector<16x32xbf16>, vector<16x32xbf16>, vector<16x16xf32> ``` }]; @@ -203,7 +203,7 @@ def TileMulIOp : AMX_Op<"tile_muli", [NoSideEffect, AllTypesMatch<["acc", "res"] Example: ```mlir - %0 = amx.tilemuli %a, %b, %c [true, true] + %0 = amx.tile_muli %a, %b, %c [true, true] : vector<16x64xi8>, vector<16x64xi8>, vector<16x16xi32> ``` }]; -- GitLab From fe990ee8159616d0739b315d0a961adb9c5695eb Mon Sep 17 00:00:00 2001 From: Vaivaswatha Nagaraj Date: Tue, 16 Mar 2021 16:42:30 +0530 Subject: [PATCH 0056/1206] [Docs] Mention linking to reviews page when committing Differential Revision: https://reviews.llvm.org/D98695 --- llvm/docs/DeveloperPolicy.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/docs/DeveloperPolicy.rst b/llvm/docs/DeveloperPolicy.rst index 6b6fe29a963e..361aedb8e6c5 100644 --- a/llvm/docs/DeveloperPolicy.rst +++ b/llvm/docs/DeveloperPolicy.rst @@ -291,6 +291,9 @@ Below are some guidelines about the format of the message itself: related commit. This could be as simple as "Revert commit NNNN because it caused PR#". +* If the patch has been reviewed, add a link to its review page, as shown + `here `_. + For minor violations of these recommendations, the community normally favors reminding the contributor of this policy over reverting. Minor corrections and omissions can be handled by sending a reply to the commits mailing list. -- GitLab From a80a33e8b55393c060e51486cfd8085b380eb36d Mon Sep 17 00:00:00 2001 From: Giorgis Georgakoudis Date: Tue, 16 Mar 2021 07:41:39 -0700 Subject: [PATCH 0057/1206] [Utils] Support lit-like substitutions in update_cc_test_checks Reviewed By: jdoerfert Differential Revision: https://reviews.llvm.org/D98712 --- .../Inputs/exec-all-runlines.c | 3 ++- .../Inputs/exec-all-runlines.c.expected | 3 ++- llvm/utils/update_cc_test_checks.py | 24 +++++++++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c index 1626eb540841..e0dfc42c4bd6 100644 --- a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c +++ b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c @@ -1,5 +1,6 @@ // Check that the non-clang/non-filechecked runlines execute -// RUN: cp %s %s.copy.c +// RUN: cp %s %S/Output/execute-all-runlines.copy.c +// RUN: cp %S/Output/execute-all-runlines.copy.c %s.copy.c // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp %s.copy.c -emit-llvm-bc -o %t-host.bc // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s diff --git a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected index 5edf11e668e4..ae9745fa9b1e 100644 --- a/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected +++ b/clang/test/utils/update_cc_test_checks/Inputs/exec-all-runlines.c.expected @@ -1,6 +1,7 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // Check that the non-clang/non-filechecked runlines execute -// RUN: cp %s %s.copy.c +// RUN: cp %s %S/Output/execute-all-runlines.copy.c +// RUN: cp %S/Output/execute-all-runlines.copy.c %s.copy.c // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp %s.copy.c -emit-llvm-bc -o %t-host.bc // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fopenmp -fopenmp-host-ir-file-path %t-host.bc %s.copy.c -emit-llvm -o - | FileCheck %s diff --git a/llvm/utils/update_cc_test_checks.py b/llvm/utils/update_cc_test_checks.py index d084bc6d0795..d3af5308ac6a 100755 --- a/llvm/utils/update_cc_test_checks.py +++ b/llvm/utils/update_cc_test_checks.py @@ -176,7 +176,7 @@ def config(): return args, parser -def get_function_body(builder, args, filename, clang_args, extra_commands, +def get_function_body(builder, args, filename, clang_args, extra_commands, prefixes): # TODO Clean up duplication of asm/common build_function_body_dictionary # Invoke external tool and extract function bodies. @@ -221,6 +221,13 @@ def main(): # Build a list of clang command lines and check prefixes from RUN lines. run_list = [] line2spell_and_mangled_list = collections.defaultdict(list) + + subs = { + '%s' : ti.path, + '%t' : tempfile.NamedTemporaryFile().name, + '%S' : os.getcwd(), + } + for l in ti.run_lines: commands = [cmd.strip() for cmd in l.split('|')] @@ -234,15 +241,18 @@ def main(): # Execute non-clang runline. if exec_args[0] not in SUBST: print('NOTE: Executing non-clang RUN line: ' + l, file=sys.stderr) - # Replace %s by `filename`. - exec_args = [i.replace('%s', ti.path) if '%s' in i else i for i in exec_args] + # Do lit-like substitutions. + for s in subs: + exec_args = [i.replace(s, subs[s]) if s in i else i for i in exec_args] exec_run_line(exec_args) continue - # This is a clang runline, apply %clang substitution rule, replace %s by `filename`, + # This is a clang runline, apply %clang substitution rule, do lit-like substitutions, # and append args.clang_args clang_args = exec_args clang_args[0:1] = SUBST[clang_args[0]] - clang_args = [i.replace('%s', ti.path) if '%s' in i else i for i in clang_args] + ti.args.clang_args + for s in subs: + clang_args = [i.replace(s, subs[s]) if s in i else i for i in clang_args] + clang_args += ti.args.clang_args # Extract -check-prefix in FileCheck args filecheck_cmd = commands[-1] @@ -271,7 +281,7 @@ def main(): common.debug('Extracted clang cmd: clang {}'.format(clang_args)) common.debug('Extracted FileCheck prefixes: {}'.format(prefixes)) - get_function_body(builder, ti.args, ti.path, clang_args, extra_commands, + get_function_body(builder, ti.args, ti.path, clang_args, extra_commands, prefixes) # Invoke clang -Xclang -ast-dump=json to get mapping from start lines to @@ -315,7 +325,7 @@ def main(): prefixes, func_dict, func) - common.add_checks_at_end(output_lines, run_list, builder.func_order(), + common.add_checks_at_end(output_lines, run_list, builder.func_order(), '//', lambda my_output_lines, prefixes, func: check_generator(my_output_lines, prefixes, func)) -- GitLab From f586de8459ce897faf532fdd49fd4aa81747589e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 16 Mar 2021 11:42:08 +0000 Subject: [PATCH 0058/1206] [VPlan] Remove PredInst2Recipe, use VP operands instead. (NFC) Instead of maintaining a separate map from predicated instructions to recipes, we can instead directly look at the VP operands. If the operand comes from a predicated instruction, the operand will be a VPPredInstPHIRecipe with a VPReplicateRecipe as its operand. --- .../Transforms/Vectorize/LoopVectorize.cpp | 25 +++++++++---------- .../Transforms/Vectorize/VPRecipeBuilder.h | 1 - llvm/lib/Transforms/Vectorize/VPlan.h | 2 ++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c92b00078c7e..f6f51e78bb27 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8534,7 +8534,6 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VPlan &Plan) const { VPBasicBlock *VPRecipeBuilder::handleReplication( Instruction *I, VFRange &Range, VPBasicBlock *VPBB, - DenseMap &PredInst2Recipe, VPlanPtr &Plan) { bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange( [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); }, @@ -8552,10 +8551,16 @@ VPBasicBlock *VPRecipeBuilder::handleReplication( // Find if I uses a predicated instruction. If so, it will use its scalar // value. Avoid hoisting the insert-element which packs the scalar value into // a vector value, as that happens iff all users use the vector value. - for (auto &Op : I->operands()) - if (auto *PredInst = dyn_cast(Op)) - if (PredInst2Recipe.find(PredInst) != PredInst2Recipe.end()) - PredInst2Recipe[PredInst]->setAlsoPack(false); + for (VPValue *Op : Recipe->operands()) { + auto *PredR = dyn_cast_or_null(Op->getDef()); + if (!PredR) + continue; + auto *RepR = + cast_or_null(PredR->getOperand(0)->getDef()); + assert(RepR->isPredicated() && + "expected Replicate recipe to be predicated"); + RepR->setAlsoPack(false); + } // Finalize the recipe for Instr, first if it is not predicated. if (!IsPredicated) { @@ -8567,7 +8572,6 @@ VPBasicBlock *VPRecipeBuilder::handleReplication( assert(VPBB->getSuccessors().empty() && "VPBB has successors when handling predicated replication."); // Record predicated instructions for above packing optimizations. - PredInst2Recipe[I] = Recipe; VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan); VPBlockUtils::insertBlockAfter(Region, VPBB); auto *RegSucc = new VPBasicBlock(); @@ -8695,11 +8699,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( VFRange &Range, SmallPtrSetImpl &DeadInstructions, const DenseMap &SinkAfter) { - // Hold a mapping from predicated instructions to their recipes, in order to - // fix their AlsoPack behavior if a user is determined to replicate and use a - // scalar instead of vector value. - DenseMap PredInst2Recipe; - SmallPtrSet *, 1> InterleaveGroups; VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder); @@ -8803,8 +8802,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( // Otherwise, if all widening options failed, Instruction is to be // replicated. This may create a successor for VPBB. - VPBasicBlock *NextVPBB = RecipeBuilder.handleReplication( - Instr, Range, VPBB, PredInst2Recipe, Plan); + VPBasicBlock *NextVPBB = + RecipeBuilder.handleReplication(Instr, Range, VPBB, Plan); if (NextVPBB != VPBB) { VPBB = NextVPBB; VPBB->setName(BB->hasName() ? BB->getName() + "." + Twine(VPBBsForBB++) diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 985bf8579ab4..89c7b127d3ba 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -162,7 +162,6 @@ public: /// \p Range.Start to \p Range.End. VPBasicBlock *handleReplication( Instruction *I, VFRange &Range, VPBasicBlock *VPBB, - DenseMap &PredInst2Recipe, VPlanPtr &Plan); }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 5a23c6839982..9b5d5d7e77be 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1221,6 +1221,8 @@ public: bool isUniform() const { return IsUniform; } bool isPacked() const { return AlsoPack; } + + bool isPredicated() const { return IsPredicated; } }; /// A recipe for generating conditional branches on the bits of a mask. -- GitLab From 6972e39d47eccc3e5fc3ded4a1e1b78f74d10af6 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Tue, 16 Mar 2021 10:52:01 -0700 Subject: [PATCH 0059/1206] [gvn] CSE gc.relocates based on meaning, not spelling (try 2) This was (partially) reverted in cfe8f8e0 because the conversion from readonly to readnone in Intrinsics.td exposed a couple of problems. This change has been reworked to not need that change (via some explicit checks in client code). This is being done to address the original optimization issue and simplify the testing of the readonly changes. I'm working on that piece under 49607. Original commit message follows: The last two operands to a gc.relocate represent indices into the associated gc.statepoint's gc bundle list. (Effectively, gc.relocates are projections from the gc.statepoints multiple return values.) We can use this to recognize when two gc.relocates are equivalent (and can be CSEd), even when the indices are non-equal. This is particular useful when considering a chain of multiple statepoints as it lets us eliminate all duplicate gc.relocates in a single pass. Differential Revision: https://reviews.llvm.org/D97974 --- llvm/lib/Transforms/Scalar/EarlyCSE.cpp | 31 +++++++++++--------- llvm/lib/Transforms/Scalar/GVN.cpp | 4 ++- llvm/test/Transforms/EarlyCSE/gc_relocate.ll | 5 +--- llvm/test/Transforms/GVN/gc_relocate.ll | 13 ++------ 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp index 180a82917fa9..9c7d43078821 100644 --- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp +++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp @@ -109,6 +109,9 @@ struct SimpleValue { static bool canHandle(Instruction *Inst) { // This can only handle non-void readnone functions. + if (isa(Inst)) + // Migration assistant for PR49607, to be removed once complete + return true; if (CallInst *CI = dyn_cast(Inst)) return CI->doesNotAccessMemory() && !CI->getType()->isVoidTy(); return isa(Inst) || isa(Inst) || @@ -280,6 +283,13 @@ static unsigned getHashValueImpl(SimpleValue Val) { return hash_combine(II->getOpcode(), LHS, RHS); } + // gc.relocate is 'special' call: its second and third operands are + // not real values, but indices into statepoint's argument list. + // Get values they point to. + if (const GCRelocateInst *GCR = dyn_cast(Inst)) + return hash_combine(GCR->getOpcode(), GCR->getOperand(0), + GCR->getBasePtr(), GCR->getDerivedPtr()); + // Mix in the opcode. return hash_combine( Inst->getOpcode(), @@ -341,6 +351,13 @@ static bool isEqualImpl(SimpleValue LHS, SimpleValue RHS) { LII->getArgOperand(1) == RII->getArgOperand(0); } + // See comment above in `getHashValue()`. + if (const GCRelocateInst *GCR1 = dyn_cast(LHSI)) + if (const GCRelocateInst *GCR2 = dyn_cast(RHSI)) + return GCR1->getOperand(0) == GCR2->getOperand(0) && + GCR1->getBasePtr() == GCR2->getBasePtr() && + GCR1->getDerivedPtr() == GCR2->getDerivedPtr(); + // Min/max can occur with commuted operands, non-canonical predicates, // and/or non-canonical operands. // Selects can be non-trivially equivalent via inverted conditions and swaps. @@ -454,13 +471,6 @@ template <> struct DenseMapInfo { unsigned DenseMapInfo::getHashValue(CallValue Val) { Instruction *Inst = Val.Inst; - // gc.relocate is 'special' call: its second and third operands are - // not real values, but indices into statepoint's argument list. - // Get values they point to. - if (const GCRelocateInst *GCR = dyn_cast(Inst)) - return hash_combine(GCR->getOpcode(), GCR->getOperand(0), - GCR->getBasePtr(), GCR->getDerivedPtr()); - // Hash all of the operands as pointers and mix in the opcode. return hash_combine( Inst->getOpcode(), @@ -472,13 +482,6 @@ bool DenseMapInfo::isEqual(CallValue LHS, CallValue RHS) { if (LHS.isSentinel() || RHS.isSentinel()) return LHSI == RHSI; - // See comment above in `getHashValue()`. - if (const GCRelocateInst *GCR1 = dyn_cast(LHSI)) - if (const GCRelocateInst *GCR2 = dyn_cast(RHSI)) - return GCR1->getOperand(0) == GCR2->getOperand(0) && - GCR1->getBasePtr() == GCR2->getBasePtr() && - GCR1->getDerivedPtr() == GCR2->getDerivedPtr(); - return LHSI->isIdenticalTo(RHSI); } diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index b9171889005a..65f7d0498adf 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -397,7 +397,9 @@ void GVN::ValueTable::add(Value *V, uint32_t num) { } uint32_t GVN::ValueTable::lookupOrAddCall(CallInst *C) { - if (AA->doesNotAccessMemory(C)) { + // The gc.relocate specific check is to simplify migration under PR49607, and + // is to be removed once complete. + if (AA->doesNotAccessMemory(C) || isa(C)) { Expression exp = createExpr(C); uint32_t e = assignExpNewValueNum(exp).first; valueNumbering[C] = e; diff --git a/llvm/test/Transforms/EarlyCSE/gc_relocate.ll b/llvm/test/Transforms/EarlyCSE/gc_relocate.ll index df32d3f85b1e..ae9001c9db66 100644 --- a/llvm/test/Transforms/EarlyCSE/gc_relocate.ll +++ b/llvm/test/Transforms/EarlyCSE/gc_relocate.ll @@ -30,11 +30,8 @@ define i1 @test_readnone(i32 addrspace(1)* %in) gc "statepoint-example" { ; CHECK-NEXT: [[SAFEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[IN:%.*]]) ] ; CHECK-NEXT: [[A:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: store i32 0, i32* @G, align 4 -; CHECK-NEXT: [[B:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i32 addrspace(1)* [[A]], null -; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 addrspace(1)* [[B]], null -; CHECK-NEXT: [[CMP:%.*]] = and i1 [[CMP1]], [[CMP2]] -; CHECK-NEXT: ret i1 [[CMP]] +; CHECK-NEXT: ret i1 [[CMP1]] ; entry: %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %in)] diff --git a/llvm/test/Transforms/GVN/gc_relocate.ll b/llvm/test/Transforms/GVN/gc_relocate.ll index 53b9e5f300fc..bd279bdf42bb 100644 --- a/llvm/test/Transforms/GVN/gc_relocate.ll +++ b/llvm/test/Transforms/GVN/gc_relocate.ll @@ -30,11 +30,8 @@ define i1 @test_readnone(i32 addrspace(1)* %in) gc "statepoint-example" { ; CHECK-NEXT: [[SAFEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[IN:%.*]]) ] ; CHECK-NEXT: [[A:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: store i32 0, i32* @G, align 4 -; CHECK-NEXT: [[B:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i32 addrspace(1)* [[A]], null -; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 addrspace(1)* [[B]], null -; CHECK-NEXT: [[CMP:%.*]] = and i1 [[CMP1]], [[CMP2]] -; CHECK-NEXT: ret i1 [[CMP]] +; CHECK-NEXT: ret i1 [[CMP1]] ; entry: %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %in)] @@ -52,14 +49,10 @@ define i1 @test_call(i32 addrspace(1)* %in) gc "statepoint-example" { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SAFEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[IN:%.*]], i32 addrspace(1)* [[IN]]) ] ; CHECK-NEXT: [[BASE:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 0) -; CHECK-NEXT: [[DERIVED:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN]], i32 0, i32 1) -; CHECK-NEXT: [[SAFEPOINT_TOKEN2:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[BASE]], i32 addrspace(1)* [[DERIVED]]) ] +; CHECK-NEXT: [[SAFEPOINT_TOKEN2:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i32 addrspace(1)* [[BASE]], i32 addrspace(1)* [[BASE]]) ] ; CHECK-NEXT: [[BASE_RELOC:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN2]], i32 0, i32 0) -; CHECK-NEXT: [[DERIVED_RELOC:%.*]] = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token [[SAFEPOINT_TOKEN2]], i32 0, i32 1) ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i32 addrspace(1)* [[BASE_RELOC]], null -; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i32 addrspace(1)* [[DERIVED_RELOC]], null -; CHECK-NEXT: [[CMP:%.*]] = and i1 [[CMP1]], [[CMP2]] -; CHECK-NEXT: ret i1 [[CMP]] +; CHECK-NEXT: ret i1 [[CMP1]] ; entry: %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @func, i32 0, i32 0, i32 0, i32 0) ["gc-live"(i32 addrspace(1)* %in, i32 addrspace(1)* %in)] -- GitLab From 56349e8b6d85621d4d95efe27716b3f6974d4324 Mon Sep 17 00:00:00 2001 From: Maksym Wezdecki Date: Tue, 16 Mar 2021 10:58:30 -0700 Subject: [PATCH 0060/1206] Fix for memory leak reported by Valgrind If llvm so lib is dlopened and dlclosed several times, then memory leak can be observed, reported by Valgrind. This patch fixes the issue. Reviewed By: lattner, dblaikie Differential Revision: https://reviews.llvm.org/D83372 --- llvm/lib/Support/ManagedStatic.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Support/ManagedStatic.cpp b/llvm/lib/Support/ManagedStatic.cpp index 053493f72fb5..a6ae67066ea0 100644 --- a/llvm/lib/Support/ManagedStatic.cpp +++ b/llvm/lib/Support/ManagedStatic.cpp @@ -18,16 +18,10 @@ using namespace llvm; static const ManagedStaticBase *StaticList = nullptr; -static std::recursive_mutex *ManagedStaticMutex = nullptr; -static llvm::once_flag mutex_init_flag; - -static void initializeMutex() { - ManagedStaticMutex = new std::recursive_mutex(); -} static std::recursive_mutex *getManagedStaticMutex() { - llvm::call_once(mutex_init_flag, initializeMutex); - return ManagedStaticMutex; + static std::recursive_mutex m; + return &m; } void ManagedStaticBase::RegisterManagedStatic(void *(*Creator)(), @@ -75,9 +69,10 @@ void ManagedStaticBase::destroy() const { } /// llvm_shutdown - Deallocate and destroy all ManagedStatic variables. +/// IMPORTANT: it's only safe to call llvm_shutdown() in single thread, +/// without any other threads executing LLVM APIs. +/// llvm_shutdown() should be the last use of LLVM APIs. void llvm::llvm_shutdown() { - std::lock_guard Lock(*getManagedStaticMutex()); - while (StaticList) StaticList->destroy(); } -- GitLab From b743bbc50586151514cd9f7f6487ad4d9838aded Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Tue, 9 Mar 2021 15:37:04 -0800 Subject: [PATCH 0061/1206] Add ConstantDataVector::getRaw() to create a constant data vector from raw data. This parallels ConstantDataArray::getRaw() and can be used with ConstantDataSequential::getRawDataValues() in the base class for both types. Update BuildConstantData{Array,Vector} tests to test the getRaw API. Also removes its unused Module. In passing, update some comments to include the support for half and bfloat. Update tests to include testing for bfloat. Differential Revision: https://reviews.llvm.org/D98302 --- llvm/include/llvm/IR/Constants.h | 28 +++++++++++++----- llvm/unittests/IR/ConstantsTest.cpp | 46 ++++++++++++++++++----------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index 510163abe6eb..223e47aa84e7 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -558,10 +558,10 @@ public: //===----------------------------------------------------------------------===// /// ConstantDataSequential - A vector or array constant whose element type is a -/// simple 1/2/4/8-byte integer or float/double, and whose elements are just -/// simple data values (i.e. ConstantInt/ConstantFP). This Constant node has no -/// operands because it stores all of the elements of the constant as densely -/// packed data, instead of as Value*'s. +/// simple 1/2/4/8-byte integer or half/bfloat/float/double, and whose elements +/// are just simple data values (i.e. ConstantInt/ConstantFP). This Constant +/// node has no operands because it stores all of the elements of the constant +/// as densely packed data, instead of as Value*'s. /// /// This is the common base class of ConstantDataArray and ConstantDataVector. /// @@ -700,11 +700,11 @@ public: return ConstantDataArray::get(Context, makeArrayRef(Elts)); } - /// get() constructor - Return a constant with array type with an element + /// getRaw() constructor - Return a constant with array type with an element /// count and element type matching the NumElements and ElementTy parameters /// passed in. Note that this can return a ConstantAggregateZero object. - /// ElementTy needs to be one of i8/i16/i32/i64/float/double. Data is the - /// buffer containing the elements. Be careful to make sure Data uses the + /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is + /// the buffer containing the elements. Be careful to make sure Data uses the /// right endianness, the buffer will be used as-is. static Constant *getRaw(StringRef Data, uint64_t NumElements, Type *ElementTy) { @@ -772,6 +772,18 @@ public: static Constant *get(LLVMContext &Context, ArrayRef Elts); static Constant *get(LLVMContext &Context, ArrayRef Elts); + /// getRaw() constructor - Return a constant with vector type with an element + /// count and element type matching the NumElements and ElementTy parameters + /// passed in. Note that this can return a ConstantAggregateZero object. + /// ElementTy must be one of i8/i16/i32/i64/half/bfloat/float/double. Data is + /// the buffer containing the elements. Be careful to make sure Data uses the + /// right endianness, the buffer will be used as-is. + static Constant *getRaw(StringRef Data, uint64_t NumElements, + Type *ElementTy) { + Type *Ty = VectorType::get(ElementTy, ElementCount::getFixed(NumElements)); + return getImpl(Data, Ty); + } + /// getFP() constructors - Return a constant of vector type with a float /// element type taken from argument `ElementType', and count taken from /// argument `Elts'. The amount of bits of the contained type must match the @@ -784,7 +796,7 @@ public: /// Return a ConstantVector with the specified constant in each element. /// The specified constant has to be a of a compatible type (i8/i16/ - /// i32/i64/float/double) and must be a ConstantFP or ConstantInt. + /// i32/i64/half/bfloat/float/double) and must be a ConstantFP or ConstantInt. static Constant *getSplat(unsigned NumElts, Constant *Elt); /// Returns true if this is a splat constant, meaning that all elements have diff --git a/llvm/unittests/IR/ConstantsTest.cpp b/llvm/unittests/IR/ConstantsTest.cpp index 44dbb90758ad..50eb3e0df1f5 100644 --- a/llvm/unittests/IR/ConstantsTest.cpp +++ b/llvm/unittests/IR/ConstantsTest.cpp @@ -418,45 +418,55 @@ static std::string getNameOfType(Type *T) { TEST(ConstantsTest, BuildConstantDataArrays) { LLVMContext Context; - std::unique_ptr M(new Module("MyModule", Context)); for (Type *T : {Type::getInt8Ty(Context), Type::getInt16Ty(Context), Type::getInt32Ty(Context), Type::getInt64Ty(Context)}) { ArrayType *ArrayTy = ArrayType::get(T, 2); Constant *Vals[] = {ConstantInt::get(T, 0), ConstantInt::get(T, 1)}; - Constant *CDV = ConstantArray::get(ArrayTy, Vals); - ASSERT_TRUE(dyn_cast(CDV) != nullptr) - << " T = " << getNameOfType(T); + Constant *CA = ConstantArray::get(ArrayTy, Vals); + ASSERT_TRUE(isa(CA)) << " T = " << getNameOfType(T); + auto *CDA = cast(CA); + Constant *CA2 = ConstantDataArray::getRaw( + CDA->getRawDataValues(), CDA->getNumElements(), CDA->getElementType()); + ASSERT_TRUE(CA == CA2) << " T = " << getNameOfType(T); } - for (Type *T : {Type::getHalfTy(Context), Type::getFloatTy(Context), - Type::getDoubleTy(Context)}) { + for (Type *T : {Type::getHalfTy(Context), Type::getBFloatTy(Context), + Type::getFloatTy(Context), Type::getDoubleTy(Context)}) { ArrayType *ArrayTy = ArrayType::get(T, 2); Constant *Vals[] = {ConstantFP::get(T, 0), ConstantFP::get(T, 1)}; - Constant *CDV = ConstantArray::get(ArrayTy, Vals); - ASSERT_TRUE(dyn_cast(CDV) != nullptr) - << " T = " << getNameOfType(T); + Constant *CA = ConstantArray::get(ArrayTy, Vals); + ASSERT_TRUE(isa(CA)) << " T = " << getNameOfType(T); + auto *CDA = cast(CA); + Constant *CA2 = ConstantDataArray::getRaw( + CDA->getRawDataValues(), CDA->getNumElements(), CDA->getElementType()); + ASSERT_TRUE(CA == CA2) << " T = " << getNameOfType(T); } } TEST(ConstantsTest, BuildConstantDataVectors) { LLVMContext Context; - std::unique_ptr M(new Module("MyModule", Context)); for (Type *T : {Type::getInt8Ty(Context), Type::getInt16Ty(Context), Type::getInt32Ty(Context), Type::getInt64Ty(Context)}) { Constant *Vals[] = {ConstantInt::get(T, 0), ConstantInt::get(T, 1)}; - Constant *CDV = ConstantVector::get(Vals); - ASSERT_TRUE(dyn_cast(CDV) != nullptr) - << " T = " << getNameOfType(T); + Constant *CV = ConstantVector::get(Vals); + ASSERT_TRUE(isa(CV)) << " T = " << getNameOfType(T); + auto *CDV = cast(CV); + Constant *CV2 = ConstantDataVector::getRaw( + CDV->getRawDataValues(), CDV->getNumElements(), CDV->getElementType()); + ASSERT_TRUE(CV == CV2) << " T = " << getNameOfType(T); } - for (Type *T : {Type::getHalfTy(Context), Type::getFloatTy(Context), - Type::getDoubleTy(Context)}) { + for (Type *T : {Type::getHalfTy(Context), Type::getBFloatTy(Context), + Type::getFloatTy(Context), Type::getDoubleTy(Context)}) { Constant *Vals[] = {ConstantFP::get(T, 0), ConstantFP::get(T, 1)}; - Constant *CDV = ConstantVector::get(Vals); - ASSERT_TRUE(dyn_cast(CDV) != nullptr) - << " T = " << getNameOfType(T); + Constant *CV = ConstantVector::get(Vals); + ASSERT_TRUE(isa(CV)) << " T = " << getNameOfType(T); + auto *CDV = cast(CV); + Constant *CV2 = ConstantDataVector::getRaw( + CDV->getRawDataValues(), CDV->getNumElements(), CDV->getElementType()); + ASSERT_TRUE(CV == CV2) << " T = " << getNameOfType(T); } } -- GitLab From 0aa637b2037d882ddf7861284169abf63f524677 Mon Sep 17 00:00:00 2001 From: Arthur O'Dwyer Date: Fri, 5 Mar 2021 20:13:35 -0500 Subject: [PATCH 0062/1206] [libc++] Improve src/filesystem's formatting of paths. This is my attempt to merge D98077 (bugfix the format strings for Windows paths, which use wchar_t not char) and D96986 (replace C++ variadic templates with C-style varargs so that `__attribute__((format(printf)))` can be applied, for better safety) and D98065 (remove an unused function overload). The one intentional functional change here is in `__create_what`. It now prints path1 and path2 in square-brackets _and_ double-quotes, rather than just square-brackets. Prior to this patch, it would print either path double-quoted if-and-only-if it was the empty string. Now the double-quotes are always present. I doubt anybody's code is relying on the current format, right? Differential Revision: https://reviews.llvm.org/D98097 --- libcxx/include/__config | 7 + libcxx/src/filesystem/directory_iterator.cpp | 7 +- libcxx/src/filesystem/filesystem_common.h | 146 ++++++++++--------- libcxx/src/filesystem/operations.cpp | 25 ++-- libcxx/test/support/filesystem_test_helper.h | 4 +- 5 files changed, 101 insertions(+), 88 deletions(-) diff --git a/libcxx/include/__config b/libcxx/include/__config index f2874e6d3f65..f4dce078e2c5 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1458,6 +1458,13 @@ extern "C" _LIBCPP_FUNC_VIS void __sanitizer_annotate_contiguous_container( # define _LIBCPP_INIT_PRIORITY_MAX #endif +#if defined(__GNUC__) || defined(__clang__) +#define _LIBCPP_FORMAT_PRINTF(a, b) \ + __attribute__((__format__(__printf__, a, b))) +#else +#define _LIBCPP_FORMAT_PRINTF(a, b) +#endif + #endif // __cplusplus #endif // _LIBCPP_CONFIG diff --git a/libcxx/src/filesystem/directory_iterator.cpp b/libcxx/src/filesystem/directory_iterator.cpp index bb3653076bfc..7b83ba9ff123 100644 --- a/libcxx/src/filesystem/directory_iterator.cpp +++ b/libcxx/src/filesystem/directory_iterator.cpp @@ -273,7 +273,7 @@ directory_iterator& directory_iterator::__increment(error_code* ec) { path root = move(__imp_->__root_); __imp_.reset(); if (m_ec) - err.report(m_ec, "at root \"%s\"", root); + err.report(m_ec, "at root " PATH_CSTR_FMT, root.c_str()); } return *this; } @@ -360,7 +360,7 @@ void recursive_directory_iterator::__advance(error_code* ec) { if (m_ec) { path root = move(stack.top().__root_); __imp_.reset(); - err.report(m_ec, "at root \"%s\"", root); + err.report(m_ec, "at root " PATH_CSTR_FMT, root.c_str()); } else { __imp_.reset(); } @@ -405,7 +405,8 @@ bool recursive_directory_iterator::__try_recursion(error_code* ec) { } else { path at_ent = move(curr_it.__entry_.__p_); __imp_.reset(); - err.report(m_ec, "attempting recursion into \"%s\"", at_ent); + err.report(m_ec, "attempting recursion into " PATH_CSTR_FMT, + at_ent.c_str()); } } return false; diff --git a/libcxx/src/filesystem/filesystem_common.h b/libcxx/src/filesystem/filesystem_common.h index 22bf8404860a..c2214d02fb80 100644 --- a/libcxx/src/filesystem/filesystem_common.h +++ b/libcxx/src/filesystem/filesystem_common.h @@ -42,8 +42,10 @@ #if defined(_LIBCPP_WIN32API) #define PS(x) (L##x) +#define PATH_CSTR_FMT "\"%ls\"" #else #define PS(x) (x) +#define PATH_CSTR_FMT "\"%s\"" #endif _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM @@ -57,68 +59,47 @@ errc __win_err_to_errc(int err); namespace { -static string format_string_imp(const char* msg, ...) { - // we might need a second shot at this, so pre-emptivly make a copy - struct GuardVAList { - va_list& target; - bool active = true; - GuardVAList(va_list& tgt) : target(tgt), active(true) {} - void clear() { - if (active) - va_end(target); - active = false; - } - ~GuardVAList() { - if (active) - va_end(target); - } - }; - va_list args; - va_start(args, msg); - GuardVAList args_guard(args); - - va_list args_cp; - va_copy(args_cp, args); - GuardVAList args_copy_guard(args_cp); - - std::string result; - - array local_buff; - size_t size_with_null = local_buff.size(); - auto ret = ::vsnprintf(local_buff.data(), size_with_null, msg, args_cp); - - args_copy_guard.clear(); - - // handle empty expansion - if (ret == 0) - return result; - if (static_cast(ret) < size_with_null) { - result.assign(local_buff.data(), static_cast(ret)); - return result; +static _LIBCPP_FORMAT_PRINTF(1, 0) string +format_string_impl(const char* msg, va_list ap) { + array buf; + + va_list apcopy; + va_copy(apcopy, ap); + int ret = ::vsnprintf(buf.data(), buf.size(), msg, apcopy); + va_end(apcopy); + + string result; + if (static_cast(ret) < buf.size()) { + result.assign(buf.data(), static_cast(ret)); + } else { + // we did not provide a long enough buffer on our first attempt. The + // return value is the number of bytes (excluding the null byte) that are + // needed for formatting. + size_t size_with_null = static_cast(ret) + 1; + result.__resize_default_init(size_with_null - 1); + ret = ::vsnprintf(&result[0], size_with_null, msg, ap); + _LIBCPP_ASSERT(static_cast(ret) == (size_with_null - 1), "TODO"); } - - // we did not provide a long enough buffer on our first attempt. The - // return value is the number of bytes (excluding the null byte) that are - // needed for formatting. - size_with_null = static_cast(ret) + 1; - result.__resize_default_init(size_with_null - 1); - ret = ::vsnprintf(&result[0], size_with_null, msg, args); - _LIBCPP_ASSERT(static_cast(ret) == (size_with_null - 1), "TODO"); - return result; } -const path::value_type* unwrap(path::string_type const& s) { return s.c_str(); } -const path::value_type* unwrap(path const& p) { return p.native().c_str(); } -template -Arg const& unwrap(Arg const& a) { - static_assert(!is_class::value, "cannot pass class here"); - return a; -} - -template -string format_string(const char* fmt, Args const&... args) { - return format_string_imp(fmt, unwrap(args)...); +static _LIBCPP_FORMAT_PRINTF(1, 2) string +format_string(const char* msg, ...) { + string ret; + va_list ap; + va_start(ap, msg); +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif // _LIBCPP_NO_EXCEPTIONS + ret = format_string_impl(msg, ap); +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + va_end(ap); + throw; + } +#endif // _LIBCPP_NO_EXCEPTIONS + va_end(ap); + return ret; } error_code capture_errno() { @@ -190,14 +171,14 @@ struct ErrorHandler { _LIBCPP_UNREACHABLE(); } - template - T report(const error_code& ec, const char* msg, Args const&... args) const { + _LIBCPP_FORMAT_PRINTF(3, 0) + void report_impl(const error_code& ec, const char* msg, va_list ap) const { if (ec_) { *ec_ = ec; - return error_value(); + return; } string what = - string("in ") + func_name_ + ": " + format_string(msg, args...); + string("in ") + func_name_ + ": " + format_string_impl(msg, ap); switch (bool(p1_) + bool(p2_)) { case 0: __throw_filesystem_error(what, ec); @@ -209,11 +190,44 @@ struct ErrorHandler { _LIBCPP_UNREACHABLE(); } - T report(errc const& err) const { return report(make_error_code(err)); } + _LIBCPP_FORMAT_PRINTF(3, 4) + T report(const error_code& ec, const char* msg, ...) const { + va_list ap; + va_start(ap, msg); +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif // _LIBCPP_NO_EXCEPTIONS + report_impl(ec, msg, ap); +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + va_end(ap); + throw; + } +#endif // _LIBCPP_NO_EXCEPTIONS + va_end(ap); + return error_value(); + } + + T report(errc const& err) const { + return report(make_error_code(err)); + } - template - T report(errc const& err, const char* msg, Args const&... args) const { - return report(make_error_code(err), msg, args...); + _LIBCPP_FORMAT_PRINTF(3, 4) + T report(errc const& err, const char* msg, ...) const { + va_list ap; + va_start(ap, msg); +#ifndef _LIBCPP_NO_EXCEPTIONS + try { +#endif // _LIBCPP_NO_EXCEPTIONS + report_impl(make_error_code(err), msg, ap); +#ifndef _LIBCPP_NO_EXCEPTIONS + } catch (...) { + va_end(ap); + throw; + } +#endif // _LIBCPP_NO_EXCEPTIONS + va_end(ap); + return error_value(); } private: diff --git a/libcxx/src/filesystem/operations.cpp b/libcxx/src/filesystem/operations.cpp index a002d0a5c93e..e604cc6c57c0 100644 --- a/libcxx/src/filesystem/operations.cpp +++ b/libcxx/src/filesystem/operations.cpp @@ -667,27 +667,20 @@ _FilesystemClock::time_point _FilesystemClock::now() noexcept { filesystem_error::~filesystem_error() {} -#if defined(_LIBCPP_WIN32API) -#define PS_FMT "%ls" -#else -#define PS_FMT "%s" -#endif - void filesystem_error::__create_what(int __num_paths) { const char* derived_what = system_error::what(); __storage_->__what_ = [&]() -> string { - const path::value_type* p1 = path1().native().empty() ? PS("\"\"") : path1().c_str(); - const path::value_type* p2 = path2().native().empty() ? PS("\"\"") : path2().c_str(); switch (__num_paths) { - default: + case 0: return detail::format_string("filesystem error: %s", derived_what); case 1: - return detail::format_string("filesystem error: %s [" PS_FMT "]", derived_what, - p1); + return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "]", + derived_what, path1().c_str()); case 2: - return detail::format_string("filesystem error: %s [" PS_FMT "] [" PS_FMT "]", - derived_what, p1, p2); + return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "] [" PATH_CSTR_FMT "]", + derived_what, path1().c_str(), path2().c_str()); } + _LIBCPP_UNREACHABLE(); }(); } @@ -1455,11 +1448,11 @@ path __temp_directory_path(error_code* ec) { error_code m_ec; file_status st = detail::posix_stat(p, &m_ec); if (!status_known(st)) - return err.report(m_ec, "cannot access path \"" PS_FMT "\"", p); + return err.report(m_ec, "cannot access path " PATH_CSTR_FMT, p.c_str()); if (!exists(st) || !is_directory(st)) - return err.report(errc::not_a_directory, "path \"" PS_FMT "\" is not a directory", - p); + return err.report(errc::not_a_directory, + "path " PATH_CSTR_FMT " is not a directory", p.c_str()); return p; } diff --git a/libcxx/test/support/filesystem_test_helper.h b/libcxx/test/support/filesystem_test_helper.h index 32d2b6ab6a86..e1607fd61899 100644 --- a/libcxx/test/support/filesystem_test_helper.h +++ b/libcxx/test/support/filesystem_test_helper.h @@ -634,9 +634,7 @@ struct ExceptionChecker { additional_msg = opt_message + ": "; } auto transform_path = [](const fs::path& p) { - if (p.native().empty()) - return std::string("\"\""); - return p.string(); + return "\"" + p.string() + "\""; }; std::string format = [&]() -> std::string { switch (num_paths) { -- GitLab From d40b4911bd9aca0573752e065f29ddd9aff280e1 Mon Sep 17 00:00:00 2001 From: Mircea Trofin Date: Mon, 8 Mar 2021 20:55:53 -0800 Subject: [PATCH 0063/1206] [regalloc] Ensure Query::collectInterferringVregs is called before interval iteration The main part of the patch is the change in RegAllocGreedy.cpp: Q.collectInterferringVregs() needs to be called before iterating the interfering live ranges. The rest of the patch offers support that is the case: instead of clearing the query's InterferingVRegs field, we invalidate it. The clearing happens when the live reg matrix is invalidated (existing triggering mechanism). Without the change in RegAllocGreedy.cpp, the compiler ices. This patch should make it more easily discoverable by developers that collectInterferringVregs needs to be called before iterating. I will follow up with a subsequent patch to improve the usability and maintainability of Query. Differential Revision: https://reviews.llvm.org/D98232 --- llvm/include/llvm/CodeGen/LiveIntervalUnion.h | 20 ++++----- llvm/lib/CodeGen/LiveIntervalUnion.cpp | 19 +++++---- llvm/lib/CodeGen/LiveRegMatrix.cpp | 16 ++++++- llvm/lib/CodeGen/RegAllocGreedy.cpp | 42 +++++++------------ 4 files changed, 50 insertions(+), 47 deletions(-) diff --git a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h index ad9e06d2bcf0..4ebe0f2dcfd8 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h @@ -114,30 +114,30 @@ public: const LiveRange *LR = nullptr; LiveRange::const_iterator LRI; ///< current position in LR ConstSegmentIter LiveUnionI; ///< current position in LiveUnion - SmallVector InterferingVRegs; + Optional> InterferingVRegs; bool CheckedFirstInterference = false; bool SeenAllInterferences = false; unsigned Tag = 0; unsigned UserTag = 0; + public: + Query() = default; + Query(const LiveRange &LR, const LiveIntervalUnion &LIU) + : LiveUnion(&LIU), LR(&LR) {} + Query(const Query &) = delete; + Query &operator=(const Query &) = delete; + void reset(unsigned NewUserTag, const LiveRange &NewLR, const LiveIntervalUnion &NewLiveUnion) { LiveUnion = &NewLiveUnion; LR = &NewLR; - InterferingVRegs.clear(); + InterferingVRegs = None; CheckedFirstInterference = false; SeenAllInterferences = false; Tag = NewLiveUnion.getTag(); UserTag = NewUserTag; } - public: - Query() = default; - Query(const LiveRange &LR, const LiveIntervalUnion &LIU): - LiveUnion(&LIU), LR(&LR) {} - Query(const Query &) = delete; - Query &operator=(const Query &) = delete; - void init(unsigned NewUserTag, const LiveRange &NewLR, const LiveIntervalUnion &NewLiveUnion) { if (UserTag == NewUserTag && LR == &NewLR && LiveUnion == &NewLiveUnion && @@ -164,7 +164,7 @@ public: // Vector generated by collectInterferingVRegs. const SmallVectorImpl &interferingVRegs() const { - return InterferingVRegs; + return *InterferingVRegs; } }; diff --git a/llvm/lib/CodeGen/LiveIntervalUnion.cpp b/llvm/lib/CodeGen/LiveIntervalUnion.cpp index 7ccb8df4bc05..dfa523d4bf41 100644 --- a/llvm/lib/CodeGen/LiveIntervalUnion.cpp +++ b/llvm/lib/CodeGen/LiveIntervalUnion.cpp @@ -112,7 +112,7 @@ LiveInterval *LiveIntervalUnion::getOneVReg() const { // Scan the vector of interfering virtual registers in this union. Assume it's // quite small. bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const { - return is_contained(InterferingVRegs, VirtReg); + return is_contained(*InterferingVRegs, VirtReg); } // Collect virtual registers in this union that interfere with this @@ -126,9 +126,12 @@ bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const { // unsigned LiveIntervalUnion::Query:: collectInterferingVRegs(unsigned MaxInterferingRegs) { + if (!InterferingVRegs) + InterferingVRegs.emplace(); + // Fast path return if we already have the desired information. - if (SeenAllInterferences || InterferingVRegs.size() >= MaxInterferingRegs) - return InterferingVRegs.size(); + if (SeenAllInterferences || InterferingVRegs->size() >= MaxInterferingRegs) + return InterferingVRegs->size(); // Set up iterators on the first call. if (!CheckedFirstInterference) { @@ -157,14 +160,14 @@ collectInterferingVRegs(unsigned MaxInterferingRegs) { LiveInterval *VReg = LiveUnionI.value(); if (VReg != RecentReg && !isSeenInterference(VReg)) { RecentReg = VReg; - InterferingVRegs.push_back(VReg); - if (InterferingVRegs.size() >= MaxInterferingRegs) - return InterferingVRegs.size(); + InterferingVRegs->push_back(VReg); + if (InterferingVRegs->size() >= MaxInterferingRegs) + return InterferingVRegs->size(); } // This LiveUnion segment is no longer interesting. if (!(++LiveUnionI).valid()) { SeenAllInterferences = true; - return InterferingVRegs.size(); + return InterferingVRegs->size(); } } @@ -185,7 +188,7 @@ collectInterferingVRegs(unsigned MaxInterferingRegs) { LiveUnionI.advanceTo(LRI->start); } SeenAllInterferences = true; - return InterferingVRegs.size(); + return InterferingVRegs->size(); } void LiveIntervalUnion::Array::init(LiveIntervalUnion::Allocator &Alloc, diff --git a/llvm/lib/CodeGen/LiveRegMatrix.cpp b/llvm/lib/CodeGen/LiveRegMatrix.cpp index a69aa6557e46..4c0172a930b5 100644 --- a/llvm/lib/CodeGen/LiveRegMatrix.cpp +++ b/llvm/lib/CodeGen/LiveRegMatrix.cpp @@ -216,7 +216,21 @@ bool LiveRegMatrix::checkInterference(SlotIndex Start, SlotIndex End, // Check for interference with that segment for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { - if (query(LR, *Units).checkInterference()) + // LR is stack-allocated. LiveRegMatrix caches queries by a key that + // includes the address of the live range. If (for the same reg unit) this + // checkInterference overload is called twice, without any other query() + // calls in between (on heap-allocated LiveRanges) - which would invalidate + // the cached query - the LR address seen the second time may well be the + // same as that seen the first time, while the Start/End/valno may not - yet + // the same cached result would be fetched. To avoid that, we don't cache + // this query. + // + // FIXME: the usability of the Query API needs to be improved to avoid + // subtle bugs due to query identity. Avoiding caching, for example, would + // greatly simplify things. + LiveIntervalUnion::Query Q; + Q.reset(UserTag, LR, Matrix[*Units]); + if (Q.checkInterference()) return true; } return false; diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp index d98a3c1bad9c..f56b5ed1bf6a 100644 --- a/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -471,12 +471,13 @@ private: bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool) const; bool canEvictInterference(LiveInterval &, MCRegister, bool, EvictionCost &, const SmallVirtRegSet &) const; - bool canEvictInterferenceInRange(LiveInterval &VirtReg, MCRegister PhysReg, - SlotIndex Start, SlotIndex End, - EvictionCost &MaxCost) const; + bool canEvictInterferenceInRange(const LiveInterval &VirtReg, + MCRegister PhysReg, SlotIndex Start, + SlotIndex End, EvictionCost &MaxCost) const; MCRegister getCheapestEvicteeWeight(const AllocationOrder &Order, - LiveInterval &VirtReg, SlotIndex Start, - SlotIndex End, float *BestEvictWeight); + const LiveInterval &VirtReg, + SlotIndex Start, SlotIndex End, + float *BestEvictWeight) const; void evictInterference(LiveInterval &, MCRegister, SmallVectorImpl &); bool mayRecolorAllInterferences(MCRegister PhysReg, LiveInterval &VirtReg, @@ -979,7 +980,7 @@ bool RAGreedy::canEvictInterference( /// \param MaxCost Only look for cheaper candidates and update with new cost /// when returning true. /// \return True when interference can be evicted cheaper than MaxCost. -bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, +bool RAGreedy::canEvictInterferenceInRange(const LiveInterval &VirtReg, MCRegister PhysReg, SlotIndex Start, SlotIndex End, EvictionCost &MaxCost) const { @@ -987,6 +988,7 @@ bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); + Q.collectInterferingVRegs(); // Check if any interfering live range is heavier than MaxWeight. for (const LiveInterval *Intf : reverse(Q.interferingVRegs())) { @@ -1031,9 +1033,9 @@ bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, /// \return The PhysReg which is the best candidate for eviction and the /// eviction cost in BestEvictweight MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, - LiveInterval &VirtReg, + const LiveInterval &VirtReg, SlotIndex Start, SlotIndex End, - float *BestEvictweight) { + float *BestEvictweight) const { EvictionCost BestEvictCost; BestEvictCost.setMax(); BestEvictCost.MaxWeight = VirtReg.weight(); @@ -1058,7 +1060,7 @@ MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, /// returned true. void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg, SmallVectorImpl &NewVRegs) { - // Make sure that VirtReg has a cascade number, and assign that cascade + // Make sure th5at VirtReg has a cascade number, and assign that cascade // number to every evicted register. These live ranges than then only be // evicted by a newer cascade, preventing infinite loops. unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade; @@ -1556,25 +1558,9 @@ bool RAGreedy::splitCanCauseLocalSpill(unsigned VirtRegToSplit, return false; } - // Check if the local interval will evict a cheaper interval. - float CheapestEvictWeight = 0; - MCRegister FutureEvictedPhysReg = getCheapestEvicteeWeight( - Order, LIS->getInterval(VirtRegToSplit), Cand.Intf.first(), - Cand.Intf.last(), &CheapestEvictWeight); - - // Have we found an interval that can be evicted? - if (FutureEvictedPhysReg) { - float splitArtifactWeight = - VRAI->futureWeight(LIS->getInterval(VirtRegToSplit), - Cand.Intf.first().getPrevIndex(), Cand.Intf.last()); - // Will the weight of the local interval be higher than the cheapest evictee - // weight? If so it will evict it and will not cause a spill. - if (splitArtifactWeight >= 0 && splitArtifactWeight > CheapestEvictWeight) - return false; - } - - // The local interval is not able to find non interferencing assignment and - // not able to evict a less worthy interval, therfore, it can cause a spill. + // The local interval is not able to find non interferencing assignment + // and not able to evict a less worthy interval, therfore, it can cause a + // spill. return true; } -- GitLab From edf9565a8665fcaf7a452f07f77fb08c484f45d7 Mon Sep 17 00:00:00 2001 From: Liam Keegan Date: Tue, 16 Mar 2021 20:30:00 +0100 Subject: [PATCH 0064/1206] [MemCpyOpt] Add missing MemorySSAWrapperPass dependency macro Add MemorySSAWrapperPass as a dependency to MemCpyOptLegacyPass, since MemCpyOpt now uses MemorySSA by default. Differential Revision: https://reviews.llvm.org/D98484 --- llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp index 5c1c6b2a2d8f..f98a06490d98 100644 --- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp +++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp @@ -308,6 +308,7 @@ INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass) +INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass) INITIALIZE_PASS_END(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization", false, false) -- GitLab From 40bc309911f0f92ff8b8f64d28cb13a2292695ff Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 16 Mar 2021 20:41:26 +0100 Subject: [PATCH 0065/1206] Revert "[regalloc] Ensure Query::collectInterferringVregs is called before interval iteration" This reverts commit d40b4911bd9aca0573752e065f29ddd9aff280e1. This causes a large compile-time regression: https://llvm-compile-time-tracker.com/compare.php?from=0aa637b2037d882ddf7861284169abf63f524677&to=d40b4911bd9aca0573752e065f29ddd9aff280e1&stat=instructions --- llvm/include/llvm/CodeGen/LiveIntervalUnion.h | 20 ++++----- llvm/lib/CodeGen/LiveIntervalUnion.cpp | 19 ++++----- llvm/lib/CodeGen/LiveRegMatrix.cpp | 16 +------ llvm/lib/CodeGen/RegAllocGreedy.cpp | 42 ++++++++++++------- 4 files changed, 47 insertions(+), 50 deletions(-) diff --git a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h index 4ebe0f2dcfd8..ad9e06d2bcf0 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h @@ -114,30 +114,30 @@ public: const LiveRange *LR = nullptr; LiveRange::const_iterator LRI; ///< current position in LR ConstSegmentIter LiveUnionI; ///< current position in LiveUnion - Optional> InterferingVRegs; + SmallVector InterferingVRegs; bool CheckedFirstInterference = false; bool SeenAllInterferences = false; unsigned Tag = 0; unsigned UserTag = 0; - public: - Query() = default; - Query(const LiveRange &LR, const LiveIntervalUnion &LIU) - : LiveUnion(&LIU), LR(&LR) {} - Query(const Query &) = delete; - Query &operator=(const Query &) = delete; - void reset(unsigned NewUserTag, const LiveRange &NewLR, const LiveIntervalUnion &NewLiveUnion) { LiveUnion = &NewLiveUnion; LR = &NewLR; - InterferingVRegs = None; + InterferingVRegs.clear(); CheckedFirstInterference = false; SeenAllInterferences = false; Tag = NewLiveUnion.getTag(); UserTag = NewUserTag; } + public: + Query() = default; + Query(const LiveRange &LR, const LiveIntervalUnion &LIU): + LiveUnion(&LIU), LR(&LR) {} + Query(const Query &) = delete; + Query &operator=(const Query &) = delete; + void init(unsigned NewUserTag, const LiveRange &NewLR, const LiveIntervalUnion &NewLiveUnion) { if (UserTag == NewUserTag && LR == &NewLR && LiveUnion == &NewLiveUnion && @@ -164,7 +164,7 @@ public: // Vector generated by collectInterferingVRegs. const SmallVectorImpl &interferingVRegs() const { - return *InterferingVRegs; + return InterferingVRegs; } }; diff --git a/llvm/lib/CodeGen/LiveIntervalUnion.cpp b/llvm/lib/CodeGen/LiveIntervalUnion.cpp index dfa523d4bf41..7ccb8df4bc05 100644 --- a/llvm/lib/CodeGen/LiveIntervalUnion.cpp +++ b/llvm/lib/CodeGen/LiveIntervalUnion.cpp @@ -112,7 +112,7 @@ LiveInterval *LiveIntervalUnion::getOneVReg() const { // Scan the vector of interfering virtual registers in this union. Assume it's // quite small. bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const { - return is_contained(*InterferingVRegs, VirtReg); + return is_contained(InterferingVRegs, VirtReg); } // Collect virtual registers in this union that interfere with this @@ -126,12 +126,9 @@ bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const { // unsigned LiveIntervalUnion::Query:: collectInterferingVRegs(unsigned MaxInterferingRegs) { - if (!InterferingVRegs) - InterferingVRegs.emplace(); - // Fast path return if we already have the desired information. - if (SeenAllInterferences || InterferingVRegs->size() >= MaxInterferingRegs) - return InterferingVRegs->size(); + if (SeenAllInterferences || InterferingVRegs.size() >= MaxInterferingRegs) + return InterferingVRegs.size(); // Set up iterators on the first call. if (!CheckedFirstInterference) { @@ -160,14 +157,14 @@ collectInterferingVRegs(unsigned MaxInterferingRegs) { LiveInterval *VReg = LiveUnionI.value(); if (VReg != RecentReg && !isSeenInterference(VReg)) { RecentReg = VReg; - InterferingVRegs->push_back(VReg); - if (InterferingVRegs->size() >= MaxInterferingRegs) - return InterferingVRegs->size(); + InterferingVRegs.push_back(VReg); + if (InterferingVRegs.size() >= MaxInterferingRegs) + return InterferingVRegs.size(); } // This LiveUnion segment is no longer interesting. if (!(++LiveUnionI).valid()) { SeenAllInterferences = true; - return InterferingVRegs->size(); + return InterferingVRegs.size(); } } @@ -188,7 +185,7 @@ collectInterferingVRegs(unsigned MaxInterferingRegs) { LiveUnionI.advanceTo(LRI->start); } SeenAllInterferences = true; - return InterferingVRegs->size(); + return InterferingVRegs.size(); } void LiveIntervalUnion::Array::init(LiveIntervalUnion::Allocator &Alloc, diff --git a/llvm/lib/CodeGen/LiveRegMatrix.cpp b/llvm/lib/CodeGen/LiveRegMatrix.cpp index 4c0172a930b5..a69aa6557e46 100644 --- a/llvm/lib/CodeGen/LiveRegMatrix.cpp +++ b/llvm/lib/CodeGen/LiveRegMatrix.cpp @@ -216,21 +216,7 @@ bool LiveRegMatrix::checkInterference(SlotIndex Start, SlotIndex End, // Check for interference with that segment for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { - // LR is stack-allocated. LiveRegMatrix caches queries by a key that - // includes the address of the live range. If (for the same reg unit) this - // checkInterference overload is called twice, without any other query() - // calls in between (on heap-allocated LiveRanges) - which would invalidate - // the cached query - the LR address seen the second time may well be the - // same as that seen the first time, while the Start/End/valno may not - yet - // the same cached result would be fetched. To avoid that, we don't cache - // this query. - // - // FIXME: the usability of the Query API needs to be improved to avoid - // subtle bugs due to query identity. Avoiding caching, for example, would - // greatly simplify things. - LiveIntervalUnion::Query Q; - Q.reset(UserTag, LR, Matrix[*Units]); - if (Q.checkInterference()) + if (query(LR, *Units).checkInterference()) return true; } return false; diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp index f56b5ed1bf6a..d98a3c1bad9c 100644 --- a/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -471,13 +471,12 @@ private: bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool) const; bool canEvictInterference(LiveInterval &, MCRegister, bool, EvictionCost &, const SmallVirtRegSet &) const; - bool canEvictInterferenceInRange(const LiveInterval &VirtReg, - MCRegister PhysReg, SlotIndex Start, - SlotIndex End, EvictionCost &MaxCost) const; + bool canEvictInterferenceInRange(LiveInterval &VirtReg, MCRegister PhysReg, + SlotIndex Start, SlotIndex End, + EvictionCost &MaxCost) const; MCRegister getCheapestEvicteeWeight(const AllocationOrder &Order, - const LiveInterval &VirtReg, - SlotIndex Start, SlotIndex End, - float *BestEvictWeight) const; + LiveInterval &VirtReg, SlotIndex Start, + SlotIndex End, float *BestEvictWeight); void evictInterference(LiveInterval &, MCRegister, SmallVectorImpl &); bool mayRecolorAllInterferences(MCRegister PhysReg, LiveInterval &VirtReg, @@ -980,7 +979,7 @@ bool RAGreedy::canEvictInterference( /// \param MaxCost Only look for cheaper candidates and update with new cost /// when returning true. /// \return True when interference can be evicted cheaper than MaxCost. -bool RAGreedy::canEvictInterferenceInRange(const LiveInterval &VirtReg, +bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, MCRegister PhysReg, SlotIndex Start, SlotIndex End, EvictionCost &MaxCost) const { @@ -988,7 +987,6 @@ bool RAGreedy::canEvictInterferenceInRange(const LiveInterval &VirtReg, for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); - Q.collectInterferingVRegs(); // Check if any interfering live range is heavier than MaxWeight. for (const LiveInterval *Intf : reverse(Q.interferingVRegs())) { @@ -1033,9 +1031,9 @@ bool RAGreedy::canEvictInterferenceInRange(const LiveInterval &VirtReg, /// \return The PhysReg which is the best candidate for eviction and the /// eviction cost in BestEvictweight MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, - const LiveInterval &VirtReg, + LiveInterval &VirtReg, SlotIndex Start, SlotIndex End, - float *BestEvictweight) const { + float *BestEvictweight) { EvictionCost BestEvictCost; BestEvictCost.setMax(); BestEvictCost.MaxWeight = VirtReg.weight(); @@ -1060,7 +1058,7 @@ MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, /// returned true. void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg, SmallVectorImpl &NewVRegs) { - // Make sure th5at VirtReg has a cascade number, and assign that cascade + // Make sure that VirtReg has a cascade number, and assign that cascade // number to every evicted register. These live ranges than then only be // evicted by a newer cascade, preventing infinite loops. unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade; @@ -1558,9 +1556,25 @@ bool RAGreedy::splitCanCauseLocalSpill(unsigned VirtRegToSplit, return false; } - // The local interval is not able to find non interferencing assignment - // and not able to evict a less worthy interval, therfore, it can cause a - // spill. + // Check if the local interval will evict a cheaper interval. + float CheapestEvictWeight = 0; + MCRegister FutureEvictedPhysReg = getCheapestEvicteeWeight( + Order, LIS->getInterval(VirtRegToSplit), Cand.Intf.first(), + Cand.Intf.last(), &CheapestEvictWeight); + + // Have we found an interval that can be evicted? + if (FutureEvictedPhysReg) { + float splitArtifactWeight = + VRAI->futureWeight(LIS->getInterval(VirtRegToSplit), + Cand.Intf.first().getPrevIndex(), Cand.Intf.last()); + // Will the weight of the local interval be higher than the cheapest evictee + // weight? If so it will evict it and will not cause a spill. + if (splitArtifactWeight >= 0 && splitArtifactWeight > CheapestEvictWeight) + return false; + } + + // The local interval is not able to find non interferencing assignment and + // not able to evict a less worthy interval, therfore, it can cause a spill. return true; } -- GitLab From 5cabf472cb3c434bbf03889623c09b14f5c62f26 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Tue, 16 Mar 2021 12:47:57 -0700 Subject: [PATCH 0066/1206] [rs4gc] don't duplicate existing values which are provably base pointers RS4GC needs to rewrite the IR to ensure that every relocated pointer has an associated base pointer. The existing code isn't particularly smart about avoiding duplication of existing IR when it turns out the original pointer we were asked to materialize a base pointer for is itself a base pointer. This patch adds a stage to the algorithm which prunes nodes proven (with a simple forward dataflow fixed point) to be base pointers from the list of nodes considered for duplication. This does require changing some of the later invariants slightly, that's probably the riskiest part of the change. Differential Revision: D98122 --- .../Scalar/RewriteStatepointsForGC.cpp | 62 +++++++--- .../RewriteStatepointsForGC/base-inference.ll | 114 +++++++++--------- .../base-pointers-4.ll | 7 +- .../RewriteStatepointsForGC/base-pointers.ll | 33 ++--- .../base-vector-inseltpoison.ll | 32 ++--- .../RewriteStatepointsForGC/base-vector.ll | 32 ++--- .../live-vector-nosplit-inseltpoison.ll | 7 +- .../live-vector-nosplit.ll | 7 +- .../rematerialize-derived-pointers.ll | 20 ++- .../scalar-base-vector.ll | 5 +- 10 files changed, 145 insertions(+), 174 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 53e256f8b852..1ab4946476cc 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -900,14 +900,51 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { } #endif + // Iterate forward through the value graph pruning any node from the state + // list where all of the inputs are base pointers. The purpose of this is to + // reuse existing values when the derived pointer we were asked to materialize + // a base pointer for happens to be a base pointer itself. (Or a sub-graph + // feeding it does.) + SmallVector ToRemove; + do { + ToRemove.clear(); + for (auto Pair : States) { + Value *BDV = Pair.first; + auto canPruneInput = [&](Value *V) { + Value *BDV = findBaseOrBDV(V, Cache); + if (V->stripPointerCasts() != BDV) + return false; + // The assumption is that anything not in the state list is + // propagates a base pointer. + return States.count(BDV) == 0; + }; + + bool CanPrune = true; + visitBDVOperands(BDV, [&](Value *Op) { + CanPrune = CanPrune && canPruneInput(Op); + }); + if (CanPrune) + ToRemove.push_back(BDV); + } + for (Value *V : ToRemove) { + States.erase(V); + // Cache the fact V is it's own base for later usage. + Cache[V] = V; + } + } while (!ToRemove.empty()); + + // Did we manage to prove that Def itself must be a base pointer? + if (!States.count(Def)) + return Def; + // Return a phi state for a base defining value. We'll generate a new // base state for known bases and expect to find a cached state otherwise. auto GetStateForBDV = [&](Value *BaseValue, Value *Input) { - if (isKnownBaseResult(BaseValue) && areBothVectorOrScalar(BaseValue, Input)) - return BDVState(BaseValue, BDVState::Base, BaseValue); auto I = States.find(BaseValue); - assert(I != States.end() && "lookup failed!"); - return I->second; + if (I != States.end()) + return I->second; + assert(areBothVectorOrScalar(BaseValue, Input)); + return BDVState(BaseValue, BDVState::Base, BaseValue); }; bool Progress = true; @@ -1071,7 +1108,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { auto getBaseForInput = [&](Value *Input, Instruction *InsertPt) { Value *BDV = findBaseOrBDV(Input, Cache); Value *Base = nullptr; - if (isKnownBaseResult(BDV) && areBothVectorOrScalar(BDV, Input)) { + if (!States.count(BDV)) { + assert(areBothVectorOrScalar(BDV, Input)); Base = BDV; } else { // Either conflict or base. @@ -1203,14 +1241,6 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none") << " to: " << Base->getName() << "\n"); - if (Cache.count(BDV)) { - assert(isKnownBaseResult(Base) && - "must be something we 'know' is a base pointer"); - // Once we transition from the BDV relation being store in the Cache to - // the base relation being stored, it must be stable - assert((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) && - "base relation should be stable"); - } Cache[BDV] = Base; } assert(Cache.count(Def)); @@ -3016,11 +3046,7 @@ static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, // We may have base pointers which are now live that weren't before. We need // to update the PointerToBase structure to reflect this. for (auto V : Updated) - if (Info.PointerToBase.insert({V, V}).second) { - assert(isKnownBaseResult(V) && - "Can't find base for unexpected live value!"); - continue; - } + Info.PointerToBase.insert({V, V}); #ifndef NDEBUG for (auto V : Updated) diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll index b5177b9628c7..ddb229234b4e 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll @@ -18,11 +18,9 @@ define i8 addrspace(1)* @test(i8 addrspace(1)* %a) gc "statepoint-example" { define i8 addrspace(1)* @test_select(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2) gc "statepoint-example" { ; CHECK-LABEL: @test_select( -; CHECK-NEXT: [[SEL_BASE:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]], !is_base_value !0 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], i8 addrspace(1)* [[A1]], i8 addrspace(1)* [[A2]] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[SEL]], i8 addrspace(1)* [[SEL_BASE]]) ] -; CHECK-NEXT: [[SEL_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[SEL_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[SEL]]) ] +; CHECK-NEXT: [[SEL_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret i8 addrspace(1)* [[SEL_RELOCATED]] ; %sel = select i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2 @@ -39,11 +37,9 @@ define i8 addrspace(1)* @test_phi1(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* ; CHECK: untaken: ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[TAKEN]] ], [ [[A2:%.*]], [[UNTAKEN]] ], !is_base_value !0 -; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1]], [[TAKEN]] ], [ [[A2]], [[UNTAKEN]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ] -; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[TAKEN]] ], [ [[A2:%.*]], [[UNTAKEN]] ] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]]) ] +; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret i8 addrspace(1)* [[PHI_RELOCATED]] ; entry: @@ -81,13 +77,11 @@ define i8 addrspace(1)* @test_loop1(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1) ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[A2:%.*]], [[LOOP]] ], !is_base_value !0 -; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1]], [[ENTRY]] ], [ [[A2]], [[LOOP]] ] +; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[A2:%.*]], [[LOOP]] ] ; CHECK-NEXT: br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ] -; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]]) ] +; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret i8 addrspace(1)* [[PHI_RELOCATED]] ; entry: @@ -106,15 +100,13 @@ define i8 addrspace(1)* @test_loop2(i1 %c, i8 addrspace(1)* %a1) gc "statepoint- ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[O2:%.*]], [[LOOP]] ], !is_base_value !0 -; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1]], [[ENTRY]] ], [ [[O2]], [[LOOP]] ] +; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[A1:%.*]], [[ENTRY:%.*]] ], [ [[O2:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[ADDR:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i8 addrspace(1)* addrspace(1)* ; CHECK-NEXT: [[O2]] = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* [[ADDR]], align 8 ; CHECK-NEXT: br i1 [[C:%.*]], label [[EXIT:%.*]], label [[LOOP]] ; CHECK: exit: -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ] -; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]]) ] +; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret i8 addrspace(1)* [[PHI_RELOCATED]] ; entry: @@ -171,11 +163,9 @@ define <2 x i8 addrspace(1)*> @test_vec_passthrough(<2 x i8 addrspace(1)*> %a) g define <2 x i8 addrspace(1)*> @test_insert(i8 addrspace(1)* %a) gc "statepoint-example" { ; CHECK-LABEL: @test_insert( -; CHECK-NEXT: [[VEC_BASE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0, !is_base_value !0 -; CHECK-NEXT: [[VEC:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A]], i64 0 -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[VEC]], <2 x i8 addrspace(1)*> [[VEC_BASE]]) ] -; CHECK-NEXT: [[VEC_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[VEC_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[VEC:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0 +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[VEC]]) ] +; CHECK-NEXT: [[VEC_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret <2 x i8 addrspace(1)*> [[VEC_RELOCATED]] ; %vec = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* %a, i64 0 @@ -185,11 +175,9 @@ define <2 x i8 addrspace(1)*> @test_insert(i8 addrspace(1)* %a) gc "statepoint-e define i8 addrspace(1)* @test_extract(<2 x i8 addrspace(1)*> %a) gc "statepoint-example" { ; CHECK-LABEL: @test_extract( -; CHECK-NEXT: [[BASE_EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[A:%.*]], i64 0, !is_base_value !0 -; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[A]], i64 0 -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]], i8 addrspace(1)* [[BASE_EE]]) ] -; CHECK-NEXT: [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[A:%.*]], i64 0 +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]]) ] +; CHECK-NEXT: [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret i8 addrspace(1)* [[EE_RELOCATED]] ; %ee = extractelement <2 x i8 addrspace(1)*> %a, i64 0 @@ -200,9 +188,8 @@ define i8 addrspace(1)* @test_extract(<2 x i8 addrspace(1)*> %a) gc "statepoint- define <2 x i8 addrspace(1)*> @test_shuffle(<2 x i8 addrspace(1)*> %a1) gc "statepoint-example" { ; CHECK-LABEL: @test_shuffle( ; CHECK-NEXT: [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A1]], <2 x i32> zeroinitializer -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]], <2 x i8 addrspace(1)*> [[A1]]) ] -; CHECK-NEXT: [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[A1_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]]) ] +; CHECK-NEXT: [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret <2 x i8 addrspace(1)*> [[RES_RELOCATED]] ; %res = shufflevector <2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a1, <2 x i32> zeroinitializer @@ -212,11 +199,9 @@ define <2 x i8 addrspace(1)*> @test_shuffle(<2 x i8 addrspace(1)*> %a1) gc "stat define <2 x i8 addrspace(1)*> @test_shuffle2(<2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2) gc "statepoint-example" { ; CHECK-LABEL: @test_shuffle2( -; CHECK-NEXT: [[RES_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <2 x i32> zeroinitializer, !is_base_value !0 -; CHECK-NEXT: [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1]], <2 x i8 addrspace(1)*> [[A2]], <2 x i32> zeroinitializer -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]], <2 x i8 addrspace(1)*> [[RES_BASE]]) ] -; CHECK-NEXT: [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[RES_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <2 x i32> zeroinitializer +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[RES]]) ] +; CHECK-NEXT: [[RES_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret <2 x i8 addrspace(1)*> [[RES_RELOCATED]] ; %res = shufflevector <2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2, <2 x i32> zeroinitializer @@ -226,11 +211,9 @@ define <2 x i8 addrspace(1)*> @test_shuffle2(<2 x i8 addrspace(1)*> %a1, <2 x i8 define <4 x i8 addrspace(1)*> @test_shuffle_concat(<2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2) gc "statepoint-example" { ; CHECK-LABEL: @test_shuffle_concat( -; CHECK-NEXT: [[RES_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <4 x i32> , !is_base_value !0 -; CHECK-NEXT: [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1]], <2 x i8 addrspace(1)*> [[A2]], <4 x i32> -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<4 x i8 addrspace(1)*> [[RES]], <4 x i8 addrspace(1)*> [[RES_BASE]]) ] -; CHECK-NEXT: [[RES_RELOCATED:%.*]] = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[RES_BASE_RELOCATED:%.*]] = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[RES:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[A1:%.*]], <2 x i8 addrspace(1)*> [[A2:%.*]], <4 x i32> +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<4 x i8 addrspace(1)*> [[RES]]) ] +; CHECK-NEXT: [[RES_RELOCATED:%.*]] = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret <4 x i8 addrspace(1)*> [[RES_RELOCATED]] ; %res = shufflevector <2 x i8 addrspace(1)*> %a1, <2 x i8 addrspace(1)*> %a2, <4 x i32> @@ -238,23 +221,40 @@ define <4 x i8 addrspace(1)*> @test_shuffle_concat(<2 x i8 addrspace(1)*> %a1, < ret <4 x i8 addrspace(1)*> %res } +; TODO: Special case worth handling - we interpret the shuffle as if we need +; to select the base pointers from either input when the mask is known. +define <2 x i8 addrspace(1)*> @test_shuffle_broadcast(i8 addrspace(1)* %a) gc "statepoint-example" { +; CHECK-LABEL: @test_shuffle_broadcast( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[IE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0 +; CHECK-NEXT: [[BROADCAST_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 +; CHECK-NEXT: [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[BROADCAST]], <2 x i8 addrspace(1)*> [[BROADCAST_BASE]]) ] +; CHECK-NEXT: [[BROADCAST_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[BROADCAST_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: ret <2 x i8 addrspace(1)*> [[BROADCAST_RELOCATED]] +; +entry: + %ie = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* %a, i64 0 + %broadcast = shufflevector <2 x i8 addrspace(1)*> %ie, <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer + call void @foo() + ret <2 x i8 addrspace(1)*> %broadcast +} ; Show a case where only a portion of the sub-graph propagates base pointers. define i8 @test_subgraph(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2) gc "statepoint-example" { ; CHECK-LABEL: @test_subgraph( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SEL_BASE:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]], !is_base_value !0 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], i8 addrspace(1)* [[A1]], i8 addrspace(1)* [[A2]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]] ; CHECK-NEXT: br i1 [[C]], label [[TAKEN:%.*]], label [[MERGE:%.*]] ; CHECK: taken: ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, i8 addrspace(1)* [[SEL]], i64 8 ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[SEL_BASE]], [[TAKEN]] ], [ [[SEL_BASE]], [[ENTRY:%.*]] ], !is_base_value !0 -; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[GEP]], [[TAKEN]] ], [ [[SEL]], [[ENTRY]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[PHI_BASE]]) ] +; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[GEP]], [[TAKEN]] ], [ [[SEL]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[PHI]], i8 addrspace(1)* [[SEL]]) ] ; CHECK-NEXT: [[PHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[PHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[SEL_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) ; CHECK-NEXT: [[RES:%.*]] = load i8, i8 addrspace(1)* [[PHI_RELOCATED]], align 1 ; CHECK-NEXT: ret i8 [[RES]] ; @@ -275,24 +275,19 @@ merge: define i8 @test_subgraph2(i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2) gc "statepoint-example" { ; CHECK-LABEL: @test_subgraph2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SEL_BASE:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]], !is_base_value !0 -; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], i8 addrspace(1)* [[A1]], i8 addrspace(1)* [[A2]] -; CHECK-NEXT: [[IE_BASE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[SEL_BASE]], i64 0, !is_base_value !0 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i8 addrspace(1)* [[A1:%.*]], i8 addrspace(1)* [[A2:%.*]] ; CHECK-NEXT: [[IE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[SEL]], i64 0 -; CHECK-NEXT: [[BROADCAST_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE_BASE]], <2 x i8 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 -; CHECK-NEXT: [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer -; CHECK-NEXT: [[EE_BASE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[BROADCAST_BASE]], i32 1, !is_base_value !0 +; CHECK-NEXT: [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> [[IE]], <2 x i32> zeroinitializer ; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[BROADCAST]], i32 1 -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]], i8 addrspace(1)* [[EE_BASE]]) ] -; CHECK-NEXT: [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[EE_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(i8 addrspace(1)* [[EE]]) ] +; CHECK-NEXT: [[EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[RES:%.*]] = load i8, i8 addrspace(1)* [[EE_RELOCATED]], align 1 ; CHECK-NEXT: ret i8 [[RES]] ; entry: %sel = select i1 %c, i8 addrspace(1)* %a1, i8 addrspace(1)* %a2 %ie = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* %sel, i64 0 - %broadcast = shufflevector <2 x i8 addrspace(1)*> %ie, <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer + %broadcast = shufflevector <2 x i8 addrspace(1)*> %ie, <2 x i8 addrspace(1)*> %ie, <2 x i32> zeroinitializer %ee = extractelement <2 x i8 addrspace(1)*> %broadcast, i32 1 call void @foo() %res = load i8, i8 addrspace(1)* %ee @@ -300,5 +295,4 @@ entry: } - declare void @foo() diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll index 31a78ae86189..2b75785bf9db 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll @@ -27,12 +27,9 @@ define void @test(i32 %condition) gc "statepoint-example" { ; CHECK: dest_c: ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[OBJ_TO_CONSUME_BASE:%.*]] = phi i64 addrspace(1)* [ [[TMP0]], [[DEST_A]] ], [ null, [[DEST_B]] ], [ null, [[DEST_C]] ], !is_base_value !0 ; CHECK-NEXT: [[OBJ_TO_CONSUME:%.*]] = phi i64 addrspace(1)* [ [[TMP0]], [[DEST_A]] ], [ null, [[DEST_B]] ], [ null, [[DEST_C]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN1:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* [[OBJ_TO_CONSUME]], i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[OBJ_TO_CONSUME_BASE]], i64 addrspace(1)* [[OBJ_TO_CONSUME]]) ] -; CHECK-NEXT: [[OBJ_TO_CONSUME_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 0) -; CHECK-NEXT: [[OBJ_TO_CONSUME_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_TO_CONSUME_BASE_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[OBJ_TO_CONSUME_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 1) +; CHECK-NEXT: [[STATEPOINT_TOKEN1:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* [[OBJ_TO_CONSUME]], i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[OBJ_TO_CONSUME]]) ] +; CHECK-NEXT: [[OBJ_TO_CONSUME_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_TO_CONSUME_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_TO_CONSUME_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: br label [[MERGE_SPLIT:%.*]] ; CHECK: merge.split: diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll index 827bb1cccc73..8c2fa5139950 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll @@ -41,7 +41,6 @@ define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1 ; CHECK-NEXT: br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]] ; CHECK: left: ; CHECK-NEXT: [[A_CAST:%.*]] = bitcast i8 addrspace(1)* [[A:%.*]] to i64 addrspace(1)* -; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[A]] to i64 addrspace(1)* ; CHECK-NEXT: switch i32 [[UNKNOWN:%.*]], label [[RIGHT]] [ ; CHECK-NEXT: i32 0, label [[MERGE:%.*]] ; CHECK-NEXT: i32 1, label [[MERGE]] @@ -49,16 +48,12 @@ define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1 ; CHECK-NEXT: ] ; CHECK: right: ; CHECK-NEXT: [[B_CAST:%.*]] = bitcast i8 addrspace(1)* [[B:%.*]] to i64 addrspace(1)* -; CHECK-NEXT: [[CAST1:%.*]] = bitcast i8 addrspace(1)* [[B]] to i64 addrspace(1)* ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[VALUE_BASE:%.*]] = phi i64 addrspace(1)* [ [[CAST]], [[LEFT]] ], [ [[CAST]], [[LEFT]] ], [ [[CAST]], [[LEFT]] ], [ [[CAST1]], [[RIGHT]] ], !is_base_value !0 ; CHECK-NEXT: [[VALUE:%.*]] = phi i64 addrspace(1)* [ [[A_CAST]], [[LEFT]] ], [ [[A_CAST]], [[LEFT]] ], [ [[A_CAST]], [[LEFT]] ], [ [[B_CAST]], [[RIGHT]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* [[VALUE]], i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[VALUE]], i64 addrspace(1)* [[VALUE_BASE]]) ] -; CHECK-NEXT: [[VALUE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* [[VALUE]], i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[VALUE]]) ] +; CHECK-NEXT: [[VALUE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[VALUE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[VALUE_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[VALUE_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[VALUE_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[VALUE_BASE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[VALUE_RELOCATED_CASTED]] ; entry: @@ -146,13 +141,10 @@ define i64 addrspace(1)* @test3(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1 ; CHECK: taken: ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[BDV_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[TAKEN]] ], !is_base_value !0 -; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ]], [[ENTRY]] ], [ [[OBJ2]], [[TAKEN]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BDV_BASE]]) ] -; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[TAKEN]] ] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ] +; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[BDV_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[BDV_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_BASE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]] ; entry: @@ -175,11 +167,9 @@ define i64 addrspace(1)* @test4(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1 ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: ; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ]], [[TAKEN]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[OBJ]]) ] -; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ] +; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]] ; entry: @@ -199,15 +189,12 @@ define i64 addrspace(1)* @test5(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1 ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[MERGE:%.*]] ; CHECK: merge: -; CHECK-NEXT: [[BDV_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[MERGE]] ], !is_base_value !0 -; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ]], [[ENTRY]] ], [ [[OBJ2]], [[MERGE]] ] +; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[MERGE]] ] ; CHECK-NEXT: br i1 [[CND:%.*]], label [[MERGE]], label [[NEXT:%.*]] ; CHECK: next: -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BDV_BASE]]) ] -; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ] +; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[BDV_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[BDV_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_BASE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]] ; entry: diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll index 6e0a8eef9010..d7f14dec9853 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll @@ -6,13 +6,10 @@ define i64 addrspace(1)* @test(<2 x i64 addrspace(1)*> %vec, i32 %idx) gc "statepoint-example" { ; CHECK-LABEL: @test( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[BASE_EE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]], !is_base_value !0 -; CHECK-NEXT: [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX]] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[BASE_EE]]) ] -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ] +; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]] ; ; Note that the second extractelement is actually redundant here. A correct output would @@ -34,25 +31,19 @@ define i64 addrspace(1)* @test2(<2 x i64 addrspace(1)*>* %ptr, i1 %cnd, i32 %idx ; CHECK-NEXT: [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16 ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[VEC_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0 ; CHECK-NEXT: [[VEC:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ] ; CHECK-NEXT: br i1 [[CND]], label [[TAKEN2:%.*]], label [[UNTAKEN2:%.*]] ; CHECK: taken2: -; CHECK-NEXT: [[OBJ0_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX1:%.*]], !is_base_value !0 -; CHECK-NEXT: [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1]] +; CHECK-NEXT: [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1:%.*]] ; CHECK-NEXT: br label [[MERGE2:%.*]] ; CHECK: untaken2: -; CHECK-NEXT: [[OBJ1_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX2:%.*]], !is_base_value !0 -; CHECK-NEXT: [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2]] +; CHECK-NEXT: [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2:%.*]] ; CHECK-NEXT: br label [[MERGE2]] ; CHECK: merge2: -; CHECK-NEXT: [[OBJ_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ0_BASE]], [[TAKEN2]] ], [ [[OBJ1_BASE]], [[UNTAKEN2]] ], !is_base_value !0 ; CHECK-NEXT: [[OBJ:%.*]] = phi i64 addrspace(1)* [ [[OBJ0]], [[TAKEN2]] ], [ [[OBJ1]], [[UNTAKEN2]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[OBJ_BASE]]) ] -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ] +; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[OBJ_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_BASE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]] ; entry: @@ -282,13 +273,10 @@ define void @test9(<4 x i64 addrspace(1)*> %vec1, i64 %idx) gc "statepoint-examp ; CHECK-LABEL: @test9( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i64 addrspace(1)*> [[VEC1:%.*]], <4 x i64 addrspace(1)*> [[VEC1]], <2 x i32> -; CHECK-NEXT: [[BASE_EE:%.*]] = extractelement <4 x i64 addrspace(1)*> [[VEC1]], i64 [[IDX:%.*]], !is_base_value !0 -; CHECK-NEXT: [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX]] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BASE_EE]]) ] -; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX:%.*]] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ] +; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: call void @use(i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]) ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll index f377989e8118..f790ef248f6e 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector.ll @@ -8,13 +8,10 @@ define i64 addrspace(1)* @test(<2 x i64 addrspace(1)*> %vec, i32 %idx) gc "state ; be to reuse the existing obj as a base since it is actually a base pointer. ; CHECK-LABEL: @test( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[BASE_EE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]], !is_base_value !0 -; CHECK-NEXT: [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX]] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[BASE_EE]]) ] -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[OBJ:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC:%.*]], i32 [[IDX:%.*]] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ] +; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]] ; entry: @@ -34,25 +31,19 @@ define i64 addrspace(1)* @test2(<2 x i64 addrspace(1)*>* %ptr, i1 %cnd, i32 %idx ; CHECK-NEXT: [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16 ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[VEC_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0 ; CHECK-NEXT: [[VEC:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ] ; CHECK-NEXT: br i1 [[CND]], label [[TAKEN2:%.*]], label [[UNTAKEN2:%.*]] ; CHECK: taken2: -; CHECK-NEXT: [[OBJ0_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX1:%.*]], !is_base_value !0 -; CHECK-NEXT: [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1]] +; CHECK-NEXT: [[OBJ0:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX1:%.*]] ; CHECK-NEXT: br label [[MERGE2:%.*]] ; CHECK: untaken2: -; CHECK-NEXT: [[OBJ1_BASE:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC_BASE]], i32 [[IDX2:%.*]], !is_base_value !0 -; CHECK-NEXT: [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2]] +; CHECK-NEXT: [[OBJ1:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i32 [[IDX2:%.*]] ; CHECK-NEXT: br label [[MERGE2]] ; CHECK: merge2: -; CHECK-NEXT: [[OBJ_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ0_BASE]], [[TAKEN2]] ], [ [[OBJ1_BASE]], [[UNTAKEN2]] ], !is_base_value !0 ; CHECK-NEXT: [[OBJ:%.*]] = phi i64 addrspace(1)* [ [[OBJ0]], [[TAKEN2]] ], [ [[OBJ1]], [[UNTAKEN2]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]], i64 addrspace(1)* [[OBJ_BASE]]) ] -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i64 addrspace(1)* [[OBJ]]) ] +; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[OBJ_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_BASE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: ret i64 addrspace(1)* [[OBJ_RELOCATED_CASTED]] ; entry: @@ -282,13 +273,10 @@ define void @test9(<4 x i64 addrspace(1)*> %vec1, i64 %idx) gc "statepoint-examp ; CHECK-LABEL: @test9( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i64 addrspace(1)*> [[VEC1:%.*]], <4 x i64 addrspace(1)*> [[VEC1]], <2 x i32> -; CHECK-NEXT: [[BASE_EE:%.*]] = extractelement <4 x i64 addrspace(1)*> [[VEC1]], i64 [[IDX:%.*]], !is_base_value !0 -; CHECK-NEXT: [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX]] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BASE_EE]]) ] -; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[BDV:%.*]] = extractelement <2 x i64 addrspace(1)*> [[VEC]], i64 [[IDX:%.*]] +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ] +; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)* -; CHECK-NEXT: [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[BASE_EE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASE_EE_RELOCATED]] to i64 addrspace(1)* ; CHECK-NEXT: call void @use(i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]) ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll index a38b556d0a37..31892b51c360 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll @@ -112,13 +112,10 @@ define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) gc ; CHECK-NEXT: [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16 ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[OBJ_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0 ; CHECK-NEXT: [[OBJ:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]], <2 x i64 addrspace(1)*> [[OBJ_BASE]]) ] -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]]) ] +; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_RELOCATED]] to <2 x i64 addrspace(1)*> -; CHECK-NEXT: [[OBJ_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_BASE_RELOCATED]] to <2 x i64 addrspace(1)*> ; CHECK-NEXT: ret <2 x i64 addrspace(1)*> [[OBJ_RELOCATED_CASTED]] ; entry: diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll index 6e6333f1dd89..76d06ac05179 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit.ll @@ -112,13 +112,10 @@ define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) gc ; CHECK-NEXT: [[OBJB:%.*]] = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* [[PTR]], align 16 ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[OBJ_BASE:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ], !is_base_value !0 ; CHECK-NEXT: [[OBJ:%.*]] = phi <2 x i64 addrspace(1)*> [ [[OBJA]], [[TAKEN]] ], [ [[OBJB]], [[UNTAKEN]] ] -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]], <2 x i64 addrspace(1)*> [[OBJ_BASE]]) ] -; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(<2 x i64 addrspace(1)*> [[OBJ]]) ] +; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_RELOCATED]] to <2 x i64 addrspace(1)*> -; CHECK-NEXT: [[OBJ_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) -; CHECK-NEXT: [[OBJ_BASE_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[OBJ_BASE_RELOCATED]] to <2 x i64 addrspace(1)*> ; CHECK-NEXT: ret <2 x i64 addrspace(1)*> [[OBJ_RELOCATED_CASTED]] ; entry: diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll b/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll index 6f674f443c94..77bedd875af0 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/rematerialize-derived-pointers.ll @@ -375,13 +375,12 @@ define void @contains_basephi(i1 %cond) gc "statepoint-example" { ; CHECK: there: ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[BASEPHI_BASE:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ], !is_base_value !0 ; CHECK-NEXT: [[BASEPHI:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ] ; CHECK-NEXT: [[PTR_GEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI]], i32 15 -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI_BASE]]) ] -; CHECK-NEXT: [[BASEPHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) -; CHECK-NEXT: [[BASEPHI_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_BASE_RELOCATED]] to i32 addrspace(1)* -; CHECK-NEXT: [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_BASE_RELOCATED_CASTED]], i32 15 +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI]]) ] +; CHECK-NEXT: [[BASEPHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) +; CHECK-NEXT: [[BASEPHI_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_RELOCATED]] to i32 addrspace(1)* +; CHECK-NEXT: [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_RELOCATED_CASTED]], i32 15 ; CHECK-NEXT: call void @use_obj32(i32 addrspace(1)* [[PTR_GEP_REMAT]]) ; CHECK-NEXT: ret void ; @@ -419,17 +418,16 @@ define void @test_intersecting_chains_with_phi(i1 %cond) gc "statepoint-example" ; CHECK: there: ; CHECK-NEXT: br label [[MERGE]] ; CHECK: merge: -; CHECK-NEXT: [[BASEPHI_BASE:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ], !is_base_value !0 ; CHECK-NEXT: [[BASEPHI:%.*]] = phi i32 addrspace(1)* [ [[BASE1]], [[HERE]] ], [ [[BASE2]], [[THERE]] ] ; CHECK-NEXT: [[PTR_GEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI]], i32 15 ; CHECK-NEXT: [[PTR_CAST:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP]] to i64 addrspace(1)* ; CHECK-NEXT: [[PTR_CAST2:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP]] to i16 addrspace(1)* -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI_BASE]]) ] -; CHECK-NEXT: [[BASEPHI_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) -; CHECK-NEXT: [[BASEPHI_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_BASE_RELOCATED]] to i32 addrspace(1)* -; CHECK-NEXT: [[PTR_GEP_REMAT1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_BASE_RELOCATED_CASTED]], i32 15 +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(i32 addrspace(1)* [[BASEPHI]]) ] +; CHECK-NEXT: [[BASEPHI_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) +; CHECK-NEXT: [[BASEPHI_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BASEPHI_RELOCATED]] to i32 addrspace(1)* +; CHECK-NEXT: [[PTR_GEP_REMAT1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_RELOCATED_CASTED]], i32 15 ; CHECK-NEXT: [[PTR_CAST_REMAT:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP_REMAT1]] to i64 addrspace(1)* -; CHECK-NEXT: [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_BASE_RELOCATED_CASTED]], i32 15 +; CHECK-NEXT: [[PTR_GEP_REMAT:%.*]] = getelementptr i32, i32 addrspace(1)* [[BASEPHI_RELOCATED_CASTED]], i32 15 ; CHECK-NEXT: [[PTR_CAST2_REMAT:%.*]] = bitcast i32 addrspace(1)* [[PTR_GEP_REMAT]] to i16 addrspace(1)* ; CHECK-NEXT: call void @use_obj64(i64 addrspace(1)* [[PTR_CAST_REMAT]]) ; CHECK-NEXT: call void @use_obj16(i16 addrspace(1)* [[PTR_CAST2_REMAT]]) diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll index 82768f2b17a3..c45a580119a7 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll @@ -14,10 +14,9 @@ define i32 addrspace(1)* @test1(i8 addrspace(1)* %base1, <2 x i64> %offsets) gc ; CHECK-NEXT: [[BASE21:%.*]] = call i8 addrspace(1)* @llvm.experimental.gc.result.p1i8(token [[STATEPOINT_TOKEN]]) ; CHECK-NEXT: br label [[SECOND]] ; CHECK: second: -; CHECK-NEXT: [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[BASE1:%.*]], [[ENTRY:%.*]] ], [ [[BASE21]], [[FIRST]] ], !is_base_value !0 -; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[BASE1]], [[ENTRY]] ], [ [[BASE21]], [[FIRST]] ] +; CHECK-NEXT: [[PHI:%.*]] = phi i8 addrspace(1)* [ [[BASE1:%.*]], [[ENTRY:%.*]] ], [ [[BASE21]], [[FIRST]] ] ; CHECK-NEXT: [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)* -; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI_BASE]] to i32 addrspace(1)* +; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)* ; CHECK-NEXT: [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0 ; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 -- GitLab From db36d882ed185534e125746f86373d434acdf2e6 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Tue, 16 Mar 2021 11:46:31 -0700 Subject: [PATCH 0067/1206] scudo: Allow TBI to be disabled on Linux with a macro. Android's native bridge (i.e. AArch64 emulator) doesn't support TBI so we need a way to disable TBI on Linux when targeting the native bridge. This can also be used to test the no-TBI code path on Linux (currently only used on Fuchsia), or make Scudo compatible with very old (pre-commit d50240a5f6ceaf690a77b0fccb17be51cfa151c2 from June 2013) Linux kernels that do not enable TBI. Differential Revision: https://reviews.llvm.org/D98732 --- compiler-rt/lib/scudo/standalone/memtag.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h index ea504bbbf7a1..0a8a0b52173a 100644 --- a/compiler-rt/lib/scudo/standalone/memtag.h +++ b/compiler-rt/lib/scudo/standalone/memtag.h @@ -26,7 +26,7 @@ void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, uptr *TaggedBegin, // We assume that Top-Byte Ignore is enabled if the architecture supports memory // tagging. Not all operating systems enable TBI, so we only claim architectural // support for memory tagging if the operating system enables TBI. -#if SCUDO_LINUX +#if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI) inline constexpr bool archSupportsMemoryTagging() { return true; } #else inline constexpr bool archSupportsMemoryTagging() { return false; } -- GitLab From ef884e155da7a46225f3441faa45d6d9e4249f8d Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Tue, 16 Mar 2021 12:57:54 -0700 Subject: [PATCH 0068/1206] [rs4gc] don't force a conflict for a canonical broadcast A broadcast is a shufflevector where only one input is used. Because of the way we handle constants (undef is a constant), the canonical shuffle sees a meet of (some value) and (nullptr). Given this, every broadcast gets treated as a conflict and a new base pointer computation is added. The other way to tackle this would be to change constant handling specifically for undefs, but this seems easier. Differential Revision: https://reviews.llvm.org/D98315 --- llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 8 ++++++-- .../Transforms/RewriteStatepointsForGC/base-inference.ll | 6 ++---- .../RewriteStatepointsForGC/scalar-base-vector.ll | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 1ab4946476cc..755ebb881622 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -855,8 +855,11 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { F(IE->getOperand(0)); F(IE->getOperand(1)); } else if (auto *SV = dyn_cast(BDV)) { + // For a canonical broadcast, ignore the undef argument + // (without this, we insert a parallel base shuffle for every broadcast) F(SV->getOperand(0)); - F(SV->getOperand(1)); + if (!SV->isZeroEltSplat()) + F(SV->getOperand(1)); } else { llvm_unreachable("unexpected BDV type"); } @@ -1214,7 +1217,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { BaseSV->setOperand(OperandIdx, Base); }; UpdateOperand(0); // vector operand - UpdateOperand(1); // vector operand + if (!BdvSV->isZeroEltSplat()) + UpdateOperand(1); // vector operand } } diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll index ddb229234b4e..0df3043fd4ca 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll @@ -227,11 +227,9 @@ define <2 x i8 addrspace(1)*> @test_shuffle_broadcast(i8 addrspace(1)* %a) gc "s ; CHECK-LABEL: @test_shuffle_broadcast( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[IE:%.*]] = insertelement <2 x i8 addrspace(1)*> zeroinitializer, i8 addrspace(1)* [[A:%.*]], i64 0 -; CHECK-NEXT: [[BROADCAST_BASE:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 ; CHECK-NEXT: [[BROADCAST:%.*]] = shufflevector <2 x i8 addrspace(1)*> [[IE]], <2 x i8 addrspace(1)*> undef, <2 x i32> zeroinitializer -; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[BROADCAST]], <2 x i8 addrspace(1)*> [[BROADCAST_BASE]]) ] -; CHECK-NEXT: [[BROADCAST_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0) -; CHECK-NEXT: [[BROADCAST_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1) +; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i8 addrspace(1)*> [[BROADCAST]]) ] +; CHECK-NEXT: [[BROADCAST_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0) ; CHECK-NEXT: ret <2 x i8 addrspace(1)*> [[BROADCAST_RELOCATED]] ; entry: diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll index c45a580119a7..2fb066608b4e 100644 --- a/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll +++ b/llvm/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll @@ -19,7 +19,7 @@ define i32 addrspace(1)* @test1(i8 addrspace(1)* %base1, <2 x i64> %offsets) gc ; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)* ; CHECK-NEXT: [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0 -; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 +; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]] ; CHECK-NEXT: [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0 @@ -54,7 +54,7 @@ define i32 addrspace(1)* @test2(i8 addrspace(1)* %base, <2 x i64> %offsets) gc " ; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)* ; CHECK-NEXT: [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0 -; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 +; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]] ; CHECK-NEXT: [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0 @@ -102,7 +102,7 @@ define i32 addrspace(1)* @test4(i8 addrspace(1)* %base, <2 x i64> %offsets) gc " ; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)* ; CHECK-NEXT: [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> poison, i32 addrspace(1)* [[BASE_I32]], i32 0 -; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0 +; CHECK-NEXT: [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer, !is_base_value !0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]] ; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0) [ "gc-live"(<2 x i32 addrspace(1)*> [[VEC]], <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]]) ] -- GitLab From cec9e7352bebe06681a9627f3fc08228129b7681 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Tue, 16 Mar 2021 13:00:23 -0700 Subject: [PATCH 0069/1206] [rs4gc] Simplify code by cloning existing instructions when inserting base chain [NFC] Previously we created a new node, then filled in the pieces. Now, we clone the existing node, then change the respective fields. The only change in handling is with phis since we have to handle multiple incoming edges from the same block a bit differently. Differential Revision: https://reviews.llvm.org/D98316 --- .../Scalar/RewriteStatepointsForGC.cpp | 85 +++++++------------ 1 file changed, 31 insertions(+), 54 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 755ebb881622..fdc1c483cb2a 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -1057,40 +1057,23 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { if (!State.isConflict()) continue; - /// Create and insert a new instruction which will represent the base of - /// the given instruction 'I'. - auto MakeBaseInstPlaceholder = [](Instruction *I) -> Instruction* { + auto getMangledName = [](Instruction *I) -> std::string { if (isa(I)) { - BasicBlock *BB = I->getParent(); - int NumPreds = pred_size(BB); - assert(NumPreds > 0 && "how did we reach here"); - std::string Name = suffixed_name_or(I, ".base", "base_phi"); - return PHINode::Create(I->getType(), NumPreds, Name, I); - } else if (SelectInst *SI = dyn_cast(I)) { - // The undef will be replaced later - UndefValue *Undef = UndefValue::get(SI->getType()); - std::string Name = suffixed_name_or(I, ".base", "base_select"); - return SelectInst::Create(SI->getCondition(), Undef, Undef, Name, SI); - } else if (auto *EE = dyn_cast(I)) { - UndefValue *Undef = UndefValue::get(EE->getVectorOperand()->getType()); - std::string Name = suffixed_name_or(I, ".base", "base_ee"); - return ExtractElementInst::Create(Undef, EE->getIndexOperand(), Name, - EE); - } else if (auto *IE = dyn_cast(I)) { - UndefValue *VecUndef = UndefValue::get(IE->getOperand(0)->getType()); - UndefValue *ScalarUndef = UndefValue::get(IE->getOperand(1)->getType()); - std::string Name = suffixed_name_or(I, ".base", "base_ie"); - return InsertElementInst::Create(VecUndef, ScalarUndef, - IE->getOperand(2), Name, IE); + return suffixed_name_or(I, ".base", "base_phi"); + } else if (isa(I)) { + return suffixed_name_or(I, ".base", "base_select"); + } else if (isa(I)) { + return suffixed_name_or(I, ".base", "base_ee"); + } else if (isa(I)) { + return suffixed_name_or(I, ".base", "base_ie"); } else { - auto *SV = cast(I); - UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType()); - std::string Name = suffixed_name_or(I, ".base", "base_sv"); - return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(), - Name, SV); + return suffixed_name_or(I, ".base", "base_sv"); } }; - Instruction *BaseInst = MakeBaseInstPlaceholder(I); + + Instruction *BaseInst = I->clone(); + BaseInst->insertBefore(I); + BaseInst->setName(getMangledName(I)); // Add metadata marking this as a base value BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {})); States[I] = BDVState(I, BDVState::Conflict, BaseInst); @@ -1145,26 +1128,21 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { if (PHINode *BasePHI = dyn_cast(State.getBaseValue())) { PHINode *PN = cast(BDV); - unsigned NumPHIValues = PN->getNumIncomingValues(); + const unsigned NumPHIValues = PN->getNumIncomingValues(); + + // The IR verifier requires phi nodes with multiple entries from the + // same basic block to have the same incoming value for each of those + // entries. Since we're inserting bitcasts in the loop, make sure we + // do so at least once per incoming block. + DenseMap BlockToValue; for (unsigned i = 0; i < NumPHIValues; i++) { Value *InVal = PN->getIncomingValue(i); BasicBlock *InBB = PN->getIncomingBlock(i); - - // If we've already seen InBB, add the same incoming value - // we added for it earlier. The IR verifier requires phi - // nodes with multiple entries from the same basic block - // to have the same incoming value for each of those - // entries. If we don't do this check here and basephi - // has a different type than base, we'll end up adding two - // bitcasts (and hence two distinct values) as incoming - // values for the same basic block. - - int BlockIndex = BasePHI->getBasicBlockIndex(InBB); - if (BlockIndex != -1) { - Value *OldBase = BasePHI->getIncomingValue(BlockIndex); - BasePHI->addIncoming(OldBase, InBB); - + if (!BlockToValue.count(InBB)) + BlockToValue[InBB] = getBaseForInput(InVal, InBB->getTerminator()); + else { #ifndef NDEBUG + Value *OldBase = BlockToValue[InBB]; Value *Base = getBaseForInput(InVal, nullptr); // In essence this assert states: the only way two values // incoming from the same basic block may be different is by @@ -1175,16 +1153,10 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { assert(Base->stripPointerCasts() == OldBase->stripPointerCasts() && "Sanity -- findBaseOrBDV should be pure!"); #endif - continue; } - - // Find the instruction which produces the base for each input. We may - // need to insert a bitcast in the incoming block. - // TODO: Need to split critical edges if insertion is needed - Value *Base = getBaseForInput(InVal, InBB->getTerminator()); - BasePHI->addIncoming(Base, InBB); + Value *Base = BlockToValue[InBB]; + BasePHI->setIncomingValue(i, Base); } - assert(BasePHI->getNumIncomingValues() == NumPHIValues); } else if (SelectInst *BaseSI = dyn_cast(State.getBaseValue())) { SelectInst *SI = cast(BDV); @@ -1219,6 +1191,11 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { UpdateOperand(0); // vector operand if (!BdvSV->isZeroEltSplat()) UpdateOperand(1); // vector operand + else { + // Never read, so just use undef + Value *InVal = BdvSV->getOperand(1); + BaseSV->setOperand(1, UndefValue::get(InVal->getType())); + } } } -- GitLab From 1bc8f5fbb4d46c54f9aa732f32aaeb77972ecad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Sat, 27 Feb 2021 00:31:18 +0200 Subject: [PATCH 0070/1206] [sanitizers] [windows] Use InternalMmapVector instead of silencing -Wframe-larger-than Also use this in ReadBinaryName which currently is producing warnings. Keep pragmas for silencing warnings in sanitizer_unwind_win.cpp, as that can be called more frequently. Differential Revision: https://reviews.llvm.org/D97726 --- .../sanitizer_symbolizer_win.cpp | 12 ++--- .../lib/sanitizer_common/sanitizer_win.cpp | 45 +++++++++---------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp index dc611a01a500..6df96d491b24 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp @@ -133,16 +133,13 @@ void InitializeDbgHelpIfNeeded() { } } -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wframe-larger-than=" -#endif bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) { InitializeDbgHelpIfNeeded(); // See https://docs.microsoft.com/en-us/windows/win32/debug/retrieving-symbol-information-by-address - char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(CHAR)]; - PSYMBOL_INFO symbol = (PSYMBOL_INFO)buffer; + InternalMmapVector buffer(sizeof(SYMBOL_INFO) + + MAX_SYM_NAME * sizeof(CHAR)); + PSYMBOL_INFO symbol = (PSYMBOL_INFO)&buffer[0]; symbol->SizeOfStruct = sizeof(SYMBOL_INFO); symbol->MaxNameLen = MAX_SYM_NAME; DWORD64 offset = 0; @@ -166,9 +163,6 @@ bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) { // Otherwise, try llvm-symbolizer. return got_fileline; } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif const char *WinSymbolizerTool::Demangle(const char *name) { CHECK(is_dbghelp_initialized); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp index 99ecfd040c6a..b4ad9d4fe36d 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp @@ -568,7 +568,7 @@ void Abort() { // load the image at this address. Therefore, we call it the preferred base. Any // addresses in the DWARF typically assume that the object has been loaded at // this address. -static uptr GetPreferredBase(const char *modname) { +static uptr GetPreferredBase(const char *modname, char *buf, size_t buf_size) { fd_t fd = OpenFile(modname, RdOnly, nullptr); if (fd == kInvalidFd) return 0; @@ -590,12 +590,10 @@ static uptr GetPreferredBase(const char *modname) { // IMAGE_FILE_HEADER // IMAGE_OPTIONAL_HEADER // Seek to e_lfanew and read all that data. - char buf[4 + sizeof(IMAGE_FILE_HEADER) + sizeof(IMAGE_OPTIONAL_HEADER)]; if (::SetFilePointer(fd, dos_header.e_lfanew, nullptr, FILE_BEGIN) == INVALID_SET_FILE_POINTER) return 0; - if (!ReadFromFile(fd, &buf[0], sizeof(buf), &bytes_read) || - bytes_read != sizeof(buf)) + if (!ReadFromFile(fd, buf, buf_size, &bytes_read) || bytes_read != buf_size) return 0; // Check for "PE\0\0" before the PE header. @@ -615,10 +613,6 @@ static uptr GetPreferredBase(const char *modname) { return (uptr)pe_header->ImageBase; } -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wframe-larger-than=" -#endif void ListOfModules::init() { clearOrInit(); HANDLE cur_process = GetCurrentProcess(); @@ -641,6 +635,10 @@ void ListOfModules::init() { } } + InternalMmapVector buf(4 + sizeof(IMAGE_FILE_HEADER) + + sizeof(IMAGE_OPTIONAL_HEADER)); + InternalMmapVector modname_utf16(kMaxPathLength); + InternalMmapVector module_name(kMaxPathLength); // |num_modules| is the number of modules actually present, size_t num_modules = bytes_required / sizeof(HMODULE); for (size_t i = 0; i < num_modules; ++i) { @@ -650,15 +648,13 @@ void ListOfModules::init() { continue; // Get the UTF-16 path and convert to UTF-8. - wchar_t modname_utf16[kMaxPathLength]; int modname_utf16_len = - GetModuleFileNameW(handle, modname_utf16, kMaxPathLength); + GetModuleFileNameW(handle, &modname_utf16[0], kMaxPathLength); if (modname_utf16_len == 0) modname_utf16[0] = '\0'; - char module_name[kMaxPathLength]; - int module_name_len = - ::WideCharToMultiByte(CP_UTF8, 0, modname_utf16, modname_utf16_len + 1, - &module_name[0], kMaxPathLength, NULL, NULL); + int module_name_len = ::WideCharToMultiByte( + CP_UTF8, 0, &modname_utf16[0], modname_utf16_len + 1, &module_name[0], + kMaxPathLength, NULL, NULL); module_name[module_name_len] = '\0'; uptr base_address = (uptr)mi.lpBaseOfDll; @@ -668,21 +664,19 @@ void ListOfModules::init() { // RVA when computing the module offset. This helps llvm-symbolizer find the // right DWARF CU. In the common case that the image is loaded at it's // preferred address, we will now print normal virtual addresses. - uptr preferred_base = GetPreferredBase(&module_name[0]); + uptr preferred_base = + GetPreferredBase(&module_name[0], &buf[0], buf.size()); uptr adjusted_base = base_address - preferred_base; - LoadedModule cur_module; - cur_module.set(module_name, adjusted_base); + modules_.push_back(LoadedModule()); + LoadedModule &cur_module = modules_.back(); + cur_module.set(&module_name[0], adjusted_base); // We add the whole module as one single address range. cur_module.addAddressRange(base_address, end_address, /*executable*/ true, /*writable*/ true); - modules_.push_back(cur_module); } UnmapOrDie(hmodules, modules_buffer_size); } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif void ListOfModules::fallbackInit() { clear(); } @@ -1057,15 +1051,16 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { return 0; // Get the UTF-16 path and convert to UTF-8. - wchar_t binname_utf16[kMaxPathLength]; + InternalMmapVector binname_utf16(kMaxPathLength); int binname_utf16_len = - GetModuleFileNameW(NULL, binname_utf16, ARRAY_SIZE(binname_utf16)); + GetModuleFileNameW(NULL, &binname_utf16[0], kMaxPathLength); if (binname_utf16_len == 0) { buf[0] = '\0'; return 0; } - int binary_name_len = ::WideCharToMultiByte( - CP_UTF8, 0, binname_utf16, binname_utf16_len, buf, buf_len, NULL, NULL); + int binary_name_len = + ::WideCharToMultiByte(CP_UTF8, 0, &binname_utf16[0], binname_utf16_len, + buf, buf_len, NULL, NULL); if ((unsigned)binary_name_len == buf_len) --binary_name_len; buf[binary_name_len] = '\0'; -- GitLab From 242762c9a3313c8aea176ca76fb77adf8edf0907 Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 16 Mar 2021 13:11:07 -0700 Subject: [PATCH 0071/1206] [mlir][pdl] Restructure how results are represented. Up until now, results have been represented as additional results to a pdl.operation. This is fairly clunky, as it mismatches the representation of the rest of the IR constructs(e.g. pdl.operand) and also isn't a viable representation for operations returned by pdl.create_native. This representation also creates much more difficult problems when factoring in support for variadic result groups, optional results, etc. To resolve some of these problems, and simplify adding support for variable length results, this revision extracts the representation for results out of pdl.operation in the form of a new `pdl.result` operation. This operation returns the result of an operation at a given index, e.g.: ``` %root = pdl.operation ... %result = pdl.result 0 of %root ``` Differential Revision: https://reviews.llvm.org/D95719 --- .../include/mlir/Dialect/PDL/IR/PDLDialect.td | 2 +- mlir/include/mlir/Dialect/PDL/IR/PDLOps.td | 55 +++- .../PDLToPDLInterp/PDLToPDLInterp.cpp | 69 +++-- .../lib/Conversion/PDLToPDLInterp/Predicate.h | 2 +- .../PDLToPDLInterp/PredicateTree.cpp | 271 ++++++++++-------- mlir/lib/Dialect/PDL/IR/PDL.cpp | 179 +++--------- .../pdl-to-pdl-interp-matcher.mlir | 60 +++- .../pdl-to-pdl-interp-rewriter.mlir | 40 +-- mlir/test/Dialect/PDL/invalid.mlir | 39 +-- mlir/test/Dialect/PDL/ops.mlir | 26 +- 10 files changed, 354 insertions(+), 389 deletions(-) diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td b/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td index afdf50673ed4..1c9de16af358 100644 --- a/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td +++ b/mlir/include/mlir/Dialect/PDL/IR/PDLDialect.td @@ -48,7 +48,7 @@ def PDL_Dialect : Dialect { %resultType = pdl.type %inputOperand = pdl.operand - %root, %results = pdl.operation "foo.op"(%inputOperand) -> %resultType + %root = pdl.operation "foo.op"(%inputOperand) -> %resultType pdl.rewrite %root { pdl.replace %root with (%inputOperand) } diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td index 60590b1fcd01..76e4c5d022a4 100644 --- a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td +++ b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td @@ -177,7 +177,7 @@ def PDL_OperandOp : PDL_Op<"operand", [HasParent<"pdl::PatternOp">]> { let description = [{ `pdl.operand` operations capture external operand edges into an operation node that originate from operations or block arguments not otherwise - specified within the pattern (e.g. via `pdl.operation`). These operations + specified within the pattern (e.g. via `pdl.result`). These operations define individual operands of a given operation. A `pdl.operand` may partially constrain an operand by specifying an expected value type (via a `pdl.type` operation). @@ -223,8 +223,8 @@ def PDL_OperationOp `pdl.operation`s are composed of a name, and a set of attribute, operand, and result type values, that map to what those that would be on a constructed instance of that operation. The results of a `pdl.operation` are - a handle to the operation itself, and a handle to each of the operation - result values. + a handle to the operation itself. Handles to the results of the operation + can be extracted via `pdl.result`. When used within a matching context, the name of the operation may be omitted. @@ -241,7 +241,7 @@ def PDL_OperationOp ```mlir // Define an instance of a `foo.op` operation. - %op, %results:4 = pdl.operation "foo.op"(%arg0, %arg1) {"attrA" = %attr0} -> %type, %type, %type, %type + %op = pdl.operation "foo.op"(%arg0, %arg1) {"attrA" = %attr0} -> %type, %type, %type, %type ``` }]; @@ -250,8 +250,13 @@ def PDL_OperationOp Variadic:$attributes, StrArrayAttr:$attributeNames, Variadic:$types); - let results = (outs PDL_Operation:$op, - Variadic:$results); + let results = (outs PDL_Operation:$op); + let assemblyFormat = [{ + ($name^)? (`(` $operands^ `)`)? + custom($attributes, $attributeNames) + (`->` $types^)? attr-dict + }]; + let builders = [ OpBuilder<(ins CArg<"Optional", "llvm::None">:$name, CArg<"ValueRange", "llvm::None">:$operandValues, @@ -259,10 +264,9 @@ def PDL_OperationOp CArg<"ValueRange", "llvm::None">:$attrValues, CArg<"ValueRange", "llvm::None">:$resultTypes), [{ auto nameAttr = name ? StringAttr() : $_builder.getStringAttr(*name); - build($_builder, $_state, $_builder.getType(), {}, nameAttr, + build($_builder, $_state, $_builder.getType(), nameAttr, operandValues, attrValues, $_builder.getStrArrayAttr(attrNames), resultTypes); - $_state.types.append(resultTypes.size(), $_builder.getType()); }]>, ]; let extraClassDeclaration = [{ @@ -293,7 +297,7 @@ def PDL_PatternOp : PDL_Op<"pattern", [IsolatedFromAbove, Symbol]> { pdl.pattern : benefit(1) { %resultType = pdl.type %inputOperand = pdl.operand - %root, %results = pdl.operation "foo.op"(%inputOperand) -> (%resultType) + %root = pdl.operation "foo.op"(%inputOperand) -> (%resultType) pdl.rewrite %root { pdl.replace %root with (%inputOperand) } @@ -368,6 +372,39 @@ def PDL_ReplaceOp : PDL_Op<"replace", [ }]; } +//===----------------------------------------------------------------------===// +// pdl::ResultOp +//===----------------------------------------------------------------------===// + +def PDL_ResultOp : PDL_Op<"result"> { + let summary = "Extract a result from an operation"; + let description = [{ + `pdl.result` operations extract result edges from an operation node within + a pattern or rewrite region. The provided index is zero-based, and + represents the concrete result to extract, i.e. this is not the result index + as defined by the ODS definition of the operation. + + Example: + + ```mlir + // Extract a result: + %operation = pdl.operation ... + %result = pdl.result 1 of %operation + + // Imagine the following IR being matched: + %result_0, %result_1 = foo.op ... + + // If the example pattern snippet above were matching against `foo.op` in + // the IR snippted, `%result` would correspond to `%result_1`. + ``` + }]; + + let arguments = (ins PDL_Operation:$parent, I32Attr:$index); + let results = (outs PDL_Value:$val); + let assemblyFormat = "$index `of` $parent attr-dict"; + let verifier = ?; +} + //===----------------------------------------------------------------------===// // pdl::RewriteOp //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp index a225699e89f7..3368ceb9be88 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp @@ -85,6 +85,9 @@ private: void generateRewriter(pdl::ReplaceOp replaceOp, DenseMap &rewriteValues, function_ref mapRewriteValue); + void generateRewriter(pdl::ResultOp resultOp, + DenseMap &rewriteValues, + function_ref mapRewriteValue); void generateRewriter(pdl::TypeOp typeOp, DenseMap &rewriteValues, function_ref mapRewriteValue); @@ -457,9 +460,10 @@ SymbolRefAttr PatternLowering::generateRewriter( for (Operation &rewriteOp : *rewriter.getBody()) { llvm::TypeSwitch(&rewriteOp) .Case([&](auto op) { - this->generateRewriter(op, rewriteValues, mapRewriteValue); - }); + pdl::OperationOp, pdl::ReplaceOp, pdl::ResultOp, pdl::TypeOp>( + [&](auto op) { + this->generateRewriter(op, rewriteValues, mapRewriteValue); + }); } } @@ -511,17 +515,15 @@ void PatternLowering::generateRewriter( operationOp.attributeNames()); rewriteValues[operationOp.op()] = createdOp; - // Make all of the new operation results available. - OperandRange resultTypes = operationOp.types(); - for (auto it : llvm::enumerate(operationOp.results())) { + // Generate accesses for any results that have their types constrained. + for (auto it : llvm::enumerate(operationOp.types())) { + Value &type = rewriteValues[it.value()]; + if (type) + continue; + Value getResultVal = builder.create( loc, builder.getType(), createdOp, it.index()); - rewriteValues[it.value()] = getResultVal; - - // If any of the types have not been resolved, make those available as well. - Value &type = rewriteValues[resultTypes[it.index()]]; - if (!type) - type = builder.create(loc, getResultVal); + type = builder.create(loc, getResultVal); } } @@ -540,29 +542,41 @@ void PatternLowering::generateRewriter( void PatternLowering::generateRewriter( pdl::ReplaceOp replaceOp, DenseMap &rewriteValues, function_ref mapRewriteValue) { + SmallVector replOperands; + // If the replacement was another operation, get its results. `pdl` allows // for using an operation for simplicitly, but the interpreter isn't as // user facing. - ValueRange origOperands; - if (Value replOp = replaceOp.replOperation()) - origOperands = cast(replOp.getDefiningOp()).results(); - else - origOperands = replaceOp.replValues(); + if (Value replOp = replaceOp.replOperation()) { + pdl::OperationOp op = cast(replOp.getDefiningOp()); + for (unsigned i = 0, e = op.types().size(); i < e; ++i) + replOperands.push_back(builder.create( + replOp.getLoc(), builder.getType(), + mapRewriteValue(replOp), i)); + } else { + for (Value operand : replaceOp.replValues()) + replOperands.push_back(mapRewriteValue(operand)); + } // If there are no replacement values, just create an erase instead. - if (origOperands.empty()) { + if (replOperands.empty()) { builder.create(replaceOp.getLoc(), mapRewriteValue(replaceOp.operation())); return; } - SmallVector replOperands; - for (Value operand : origOperands) - replOperands.push_back(mapRewriteValue(operand)); builder.create( replaceOp.getLoc(), mapRewriteValue(replaceOp.operation()), replOperands); } +void PatternLowering::generateRewriter( + pdl::ResultOp resultOp, DenseMap &rewriteValues, + function_ref mapRewriteValue) { + rewriteValues[resultOp] = builder.create( + resultOp.getLoc(), builder.getType(), + mapRewriteValue(resultOp.parent()), resultOp.index()); +} + void PatternLowering::generateRewriter( pdl::TypeOp typeOp, DenseMap &rewriteValues, function_ref mapRewriteValue) { @@ -602,8 +616,8 @@ void PatternLowering::generateOperationResultTypeRewriter( bool hasTypeInference = op.hasTypeInference(); auto resultTypeValues = op.types(); types.reserve(resultTypeValues.size()); - for (auto it : llvm::enumerate(op.results())) { - Value result = it.value(), resultType = resultTypeValues[it.index()]; + for (auto it : llvm::enumerate(resultTypeValues)) { + Value resultType = it.value(); // Check for an already translated value. if (Value existingRewriteValue = rewriteValues.lookup(resultType)) { @@ -633,16 +647,11 @@ void PatternLowering::generateOperationResultTypeRewriter( if ((replacedOp = getReplacedOperationFrom(use))) break; fullReplacedOperation = replacedOp; + assert(fullReplacedOperation && + "expected replaced op to infer a result type from"); } else { replacedOp = fullReplacedOperation.getValue(); } - // Infer from the result, as there was no fully replaced op. - if (!replacedOp) { - for (OpOperand &use : result.getUses()) - if ((replacedOp = getReplacedOperationFrom(use))) - break; - assert(replacedOp && "expected replaced op to infer a result type from"); - } auto replOpOp = cast(replacedOp); types.push_back(mapRewriteValue(replOpOp.types()[it.index()])); diff --git a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h index b3919609a640..4d5c909465da 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h +++ b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h @@ -433,7 +433,7 @@ public: Position *getRoot() { return OperationPosition::getRoot(uniquer); } /// Returns the parent position defining the value held by the given operand. - Position *getParent(OperandPosition *p) { + OperationPosition *getParent(OperandPosition *p) { std::vector index = p->getIndex(); index.push_back(p->getOperandNumber()); return OperationPosition::get(uniquer, index); diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp index 22794aa4d991..0db35f050515 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp @@ -12,6 +12,7 @@ #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Interfaces/InferTypeOpInterface.h" +#include "llvm/ADT/TypeSwitch.h" using namespace mlir; using namespace mlir::pdl_to_pdl_interp; @@ -20,151 +21,181 @@ using namespace mlir::pdl_to_pdl_interp; // Predicate List Building //===----------------------------------------------------------------------===// +static void getTreePredicates(std::vector &predList, + Value val, PredicateBuilder &builder, + DenseMap &inputs, + Position *pos); + /// Compares the depths of two positions. static bool comparePosDepth(Position *lhs, Position *rhs) { return lhs->getIndex().size() < rhs->getIndex().size(); } -/// Collect the tree predicates anchored at the given value. static void getTreePredicates(std::vector &predList, Value val, PredicateBuilder &builder, DenseMap &inputs, - Position *pos) { - // Make sure this input value is accessible to the rewrite. - auto it = inputs.try_emplace(val, pos); + AttributePosition *pos) { + assert(val.getType().isa() && "expected attribute type"); + pdl::AttributeOp attr = cast(val.getDefiningOp()); + predList.emplace_back(pos, builder.getIsNotNull()); + + // If the attribute has a type or value, add a constraint. + if (Value type = attr.type()) + getTreePredicates(predList, type, builder, inputs, builder.getType(pos)); + else if (Attribute value = attr.valueAttr()) + predList.emplace_back(pos, builder.getAttributeConstraint(value)); +} - // If this is an input value that has been visited in the tree, add a - // constraint to ensure that both instances refer to the same value. - if (!it.second && - isa(val.getDefiningOp())) { - auto minMaxPositions = std::minmax(pos, it.first->second, comparePosDepth); - predList.emplace_back(minMaxPositions.second, - builder.getEqualTo(minMaxPositions.first)); - return; - } +static void getTreePredicates(std::vector &predList, + Value val, PredicateBuilder &builder, + DenseMap &inputs, + OperandPosition *pos) { + assert(val.getType().isa() && "expected value type"); - // Check for a per-position predicate to apply. - switch (pos->getKind()) { - case Predicates::AttributePos: { - assert(val.getType().isa() && - "expected attribute type"); - pdl::AttributeOp attr = cast(val.getDefiningOp()); - predList.emplace_back(pos, builder.getIsNotNull()); + // Prevent traversal into a null value. + predList.emplace_back(pos, builder.getIsNotNull()); - // If the attribute has a type, add a type constraint. - if (Value type = attr.type()) { + // If this is a typed operand, add a type constraint. + if (auto in = val.getDefiningOp()) { + if (Value type = in.type()) getTreePredicates(predList, type, builder, inputs, builder.getType(pos)); - // Check for a constant value of the attribute. - } else if (Optional value = attr.value()) { - predList.emplace_back(pos, builder.getAttributeConstraint(*value)); - } - break; + // Otherwise, recurse into a result node. + } else if (auto resultOp = val.getDefiningOp()) { + OperationPosition *parentPos = builder.getParent(pos); + Position *resultPos = builder.getResult(parentPos, resultOp.index()); + predList.emplace_back(parentPos, builder.getIsNotNull()); + predList.emplace_back(resultPos, builder.getEqualTo(pos)); + getTreePredicates(predList, resultOp.parent(), builder, inputs, parentPos); } - case Predicates::OperandPos: { - assert(val.getType().isa() && "expected value type"); +} + +static void getTreePredicates(std::vector &predList, + Value val, PredicateBuilder &builder, + DenseMap &inputs, + OperationPosition *pos) { + assert(val.getType().isa() && "expected operation"); + pdl::OperationOp op = cast(val.getDefiningOp()); + OperationPosition *opPos = cast(pos); - // Prevent traversal into a null value. + // Ensure getDefiningOp returns a non-null operation. + if (!opPos->isRoot()) predList.emplace_back(pos, builder.getIsNotNull()); - // If this is a typed operand, add a type constraint. - if (auto in = val.getDefiningOp()) { - if (Value type = in.type()) { - getTreePredicates(predList, type, builder, inputs, - builder.getType(pos)); - } - - // Otherwise, recurse into the parent node. - } else if (auto parentOp = val.getDefiningOp()) { - getTreePredicates(predList, parentOp.op(), builder, inputs, - builder.getParent(cast(pos))); - } - break; + // Check that this is the correct root operation. + if (Optional opName = op.name()) + predList.emplace_back(pos, builder.getOperationName(*opName)); + + // Check that the operation has the proper number of operands and results. + OperandRange operands = op.operands(); + OperandRange types = op.types(); + predList.emplace_back(pos, builder.getOperandCount(operands.size())); + predList.emplace_back(pos, builder.getResultCount(types.size())); + + // Recurse into any attributes, operands, or results. + for (auto it : llvm::zip(op.attributeNames(), op.attributes())) { + getTreePredicates( + predList, std::get<1>(it), builder, inputs, + builder.getAttribute(opPos, + std::get<0>(it).cast().getValue())); } - case Predicates::OperationPos: { - assert(val.getType().isa() && "expected operation"); - pdl::OperationOp op = cast(val.getDefiningOp()); - OperationPosition *opPos = cast(pos); - - // Ensure getDefiningOp returns a non-null operation. - if (!opPos->isRoot()) - predList.emplace_back(pos, builder.getIsNotNull()); - - // Check that this is the correct root operation. - if (Optional opName = op.name()) - predList.emplace_back(pos, builder.getOperationName(*opName)); - - // Check that the operation has the proper number of operands and results. - OperandRange operands = op.operands(); - ResultRange results = op.results(); - predList.emplace_back(pos, builder.getOperandCount(operands.size())); - predList.emplace_back(pos, builder.getResultCount(results.size())); - - // Recurse into any attributes, operands, or results. - for (auto it : llvm::zip(op.attributeNames(), op.attributes())) { - getTreePredicates( - predList, std::get<1>(it), builder, inputs, - builder.getAttribute(opPos, - std::get<0>(it).cast().getValue())); - } - for (auto operandIt : llvm::enumerate(operands)) - getTreePredicates(predList, operandIt.value(), builder, inputs, - builder.getOperand(opPos, operandIt.index())); - - // Only recurse into results that are not referenced in the source tree. - for (auto resultIt : llvm::enumerate(results)) { - getTreePredicates(predList, resultIt.value(), builder, inputs, - builder.getResult(opPos, resultIt.index())); - } - break; + for (auto operandIt : llvm::enumerate(operands)) { + getTreePredicates(predList, operandIt.value(), builder, inputs, + builder.getOperand(opPos, operandIt.index())); + } + for (auto &resultIt : llvm::enumerate(types)) { + auto *resultPos = builder.getResult(pos, resultIt.index()); + predList.emplace_back(resultPos, builder.getIsNotNull()); + getTreePredicates(predList, resultIt.value(), builder, inputs, + builder.getType(resultPos)); } - case Predicates::ResultPos: { - assert(val.getType().isa() && "expected value type"); - pdl::OperationOp parentOp = cast(val.getDefiningOp()); +} - // Prevent traversing a null value. - predList.emplace_back(pos, builder.getIsNotNull()); +static void getTreePredicates(std::vector &predList, + Value val, PredicateBuilder &builder, + DenseMap &inputs, + TypePosition *pos) { + assert(val.getType().isa() && "expected value type"); + pdl::TypeOp typeOp = cast(val.getDefiningOp()); - // Traverse the type constraint. - unsigned resultNo = cast(pos)->getResultNumber(); - getTreePredicates(predList, parentOp.types()[resultNo], builder, inputs, - builder.getType(pos)); - break; - } - case Predicates::TypePos: { - assert(val.getType().isa() && "expected value type"); - pdl::TypeOp typeOp = cast(val.getDefiningOp()); - - // Check for a constraint on a constant type. - if (Optional type = typeOp.type()) - predList.emplace_back(pos, builder.getTypeConstraint(*type)); - break; - } - default: - llvm_unreachable("unknown position kind"); + // Check for a constraint on a constant type. + if (Optional type = typeOp.type()) + predList.emplace_back(pos, builder.getTypeConstraint(*type)); +} + +/// Collect the tree predicates anchored at the given value. +static void getTreePredicates(std::vector &predList, + Value val, PredicateBuilder &builder, + DenseMap &inputs, + Position *pos) { + // Make sure this input value is accessible to the rewrite. + auto it = inputs.try_emplace(val, pos); + if (!it.second) { + // If this is an input value that has been visited in the tree, add a + // constraint to ensure that both instances refer to the same value. + if (isa( + val.getDefiningOp())) { + auto minMaxPositions = + std::minmax(pos, it.first->second, comparePosDepth); + predList.emplace_back(minMaxPositions.second, + builder.getEqualTo(minMaxPositions.first)); + } + return; } + + TypeSwitch(pos) + .Case([&](auto *derivedPos) { + getTreePredicates(predList, val, builder, inputs, derivedPos); + }) + .Default([](auto *) { llvm_unreachable("unexpected position kind"); }); } /// Collect all of the predicates related to constraints within the given /// pattern operation. -static void collectConstraintPredicates( - pdl::PatternOp pattern, std::vector &predList, - PredicateBuilder &builder, DenseMap &inputs) { - for (auto op : pattern.body().getOps()) { - OperandRange arguments = op.args(); - ArrayAttr parameters = op.constParamsAttr(); - - std::vector allPositions; - allPositions.reserve(arguments.size()); - for (Value arg : arguments) - allPositions.push_back(inputs.lookup(arg)); - - // Push the constraint to the furthest position. - Position *pos = *std::max_element(allPositions.begin(), allPositions.end(), - comparePosDepth); - PredicateBuilder::Predicate pred = - builder.getConstraint(op.name(), std::move(allPositions), parameters); - predList.emplace_back(pos, pred); +static void getConstraintPredicates(pdl::ApplyConstraintOp op, + std::vector &predList, + PredicateBuilder &builder, + DenseMap &inputs) { + OperandRange arguments = op.args(); + ArrayAttr parameters = op.constParamsAttr(); + + std::vector allPositions; + allPositions.reserve(arguments.size()); + for (Value arg : arguments) + allPositions.push_back(inputs.lookup(arg)); + + // Push the constraint to the furthest position. + Position *pos = *std::max_element(allPositions.begin(), allPositions.end(), + comparePosDepth); + PredicateBuilder::Predicate pred = + builder.getConstraint(op.name(), std::move(allPositions), parameters); + predList.emplace_back(pos, pred); +} + +static void getResultPredicates(pdl::ResultOp op, + std::vector &predList, + PredicateBuilder &builder, + DenseMap &inputs) { + Position *&resultPos = inputs[op]; + if (resultPos) + return; + auto *parentPos = cast(inputs.lookup(op.parent())); + resultPos = builder.getResult(parentPos, op.index()); + predList.emplace_back(resultPos, builder.getIsNotNull()); +} + +/// Collect all of the predicates that cannot be determined via walking the +/// tree. +static void getNonTreePredicates(pdl::PatternOp pattern, + std::vector &predList, + PredicateBuilder &builder, + DenseMap &inputs) { + for (Operation &op : pattern.body().getOps()) { + if (auto constraintOp = dyn_cast(&op)) + getConstraintPredicates(constraintOp, predList, builder, inputs); + else if (auto resultOp = dyn_cast(&op)) + getResultPredicates(resultOp, predList, builder, inputs); } } @@ -176,7 +207,7 @@ static void buildPredicateList(pdl::PatternOp pattern, DenseMap &valueToPosition) { getTreePredicates(predList, pattern.getRewriter().root(), builder, valueToPosition, builder.getRoot()); - collectConstraintPredicates(pattern, predList, builder, valueToPosition); + getNonTreePredicates(pattern, predList, builder, valueToPosition); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp index beb43d7072f2..d35aab41ba8f 100644 --- a/mlir/lib/Dialect/PDL/IR/PDL.cpp +++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp @@ -28,21 +28,36 @@ void PDLDialect::initialize() { registerTypes(); } +//===----------------------------------------------------------------------===// +// PDL Operations +//===----------------------------------------------------------------------===// + /// Returns true if the given operation is used by a "binding" pdl operation /// within the main matcher body of a `pdl.pattern`. +static bool hasBindingUseInMatcher(Operation *op, Block *matcherBlock) { + for (Operation *user : op->getUsers()) { + if (user->getBlock() != matcherBlock) + continue; + if (isa(user)) + return true; + // A result by itself is not binding, it must also be bound. + if (isa(user) && hasBindingUseInMatcher(user, matcherBlock)) + return true; + } + return false; +} + +/// Returns success if the given operation is used by a "binding" pdl operation +/// within the main matcher body of a `pdl.pattern`. On failure, emits an error +/// with the given context message. static LogicalResult verifyHasBindingUseInMatcher(Operation *op, StringRef bindableContextStr = "`pdl.operation`") { // If the pattern is not a pattern, there is nothing to do. if (!isa(op->getParentOp())) return success(); - Block *matcherBlock = op->getBlock(); - for (Operation *user : op->getUsers()) { - if (user->getBlock() != matcherBlock) - continue; - if (isa(user)) - return success(); - } + if (hasBindingUseInMatcher(op, op->getBlock())) + return success(); return op->emitOpError() << "expected a bindable (i.e. " << bindableContextStr << ") user when defined in the matcher body of a `pdl.pattern`"; @@ -86,37 +101,12 @@ static LogicalResult verify(OperandOp op) { // pdl::OperationOp //===----------------------------------------------------------------------===// -static ParseResult parseOperationOp(OpAsmParser &p, OperationState &state) { +static ParseResult parseOperationOpAttributes( + OpAsmParser &p, SmallVectorImpl &attrOperands, + ArrayAttr &attrNamesAttr) { Builder &builder = p.getBuilder(); - - // Parse the optional operation name. - bool startsWithOperands = succeeded(p.parseOptionalLParen()); - bool startsWithAttributes = - !startsWithOperands && succeeded(p.parseOptionalLBrace()); - bool startsWithOpName = false; - if (!startsWithAttributes && !startsWithOperands) { - StringAttr opName; - OptionalParseResult opNameResult = - p.parseOptionalAttribute(opName, "name", state.attributes); - startsWithOpName = opNameResult.hasValue(); - if (startsWithOpName && failed(*opNameResult)) - return failure(); - } - - // Parse the operands. - SmallVector operands; - if (startsWithOperands || - (!startsWithAttributes && succeeded(p.parseOptionalLParen()))) { - if (p.parseOperandList(operands) || p.parseRParen() || - p.resolveOperands(operands, builder.getType(), - state.operands)) - return failure(); - } - - // Parse the attributes. SmallVector attrNames; - if (startsWithAttributes || succeeded(p.parseOptionalLBrace())) { - SmallVector attrOps; + if (succeeded(p.parseOptionalLBrace())) { do { StringAttr nameAttr; OpAsmParser::OperandType operand; @@ -124,68 +114,29 @@ static ParseResult parseOperationOp(OpAsmParser &p, OperationState &state) { p.parseOperand(operand)) return failure(); attrNames.push_back(nameAttr); - attrOps.push_back(operand); + attrOperands.push_back(operand); } while (succeeded(p.parseOptionalComma())); - - if (p.parseRBrace() || - p.resolveOperands(attrOps, builder.getType(), - state.operands)) - return failure(); - } - state.addAttribute("attributeNames", builder.getArrayAttr(attrNames)); - state.addTypes(builder.getType()); - - // Parse the result types. - SmallVector opResultTypes; - if (succeeded(p.parseOptionalArrow())) { - if (p.parseOperandList(opResultTypes) || - p.resolveOperands(opResultTypes, builder.getType(), - state.operands)) + if (p.parseRBrace()) return failure(); - state.types.append(opResultTypes.size(), builder.getType()); } - - if (p.parseOptionalAttrDict(state.attributes)) - return failure(); - - int32_t operandSegmentSizes[] = {static_cast(operands.size()), - static_cast(attrNames.size()), - static_cast(opResultTypes.size())}; - state.addAttribute("operand_segment_sizes", - builder.getI32VectorAttr(operandSegmentSizes)); + attrNamesAttr = builder.getArrayAttr(attrNames); return success(); } -static void print(OpAsmPrinter &p, OperationOp op) { - p << "pdl.operation "; - if (Optional name = op.name()) - p << '"' << *name << '"'; - - auto operandValues = op.operands(); - if (!operandValues.empty()) - p << '(' << operandValues << ')'; - - // Emit the optional attributes. - ArrayAttr attrNames = op.attributeNames(); - if (!attrNames.empty()) { - Operation::operand_range attrArgs = op.attributes(); - p << " {"; - interleaveComma(llvm::seq(0, attrNames.size()), p, - [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; }); - p << '}'; - } - - // Print the result type constraints of the operation. - if (!op.results().empty()) - p << " -> " << op.types(); - p.printOptionalAttrDict(op->getAttrs(), - {"attributeNames", "name", "operand_segment_sizes"}); +static void printOperationOpAttributes(OpAsmPrinter &p, OperationOp op, + OperandRange attrArgs, + ArrayAttr attrNames) { + if (attrNames.empty()) + return; + p << " {"; + interleaveComma(llvm::seq(0, attrNames.size()), p, + [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; }); + p << '}'; } /// Verifies that the result types of this operation, defined within a /// `pdl.rewrite`, can be inferred. static LogicalResult verifyResultTypesAreInferrable(OperationOp op, - ResultRange opResults, OperandRange resultTypes) { // Functor that returns if the given use can be used to infer a type. Block *rewriterBlock = op->getBlock(); @@ -207,8 +158,8 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op, return success(); // Otherwise, make sure each of the types can be inferred. - for (int i : llvm::seq(0, opResults.size())) { - Operation *resultTypeOp = resultTypes[i].getDefiningOp(); + for (auto it : llvm::enumerate(resultTypes)) { + Operation *resultTypeOp = it.value().getDefiningOp(); assert(resultTypeOp && "expected valid result type operation"); // If the op was defined by a `create_native`, it is guaranteed to be @@ -229,14 +180,11 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op, if (llvm::any_of(typeOp.getResult().getUsers(), constrainsInputOp)) continue; - // Otherwise, check to see if any uses of the result can infer the type. - if (llvm::any_of(opResults[i].getUses(), canInferTypeFromUse)) - continue; return op .emitOpError("must have inferable or constrained result types when " "nested within `pdl.rewrite`") .attachNote() - .append("result type #", i, " was not constrained"); + .append("result type #", it.index(), " was not constrained"); } return success(); } @@ -256,19 +204,10 @@ static LogicalResult verify(OperationOp op) { << " values"; } - OperandRange resultTypes = op.types(); - auto opResults = op.results(); - if (resultTypes.size() != opResults.size()) { - return op.emitOpError() << "expected the same number of result values and " - "result type constraints, got " - << opResults.size() << " results and " - << resultTypes.size() << " constraints"; - } - // If the operation is within a rewrite body and doesn't have type inference, // ensure that the result types can be resolved. if (isWithinRewrite && !op.hasTypeInference()) { - if (failed(verifyResultTypesAreInferrable(op, opResults, resultTypes))) + if (failed(verifyResultTypesAreInferrable(op, op.types()))) return failure(); } @@ -341,37 +280,9 @@ Optional PatternOp::getRootKind() { //===----------------------------------------------------------------------===// static LogicalResult verify(ReplaceOp op) { - auto sourceOp = cast(op.operation().getDefiningOp()); - auto sourceOpResults = sourceOp.results(); - auto replValues = op.replValues(); - - if (Value replOpVal = op.replOperation()) { - auto replOp = cast(replOpVal.getDefiningOp()); - auto replOpResults = replOp.results(); - if (sourceOpResults.size() != replOpResults.size()) { - return op.emitOpError() - << "expected source operation to have the same number of results " - "as the replacement operation, replacement operation provided " - << replOpResults.size() << " but expected " - << sourceOpResults.size(); - } - - if (!replValues.empty()) { - return op.emitOpError() << "expected no replacement values to be provided" - " when the replacement operation is present"; - } - - return success(); - } - - if (sourceOpResults.size() != replValues.size()) { - return op.emitOpError() - << "expected source operation to have the same number of results " - "as the provided replacement values, found " - << replValues.size() << " replacement values but expected " - << sourceOpResults.size(); - } - + if (op.replOperation() && !op.replValues().empty()) + return op.emitOpError() << "expected no replacement values to be provided" + " when the replacement operation is present"; return success(); } diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir index 9d87ba5a21f0..c856ab5c9f6f 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir @@ -63,15 +63,16 @@ module @constraints { // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) // CHECK-DAG: %[[INPUT:.*]] = pdl_interp.get_operand 0 of %[[ROOT]] // CHECK-DAG: %[[INPUT1:.*]] = pdl_interp.get_operand 1 of %[[ROOT]] - // CHECK: pdl_interp.apply_constraint "multi_constraint" [true](%[[INPUT]], %[[INPUT1]] : !pdl.value, !pdl.value) + // CHECK-DAG: %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]] + // CHECK: pdl_interp.apply_constraint "multi_constraint" [true](%[[INPUT]], %[[INPUT1]], %[[RESULT]] pdl.pattern : benefit(1) { %input0 = pdl.operand %input1 = pdl.operand - - pdl.apply_constraint "multi_constraint"[true](%input0, %input1 : !pdl.value, !pdl.value) - %root = pdl.operation(%input0, %input1) + %result0 = pdl.result 0 of %root + + pdl.apply_constraint "multi_constraint"[true](%input0, %input1, %result0 : !pdl.value, !pdl.value, !pdl.value) pdl.rewrite %root with "rewriter" } } @@ -107,19 +108,52 @@ module @results { // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) // CHECK: pdl_interp.check_result_count of %[[ROOT]] is 2 - // Get the input and check the type. + // Get the result and check the type. // CHECK-DAG: %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]] // CHECK-DAG: pdl_interp.is_not_null %[[RESULT]] : !pdl.value // CHECK-DAG: %[[RESULT_TYPE:.*]] = pdl_interp.get_value_type of %[[RESULT]] // CHECK-DAG: pdl_interp.check_type %[[RESULT_TYPE]] is i32 - // Get the second operand and check that it is equal to the first. - // CHECK-DAG: %[[RESULT1:.*]] = pdl_interp.get_result 1 of %[[ROOT]] - // CHECK-NOT: pdl_interp.get_value_type of %[[RESULT1]] + // The second result doesn't have any constraints, so we don't generate an + // access for it. + // CHECK-NOT: pdl_interp.get_result 1 of %[[ROOT]] pdl.pattern : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %root, %results:2 = pdl.operation -> %type1, %type2 + %root = pdl.operation -> %type1, %type2 + pdl.rewrite %root with "rewriter" + } +} + +// ----- + +// CHECK-LABEL: module @results_as_operands +module @results_as_operands { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + + // Get the first result and check it matches the first operand. + // CHECK-DAG: %[[OPERAND_0:.*]] = pdl_interp.get_operand 0 of %[[ROOT]] + // CHECK-DAG: %[[DEF_OP_0:.*]] = pdl_interp.get_defining_op of %[[OPERAND_0]] + // CHECK-DAG: %[[RESULT_0:.*]] = pdl_interp.get_result 0 of %[[DEF_OP_0]] + // CHECK-DAG: pdl_interp.are_equal %[[RESULT_0]], %[[OPERAND_0]] + + // Get the second result and check it matches the second operand. + // CHECK-DAG: %[[OPERAND_1:.*]] = pdl_interp.get_operand 1 of %[[ROOT]] + // CHECK-DAG: %[[DEF_OP_1:.*]] = pdl_interp.get_defining_op of %[[OPERAND_1]] + // CHECK-DAG: %[[RESULT_1:.*]] = pdl_interp.get_result 1 of %[[DEF_OP_1]] + // CHECK-DAG: pdl_interp.are_equal %[[RESULT_1]], %[[OPERAND_1]] + + // Check that the parent operation of both results is the same. + // CHECK-DAG: pdl_interp.are_equal %[[DEF_OP_0]], %[[DEF_OP_1]] + + pdl.pattern : benefit(1) { + %type1 = pdl.type : i32 + %type2 = pdl.type + %inputOp = pdl.operation -> %type1, %type2 + %result1 = pdl.result 0 of %inputOp + %result2 = pdl.result 1 of %inputOp + + %root = pdl.operation(%result1, %result2) pdl.rewrite %root with "rewriter" } } @@ -134,12 +168,12 @@ module @switch_result_types { // CHECK: pdl_interp.switch_type %[[RESULT_TYPE]] to [i32, i64] pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root, %result = pdl.operation -> %type + %root = pdl.operation -> %type pdl.rewrite %root with "rewriter" } pdl.pattern : benefit(1) { %type = pdl.type : i64 - %root, %result = pdl.operation -> %type + %root = pdl.operation -> %type pdl.rewrite %root with "rewriter" } } @@ -161,13 +195,13 @@ module @predicate_ordering { pdl.pattern : benefit(1) { %resultType = pdl.type pdl.apply_constraint "typeConstraint"[](%resultType : !pdl.type) - %root, %result = pdl.operation -> %resultType + %root = pdl.operation -> %resultType pdl.rewrite %root with "rewriter" } pdl.pattern : benefit(1) { %resultType = pdl.type - %apply, %applyRes = pdl.operation -> %resultType + %apply = pdl.operation -> %resultType pdl.rewrite %apply with "rewriter" } } diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir index 4b6b1ae75700..5652b2118afe 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir @@ -63,7 +63,8 @@ module @operation_operands { %root = pdl.operation "foo.op"(%operand) pdl.rewrite %root { %type = pdl.type : i32 - %newOp, %result = pdl.operation "foo.op"(%operand) -> %type + %newOp = pdl.operation "foo.op"(%operand) -> %type + %result = pdl.result 0 of %newOp %newOp1 = pdl.operation "foo.op2"(%result) pdl.erase %root } @@ -84,7 +85,8 @@ module @operation_operands { %root = pdl.operation "foo.op"(%operand) pdl.rewrite %root { %type = pdl.type : i32 - %newOp, %result = pdl.operation "foo.op"(%operand) -> %type + %newOp = pdl.operation "foo.op"(%operand) -> %type + %result = pdl.result 0 of %newOp %newOp1 = pdl.operation "foo.op2"(%result) pdl.erase %root } @@ -101,10 +103,10 @@ module @operation_result_types { pdl.pattern : benefit(1) { %rootType = pdl.type %rootType1 = pdl.type - %root, %results:2 = pdl.operation "foo.op" -> %rootType, %rootType1 + %root = pdl.operation "foo.op" -> %rootType, %rootType1 pdl.rewrite %root { %newType1 = pdl.type - %newOp, %newResults:2 = pdl.operation "foo.op" -> %rootType, %newType1 + %newOp = pdl.operation "foo.op" -> %rootType, %newType1 pdl.replace %root with %newOp } } @@ -112,23 +114,6 @@ module @operation_result_types { // ----- -// CHECK-LABEL: module @operation_result_types_infer_from_value_replacement -module @operation_result_types_infer_from_value_replacement { - // CHECK: module @rewriters - // CHECK: func @pdl_generated_rewriter(%[[TYPE:.*]]: !pdl.type - // CHECK: pdl_interp.create_operation "foo.op"() -> %[[TYPE]] - pdl.pattern : benefit(1) { - %rootType = pdl.type - %root, %result = pdl.operation "foo.op" -> %rootType - pdl.rewrite %root { - %newType = pdl.type - %newOp, %newResult = pdl.operation "foo.op" -> %newType - pdl.replace %root with (%newResult) - } - } -} -// ----- - // CHECK-LABEL: module @replace_with_op module @replace_with_op { // CHECK: module @rewriters @@ -138,9 +123,9 @@ module @replace_with_op { // CHECK: pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]]) pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root, %result = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> %type pdl.rewrite %root { - %newOp, %newResult = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> %type pdl.replace %root with %newOp } } @@ -157,9 +142,10 @@ module @replace_with_values { // CHECK: pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]]) pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root, %result = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> %type pdl.rewrite %root { - %newOp, %newResult = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> %type + %newResult = pdl.result 0 of %newOp pdl.replace %root with (%newResult) } } @@ -192,10 +178,10 @@ module @create_native { // CHECK: pdl_interp.create_operation "foo.op"() -> %[[TYPE]] pdl.pattern : benefit(1) { %type = pdl.type - %root, %result = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> %type pdl.rewrite %root { %newType = pdl.create_native "functor"[true](%root : !pdl.operation) : !pdl.type - %newOp, %newResult = pdl.operation "foo.op" -> %newType + %newOp = pdl.operation "foo.op" -> %newType pdl.replace %root with %newOp } } diff --git a/mlir/test/Dialect/PDL/invalid.mlir b/mlir/test/Dialect/PDL/invalid.mlir index 0f4d96778277..0f900bbe3f53 100644 --- a/mlir/test/Dialect/PDL/invalid.mlir +++ b/mlir/test/Dialect/PDL/invalid.mlir @@ -24,7 +24,7 @@ pdl.pattern : benefit(1) { // expected-error@below {{expected only one of [`type`, `value`] to be set}} %attr = pdl.attribute : %type 10 - %op, %result = pdl.operation "foo.op" {"attr" = %attr} -> %type + %op = pdl.operation "foo.op" {"attr" = %attr} -> %type pdl.rewrite %op with "rewriter" } @@ -108,7 +108,7 @@ pdl.pattern : benefit(1) { // expected-error@below {{op must have inferable or constrained result types when nested within `pdl.rewrite`}} // expected-note@below {{result type #0 was not constrained}} - %newOp, %result = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> %type } } @@ -147,28 +147,12 @@ pdl.pattern : benefit(1) { // ----- -//===----------------------------------------------------------------------===// -// pdl::ReplaceOp -//===----------------------------------------------------------------------===// - -pdl.pattern : benefit(1) { - %root = pdl.operation "foo.op" - pdl.rewrite %root { - %type = pdl.type : i32 - %newOp, %newResult = pdl.operation "foo.op" -> %type - - // expected-error@below {{to have the same number of results as the replacement operation}} - pdl.replace %root with %newOp - } -} - -// ----- - pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root, %oldResult = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> %type pdl.rewrite %root { - %newOp, %newResult = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> %type + %newResult = pdl.result 0 of %newOp // expected-error@below {{expected no replacement values to be provided when the replacement operation is present}} "pdl.replace"(%root, %newOp, %newResult) { @@ -179,19 +163,6 @@ pdl.pattern : benefit(1) { // ----- -pdl.pattern : benefit(1) { - %root = pdl.operation "foo.op" - pdl.rewrite %root { - %type = pdl.type : i32 - %newOp, %newResult = pdl.operation "foo.op" -> %type - - // expected-error@below {{to have the same number of results as the provided replacement values}} - pdl.replace %root with (%newResult) - } -} - -// ----- - //===----------------------------------------------------------------------===// // pdl::RewriteOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/PDL/ops.mlir b/mlir/test/Dialect/PDL/ops.mlir index 5b6a642daf83..d376f001fcfa 100644 --- a/mlir/test/Dialect/PDL/ops.mlir +++ b/mlir/test/Dialect/PDL/ops.mlir @@ -8,7 +8,8 @@ pdl.pattern @operations : benefit(1) { // Operation with attributes and results. %attribute = pdl.attribute %type = pdl.type - %op0, %op0_result = pdl.operation {"attr" = %attribute} -> %type + %op0 = pdl.operation {"attr" = %attribute} -> %type + %op0_result = pdl.result 0 of %op0 // Operation with input. %input = pdl.operand @@ -46,38 +47,23 @@ pdl.pattern @rewrite_with_args_and_params : benefit(1) { pdl.pattern @infer_type_from_operation_replace : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %root, %results:2 = pdl.operation -> %type1, %type2 + %root = pdl.operation -> %type1, %type2 pdl.rewrite %root { %type3 = pdl.type - %newOp, %newResults:2 = pdl.operation "foo.op" -> %type1, %type3 + %newOp = pdl.operation "foo.op" -> %type1, %type3 pdl.replace %root with %newOp } } // ----- -// Check that the result type of an operation within a rewrite can be inferred -// from a pdl.replace. -pdl.pattern @infer_type_from_result_replace : benefit(1) { - %type1 = pdl.type : i32 - %type2 = pdl.type - %root, %results:2 = pdl.operation -> %type1, %type2 - pdl.rewrite %root { - %type3 = pdl.type - %newOp, %newResults:2 = pdl.operation "foo.op" -> %type1, %type3 - pdl.replace %root with (%newResults#0, %newResults#1) - } -} - -// ----- - // Check that the result type of an operation within a rewrite can be inferred // from a pdl.replace. pdl.pattern @infer_type_from_type_used_in_match : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %root, %results:2 = pdl.operation -> %type1, %type2 + %root = pdl.operation -> %type1, %type2 pdl.rewrite %root { - %newOp, %newResults:2 = pdl.operation "foo.op" -> %type1, %type2 + %newOp = pdl.operation "foo.op" -> %type1, %type2 } } -- GitLab From 02c4c0d5b2adc79c122bd2662a4458f75771aecf Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 16 Mar 2021 13:11:22 -0700 Subject: [PATCH 0072/1206] [mlir][pdl] Remove CreateNativeOp in favor of a more general ApplyNativeRewriteOp. This has a numerous amount of benefits, given the overly clunky nature of CreateNativeOp: * Users can now call into arbitrary rewrite functions from inside of PDL, allowing for more natural interleaving of PDL/C++ and enabling for more of the pattern to be in PDL. * Removes the need for an additional set of C++ functions/registry/etc. The new ApplyNativeRewriteOp will use the same PDLRewriteFunction as the existing RewriteOp. This reduces the API surface area exposed to users. This revision also introduces a new PDLResultList class. This class is used to provide results of native rewrite functions back to PDL. We introduce a new class instead of using a SmallVector to simplify the work necessary for variadics, given that ranges will require some changes to the structure of PDLValue. Differential Revision: https://reviews.llvm.org/D95720 --- mlir/include/mlir/Dialect/PDL/IR/PDLOps.td | 115 ++++++++++------- .../mlir/Dialect/PDLInterp/IR/PDLInterpOps.td | 53 ++------ mlir/include/mlir/IR/PatternMatch.h | 56 ++++---- .../PDLToPDLInterp/PDLToPDLInterp.cpp | 41 +++--- .../PDLToPDLInterp/PredicateTree.cpp | 4 +- mlir/lib/Dialect/PDL/IR/PDL.cpp | 18 ++- mlir/lib/IR/PatternMatch.cpp | 11 +- mlir/lib/Rewrite/ByteCode.cpp | 122 ++++++++---------- mlir/lib/Rewrite/ByteCode.h | 2 - mlir/lib/Rewrite/FrozenRewritePatternList.cpp | 2 +- .../pdl-to-pdl-interp-matcher.mlir | 6 +- .../pdl-to-pdl-interp-rewriter.mlir | 10 +- mlir/test/Dialect/PDL/invalid.mlir | 18 ++- mlir/test/Rewrite/pdl-bytecode.mlir | 63 +++++---- mlir/test/lib/Rewrite/TestPDLByteCode.cpp | 18 +-- 15 files changed, 267 insertions(+), 272 deletions(-) diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td index 76e4c5d022a4..74f3fce08933 100644 --- a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td +++ b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td @@ -29,17 +29,17 @@ class PDL_Op traits = []> } //===----------------------------------------------------------------------===// -// pdl::ApplyConstraintOp +// pdl::ApplyNativeConstraintOp //===----------------------------------------------------------------------===// -def PDL_ApplyConstraintOp - : PDL_Op<"apply_constraint", [HasParent<"pdl::PatternOp">]> { - let summary = "Apply a generic constraint to a set of provided entities"; +def PDL_ApplyNativeConstraintOp + : PDL_Op<"apply_native_constraint", [HasParent<"pdl::PatternOp">]> { + let summary = "Apply a native constraint to a set of provided entities"; let description = [{ - `apply_constraint` operations apply a generic constraint, that has been - registered externally with the consumer of PDL, to a given set of entities. - The constraint is permitted to accept any number of constant valued - parameters. + `pdl.apply_native_constraint` operations apply a native C++ constraint, that + has been registered externally with the consumer of PDL, to a given set of + entities. The constraint is permitted to accept any number of constant + valued parameters. Example: @@ -47,7 +47,7 @@ def PDL_ApplyConstraintOp // Apply `myConstraint` to the entities defined by `input`, `attr`, and // `op`. `42`, `"abc"`, and `i32` are constant parameters passed to the // constraint. - pdl.apply_constraint "myConstraint"[42, "abc", i32](%input, %attr, %op : !pdl.value, !pdl.attribute, !pdl.operation) + pdl.apply_native_constraint "myConstraint"[42, "abc", i32](%input, %attr, %op : !pdl.value, !pdl.attribute, !pdl.operation) ``` }]; @@ -67,6 +67,58 @@ def PDL_ApplyConstraintOp ]; } +//===----------------------------------------------------------------------===// +// pdl::ApplyNativeRewriteOp +//===----------------------------------------------------------------------===// + +def PDL_ApplyNativeRewriteOp + : PDL_Op<"apply_native_rewrite", [HasParent<"pdl::RewriteOp">]> { + let summary = "Apply a native rewrite method inside of pdl.rewrite region"; + let description = [{ + `pdl.apply_native_rewrite` operations apply a native C++ function, that has + been registered externally with the consumer of PDL, to perform a rewrite + and optionally return a number of values. The native function may accept any + number of arguments and constant attribute parameters. This operation is + used within a pdl.rewrite region to enable the interleaving of native + rewrite methods with other pdl constructs. + + Example: + + ```mlir + // Apply a native rewrite method that returns an attribute. + %ret = pdl.apply_native_rewrite "myNativeFunc"[42, "gt"](%arg0, %arg1) : !pdl.attribute + ``` + + ```c++ + // The native rewrite as defined in C++: + static void myNativeFunc(ArrayRef args, ArrayAttr constantParams, + PatternRewriter &rewriter, + PDLResultList &results) { + Value arg0 = args[0].cast(); + Value arg1 = args[1].cast(); + IntegerAttr param0 = constantParams[0].cast(); + StringAttr param1 = constantParams[1].cast(); + + // Just push back the first param attribute. + results.push_back(param0); + } + + void registerNativeRewrite(PDLPatternModule &pdlModule) { + pdlModule.registerRewriteFunction("myNativeFunc", myNativeFunc); + } + ``` + }]; + + let arguments = (ins StrAttr:$name, + Variadic:$args, + OptionalAttr:$constParams); + let results = (outs Variadic:$results); + let assemblyFormat = [{ + $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `:` type($results) + attr-dict + }]; +} + //===----------------------------------------------------------------------===// // pdl::AttributeOp //===----------------------------------------------------------------------===// @@ -113,39 +165,6 @@ def PDL_AttributeOp : PDL_Op<"attribute"> { ]; } -//===----------------------------------------------------------------------===// -// pdl::CreateNativeOp -//===----------------------------------------------------------------------===// - -def PDL_CreateNativeOp - : PDL_Op<"create_native", [HasParent<"pdl::RewriteOp">]> { - let summary = "Call a native creation method to construct an `Attribute`, " - "`Operation`, `Type`, or `Value`"; - let description = [{ - `pdl.create_native` operations invoke a native C++ function, that has been - registered externally with the consumer of PDL, to create an `Attribute`, - `Operation`, `Type`, or `Value`. The native function must produce a value - of the specified return type, and may accept any number of positional - arguments and constant attribute parameters. - - Example: - - ```mlir - %ret = pdl.create_native "myNativeFunc"[42, "gt"](%arg0, %arg1) : !pdl.attribute - ``` - }]; - - let arguments = (ins StrAttr:$name, - Variadic:$args, - OptionalAttr:$constParams); - let results = (outs PDL_AnyType:$result); - let assemblyFormat = [{ - $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `:` type($result) - attr-dict - }]; - let verifier = ?; -} - //===----------------------------------------------------------------------===// // pdl::EraseOp //===----------------------------------------------------------------------===// @@ -233,9 +252,10 @@ def PDL_OperationOp `pdl.rewrite`, all of the result types must be "inferable". This means that the type must be attributable to either a constant type value or the result type of another entity, such as an attribute, the result of a - `createNative`, or the result type of another operation. If the result type - value does not meet any of these criteria, the operation must provide the - `InferTypeOpInterface` to ensure that the result types can be inferred. + `apply_native_rewrite`, or the result type of another operation. If the + result type value does not meet any of these criteria, the operation must + override the `InferTypeOpInterface` to ensure that the result types can be + inferred. Example: @@ -416,13 +436,14 @@ def PDL_RewriteOp : PDL_Op<"rewrite", [ let summary = "Specify the rewrite of a matched pattern"; let description = [{ `pdl.rewrite` operations terminate the region of a `pdl.pattern` and specify - the rewrite of a `pdl.pattern`, on the specified root operation. The + the main rewrite of a `pdl.pattern`, on the specified root operation. The rewrite is specified either via a string name (`name`) to an external rewrite function, or via the region body. The rewrite region, if specified, must contain a single block and terminate via the `pdl.rewrite_end` operation. If the rewrite is external, it also takes a set of constant parameters and a set of additional positional values defined within the - matcher as arguments. + matcher as arguments. If the rewrite is external, the root operation is + passed to the native function as the first argument. Example: diff --git a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td index 517a0f4f0af0..8f8a5b130175 100644 --- a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td +++ b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td @@ -130,32 +130,35 @@ def PDLInterp_ApplyRewriteOp : PDLInterp_Op<"apply_rewrite"> { let description = [{ `pdl_interp.apply_rewrite` operations invoke an external rewriter that has been registered with the interpreter to perform the rewrite after a - successful match. The rewrite is passed the root operation being matched, a - set of additional positional arguments generated within the matcher, and a - set of constant parameters. + successful match. The rewrite is passed a set of positional arguments, + and a set of constant parameters. The rewrite function may return any + number of results. Example: ```mlir // Rewriter operating solely on the root operation. - pdl_interp.apply_rewrite "rewriter" on %root + pdl_interp.apply_rewrite "rewriter"(%root : !pdl.operation) + + // Rewriter operating solely on the root operation and return an attribute. + %attr = pdl_interp.apply_rewrite "rewriter"(%root : !pdl.operation) : !pdl.attribute // Rewriter operating on the root operation along with additional arguments // from the matcher. - pdl_interp.apply_rewrite "rewriter"(%value : !pdl.value) on %root + pdl_interp.apply_rewrite "rewriter"(%root : !pdl.operation, %value : !pdl.value) // Rewriter operating on the root operation along with additional arguments // and constant parameters. - pdl_interp.apply_rewrite "rewriter"[42](%value : !pdl.value) on %root + pdl_interp.apply_rewrite "rewriter"[42](%root : !pdl.operation, %value : !pdl.value) ``` }]; let arguments = (ins StrAttr:$name, - PDL_Operation:$root, Variadic:$args, OptionalAttr:$constParams); + let results = (outs Variadic:$results); let assemblyFormat = [{ - $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `on` $root - attr-dict + $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? + (`:` type($results)^)? attr-dict }]; } @@ -351,38 +354,6 @@ def PDLInterp_CreateAttributeOp }]>]; } -//===----------------------------------------------------------------------===// -// pdl_interp::CreateNativeOp -//===----------------------------------------------------------------------===// - -def PDLInterp_CreateNativeOp : PDLInterp_Op<"create_native"> { - let summary = "Call a native creation method to construct an `Attribute`, " - "`Operation`, `Type`, or `Value`"; - let description = [{ - `pdl_interp.create_native` operations invoke a native C++ function, that has - been registered externally with the consumer of PDL, to create an - `Attribute`, `Operation`, `Type`, or `Value`. The native function must - produce a value of the specified return type, and may accept any number of - positional arguments and constant attribute parameters. - - Example: - - ```mlir - %ret = pdl_interp.create_native "myNativeFunc"[42, "gt"](%arg0, %arg1 : !pdl.value, !pdl.value) : !pdl.attribute - ``` - }]; - - let arguments = (ins StrAttr:$name, - Variadic:$args, - OptionalAttr:$constParams); - let results = (outs PDL_AnyType:$result); - let assemblyFormat = [{ - $name ($constParams^)? (`(` $args^ `:` type($args) `)`)? `:` type($result) - attr-dict - }]; - let verifier = ?; -} - //===----------------------------------------------------------------------===// // pdl_interp::CreateOperationOp //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h index 8e1a5b98c318..56da9b870948 100644 --- a/mlir/include/mlir/IR/PatternMatch.h +++ b/mlir/include/mlir/IR/PatternMatch.h @@ -302,6 +302,33 @@ inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) { return os; } +//===----------------------------------------------------------------------===// +// PDLResultList + +/// The class represents a list of PDL results, returned by a native rewrite +/// method. It provides the mechanism with which to pass PDLValues back to the +/// PDL bytecode. +class PDLResultList { +public: + /// Push a new Attribute value onto the result list. + void push_back(Attribute value) { results.push_back(value); } + + /// Push a new Operation onto the result list. + void push_back(Operation *value) { results.push_back(value); } + + /// Push a new Type onto the result list. + void push_back(Type value) { results.push_back(value); } + + /// Push a new Value onto the result list. + void push_back(Value value) { results.push_back(value); } + +protected: + PDLResultList() = default; + + /// The PDL results held by this list. + SmallVector results; +}; + //===----------------------------------------------------------------------===// // PDLPatternModule @@ -311,16 +338,13 @@ inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) { /// success if the constraint successfully held, failure otherwise. using PDLConstraintFunction = std::function, ArrayAttr, PatternRewriter &)>; -/// A native PDL creation function. This function creates a new PDLValue given -/// a set of existing PDL values, a set of constant parameters specified in -/// Attribute form, and a PatternRewriter. Returns the newly created PDLValue. -using PDLCreateFunction = - std::function, ArrayAttr, PatternRewriter &)>; -/// A native PDL rewrite function. This function rewrites the given root -/// operation using the provided PatternRewriter. This method is only invoked -/// when the corresponding match was successful. -using PDLRewriteFunction = std::function, - ArrayAttr, PatternRewriter &)>; +/// A native PDL rewrite function. This function performs a rewrite on the +/// given set of values and constant parameters. Any results from this rewrite +/// that should be passed back to PDL should be added to the provided result +/// list. This method is only invoked when the corresponding match was +/// successful. +using PDLRewriteFunction = std::function, ArrayAttr, PatternRewriter &, PDLResultList &)>; /// A generic PDL pattern constraint function. This function applies a /// constraint to a given opaque PDLValue entity. The second parameter is a set /// of constant value parameters specified in Attribute form. Returns success if @@ -367,9 +391,6 @@ public: }); } - /// Register a creation function. - void registerCreateFunction(StringRef name, PDLCreateFunction createFn); - /// Register a rewrite function. void registerRewriteFunction(StringRef name, PDLRewriteFunction rewriteFn); @@ -380,13 +401,6 @@ public: llvm::StringMap takeConstraintFunctions() { return constraintFunctions; } - /// Return the set of the registered create functions. - const llvm::StringMap &getCreateFunctions() const { - return createFunctions; - } - llvm::StringMap takeCreateFunctions() { - return createFunctions; - } /// Return the set of the registered rewrite functions. const llvm::StringMap &getRewriteFunctions() const { return rewriteFunctions; @@ -399,7 +413,6 @@ public: void clear() { pdlModule = nullptr; constraintFunctions.clear(); - createFunctions.clear(); rewriteFunctions.clear(); } @@ -409,7 +422,6 @@ private: /// The external functions referenced from within the PDL module. llvm::StringMap constraintFunctions; - llvm::StringMap createFunctions; llvm::StringMap rewriteFunctions; }; diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp index 3368ceb9be88..d1da22671d95 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp @@ -70,6 +70,9 @@ private: SmallVectorImpl &usedMatchValues); /// Generate the rewriter code for the given operation. + void generateRewriter(pdl::ApplyNativeRewriteOp rewriteOp, + DenseMap &rewriteValues, + function_ref mapRewriteValue); void generateRewriter(pdl::AttributeOp attrOp, DenseMap &rewriteValues, function_ref mapRewriteValue); @@ -79,9 +82,6 @@ private: void generateRewriter(pdl::OperationOp operationOp, DenseMap &rewriteValues, function_ref mapRewriteValue); - void generateRewriter(pdl::CreateNativeOp createNativeOp, - DenseMap &rewriteValues, - function_ref mapRewriteValue); void generateRewriter(pdl::ReplaceOp replaceOp, DenseMap &rewriteValues, function_ref mapRewriteValue); @@ -449,17 +449,17 @@ SymbolRefAttr PatternLowering::generateRewriter( // method. pdl::RewriteOp rewriter = pattern.getRewriter(); if (StringAttr rewriteName = rewriter.nameAttr()) { - Value root = mapRewriteValue(rewriter.root()); - SmallVector args = llvm::to_vector<4>( - llvm::map_range(rewriter.externalArgs(), mapRewriteValue)); + auto mappedArgs = llvm::map_range(rewriter.externalArgs(), mapRewriteValue); + SmallVector args(1, mapRewriteValue(rewriter.root())); + args.append(mappedArgs.begin(), mappedArgs.end()); builder.create( - rewriter.getLoc(), rewriteName, root, args, + rewriter.getLoc(), /*resultTypes=*/TypeRange(), rewriteName, args, rewriter.externalConstParamsAttr()); } else { // Otherwise this is a dag rewriter defined using PDL operations. for (Operation &rewriteOp : *rewriter.getBody()) { llvm::TypeSwitch(&rewriteOp) - .Case( [&](auto op) { this->generateRewriter(op, rewriteValues, mapRewriteValue); @@ -478,6 +478,19 @@ SymbolRefAttr PatternLowering::generateRewriter( builder.getSymbolRefAttr(rewriterFunc)); } +void PatternLowering::generateRewriter( + pdl::ApplyNativeRewriteOp rewriteOp, DenseMap &rewriteValues, + function_ref mapRewriteValue) { + SmallVector arguments; + for (Value argument : rewriteOp.args()) + arguments.push_back(mapRewriteValue(argument)); + auto interpOp = builder.create( + rewriteOp.getLoc(), rewriteOp.getResultTypes(), rewriteOp.nameAttr(), + arguments, rewriteOp.constParamsAttr()); + for (auto it : llvm::zip(rewriteOp.results(), interpOp.results())) + rewriteValues[std::get<0>(it)] = std::get<1>(it); +} + void PatternLowering::generateRewriter( pdl::AttributeOp attrOp, DenseMap &rewriteValues, function_ref mapRewriteValue) { @@ -527,18 +540,6 @@ void PatternLowering::generateRewriter( } } -void PatternLowering::generateRewriter( - pdl::CreateNativeOp createNativeOp, DenseMap &rewriteValues, - function_ref mapRewriteValue) { - SmallVector arguments; - for (Value argument : createNativeOp.args()) - arguments.push_back(mapRewriteValue(argument)); - Value result = builder.create( - createNativeOp.getLoc(), createNativeOp.result().getType(), - createNativeOp.nameAttr(), arguments, createNativeOp.constParamsAttr()); - rewriteValues[createNativeOp] = result; -} - void PatternLowering::generateRewriter( pdl::ReplaceOp replaceOp, DenseMap &rewriteValues, function_ref mapRewriteValue) { diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp index 0db35f050515..885fbad0f976 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp @@ -153,7 +153,7 @@ static void getTreePredicates(std::vector &predList, /// Collect all of the predicates related to constraints within the given /// pattern operation. -static void getConstraintPredicates(pdl::ApplyConstraintOp op, +static void getConstraintPredicates(pdl::ApplyNativeConstraintOp op, std::vector &predList, PredicateBuilder &builder, DenseMap &inputs) { @@ -192,7 +192,7 @@ static void getNonTreePredicates(pdl::PatternOp pattern, PredicateBuilder &builder, DenseMap &inputs) { for (Operation &op : pattern.body().getOps()) { - if (auto constraintOp = dyn_cast(&op)) + if (auto constraintOp = dyn_cast(&op)) getConstraintPredicates(constraintOp, predList, builder, inputs); else if (auto resultOp = dyn_cast(&op)) getResultPredicates(resultOp, predList, builder, inputs); diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp index d35aab41ba8f..dc1f501825bd 100644 --- a/mlir/lib/Dialect/PDL/IR/PDL.cpp +++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp @@ -64,15 +64,25 @@ verifyHasBindingUseInMatcher(Operation *op, } //===----------------------------------------------------------------------===// -// pdl::ApplyConstraintOp +// pdl::ApplyNativeConstraintOp //===----------------------------------------------------------------------===// -static LogicalResult verify(ApplyConstraintOp op) { +static LogicalResult verify(ApplyNativeConstraintOp op) { if (op.getNumOperands() == 0) return op.emitOpError("expected at least one argument"); return success(); } +//===----------------------------------------------------------------------===// +// pdl::ApplyNativeRewriteOp +//===----------------------------------------------------------------------===// + +static LogicalResult verify(ApplyNativeRewriteOp op) { + if (op.getNumOperands() == 0 && op.getNumResults() == 0) + return op.emitOpError("expected at least one argument or result"); + return success(); +} + //===----------------------------------------------------------------------===// // pdl::AttributeOp //===----------------------------------------------------------------------===// @@ -162,9 +172,9 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op, Operation *resultTypeOp = it.value().getDefiningOp(); assert(resultTypeOp && "expected valid result type operation"); - // If the op was defined by a `create_native`, it is guaranteed to be + // If the op was defined by a `apply_native_rewrite`, it is guaranteed to be // usable. - if (isa(resultTypeOp)) + if (isa(resultTypeOp)) continue; // If the type is already constrained, there is nothing to do. diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp index 90e89a536405..034698d85cb1 100644 --- a/mlir/lib/IR/PatternMatch.cpp +++ b/mlir/lib/IR/PatternMatch.cpp @@ -102,7 +102,6 @@ void PDLPatternModule::mergeIn(PDLPatternModule &&other) { // Steal the other state if we have no patterns. if (!pdlModule) { constraintFunctions = std::move(other.constraintFunctions); - createFunctions = std::move(other.createFunctions); rewriteFunctions = std::move(other.rewriteFunctions); pdlModule = std::move(other.pdlModule); return; @@ -110,8 +109,6 @@ void PDLPatternModule::mergeIn(PDLPatternModule &&other) { // Steal the functions of the other module. for (auto &it : constraintFunctions) registerConstraintFunction(it.first(), std::move(it.second)); - for (auto &it : createFunctions) - registerCreateFunction(it.first(), std::move(it.second)); for (auto &it : rewriteFunctions) registerRewriteFunction(it.first(), std::move(it.second)); @@ -132,13 +129,7 @@ void PDLPatternModule::registerConstraintFunction( assert(it.second && "constraint with the given name has already been registered"); } -void PDLPatternModule::registerCreateFunction(StringRef name, - PDLCreateFunction createFn) { - auto it = createFunctions.try_emplace(name, std::move(createFn)); - (void)it; - assert(it.second && "native create function with the given name has " - "already been registered"); -} + void PDLPatternModule::registerRewriteFunction(StringRef name, PDLRewriteFunction rewriteFn) { auto it = rewriteFunctions.try_emplace(name, std::move(rewriteFn)); diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp index 1986b3f87d96..c09892caec1b 100644 --- a/mlir/lib/Rewrite/ByteCode.cpp +++ b/mlir/lib/Rewrite/ByteCode.cpp @@ -80,8 +80,6 @@ enum OpCode : ByteCodeField { CheckOperationName, /// Compare the result count of an operation with a constant. CheckResultCount, - /// Invoke a native creation method. - CreateNative, /// Create an operation. CreateOperation, /// Erase an operation. @@ -148,15 +146,12 @@ public: SmallVectorImpl &patterns, ByteCodeField &maxValueMemoryIndex, llvm::StringMap &constraintFns, - llvm::StringMap &createFns, llvm::StringMap &rewriteFns) : ctx(ctx), uniquedData(uniquedData), matcherByteCode(matcherByteCode), rewriterByteCode(rewriterByteCode), patterns(patterns), maxValueMemoryIndex(maxValueMemoryIndex) { for (auto it : llvm::enumerate(constraintFns)) constraintToMemIndex.try_emplace(it.value().first(), it.index()); - for (auto it : llvm::enumerate(createFns)) - nativeCreateToMemIndex.try_emplace(it.value().first(), it.index()); for (auto it : llvm::enumerate(rewriteFns)) externalRewriterToMemIndex.try_emplace(it.value().first(), it.index()); } @@ -203,7 +198,6 @@ private: void generate(pdl_interp::CheckResultCountOp op, ByteCodeWriter &writer); void generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer); void generate(pdl_interp::CreateAttributeOp op, ByteCodeWriter &writer); - void generate(pdl_interp::CreateNativeOp op, ByteCodeWriter &writer); void generate(pdl_interp::CreateOperationOp op, ByteCodeWriter &writer); void generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer); void generate(pdl_interp::EraseOp op, ByteCodeWriter &writer); @@ -235,10 +229,6 @@ private: /// in the bytecode registry. llvm::StringMap constraintToMemIndex; - /// Mapping from the name of an externally registered creation method to its - /// index in the bytecode registry. - llvm::StringMap nativeCreateToMemIndex; - /// Mapping from rewriter function name to the bytecode address of the /// rewriter function in byte. llvm::StringMap rewriterToAddr; @@ -492,16 +482,16 @@ void Generator::generate(Operation *op, ByteCodeWriter &writer) { pdl_interp::CheckAttributeOp, pdl_interp::CheckOperandCountOp, pdl_interp::CheckOperationNameOp, pdl_interp::CheckResultCountOp, pdl_interp::CheckTypeOp, pdl_interp::CreateAttributeOp, - pdl_interp::CreateNativeOp, pdl_interp::CreateOperationOp, - pdl_interp::CreateTypeOp, pdl_interp::EraseOp, - pdl_interp::FinalizeOp, pdl_interp::GetAttributeOp, - pdl_interp::GetAttributeTypeOp, pdl_interp::GetDefiningOpOp, - pdl_interp::GetOperandOp, pdl_interp::GetResultOp, - pdl_interp::GetValueTypeOp, pdl_interp::InferredTypeOp, - pdl_interp::IsNotNullOp, pdl_interp::RecordMatchOp, - pdl_interp::ReplaceOp, pdl_interp::SwitchAttributeOp, - pdl_interp::SwitchTypeOp, pdl_interp::SwitchOperandCountOp, - pdl_interp::SwitchOperationNameOp, pdl_interp::SwitchResultCountOp>( + pdl_interp::CreateOperationOp, pdl_interp::CreateTypeOp, + pdl_interp::EraseOp, pdl_interp::FinalizeOp, + pdl_interp::GetAttributeOp, pdl_interp::GetAttributeTypeOp, + pdl_interp::GetDefiningOpOp, pdl_interp::GetOperandOp, + pdl_interp::GetResultOp, pdl_interp::GetValueTypeOp, + pdl_interp::InferredTypeOp, pdl_interp::IsNotNullOp, + pdl_interp::RecordMatchOp, pdl_interp::ReplaceOp, + pdl_interp::SwitchAttributeOp, pdl_interp::SwitchTypeOp, + pdl_interp::SwitchOperandCountOp, pdl_interp::SwitchOperationNameOp, + pdl_interp::SwitchResultCountOp>( [&](auto interpOp) { this->generate(interpOp, writer); }) .Default([](Operation *) { llvm_unreachable("unknown `pdl_interp` operation"); @@ -522,8 +512,16 @@ void Generator::generate(pdl_interp::ApplyRewriteOp op, assert(externalRewriterToMemIndex.count(op.name()) && "expected index for rewrite function"); writer.append(OpCode::ApplyRewrite, externalRewriterToMemIndex[op.name()], - op.constParamsAttr(), op.root()); + op.constParamsAttr()); writer.appendPDLValueList(op.args()); + +#ifndef NDEBUG + // In debug mode we also append the number of results so that we can assert + // that the native creation function gave us the correct number of results. + writer.append(ByteCodeField(op.results().size())); +#endif + for (Value result : op.results()) + writer.append(result); } void Generator::generate(pdl_interp::AreEqualOp op, ByteCodeWriter &writer) { writer.append(OpCode::AreEqual, op.lhs(), op.rhs(), op.getSuccessors()); @@ -559,14 +557,6 @@ void Generator::generate(pdl_interp::CreateAttributeOp op, // Simply repoint the memory index of the result to the constant. getMemIndex(op.attribute()) = getMemIndex(op.value()); } -void Generator::generate(pdl_interp::CreateNativeOp op, - ByteCodeWriter &writer) { - assert(nativeCreateToMemIndex.count(op.name()) && - "expected index for creation function"); - writer.append(OpCode::CreateNative, nativeCreateToMemIndex[op.name()], - op.result(), op.constParamsAttr()); - writer.appendPDLValueList(op.args()); -} void Generator::generate(pdl_interp::CreateOperationOp op, ByteCodeWriter &writer) { writer.append(OpCode::CreateOperation, op.operation(), @@ -678,18 +668,15 @@ void Generator::generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer) { PDLByteCode::PDLByteCode(ModuleOp module, llvm::StringMap constraintFns, - llvm::StringMap createFns, llvm::StringMap rewriteFns) { Generator generator(module.getContext(), uniquedData, matcherByteCode, rewriterByteCode, patterns, maxValueMemoryIndex, - constraintFns, createFns, rewriteFns); + constraintFns, rewriteFns); generator.generate(module); // Initialize the external functions. for (auto &it : constraintFns) constraintFunctions.push_back(std::move(it.second)); - for (auto &it : createFns) - createFunctions.push_back(std::move(it.second)); for (auto &it : rewriteFns) rewriteFunctions.push_back(std::move(it.second)); } @@ -717,12 +704,11 @@ public: ArrayRef currentPatternBenefits, ArrayRef patterns, ArrayRef constraintFunctions, - ArrayRef createFunctions, ArrayRef rewriteFunctions) : curCodeIt(curCodeIt), memory(memory), uniquedMemory(uniquedMemory), code(code), currentPatternBenefits(currentPatternBenefits), patterns(patterns), constraintFunctions(constraintFunctions), - createFunctions(createFunctions), rewriteFunctions(rewriteFunctions) {} + rewriteFunctions(rewriteFunctions) {} /// Start executing the code at the current bytecode index. `matches` is an /// optional field provided when this function is executed in a matching @@ -740,7 +726,6 @@ private: void executeCheckOperandCount(); void executeCheckOperationName(); void executeCheckResultCount(); - void executeCreateNative(PatternRewriter &rewriter); void executeCreateOperation(PatternRewriter &rewriter, Location mainRewriteLoc); void executeEraseOp(PatternRewriter &rewriter); @@ -866,9 +851,17 @@ private: ArrayRef currentPatternBenefits; ArrayRef patterns; ArrayRef constraintFunctions; - ArrayRef createFunctions; ArrayRef rewriteFunctions; }; + +/// This class is an instantiation of the PDLResultList that provides access to +/// the returned results. This API is not on `PDLResultList` to avoid +/// overexposing access to information specific solely to the ByteCode. +class ByteCodeRewriteResultList : public PDLResultList { +public: + /// Return the list of PDL results. + MutableArrayRef getResults() { return results; } +}; } // end anonymous namespace void ByteCodeExecutor::executeApplyConstraint(PatternRewriter &rewriter) { @@ -892,18 +885,29 @@ void ByteCodeExecutor::executeApplyRewrite(PatternRewriter &rewriter) { LLVM_DEBUG(llvm::dbgs() << "Executing ApplyRewrite:\n"); const PDLRewriteFunction &rewriteFn = rewriteFunctions[read()]; ArrayAttr constParams = read(); - Operation *root = read(); SmallVector args; readList(args); LLVM_DEBUG({ - llvm::dbgs() << " * Root: " << *root << "\n * Arguments: "; + llvm::dbgs() << " * Arguments: "; llvm::interleaveComma(args, llvm::dbgs()); llvm::dbgs() << "\n * Parameters: " << constParams << "\n"; }); - - // Invoke the native rewrite function. - rewriteFn(root, args, constParams, rewriter); + ByteCodeRewriteResultList results; + rewriteFn(args, constParams, rewriter, results); + + // Store the results in the bytecode memory. +#ifndef NDEBUG + ByteCodeField expectedNumberOfResults = read(); + assert(results.getResults().size() == expectedNumberOfResults && + "native PDL rewrite function returned unexpected number of results"); +#endif + + // Store the results in the bytecode memory. + for (PDLValue &result : results.getResults()) { + LLVM_DEBUG(llvm::dbgs() << " * Result: " << result << "\n"); + memory[read()] = result.getAsOpaquePointer(); + } } void ByteCodeExecutor::executeAreEqual() { @@ -950,26 +954,6 @@ void ByteCodeExecutor::executeCheckResultCount() { selectJump(op->getNumResults() == expectedCount); } -void ByteCodeExecutor::executeCreateNative(PatternRewriter &rewriter) { - LLVM_DEBUG(llvm::dbgs() << "Executing CreateNative:\n"); - const PDLCreateFunction &createFn = createFunctions[read()]; - ByteCodeField resultIndex = read(); - ArrayAttr constParams = read(); - SmallVector args; - readList(args); - - LLVM_DEBUG({ - llvm::dbgs() << " * Arguments: "; - llvm::interleaveComma(args, llvm::dbgs()); - llvm::dbgs() << "\n * Parameters: " << constParams << "\n"; - }); - - PDLValue result = createFn(args, constParams, rewriter); - memory[resultIndex] = result.getAsOpaquePointer(); - - LLVM_DEBUG(llvm::dbgs() << " * Result: " << result << "\n"); -} - void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter, Location mainRewriteLoc) { LLVM_DEBUG(llvm::dbgs() << "Executing CreateOperation:\n"); @@ -1246,9 +1230,6 @@ void ByteCodeExecutor::execute( case CheckResultCount: executeCheckResultCount(); break; - case CreateNative: - executeCreateNative(rewriter); - break; case CreateOperation: executeCreateOperation(rewriter, *mainRewriteLoc); break; @@ -1338,8 +1319,7 @@ void PDLByteCode::match(Operation *op, PatternRewriter &rewriter, // The matcher function always starts at code address 0. ByteCodeExecutor executor(matcherByteCode.data(), state.memory, uniquedData, matcherByteCode, state.currentPatternBenefits, - patterns, constraintFunctions, createFunctions, - rewriteFunctions); + patterns, constraintFunctions, rewriteFunctions); executor.execute(rewriter, &matches); // Order the found matches by benefit. @@ -1356,9 +1336,9 @@ void PDLByteCode::rewrite(PatternRewriter &rewriter, const MatchResult &match, // memory buffer. llvm::copy(match.values, state.memory.begin()); - ByteCodeExecutor executor( - &rewriterByteCode[match.pattern->getRewriterAddr()], state.memory, - uniquedData, rewriterByteCode, state.currentPatternBenefits, patterns, - constraintFunctions, createFunctions, rewriteFunctions); + ByteCodeExecutor executor(&rewriterByteCode[match.pattern->getRewriterAddr()], + state.memory, uniquedData, rewriterByteCode, + state.currentPatternBenefits, patterns, + constraintFunctions, rewriteFunctions); executor.execute(rewriter, /*matches=*/nullptr, match.location); } diff --git a/mlir/lib/Rewrite/ByteCode.h b/mlir/lib/Rewrite/ByteCode.h index 38dbbcd855ce..f6a3bcbe54f9 100644 --- a/mlir/lib/Rewrite/ByteCode.h +++ b/mlir/lib/Rewrite/ByteCode.h @@ -114,7 +114,6 @@ public: /// the PDL interpreter dialect. PDLByteCode(ModuleOp module, llvm::StringMap constraintFns, - llvm::StringMap createFns, llvm::StringMap rewriteFns); /// Return the patterns held by the bytecode. @@ -160,7 +159,6 @@ private: /// A set of user defined functions invoked via PDL. std::vector constraintFunctions; - std::vector createFunctions; std::vector rewriteFunctions; /// The maximum memory index used by a value. diff --git a/mlir/lib/Rewrite/FrozenRewritePatternList.cpp b/mlir/lib/Rewrite/FrozenRewritePatternList.cpp index 40f7aec44e51..c2de51a647dd 100644 --- a/mlir/lib/Rewrite/FrozenRewritePatternList.cpp +++ b/mlir/lib/Rewrite/FrozenRewritePatternList.cpp @@ -70,7 +70,7 @@ FrozenRewritePatternList::FrozenRewritePatternList( // Generate the pdl bytecode. impl->pdlByteCode = std::make_unique( pdlModule, pdlPatterns.takeConstraintFunctions(), - pdlPatterns.takeCreateFunctions(), pdlPatterns.takeRewriteFunctions()); + pdlPatterns.takeRewriteFunctions()); } FrozenRewritePatternList::~FrozenRewritePatternList() {} diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir index c856ab5c9f6f..a42b51604945 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir @@ -24,7 +24,7 @@ module @simple { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[REWRITE_ROOT:.*]]: !pdl.operation) - // CHECK: pdl_interp.apply_rewrite "rewriter" on %[[REWRITE_ROOT]] + // CHECK: pdl_interp.apply_rewrite "rewriter"(%[[REWRITE_ROOT]] // CHECK: pdl_interp.finalize pdl.pattern : benefit(1) { %root = pdl.operation "foo.op"() @@ -72,7 +72,7 @@ module @constraints { %root = pdl.operation(%input0, %input1) %result0 = pdl.result 0 of %root - pdl.apply_constraint "multi_constraint"[true](%input0, %input1, %result0 : !pdl.value, !pdl.value, !pdl.value) + pdl.apply_native_constraint "multi_constraint"[true](%input0, %input1, %result0 : !pdl.value, !pdl.value, !pdl.value) pdl.rewrite %root with "rewriter" } } @@ -194,7 +194,7 @@ module @predicate_ordering { pdl.pattern : benefit(1) { %resultType = pdl.type - pdl.apply_constraint "typeConstraint"[](%resultType : !pdl.type) + pdl.apply_native_constraint "typeConstraint"[](%resultType : !pdl.type) %root = pdl.operation -> %resultType pdl.rewrite %root with "rewriter" } diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir index 5652b2118afe..3d0d565c547f 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir @@ -6,7 +6,7 @@ module @external { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation, %[[INPUT:.*]]: !pdl.value) - // CHECK: pdl_interp.apply_rewrite "rewriter" [true](%[[INPUT]] : !pdl.value) on %[[ROOT]] + // CHECK: pdl_interp.apply_rewrite "rewriter" [true](%[[ROOT]], %[[INPUT]] : !pdl.operation, !pdl.value) pdl.pattern : benefit(1) { %input = pdl.operand %root = pdl.operation "foo.op"(%input) @@ -170,17 +170,17 @@ module @replace_with_no_results { // ----- -// CHECK-LABEL: module @create_native -module @create_native { +// CHECK-LABEL: module @apply_native_rewrite +module @apply_native_rewrite { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation) - // CHECK: %[[TYPE:.*]] = pdl_interp.create_native "functor" [true](%[[ROOT]] : !pdl.operation) : !pdl.type + // CHECK: %[[TYPE:.*]] = pdl_interp.apply_rewrite "functor" [true](%[[ROOT]] : !pdl.operation) : !pdl.type // CHECK: pdl_interp.create_operation "foo.op"() -> %[[TYPE]] pdl.pattern : benefit(1) { %type = pdl.type %root = pdl.operation "foo.op" -> %type pdl.rewrite %root { - %newType = pdl.create_native "functor"[true](%root : !pdl.operation) : !pdl.type + %newType = pdl.apply_native_rewrite "functor"[true](%root : !pdl.operation) : !pdl.type %newOp = pdl.operation "foo.op" -> %newType pdl.replace %root with %newOp } diff --git a/mlir/test/Dialect/PDL/invalid.mlir b/mlir/test/Dialect/PDL/invalid.mlir index 0f900bbe3f53..a054da24ba4d 100644 --- a/mlir/test/Dialect/PDL/invalid.mlir +++ b/mlir/test/Dialect/PDL/invalid.mlir @@ -1,19 +1,33 @@ // RUN: mlir-opt %s -split-input-file -verify-diagnostics //===----------------------------------------------------------------------===// -// pdl::ApplyConstraintOp +// pdl::ApplyNativeConstraintOp //===----------------------------------------------------------------------===// pdl.pattern : benefit(1) { %op = pdl.operation "foo.op" // expected-error@below {{expected at least one argument}} - "pdl.apply_constraint"() {name = "foo", params = []} : () -> () + "pdl.apply_native_constraint"() {name = "foo", params = []} : () -> () pdl.rewrite %op with "rewriter" } // ----- +//===----------------------------------------------------------------------===// +// pdl::ApplyNativeRewriteOp +//===----------------------------------------------------------------------===// + +pdl.pattern : benefit(1) { + %op = pdl.operation "foo.op" + pdl.rewrite %op { + // expected-error@below {{expected at least one argument}} + "pdl.apply_native_rewrite"() {name = "foo", params = []} : () -> () + } +} + +// ----- + //===----------------------------------------------------------------------===// // pdl::AttributeOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/Rewrite/pdl-bytecode.mlir b/mlir/test/Rewrite/pdl-bytecode.mlir index b2a22d0a8749..2093d03bbf25 100644 --- a/mlir/test/Rewrite/pdl-bytecode.mlir +++ b/mlir/test/Rewrite/pdl-bytecode.mlir @@ -58,7 +58,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { %operand = pdl_interp.get_operand 0 of %root - pdl_interp.apply_rewrite "rewriter"[42](%operand : !pdl.value) on %root + pdl_interp.apply_rewrite "rewriter"[42](%root, %operand : !pdl.operation, !pdl.value) pdl_interp.finalize } } @@ -72,6 +72,35 @@ module @ir attributes { test.apply_rewrite_1 } { %input = "test.op_input"() : () -> i32 "test.op"(%input) : (i32) -> () } + +// ----- + +module @patterns { + func @matcher(%root : !pdl.operation) { + pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end + + ^pat: + pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end + + ^end: + pdl_interp.finalize + } + + module @rewriters { + func @success(%root : !pdl.operation) { + %op = pdl_interp.apply_rewrite "creator"(%root : !pdl.operation) : !pdl.operation + pdl_interp.erase %root + pdl_interp.finalize + } + } +} + +// CHECK-LABEL: test.apply_rewrite_2 +// CHECK: "test.success" +module @ir attributes { test.apply_rewrite_2 } { + "test.op"() : () -> () +} + // ----- //===----------------------------------------------------------------------===// @@ -317,38 +346,6 @@ module @ir attributes { test.check_type_1 } { // Fully tested within the tests for other operations. -//===----------------------------------------------------------------------===// -// pdl_interp::CreateNativeOp -//===----------------------------------------------------------------------===// - -// ----- - -module @patterns { - func @matcher(%root : !pdl.operation) { - pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end - - ^pat: - pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end - - ^end: - pdl_interp.finalize - } - - module @rewriters { - func @success(%root : !pdl.operation) { - %op = pdl_interp.create_native "creator"(%root : !pdl.operation) : !pdl.operation - pdl_interp.erase %root - pdl_interp.finalize - } - } -} - -// CHECK-LABEL: test.create_native_1 -// CHECK: "test.success" -module @ir attributes { test.create_native_1 } { - "test.op"() : () -> () -} - //===----------------------------------------------------------------------===// // pdl_interp::CreateOperationOp //===----------------------------------------------------------------------===// diff --git a/mlir/test/lib/Rewrite/TestPDLByteCode.cpp b/mlir/test/lib/Rewrite/TestPDLByteCode.cpp index 3b23cb103675..e60022ba94cc 100644 --- a/mlir/test/lib/Rewrite/TestPDLByteCode.cpp +++ b/mlir/test/lib/Rewrite/TestPDLByteCode.cpp @@ -26,18 +26,18 @@ static LogicalResult customMultiEntityConstraint(ArrayRef values, } // Custom creator invoked from PDL. -static PDLValue customCreate(ArrayRef args, ArrayAttr constantParams, - PatternRewriter &rewriter) { - return rewriter.createOperation( - OperationState(args[0].cast()->getLoc(), "test.success")); +static void customCreate(ArrayRef args, ArrayAttr constantParams, + PatternRewriter &rewriter, PDLResultList &results) { + results.push_back(rewriter.createOperation( + OperationState(args[0].cast()->getLoc(), "test.success"))); } /// Custom rewriter invoked from PDL. -static void customRewriter(Operation *root, ArrayRef args, - ArrayAttr constantParams, - PatternRewriter &rewriter) { +static void customRewriter(ArrayRef args, ArrayAttr constantParams, + PatternRewriter &rewriter, PDLResultList &results) { + Operation *root = args[0].cast(); OperationState successOpState(root->getLoc(), "test.success"); - successOpState.addOperands(args[0].cast()); + successOpState.addOperands(args[1].cast()); successOpState.addAttribute("constantParams", constantParams); rewriter.createOperation(successOpState); rewriter.eraseOp(root); @@ -63,7 +63,7 @@ struct TestPDLByteCodePass customMultiEntityConstraint); pdlPattern.registerConstraintFunction("single_entity_constraint", customSingleEntityConstraint); - pdlPattern.registerCreateFunction("creator", customCreate); + pdlPattern.registerRewriteFunction("creator", customCreate); pdlPattern.registerRewriteFunction("rewriter", customRewriter); OwningRewritePatternList patternList(std::move(pdlPattern)); -- GitLab From 1eb6994d6ab18d5f6555acf515d27e2076fbea8a Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 16 Mar 2021 13:11:34 -0700 Subject: [PATCH 0073/1206] [mlir][PDL] Add support for variadic operands and results in PDL This revision extends the PDL dialect to add support for variadic operands and results, with ranges of these values represented via the recently added !pdl.range type. To support this extension, three new operations have been added that closely match the single variant: * pdl.operands : Define a range of input operands. * pdl.results : Extract a result group from an operation. * pdl.types : Define a handle to a range of types. Support for these in the pdl interpreter dialect and byte code will be added in followup revisions. Differential Revision: https://reviews.llvm.org/D95721 --- mlir/include/mlir/Dialect/PDL/IR/PDLOps.td | 217 ++++++++++++++++-- mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td | 14 ++ mlir/lib/Dialect/PDL/IR/PDL.cpp | 72 +++++- .../pdl-to-pdl-interp-matcher.mlir | 20 +- .../pdl-to-pdl-interp-rewriter.mlir | 32 +-- mlir/test/Dialect/PDL/invalid.mlir | 51 +++- mlir/test/Dialect/PDL/ops.mlir | 31 ++- 7 files changed, 362 insertions(+), 75 deletions(-) diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td index 74f3fce08933..32de9f438c00 100644 --- a/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td +++ b/mlir/include/mlir/Dialect/PDL/IR/PDLOps.td @@ -60,7 +60,7 @@ def PDL_ApplyNativeConstraintOp let builders = [ OpBuilder<(ins "StringRef":$name, CArg<"ValueRange", "{}">:$args, - CArg<"ArrayRef", "{}">:$params), [{ + CArg<"ArrayRef", "{}">:$params), [{ build($_builder, $_state, $_builder.getStringAttr(name), args, params.empty() ? ArrayAttr() : $_builder.getArrayAttr(params)); }]>, @@ -196,9 +196,9 @@ def PDL_OperandOp : PDL_Op<"operand", [HasParent<"pdl::PatternOp">]> { let description = [{ `pdl.operand` operations capture external operand edges into an operation node that originate from operations or block arguments not otherwise - specified within the pattern (e.g. via `pdl.result`). These operations - define individual operands of a given operation. A `pdl.operand` may - partially constrain an operand by specifying an expected value type + specified within the pattern (i.e. via `pdl.result` or `pdl.results`). These + operations define individual operands of a given operation. A `pdl.operand` + may partially constrain an operand by specifying an expected value type (via a `pdl.type` operation). Example: @@ -224,6 +224,44 @@ def PDL_OperandOp : PDL_Op<"operand", [HasParent<"pdl::PatternOp">]> { ]; } +//===----------------------------------------------------------------------===// +// pdl::OperandsOp +//===----------------------------------------------------------------------===// + +def PDL_OperandsOp : PDL_Op<"operands", [HasParent<"pdl::PatternOp">]> { + let summary = "Define a range of input operands in a pattern"; + let description = [{ + `pdl.operands` operations capture external operand range edges into an + operation node that originate from operations or block arguments not + otherwise specified within the pattern (i.e. via `pdl.result` or + `pdl.results`). These operations define groups of input operands into a + given operation. A `pdl.operands` may partially constrain a set of input + operands by specifying expected value types (via `pdl.types` operations). + + Example: + + ```mlir + // Define a range of input operands: + %operands = pdl.operands + + // Define a range of input operands with expected types: + %types = pdl.types : [i32, i64, i32] + %typed_operands = pdl.operands : %types + ``` + }]; + + let arguments = (ins Optional>:$type); + let results = (outs PDL_RangeOf:$val); + let assemblyFormat = "(`:` $type^)? attr-dict"; + + let builders = [ + OpBuilder<(ins), [{ + build($_builder, $_state, RangeType::get($_builder.getType()), + Value()); + }]>, + ]; +} + //===----------------------------------------------------------------------===// // pdl::OperationOp //===----------------------------------------------------------------------===// @@ -245,6 +283,14 @@ def PDL_OperationOp a handle to the operation itself. Handles to the results of the operation can be extracted via `pdl.result`. + Example: + + ```mlir + // Define an instance of a `foo.op` operation. + %op = pdl.operation "foo.op"(%arg0, %arg1 : !pdl.value, !pdl.value) + {"attrA" = %attr0} -> (%type, %type : !pdl.type, !pdl.type) + ``` + When used within a matching context, the name of the operation may be omitted. @@ -257,24 +303,78 @@ def PDL_OperationOp override the `InferTypeOpInterface` to ensure that the result types can be inferred. - Example: + The operands of the operation are interpreted in the following ways: + + 1) A single !pdl.range: + + In this case, the single range is treated as all of the operands of the + operation. ```mlir - // Define an instance of a `foo.op` operation. - %op = pdl.operation "foo.op"(%arg0, %arg1) {"attrA" = %attr0} -> %type, %type, %type, %type + // Define an instance with single range of operands. + %op = pdl.operation "std.return"(%allArgs : !pdl.range) + ``` + + 2) A variadic number of either !pdl.value or !pdl.range: + + In this case, the inputs are expected to correspond with the operand groups + defined on the operation in ODS. + + ```tablgen + // Given the following operation definition in ODS: + def MyIndirectCallOp { + let results = (outs FunctionType:$call, Variadic:$args); + } + ``` + + ```mlir + // We can match the operands as so: + %op = pdl.operation "my.indirect_call"(%call, %args : !pdl.value, !pdl.range) + ``` + + The results of the operation are interpreted in the following ways: + + 1) A single !pdl.range: + + In this case, the single range is treated as all of the result types of the + operation. + + ```mlir + // Define an instance with single range of types. + %allResultTypes = pdl.types + %op = pdl.operation "unrealized_conversion_cast" -> (%allResultTypes : !pdl.types) + ``` + + 2) A variadic number of either !pdl.type or !pdl.range: + + In this case, the inputs are expected to correspond with the result groups + defined on the operation in ODS. + + ```tablgen + // Given the following operation definition in ODS: + def MyOp { + let results = (outs SomeType:$result, Variadic:$otherResults); + } + ``` + + ```mlir + // We can match the results as so: + %result = pdl.type + %otherResults = pdl.types + %op = pdl.operation "foo.op" -> (%result, %otherResults : !pdl.type, !pdl.range) ``` }]; let arguments = (ins OptionalAttr:$name, - Variadic:$operands, + Variadic>:$operands, Variadic:$attributes, StrArrayAttr:$attributeNames, - Variadic:$types); + Variadic>:$types); let results = (outs PDL_Operation:$op); let assemblyFormat = [{ - ($name^)? (`(` $operands^ `)`)? + ($name^)? (`(` $operands^ `:` type($operands) `)`)? custom($attributes, $attributeNames) - (`->` $types^)? attr-dict + (`->` `(` $types^ `:` type($types) `)`)? attr-dict }]; let builders = [ @@ -378,7 +478,10 @@ def PDL_ReplaceOp : PDL_Op<"replace", [ ```mlir // Replace root node with 2 values: - pdl.replace %root with (%val0, %val1) + pdl.replace %root with (%val0, %val1 : !pdl.value, !pdl.value) + + // Replace root node with a range of values: + pdl.replace %root with (%vals : !pdl.range) // Replace root with another operation: pdl.replace %root with %otherOp @@ -386,9 +489,10 @@ def PDL_ReplaceOp : PDL_Op<"replace", [ }]; let arguments = (ins PDL_Operation:$operation, Optional:$replOperation, - Variadic:$replValues); + Variadic>:$replValues); let assemblyFormat = [{ - $operation `with` (`(` $replValues^ `)`)? ($replOperation^)? attr-dict + $operation `with` (`(` $replValues^ `:` type($replValues) `)`)? + ($replOperation^)? attr-dict }]; } @@ -409,13 +513,13 @@ def PDL_ResultOp : PDL_Op<"result"> { ```mlir // Extract a result: %operation = pdl.operation ... - %result = pdl.result 1 of %operation + %pdl_result = pdl.result 1 of %operation // Imagine the following IR being matched: %result_0, %result_1 = foo.op ... // If the example pattern snippet above were matching against `foo.op` in - // the IR snippted, `%result` would correspond to `%result_1`. + // the IR snippet, `%pdl_result` would correspond to `%result_1`. ``` }]; @@ -425,6 +529,48 @@ def PDL_ResultOp : PDL_Op<"result"> { let verifier = ?; } +//===----------------------------------------------------------------------===// +// pdl::ResultsOp +//===----------------------------------------------------------------------===// + +def PDL_ResultsOp : PDL_Op<"results"> { + let summary = "Extract a result group from an operation"; + let description = [{ + `pdl.results` operations extract a result group from an operation within a + pattern or rewrite region. If an index is provided, this operation extracts + a result group as defined by the ODS definition of the operation. In this + case the result of this operation may be either a single `pdl.value` or + a `pdl.range`, depending on the constraint of the result in ODS. If + no index is provided, this operation extracts the full result range of the + operation. + + Example: + + ```mlir + // Extract all of the results of an operation: + %operation = pdl.operation ... + %results = pdl.results of %operation + + // Extract the results in the first result group of an operation, which is + // variadic: + %operation = pdl.operation ... + %results = pdl.results 0 of %operation -> !pdl.range + + // Extract the results in the second result group of an operation, which is + // not variadic: + %operation = pdl.operation ... + %results = pdl.results 1 of %operation -> !pdl.value + ``` + }]; + + let arguments = (ins PDL_Operation:$parent, OptionalAttr:$index); + let results = (outs PDL_InstOrRangeOf:$val); + let assemblyFormat = [{ + ($index^)? `of` $parent custom(ref($index), type($val)) + attr-dict + }]; +} + //===----------------------------------------------------------------------===// // pdl::RewriteOp //===----------------------------------------------------------------------===// @@ -489,7 +635,7 @@ def PDL_RewriteEndOp : PDL_Op<"rewrite_end", [Terminator, def PDL_TypeOp : PDL_Op<"type"> { let summary = "Define a type handle within a pattern"; let description = [{ - `pdl.type` operations capture result type constraints of an `Attributes`, + `pdl.type` operations capture result type constraints of `Attributes`, `Values`, and `Operations`. Instances of this operation define, and partially constrain, results types of a given entity. A `pdl.type` may partially constrain the result by specifying a constant `Type`. @@ -498,23 +644,44 @@ def PDL_TypeOp : PDL_Op<"type"> { ```mlir // Define a type: - %attr = pdl.type + %type = pdl.type // Define a type with a constant value: - %attr = pdl.type : i32 + %type = pdl.type : i32 ``` }]; let arguments = (ins OptionalAttr:$type); let results = (outs PDL_Type:$result); let assemblyFormat = "attr-dict (`:` $type^)?"; +} - let builders = [ - OpBuilder<(ins CArg<"Type", "Type()">:$ty), [{ - build($_builder, $_state, $_builder.getType(), - ty ? TypeAttr::get(ty) : TypeAttr()); - }]>, - ]; +//===----------------------------------------------------------------------===// +// pdl::TypesOp +//===----------------------------------------------------------------------===// + +def PDL_TypesOp : PDL_Op<"types"> { + let summary = "Define a range of type handles within a pattern"; + let description = [{ + `pdl.types` operations capture result type constraints of `Value`s, and + `Operation`s. Instances of this operation define results types of a given + entity. A `pdl.types` may partially constrain the results by specifying + an array of `Type`s. + + Example: + + ```mlir + // Define a range of types: + %types = pdl.types + + // Define a range of types with a range of constant values: + %types = pdl.types : [i32, i64, i32] + ``` + }]; + + let arguments = (ins OptionalAttr:$types); + let results = (outs PDL_RangeOf:$result); + let assemblyFormat = "attr-dict (`:` $types^)?"; } #endif // MLIR_DIALECT_PDL_IR_PDLOPS diff --git a/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td b/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td index c854616fbc8f..1e0578339ad8 100644 --- a/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td +++ b/mlir/include/mlir/Dialect/PDL/IR/PDLTypes.td @@ -101,4 +101,18 @@ def PDL_AnyType : Type< CPred<"$_self.isa<::mlir::pdl::PDLType>()">, "pdl type", "::mlir::pdl::PDLType">; +// A range of positional values of one of the provided types. +class PDL_RangeOf : + ContainerType, PDL_Range.predicate, + "$_self.cast<::mlir::pdl::RangeType>().getElementType()", + "range", "::mlir::pdl::RangeType">, + BuildableType<"::mlir::pdl::RangeType::get(" # positionalType.builderCall # + ")">; + +// Either a positional value or a range of positional values for a given type. +class PDL_InstOrRangeOf : + AnyTypeOf<[positionalType, PDL_RangeOf], + "single element or range of " # positionalType.summary, + "::mlir::pdl::PDLType">; + #endif // MLIR_DIALECT_PDL_IR_PDLTYPES diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp index dc1f501825bd..8164c89dac54 100644 --- a/mlir/lib/Dialect/PDL/IR/PDL.cpp +++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp @@ -35,13 +35,19 @@ void PDLDialect::initialize() { /// Returns true if the given operation is used by a "binding" pdl operation /// within the main matcher body of a `pdl.pattern`. static bool hasBindingUseInMatcher(Operation *op, Block *matcherBlock) { - for (Operation *user : op->getUsers()) { + for (OpOperand &use : op->getUses()) { + Operation *user = use.getOwner(); if (user->getBlock() != matcherBlock) continue; - if (isa(user)) + if (isa(user)) + return true; + // Only the first operand of RewriteOp may be bound to, i.e. the root + // operation of the pattern. + if (isa(user) && use.getOperandNumber() == 0) return true; // A result by itself is not binding, it must also be bound. - if (isa(user) && hasBindingUseInMatcher(user, matcherBlock)) + if (isa(user) && + hasBindingUseInMatcher(user, matcherBlock)) return true; } return false; @@ -107,6 +113,14 @@ static LogicalResult verify(OperandOp op) { return verifyHasBindingUseInMatcher(op); } +//===----------------------------------------------------------------------===// +// pdl::OperandsOp +//===----------------------------------------------------------------------===// + +static LogicalResult verify(OperandsOp op) { + return verifyHasBindingUseInMatcher(op); +} + //===----------------------------------------------------------------------===// // pdl::OperationOp //===----------------------------------------------------------------------===// @@ -177,18 +191,18 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op, if (isa(resultTypeOp)) continue; - // If the type is already constrained, there is nothing to do. - TypeOp typeOp = cast(resultTypeOp); - if (typeOp.type()) - continue; - // If the type operation was defined in the matcher and constrains the // result of an input operation, it can be used. auto constrainsInputOp = [rewriterBlock](Operation *user) { return user->getBlock() != rewriterBlock && isa(user); }; - if (llvm::any_of(typeOp.getResult().getUsers(), constrainsInputOp)) - continue; + if (TypeOp typeOp = dyn_cast(resultTypeOp)) { + if (typeOp.type() || llvm::any_of(typeOp->getUsers(), constrainsInputOp)) + continue; + } else if (TypesOp typeOp = dyn_cast(resultTypeOp)) { + if (typeOp.types() || llvm::any_of(typeOp->getUsers(), constrainsInputOp)) + continue; + } return op .emitOpError("must have inferable or constrained result types when " @@ -296,6 +310,36 @@ static LogicalResult verify(ReplaceOp op) { return success(); } +//===----------------------------------------------------------------------===// +// pdl::ResultsOp +//===----------------------------------------------------------------------===// + +static ParseResult parseResultsValueType(OpAsmParser &p, IntegerAttr index, + Type &resultType) { + if (!index) { + resultType = RangeType::get(p.getBuilder().getType()); + return success(); + } + if (p.parseArrow() || p.parseType(resultType)) + return failure(); + return success(); +} + +static void printResultsValueType(OpAsmPrinter &p, ResultsOp op, + IntegerAttr index, Type resultType) { + if (index) + p << " -> " << resultType; +} + +static LogicalResult verify(ResultsOp op) { + if (!op.index() && op.getType().isa()) { + return op.emitOpError() << "expected `pdl.range` result type when " + "no index is specified, but got: " + << op.getType(); + } + return success(); +} + //===----------------------------------------------------------------------===// // pdl::RewriteOp //===----------------------------------------------------------------------===// @@ -340,6 +384,14 @@ static LogicalResult verify(TypeOp op) { op, "`pdl.attribute`, `pdl.operand`, or `pdl.operation`"); } +//===----------------------------------------------------------------------===// +// pdl::TypesOp +//===----------------------------------------------------------------------===// + +static LogicalResult verify(TypesOp op) { + return verifyHasBindingUseInMatcher(op, "`pdl.operands`, or `pdl.operation`"); +} + //===----------------------------------------------------------------------===// // TableGen'd op method definitions //===----------------------------------------------------------------------===// diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir index a42b51604945..0792f76cba7a 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir @@ -27,7 +27,7 @@ module @simple { // CHECK: pdl_interp.apply_rewrite "rewriter"(%[[REWRITE_ROOT]] // CHECK: pdl_interp.finalize pdl.pattern : benefit(1) { - %root = pdl.operation "foo.op"() + %root = pdl.operation "foo.op" pdl.rewrite %root with "rewriter" } } @@ -69,7 +69,7 @@ module @constraints { pdl.pattern : benefit(1) { %input0 = pdl.operand %input1 = pdl.operand - %root = pdl.operation(%input0, %input1) + %root = pdl.operation(%input0, %input1 : !pdl.value, !pdl.value) %result0 = pdl.result 0 of %root pdl.apply_native_constraint "multi_constraint"[true](%input0, %input1, %result0 : !pdl.value, !pdl.value, !pdl.value) @@ -96,7 +96,7 @@ module @inputs { pdl.pattern : benefit(1) { %type = pdl.type : i64 %input = pdl.operand : %type - %root = pdl.operation(%input, %input) + %root = pdl.operation(%input, %input : !pdl.value, !pdl.value) pdl.rewrite %root with "rewriter" } } @@ -120,7 +120,7 @@ module @results { pdl.pattern : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %root = pdl.operation -> %type1, %type2 + %root = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type) pdl.rewrite %root with "rewriter" } } @@ -149,11 +149,11 @@ module @results_as_operands { pdl.pattern : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %inputOp = pdl.operation -> %type1, %type2 + %inputOp = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type) %result1 = pdl.result 0 of %inputOp %result2 = pdl.result 1 of %inputOp - %root = pdl.operation(%result1, %result2) + %root = pdl.operation(%result1, %result2 : !pdl.value, !pdl.value) pdl.rewrite %root with "rewriter" } } @@ -168,12 +168,12 @@ module @switch_result_types { // CHECK: pdl_interp.switch_type %[[RESULT_TYPE]] to [i32, i64] pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root = pdl.operation -> %type + %root = pdl.operation -> (%type : !pdl.type) pdl.rewrite %root with "rewriter" } pdl.pattern : benefit(1) { %type = pdl.type : i64 - %root = pdl.operation -> %type + %root = pdl.operation -> (%type : !pdl.type) pdl.rewrite %root with "rewriter" } } @@ -195,13 +195,13 @@ module @predicate_ordering { pdl.pattern : benefit(1) { %resultType = pdl.type pdl.apply_native_constraint "typeConstraint"[](%resultType : !pdl.type) - %root = pdl.operation -> %resultType + %root = pdl.operation -> (%resultType : !pdl.type) pdl.rewrite %root with "rewriter" } pdl.pattern : benefit(1) { %resultType = pdl.type - %apply = pdl.operation -> %resultType + %apply = pdl.operation -> (%resultType : !pdl.type) pdl.rewrite %apply with "rewriter" } } diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir index 3d0d565c547f..67ac7c811ab7 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir @@ -9,7 +9,7 @@ module @external { // CHECK: pdl_interp.apply_rewrite "rewriter" [true](%[[ROOT]], %[[INPUT]] : !pdl.operation, !pdl.value) pdl.pattern : benefit(1) { %input = pdl.operand - %root = pdl.operation "foo.op"(%input) + %root = pdl.operation "foo.op"(%input : !pdl.value) pdl.rewrite %root with "rewriter"[true](%input : !pdl.value) } } @@ -60,12 +60,12 @@ module @operation_operands { // CHECK: pdl_interp.create_operation "foo.op2"(%[[OPERAND1]]) pdl.pattern : benefit(1) { %operand = pdl.operand - %root = pdl.operation "foo.op"(%operand) + %root = pdl.operation "foo.op"(%operand : !pdl.value) pdl.rewrite %root { %type = pdl.type : i32 - %newOp = pdl.operation "foo.op"(%operand) -> %type + %newOp = pdl.operation "foo.op"(%operand : !pdl.value) -> (%type : !pdl.type) %result = pdl.result 0 of %newOp - %newOp1 = pdl.operation "foo.op2"(%result) + %newOp1 = pdl.operation "foo.op2"(%result : !pdl.value) pdl.erase %root } } @@ -82,12 +82,12 @@ module @operation_operands { // CHECK: pdl_interp.create_operation "foo.op2"(%[[OPERAND1]]) pdl.pattern : benefit(1) { %operand = pdl.operand - %root = pdl.operation "foo.op"(%operand) + %root = pdl.operation "foo.op"(%operand : !pdl.value) pdl.rewrite %root { %type = pdl.type : i32 - %newOp = pdl.operation "foo.op"(%operand) -> %type + %newOp = pdl.operation "foo.op"(%operand : !pdl.value) -> (%type : !pdl.type) %result = pdl.result 0 of %newOp - %newOp1 = pdl.operation "foo.op2"(%result) + %newOp1 = pdl.operation "foo.op2"(%result : !pdl.value) pdl.erase %root } } @@ -103,10 +103,10 @@ module @operation_result_types { pdl.pattern : benefit(1) { %rootType = pdl.type %rootType1 = pdl.type - %root = pdl.operation "foo.op" -> %rootType, %rootType1 + %root = pdl.operation "foo.op" -> (%rootType, %rootType1 : !pdl.type, !pdl.type) pdl.rewrite %root { %newType1 = pdl.type - %newOp = pdl.operation "foo.op" -> %rootType, %newType1 + %newOp = pdl.operation "foo.op" -> (%rootType, %newType1 : !pdl.type, !pdl.type) pdl.replace %root with %newOp } } @@ -123,9 +123,9 @@ module @replace_with_op { // CHECK: pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]]) pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> (%type : !pdl.type) pdl.rewrite %root { - %newOp = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> (%type : !pdl.type) pdl.replace %root with %newOp } } @@ -142,11 +142,11 @@ module @replace_with_values { // CHECK: pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]]) pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> (%type : !pdl.type) pdl.rewrite %root { - %newOp = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> (%type : !pdl.type) %newResult = pdl.result 0 of %newOp - pdl.replace %root with (%newResult) + pdl.replace %root with (%newResult : !pdl.value) } } } @@ -178,10 +178,10 @@ module @apply_native_rewrite { // CHECK: pdl_interp.create_operation "foo.op"() -> %[[TYPE]] pdl.pattern : benefit(1) { %type = pdl.type - %root = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> (%type : !pdl.type) pdl.rewrite %root { %newType = pdl.apply_native_rewrite "functor"[true](%root : !pdl.operation) : !pdl.type - %newOp = pdl.operation "foo.op" -> %newType + %newOp = pdl.operation "foo.op" -> (%newType : !pdl.type) pdl.replace %root with %newOp } } diff --git a/mlir/test/Dialect/PDL/invalid.mlir b/mlir/test/Dialect/PDL/invalid.mlir index a054da24ba4d..e371d8408670 100644 --- a/mlir/test/Dialect/PDL/invalid.mlir +++ b/mlir/test/Dialect/PDL/invalid.mlir @@ -38,7 +38,7 @@ pdl.pattern : benefit(1) { // expected-error@below {{expected only one of [`type`, `value`] to be set}} %attr = pdl.attribute : %type 10 - %op = pdl.operation "foo.op" {"attr" = %attr} -> %type + %op = pdl.operation "foo.op" {"attr" = %attr} -> (%type : !pdl.type) pdl.rewrite %op with "rewriter" } @@ -90,6 +90,20 @@ pdl.pattern : benefit(1) { // ----- +//===----------------------------------------------------------------------===// +// pdl::OperandsOp +//===----------------------------------------------------------------------===// + +pdl.pattern : benefit(1) { + // expected-error@below {{expected a bindable (i.e. `pdl.operation`) user when defined in the matcher body of a `pdl.pattern`}} + %unused = pdl.operands + + %op = pdl.operation "foo.op" + pdl.rewrite %op with "rewriter" +} + +// ----- + //===----------------------------------------------------------------------===// // pdl::OperationOp //===----------------------------------------------------------------------===// @@ -116,13 +130,13 @@ pdl.pattern : benefit(1) { // ----- pdl.pattern : benefit(1) { - %op = pdl.operation "foo.op"() + %op = pdl.operation "foo.op" pdl.rewrite %op { %type = pdl.type // expected-error@below {{op must have inferable or constrained result types when nested within `pdl.rewrite`}} // expected-note@below {{result type #0 was not constrained}} - %newOp = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> (%type : !pdl.type) } } @@ -163,9 +177,9 @@ pdl.pattern : benefit(1) { pdl.pattern : benefit(1) { %type = pdl.type : i32 - %root = pdl.operation "foo.op" -> %type + %root = pdl.operation "foo.op" -> (%type : !pdl.type) pdl.rewrite %root { - %newOp = pdl.operation "foo.op" -> %type + %newOp = pdl.operation "foo.op" -> (%type : !pdl.type) %newResult = pdl.result 0 of %newOp // expected-error@below {{expected no replacement values to be provided when the replacement operation is present}} @@ -177,6 +191,19 @@ pdl.pattern : benefit(1) { // ----- +//===----------------------------------------------------------------------===// +// pdl::ResultsOp +//===----------------------------------------------------------------------===// + +pdl.pattern : benefit(1) { + %root = pdl.operation "foo.op" + // expected-error@below {{expected `pdl.range` result type when no index is specified, but got: '!pdl.value'}} + %results = "pdl.results"(%root) : (!pdl.operation) -> !pdl.value + pdl.rewrite %root with "rewriter" +} + +// ----- + //===----------------------------------------------------------------------===// // pdl::RewriteOp //===----------------------------------------------------------------------===// @@ -237,3 +264,17 @@ pdl.pattern : benefit(1) { %op = pdl.operation "foo.op" pdl.rewrite %op with "rewriter" } + +// ----- + +//===----------------------------------------------------------------------===// +// pdl::TypesOp +//===----------------------------------------------------------------------===// + +pdl.pattern : benefit(1) { + // expected-error@below {{expected a bindable (i.e. `pdl.operands`, or `pdl.operation`) user when defined in the matcher body of a `pdl.pattern`}} + %unused = pdl.types + + %op = pdl.operation "foo.op" + pdl.rewrite %op with "rewriter" +} diff --git a/mlir/test/Dialect/PDL/ops.mlir b/mlir/test/Dialect/PDL/ops.mlir index d376f001fcfa..07e98f9e5868 100644 --- a/mlir/test/Dialect/PDL/ops.mlir +++ b/mlir/test/Dialect/PDL/ops.mlir @@ -8,12 +8,12 @@ pdl.pattern @operations : benefit(1) { // Operation with attributes and results. %attribute = pdl.attribute %type = pdl.type - %op0 = pdl.operation {"attr" = %attribute} -> %type + %op0 = pdl.operation {"attr" = %attribute} -> (%type : !pdl.type) %op0_result = pdl.result 0 of %op0 // Operation with input. %input = pdl.operand - %root = pdl.operation(%op0_result, %input) + %root = pdl.operation(%op0_result, %input : !pdl.value, !pdl.value) pdl.rewrite %root with "rewriter" } @@ -21,7 +21,7 @@ pdl.pattern @operations : benefit(1) { pdl.pattern @rewrite_with_args : benefit(1) { %input = pdl.operand - %root = pdl.operation(%input) + %root = pdl.operation(%input : !pdl.value) pdl.rewrite %root with "rewriter"(%input : !pdl.value) } @@ -36,7 +36,7 @@ pdl.pattern @rewrite_with_params : benefit(1) { pdl.pattern @rewrite_with_args_and_params : benefit(1) { %input = pdl.operand - %root = pdl.operation(%input) + %root = pdl.operation(%input : !pdl.value) pdl.rewrite %root with "rewriter"["I am param"](%input : !pdl.value) } @@ -47,10 +47,10 @@ pdl.pattern @rewrite_with_args_and_params : benefit(1) { pdl.pattern @infer_type_from_operation_replace : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %root = pdl.operation -> %type1, %type2 + %root = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type) pdl.rewrite %root { %type3 = pdl.type - %newOp = pdl.operation "foo.op" -> %type1, %type3 + %newOp = pdl.operation "foo.op" -> (%type1, %type3 : !pdl.type, !pdl.type) pdl.replace %root with %newOp } } @@ -58,12 +58,25 @@ pdl.pattern @infer_type_from_operation_replace : benefit(1) { // ----- // Check that the result type of an operation within a rewrite can be inferred -// from a pdl.replace. +// from types used within the match block. pdl.pattern @infer_type_from_type_used_in_match : benefit(1) { %type1 = pdl.type : i32 %type2 = pdl.type - %root = pdl.operation -> %type1, %type2 + %root = pdl.operation -> (%type1, %type2 : !pdl.type, !pdl.type) + pdl.rewrite %root { + %newOp = pdl.operation "foo.op" -> (%type1, %type2 : !pdl.type, !pdl.type) + } +} + +// ----- + +// Check that the result type of an operation within a rewrite can be inferred +// from types used within the match block. +pdl.pattern @infer_type_from_type_used_in_match : benefit(1) { + %types = pdl.types + %root = pdl.operation -> (%types : !pdl.range) pdl.rewrite %root { - %newOp = pdl.operation "foo.op" -> %type1, %type2 + %otherTypes = pdl.types : [i32, i64] + %newOp = pdl.operation "foo.op" -> (%types, %otherTypes : !pdl.range, !pdl.range) } } -- GitLab From 3a833a0e0e526d4ef3f0037eaa2ace3511f216ce Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 16 Mar 2021 13:11:50 -0700 Subject: [PATCH 0074/1206] [mlir][PDL] Add support for variadic operands and results in the PDL Interpreter This revision extends the PDL Interpreter dialect to add support for variadic operands and results, with ranges of these values represented via the recently added !pdl.range type. To support this extension, three new operations have been added that closely match the single variant: * pdl_interp.check_types : Compare a range of types with a known range. * pdl_interp.create_types : Create a constant range of types. * pdl_interp.get_operands : Get a range of operands from an operation. * pdl_interp.get_results : Get a range of results from an operation. * pdl_interp.switch_types : Switch on a range of types. This revision handles adding support in the interpreter dialect and the conversion from PDL to PDLInterp. Support for variadic operands and results in the bytecode will be added in a followup revision. Differential Revision: https://reviews.llvm.org/D95722 --- .../mlir/Dialect/PDLInterp/IR/PDLInterpOps.td | 315 +++++++++++++++--- mlir/include/mlir/IR/OpBase.td | 2 +- .../PDLToPDLInterp/PDLToPDLInterp.cpp | 293 +++++++++++----- .../Conversion/PDLToPDLInterp/Predicate.cpp | 25 +- .../lib/Conversion/PDLToPDLInterp/Predicate.h | 171 +++++++--- .../PDLToPDLInterp/PredicateTree.cpp | 196 ++++++++--- .../Conversion/PDLToPDLInterp/PredicateTree.h | 6 + mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp | 89 ++--- mlir/lib/Rewrite/ByteCode.cpp | 16 +- mlir/lib/TableGen/Predicate.cpp | 39 ++- .../pdl-to-pdl-interp-matcher.mlir | 207 +++++++++++- .../pdl-to-pdl-interp-rewriter.mlir | 78 +++-- mlir/test/Dialect/PDLInterp/ops.mlir | 8 +- mlir/test/Rewrite/pdl-bytecode.mlir | 46 +-- mlir/test/mlir-tblgen/op-attribute.td | 2 +- 15 files changed, 1105 insertions(+), 388 deletions(-) diff --git a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td index 8f8a5b130175..e35208747ade 100644 --- a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td +++ b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td @@ -168,7 +168,7 @@ def PDLInterp_ApplyRewriteOp : PDLInterp_Op<"apply_rewrite"> { def PDLInterp_AreEqualOp : PDLInterp_PredicateOp<"are_equal", [NoSideEffect, SameTypeOperands]> { - let summary = "Check if two positional values are equivalent"; + let summary = "Check if two positional values or ranges are equivalent"; let description = [{ `pdl_interp.are_equal` operations compare two positional values for equality. On success, this operation branches to the true destination, @@ -241,19 +241,29 @@ def PDLInterp_CheckOperandCountOp let summary = "Check the number of operands of an `Operation`"; let description = [{ `pdl_interp.check_operand_count` operations compare the number of operands - of a given operation value with a constant. On success, this operation - branches to the true destination, otherwise the false destination is taken. + of a given operation value with a constant. The comparison is either exact + or at_least, with the latter used to compare against a minimum number of + expected operands. On success, this operation branches to the true + destination, otherwise the false destination is taken. Example: ```mlir + // Check for exact equality. pdl_interp.check_operand_count of %op is 2 -> ^matchDest, ^failureDest + + // Check for at least N operands. + pdl_interp.check_operand_count of %op is at_least 2 -> ^matchDest, ^failureDest ``` }]; let arguments = (ins PDL_Operation:$operation, - Confined:$count); - let assemblyFormat = "`of` $operation `is` $count attr-dict `->` successors"; + Confined:$count, + UnitAttr:$compareAtLeast); + let assemblyFormat = [{ + `of` $operation `is` (`at_least` $compareAtLeast^)? $count attr-dict + `->` successors + }]; } //===----------------------------------------------------------------------===// @@ -288,19 +298,29 @@ def PDLInterp_CheckResultCountOp let summary = "Check the number of results of an `Operation`"; let description = [{ `pdl_interp.check_result_count` operations compare the number of results - of a given operation value with a constant. On success, this operation - branches to the true destination, otherwise the false destination is taken. + of a given operation value with a constant. The comparison is either exact + or at_least, with the latter used to compare against a minimum number of + expected results. On success, this operation branches to the true + destination, otherwise the false destination is taken. Example: ```mlir - pdl_interp.check_result_count of %op is 0 -> ^matchDest, ^failureDest + // Check for exact equality. + pdl_interp.check_result_count of %op is 2 -> ^matchDest, ^failureDest + + // Check for at least N results. + pdl_interp.check_result_count of %op is at_least 2 -> ^matchDest, ^failureDest ``` }]; let arguments = (ins PDL_Operation:$operation, - Confined:$count); - let assemblyFormat = "`of` $operation `is` $count attr-dict `->` successors"; + Confined:$count, + UnitAttr:$compareAtLeast); + let assemblyFormat = [{ + `of` $operation `is` (`at_least` $compareAtLeast^)? $count attr-dict + `->` successors + }]; } //===----------------------------------------------------------------------===// @@ -326,6 +346,30 @@ def PDLInterp_CheckTypeOp let assemblyFormat = "$value `is` $type attr-dict `->` successors"; } +//===----------------------------------------------------------------------===// +// pdl_interp::CheckTypesOp +//===----------------------------------------------------------------------===// + +def PDLInterp_CheckTypesOp + : PDLInterp_PredicateOp<"check_types", [NoSideEffect]> { + let summary = "Compare a range of types to a range of known values"; + let description = [{ + `pdl_interp.check_types` operations compare a range of types with a + statically known range of types. On success, this operation branches + to the true destination, otherwise the false destination is taken. + + Example: + + ```mlir + pdl_interp.check_types %type are [i32, i64] -> ^matchDest, ^failureDest + ``` + }]; + + let arguments = (ins PDL_RangeOf:$value, + TypeArrayAttr:$types); + let assemblyFormat = "$value `are` $types attr-dict `->` successors"; +} + //===----------------------------------------------------------------------===// // pdl_interp::CreateAttributeOp //===----------------------------------------------------------------------===// @@ -363,21 +407,23 @@ def PDLInterp_CreateOperationOp let summary = "Create an instance of a specific `Operation`"; let description = [{ `pdl_interp.create_operation` operations create an `Operation` instance with - the specified attributes, operands, and result types. + the specified attributes, operands, and result types. See `pdl.operation` + for a more detailed description on the interpretation of the arguments to + this operation. Example: ```mlir // Create an instance of a `foo.op` operation. - %op = pdl_interp.create_operation "foo.op"(%arg0) {"attrA" = %attr0} -> %type, %type + %op = pdl_interp.create_operation "foo.op"(%arg0 : !pdl.value) {"attrA" = %attr0} -> (%type : !pdl.type) ``` }]; let arguments = (ins StrAttr:$name, - Variadic:$operands, + Variadic>:$operands, Variadic:$attributes, StrArrayAttr:$attributeNames, - Variadic:$types); + Variadic>:$types); let results = (outs PDL_Operation:$operation); let builders = [ @@ -386,9 +432,13 @@ def PDLInterp_CreateOperationOp "ArrayAttr":$attributeNames), [{ build($_builder, $_state, $_builder.getType(), name, operands, attributes, attributeNames, types); - }]>]; - let parser = [{ return ::parseCreateOperationOp(parser, result); }]; - let printer = [{ ::print(p, *this); }]; + }]> + ]; + let assemblyFormat = [{ + $name (`(` $operands^ `:` type($operands) `)`)? + custom($attributes, $attributeNames) + (`->` `(` $types^ `:` type($types) `)`)? attr-dict + }]; } //===----------------------------------------------------------------------===// @@ -419,6 +469,28 @@ def PDLInterp_CreateTypeOp : PDLInterp_Op<"create_type", [NoSideEffect]> { ]; } +//===----------------------------------------------------------------------===// +// pdl_interp::CreateTypesOp +//===----------------------------------------------------------------------===// + +def PDLInterp_CreateTypesOp : PDLInterp_Op<"create_types", [NoSideEffect]> { + let summary = "Create an interpreter handle to a range of constant `Type`s"; + let description = [{ + `pdl_interp.create_types` operations generate a handle within the + interpreter for a specific range of constant type values. + + Example: + + ```mlir + pdl_interp.create_types [i64, i64] + ``` + }]; + + let arguments = (ins TypeArrayAttr:$value); + let results = (outs PDL_RangeOf:$result); + let assemblyFormat = "$value attr-dict"; +} + //===----------------------------------------------------------------------===// // pdl_interp::EraseOp //===----------------------------------------------------------------------===// @@ -523,19 +595,20 @@ def PDLInterp_GetDefiningOpOp let summary = "Get the defining operation of a `Value`"; let description = [{ `pdl_interp.get_defining_op` operations try to get the defining operation - of a specific value. If the value is not an operation result, null is - returned. + of a specific value or range of values. In the case of range, the defining + op of the first value is returned. If the value is not an operation result + or range of operand results, null is returned. Example: ```mlir - %op = pdl_interp.get_defining_op of %value + %op = pdl_interp.get_defining_op of %value : !pdl.value ``` }]; - let arguments = (ins PDL_Value:$value); + let arguments = (ins PDL_InstOrRangeOf:$value); let results = (outs PDL_Operation:$operation); - let assemblyFormat = "`of` $value attr-dict"; + let assemblyFormat = "`of` $value `:` type($value) attr-dict"; } //===----------------------------------------------------------------------===// @@ -562,6 +635,49 @@ def PDLInterp_GetOperandOp : PDLInterp_Op<"get_operand", [NoSideEffect]> { let assemblyFormat = "$index `of` $operation attr-dict"; } +//===----------------------------------------------------------------------===// +// pdl_interp::GetOperandsOp +//===----------------------------------------------------------------------===// + +def PDLInterp_GetOperandsOp : PDLInterp_Op<"get_operands", [NoSideEffect]> { + let summary = "Get a specified operand group from an `Operation`"; + let description = [{ + `pdl_interp.get_operands` operations try to get a specific operand + group from an operation. If the expected result is a single Value, null is + returned if the operand group is not of size 1. If a range is expected, + null is returned if the operand group is invalid. If no index is provided, + the returned operand group corresponds to all operands of the operation. + + Example: + + ```mlir + // Get the first group of operands from an operation, and expect a single + // element. + %operand = pdl_interp.get_operands 0 of %op : !pdl.value + + // Get the first group of operands from an operation. + %operands = pdl_interp.get_operands 0 of %op : !pdl.range + + // Get all of the operands from an operation. + %operands = pdl_interp.get_operands of %op : !pdl.range + ``` + }]; + + let arguments = (ins + PDL_Operation:$operation, + OptionalAttr>:$index + ); + let results = (outs PDL_InstOrRangeOf:$value); + let assemblyFormat = "($index^)? `of` $operation `:` type($value) attr-dict"; + let builders = [ + OpBuilder<(ins "Type":$resultType, "Value":$operation, + "Optional":$index), [{ + build($_builder, $_state, resultType, operation, + index ? $_builder.getI32IntegerAttr(*index) : IntegerAttr()); + }]>, + ]; +} + //===----------------------------------------------------------------------===// // pdl_interp::GetResultOp //===----------------------------------------------------------------------===// @@ -586,59 +702,117 @@ def PDLInterp_GetResultOp : PDLInterp_Op<"get_result", [NoSideEffect]> { let assemblyFormat = "$index `of` $operation attr-dict"; } +//===----------------------------------------------------------------------===// +// pdl_interp::GetResultsOp +//===----------------------------------------------------------------------===// + +def PDLInterp_GetResultsOp : PDLInterp_Op<"get_results", [NoSideEffect]> { + let summary = "Get a specified result group from an `Operation`"; + let description = [{ + `pdl_interp.get_results` operations try to get a specific result group + from an operation. If the expected result is a single Value, null is + returned if the result group is not of size 1. If a range is expected, + null is returned if the result group is invalid. If no index is provided, + the returned operand group corresponds to all results of the operation. + + Example: + + ```mlir + // Get the first group of results from an operation, and expect a single + // element. + %result = pdl_interp.get_results 0 of %op : !pdl.value + + // Get the first group of results from an operation. + %results = pdl_interp.get_results 0 of %op : !pdl.range + + // Get all of the results from an operation. + %results = pdl_interp.get_results of %op : !pdl.range + ``` + }]; + + let arguments = (ins + PDL_Operation:$operation, + OptionalAttr>:$index + ); + let results = (outs PDL_InstOrRangeOf:$value); + let assemblyFormat = "($index^)? `of` $operation `:` type($value) attr-dict"; + let builders = [ + OpBuilder<(ins "Type":$resultType, "Value":$operation, + "Optional":$index), [{ + build($_builder, $_state, resultType, operation, + index ? $_builder.getI32IntegerAttr(*index) : IntegerAttr()); + }]>, + OpBuilder<(ins "Value":$operation), [{ + build($_builder, $_state, + pdl::RangeType::get($_builder.getType()), operation, + IntegerAttr()); + }]>, + ]; +} + //===----------------------------------------------------------------------===// // pdl_interp::GetValueTypeOp //===----------------------------------------------------------------------===// -// Get a type from the root operation, held in the rewriter context. -def PDLInterp_GetValueTypeOp : PDLInterp_Op<"get_value_type", [NoSideEffect]> { +def PDLInterp_GetValueTypeOp : PDLInterp_Op<"get_value_type", [NoSideEffect, + TypesMatchWith<"`value` type matches arity of `result`", + "result", "value", "getGetValueTypeOpValueType($_self)">]> { let summary = "Get the result type of a specified `Value`"; let description = [{ `pdl_interp.get_value_type` operations get the resulting type of a specific - value. + value or range thereof. Example: ```mlir - %type = pdl_interp.get_value_type of %value + // Get the type of a single value. + %type = pdl_interp.get_value_type of %value : !pdl.type + + // Get the types of a value range. + %type = pdl_interp.get_value_type of %values : !pdl.range ``` }]; - let arguments = (ins PDL_Value:$value); - let results = (outs PDL_Type:$result); - let assemblyFormat = "`of` $value attr-dict"; + let arguments = (ins PDL_InstOrRangeOf:$value); + let results = (outs PDL_InstOrRangeOf:$result); + let assemblyFormat = "`of` $value `:` type($result) attr-dict"; let builders = [ OpBuilder<(ins "Value":$value), [{ - build($_builder, $_state, $_builder.getType(), value); + Type valType = value.getType(); + Type typeType = $_builder.getType(); + build($_builder, $_state, + valType.isa() ? pdl::RangeType::get(typeType) + : typeType, + value); }]> ]; } //===----------------------------------------------------------------------===// -// pdl_interp::InferredTypeOp +// pdl_interp::InferredTypesOp //===----------------------------------------------------------------------===// -def PDLInterp_InferredTypeOp : PDLInterp_Op<"inferred_type"> { - let summary = "Generate a handle to a Type that is \"inferred\""; +def PDLInterp_InferredTypesOp : PDLInterp_Op<"inferred_types"> { + let summary = "Generate a handle to a range of Types that are \"inferred\""; let description = [{ - `pdl_interp.inferred_type` operations generate a handle to a type that - should be inferred. This signals to other operations, such as - `pdl_interp.create_operation`, that this type should be inferred. + `pdl_interp.inferred_types` operations generate handles to ranges of types + that should be inferred. This signals to other operations, such as + `pdl_interp.create_operation`, that these types should be inferred. Example: ```mlir - pdl_interp.inferred_type + %types = pdl_interp.inferred_types ``` }]; - let results = (outs PDL_Type:$type); + let results = (outs PDL_RangeOf:$type); let assemblyFormat = "attr-dict"; - let builders = [ OpBuilder<(ins), [{ - build($_builder, $_state, $_builder.getType()); - }]>, + build($_builder, $_state, + pdl::RangeType::get($_builder.getType())); + }]> ]; } @@ -650,7 +824,8 @@ def PDLInterp_IsNotNullOp : PDLInterp_PredicateOp<"is_not_null", [NoSideEffect]> { let summary = "Check if a positional value is non-null"; let description = [{ - `pdl_interp.is_not_null` operations check that a positional value exists. On + `pdl_interp.is_not_null` operations check that a positional value or range + exists. For ranges, this does not mean that the range was simply empty. On success, this operation branches to the true destination. Otherwise, the false destination is taken. @@ -718,12 +893,15 @@ def PDLInterp_ReplaceOp : PDLInterp_Op<"replace"> { ```mlir // Replace root node with 2 values: - pdl_interp.replace %root with (%val0, %val1) + pdl_interp.replace %root with (%val0, %val1 : !pdl.type, !pdl.type) ``` }]; let arguments = (ins PDL_Operation:$operation, - Variadic:$replValues); - let assemblyFormat = "$operation `with` `(` $replValues `)` attr-dict"; + Variadic>:$replValues); + let assemblyFormat = [{ + $operation `with` ` ` `(` ($replValues^ `:` type($replValues))? `)` + attr-dict + }]; } //===----------------------------------------------------------------------===// @@ -886,9 +1064,9 @@ def PDLInterp_SwitchTypeOp : PDLInterp_SwitchOp<"switch_type", [NoSideEffect]> { }]; let builders = [ - OpBuilder<(ins "Value":$edge, "TypeRange":$types, "Block *":$defaultDest, - "BlockRange":$dests), [{ - build($_builder, $_state, edge, $_builder.getTypeArrayAttr(types), + OpBuilder<(ins "Value":$edge, "ArrayRef":$types, + "Block *":$defaultDest, "BlockRange":$dests), [{ + build($_builder, $_state, edge, $_builder.getArrayAttr(types), defaultDest, dests); }]>, ]; @@ -898,4 +1076,45 @@ def PDLInterp_SwitchTypeOp : PDLInterp_SwitchOp<"switch_type", [NoSideEffect]> { }]; } +//===----------------------------------------------------------------------===// +// pdl_interp::SwitchTypesOp +//===----------------------------------------------------------------------===// + +def PDLInterp_SwitchTypesOp : PDLInterp_SwitchOp<"switch_types", + [NoSideEffect]> { + let summary = "Switch on a range of `Type` values"; + let description = [{ + `pdl_interp.switch_types` operations compare a range of types with a set of + statically known ranges. If the value matches one of the provided case + values the destination for that case value is taken, otherwise the default + destination is taken. + + Example: + + ```mlir + pdl_interp.switch_types %type is [[i32], [i64, i64]] -> ^i32Dest, ^i64Dest, ^defaultDest + ``` + }]; + + let arguments = (ins + PDL_RangeOf:$value, + TypedArrayAttrBase:$caseValues + ); + let assemblyFormat = [{ + $value `to` $caseValues `(` $cases `)` attr-dict `->` $defaultDest + }]; + + let builders = [ + OpBuilder<(ins "Value":$edge, "ArrayRef":$types, + "Block *":$defaultDest, "BlockRange":$dests), [{ + build($_builder, $_state, edge, $_builder.getArrayAttr(types), + defaultDest, dests); + }]>, + ]; + + let extraClassDeclaration = [{ + auto getCaseTypes() { return caseValues().getAsRange(); } + }]; +} + #endif // MLIR_DIALECT_PDLINTERP_IR_PDLINTERPOPS diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td index 70a5236d885f..5a7037af63d2 100644 --- a/mlir/include/mlir/IR/OpBase.td +++ b/mlir/include/mlir/IR/OpBase.td @@ -1439,7 +1439,7 @@ class TypedArrayAttrBase: ArrayAttrBase< CPred<"$_self.isa<::mlir::ArrayAttr>()">, // Guarantee all elements satisfy the constraints from `element` Concat<"::llvm::all_of($_self.cast<::mlir::ArrayAttr>(), " - "[](::mlir::Attribute attr) { return ", + "[&](::mlir::Attribute attr) { return ", SubstLeaves<"$_self", "attr", element.predicate>, "; })">]>, summary> { diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp index d1da22671d95..57a0885c03c8 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp @@ -56,9 +56,8 @@ private: /// Create an interpreter switch predicate operation, with a provided default /// and several case destinations. - void generateSwitch(Block *currentBlock, Qualifier *question, Value val, - Block *defaultDest, - ArrayRef> dests); + void generateSwitch(SwitchNode *switchNode, Block *currentBlock, + Qualifier *question, Value val, Block *defaultDest); /// Create the interpreter operations to record a successful pattern match. void generateRecordMatch(Block *currentBlock, Block *nextBlock, @@ -88,9 +87,15 @@ private: void generateRewriter(pdl::ResultOp resultOp, DenseMap &rewriteValues, function_ref mapRewriteValue); + void generateRewriter(pdl::ResultsOp resultOp, + DenseMap &rewriteValues, + function_ref mapRewriteValue); void generateRewriter(pdl::TypeOp typeOp, DenseMap &rewriteValues, function_ref mapRewriteValue); + void generateRewriter(pdl::TypesOp typeOp, + DenseMap &rewriteValues, + function_ref mapRewriteValue); /// Generate the values used for resolving the result types of an operation /// created within a dag rewriter region. @@ -200,12 +205,7 @@ Block *PatternLowering::generateMatcher(MatcherNode &node) { // Generate code for a switch node. } else if (auto *switchNode = dyn_cast(&node)) { - // Collect the next blocks for all of the children and generate a switch. - llvm::MapVector children; - for (auto &it : switchNode->getChildren()) - children.insert({it.first, generateMatcher(*it.second)}); - generateSwitch(block, node.getQuestion(), val, nextBlock, - children.takeVector()); + generateSwitch(switchNode, block, node.getQuestion(), val, nextBlock); // Generate code for a success node. } else if (auto *successNode = dyn_cast(&node)) { @@ -242,6 +242,14 @@ Value PatternLowering::getValueAt(Block *cur, Position *pos) { operandPos->getOperandNumber()); break; } + case Predicates::OperandGroupPos: { + auto *operandPos = cast(pos); + Type valueTy = builder.getType(); + value = builder.create( + loc, operandPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy, + parentVal, operandPos->getOperandGroupNumber()); + break; + } case Predicates::AttributePos: { auto *attrPos = cast(pos); value = builder.create( @@ -250,10 +258,10 @@ Value PatternLowering::getValueAt(Block *cur, Position *pos) { break; } case Predicates::TypePos: { - if (parentVal.getType().isa()) - value = builder.create(loc, parentVal); - else + if (parentVal.getType().isa()) value = builder.create(loc, parentVal); + else + value = builder.create(loc, parentVal); break; } case Predicates::ResultPos: { @@ -263,6 +271,14 @@ Value PatternLowering::getValueAt(Block *cur, Position *pos) { resPos->getResultNumber()); break; } + case Predicates::ResultGroupPos: { + auto *resPos = cast(pos); + Type valueTy = builder.getType(); + value = builder.create( + loc, resPos->isVariadic() ? pdl::RangeType::get(valueTy) : valueTy, + parentVal, resPos->getResultGroupNumber()); + break; + } default: llvm_unreachable("Generating unknown Position getter"); break; @@ -277,7 +293,8 @@ void PatternLowering::generatePredicate(Block *currentBlock, Block *falseDest) { builder.setInsertionPointToEnd(currentBlock); Location loc = val.getLoc(); - switch (question->getKind()) { + Predicates::Kind kind = question->getKind(); + switch (kind) { case Predicates::IsNotNullQuestion: builder.create(loc, val, trueDest, falseDest); break; @@ -289,8 +306,12 @@ void PatternLowering::generatePredicate(Block *currentBlock, } case Predicates::TypeQuestion: { auto *ans = cast(answer); - builder.create( - loc, val, TypeAttr::get(ans->getValue()), trueDest, falseDest); + if (val.getType().isa()) + builder.create( + loc, val, ans->getValue().cast(), trueDest, falseDest); + else + builder.create( + loc, val, ans->getValue().cast(), trueDest, falseDest); break; } case Predicates::AttributeQuestion: { @@ -299,18 +320,20 @@ void PatternLowering::generatePredicate(Block *currentBlock, trueDest, falseDest); break; } - case Predicates::OperandCountQuestion: { - auto *unsignedAnswer = cast(answer); + case Predicates::OperandCountAtLeastQuestion: + case Predicates::OperandCountQuestion: builder.create( - loc, val, unsignedAnswer->getValue(), trueDest, falseDest); + loc, val, cast(answer)->getValue(), + /*compareAtLeast=*/kind == Predicates::OperandCountAtLeastQuestion, + trueDest, falseDest); break; - } - case Predicates::ResultCountQuestion: { - auto *unsignedAnswer = cast(answer); + case Predicates::ResultCountAtLeastQuestion: + case Predicates::ResultCountQuestion: builder.create( - loc, val, unsignedAnswer->getValue(), trueDest, falseDest); + loc, val, cast(answer)->getValue(), + /*compareAtLeast=*/kind == Predicates::ResultCountAtLeastQuestion, + trueDest, falseDest); break; - } case Predicates::EqualToQuestion: { auto *equalToQuestion = cast(question); builder.create( @@ -336,7 +359,7 @@ void PatternLowering::generatePredicate(Block *currentBlock, template static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder, - ArrayRef> dests) { + llvm::MapVector &dests) { std::vector values; std::vector blocks; values.reserve(dests.size()); @@ -348,27 +371,83 @@ static void createSwitchOp(Value val, Block *defaultDest, OpBuilder &builder, builder.create(val.getLoc(), val, values, defaultDest, blocks); } -void PatternLowering::generateSwitch( - Block *currentBlock, Qualifier *question, Value val, Block *defaultDest, - ArrayRef> dests) { +void PatternLowering::generateSwitch(SwitchNode *switchNode, + Block *currentBlock, Qualifier *question, + Value val, Block *defaultDest) { + // If the switch question is not an exact answer, i.e. for the `at_least` + // cases, we generate a special block sequence. + Predicates::Kind kind = question->getKind(); + if (kind == Predicates::OperandCountAtLeastQuestion || + kind == Predicates::ResultCountAtLeastQuestion) { + // Order the children such that the cases are in reverse numerical order. + SmallVector sortedChildren( + llvm::seq(0, switchNode->getChildren().size())); + llvm::sort(sortedChildren, [&](unsigned lhs, unsigned rhs) { + return cast(switchNode->getChild(lhs).first)->getValue() > + cast(switchNode->getChild(rhs).first)->getValue(); + }); + + // Build the destination for each child using the next highest child as a + // a failure destination. This essentially creates the following control + // flow: + // + // if (operand_count < 1) + // goto failure + // if (child1.match()) + // ... + // + // if (operand_count < 2) + // goto failure + // if (child2.match()) + // ... + // + // failure: + // ... + // + failureBlockStack.push_back(defaultDest); + for (unsigned idx : sortedChildren) { + auto &child = switchNode->getChild(idx); + Block *childBlock = generateMatcher(*child.second); + Block *predicateBlock = builder.createBlock(childBlock); + generatePredicate(predicateBlock, question, child.first, val, childBlock, + defaultDest); + failureBlockStack.back() = predicateBlock; + } + Block *firstPredicateBlock = failureBlockStack.pop_back_val(); + currentBlock->getOperations().splice(currentBlock->end(), + firstPredicateBlock->getOperations()); + firstPredicateBlock->erase(); + return; + } + + // Otherwise, generate each of the children and generate an interpreter + // switch. + llvm::MapVector children; + for (auto &it : switchNode->getChildren()) + children.insert({it.first, generateMatcher(*it.second)}); builder.setInsertionPointToEnd(currentBlock); + switch (question->getKind()) { case Predicates::OperandCountQuestion: return createSwitchOp(val, defaultDest, builder, dests); + int32_t>(val, defaultDest, builder, children); case Predicates::ResultCountQuestion: return createSwitchOp(val, defaultDest, builder, dests); + int32_t>(val, defaultDest, builder, children); case Predicates::OperationNameQuestion: return createSwitchOp(val, defaultDest, builder, - dests); + children); case Predicates::TypeQuestion: + if (val.getType().isa()) { + return createSwitchOp( + val, defaultDest, builder, children); + } return createSwitchOp( - val, defaultDest, builder, dests); + val, defaultDest, builder, children); case Predicates::AttributeQuestion: return createSwitchOp( - val, defaultDest, builder, dests); + val, defaultDest, builder, children); default: llvm_unreachable("Generating unknown switch predicate."); } @@ -436,6 +515,11 @@ SymbolRefAttr PatternLowering::generateRewriter( return newValue = builder.create( typeOp.getLoc(), type); } + } else if (pdl::TypesOp typeOp = dyn_cast(oldOp)) { + if (ArrayAttr type = typeOp.typesAttr()) { + return newValue = builder.create( + typeOp.getLoc(), typeOp.getType(), type); + } } // Otherwise, add this as an input to the rewriter. @@ -460,10 +544,10 @@ SymbolRefAttr PatternLowering::generateRewriter( for (Operation &rewriteOp : *rewriter.getBody()) { llvm::TypeSwitch(&rewriteOp) .Case( - [&](auto op) { - this->generateRewriter(op, rewriteValues, mapRewriteValue); - }); + pdl::OperationOp, pdl::ReplaceOp, pdl::ResultOp, pdl::ResultsOp, + pdl::TypeOp, pdl::TypesOp>([&](auto op) { + this->generateRewriter(op, rewriteValues, mapRewriteValue); + }); } } @@ -529,14 +613,39 @@ void PatternLowering::generateRewriter( rewriteValues[operationOp.op()] = createdOp; // Generate accesses for any results that have their types constrained. - for (auto it : llvm::enumerate(operationOp.types())) { + // Handle the case where there is a single range representing all of the + // result types. + OperandRange resultTys = operationOp.types(); + if (resultTys.size() == 1 && resultTys[0].getType().isa()) { + Value &type = rewriteValues[resultTys[0]]; + if (!type) { + auto results = builder.create(loc, createdOp); + type = builder.create(loc, results); + } + return; + } + + // Otherwise, populate the individual results. + bool seenVariableLength = false; + Type valueTy = builder.getType(); + Type valueRangeTy = pdl::RangeType::get(valueTy); + for (auto it : llvm::enumerate(resultTys)) { Value &type = rewriteValues[it.value()]; if (type) continue; - - Value getResultVal = builder.create( - loc, builder.getType(), createdOp, it.index()); - type = builder.create(loc, getResultVal); + bool isVariadic = it.value().getType().isa(); + seenVariableLength |= isVariadic; + + // After a variable length result has been seen, we need to use result + // groups because the exact index of the result is not statically known. + Value resultVal; + if (seenVariableLength) + resultVal = builder.create( + loc, isVariadic ? valueRangeTy : valueTy, createdOp, it.index()); + else + resultVal = builder.create( + loc, valueTy, createdOp, it.index()); + type = builder.create(loc, resultVal); } } @@ -549,11 +658,12 @@ void PatternLowering::generateRewriter( // for using an operation for simplicitly, but the interpreter isn't as // user facing. if (Value replOp = replaceOp.replOperation()) { - pdl::OperationOp op = cast(replOp.getDefiningOp()); - for (unsigned i = 0, e = op.types().size(); i < e; ++i) - replOperands.push_back(builder.create( - replOp.getLoc(), builder.getType(), - mapRewriteValue(replOp), i)); + // Don't use replace if we know the replaced operation has no results. + auto opOp = replaceOp.operation().getDefiningOp(); + if (!opOp || !opOp.types().empty()) { + replOperands.push_back(builder.create( + replOp.getLoc(), mapRewriteValue(replOp))); + } } else { for (Value operand : replaceOp.replValues()) replOperands.push_back(mapRewriteValue(operand)); @@ -578,15 +688,33 @@ void PatternLowering::generateRewriter( mapRewriteValue(resultOp.parent()), resultOp.index()); } +void PatternLowering::generateRewriter( + pdl::ResultsOp resultOp, DenseMap &rewriteValues, + function_ref mapRewriteValue) { + rewriteValues[resultOp] = builder.create( + resultOp.getLoc(), resultOp.getType(), mapRewriteValue(resultOp.parent()), + resultOp.index()); +} + void PatternLowering::generateRewriter( pdl::TypeOp typeOp, DenseMap &rewriteValues, function_ref mapRewriteValue) { // If the type isn't constant, the users (e.g. OperationOp) will resolve this // type. if (TypeAttr typeAttr = typeOp.typeAttr()) { - Value newType = + rewriteValues[typeOp] = builder.create(typeOp.getLoc(), typeAttr); - rewriteValues[typeOp] = newType; + } +} + +void PatternLowering::generateRewriter( + pdl::TypesOp typeOp, DenseMap &rewriteValues, + function_ref mapRewriteValue) { + // If the type isn't constant, the users (e.g. OperationOp) will resolve this + // type. + if (ArrayAttr typeAttr = typeOp.typesAttr()) { + rewriteValues[typeOp] = builder.create( + typeOp.getLoc(), typeOp.getType(), typeAttr); } } @@ -594,28 +722,38 @@ void PatternLowering::generateOperationResultTypeRewriter( pdl::OperationOp op, SmallVectorImpl &types, DenseMap &rewriteValues, function_ref mapRewriteValue) { - // Functor that returns if the given use can be used to infer a type. + // Look for an operation that was replaced by `op`. The result types will be + // inferred from the results that were replaced. Block *rewriterBlock = op->getBlock(); - auto getReplacedOperationFrom = [&](OpOperand &use) -> Operation * { + Value replacedOp; + for (OpOperand &use : op.op().getUses()) { // Check that the use corresponds to a ReplaceOp and that it is the // replacement value, not the operation being replaced. pdl::ReplaceOp replOpUser = dyn_cast(use.getOwner()); if (!replOpUser || use.getOperandNumber() == 0) - return nullptr; + continue; // Make sure the replaced operation was defined before this one. - Operation *replacedOp = replOpUser.operation().getDefiningOp(); - if (replacedOp->getBlock() != rewriterBlock || - replacedOp->isBeforeInBlock(op)) - return replacedOp; - return nullptr; - }; + Value replOpVal = replOpUser.operation(); + Operation *replacedOp = replOpVal.getDefiningOp(); + if (replacedOp->getBlock() == rewriterBlock && + !replacedOp->isBeforeInBlock(op)) + continue; + + Value replacedOpResults = builder.create( + replacedOp->getLoc(), mapRewriteValue(replOpVal)); + types.push_back(builder.create( + replacedOp->getLoc(), replacedOpResults)); + return; + } + + // Check if the operation has type inference support. + if (op.hasTypeInference()) { + types.push_back(builder.create(op.getLoc())); + return; + } - // If non-None/non-Null, this is an operation that is replaced by `op`. - // If Null, there is no full replacement operation for `op`. - // If None, a replacement operation hasn't been searched for. - Optional fullReplacedOperation; - bool hasTypeInference = op.hasTypeInference(); - auto resultTypeValues = op.types(); + // Otherwise, handle inference for each of the result types individually. + OperandRange resultTypeValues = op.types(); types.reserve(resultTypeValues.size()); for (auto it : llvm::enumerate(resultTypeValues)) { Value resultType = it.value(); @@ -632,30 +770,11 @@ void PatternLowering::generateOperationResultTypeRewriter( continue; } - // Check if the operation has type inference support. - if (hasTypeInference) { - types.push_back(builder.create(op.getLoc())); - continue; - } - - // Look for an operation that was replaced by `op`. The result type will be - // inferred from the result that was replaced. There is guaranteed to be a - // replacement for either the op, or this specific result. Note that this is - // guaranteed by the verifier of `pdl::OperationOp`. - Operation *replacedOp = nullptr; - if (!fullReplacedOperation.hasValue()) { - for (OpOperand &use : op.op().getUses()) - if ((replacedOp = getReplacedOperationFrom(use))) - break; - fullReplacedOperation = replacedOp; - assert(fullReplacedOperation && - "expected replaced op to infer a result type from"); - } else { - replacedOp = fullReplacedOperation.getValue(); - } - - auto replOpOp = cast(replacedOp); - types.push_back(mapRewriteValue(replOpOp.types()[it.index()])); + // The verifier asserts that the result types of each pdl.operation can be + // inferred. If we reach here, there is a bug either in the logic above or + // in the verifier for pdl.operation. + op->emitOpError() << "unable to infer result type for operation"; + llvm_unreachable("unable to infer result type for operation"); } } diff --git a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp index 3eaeb13cffc0..8983ecb8d324 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.cpp @@ -17,6 +17,13 @@ using namespace mlir::pdl_to_pdl_interp; Position::~Position() {} +/// Returns the depth of the first ancestor operation position. +unsigned Position::getOperationDepth() const { + if (const auto *operationPos = dyn_cast(this)) + return operationPos->getDepth(); + return parent->getOperationDepth(); +} + //===----------------------------------------------------------------------===// // AttributePosition @@ -32,18 +39,8 @@ OperandPosition::OperandPosition(const KeyTy &key) : Base(key) { } //===----------------------------------------------------------------------===// -// OperationPosition - -OperationPosition *OperationPosition::get(StorageUniquer &uniquer, - ArrayRef index) { - assert(!index.empty() && "expected at least two indices"); - - // Set the parent position if this isn't the root. - Position *parent = nullptr; - if (index.size() > 1) { - auto *node = OperationPosition::get(uniquer, index.drop_back()); - parent = OperandPosition::get(uniquer, std::make_pair(node, index.back())); - } - return uniquer.get( - [parent](OperationPosition *node) { node->parent = parent; }, index); +// OperandGroupPosition + +OperandGroupPosition::OperandGroupPosition(const KeyTy &key) : Base(key) { + parent = std::get<0>(key); } diff --git a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h index 4d5c909465da..1c8fece05e07 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h +++ b/mlir/lib/Conversion/PDLToPDLInterp/Predicate.h @@ -45,8 +45,10 @@ enum Kind : unsigned { /// Positions, ordered by decreasing priority. OperationPos, OperandPos, + OperandGroupPos, AttributePos, ResultPos, + ResultGroupPos, TypePos, // Questions, ordered by dependency and decreasing priority. @@ -54,7 +56,9 @@ enum Kind : unsigned { OperationNameQuestion, TypeQuestion, AttributeQuestion, + OperandCountAtLeastQuestion, OperandCountQuestion, + ResultCountAtLeastQuestion, ResultCountQuestion, EqualToQuestion, ConstraintQuestion, @@ -129,21 +133,15 @@ struct OperationPosition; /// predicates, and assists generating bytecode and memory management. /// /// Operation positions form the base of other positions, which are formed -/// relative to a parent operation, e.g. OperandPosition<[0] -> 1>. Operations -/// are indexed by child index: [0, 1, 2] refers to the 3rd child of the 2nd -/// child of the root operation. -/// -/// Positions are linked to their parent position, which describes how to obtain -/// a positional value. As a concrete example, getting OperationPosition<[0, 1]> -/// would be `root->getOperand(1)->getDefiningOp()`, so its parent is -/// OperandPosition<[0] -> 1>, whose parent is OperationPosition<[0]>. +/// relative to a parent operation. Operations are anchored at Operand nodes, +/// except for the root operation which is parentless. class Position : public StorageUniquer::BaseStorage { public: explicit Position(Predicates::Kind kind) : kind(kind) {} virtual ~Position(); - /// Returns the base node position. This is an array of indices. - virtual ArrayRef getIndex() const = 0; + /// Returns the depth of the first ancestor operation position. + unsigned getOperationDepth() const; /// Returns the parent position. The root operation position has no parent. Position *getParent() const { return parent; } @@ -170,9 +168,6 @@ struct AttributePosition Predicates::AttributePos> { explicit AttributePosition(const KeyTy &key); - /// Returns the index of this position. - ArrayRef getIndex() const final { return parent->getIndex(); } - /// Returns the attribute name of this position. Identifier getName() const { return key.second; } }; @@ -187,42 +182,61 @@ struct OperandPosition Predicates::OperandPos> { explicit OperandPosition(const KeyTy &key); - /// Returns the index of this position. - ArrayRef getIndex() const final { return parent->getIndex(); } - /// Returns the operand number of this position. unsigned getOperandNumber() const { return key.second; } }; +//===----------------------------------------------------------------------===// +// OperandGroupPosition + +/// A position describing an operand group of an operation. +struct OperandGroupPosition + : public PredicateBase< + OperandGroupPosition, Position, + std::tuple, bool>, + Predicates::OperandGroupPos> { + explicit OperandGroupPosition(const KeyTy &key); + + /// Returns a hash suitable for the given keytype. + static llvm::hash_code hashKey(const KeyTy &key) { + return llvm::hash_value(key); + } + + /// Returns the group number of this position. If None, this group refers to + /// all operands. + Optional getOperandGroupNumber() const { return std::get<1>(key); } + + /// Returns if the operand group has unknown size. If false, the operand group + /// has at max one element. + bool isVariadic() const { return std::get<2>(key); } +}; + //===----------------------------------------------------------------------===// // OperationPosition /// An operation position describes an operation node in the IR. Other position /// kinds are formed with respect to an operation position. -struct OperationPosition - : public PredicateBase, - Predicates::OperationPos> { - using Base::Base; +struct OperationPosition : public PredicateBase, + Predicates::OperationPos> { + explicit OperationPosition(const KeyTy &key) : Base(key) { + parent = key.first; + } - /// Gets the root position, which is always [0]. + /// Gets the root position. static OperationPosition *getRoot(StorageUniquer &uniquer) { - return get(uniquer, ArrayRef(0)); + return Base::get(uniquer, nullptr, 0); } - /// Gets a node position for the given index. - static OperationPosition *get(StorageUniquer &uniquer, - ArrayRef index); - - /// Constructs an instance with the given storage allocator. - static OperationPosition *construct(StorageUniquer::StorageAllocator &alloc, - ArrayRef key) { - return Base::construct(alloc, alloc.copyInto(key)); + /// Gets an operation position with the given parent. + static OperationPosition *get(StorageUniquer &uniquer, Position *parent) { + return Base::get(uniquer, parent, parent->getOperationDepth() + 1); } - /// Returns the index of this position. - ArrayRef getIndex() const final { return key; } + /// Returns the depth of this position. + unsigned getDepth() const { return key.second; } /// Returns if this operation position corresponds to the root. - bool isRoot() const { return key.size() == 1 && key[0] == 0; } + bool isRoot() const { return getDepth() == 0; } }; //===----------------------------------------------------------------------===// @@ -235,13 +249,37 @@ struct ResultPosition Predicates::ResultPos> { explicit ResultPosition(const KeyTy &key) : Base(key) { parent = key.first; } - /// Returns the index of this position. - ArrayRef getIndex() const final { return key.first->getIndex(); } - /// Returns the result number of this position. unsigned getResultNumber() const { return key.second; } }; +//===----------------------------------------------------------------------===// +// ResultGroupPosition + +/// A position describing a result group of an operation. +struct ResultGroupPosition + : public PredicateBase< + ResultGroupPosition, Position, + std::tuple, bool>, + Predicates::ResultGroupPos> { + explicit ResultGroupPosition(const KeyTy &key) : Base(key) { + parent = std::get<0>(key); + } + + /// Returns a hash suitable for the given keytype. + static llvm::hash_code hashKey(const KeyTy &key) { + return llvm::hash_value(key); + } + + /// Returns the group number of this position. If None, this group refers to + /// all results. + Optional getResultGroupNumber() const { return std::get<1>(key); } + + /// Returns if the result group has unknown size. If false, the result group + /// has at max one element. + bool isVariadic() const { return std::get<2>(key); } +}; + //===----------------------------------------------------------------------===// // TypePosition @@ -250,14 +288,11 @@ struct ResultPosition struct TypePosition : public PredicateBase { explicit TypePosition(const KeyTy &key) : Base(key) { - assert((isa(key) || isa(key) || - isa(key)) && + assert((isa(key)) && "expected parent to be an attribute, operand, or result"); parent = key; } - - /// Returns the index of this position. - ArrayRef getIndex() const final { return key->getIndex(); } }; //===----------------------------------------------------------------------===// @@ -311,8 +346,9 @@ struct TrueAnswer using Base::Base; }; -/// An Answer representing a `Type` value. -struct TypeAnswer : public PredicateBase { using Base::Base; }; @@ -365,6 +401,9 @@ struct IsNotNullQuestion struct OperandCountQuestion : public PredicateBase {}; +struct OperandCountAtLeastQuestion + : public PredicateBase {}; /// Compare the name of an operation with a known value. struct OperationNameQuestion @@ -375,6 +414,9 @@ struct OperationNameQuestion struct ResultCountQuestion : public PredicateBase {}; +struct ResultCountAtLeastQuestion + : public PredicateBase {}; /// Compare the type of an attribute or value with a known type. struct TypeQuestion : public PredicateBase(); registerParametricStorageType(); + registerParametricStorageType(); registerParametricStorageType(); registerParametricStorageType(); + registerParametricStorageType(); registerParametricStorageType(); // Register the types of Questions with the uniquer. @@ -409,8 +453,10 @@ public: registerSingletonStorageType(); registerSingletonStorageType(); registerSingletonStorageType(); + registerSingletonStorageType(); registerSingletonStorageType(); registerSingletonStorageType(); + registerSingletonStorageType(); registerSingletonStorageType(); } }; @@ -433,10 +479,10 @@ public: Position *getRoot() { return OperationPosition::getRoot(uniquer); } /// Returns the parent position defining the value held by the given operand. - OperationPosition *getParent(OperandPosition *p) { - std::vector index = p->getIndex(); - index.push_back(p->getOperandNumber()); - return OperationPosition::get(uniquer, index); + OperationPosition *getOperandDefiningOp(Position *p) { + assert((isa(p)) && + "expected operand position"); + return OperationPosition::get(uniquer, p); } /// Returns an attribute position for an attribute of the given operation. @@ -449,11 +495,29 @@ public: return OperandPosition::get(uniquer, p, operand); } + /// Returns a position for a group of operands of the given operation. + Position *getOperandGroup(OperationPosition *p, Optional group, + bool isVariadic) { + return OperandGroupPosition::get(uniquer, p, group, isVariadic); + } + Position *getAllOperands(OperationPosition *p) { + return getOperandGroup(p, /*group=*/llvm::None, /*isVariadic=*/true); + } + /// Returns a result position for a result of the given operation. Position *getResult(OperationPosition *p, unsigned result) { return ResultPosition::get(uniquer, p, result); } + /// Returns a position for a group of results of the given operation. + Position *getResultGroup(OperationPosition *p, Optional group, + bool isVariadic) { + return ResultGroupPosition::get(uniquer, p, group, isVariadic); + } + Position *getAllResults(OperationPosition *p) { + return getResultGroup(p, /*group=*/llvm::None, /*isVariadic=*/true); + } + /// Returns a type position for the given entity. Position *getType(Position *p) { return TypePosition::get(uniquer, p); } @@ -496,6 +560,10 @@ public: return {OperandCountQuestion::get(uniquer), UnsignedAnswer::get(uniquer, count)}; } + Predicate getOperandCountAtLeast(unsigned count) { + return {OperandCountAtLeastQuestion::get(uniquer), + UnsignedAnswer::get(uniquer, count)}; + } /// Create a predicate comparing the name of an operation to a known value. Predicate getOperationName(StringRef name) { @@ -509,10 +577,15 @@ public: return {ResultCountQuestion::get(uniquer), UnsignedAnswer::get(uniquer, count)}; } + Predicate getResultCountAtLeast(unsigned count) { + return {ResultCountAtLeastQuestion::get(uniquer), + UnsignedAnswer::get(uniquer, count)}; + } /// Create a predicate comparing the type of an attribute or value to a known - /// type. - Predicate getTypeConstraint(Type type) { + /// type. The value is stored as either a TypeAttr, or an ArrayAttr of + /// TypeAttr. + Predicate getTypeConstraint(Attribute type) { return {TypeQuestion::get(uniquer), TypeAnswer::get(uniquer, type)}; } diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp index 885fbad0f976..bcd32dfa4bef 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp @@ -28,7 +28,13 @@ static void getTreePredicates(std::vector &predList, /// Compares the depths of two positions. static bool comparePosDepth(Position *lhs, Position *rhs) { - return lhs->getIndex().size() < rhs->getIndex().size(); + return lhs->getOperationDepth() < rhs->getOperationDepth(); +} + +/// Returns the number of non-range elements within `values`. +static unsigned getNumNonRangeValues(ValueRange values) { + return llvm::count_if(values.getTypes(), + [](Type type) { return !type.isa(); }); } static void getTreePredicates(std::vector &predList, @@ -46,28 +52,50 @@ static void getTreePredicates(std::vector &predList, predList.emplace_back(pos, builder.getAttributeConstraint(value)); } -static void getTreePredicates(std::vector &predList, - Value val, PredicateBuilder &builder, - DenseMap &inputs, - OperandPosition *pos) { - assert(val.getType().isa() && "expected value type"); - - // Prevent traversal into a null value. - predList.emplace_back(pos, builder.getIsNotNull()); +/// Collect all of the predicates for the given operand position. +static void getOperandTreePredicates(std::vector &predList, + Value val, PredicateBuilder &builder, + DenseMap &inputs, + Position *pos) { + Type valueType = val.getType(); + bool isVariadic = valueType.isa(); // If this is a typed operand, add a type constraint. - if (auto in = val.getDefiningOp()) { - if (Value type = in.type()) - getTreePredicates(predList, type, builder, inputs, builder.getType(pos)); - - // Otherwise, recurse into a result node. - } else if (auto resultOp = val.getDefiningOp()) { - OperationPosition *parentPos = builder.getParent(pos); - Position *resultPos = builder.getResult(parentPos, resultOp.index()); - predList.emplace_back(parentPos, builder.getIsNotNull()); - predList.emplace_back(resultPos, builder.getEqualTo(pos)); - getTreePredicates(predList, resultOp.parent(), builder, inputs, parentPos); - } + TypeSwitch(val.getDefiningOp()) + .Case([&](auto op) { + // Prevent traversal into a null value if the operand has a proper + // index. + if (std::is_same::value || + cast(pos)->getOperandGroupNumber()) + predList.emplace_back(pos, builder.getIsNotNull()); + + if (Value type = op.type()) + getTreePredicates(predList, type, builder, inputs, + builder.getType(pos)); + }) + .Case([&](auto op) { + Optional index = op.index(); + + // Prevent traversal into a null value if the result has a proper index. + if (index) + predList.emplace_back(pos, builder.getIsNotNull()); + + // Get the parent operation of this operand. + OperationPosition *parentPos = builder.getOperandDefiningOp(pos); + predList.emplace_back(parentPos, builder.getIsNotNull()); + + // Ensure that the operands match the corresponding results of the + // parent operation. + Position *resultPos = nullptr; + if (std::is_same::value) + resultPos = builder.getResult(parentPos, *index); + else + resultPos = builder.getResultGroup(parentPos, index, isVariadic); + predList.emplace_back(resultPos, builder.getEqualTo(pos)); + + // Collect the predicates of the parent operation. + getTreePredicates(predList, op.parent(), builder, inputs, parentPos); + }); } static void getTreePredicates(std::vector &predList, @@ -86,11 +114,25 @@ static void getTreePredicates(std::vector &predList, if (Optional opName = op.name()) predList.emplace_back(pos, builder.getOperationName(*opName)); - // Check that the operation has the proper number of operands and results. + // Check that the operation has the proper number of operands. If there are + // any variable length operands, we check a minimum instead of an exact count. OperandRange operands = op.operands(); + unsigned minOperands = getNumNonRangeValues(operands); + if (minOperands != operands.size()) { + if (minOperands) + predList.emplace_back(pos, builder.getOperandCountAtLeast(minOperands)); + } else { + predList.emplace_back(pos, builder.getOperandCount(minOperands)); + } + + // Check that the operation has the proper number of results. If there are + // any variable length results, we check a minimum instead of an exact count. OperandRange types = op.types(); - predList.emplace_back(pos, builder.getOperandCount(operands.size())); - predList.emplace_back(pos, builder.getResultCount(types.size())); + unsigned minResults = getNumNonRangeValues(types); + if (minResults == types.size()) + predList.emplace_back(pos, builder.getResultCount(types.size())); + else if (minResults) + predList.emplace_back(pos, builder.getResultCountAtLeast(minResults)); // Recurse into any attributes, operands, or results. for (auto it : llvm::zip(op.attributeNames(), op.attributes())) { @@ -99,15 +141,47 @@ static void getTreePredicates(std::vector &predList, builder.getAttribute(opPos, std::get<0>(it).cast().getValue())); } - for (auto operandIt : llvm::enumerate(operands)) { - getTreePredicates(predList, operandIt.value(), builder, inputs, - builder.getOperand(opPos, operandIt.index())); + + // Process the operands and results of the operation. For all values up to + // the first variable length value, we use the concrete operand/result + // number. After that, we use the "group" given that we can't know the + // concrete indices until runtime. If there is only one variadic operand + // group, we treat it as all of the operands/results of the operation. + /// Operands. + if (operands.size() == 1 && operands[0].getType().isa()) { + getTreePredicates(predList, operands.front(), builder, inputs, + builder.getAllOperands(opPos)); + } else { + bool foundVariableLength = false; + for (auto operandIt : llvm::enumerate(operands)) { + bool isVariadic = operandIt.value().getType().isa(); + foundVariableLength |= isVariadic; + + Position *pos = + foundVariableLength + ? builder.getOperandGroup(opPos, operandIt.index(), isVariadic) + : builder.getOperand(opPos, operandIt.index()); + getTreePredicates(predList, operandIt.value(), builder, inputs, pos); + } } - for (auto &resultIt : llvm::enumerate(types)) { - auto *resultPos = builder.getResult(pos, resultIt.index()); - predList.emplace_back(resultPos, builder.getIsNotNull()); - getTreePredicates(predList, resultIt.value(), builder, inputs, - builder.getType(resultPos)); + /// Results. + if (types.size() == 1 && types[0].getType().isa()) { + getTreePredicates(predList, types.front(), builder, inputs, + builder.getType(builder.getAllResults(opPos))); + } else { + bool foundVariableLength = false; + for (auto &resultIt : llvm::enumerate(types)) { + bool isVariadic = resultIt.value().getType().isa(); + foundVariableLength |= isVariadic; + + auto *resultPos = + foundVariableLength + ? builder.getResultGroup(pos, resultIt.index(), isVariadic) + : builder.getResult(pos, resultIt.index()); + predList.emplace_back(resultPos, builder.getIsNotNull()); + getTreePredicates(predList, resultIt.value(), builder, inputs, + builder.getType(resultPos)); + } } } @@ -115,12 +189,14 @@ static void getTreePredicates(std::vector &predList, Value val, PredicateBuilder &builder, DenseMap &inputs, TypePosition *pos) { - assert(val.getType().isa() && "expected value type"); - pdl::TypeOp typeOp = cast(val.getDefiningOp()); - // Check for a constraint on a constant type. - if (Optional type = typeOp.type()) - predList.emplace_back(pos, builder.getTypeConstraint(*type)); + if (pdl::TypeOp typeOp = val.getDefiningOp()) { + if (Attribute type = typeOp.typeAttr()) + predList.emplace_back(pos, builder.getTypeConstraint(type)); + } else if (pdl::TypesOp typeOp = val.getDefiningOp()) { + if (Attribute typeAttr = typeOp.typesAttr()) + predList.emplace_back(pos, builder.getTypeConstraint(typeAttr)); + } } /// Collect the tree predicates anchored at the given value. @@ -133,8 +209,8 @@ static void getTreePredicates(std::vector &predList, if (!it.second) { // If this is an input value that has been visited in the tree, add a // constraint to ensure that both instances refer to the same value. - if (isa( - val.getDefiningOp())) { + if (isa(val.getDefiningOp())) { auto minMaxPositions = std::minmax(pos, it.first->second, comparePosDepth); predList.emplace_back(minMaxPositions.second, @@ -144,9 +220,11 @@ static void getTreePredicates(std::vector &predList, } TypeSwitch(pos) - .Case([&](auto *derivedPos) { - getTreePredicates(predList, val, builder, inputs, derivedPos); + .Case([&](auto *pos) { + getTreePredicates(predList, val, builder, inputs, pos); + }) + .Case([&](auto *pos) { + getOperandTreePredicates(predList, val, builder, inputs, pos); }) .Default([](auto *) { llvm_unreachable("unexpected position kind"); }); } @@ -180,11 +258,30 @@ static void getResultPredicates(pdl::ResultOp op, Position *&resultPos = inputs[op]; if (resultPos) return; + + // Ensure that the result isn't null. auto *parentPos = cast(inputs.lookup(op.parent())); resultPos = builder.getResult(parentPos, op.index()); predList.emplace_back(resultPos, builder.getIsNotNull()); } +static void getResultPredicates(pdl::ResultsOp op, + std::vector &predList, + PredicateBuilder &builder, + DenseMap &inputs) { + Position *&resultPos = inputs[op]; + if (resultPos) + return; + + // Ensure that the result isn't null if the result has an index. + auto *parentPos = cast(inputs.lookup(op.parent())); + bool isVariadic = op.getType().isa(); + Optional index = op.index(); + resultPos = builder.getResultGroup(parentPos, index, isVariadic); + if (index) + predList.emplace_back(resultPos, builder.getIsNotNull()); +} + /// Collect all of the predicates that cannot be determined via walking the /// tree. static void getNonTreePredicates(pdl::PatternOp pattern, @@ -192,10 +289,13 @@ static void getNonTreePredicates(pdl::PatternOp pattern, PredicateBuilder &builder, DenseMap &inputs) { for (Operation &op : pattern.body().getOps()) { - if (auto constraintOp = dyn_cast(&op)) - getConstraintPredicates(constraintOp, predList, builder, inputs); - else if (auto resultOp = dyn_cast(&op)) - getResultPredicates(resultOp, predList, builder, inputs); + TypeSwitch(&op) + .Case([&](auto constraintOp) { + getConstraintPredicates(constraintOp, predList, builder, inputs); + }) + .Case([&](auto resultOp) { + getResultPredicates(resultOp, predList, builder, inputs); + }); } } @@ -254,10 +354,10 @@ struct OrderedPredicate { // * lower position dependency // * lower predicate dependency auto *rhsPos = rhs.position; - return std::make_tuple(primary, secondary, rhsPos->getIndex().size(), + return std::make_tuple(primary, secondary, rhsPos->getOperationDepth(), rhsPos->getKind(), rhs.question->getKind()) > std::make_tuple(rhs.primary, rhs.secondary, - position->getIndex().size(), position->getKind(), + position->getOperationDepth(), position->getKind(), question->getKind()); } }; diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h index 1621fa96747b..ac2fa98d7c7b 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h +++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h @@ -190,6 +190,12 @@ struct SwitchNode : public MatcherNode { using ChildMapT = llvm::MapVector>; ChildMapT &getChildren() { return children; } + /// Returns the child at the given index. + std::pair> &getChild(unsigned i) { + assert(i < children.size() && "invalid child index"); + return *std::next(children.begin(), i); + } + private: /// Switch predicate "answers" select the child. Answers that are not found /// default to the failure node. diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp index 8b9c27c63e82..a93f3c48503c 100644 --- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp +++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp @@ -29,28 +29,12 @@ void PDLInterpDialect::initialize() { // pdl_interp::CreateOperationOp //===----------------------------------------------------------------------===// -static ParseResult parseCreateOperationOp(OpAsmParser &p, - OperationState &state) { - if (p.parseOptionalAttrDict(state.attributes)) - return failure(); +static ParseResult parseCreateOperationOpAttributes( + OpAsmParser &p, SmallVectorImpl &attrOperands, + ArrayAttr &attrNamesAttr) { Builder &builder = p.getBuilder(); - - // Parse the operation name. - StringAttr opName; - if (p.parseAttribute(opName, "name", state.attributes)) - return failure(); - - // Parse the operands. - SmallVector operands; - if (p.parseLParen() || p.parseOperandList(operands) || p.parseRParen() || - p.resolveOperands(operands, builder.getType(), - state.operands)) - return failure(); - - // Parse the attributes. SmallVector attrNames; if (succeeded(p.parseOptionalLBrace())) { - SmallVector attrOps; do { StringAttr nameAttr; OpAsmParser::OperandType operand; @@ -58,60 +42,35 @@ static ParseResult parseCreateOperationOp(OpAsmParser &p, p.parseOperand(operand)) return failure(); attrNames.push_back(nameAttr); - attrOps.push_back(operand); + attrOperands.push_back(operand); } while (succeeded(p.parseOptionalComma())); - - if (p.parseRBrace() || - p.resolveOperands(attrOps, builder.getType(), - state.operands)) - return failure(); - } - state.addAttribute("attributeNames", builder.getArrayAttr(attrNames)); - state.addTypes(builder.getType()); - - // Parse the result types. - SmallVector opResultTypes; - if (p.parseArrow()) - return failure(); - if (succeeded(p.parseOptionalLParen())) { - if (p.parseRParen()) + if (p.parseRBrace()) return failure(); - } else if (p.parseOperandList(opResultTypes) || - p.resolveOperands(opResultTypes, builder.getType(), - state.operands)) { - return failure(); } - - int32_t operandSegmentSizes[] = {static_cast(operands.size()), - static_cast(attrNames.size()), - static_cast(opResultTypes.size())}; - state.addAttribute("operand_segment_sizes", - builder.getI32VectorAttr(operandSegmentSizes)); + attrNamesAttr = builder.getArrayAttr(attrNames); return success(); } -static void print(OpAsmPrinter &p, CreateOperationOp op) { - p << "pdl_interp.create_operation "; - p.printOptionalAttrDict(op->getAttrs(), - {"attributeNames", "name", "operand_segment_sizes"}); - p << '"' << op.name() << "\"(" << op.operands() << ')'; +static void printCreateOperationOpAttributes(OpAsmPrinter &p, + CreateOperationOp op, + OperandRange attrArgs, + ArrayAttr attrNames) { + if (attrNames.empty()) + return; + p << " {"; + interleaveComma(llvm::seq(0, attrNames.size()), p, + [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; }); + p << '}'; +} - // Emit the optional attributes. - ArrayAttr attrNames = op.attributeNames(); - if (!attrNames.empty()) { - Operation::operand_range attrArgs = op.attributes(); - p << " {"; - interleaveComma(llvm::seq(0, attrNames.size()), p, - [&](int i) { p << attrNames[i] << " = " << attrArgs[i]; }); - p << '}'; - } +//===----------------------------------------------------------------------===// +// pdl_interp::GetValueTypeOp +//===----------------------------------------------------------------------===// - // Print the result type constraints of the operation. - auto types = op.types(); - if (types.empty()) - p << " -> ()"; - else - p << " -> " << op.types(); +/// Given the result type of a `GetValueTypeOp`, return the expected input type. +static Type getGetValueTypeOpValueType(Type type) { + Type valueTy = pdl::ValueType::get(type.getContext()); + return type.isa() ? pdl::RangeType::get(valueTy) : valueTy; } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp index c09892caec1b..ef96e25c7be3 100644 --- a/mlir/lib/Rewrite/ByteCode.cpp +++ b/mlir/lib/Rewrite/ByteCode.cpp @@ -208,7 +208,7 @@ private: void generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetValueTypeOp op, ByteCodeWriter &writer); - void generate(pdl_interp::InferredTypeOp op, ByteCodeWriter &writer); + void generate(pdl_interp::InferredTypesOp op, ByteCodeWriter &writer); void generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer); void generate(pdl_interp::RecordMatchOp op, ByteCodeWriter &writer); void generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer); @@ -487,7 +487,7 @@ void Generator::generate(Operation *op, ByteCodeWriter &writer) { pdl_interp::GetAttributeOp, pdl_interp::GetAttributeTypeOp, pdl_interp::GetDefiningOpOp, pdl_interp::GetOperandOp, pdl_interp::GetResultOp, pdl_interp::GetValueTypeOp, - pdl_interp::InferredTypeOp, pdl_interp::IsNotNullOp, + pdl_interp::InferredTypesOp, pdl_interp::IsNotNullOp, pdl_interp::RecordMatchOp, pdl_interp::ReplaceOp, pdl_interp::SwitchAttributeOp, pdl_interp::SwitchTypeOp, pdl_interp::SwitchOperandCountOp, pdl_interp::SwitchOperationNameOp, @@ -615,9 +615,9 @@ void Generator::generate(pdl_interp::GetValueTypeOp op, ByteCodeWriter &writer) { writer.append(OpCode::GetValueType, op.result(), op.value()); } -void Generator::generate(pdl_interp::InferredTypeOp op, +void Generator::generate(pdl_interp::InferredTypesOp op, ByteCodeWriter &writer) { - // InferType maps to a null type as a marker for inferring a result type. + // InferType maps to a null type as a marker for inferring result types. getMemIndex(op.type()) = getMemIndex(Type()); } void Generator::generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer) { @@ -980,16 +980,12 @@ void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter, state.name.getAbstractOperation()->getInterface(); // TODO: Handle failure. - SmallVector inferredTypes; + state.types.clear(); if (failed(concept->inferReturnTypes( state.getContext(), state.location, state.operands, state.attributes.getDictionary(state.getContext()), state.regions, - inferredTypes))) + state.types))) return; - - for (unsigned i = 0, e = state.types.size(); i != e; ++i) - if (!state.types[i]) - state.types[i] = inferredTypes[i]; } Operation *resultOp = rewriter.createOperation(state); memory[memIndex] = resultOp; diff --git a/mlir/lib/TableGen/Predicate.cpp b/mlir/lib/TableGen/Predicate.cpp index a37847f0d489..dd71540c15d4 100644 --- a/mlir/lib/TableGen/Predicate.cpp +++ b/mlir/lib/TableGen/Predicate.cpp @@ -133,6 +133,23 @@ namespace { using Subst = std::pair; } // end anonymous namespace +/// Perform the given substitutions on 'str' in-place. +static void performSubstitutions(std::string &str, + ArrayRef substitutions) { + // Apply all parent substitutions from innermost to outermost. + for (const auto &subst : llvm::reverse(substitutions)) { + auto pos = str.find(std::string(subst.first)); + while (pos != std::string::npos) { + str.replace(pos, subst.first.size(), std::string(subst.second)); + // Skip the newly inserted substring, which itself may consider the + // pattern to match. + pos += subst.second.size(); + // Find the next possible match position. + pos = str.find(std::string(subst.first), pos); + } + } +} + // Build the predicate tree starting from the top-level predicate, which may // have children, and perform leaf substitutions inplace. Note that after // substitution, nodes are still pointing to the original TableGen record. @@ -147,19 +164,7 @@ buildPredicateTree(const Pred &root, rootNode->predicate = &root; if (!root.isCombined()) { rootNode->expr = root.getCondition(); - // Apply all parent substitutions from innermost to outermost. - for (const auto &subst : llvm::reverse(substitutions)) { - auto pos = rootNode->expr.find(std::string(subst.first)); - while (pos != std::string::npos) { - rootNode->expr.replace(pos, subst.first.size(), - std::string(subst.second)); - // Skip the newly inserted substring, which itself may consider the - // pattern to match. - pos += subst.second.size(); - // Find the next possible match position. - pos = rootNode->expr.find(std::string(subst.first), pos); - } - } + performSubstitutions(rootNode->expr, substitutions); return rootNode; } @@ -170,12 +175,14 @@ buildPredicateTree(const Pred &root, const auto &substPred = static_cast(root); allSubstitutions.push_back( {substPred.getPattern(), substPred.getReplacement()}); - } - // If the current predicate is a ConcatPred, record the prefix and suffix. - else if (rootNode->kind == PredCombinerKind::Concat) { + + // If the current predicate is a ConcatPred, record the prefix and suffix. + } else if (rootNode->kind == PredCombinerKind::Concat) { const auto &concatPred = static_cast(root); rootNode->prefix = std::string(concatPred.getPrefix()); + performSubstitutions(rootNode->prefix, substitutions); rootNode->suffix = std::string(concatPred.getSuffix()); + performSubstitutions(rootNode->suffix, substitutions); } // Build child subtrees. diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir index 0792f76cba7a..0af77a24efb4 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-matcher.mlir @@ -103,6 +103,59 @@ module @inputs { // ----- +// CHECK-LABEL: module @variadic_inputs +module @variadic_inputs { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + // CHECK-DAG: pdl_interp.check_operand_count of %[[ROOT]] is at_least 2 + + // The first operand has a known index. + // CHECK-DAG: %[[INPUT:.*]] = pdl_interp.get_operand 0 of %[[ROOT]] + // CHECK-DAG: pdl_interp.is_not_null %[[INPUT]] : !pdl.value + + // The second operand is a group of unknown size, with a type constraint. + // CHECK-DAG: %[[VAR_INPUTS:.*]] = pdl_interp.get_operands 1 of %[[ROOT]] : !pdl.range + // CHECK-DAG: pdl_interp.is_not_null %[[VAR_INPUTS]] : !pdl.range + + // CHECK-DAG: %[[INPUT_TYPE:.*]] = pdl_interp.get_value_type of %[[VAR_INPUTS]] : !pdl.range + // CHECK-DAG: pdl_interp.check_types %[[INPUT_TYPE]] are [i64] + + // The third operand is at an unknown offset due to operand 2, but is expected + // to be of size 1. + // CHECK-DAG: %[[INPUT2:.*]] = pdl_interp.get_operands 2 of %[[ROOT]] : !pdl.value + // CHECK-DAG: pdl_interp.are_equal %[[INPUT]], %[[INPUT2]] : !pdl.value + pdl.pattern : benefit(1) { + %types = pdl.types : [i64] + %inputs = pdl.operands : %types + %input = pdl.operand + %root = pdl.operation(%input, %inputs, %input : !pdl.value, !pdl.range, !pdl.value) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + +// CHECK-LABEL: module @single_operand_range +module @single_operand_range { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + + // Check that the operand range is treated as all of the operands of the + // operation. + // CHECK-DAG: %[[RESULTS:.*]] = pdl_interp.get_operands of %[[ROOT]] + // CHECK-DAG: %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]] : !pdl.range + // CHECK-DAG: pdl_interp.check_types %[[RESULT_TYPES]] are [i64] + + // The operand count is unknown, so there is no need to check for it. + // CHECK-NOT: pdl_interp.check_operand_count + pdl.pattern : benefit(1) { + %types = pdl.types : [i64] + %operands = pdl.operands : %types + %root = pdl.operation(%operands : !pdl.range) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + // CHECK-LABEL: module @results module @results { // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) @@ -127,6 +180,57 @@ module @results { // ----- +// CHECK-LABEL: module @variadic_results +module @variadic_results { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + // CHECK-DAG: pdl_interp.check_result_count of %[[ROOT]] is at_least 2 + + // The first result has a known index. + // CHECK-DAG: %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]] + // CHECK-DAG: pdl_interp.is_not_null %[[RESULT]] : !pdl.value + + // The second result is a group of unknown size, with a type constraint. + // CHECK-DAG: %[[VAR_RESULTS:.*]] = pdl_interp.get_results 1 of %[[ROOT]] : !pdl.range + // CHECK-DAG: pdl_interp.is_not_null %[[VAR_RESULTS]] : !pdl.range + + // CHECK-DAG: %[[RESULT_TYPE:.*]] = pdl_interp.get_value_type of %[[VAR_RESULTS]] : !pdl.range + // CHECK-DAG: pdl_interp.check_types %[[RESULT_TYPE]] are [i64] + + // The third result is at an unknown offset due to result 1, but is expected + // to be of size 1. + // CHECK-DAG: %[[RESULT2:.*]] = pdl_interp.get_results 2 of %[[ROOT]] : !pdl.value + // CHECK-DAG: pdl_interp.is_not_null %[[RESULT2]] : !pdl.value + pdl.pattern : benefit(1) { + %types = pdl.types : [i64] + %type = pdl.type + %root = pdl.operation -> (%type, %types, %type : !pdl.type, !pdl.range, !pdl.type) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + +// CHECK-LABEL: module @single_result_range +module @single_result_range { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + + // Check that the result range is treated as all of the results of the + // operation. + // CHECK-DAG: %[[RESULTS:.*]] = pdl_interp.get_results of %[[ROOT]] + // CHECK-DAG: %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]] : !pdl.range + // CHECK-DAG: pdl_interp.check_types %[[RESULT_TYPES]] are [i64] + + // The result count is unknown, so there is no need to check for it. + // CHECK-NOT: pdl_interp.check_result_count + pdl.pattern : benefit(1) { + %types = pdl.types : [i64] + %root = pdl.operation -> (%types : !pdl.range) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + // CHECK-LABEL: module @results_as_operands module @results_as_operands { // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) @@ -160,8 +264,29 @@ module @results_as_operands { // ----- -// CHECK-LABEL: module @switch_result_types -module @switch_result_types { +// CHECK-LABEL: module @single_result_range_as_operands +module @single_result_range_as_operands { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + // CHECK-DAG: %[[OPERANDS:.*]] = pdl_interp.get_operands of %[[ROOT]] : !pdl.range + // CHECK-DAG: %[[OP:.*]] = pdl_interp.get_defining_op of %[[OPERANDS]] : !pdl.range + // CHECK-DAG: pdl_interp.is_not_null %[[OP]] + // CHECK-DAG: %[[RESULTS:.*]] = pdl_interp.get_results of %[[OP]] : !pdl.range + // CHECK-DAG: pdl_interp.are_equal %[[RESULTS]], %[[OPERANDS]] : !pdl.range + + pdl.pattern : benefit(1) { + %types = pdl.types + %inputOp = pdl.operation -> (%types : !pdl.range) + %results = pdl.results of %inputOp + + %root = pdl.operation(%results : !pdl.range) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + +// CHECK-LABEL: module @switch_single_result_type +module @switch_single_result_type { // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) // CHECK: %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[ROOT]] // CHECK: %[[RESULT_TYPE:.*]] = pdl_interp.get_value_type of %[[RESULT]] @@ -178,6 +303,84 @@ module @switch_result_types { } } +// ----- + +// CHECK-LABEL: module @switch_result_types +module @switch_result_types { + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + // CHECK: %[[RESULTS:.*]] = pdl_interp.get_results of %[[ROOT]] + // CHECK: %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]] + // CHECK: pdl_interp.switch_types %[[RESULT_TYPES]] to {{\[\[}}i32], [i64, i32]] + pdl.pattern : benefit(1) { + %types = pdl.types : [i32] + %root = pdl.operation -> (%types : !pdl.range) + pdl.rewrite %root with "rewriter" + } + pdl.pattern : benefit(1) { + %types = pdl.types : [i64, i32] + %root = pdl.operation -> (%types : !pdl.range) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + +// CHECK-LABEL: module @switch_operand_count_at_least +module @switch_operand_count_at_least { + // Check that when there are multiple "at_least" checks, the failure branch + // goes to the next one in increasing order. + + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + // CHECK: pdl_interp.check_operand_count of %[[ROOT]] is at_least 1 -> ^[[PATTERN_1_NEXT_BLOCK:.*]], + // CHECK: ^bb2: + // CHECK-NEXT: pdl_interp.check_operand_count of %[[ROOT]] is at_least 2 + // CHECK: ^[[PATTERN_1_NEXT_BLOCK]]: + // CHECK-NEXT: {{.*}} -> ^{{.*}}, ^bb2 + pdl.pattern : benefit(1) { + %operand = pdl.operand + %operands = pdl.operands + %root = pdl.operation(%operand, %operands : !pdl.value, !pdl.range) + pdl.rewrite %root with "rewriter" + } + pdl.pattern : benefit(1) { + %operand = pdl.operand + %operand2 = pdl.operand + %operands = pdl.operands + %root = pdl.operation(%operand, %operand2, %operands : !pdl.value, !pdl.value, !pdl.range) + pdl.rewrite %root with "rewriter" + } +} + +// ----- + +// CHECK-LABEL: module @switch_result_count_at_least +module @switch_result_count_at_least { + // Check that when there are multiple "at_least" checks, the failure branch + // goes to the next one in increasing order. + + // CHECK: func @matcher(%[[ROOT:.*]]: !pdl.operation) + // CHECK: pdl_interp.check_result_count of %[[ROOT]] is at_least 1 -> ^[[PATTERN_1_NEXT_BLOCK:.*]], + // CHECK: ^[[PATTERN_2_BLOCK:[a-zA-Z_0-9]*]]: + // CHECK: pdl_interp.check_result_count of %[[ROOT]] is at_least 2 + // CHECK: ^[[PATTERN_1_NEXT_BLOCK]]: + // CHECK-NEXT: pdl_interp.get_result + // CHECK-NEXT: pdl_interp.is_not_null {{.*}} -> ^{{.*}}, ^[[PATTERN_2_BLOCK]] + pdl.pattern : benefit(1) { + %type = pdl.type + %types = pdl.types + %root = pdl.operation -> (%type, %types : !pdl.type, !pdl.range) + pdl.rewrite %root with "rewriter" + } + pdl.pattern : benefit(1) { + %type = pdl.type + %type2 = pdl.type + %types = pdl.types + %root = pdl.operation -> (%type, %type2, %types : !pdl.type, !pdl.type, !pdl.range) + pdl.rewrite %root with "rewriter" + } +} + + // ----- // CHECK-LABEL: module @predicate_ordering diff --git a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir index 67ac7c811ab7..58d1c3177dad 100644 --- a/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir +++ b/mlir/test/Conversion/PDLToPDLInterp/pdl-to-pdl-interp-rewriter.mlir @@ -37,7 +37,7 @@ module @operation_attributes { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[ATTR:.*]]: !pdl.attribute, %[[ROOT:.*]]: !pdl.operation) // CHECK: %[[ATTR1:.*]] = pdl_interp.create_attribute true - // CHECK: pdl_interp.create_operation "foo.op"() {"attr" = %[[ATTR]], "attr1" = %[[ATTR1]]} + // CHECK: pdl_interp.create_operation "foo.op" {"attr" = %[[ATTR]], "attr1" = %[[ATTR1]]} pdl.pattern : benefit(1) { %attr = pdl.attribute %root = pdl.operation "foo.op" {"attr" = %attr} @@ -55,9 +55,9 @@ module @operation_attributes { module @operation_operands { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[OPERAND:.*]]: !pdl.value, %[[ROOT:.*]]: !pdl.operation) - // CHECK: %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]]) + // CHECK: %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]] : !pdl.value) // CHECK: %[[OPERAND1:.*]] = pdl_interp.get_result 0 of %[[NEWOP]] - // CHECK: pdl_interp.create_operation "foo.op2"(%[[OPERAND1]]) + // CHECK: pdl_interp.create_operation "foo.op2"(%[[OPERAND1]] : !pdl.value) pdl.pattern : benefit(1) { %operand = pdl.operand %root = pdl.operation "foo.op"(%operand : !pdl.value) @@ -77,9 +77,9 @@ module @operation_operands { module @operation_operands { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[OPERAND:.*]]: !pdl.value, %[[ROOT:.*]]: !pdl.operation) - // CHECK: %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]]) + // CHECK: %[[NEWOP:.*]] = pdl_interp.create_operation "foo.op"(%[[OPERAND]] : !pdl.value) // CHECK: %[[OPERAND1:.*]] = pdl_interp.get_result 0 of %[[NEWOP]] - // CHECK: pdl_interp.create_operation "foo.op2"(%[[OPERAND1]]) + // CHECK: pdl_interp.create_operation "foo.op2"(%[[OPERAND1]] : !pdl.value) pdl.pattern : benefit(1) { %operand = pdl.operand %root = pdl.operation "foo.op"(%operand : !pdl.value) @@ -95,11 +95,13 @@ module @operation_operands { // ----- -// CHECK-LABEL: module @operation_result_types -module @operation_result_types { +// CHECK-LABEL: module @operation_infer_types_from_replaceop +module @operation_infer_types_from_replaceop { // CHECK: module @rewriters - // CHECK: func @pdl_generated_rewriter(%[[TYPE:.*]]: !pdl.type, %[[TYPE1:.*]]: !pdl.type - // CHECK: pdl_interp.create_operation "foo.op"() -> %[[TYPE]], %[[TYPE1]] + // CHECK: func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation + // CHECK: %[[RESULTS:.*]] = pdl_interp.get_results of %[[ROOT]] + // CHECK: %[[RESULT_TYPES:.*]] = pdl_interp.get_value_type of %[[RESULTS]] + // CHECK: pdl_interp.create_operation "foo.op" -> (%[[RESULT_TYPES]] : !pdl.range) pdl.pattern : benefit(1) { %rootType = pdl.type %rootType1 = pdl.type @@ -114,13 +116,46 @@ module @operation_result_types { // ----- +// CHECK-LABEL: module @operation_infer_types_from_otherop_individual_results +module @operation_infer_types_from_otherop_individual_results { + // CHECK: module @rewriters + // CHECK: func @pdl_generated_rewriter(%[[TYPE:.*]]: !pdl.type, %[[TYPES:.*]]: !pdl.range + // CHECK: pdl_interp.create_operation "foo.op" -> (%[[TYPE]], %[[TYPES]] : !pdl.type, !pdl.range) + pdl.pattern : benefit(1) { + %rootType = pdl.type + %rootTypes = pdl.types + %root = pdl.operation "foo.op" -> (%rootType, %rootTypes : !pdl.type, !pdl.range) + pdl.rewrite %root { + %newOp = pdl.operation "foo.op" -> (%rootType, %rootTypes : !pdl.type, !pdl.range) + } + } +} + +// ----- + +// CHECK-LABEL: module @operation_infer_types_from_otherop_results +module @operation_infer_types_from_otherop_results { + // CHECK: module @rewriters + // CHECK: func @pdl_generated_rewriter(%[[TYPES:.*]]: !pdl.range + // CHECK: pdl_interp.create_operation "foo.op" -> (%[[TYPES]] : !pdl.range) + pdl.pattern : benefit(1) { + %rootTypes = pdl.types + %root = pdl.operation "foo.op" -> (%rootTypes : !pdl.range) + pdl.rewrite %root { + %newOp = pdl.operation "foo.op" -> (%rootTypes : !pdl.range) + } + } +} + +// ----- + // CHECK-LABEL: module @replace_with_op module @replace_with_op { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation) // CHECK: %[[NEWOP:.*]] = pdl_interp.create_operation - // CHECK: %[[OP_RESULT:.*]] = pdl_interp.get_result 0 of %[[NEWOP]] - // CHECK: pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]]) + // CHECK: %[[RESULTS:.*]] = pdl_interp.get_results of %[[NEWOP]] + // CHECK: pdl_interp.replace %[[ROOT]] with (%[[RESULTS]] : !pdl.range) pdl.pattern : benefit(1) { %type = pdl.type : i32 %root = pdl.operation "foo.op" -> (%type : !pdl.type) @@ -136,17 +171,21 @@ module @replace_with_op { // CHECK-LABEL: module @replace_with_values module @replace_with_values { // CHECK: module @rewriters - // CHECK: func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation) + // CHECK: func @pdl_generated_rewriter({{.*}}, %[[ROOT:.*]]: !pdl.operation) // CHECK: %[[NEWOP:.*]] = pdl_interp.create_operation - // CHECK: %[[OP_RESULT:.*]] = pdl_interp.get_result 0 of %[[NEWOP]] - // CHECK: pdl_interp.replace %[[ROOT]] with(%[[OP_RESULT]]) + // CHECK: %[[RESULT:.*]] = pdl_interp.get_result 0 of %[[NEWOP]] + // CHECK: %[[RESULTS:.*]] = pdl_interp.get_results 1 of %[[NEWOP]] : !pdl.range + // CHECK: %[[RESULTS_2:.*]] = pdl_interp.get_results 2 of %[[NEWOP]] : !pdl.value + // CHECK: pdl_interp.replace %[[ROOT]] with (%[[RESULT]], %[[RESULTS]], %[[RESULTS_2]] : !pdl.value, !pdl.range, !pdl.value) pdl.pattern : benefit(1) { - %type = pdl.type : i32 - %root = pdl.operation "foo.op" -> (%type : !pdl.type) + %types = pdl.types + %root = pdl.operation "foo.op" -> (%types : !pdl.range) pdl.rewrite %root { - %newOp = pdl.operation "foo.op" -> (%type : !pdl.type) + %newOp = pdl.operation "foo.op" -> (%types : !pdl.range) %newResult = pdl.result 0 of %newOp - pdl.replace %root with (%newResult : !pdl.value) + %newResults = pdl.results 1 of %newOp -> !pdl.range + %newResults2 = pdl.results 2 of %newOp -> !pdl.value + pdl.replace %root with (%newResult, %newResults, %newResults2 : !pdl.value, !pdl.range, !pdl.value) } } } @@ -175,14 +214,13 @@ module @apply_native_rewrite { // CHECK: module @rewriters // CHECK: func @pdl_generated_rewriter(%[[ROOT:.*]]: !pdl.operation) // CHECK: %[[TYPE:.*]] = pdl_interp.apply_rewrite "functor" [true](%[[ROOT]] : !pdl.operation) : !pdl.type - // CHECK: pdl_interp.create_operation "foo.op"() -> %[[TYPE]] + // CHECK: pdl_interp.create_operation "foo.op" -> (%[[TYPE]] : !pdl.type) pdl.pattern : benefit(1) { %type = pdl.type %root = pdl.operation "foo.op" -> (%type : !pdl.type) pdl.rewrite %root { %newType = pdl.apply_native_rewrite "functor"[true](%root : !pdl.operation) : !pdl.type %newOp = pdl.operation "foo.op" -> (%newType : !pdl.type) - pdl.replace %root with %newOp } } } diff --git a/mlir/test/Dialect/PDLInterp/ops.mlir b/mlir/test/Dialect/PDLInterp/ops.mlir index d76b17c394e8..072dfaddcda2 100644 --- a/mlir/test/Dialect/PDLInterp/ops.mlir +++ b/mlir/test/Dialect/PDLInterp/ops.mlir @@ -10,16 +10,16 @@ func @operations(%attribute: !pdl.attribute, %input: !pdl.value, %type: !pdl.type) { // attributes, operands, and results - %op0 = pdl_interp.create_operation "foo.op"(%input) {"attr" = %attribute} -> %type + %op0 = pdl_interp.create_operation "foo.op"(%input : !pdl.value) {"attr" = %attribute} -> (%type : !pdl.type) // attributes, and results - %op1 = pdl_interp.create_operation "foo.op"() {"attr" = %attribute} -> %type + %op1 = pdl_interp.create_operation "foo.op" {"attr" = %attribute} -> (%type : !pdl.type) // attributes - %op2 = pdl_interp.create_operation "foo.op"() {"attr" = %attribute, "attr1" = %attribute} -> () + %op2 = pdl_interp.create_operation "foo.op" {"attr" = %attribute, "attr1" = %attribute} // operands, and results - %op3 = pdl_interp.create_operation "foo.op"(%input) -> %type + %op3 = pdl_interp.create_operation "foo.op"(%input : !pdl.value) -> (%type : !pdl.type) pdl_interp.finalize } diff --git a/mlir/test/Rewrite/pdl-bytecode.mlir b/mlir/test/Rewrite/pdl-bytecode.mlir index 2093d03bbf25..b0acd328147a 100644 --- a/mlir/test/Rewrite/pdl-bytecode.mlir +++ b/mlir/test/Rewrite/pdl-bytecode.mlir @@ -25,7 +25,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.replaced_by_pattern"() -> () + %op = pdl_interp.create_operation "test.replaced_by_pattern" pdl_interp.erase %root pdl_interp.finalize } @@ -122,7 +122,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -157,7 +157,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -190,7 +190,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -222,7 +222,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -256,7 +256,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -288,7 +288,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -325,7 +325,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -375,7 +375,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -425,8 +425,8 @@ module @patterns { ^pat1: %operand0 = pdl_interp.get_operand 0 of %root %operand4 = pdl_interp.get_operand 4 of %root - %defOp0 = pdl_interp.get_defining_op of %operand0 - %defOp4 = pdl_interp.get_defining_op of %operand4 + %defOp0 = pdl_interp.get_defining_op of %operand0 : !pdl.value + %defOp4 = pdl_interp.get_defining_op of %operand4 : !pdl.value pdl_interp.are_equal %defOp0, %defOp4 : !pdl.operation -> ^pat2, ^end ^pat2: @@ -438,7 +438,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -476,8 +476,8 @@ module @patterns { ^pat1: %result0 = pdl_interp.get_result 0 of %root %result4 = pdl_interp.get_result 4 of %root - %result0_type = pdl_interp.get_value_type of %result0 - %result4_type = pdl_interp.get_value_type of %result4 + %result0_type = pdl_interp.get_value_type of %result0 : !pdl.type + %result4_type = pdl_interp.get_value_type of %result4 : !pdl.type pdl_interp.are_equal %result0_type, %result4_type : !pdl.type -> ^pat2, ^end ^pat2: @@ -489,7 +489,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -513,7 +513,7 @@ module @ir attributes { test.get_result_1 } { // Fully tested within the tests for other operations. //===----------------------------------------------------------------------===// -// pdl_interp::InferredTypeOp +// pdl_interp::InferredTypesOp //===----------------------------------------------------------------------===// // Fully tested within the tests for other operations. @@ -549,7 +549,7 @@ module @patterns { pdl_interp.finalize } func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -582,7 +582,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { %operand = pdl_interp.get_operand 0 of %root - pdl_interp.replace %root with (%operand) + pdl_interp.replace %root with (%operand : !pdl.value) pdl_interp.finalize } } @@ -622,7 +622,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -657,7 +657,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -693,7 +693,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -728,7 +728,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } @@ -768,7 +768,7 @@ module @patterns { module @rewriters { func @success(%root : !pdl.operation) { - %op = pdl_interp.create_operation "test.success"() -> () + %op = pdl_interp.create_operation "test.success" pdl_interp.erase %root pdl_interp.finalize } diff --git a/mlir/test/mlir-tblgen/op-attribute.td b/mlir/test/mlir-tblgen/op-attribute.td index 725afd9bc1aa..987f417d867c 100644 --- a/mlir/test/mlir-tblgen/op-attribute.td +++ b/mlir/test/mlir-tblgen/op-attribute.td @@ -136,7 +136,7 @@ def BOp : NS_Op<"b_op", []> { // DEF: if (!((tblgen_function_attr.isa<::mlir::FlatSymbolRefAttr>()))) // DEF: if (!(((tblgen_some_type_attr.isa<::mlir::TypeAttr>())) && ((tblgen_some_type_attr.cast<::mlir::TypeAttr>().getValue().isa())))) // DEF: if (!((tblgen_array_attr.isa<::mlir::ArrayAttr>()))) -// DEF: if (!(((tblgen_some_attr_array.isa<::mlir::ArrayAttr>())) && (::llvm::all_of(tblgen_some_attr_array.cast<::mlir::ArrayAttr>(), [](::mlir::Attribute attr) { return (some-condition); })))) +// DEF: if (!(((tblgen_some_attr_array.isa<::mlir::ArrayAttr>())) && (::llvm::all_of(tblgen_some_attr_array.cast<::mlir::ArrayAttr>(), [&](::mlir::Attribute attr) { return (some-condition); })))) // DEF: if (!(((tblgen_type_attr.isa<::mlir::TypeAttr>())) && ((tblgen_type_attr.cast<::mlir::TypeAttr>().getValue().isa<::mlir::Type>())))) // Test common attribute kind getters' return types -- GitLab From 85ab413b53aeb135eb58dab066afcbf20bef0cf8 Mon Sep 17 00:00:00 2001 From: River Riddle Date: Tue, 16 Mar 2021 13:12:01 -0700 Subject: [PATCH 0075/1206] [mlir][PDL] Add support for variadic operands and results in the PDL byte code Supporting ranges in the byte code requires additional complexity, given that a range can't be easily representable as an opaque void *, as is possible with the existing bytecode value types (Attribute, Type, Value, etc.). To enable representing a range with void *, an auxillary storage is used for the actual range itself, with the pointer being passed around in the normal byte code memory. For type ranges, a TypeRange is stored. For value ranges, a ValueRange is stored. The above problem represents a majority of the complexity involved in this revision, the rest is adapting/adding byte code operations to support the changes made to the PDL interpreter in the parent revision. After this revision, PDL will have initial end-to-end support for variadic operands/results. Differential Revision: https://reviews.llvm.org/D95723 --- .../mlir/Dialect/PDLInterp/IR/PDLInterpOps.td | 4 +- mlir/include/mlir/IR/PatternMatch.h | 161 +++- mlir/include/mlir/IR/TypeRange.h | 6 + mlir/lib/IR/PatternMatch.cpp | 35 +- mlir/lib/Rewrite/ByteCode.cpp | 748 +++++++++++++++--- mlir/lib/Rewrite/ByteCode.h | 39 +- mlir/lib/Rewrite/PatternApplicator.cpp | 57 +- mlir/test/Rewrite/pdl-bytecode.mlir | 477 ++++++++++- mlir/test/lib/Rewrite/TestPDLByteCode.cpp | 30 + 9 files changed, 1363 insertions(+), 194 deletions(-) diff --git a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td index e35208747ade..ff9b3dda0f48 100644 --- a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td +++ b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td @@ -963,7 +963,7 @@ def PDLInterp_SwitchOperandCountOp let builders = [ OpBuilder<(ins "Value":$operation, "ArrayRef":$counts, - "Block *":$defaultDest, "BlockRange":$dests), [{ + "Block *":$defaultDest, "BlockRange":$dests), [{ build($_builder, $_state, operation, $_builder.getI32VectorAttr(counts), defaultDest, dests); }]>]; @@ -1033,7 +1033,7 @@ def PDLInterp_SwitchResultCountOp let builders = [ OpBuilder<(ins "Value":$operation, "ArrayRef":$counts, - "Block *":$defaultDest, "BlockRange":$dests), [{ + "Block *":$defaultDest, "BlockRange":$dests), [{ build($_builder, $_state, operation, $_builder.getI32VectorAttr(counts), defaultDest, dests); }]>]; diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h index 56da9b870948..c797f5329bd5 100644 --- a/mlir/include/mlir/IR/PatternMatch.h +++ b/mlir/include/mlir/IR/PatternMatch.h @@ -238,63 +238,92 @@ struct OpRewritePattern : public RewritePattern { /// Storage type of byte-code interpreter values. These are passed to constraint /// functions as arguments. class PDLValue { - /// The internal implementation type when the value is an Attribute, - /// Operation*, or Type. See `impl` below for more details. - using AttrOpTypeImplT = llvm::PointerUnion; - public: - PDLValue(const PDLValue &other) : impl(other.impl) {} - PDLValue(std::nullptr_t = nullptr) : impl() {} - PDLValue(Attribute value) : impl(value) {} - PDLValue(Operation *value) : impl(value) {} - PDLValue(Type value) : impl(value) {} - PDLValue(Value value) : impl(value) {} + /// The underlying kind of a PDL value. + enum class Kind { Attribute, Operation, Type, TypeRange, Value, ValueRange }; + + /// Construct a new PDL value. + PDLValue(const PDLValue &other) = default; + PDLValue(std::nullptr_t = nullptr) : value(nullptr), kind(Kind::Attribute) {} + PDLValue(Attribute value) + : value(value.getAsOpaquePointer()), kind(Kind::Attribute) {} + PDLValue(Operation *value) : value(value), kind(Kind::Operation) {} + PDLValue(Type value) : value(value.getAsOpaquePointer()), kind(Kind::Type) {} + PDLValue(TypeRange *value) : value(value), kind(Kind::TypeRange) {} + PDLValue(Value value) + : value(value.getAsOpaquePointer()), kind(Kind::Value) {} + PDLValue(ValueRange *value) : value(value), kind(Kind::ValueRange) {} /// Returns true if the type of the held value is `T`. - template - std::enable_if_t::value, bool> isa() const { - return impl.is(); - } - template - std::enable_if_t::value, bool> isa() const { - auto attrOpTypeImpl = impl.dyn_cast(); - return attrOpTypeImpl && attrOpTypeImpl.is(); + template bool isa() const { + assert(value && "isa<> used on a null value"); + return kind == getKindOf(); } /// Attempt to dynamically cast this value to type `T`, returns null if this /// value is not an instance of `T`. - template - std::enable_if_t::value, T> dyn_cast() const { - return impl.dyn_cast(); - } - template - std::enable_if_t::value, T> dyn_cast() const { - auto attrOpTypeImpl = impl.dyn_cast(); - return attrOpTypeImpl && attrOpTypeImpl.dyn_cast(); + template ::value, T, Optional>> + ResultT dyn_cast() const { + return isa() ? castImpl() : ResultT(); } /// Cast this value to type `T`, asserts if this value is not an instance of /// `T`. - template - std::enable_if_t::value, T> cast() const { - return impl.get(); - } - template - std::enable_if_t::value, T> cast() const { - return impl.get().get(); + template T cast() const { + assert(isa() && "expected value to be of type `T`"); + return castImpl(); } /// Get an opaque pointer to the value. - void *getAsOpaquePointer() { return impl.getOpaqueValue(); } + const void *getAsOpaquePointer() const { return value; } + + /// Return if this value is null or not. + explicit operator bool() const { return value; } + + /// Return the kind of this value. + Kind getKind() const { return kind; } /// Print this value to the provided output stream. - void print(raw_ostream &os); + void print(raw_ostream &os) const; private: - /// The internal opaque representation of a PDLValue. We use a nested - /// PointerUnion structure here because `Value` only has 1 low bit - /// available, where as the remaining types all have 3. - llvm::PointerUnion impl; + /// Find the index of a given type in a range of other types. + template struct index_of_t; + template + struct index_of_t : std::integral_constant {}; + template + struct index_of_t + : std::integral_constant::value> {}; + + /// Return the kind used for the given T. + template static Kind getKindOf() { + return static_cast(index_of_t::value); + } + + /// The internal implementation of `cast`, that returns the underlying value + /// as the given type `T`. + template + std::enable_if_t::value, T> + castImpl() const { + return T::getFromOpaquePointer(value); + } + template + std::enable_if_t::value, T> + castImpl() const { + return *reinterpret_cast(const_cast(value)); + } + template + std::enable_if_t::value, T> castImpl() const { + return reinterpret_cast(const_cast(value)); + } + + /// The internal opaque representation of a PDLValue. + const void *value; + /// The kind of the opaque value. + Kind kind; }; inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) { @@ -319,14 +348,66 @@ public: /// Push a new Type onto the result list. void push_back(Type value) { results.push_back(value); } + /// Push a new TypeRange onto the result list. + void push_back(TypeRange value) { + // The lifetime of a TypeRange can't be guaranteed, so we'll need to + // allocate a storage for it. + llvm::OwningArrayRef storage(value.size()); + llvm::copy(value, storage.begin()); + allocatedTypeRanges.emplace_back(std::move(storage)); + typeRanges.push_back(allocatedTypeRanges.back()); + results.push_back(&typeRanges.back()); + } + void push_back(ValueTypeRange value) { + typeRanges.push_back(value); + results.push_back(&typeRanges.back()); + } + void push_back(ValueTypeRange value) { + typeRanges.push_back(value); + results.push_back(&typeRanges.back()); + } + /// Push a new Value onto the result list. void push_back(Value value) { results.push_back(value); } + /// Push a new ValueRange onto the result list. + void push_back(ValueRange value) { + // The lifetime of a ValueRange can't be guaranteed, so we'll need to + // allocate a storage for it. + llvm::OwningArrayRef storage(value.size()); + llvm::copy(value, storage.begin()); + allocatedValueRanges.emplace_back(std::move(storage)); + valueRanges.push_back(allocatedValueRanges.back()); + results.push_back(&valueRanges.back()); + } + void push_back(OperandRange value) { + valueRanges.push_back(value); + results.push_back(&valueRanges.back()); + } + void push_back(ResultRange value) { + valueRanges.push_back(value); + results.push_back(&valueRanges.back()); + } + protected: - PDLResultList() = default; + /// Create a new result list with the expected number of results. + PDLResultList(unsigned maxNumResults) { + // For now just reserve enough space for all of the results. We could do + // separate counts per range type, but it isn't really worth it unless there + // are a "large" number of results. + typeRanges.reserve(maxNumResults); + valueRanges.reserve(maxNumResults); + } /// The PDL results held by this list. SmallVector results; + /// Memory used to store ranges held by the list. + SmallVector typeRanges; + SmallVector valueRanges; + /// Memory allocated to store ranges in the result list whose lifetime was + /// generated in the native function. + SmallVector> allocatedTypeRanges; + SmallVector> allocatedValueRanges; }; //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/IR/TypeRange.h b/mlir/include/mlir/IR/TypeRange.h index fe11fde58793..4fb40e127f9f 100644 --- a/mlir/include/mlir/IR/TypeRange.h +++ b/mlir/include/mlir/IR/TypeRange.h @@ -82,6 +82,12 @@ inline ::llvm::hash_code hash_value(TypeRange arg) { return ::llvm::hash_combine_range(arg.begin(), arg.end()); } +/// Emit a type range to the given output stream. +inline raw_ostream &operator<<(raw_ostream &os, const TypeRange &types) { + llvm::interleaveComma(types, os); + return os; +} + //===----------------------------------------------------------------------===// // ValueTypeRange diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp index 034698d85cb1..354d5f31bf74 100644 --- a/mlir/lib/IR/PatternMatch.cpp +++ b/mlir/lib/IR/PatternMatch.cpp @@ -73,22 +73,31 @@ void RewritePattern::anchor() {} // PDLValue //===----------------------------------------------------------------------===// -void PDLValue::print(raw_ostream &os) { - if (!impl) { - os << ""; +void PDLValue::print(raw_ostream &os) const { + if (!value) { + os << ""; return; } - if (Value val = impl.dyn_cast()) { - os << val; - return; + switch (kind) { + case Kind::Attribute: + os << cast(); + break; + case Kind::Operation: + os << *cast(); + break; + case Kind::Type: + os << cast(); + break; + case Kind::TypeRange: + llvm::interleaveComma(cast(), os); + break; + case Kind::Value: + os << cast(); + break; + case Kind::ValueRange: + llvm::interleaveComma(cast(), os); + break; } - AttrOpTypeImplT aotImpl = impl.get(); - if (Attribute attr = aotImpl.dyn_cast()) - os << attr; - else if (Operation *op = aotImpl.dyn_cast()) - os << *op; - else - os << aotImpl.get(); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp index ef96e25c7be3..ea17f99deb9c 100644 --- a/mlir/lib/Rewrite/ByteCode.cpp +++ b/mlir/lib/Rewrite/ByteCode.cpp @@ -20,6 +20,9 @@ #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/FormatVariadic.h" +#include #define DEBUG_TYPE "pdl-bytecode" @@ -60,6 +63,14 @@ void PDLByteCodeMutableState::updatePatternBenefit(unsigned patternIndex, currentPatternBenefits[patternIndex] = benefit; } +/// Cleanup any allocated state after a full match/rewrite has been completed. +/// This method should be called irregardless of whether the match+rewrite was a +/// success or not. +void PDLByteCodeMutableState::cleanupAfterMatchAndRewrite() { + allocatedTypeRangeMemory.clear(); + allocatedValueRangeMemory.clear(); +} + //===----------------------------------------------------------------------===// // Bytecode OpCodes //===----------------------------------------------------------------------===// @@ -72,6 +83,8 @@ enum OpCode : ByteCodeField { ApplyRewrite, /// Check if two generic values are equal. AreEqual, + /// Check if two ranges are equal. + AreRangesEqual, /// Unconditional branch. Branch, /// Compare the operand count of an operation with a constant. @@ -80,8 +93,12 @@ enum OpCode : ByteCodeField { CheckOperationName, /// Compare the result count of an operation with a constant. CheckResultCount, + /// Compare a range of types to a constant range of types. + CheckTypes, /// Create an operation. CreateOperation, + /// Create a range of types. + CreateTypes, /// Erase an operation. EraseOp, /// Terminate a matcher or rewrite sequence. @@ -98,14 +115,20 @@ enum OpCode : ByteCodeField { GetOperand2, GetOperand3, GetOperandN, + /// Get a specific operand group of an operation. + GetOperands, /// Get a specific result of an operation. GetResult0, GetResult1, GetResult2, GetResult3, GetResultN, + /// Get a specific result group of an operation. + GetResults, /// Get the type of a value. GetValueType, + /// Get the types of a value range. + GetValueRangeTypes, /// Check if a generic value is not null. IsNotNull, /// Record a successful pattern match. @@ -122,9 +145,9 @@ enum OpCode : ByteCodeField { SwitchResultCount, /// Compare a type with a set of constants. SwitchType, + /// Compare a range of types with a set of constants. + SwitchTypes, }; - -enum class PDLValueKind { Attribute, Operation, Type, Value }; } // end anonymous namespace //===----------------------------------------------------------------------===// @@ -145,11 +168,15 @@ public: SmallVectorImpl &rewriterByteCode, SmallVectorImpl &patterns, ByteCodeField &maxValueMemoryIndex, + ByteCodeField &maxTypeRangeMemoryIndex, + ByteCodeField &maxValueRangeMemoryIndex, llvm::StringMap &constraintFns, llvm::StringMap &rewriteFns) : ctx(ctx), uniquedData(uniquedData), matcherByteCode(matcherByteCode), rewriterByteCode(rewriterByteCode), patterns(patterns), - maxValueMemoryIndex(maxValueMemoryIndex) { + maxValueMemoryIndex(maxValueMemoryIndex), + maxTypeRangeMemoryIndex(maxTypeRangeMemoryIndex), + maxValueRangeMemoryIndex(maxValueRangeMemoryIndex) { for (auto it : llvm::enumerate(constraintFns)) constraintToMemIndex.try_emplace(it.value().first(), it.index()); for (auto it : llvm::enumerate(rewriteFns)) @@ -166,6 +193,13 @@ public: return valueToMemIndex[value]; } + /// Return the range memory index used to store the given range value. + ByteCodeField &getRangeStorageIndex(Value value) { + assert(valueToRangeIndex.count(value) && + "expected range index to be assigned"); + return valueToRangeIndex[value]; + } + /// Return an index to use when referring to the given data that is uniqued in /// the MLIR context. template @@ -197,16 +231,20 @@ private: void generate(pdl_interp::CheckOperationNameOp op, ByteCodeWriter &writer); void generate(pdl_interp::CheckResultCountOp op, ByteCodeWriter &writer); void generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer); + void generate(pdl_interp::CheckTypesOp op, ByteCodeWriter &writer); void generate(pdl_interp::CreateAttributeOp op, ByteCodeWriter &writer); void generate(pdl_interp::CreateOperationOp op, ByteCodeWriter &writer); void generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer); + void generate(pdl_interp::CreateTypesOp op, ByteCodeWriter &writer); void generate(pdl_interp::EraseOp op, ByteCodeWriter &writer); void generate(pdl_interp::FinalizeOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetAttributeOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetAttributeTypeOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetDefiningOpOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer); + void generate(pdl_interp::GetOperandsOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer); + void generate(pdl_interp::GetResultsOp op, ByteCodeWriter &writer); void generate(pdl_interp::GetValueTypeOp op, ByteCodeWriter &writer); void generate(pdl_interp::InferredTypesOp op, ByteCodeWriter &writer); void generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer); @@ -214,6 +252,7 @@ private: void generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer); void generate(pdl_interp::SwitchAttributeOp op, ByteCodeWriter &writer); void generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer); + void generate(pdl_interp::SwitchTypesOp op, ByteCodeWriter &writer); void generate(pdl_interp::SwitchOperandCountOp op, ByteCodeWriter &writer); void generate(pdl_interp::SwitchOperationNameOp op, ByteCodeWriter &writer); void generate(pdl_interp::SwitchResultCountOp op, ByteCodeWriter &writer); @@ -221,6 +260,9 @@ private: /// Mapping from value to its corresponding memory index. DenseMap valueToMemIndex; + /// Mapping from a range value to its corresponding range storage index. + DenseMap valueToRangeIndex; + /// Mapping from the name of an externally registered rewrite to its index in /// the bytecode registry. llvm::StringMap externalRewriterToMemIndex; @@ -246,6 +288,8 @@ private: SmallVectorImpl &rewriterByteCode; SmallVectorImpl &patterns; ByteCodeField &maxValueMemoryIndex; + ByteCodeField &maxTypeRangeMemoryIndex; + ByteCodeField &maxValueRangeMemoryIndex; }; /// This class provides utilities for writing a bytecode stream. @@ -281,19 +325,33 @@ struct ByteCodeWriter { /// Append a range of values that will be read as generic PDLValues. void appendPDLValueList(OperandRange values) { bytecode.push_back(values.size()); - for (Value value : values) { - // Append the type of the value in addition to the value itself. - PDLValueKind kind = - TypeSwitch(value.getType()) - .Case( - [](Type) { return PDLValueKind::Attribute; }) - .Case( - [](Type) { return PDLValueKind::Operation; }) - .Case([](Type) { return PDLValueKind::Type; }) - .Case([](Type) { return PDLValueKind::Value; }); - bytecode.push_back(static_cast(kind)); - append(value); - } + for (Value value : values) + appendPDLValue(value); + } + + /// Append a value as a PDLValue. + void appendPDLValue(Value value) { + appendPDLValueKind(value); + append(value); + } + + /// Append the PDLValue::Kind of the given value. + void appendPDLValueKind(Value value) { + // Append the type of the value in addition to the value itself. + PDLValue::Kind kind = + TypeSwitch(value.getType()) + .Case( + [](Type) { return PDLValue::Kind::Attribute; }) + .Case( + [](Type) { return PDLValue::Kind::Operation; }) + .Case([](pdl::RangeType rangeTy) { + if (rangeTy.getElementType().isa()) + return PDLValue::Kind::TypeRange; + return PDLValue::Kind::ValueRange; + }) + .Case([](Type) { return PDLValue::Kind::Type; }) + .Case([](Type) { return PDLValue::Kind::Value; }); + bytecode.push_back(static_cast(kind)); } /// Check if the given class `T` has an iterator type. @@ -334,6 +392,36 @@ struct ByteCodeWriter { /// The main generator producing PDL. Generator &generator; }; + +/// This class represents a live range of PDL Interpreter values, containing +/// information about when values are live within a match/rewrite. +struct ByteCodeLiveRange { + using Set = llvm::IntervalMap; + using Allocator = Set::Allocator; + + ByteCodeLiveRange(Allocator &alloc) : liveness(alloc) {} + + /// Union this live range with the one provided. + void unionWith(const ByteCodeLiveRange &rhs) { + for (auto it = rhs.liveness.begin(), e = rhs.liveness.end(); it != e; ++it) + liveness.insert(it.start(), it.stop(), /*dummyValue*/ 0); + } + + /// Returns true if this range overlaps with the one provided. + bool overlaps(const ByteCodeLiveRange &rhs) const { + return llvm::IntervalMapOverlaps(liveness, rhs.liveness).valid(); + } + + /// A map representing the ranges of the match/rewrite that a value is live in + /// the interpreter. + llvm::IntervalMap liveness; + + /// The type range storage index for this range. + Optional typeRangeIndex; + + /// The value range storage index for this range. + Optional valueRangeIndex; +}; } // end anonymous namespace void Generator::generate(ModuleOp module) { @@ -381,15 +469,30 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc, // Rewriters use simplistic allocation scheme that simply assigns an index to // each result. for (FuncOp rewriterFunc : rewriterModule.getOps()) { - ByteCodeField index = 0; + ByteCodeField index = 0, typeRangeIndex = 0, valueRangeIndex = 0; + auto processRewriterValue = [&](Value val) { + valueToMemIndex.try_emplace(val, index++); + if (pdl::RangeType rangeType = val.getType().dyn_cast()) { + Type elementTy = rangeType.getElementType(); + if (elementTy.isa()) + valueToRangeIndex.try_emplace(val, typeRangeIndex++); + else if (elementTy.isa()) + valueToRangeIndex.try_emplace(val, valueRangeIndex++); + } + }; + for (BlockArgument arg : rewriterFunc.getArguments()) - valueToMemIndex.try_emplace(arg, index++); + processRewriterValue(arg); rewriterFunc.getBody().walk([&](Operation *op) { for (Value result : op->getResults()) - valueToMemIndex.try_emplace(result, index++); + processRewriterValue(result); }); if (index > maxValueMemoryIndex) maxValueMemoryIndex = index; + if (typeRangeIndex > maxTypeRangeMemoryIndex) + maxTypeRangeMemoryIndex = typeRangeIndex; + if (valueRangeIndex > maxValueRangeMemoryIndex) + maxValueRangeMemoryIndex = valueRangeIndex; } // The matcher function uses a more sophisticated numbering that tries to @@ -404,9 +507,8 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc, }); // Liveness info for each of the defs within the matcher. - using LivenessSet = llvm::IntervalMap; - LivenessSet::Allocator allocator; - DenseMap valueDefRanges; + ByteCodeLiveRange::Allocator allocator; + DenseMap valueDefRanges; // Assign the root operation being matched to slot 0. BlockArgument rootOpArg = matcherFunc.getArgument(0); @@ -425,10 +527,19 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc, // Set indices for the range of this block that the value is used. auto defRangeIt = valueDefRanges.try_emplace(value, allocator).first; - defRangeIt->second.insert( + defRangeIt->second.liveness.insert( opToIndex[firstUseOrDef], opToIndex[info->getEndOperation(value, firstUseOrDef)], /*dummyValue*/ 0); + + // Check to see if this value is a range type. + if (auto rangeTy = value.getType().dyn_cast()) { + Type eleType = rangeTy.getElementType(); + if (eleType.isa()) + defRangeIt->second.typeRangeIndex = 0; + else if (eleType.isa()) + defRangeIt->second.valueRangeIndex = 0; + } }; // Process the live-ins of this block. @@ -442,37 +553,59 @@ void Generator::allocateMemoryIndices(FuncOp matcherFunc, } // Greedily allocate memory slots using the computed def live ranges. - std::vector allocatedIndices; + std::vector allocatedIndices; + ByteCodeField numIndices = 1, numTypeRanges = 0, numValueRanges = 0; for (auto &defIt : valueDefRanges) { ByteCodeField &memIndex = valueToMemIndex[defIt.first]; - LivenessSet &defSet = defIt.second; + ByteCodeLiveRange &defRange = defIt.second; // Try to allocate to an existing index. for (auto existingIndexIt : llvm::enumerate(allocatedIndices)) { - LivenessSet &existingIndex = existingIndexIt.value(); - llvm::IntervalMapOverlaps overlaps( - defIt.second, existingIndex); - if (overlaps.valid()) - continue; - // Union the range of the def within the existing index. - for (auto it = defSet.begin(), e = defSet.end(); it != e; ++it) - existingIndex.insert(it.start(), it.stop(), /*dummyValue*/ 0); - memIndex = existingIndexIt.index() + 1; + ByteCodeLiveRange &existingRange = existingIndexIt.value(); + if (!defRange.overlaps(existingRange)) { + existingRange.unionWith(defRange); + memIndex = existingIndexIt.index() + 1; + + if (defRange.typeRangeIndex) { + if (!existingRange.typeRangeIndex) + existingRange.typeRangeIndex = numTypeRanges++; + valueToRangeIndex[defIt.first] = *existingRange.typeRangeIndex; + } else if (defRange.valueRangeIndex) { + if (!existingRange.valueRangeIndex) + existingRange.valueRangeIndex = numValueRanges++; + valueToRangeIndex[defIt.first] = *existingRange.valueRangeIndex; + } + break; + } } // If no existing index could be used, add a new one. if (memIndex == 0) { allocatedIndices.emplace_back(allocator); - for (auto it = defSet.begin(), e = defSet.end(); it != e; ++it) - allocatedIndices.back().insert(it.start(), it.stop(), /*dummyValue*/ 0); + ByteCodeLiveRange &newRange = allocatedIndices.back(); + newRange.unionWith(defRange); + + // Allocate an index for type/value ranges. + if (defRange.typeRangeIndex) { + newRange.typeRangeIndex = numTypeRanges; + valueToRangeIndex[defIt.first] = numTypeRanges++; + } else if (defRange.valueRangeIndex) { + newRange.valueRangeIndex = numValueRanges; + valueToRangeIndex[defIt.first] = numValueRanges++; + } + memIndex = allocatedIndices.size(); + ++numIndices; } } // Update the max number of indices. - ByteCodeField numMatcherIndices = allocatedIndices.size() + 1; - if (numMatcherIndices > maxValueMemoryIndex) - maxValueMemoryIndex = numMatcherIndices; + if (numIndices > maxValueMemoryIndex) + maxValueMemoryIndex = numIndices; + if (numTypeRanges > maxTypeRangeMemoryIndex) + maxTypeRangeMemoryIndex = numTypeRanges; + if (numValueRanges > maxValueRangeMemoryIndex) + maxValueRangeMemoryIndex = numValueRanges; } void Generator::generate(Operation *op, ByteCodeWriter &writer) { @@ -481,17 +614,19 @@ void Generator::generate(Operation *op, ByteCodeWriter &writer) { pdl_interp::AreEqualOp, pdl_interp::BranchOp, pdl_interp::CheckAttributeOp, pdl_interp::CheckOperandCountOp, pdl_interp::CheckOperationNameOp, pdl_interp::CheckResultCountOp, - pdl_interp::CheckTypeOp, pdl_interp::CreateAttributeOp, - pdl_interp::CreateOperationOp, pdl_interp::CreateTypeOp, + pdl_interp::CheckTypeOp, pdl_interp::CheckTypesOp, + pdl_interp::CreateAttributeOp, pdl_interp::CreateOperationOp, + pdl_interp::CreateTypeOp, pdl_interp::CreateTypesOp, pdl_interp::EraseOp, pdl_interp::FinalizeOp, pdl_interp::GetAttributeOp, pdl_interp::GetAttributeTypeOp, pdl_interp::GetDefiningOpOp, pdl_interp::GetOperandOp, - pdl_interp::GetResultOp, pdl_interp::GetValueTypeOp, + pdl_interp::GetOperandsOp, pdl_interp::GetResultOp, + pdl_interp::GetResultsOp, pdl_interp::GetValueTypeOp, pdl_interp::InferredTypesOp, pdl_interp::IsNotNullOp, pdl_interp::RecordMatchOp, pdl_interp::ReplaceOp, pdl_interp::SwitchAttributeOp, pdl_interp::SwitchTypeOp, - pdl_interp::SwitchOperandCountOp, pdl_interp::SwitchOperationNameOp, - pdl_interp::SwitchResultCountOp>( + pdl_interp::SwitchTypesOp, pdl_interp::SwitchOperandCountOp, + pdl_interp::SwitchOperationNameOp, pdl_interp::SwitchResultCountOp>( [&](auto interpOp) { this->generate(interpOp, writer); }) .Default([](Operation *) { llvm_unreachable("unknown `pdl_interp` operation"); @@ -515,16 +650,31 @@ void Generator::generate(pdl_interp::ApplyRewriteOp op, op.constParamsAttr()); writer.appendPDLValueList(op.args()); + ResultRange results = op.results(); + writer.append(ByteCodeField(results.size())); + for (Value result : results) { + // In debug mode we also record the expected kind of the result, so that we + // can provide extra verification of the native rewrite function. #ifndef NDEBUG - // In debug mode we also append the number of results so that we can assert - // that the native creation function gave us the correct number of results. - writer.append(ByteCodeField(op.results().size())); + writer.appendPDLValueKind(result); #endif - for (Value result : op.results()) + + // Range results also need to append the range storage index. + if (result.getType().isa()) + writer.append(getRangeStorageIndex(result)); writer.append(result); + } } void Generator::generate(pdl_interp::AreEqualOp op, ByteCodeWriter &writer) { - writer.append(OpCode::AreEqual, op.lhs(), op.rhs(), op.getSuccessors()); + Value lhs = op.lhs(); + if (lhs.getType().isa()) { + writer.append(OpCode::AreRangesEqual); + writer.appendPDLValueKind(lhs); + writer.append(op.lhs(), op.rhs(), op.getSuccessors()); + return; + } + + writer.append(OpCode::AreEqual, lhs, op.rhs(), op.getSuccessors()); } void Generator::generate(pdl_interp::BranchOp op, ByteCodeWriter &writer) { writer.append(OpCode::Branch, SuccessorRange(op.getOperation())); @@ -537,6 +687,7 @@ void Generator::generate(pdl_interp::CheckAttributeOp op, void Generator::generate(pdl_interp::CheckOperandCountOp op, ByteCodeWriter &writer) { writer.append(OpCode::CheckOperandCount, op.operation(), op.count(), + static_cast(op.compareAtLeast()), op.getSuccessors()); } void Generator::generate(pdl_interp::CheckOperationNameOp op, @@ -547,11 +698,15 @@ void Generator::generate(pdl_interp::CheckOperationNameOp op, void Generator::generate(pdl_interp::CheckResultCountOp op, ByteCodeWriter &writer) { writer.append(OpCode::CheckResultCount, op.operation(), op.count(), + static_cast(op.compareAtLeast()), op.getSuccessors()); } void Generator::generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer) { writer.append(OpCode::AreEqual, op.value(), op.type(), op.getSuccessors()); } +void Generator::generate(pdl_interp::CheckTypesOp op, ByteCodeWriter &writer) { + writer.append(OpCode::CheckTypes, op.value(), op.types(), op.getSuccessors()); +} void Generator::generate(pdl_interp::CreateAttributeOp op, ByteCodeWriter &writer) { // Simply repoint the memory index of the result to the constant. @@ -560,7 +715,8 @@ void Generator::generate(pdl_interp::CreateAttributeOp op, void Generator::generate(pdl_interp::CreateOperationOp op, ByteCodeWriter &writer) { writer.append(OpCode::CreateOperation, op.operation(), - OperationName(op.name(), ctx), op.operands()); + OperationName(op.name(), ctx)); + writer.appendPDLValueList(op.operands()); // Add the attributes. OperandRange attributes = op.attributes(); @@ -570,12 +726,16 @@ void Generator::generate(pdl_interp::CreateOperationOp op, Identifier::get(std::get<0>(it).cast().getValue(), ctx), std::get<1>(it)); } - writer.append(op.types()); + writer.appendPDLValueList(op.types()); } void Generator::generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer) { // Simply repoint the memory index of the result to the constant. getMemIndex(op.result()) = getMemIndex(op.value()); } +void Generator::generate(pdl_interp::CreateTypesOp op, ByteCodeWriter &writer) { + writer.append(OpCode::CreateTypes, op.result(), + getRangeStorageIndex(op.result()), op.value()); +} void Generator::generate(pdl_interp::EraseOp op, ByteCodeWriter &writer) { writer.append(OpCode::EraseOp, op.operation()); } @@ -593,7 +753,8 @@ void Generator::generate(pdl_interp::GetAttributeTypeOp op, } void Generator::generate(pdl_interp::GetDefiningOpOp op, ByteCodeWriter &writer) { - writer.append(OpCode::GetDefiningOp, op.operation(), op.value()); + writer.append(OpCode::GetDefiningOp, op.operation()); + writer.appendPDLValue(op.value()); } void Generator::generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer) { uint32_t index = op.index(); @@ -603,6 +764,18 @@ void Generator::generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer) { writer.append(OpCode::GetOperandN, index); writer.append(op.operation(), op.value()); } +void Generator::generate(pdl_interp::GetOperandsOp op, ByteCodeWriter &writer) { + Value result = op.value(); + Optional index = op.index(); + writer.append(OpCode::GetOperands, + index.getValueOr(std::numeric_limits::max()), + op.operation()); + if (result.getType().isa()) + writer.append(getRangeStorageIndex(result)); + else + writer.append(std::numeric_limits::max()); + writer.append(result); +} void Generator::generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer) { uint32_t index = op.index(); if (index < 4) @@ -611,10 +784,29 @@ void Generator::generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer) { writer.append(OpCode::GetResultN, index); writer.append(op.operation(), op.value()); } +void Generator::generate(pdl_interp::GetResultsOp op, ByteCodeWriter &writer) { + Value result = op.value(); + Optional index = op.index(); + writer.append(OpCode::GetResults, + index.getValueOr(std::numeric_limits::max()), + op.operation()); + if (result.getType().isa()) + writer.append(getRangeStorageIndex(result)); + else + writer.append(std::numeric_limits::max()); + writer.append(result); +} void Generator::generate(pdl_interp::GetValueTypeOp op, ByteCodeWriter &writer) { - writer.append(OpCode::GetValueType, op.result(), op.value()); + if (op.getType().isa()) { + Value result = op.result(); + writer.append(OpCode::GetValueRangeTypes, result, + getRangeStorageIndex(result), op.value()); + } else { + writer.append(OpCode::GetValueType, op.result(), op.value()); + } } + void Generator::generate(pdl_interp::InferredTypesOp op, ByteCodeWriter &writer) { // InferType maps to a null type as a marker for inferring result types. @@ -628,11 +820,12 @@ void Generator::generate(pdl_interp::RecordMatchOp op, ByteCodeWriter &writer) { patterns.emplace_back(PDLByteCodePattern::create( op, rewriterToAddr[op.rewriter().getLeafReference()])); writer.append(OpCode::RecordMatch, patternIndex, - SuccessorRange(op.getOperation()), op.matchedOps(), - op.inputs()); + SuccessorRange(op.getOperation()), op.matchedOps()); + writer.appendPDLValueList(op.inputs()); } void Generator::generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer) { - writer.append(OpCode::ReplaceOp, op.operation(), op.replValues()); + writer.append(OpCode::ReplaceOp, op.operation()); + writer.appendPDLValueList(op.replValues()); } void Generator::generate(pdl_interp::SwitchAttributeOp op, ByteCodeWriter &writer) { @@ -661,6 +854,10 @@ void Generator::generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer) { writer.append(OpCode::SwitchType, op.value(), op.caseValuesAttr(), op.getSuccessors()); } +void Generator::generate(pdl_interp::SwitchTypesOp op, ByteCodeWriter &writer) { + writer.append(OpCode::SwitchTypes, op.value(), op.caseValuesAttr(), + op.getSuccessors()); +} //===----------------------------------------------------------------------===// // PDLByteCode @@ -671,7 +868,8 @@ PDLByteCode::PDLByteCode(ModuleOp module, llvm::StringMap rewriteFns) { Generator generator(module.getContext(), uniquedData, matcherByteCode, rewriterByteCode, patterns, maxValueMemoryIndex, - constraintFns, rewriteFns); + maxTypeRangeCount, maxValueRangeCount, constraintFns, + rewriteFns); generator.generate(module); // Initialize the external functions. @@ -685,6 +883,8 @@ PDLByteCode::PDLByteCode(ModuleOp module, /// bytecode. void PDLByteCode::initializeMutableState(PDLByteCodeMutableState &state) const { state.memory.resize(maxValueMemoryIndex, nullptr); + state.typeRangeMemory.resize(maxTypeRangeCount, TypeRange()); + state.valueRangeMemory.resize(maxValueRangeCount, ValueRange()); state.currentPatternBenefits.reserve(patterns.size()); for (const PDLByteCodePattern &pattern : patterns) state.currentPatternBenefits.push_back(pattern.getBenefit()); @@ -697,17 +897,24 @@ namespace { /// This class provides support for executing a bytecode stream. class ByteCodeExecutor { public: - ByteCodeExecutor(const ByteCodeField *curCodeIt, - MutableArrayRef memory, - ArrayRef uniquedMemory, - ArrayRef code, - ArrayRef currentPatternBenefits, - ArrayRef patterns, - ArrayRef constraintFunctions, - ArrayRef rewriteFunctions) - : curCodeIt(curCodeIt), memory(memory), uniquedMemory(uniquedMemory), - code(code), currentPatternBenefits(currentPatternBenefits), - patterns(patterns), constraintFunctions(constraintFunctions), + ByteCodeExecutor( + const ByteCodeField *curCodeIt, MutableArrayRef memory, + MutableArrayRef typeRangeMemory, + std::vector> &allocatedTypeRangeMemory, + MutableArrayRef valueRangeMemory, + std::vector> &allocatedValueRangeMemory, + ArrayRef uniquedMemory, ArrayRef code, + ArrayRef currentPatternBenefits, + ArrayRef patterns, + ArrayRef constraintFunctions, + ArrayRef rewriteFunctions) + : curCodeIt(curCodeIt), memory(memory), typeRangeMemory(typeRangeMemory), + allocatedTypeRangeMemory(allocatedTypeRangeMemory), + valueRangeMemory(valueRangeMemory), + allocatedValueRangeMemory(allocatedValueRangeMemory), + uniquedMemory(uniquedMemory), code(code), + currentPatternBenefits(currentPatternBenefits), patterns(patterns), + constraintFunctions(constraintFunctions), rewriteFunctions(rewriteFunctions) {} /// Start executing the code at the current bytecode index. `matches` is an @@ -722,19 +929,25 @@ private: void executeApplyConstraint(PatternRewriter &rewriter); void executeApplyRewrite(PatternRewriter &rewriter); void executeAreEqual(); + void executeAreRangesEqual(); void executeBranch(); void executeCheckOperandCount(); void executeCheckOperationName(); void executeCheckResultCount(); + void executeCheckTypes(); void executeCreateOperation(PatternRewriter &rewriter, Location mainRewriteLoc); + void executeCreateTypes(); void executeEraseOp(PatternRewriter &rewriter); void executeGetAttribute(); void executeGetAttributeType(); void executeGetDefiningOp(); void executeGetOperand(unsigned index); + void executeGetOperands(); void executeGetResult(unsigned index); + void executeGetResults(); void executeGetValueType(); + void executeGetValueRangeTypes(); void executeIsNotNull(); void executeRecordMatch(PatternRewriter &rewriter, SmallVectorImpl &matches); @@ -744,6 +957,7 @@ private: void executeSwitchOperationName(); void executeSwitchResultCount(); void executeSwitchType(); + void executeSwitchTypes(); /// Read a value from the bytecode buffer, optionally skipping a certain /// number of prefix values. These methods always update the buffer to point @@ -763,6 +977,19 @@ private: list.push_back(read()); } + /// Read a list of values from the bytecode buffer. The values may be encoded + /// as either Value or ValueRange elements. + void readValueList(SmallVectorImpl &list) { + for (unsigned i = 0, e = read(); i != e; ++i) { + if (read() == PDLValue::Kind::Value) { + list.push_back(read()); + } else { + ValueRange *values = read(); + list.append(values->begin(), values->end()); + } + } + } + /// Jump to a specific successor based on a predicate value. void selectJump(bool isTrue) { selectJump(size_t(isTrue ? 0 : 1)); } /// Jump to a specific successor based on a destination index. @@ -771,8 +998,8 @@ private: } /// Handle a switch operation with the provided value and cases. - template - void handleSwitch(const T &value, RangeT &&cases) { + template > + void handleSwitch(const T &value, RangeT &&cases, Comparator cmp = {}) { LLVM_DEBUG({ llvm::dbgs() << " * Value: " << value << "\n" << " * Cases: "; @@ -783,7 +1010,7 @@ private: // Check to see if the attribute value is within the case list. Jump to // the correct successor index based on the result. for (auto it = cases.begin(), e = cases.end(); it != e; ++it) - if (*it == value) + if (cmp(*it, value)) return selectJump(size_t((it - cases.begin()) + 1)); selectJump(size_t(0)); } @@ -795,7 +1022,9 @@ private: size_t index = *curCodeIt++; // If this type is an SSA value, it can only be stored in non-const memory. - if (llvm::is_one_of::value || index < memory.size()) + if (llvm::is_one_of::value || + index < memory.size()) return memory[index]; // Otherwise, if this index is not inbounds it is uniqued. @@ -813,17 +1042,21 @@ private: } template std::enable_if_t::value, T> readImpl() { - switch (static_cast(read())) { - case PDLValueKind::Attribute: + switch (read()) { + case PDLValue::Kind::Attribute: return read(); - case PDLValueKind::Operation: + case PDLValue::Kind::Operation: return read(); - case PDLValueKind::Type: + case PDLValue::Kind::Type: return read(); - case PDLValueKind::Value: + case PDLValue::Kind::Value: return read(); + case PDLValue::Kind::TypeRange: + return read(); + case PDLValue::Kind::ValueRange: + return read(); } - llvm_unreachable("unhandled PDLValueKind"); + llvm_unreachable("unhandled PDLValue::Kind"); } template std::enable_if_t::value, T> readImpl() { @@ -838,12 +1071,20 @@ private: std::enable_if_t::value, T> readImpl() { return *curCodeIt++; } + template + std::enable_if_t::value, T> readImpl() { + return static_cast(readImpl()); + } /// The underlying bytecode buffer. const ByteCodeField *curCodeIt; /// The current execution memory. MutableArrayRef memory; + MutableArrayRef typeRangeMemory; + std::vector> &allocatedTypeRangeMemory; + MutableArrayRef valueRangeMemory; + std::vector> &allocatedValueRangeMemory; /// References to ByteCode data necessary for execution. ArrayRef uniquedMemory; @@ -859,8 +1100,21 @@ private: /// overexposing access to information specific solely to the ByteCode. class ByteCodeRewriteResultList : public PDLResultList { public: + ByteCodeRewriteResultList(unsigned maxNumResults) + : PDLResultList(maxNumResults) {} + /// Return the list of PDL results. MutableArrayRef getResults() { return results; } + + /// Return the type ranges allocated by this list. + MutableArrayRef> getAllocatedTypeRanges() { + return allocatedTypeRanges; + } + + /// Return the value ranges allocated by this list. + MutableArrayRef> getAllocatedValueRanges() { + return allocatedValueRanges; + } }; } // end anonymous namespace @@ -893,21 +1147,46 @@ void ByteCodeExecutor::executeApplyRewrite(PatternRewriter &rewriter) { llvm::interleaveComma(args, llvm::dbgs()); llvm::dbgs() << "\n * Parameters: " << constParams << "\n"; }); - ByteCodeRewriteResultList results; + + // Execute the rewrite function. + ByteCodeField numResults = read(); + ByteCodeRewriteResultList results(numResults); rewriteFn(args, constParams, rewriter, results); - // Store the results in the bytecode memory. -#ifndef NDEBUG - ByteCodeField expectedNumberOfResults = read(); - assert(results.getResults().size() == expectedNumberOfResults && + assert(results.getResults().size() == numResults && "native PDL rewrite function returned unexpected number of results"); -#endif // Store the results in the bytecode memory. for (PDLValue &result : results.getResults()) { LLVM_DEBUG(llvm::dbgs() << " * Result: " << result << "\n"); - memory[read()] = result.getAsOpaquePointer(); + +// In debug mode we also verify the expected kind of the result. +#ifndef NDEBUG + assert(result.getKind() == read() && + "native PDL rewrite function returned an unexpected type of result"); +#endif + + // If the result is a range, we need to copy it over to the bytecodes + // range memory. + if (Optional typeRange = result.dyn_cast()) { + unsigned rangeIndex = read(); + typeRangeMemory[rangeIndex] = *typeRange; + memory[read()] = &typeRangeMemory[rangeIndex]; + } else if (Optional valueRange = + result.dyn_cast()) { + unsigned rangeIndex = read(); + valueRangeMemory[rangeIndex] = *valueRange; + memory[read()] = &valueRangeMemory[rangeIndex]; + } else { + memory[read()] = result.getAsOpaquePointer(); + } } + + // Copy over any underlying storage allocated for result ranges. + for (auto &it : results.getAllocatedTypeRanges()) + allocatedTypeRangeMemory.push_back(std::move(it)); + for (auto &it : results.getAllocatedValueRanges()) + allocatedValueRangeMemory.push_back(std::move(it)); } void ByteCodeExecutor::executeAreEqual() { @@ -919,6 +1198,32 @@ void ByteCodeExecutor::executeAreEqual() { selectJump(lhs == rhs); } +void ByteCodeExecutor::executeAreRangesEqual() { + LLVM_DEBUG(llvm::dbgs() << "Executing AreRangesEqual:\n"); + PDLValue::Kind valueKind = read(); + const void *lhs = read(); + const void *rhs = read(); + + switch (valueKind) { + case PDLValue::Kind::TypeRange: { + const TypeRange *lhsRange = reinterpret_cast(lhs); + const TypeRange *rhsRange = reinterpret_cast(rhs); + LLVM_DEBUG(llvm::dbgs() << " * " << lhs << " == " << rhs << "\n\n"); + selectJump(*lhsRange == *rhsRange); + break; + } + case PDLValue::Kind::ValueRange: { + const auto *lhsRange = reinterpret_cast(lhs); + const auto *rhsRange = reinterpret_cast(rhs); + LLVM_DEBUG(llvm::dbgs() << " * " << lhs << " == " << rhs << "\n\n"); + selectJump(*lhsRange == *rhsRange); + break; + } + default: + llvm_unreachable("unexpected `AreRangesEqual` value kind"); + } +} + void ByteCodeExecutor::executeBranch() { LLVM_DEBUG(llvm::dbgs() << "Executing Branch\n"); curCodeIt = &code[read()]; @@ -928,10 +1233,16 @@ void ByteCodeExecutor::executeCheckOperandCount() { LLVM_DEBUG(llvm::dbgs() << "Executing CheckOperandCount:\n"); Operation *op = read(); uint32_t expectedCount = read(); + bool compareAtLeast = read(); LLVM_DEBUG(llvm::dbgs() << " * Found: " << op->getNumOperands() << "\n" - << " * Expected: " << expectedCount << "\n"); - selectJump(op->getNumOperands() == expectedCount); + << " * Expected: " << expectedCount << "\n" + << " * Comparator: " + << (compareAtLeast ? ">=" : "==") << "\n"); + if (compareAtLeast) + selectJump(op->getNumOperands() >= expectedCount); + else + selectJump(op->getNumOperands() == expectedCount); } void ByteCodeExecutor::executeCheckOperationName() { @@ -948,10 +1259,44 @@ void ByteCodeExecutor::executeCheckResultCount() { LLVM_DEBUG(llvm::dbgs() << "Executing CheckResultCount:\n"); Operation *op = read(); uint32_t expectedCount = read(); + bool compareAtLeast = read(); LLVM_DEBUG(llvm::dbgs() << " * Found: " << op->getNumResults() << "\n" - << " * Expected: " << expectedCount << "\n"); - selectJump(op->getNumResults() == expectedCount); + << " * Expected: " << expectedCount << "\n" + << " * Comparator: " + << (compareAtLeast ? ">=" : "==") << "\n"); + if (compareAtLeast) + selectJump(op->getNumResults() >= expectedCount); + else + selectJump(op->getNumResults() == expectedCount); +} + +void ByteCodeExecutor::executeCheckTypes() { + LLVM_DEBUG(llvm::dbgs() << "Executing AreEqual:\n"); + TypeRange *lhs = read(); + Attribute rhs = read(); + LLVM_DEBUG(llvm::dbgs() << " * " << lhs << " == " << rhs << "\n\n"); + + selectJump(*lhs == rhs.cast().getAsValueRange()); +} + +void ByteCodeExecutor::executeCreateTypes() { + LLVM_DEBUG(llvm::dbgs() << "Executing CreateTypes:\n"); + unsigned memIndex = read(); + unsigned rangeIndex = read(); + ArrayAttr typesAttr = read().cast(); + + LLVM_DEBUG(llvm::dbgs() << " * Types: " << typesAttr << "\n\n"); + + // Allocate a buffer for this type range. + llvm::OwningArrayRef storage(typesAttr.size()); + llvm::copy(typesAttr.getAsValueRange(), storage.begin()); + allocatedTypeRangeMemory.emplace_back(std::move(storage)); + + // Assign this to the range slot and use the range as the value for the + // memory index. + typeRangeMemory[rangeIndex] = allocatedTypeRangeMemory.back(); + memory[memIndex] = &typeRangeMemory[rangeIndex]; } void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter, @@ -960,22 +1305,26 @@ void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter, unsigned memIndex = read(); OperationState state(mainRewriteLoc, read()); - readList(state.operands); + readValueList(state.operands); for (unsigned i = 0, e = read(); i != e; ++i) { Identifier name = read(); if (Attribute attr = read()) state.addAttribute(name, attr); } - bool hasInferredTypes = false; for (unsigned i = 0, e = read(); i != e; ++i) { - Type resultType = read(); - hasInferredTypes |= !resultType; - state.types.push_back(resultType); - } + if (read() == PDLValue::Kind::Type) { + state.types.push_back(read()); + continue; + } + + // If we find a null range, this signals that the types are infered. + if (TypeRange *resultTypes = read()) { + state.types.append(resultTypes->begin(), resultTypes->end()); + continue; + } - // Handle the case where the operation has inferred types. - if (hasInferredTypes) { + // Handle the case where the operation has inferred types. InferTypeOpInterface::Concept *concept = state.name.getAbstractOperation()->getInterface(); @@ -986,7 +1335,9 @@ void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter, state.attributes.getDictionary(state.getContext()), state.regions, state.types))) return; + break; } + Operation *resultOp = rewriter.createOperation(state); memory[memIndex] = resultOp; @@ -1036,11 +1387,21 @@ void ByteCodeExecutor::executeGetAttributeType() { void ByteCodeExecutor::executeGetDefiningOp() { LLVM_DEBUG(llvm::dbgs() << "Executing GetDefiningOp:\n"); unsigned memIndex = read(); - Value value = read(); - Operation *op = value ? value.getDefiningOp() : nullptr; + Operation *op = nullptr; + if (read() == PDLValue::Kind::Value) { + Value value = read(); + if (value) + op = value.getDefiningOp(); + LLVM_DEBUG(llvm::dbgs() << " * Value: " << value << "\n"); + } else { + ValueRange *values = read(); + if (values && !values->empty()) { + op = values->front().getDefiningOp(); + } + LLVM_DEBUG(llvm::dbgs() << " * Values: " << values << "\n"); + } - LLVM_DEBUG(llvm::dbgs() << " * Value: " << value << "\n" - << " * Result: " << *op << "\n"); + LLVM_DEBUG(llvm::dbgs() << " * Result: " << op << "\n"); memory[memIndex] = op; } @@ -1056,6 +1417,75 @@ void ByteCodeExecutor::executeGetOperand(unsigned index) { memory[memIndex] = operand.getAsOpaquePointer(); } +/// This function is the internal implementation of `GetResults` and +/// `GetOperands` that provides support for extracting a value range from the +/// given operation. +template