diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp index 9fbc00fbb01b009f875c00d331e6c438d7a10344..5170c68e2915ae9b3a51eb57bde4a76d013e2a64 100644 --- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -427,6 +427,50 @@ CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop, return nullptr; } +/// Returns true if we can safely unroll a multi-exit/exiting loop. OtherExits +/// is populated with all the loop exit blocks other than the LatchExit block. +static bool +canSafelyUnrollMultiExitLoop(Loop *L, SmallVectorImpl &OtherExits, + BasicBlock *LatchExit, bool PreserveLCSSA, + bool UseEpilogRemainder) { + + // Support runtime unrolling for multiple exit blocks and multiple exiting + // blocks. + if (!UnrollRuntimeMultiExit) + return false; + // Even if runtime multi exit is enabled, we currently have some correctness + // constrains in unrolling a multi-exit loop. + // We rely on LCSSA form being preserved when the exit blocks are transformed. + if (!PreserveLCSSA) + return false; + SmallVector Exits; + L->getUniqueExitBlocks(Exits); + for (auto *BB : Exits) + if (BB != LatchExit) + OtherExits.push_back(BB); + + // TODO: Support multiple exiting blocks jumping to the `LatchExit` when + // UnrollRuntimeMultiExit is true. This will need updating the logic in + // connectEpilog/connectProlog. + if (!LatchExit->getSinglePredecessor()) { + DEBUG(dbgs() << "Bailout for multi-exit handling when latch exit has >1 " + "predecessor.\n"); + return false; + } + // FIXME: We bail out of multi-exit unrolling when epilog loop is generated + // and L is an inner loop. This is because in presence of multiple exits, the + // outer loop is incorrect: we do not add the EpilogPreheader and exit to the + // outer loop. This is automatically handled in the prolog case, so we do not + // have that bug in prolog generation. + if (UseEpilogRemainder && L->getParentLoop()) + return false; + + // All constraints have been satisfied. + return true; +} + + + /// Insert code in the prolog/epilog code when unrolling a loop with a /// run-time trip-count. /// @@ -470,16 +514,8 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool UseEpilogRemainder, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, bool PreserveLCSSA) { - bool hasMultipleExitingBlocks = !L->getExitingBlock(); DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n"); DEBUG(L->dump()); - // Support only single exiting block unless UnrollRuntimeMultiExit is true. - if (!UnrollRuntimeMultiExit && hasMultipleExitingBlocks) { - DEBUG( - dbgs() - << "Multiple exiting blocks and UnrollRuntimeMultiExit not enabled!\n"); - return false; - } // Make sure the loop is in canonical form. if (!L->isLoopSimplifyForm()) { @@ -491,52 +527,27 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, BasicBlock *Latch = L->getLoopLatch(); BasicBlock *Header = L->getHeader(); - BasicBlock *LatchExit = L->getUniqueExitBlock(); // successor out of loop - if (!LatchExit && !UnrollRuntimeMultiExit) { - DEBUG(dbgs() << "No unique exit block and UnrollRuntimeMultiExit not enabled\n"); - return false; - } - // These are exit blocks other than the target of the latch exiting block. - SmallVector OtherExits; BranchInst *LatchBR = cast(Latch->getTerminator()); - unsigned int ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0; + unsigned ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0; + BasicBlock *LatchExit = LatchBR->getSuccessor(ExitIndex); // Cloning the loop basic blocks (`CloneLoopBlocks`) requires that one of the // targets of the Latch be an exit block out of the loop. This needs // to be guaranteed by the callers of UnrollRuntimeLoopRemainder. - assert(!L->contains(LatchBR->getSuccessor(ExitIndex)) && + assert(!L->contains(LatchExit) && "one of the loop latch successors should be the exit block!"); - // Support runtime unrolling for multiple exit blocks and multiple exiting - // blocks. - if (!LatchExit) { - LatchExit = LatchBR->getSuccessor(ExitIndex); - // We rely on LCSSA form being preserved when the exit blocks are - // transformed. - if (!PreserveLCSSA) - return false; - SmallVector Exits; - L->getUniqueExitBlocks(Exits); - for (auto *BB : Exits) - if (BB != LatchExit) - OtherExits.push_back(BB); - } - - assert(LatchExit && "Latch Exit should exist!"); - - // TODO: Support multiple exiting blocks jumping to the `LatchExit` when - // UnrollRuntimeMultiExit is true. This will need updating the logic in - // connectEpilog. - if (!LatchExit->getSinglePredecessor()) { - DEBUG(dbgs() << "Bailout for multi-exit handling when latch exit has >1 " - "predecessor.\n"); + // These are exit blocks other than the target of the latch exiting block. + SmallVector OtherExits; + bool isMultiExitUnrollingEnabled = canSafelyUnrollMultiExitLoop( + L, OtherExits, LatchExit, PreserveLCSSA, UseEpilogRemainder); + // Support only single exit and exiting block unless multi-exit loop unrolling is enabled. + if (!isMultiExitUnrollingEnabled && + (!L->getExitingBlock() || OtherExits.size())) { + DEBUG( + dbgs() + << "Multiple exit/exiting blocks in loop and multi-exit unrolling not " + "enabled!\n"); return false; } - // FIXME: We bail out of multi-exit unrolling when epilog loop is generated - // and L is an inner loop. This is because in presence of multiple exits, the - // outer loop is incorrect: we do not add the EpilogPreheader and exit to the - // outer loop. This is automatically handled in the prolog case, so we do not - // have that bug in prolog generation. - if (hasMultipleExitingBlocks && UseEpilogRemainder && L->getParentLoop()) - return false; // Use Scalar Evolution to compute the trip count. This allows more loops to // be unrolled than relying on induction var simplification. if (!SE) diff --git a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll index 04661314eb1d74d722245652fcfa0fdc863e8247..878f4e8c78f0fd8a090ba848986c4cbf6244a63c 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-loop.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-loop.ll @@ -170,6 +170,74 @@ for.end: ; preds = %for.cond.for.end_cr ret i16 %res.0.lcssa } +; dont unroll loop with multiple exit/exiting blocks, unless +; -runtime-unroll-multi-exit=true +; single exit, multiple exiting blocks. +define void @unique_exit(i32 %arg) { +; PROLOG: unique_exit( +; PROLOG-NOT: .unr + +; EPILOG: unique_exit( +; EPILOG-NOT: .unr +entry: + %tmp = icmp sgt i32 undef, %arg + br i1 %tmp, label %preheader, label %returnblock + +preheader: ; preds = %entry + br label %header + +LoopExit: ; preds = %header, %latch + %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ] + br label %returnblock + +returnblock: ; preds = %LoopExit, %entry + %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %LoopExit ] + ret void + +header: ; preds = %preheader, %latch + %tmp4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ] + %inc = add nsw i32 %tmp4, 1 + br i1 true, label %LoopExit, label %latch + +latch: ; preds = %header + %cmp = icmp slt i32 %inc, undef + br i1 %cmp, label %header, label %LoopExit +} + +; multiple exit blocks. don't unroll +define void @multi_exit(i64 %trip, i1 %cond) { +; PROLOG: multi_exit( +; PROLOG-NOT: .unr + +; EPILOG: multi_exit( +; EPILOG-NOT: .unr +entry: + br label %loop_header + +loop_header: + %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop_latch ] + br i1 %cond, label %loop_latch, label %loop_exiting_bb1 + +loop_exiting_bb1: + br i1 false, label %loop_exiting_bb2, label %exit1 + +loop_exiting_bb2: + br i1 false, label %loop_latch, label %exit3 + +exit3: + ret void + +loop_latch: + %iv_next = add i64 %iv, 1 + %cmp = icmp ne i64 %iv_next, %trip + br i1 %cmp, label %loop_header, label %exit2.loopexit + +exit1: + ret void + +exit2.loopexit: + ret void +} !0 = distinct !{!0, !1} !1 = !{!"llvm.loop.unroll.runtime.disable"}