[InstCombine] Check for out of range ashr values using APInt before calling getZExtValue

Reduced from oss-fuzz #5032 test case

llvm-svn: 322078

llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp

@@ -818,7 +818,7 @@ Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
   Type *Ty = I.getType();
   unsigned BitWidth = Ty->getScalarSizeInBits();
   const APInt *ShAmtAPInt;
-  if (match(Op1, m_APInt(ShAmtAPInt))) {
+  if (match(Op1, m_APInt(ShAmtAPInt)) && ShAmtAPInt->ult(BitWidth)) {
     unsigned ShAmt = ShAmtAPInt->getZExtValue();
 
     // If the shift amount equals the difference in width of the destination
@@ -832,7 +832,8 @@ Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
     // We can't handle (X << C1) >>s C2. It shifts arbitrary bits in. However,
     // we can handle (X <<nsw C1) >>s C2 since it only shifts in sign bits.
     const APInt *ShOp1;
-    if (match(Op0, m_NSWShl(m_Value(X), m_APInt(ShOp1)))) {
+    if (match(Op0, m_NSWShl(m_Value(X), m_APInt(ShOp1))) &&
+        ShOp1->ult(BitWidth)) {
       unsigned ShlAmt = ShOp1->getZExtValue();
       if (ShlAmt < ShAmt) {
         // (X <<nsw C1) >>s C2 --> X >>s (C2 - C1)
@@ -850,7 +851,8 @@ Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
       }
     }
 
-    if (match(Op0, m_AShr(m_Value(X), m_APInt(ShOp1)))) {
+    if (match(Op0, m_AShr(m_Value(X), m_APInt(ShOp1))) &&
+        ShOp1->ult(BitWidth)) {
       unsigned AmtSum = ShAmt + ShOp1->getZExtValue();
       // Oversized arithmetic shifts replicate the sign bit.
       AmtSum = std::min(AmtSum, BitWidth - 1);

llvm/test/Transforms/InstCombine/shift.ll

@@ -1613,3 +1613,26 @@ define i177 @lshr_out_of_range(i177 %Y, i177** %A2) {
   %B1 = udiv i177 %B10, %B6
   ret i177 %B1
 }
+
+; OSS Fuzz #5032
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=5032
+define void @ashr_out_of_range(i177* %A) {
+; CHECK-LABEL: @ashr_out_of_range(
+; CHECK-NEXT:    ret void
+;
+  %L = load i177, i177* %A
+  %B5 = udiv i177 %L, -1
+  %B4 = add i177 %B5, -1
+  %B2 = add i177 %B4, -1
+  %G11 = getelementptr i177, i177* %A, i177 %B2
+  %L7 = load i177, i177* %G11
+  %B6 = mul i177 %B5, %B2
+  %B24 = ashr i177 %L7, %B6
+  %B36 = and i177 %L7, %B4
+  %C17 = icmp sgt i177 %B36, %B24
+  %G62 = getelementptr i177, i177* %G11, i1 %C17
+  %B28 = urem i177 %B24, %B6
+  store i177 %B28, i177* %G62
+  ret void
+}
+