Use IntegralRange in fgOptimizeCast (#59897)

* Set call return type in fgMorphIntoHelperCall

It was just forgotten.

* Introduce IntegralRange::ForNode

* Use Range::ForNode in assertion propagation

Couple good diffs from the handling of GT_CALL.

* Fix IntegralRange::ForCastInput

The comments are right, the code is wrong...

Fortunately, the bug was a pessimizing one, not a correctness issue.

* Use IntegralRange in fgOptimizeCast

And reap the benefits.

src/coreclr/jit/assertionprop.cpp

@@ -118,6 +118,59 @@ bool IntegralRange::Contains(int64_t value) const
     }
 }
 
+//------------------------------------------------------------------------
+// ForNode: Compute the integral range for a node.
+//
+// Arguments:
+//    node     - the node, of an integral type, in question
+//    compiler - the Compiler, used to retrieve additional info
+//
+// Return Value:
+//    The integral range this node produces.
+//
+/* static */ IntegralRange IntegralRange::ForNode(GenTree* node, Compiler* compiler)
+{
+    assert(varTypeIsIntegral(node));
+
+    var_types rangeType = node->TypeGet();
+
+    switch (node->OperGet())
+    {
+        case GT_EQ:
+        case GT_NE:
+        case GT_LT:
+        case GT_LE:
+        case GT_GE:
+        case GT_GT:
+            return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One};
+
+        case GT_ARR_LENGTH:
+            return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::IntMax};
+
+        case GT_CALL:
+            if (node->AsCall()->NormalizesSmallTypesOnReturn())
+            {
+                rangeType = static_cast<var_types>(node->AsCall()->gtReturnType);
+            }
+            break;
+
+        case GT_LCL_VAR:
+            if (compiler->lvaGetDesc(node->AsLclVar())->lvNormalizeOnStore())
+            {
+                rangeType = compiler->lvaGetDesc(node->AsLclVar())->TypeGet();
+            }
+            break;
+
+        case GT_CAST:
+            return ForCastOutput(node->AsCast());
+
+        default:
+            break;
+    }
+
+    return ForType(rangeType);
+}
+
 //------------------------------------------------------------------------
 // ForCastInput: Get the non-overflowing input range for a cast.
 //
@@ -214,7 +267,14 @@ bool IntegralRange::Contains(int64_t value) const
             // CAST_OVF(long <- ulong)    - [0..LONG_MAX]
             // CAST_OVF(long <- long)     - [LONG_MIN..LONG_MAX]
             case TYP_LONG:
-                lowerBound = fromUnsigned ? SymbolicIntegerValue::Zero : LowerBoundForType(fromType);
+                if (fromUnsigned && (fromType == TYP_LONG))
+                {
+                    lowerBound = SymbolicIntegerValue::Zero;
+                }
+                else
+                {
+                    lowerBound = LowerBoundForType(fromType);
+                }
                 upperBound = UpperBoundForType(fromType);
                 break;
 
@@ -1545,12 +1605,17 @@ AssertionIndex Compiler::optCreateAssertion(GenTree*         op1,
             }
 
             // Try and see if we can make a subrange assertion.
-            if (((assertionKind == OAK_SUBRANGE) || (assertionKind == OAK_EQUAL)))
+            if (((assertionKind == OAK_SUBRANGE) || (assertionKind == OAK_EQUAL)) && varTypeIsIntegral(op2))
             {
-                if (optTryExtractSubrangeAssertion(op2, &assertion.op2.u2))
+                IntegralRange nodeRange = IntegralRange::ForNode(op2, this);
+                IntegralRange typeRange = IntegralRange::ForType(genActualType(op2));
+                assert(typeRange.Contains(nodeRange));
+
+                if (!typeRange.Equals(nodeRange))
                 {
                     assertion.op2.kind      = O2K_SUBRANGE;
                     assertion.assertionKind = OAK_SUBRANGE;
+                    assertion.op2.u2        = nodeRange;
                 }
             }
         }
@@ -1689,73 +1754,6 @@ AssertionIndex Compiler::optFinalizeCreatingAssertion(AssertionDsc* assertion)
     return optAddAssertion(assertion);
 }
 
-//------------------------------------------------------------------------
-// optTryExtractSubrangeAssertion: Extract the bounds of the value a tree produces.
-//
-// Generates [0..1] ranges for relops, [T_MIN..T_MAX] for small-typed indirections
-// and casts to small types. Generates various ranges for casts to and from "large"
-// types - see "IntegralRange::ForCastOutput".
-//
-// Arguments:
-//    source - tree producing the value
-//    pRange - [out] parameter for the range
-//
-// Return Value:
-//    "true" if the "source" computes a value that could be used for a subrange
-//    assertion, i. e. narrower than the range of the node's type.
-//
-// Notes:
-//   The "pRange" parameter is only written to if the function returns "true".
-//
-bool Compiler::optTryExtractSubrangeAssertion(GenTree* source, IntegralRange* pRange)
-{
-    var_types sourceType = TYP_UNDEF;
-
-    switch (source->OperGet())
-    {
-        case GT_EQ:
-        case GT_NE:
-        case GT_LT:
-        case GT_LE:
-        case GT_GT:
-        case GT_GE:
-            *pRange = {SymbolicIntegerValue::Zero, SymbolicIntegerValue::One};
-            return true;
-
-        case GT_CLS_VAR:
-        case GT_LCL_FLD:
-        case GT_IND:
-            sourceType = source->TypeGet();
-            break;
-
-        case GT_CAST:
-            if (varTypeIsIntegral(source))
-            {
-                IntegralRange castRange = IntegralRange::ForCastOutput(source->AsCast());
-                IntegralRange nodeRange = IntegralRange::ForType(source->TypeGet());
-                assert(nodeRange.Contains(castRange));
-
-                if (!castRange.Equals(nodeRange))
-                {
-                    *pRange = castRange;
-                    return true;
-                }
-            }
-            return false;
-
-        default:
-            return false;
-    }
-
-    if (varTypeIsSmall(sourceType))
-    {
-        *pRange = IntegralRange::ForType(sourceType);
-        return true;
-    }
-
-    return false;
-}
-
 /*****************************************************************************
  *
  * If tree is a constant node holding an integral value, retrieve the value in

src/coreclr/jit/compiler.h

@@ -1216,6 +1216,11 @@ class IntegralRange
         return (m_lowerBound <= other.m_lowerBound) && (other.m_upperBound <= m_upperBound);
     }
 
+    bool IsPositive()
+    {
+        return m_lowerBound >= SymbolicIntegerValue::Zero;
+    }
+
     bool Equals(IntegralRange other) const
     {
         return (m_lowerBound == other.m_lowerBound) && (m_upperBound == other.m_upperBound);
@@ -1230,6 +1235,7 @@ class IntegralRange
         return {LowerBoundForType(type), UpperBoundForType(type)};
     }
 
+    static IntegralRange ForNode(GenTree* node, Compiler* compiler);
     static IntegralRange ForCastInput(GenTreeCast* cast);
     static IntegralRange ForCastOutput(GenTreeCast* cast);
 
@@ -6363,7 +6369,7 @@ class Compiler
     GenTree* fgMorphCopyBlock(GenTree* tree);
     GenTree* fgMorphForRegisterFP(GenTree* tree);
     GenTree* fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac = nullptr);
-    GenTree* fgOptimizeCast(GenTree* tree);
+    GenTree* fgOptimizeCast(GenTreeCast* cast);
     GenTree* fgOptimizeEqualityComparisonWithConst(GenTreeOp* cmp);
     GenTree* fgOptimizeRelationalComparisonWithConst(GenTreeOp* cmp);
     GenTree* fgPropagateCommaThrow(GenTree* parent, GenTreeOp* commaThrow, GenTreeFlags precedingSideEffects);

src/coreclr/jit/gentree.h

@@ -4636,6 +4636,11 @@ struct GenTreeCall final : public GenTree
     }
 #endif // !FEATURE_TAILCALL_OPT
 
+    bool NormalizesSmallTypesOnReturn()
+    {
+        return GetUnmanagedCallConv() == CorInfoCallConvExtension::Managed;
+    }
+
     bool IsSameThis() const
     {
         return (gtCallMoreFlags & GTF_CALL_M_NONVIRT_SAME_THIS) != 0;