Skip to content

Commit

Permalink
JIT: Always track the context for late devirt (#112396)
Browse files Browse the repository at this point in the history 
* Track InlinersContext in LateDevirtualizationInfo

* Rework fix

* Update inline context for statement as well

* Use IL offset from call

* Always set DebugInfo

* Track InlineContext

* Cleanup

* Always record late devirt info

* Always record late devirt info

* Copy gtLateDevirtualizationInfo

* Refactor to take InlineContext explicitly

* Set inlinersContext to nullptr

* Late devirt info always available

* Oops

* Remove the reduandant flag
  • Loading branch information
@hez2010
hez2010 authored Feb 19, 2025
1 parent bde9ba9 commit 54e55db
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 46 deletions.
5 changes: 4 additions & 1 deletion src/coreclr/jit/compiler.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5247,6 +5247,7 @@ class Compiler
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
InlineContext* inlinersContext,
InlineCandidateInfo** ppInlineCandidateInfo,
InlineResult* inlineResult);

Expand All @@ -5268,13 +5269,15 @@ class Compiler
void impMarkInlineCandidate(GenTree* call,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo);
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext);

void impMarkInlineCandidateHelper(GenTreeCall* call,
uint8_t candidateIndex,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext,
InlineResult* inlineResult);

bool impTailCallRetTypeCompatible(bool allowWidening,
Expand Down
19 changes: 4 additions & 15 deletions src/coreclr/jit/fginline.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -589,22 +589,13 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
}
#endif // DEBUG

CORINFO_CONTEXT_HANDLE context = nullptr;
CORINFO_CONTEXT_HANDLE context = call->gtLateDevirtualizationInfo->exactContextHnd;
InlineContext* inlinersContext = call->gtLateDevirtualizationInfo->inlinersContext;
CORINFO_METHOD_HANDLE method = call->gtCallMethHnd;
unsigned methodFlags = 0;
const bool isLateDevirtualization = true;
const bool explicitTailCall = call->IsTailPrefixedCall();

if ((call->gtCallMoreFlags & GTF_CALL_M_HAS_LATE_DEVIRT_INFO) != 0)
{
context = call->gtLateDevirtualizationInfo->exactContextHnd;
// Note: we might call this multiple times for the same trees.
// If the devirtualization below succeeds, the call becomes
// non-virtual and we won't get here again. If it does not
// succeed we might get here again so we keep the late devirt
// info.
}

CORINFO_CONTEXT_HANDLE contextInput = context;
context = nullptr;
m_compiler->impDevirtualizeCall(call, nullptr, &method, &methodFlags, &contextInput, &context,
Expand All @@ -613,10 +604,11 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
if (!call->IsVirtual())
{
assert(context != nullptr);
assert(inlinersContext != nullptr);
CORINFO_CALL_INFO callInfo = {};
callInfo.hMethod = method;
callInfo.methodFlags = methodFlags;
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo);
m_compiler->impMarkInlineCandidate(call, context, false, &callInfo, inlinersContext);

if (call->IsInlineCandidate())
{
Expand Down Expand Up @@ -652,9 +644,6 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitor<Substi
*pTree = retExpr;
}

call->GetSingleInlineCandidateInfo()->exactContextHandle = context;
INDEBUG(call->GetSingleInlineCandidateInfo()->inlinersContext = call->gtInlineContext);

JITDUMP("New inline candidate due to late devirtualization:\n");
DISPTREE(call);
}
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/gentree.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10002,6 +10002,8 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree)
copy->gtInlineInfoCount = tree->gtInlineInfoCount;
}

copy->gtLateDevirtualizationInfo = tree->gtLateDevirtualizationInfo;

copy->gtCallType = tree->gtCallType;
copy->gtReturnType = tree->gtReturnType;

Expand Down
13 changes: 7 additions & 6 deletions src/coreclr/jit/gentree.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4225,12 +4225,11 @@ enum GenTreeCallFlags : unsigned int
GTF_CALL_M_ALLOC_SIDE_EFFECTS = 0x00100000, // this is a call to an allocator with side effects
GTF_CALL_M_SUPPRESS_GC_TRANSITION = 0x00200000, // suppress the GC transition (i.e. during a pinvoke) but a separate GC safe point is required.
GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower
GTF_CALL_M_HAS_LATE_DEVIRT_INFO = 0x01000000, // this call has late devirtualzation info
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x02000000, // ldvirtftn on an interface type
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x01000000, // ldvirtftn on an interface type
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x02000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
GTF_CALL_M_CAST_OBJ_NONNULL = 0x04000000, // if we expand this specific cast we don't need to check the input object for null
// NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers
GTF_CALL_M_STACK_ARRAY = 0x10000000, // this call is a new array helper for a stack allocated array.
GTF_CALL_M_STACK_ARRAY = 0x08000000, // this call is a new array helper for a stack allocated array.
};

inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a)
Expand Down Expand Up @@ -5742,11 +5741,13 @@ struct GenTreeCall final : public GenTree
jitstd::vector<InlineCandidateInfo*>* gtInlineCandidateInfoList;

HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo;
LateDevirtualizationInfo* gtLateDevirtualizationInfo;

CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers
void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen
};

LateDevirtualizationInfo* gtLateDevirtualizationInfo; // Always available for user virtual calls

// expression evaluated after args are placed which determines the control target
GenTree* gtControlExpr;

Expand Down
55 changes: 31 additions & 24 deletions src/coreclr/jit/importercalls.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1026,7 +1026,8 @@ var_types Compiler::impImportCall(OPCODE opcode,
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);

// Is it an inline candidate?
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
compInlineContext);
}

// append the call node.
Expand Down Expand Up @@ -1236,7 +1237,20 @@ var_types Compiler::impImportCall(OPCODE opcode,
INDEBUG(call->AsCall()->gtRawILOffset = rawILOffset);

// Is it an inline candidate?
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo);
impMarkInlineCandidate(call, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, compInlineContext);

// If the call is virtual, record the inliner's context for possible use during late devirt inlining.
// Also record the generics context if there is any.
//
if (call->AsCall()->IsVirtual() && (call->AsCall()->gtCallType != CT_INDIRECT))
{
JITDUMP("\nSaving generic context %p and inline context %p for call [%06u]\n", dspPtr(exactContextHnd),
dspPtr(compInlineContext), dspTreeID(call->AsCall()));
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
info->exactContextHnd = exactContextHnd;
info->inlinersContext = compInlineContext;
call->AsCall()->gtLateDevirtualizationInfo = info;
}
}

// Extra checks for tail calls and tail recursion.
Expand Down Expand Up @@ -1427,22 +1441,6 @@ var_types Compiler::impImportCall(OPCODE opcode,
}
else
{
// If the call is virtual, and has a generics context, and is not going to have a class probe,
// record the context for possible use during late devirt.
//
// If we ever want to devirt at Tier0, and/or see issues where OSR methods under PGO lose
// important devirtualizations, we'll want to allow both a class probe and a captured context.
//
if (origCall->IsVirtual() && (origCall->gtCallType != CT_INDIRECT) && (exactContextHnd != nullptr) &&
(origCall->gtHandleHistogramProfileCandidateInfo == nullptr))
{
JITDUMP("\nSaving context %p for call [%06u]\n", dspPtr(exactContextHnd), dspTreeID(origCall));
origCall->gtCallMoreFlags |= GTF_CALL_M_HAS_LATE_DEVIRT_INFO;
LateDevirtualizationInfo* const info = new (this, CMK_Inlining) LateDevirtualizationInfo;
info->exactContextHnd = exactContextHnd;
origCall->gtLateDevirtualizationInfo = info;
}

if (isFatPointerCandidate)
{
// fatPointer candidates should be in statements of the form call() or var = call().
Expand Down Expand Up @@ -7451,6 +7449,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// inlinersContext -- the inliner's context
//
// Notes:
// Mostly a wrapper for impMarkInlineCandidateHelper that also undoes
Expand All @@ -7460,7 +7459,8 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call,
void Compiler::impMarkInlineCandidate(GenTree* callNode,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo)
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext)
{
if (!opts.OptEnabled(CLFLG_INLINING))
{
Expand All @@ -7483,7 +7483,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,

// Do the actual evaluation
impMarkInlineCandidateHelper(call, candidateId, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
&inlineResult);
inlinersContext, &inlineResult);
// Ignore non-inlineable candidates
// TODO: Consider keeping them to just devirtualize without inlining, at least for interface
// calls on NativeAOT, but that requires more changes elsewhere too.
Expand All @@ -7506,7 +7506,8 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
const uint8_t candidatesCount = call->GetInlineCandidatesCount();
assert(candidatesCount <= 1);
InlineResult inlineResult(this, call, nullptr, "impMarkInlineCandidate");
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo, &inlineResult);
impMarkInlineCandidateHelper(call, 0, exactContextHnd, exactContextNeedsRuntimeLookup, callInfo,
inlinersContext, &inlineResult);
}

// If this call is an inline candidate or is not a guarded devirtualization
Expand Down Expand Up @@ -7539,6 +7540,7 @@ void Compiler::impMarkInlineCandidate(GenTree* callNode,
// exactContextHnd -- context handle for inlining
// exactContextNeedsRuntimeLookup -- true if context required runtime lookup
// callInfo -- call info from VM
// inlinersContext -- the inliner's context
//
// Notes:
// If callNode is an inline candidate, this method sets the flag
Expand All @@ -7555,6 +7557,7 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
CORINFO_CONTEXT_HANDLE exactContextHnd,
bool exactContextNeedsRuntimeLookup,
CORINFO_CALL_INFO* callInfo,
InlineContext* inlinersContext,
InlineResult* inlineResult)
{
// Let the strategy know there's another call
Expand Down Expand Up @@ -7749,7 +7752,8 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call,
}

InlineCandidateInfo* inlineCandidateInfo = nullptr;
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, &inlineCandidateInfo, inlineResult);
impCheckCanInline(call, candidateIndex, fncHandle, methAttr, exactContextHnd, inlinersContext, &inlineCandidateInfo,
inlineResult);

if (inlineResult->IsFailure())
{
Expand Down Expand Up @@ -8378,7 +8382,6 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call,
// it's a union field used for other things by virtual
// stubs)
call->ClearInlineInfo();
call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO;

#if defined(DEBUG)
if (verbose)
Expand Down Expand Up @@ -9057,6 +9060,7 @@ bool Compiler::impTailCallRetTypeCompatible(bool allowWideni
// fncHandle - method that will be called
// methAttr - attributes for the method
// exactContextHnd - exact context for the method
// inlinersContext - the inliner's context
// ppInlineCandidateInfo [out] - information needed later for inlining
// inlineResult - result of ongoing inline evaluation
//
Expand All @@ -9069,6 +9073,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
CORINFO_METHOD_HANDLE fncHandle,
unsigned methAttr,
CORINFO_CONTEXT_HANDLE exactContextHnd,
InlineContext* inlinersContext,
InlineCandidateInfo** ppInlineCandidateInfo,
InlineResult* inlineResult)
{
Expand All @@ -9083,6 +9088,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
CORINFO_METHOD_HANDLE fncHandle;
unsigned methAttr;
CORINFO_CONTEXT_HANDLE exactContextHnd;
InlineContext* inlinersContext;
InlineResult* result;
InlineCandidateInfo** ppInlineCandidateInfo;
} param;
Expand All @@ -9094,6 +9100,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
param.fncHandle = fncHandle;
param.methAttr = methAttr;
param.exactContextHnd = (exactContextHnd != nullptr) ? exactContextHnd : MAKE_METHODCONTEXT(fncHandle);
param.inlinersContext = inlinersContext;
param.result = inlineResult;
param.ppInlineCandidateInfo = ppInlineCandidateInfo;

Expand Down Expand Up @@ -9244,7 +9251,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
pInfo->methAttr = pParam->methAttr;
pInfo->initClassResult = initClassResult;
pInfo->exactContextNeedsRuntimeLookup = false;
pInfo->inlinersContext = pParam->pThis->compInlineContext;
pInfo->inlinersContext = pParam->inlinersContext;

// Note exactContextNeedsRuntimeLookup is reset later on,
// over in impMarkInlineCandidate.
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/inline.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,7 @@ struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo
struct LateDevirtualizationInfo
{
CORINFO_CONTEXT_HANDLE exactContextHnd;
InlineContext* inlinersContext;
};

// InlArgInfo describes inline candidate argument properties.
Expand Down

0 comments on commit 54e55db

Please sign in to comment.