: m_couldTakeSlowPath(false)

, m_isProved(false)

{

if (!value || !value.isCell()) {

m_couldTakeSlowPath = true;

return;

}

m_variants.append(CallVariant(value.asCell()));

CodeBlock* profiledBlock, BytecodeIndex bytecodeIndex, const ICStatusMap& map,

ExitSiteData exitSiteData)

{

ConcurrentJSLocker locker(profiledBlock->m_lock);

UNUSED_PARAM(profiledBlock);

UNUSED_PARAM(bytecodeIndex);

UNUSED_PARAM(map);

UNUSED_PARAM(exitSiteData);

#if ENABLE(DFG_JIT)

CallLinkInfo* callLinkInfo = map.get(CodeOrigin(bytecodeIndex)).callLinkInfo;

if (!callLinkInfo)

return CallLinkStatus();

// m_jitData is nullptr when it is tied to LLInt (not Baseline).

if (callLinkInfo->type() == CallLinkInfo::Type::DataOnly && !profiledBlock->m_jitData) {

if (exitSiteData.takesSlowPath)

return takesSlowPath();

#if ENABLE(DFG_JIT)

if (profiledBlock->unlinkedCodeBlock()->hasExitSite(DFG::FrequentExitSite(bytecodeIndex, BadConstantValue))) {

// We could force this to be a closure call, but instead we'll just assume that it

// takes slow path.

return takesSlowPath();

}

#endif

}

return computeFor(locker, profiledBlock, *callLinkInfo, exitSiteData);

#else

return CallLinkStatus();

#endif

CodeBlock* profiledBlock, BytecodeIndex bytecodeIndex, const ICStatusMap& map)

{

return computeFor(profiledBlock, bytecodeIndex, map, computeExitSiteData(profiledBlock, bytecodeIndex));

{

ExitSiteData exitSiteData;

#if ENABLE(DFG_JIT)

UnlinkedCodeBlock* codeBlock = profiledBlock->unlinkedCodeBlock();

ConcurrentJSLocker locker(codeBlock->m_lock);

auto takesSlowPath = [&] (ExitingInlineKind inlineKind) -> ExitFlag {

return ExitFlag(

codeBlock->hasExitSite(locker, DFG::FrequentExitSite(bytecodeIndex, BadType, ExitFromAnything, inlineKind))

|| codeBlock->hasExitSite(locker, DFG::FrequentExitSite(bytecodeIndex, BadExecutable, ExitFromAnything, inlineKind)),

inlineKind);

};

auto badFunction = [&] (ExitingInlineKind inlineKind) -> ExitFlag {

return ExitFlag(codeBlock->hasExitSite(locker, DFG::FrequentExitSite(bytecodeIndex, BadConstantValue, ExitFromAnything, inlineKind)), inlineKind);

};

exitSiteData.takesSlowPath |= takesSlowPath(ExitFromNotInlined);

exitSiteData.takesSlowPath |= takesSlowPath(ExitFromInlined);

exitSiteData.badFunction |= badFunction(ExitFromNotInlined);

exitSiteData.badFunction |= badFunction(ExitFromInlined);

#else

UNUSED_PARAM(profiledBlock);

UNUSED_PARAM(bytecodeIndex);

#endif

return exitSiteData;

const ConcurrentJSLocker& locker, CodeBlock* profiledBlock, CallLinkInfo& callLinkInfo)

{

// We don't really need this, but anytime we have to debug this code, it becomes indispensable.

UNUSED_PARAM(profiledBlock);

CallLinkStatus result = computeFromCallLinkInfo(locker, callLinkInfo);

result.m_maxArgumentCountIncludingThisForVarargs = callLinkInfo.maxArgumentCountIncludingThisForVarargs();

return result;

const ConcurrentJSLocker&, CallLinkInfo& callLinkInfo)

{

if (callLinkInfo.clearedByGC() || callLinkInfo.clearedByVirtual())

return takesSlowPath();

// Note that despite requiring that the locker is held, this code is racy with respect

// to the CallLinkInfo: it may get cleared while this code runs! This is because

// CallLinkInfoBase::unlinkOrUpgrade() may be called from a different CodeBlock than the one that owns

// the CallLinkInfo and currently we save space by not having CallLinkInfos know who owns

// them. So, there is no way for either the caller of CallLinkInfo::unlock() or unlock()

// itself to figure out which lock to lock.

//

// Fortunately, that doesn't matter. The only things we ask of CallLinkInfo - the stub

// and the target - can all be asked racily. Stubs and targets can

// only be deleted at next GC, so if we load a non-null one, then it must contain data

// that is still marginally valid (i.e. the pointers ain't stale). This kind of raciness

// is probably OK for now.

// PolymorphicCallStubRoutine is a GCAwareJITStubRoutine, so if non-null, it will stay alive

// until next GC even if the CallLinkInfo is concurrently cleared. Also, the variants list is

// never mutated after the PolymorphicCallStubRoutine is instantiated. We have some conservative

// fencing in place to make sure that we see the variants list after construction.

if (PolymorphicCallStubRoutine* stub = callLinkInfo.stub()) {

WTF::dependentLoadLoadFence();

if (!stub->hasEdges()) {

// This means we have an FTL profile, which has incomplete information.

//

// It's not clear if takesSlowPath() or CallLinkStatus() are appropriate here, but I

// think that takesSlowPath() is a narrow winner.

//

// Either way, this is telling us that the FTL had been led to believe that it's

// profitable not to inline anything.

return takesSlowPath();

}

CallEdgeList edges = stub->edges();

// Now that we've loaded the edges list, there are no further concurrency concerns. We will

// just manipulate and prune this list to our liking - mostly removing entries that are too

// infrequent and ensuring that it's sorted in descending order of frequency.

RELEASE_ASSERT(edges.size());

std::ranges::sort(edges, std::ranges::greater { }, &CallEdge::count);

RELEASE_ASSERT(edges.first().count() >= edges.last().count());

double totalCallsToKnown = 0;

double totalCallsToUnknown = 0;

CallVariantList variants;

for (size_t i = 0; i < edges.size(); ++i) {

CallEdge edge = edges[i];

// If the call is at the tail of the distribution, then we don't optimize it and we

// treat it as if it was a call to something unknown. We define the tail as being either

// a call that doesn't belong to the N most frequent callees (N =

// maxPolymorphicCallVariantsForInlining) or that has a total call count that is too

// small.

if (i >= Options::maxPolymorphicCallVariantsForInlining()

|| edge.count() < Options::frequentCallThreshold())

totalCallsToUnknown += edge.count();

else {

totalCallsToKnown += edge.count();

variants.append(edge.callee());

}

}

// Bail if we didn't find any calls that qualified.

RELEASE_ASSERT(!!totalCallsToKnown == !!variants.size());

if (variants.isEmpty())

return takesSlowPath();

// We require that the distribution of callees is skewed towards a handful of common ones.

if (totalCallsToKnown / totalCallsToUnknown < Options::minimumCallToKnownRate())

return takesSlowPath();

RELEASE_ASSERT(totalCallsToKnown);

RELEASE_ASSERT(variants.size());

CallLinkStatus result;

result.m_variants = variants;

result.m_couldTakeSlowPath = !!totalCallsToUnknown;

result.m_isBasedOnStub = true;

return result;

}

CallLinkStatus result;

if (JSObject* class="pl-c1">lastSeenCallee()) {

CallVariant variant(target);

if (callLinkInfo.hasSeenClosure())

variant = variant.despecifiedClosure();

result.m_variants.append(variant);

}

return result;

const ConcurrentJSLocker& locker, CodeBlock* profiledBlock, CallLinkInfo& callLinkInfo,

ExitSiteData exitSiteData, ExitingInlineKind inlineKind)

{

CallLinkStatus result = computeFor(locker, profiledBlock, callLinkInfo);

result.accountForExits(exitSiteData, inlineKind);

return result;

{

if (exitSiteData.badFunction.isSet(inlineKind)) {

if (isBasedOnStub()) {

// If we have a polymorphic stub, then having an exit site is not quite so useful. In

// most cases, the information in the stub has higher fidelity.

makeClosureCall();

} else {

// We might not have a polymorphic stub for any number of reasons. When this happens, we

// are in less certain territory, so exit sites mean a lot.

m_couldTakeSlowPath = true;

}

}

if (exitSiteData.takesSlowPath.isSet(inlineKind))

m_couldTakeSlowPath = true;

CodeBlock* profiledBlock, CodeOrigin codeOrigin,

const ICStatusMap& baselineMap, const ICStatusContextStack& optimizedStack)

{

if (CallLinkStatusInternal::verbose)

dataLog("Figuring out call profiling for ", codeOrigin, "\n");

ExitSiteData exitSiteData = computeExitSiteData(profiledBlock, codeOrigin.bytecodeIndex());

if (CallLinkStatusInternal::verbose) {

dataLog("takesSlowPath = ", exitSiteData.takesSlowPath, "\n");

dataLog("badFunction = ", exitSiteData.badFunction, "\n");

}

for (const ICStatusContext* context : optimizedStack) {

if (CallLinkStatusInternal::verbose)

dataLog("Examining status in ", pointerDump(context->optimizedCodeBlock), "\n");

ICStatus status = context->get(codeOrigin);

// If we have both a CallLinkStatus and a callLinkInfo for the same origin, then it means

// one of two things:

//

// 1) We initially thought that we'd be able to inline this call so we recorded a status

// but then we realized that it was pointless and gave up and emitted a normal call

// anyway.

//

// 2) We did a polymorphic call inline that had a slow path case.

//

// In the first case, it's essential that we use the callLinkInfo, since the status may

// be polymorphic but the link info benefits from polyvariant profiling.

//

// In the second case, it's essential that we use the status, since the callLinkInfo

// corresponds to the slow case.

//

// It would be annoying to distinguish those two cases. However, we know that:

//

// If the first case happens in the FTL, then it means that even with polyvariant

// profiling, we failed to do anything useful. That suggests that in the FTL, it's OK to

// prioritize the status. That status will again tell us to not do anything useful.

//

// The second case only happens in the FTL.

//

// Therefore, we prefer the status in the FTL and the info in the DFG.

//

// Luckily, this case doesn't matter for the other ICStatuses, since they never do the

// fast-path-slow-path control-flow-diamond style of IC inlining. It's either all fast

// path or it's a full IC. So, for them, if there is an IC status then it means case (1).

bool checkStatusFirst = context->optimizedCodeBlock->jitType() == JITType::FTLJIT;

auto bless = [&] (CallLinkStatus& result) {

if (!context->isInlined(codeOrigin))

result.merge(computeFor(profiledBlock, codeOrigin.bytecodeIndex(), baselineMap, exitSiteData));

};

auto checkInfo = [&] () -> CallLinkStatus {

if (!status.callLinkInfo)

return CallLinkStatus();

dataLogLnIf(CallLinkStatusInternal::verbose, "Have CallLinkInfo with CodeOrigin = ", codeOrigin, "\n");

CallLinkStatus result;

{

ConcurrentJSLocker locker(context->optimizedCodeBlock->m_lock);

result = computeFor(

locker, context->optimizedCodeBlock, *status.callLinkInfo, exitSiteData,

context->inlineKind(codeOrigin));

if (CallLinkStatusInternal::verbose)

dataLog("Got result: ", result, "\n");

}

bless(result);

return result;

};

auto checkStatus = [&] () -> CallLinkStatus {

if (!status.callStatus)

return CallLinkStatus();

CallLinkStatus result = *status.callStatus;

if (CallLinkStatusInternal::verbose)

dataLog("Have callStatus: ", result, "\n");

result.accountForExits(exitSiteData, context->inlineKind(codeOrigin));

bless(result);

return result;

};

if (checkStatusFirst) {

if (CallLinkStatus result = checkStatus())

return result;

if (CallLinkStatus result = checkInfo())

return result;

continue;

}

if (CallLinkStatus result = checkInfo())

return result;

if (CallLinkStatus result = checkStatus())

return result;

}

return computeFor(profiledBlock, codeOrigin.bytecodeIndex(), baselineMap, exitSiteData);

{

m_variants = CallVariantList{ variant };

m_couldTakeSlowPath = false;

m_isProved = true;

{

for (unsigned i = m_variants.size(); i--;) {

if (m_variants[i].isClosureCall())

return true;

}

return false;

{

for (CallVariant& variant : m_variants) {

if (!variant.isStillLive(vm))

return false;

}

return true;

{

m_couldTakeSlowPath |= other.m_couldTakeSlowPath;

for (const CallVariant& otherVariant : other.m_variants) {

bool found = false;

for (CallVariant& thisVariant : m_variants) {

if (thisVariant.merge(otherVariant)) {

found = true;

break;

}

}

if (!found)

m_variants.append(otherVariant);

}

{

m_variants.removeAllMatching(

[&] (CallVariant& variant) -> bool {

variant.filter(value);

return !variant;

});

{

if (!isSet()) {

out.print("Not Set"_s);

return;

}

CommaPrinter comma;

if (m_isProved)

out.print(comma, "Statically Proved"_s);

if (m_couldTakeSlowPath)

out.print(comma, "Could Take Slow Path"_s);

if (m_isBasedOnStub)

out.print(comma, "Based On Stub"_s);

if (!m_variants.isEmpty())

out.print(comma, listDump(m_variants));

if (m_maxArgumentCountIncludingThisForVarargs)

out.print(comma, "maxArgumentCountIncludingThisForVarargs = "_s, m_maxArgumentCountIncludingThisForVarargs);