{

dataLogLnIf(WasmOperationsInternal::verbose, "operationJSToWasmEntryWrapperBuildFrame sp: ", RawPointer(sp), " fp: ", RawPointer(callFrame));

auto* globalObject = function->realm();

VM& vm = globalObject->vm();

if (function->taintedness().isTainted())

vm.setMightBeExecutingTaintedCode();

WasmOperationPrologueCallFrameTracer tracer(vm, callFrame, OUR_RETURN_ADDRESS);

auto* callee = function->jsToWasmCallee();

ASSERT(function);

ASSERT(callee->compilationMode() == CompilationMode::JSToWasmMode);

ASSERT(&callee->rtt() == function->rtt());

ASSERT(callee->frameSize() + JSToWasmCallee::SpillStackSpaceAligned == (reinterpret_cast<uintptr_t>(callFrame) - reinterpret_cast<uintptr_t>(sp)));

dataLogLnIf(WasmOperationsInternal::verbose, "operationJSToWasmEntryWrapperBuildFrame setting callee: ", RawHex(CalleeBits::encodeNativeCallee(callee)));

dataLogLnIf(WasmOperationsInternal::verbose, "operationJSToWasmEntryWrapperBuildFrame wasm callee: ", RawHex(callee->wasmCallee().encodedBits()));

auto scope = DECLARE_THROW_SCOPE(vm);

auto calleeSPOffsetFromFP = -(static_cast<intptr_t>(callee->frameSize()) + JSToWasmCallee::SpillStackSpaceAligned - JSToWasmCallee::RegisterStackSpaceAligned);

const RTT& signature = *function->rtt();

if (signature.argumentsOrResultsIncludeV128() || signature.argumentsOrResultsIncludeExnref()) [[unlikely]] {

throwVMTypeError(globalObject, scope, Wasm::errorMessageForExceptionType(Wasm::ExceptionType::TypeErrorInvalidValueUse));

OPERATION_RETURN(scope, callee);

}

auto access = [sp, callFrame]<typename V>(auto* arr, int i) -> V* {

dataLogLnIf(WasmOperationsInternal::verbose, "fp[", (&reinterpret_cast<uint8_t*>(arr)[i / sizeof(uint8_t)] - reinterpret_cast<uint8_t*>(callFrame)), "] sp[", (&reinterpret_cast<uint8_t*>(arr)[i / sizeof(uint8_t)] - reinterpret_cast<uint8_t*>(sp)), "](", RawHex(reinterpret_cast<V*>(arr)[i / sizeof(V)]), ")");

return &reinterpret_cast<V*>(arr)[i / sizeof(V)];

};

CallInformation wasmFrameConvention = wasmCallingConvention().callInformationFor(signature, CallRole::Caller);

uint64_t* registerSpace = reinterpret_cast<uint64_t*>(sp);

for (unsigned i = 0; i < signature.argumentCount(); ++i) {

JSValue jsArg = callFrame->argument(i);

Type type = signature.argumentType(i);

dataLogLnIf(WasmOperationsInternal::verbose, "Arg ", i, " ", wasmFrameConvention.params[i].location);

uint64_t value = toWebAssemblyValue(globalObject, type, jsArg);

OPERATION_RETURN_IF_EXCEPTION(scope, callee);

if (wasmFrameConvention.params[i].location.isStackArgument()) {

auto dst = wasmFrameConvention.params[i].location.offsetFromSP() + calleeSPOffsetFromFP;

if (type.isI32() || type.isF32())

*access.operator()<uint32_t>(callFrame, dst) = static_cast<uint32_t>(value);

else

*access.operator()<uint64_t>(callFrame, dst) = value;

} else {

int dst = 0;

if (wasmFrameConvention.params[i].location.isFPR())

dst = GPRInfo::numberOfArgumentRegisters * sizeof(UCPURegister) + FPRInfo::toArgumentIndex(wasmFrameConvention.params[i].location.fpr()) * bytesForWidth(Width::Width64);

else

dst = GPRInfo::toArgumentIndex(wasmFrameConvention.params[i].location.gpr()) * sizeof(UCPURegister);

ASSERT(dst >= 0);

dataLogLnIf(WasmOperationsInternal::verbose, "* Register Arg ", i, " ", dst);

if (type.isI32() || type.isF32())

value = static_cast<uint64_t>(static_cast<uint32_t>(value));

*access.operator()<uint64_t>(registerSpace, dst) = value;

}

}

OPERATION_RETURN(scope, callee);

{

auto access = []<typename V>(auto* arr, int i) -> V* {

return &reinterpret_cast<V*>(arr)[i / sizeof(V)];

};

// We need to set up them immediately before potentially throwing anything.

auto singletonCallee = CalleeBits::boxNativeCallee(&WasmToJSCallee::singleton());

*access.operator()<uintptr_t>(callFrame, CallFrameSlot::codeBlock * sizeof(Register)) = std::bit_cast<uintptr_t>(instance);

*access.operator()<uintptr_t>(callFrame, CallFrameSlot::callee * sizeof(Register)) = std::bit_cast<uintptr_t>(singletonCallee);

CallFrame* calleeFrame = std::bit_cast<CallFrame*>(reinterpret_cast<uintptr_t>(sp) - sizeof(CallerFrameAndPC));

ASSERT(instance);

JSGlobalObject* globalObject = instance->realm();

ASSERT(globalObject);

VM& vm = instance->vm();

WasmOperationPrologueCallFrameTracer tracer(vm, callFrame, OUR_RETURN_ADDRESS);

auto scope = DECLARE_THROW_SCOPE(vm);

auto* importableFunction = *access.operator()<WasmOrJSImportableFunctionCallLinkInfo*>(callFrame, WasmToJSCallableFunctionSlot);

const RTT& signature = *importableFunction->rtt;

unsigned argCount = signature.argumentCount();

if (signature.argumentsOrResultsIncludeV128() || signature.argumentsOrResultsIncludeExnref()) [[unlikely]] {

throwVMTypeError(globalObject, scope, Wasm::errorMessageForExceptionType(Wasm::ExceptionType::TypeErrorInvalidValueUse));

OPERATION_RETURN(scope);

}

const auto& wasmCC = wasmCallingConvention().callInformationFor(signature, CallRole::Callee);

const auto& jsCC = jsCallingConvention().callInformationFor(signature, CallRole::Callee);

for (unsigned argNum = 0; argNum < argCount; ++argNum) {

Type argType = signature.argumentType(argNum);

auto wasmParam = wasmCC.params[argNum].location;

auto dst = jsCC.params[argNum].location.offsetFromFP();

switch (argType.kind()) {

case TypeKind::Void:

case TypeKind::Structref:

case TypeKind::Arrayref:

case TypeKind::Eqref:

case TypeKind::Anyref:

case TypeKind::Noexnref:

case TypeKind::Noneref:

case TypeKind::Nofuncref:

case TypeKind::Noexternref:

case TypeKind::I31ref:

case TypeKind::V128:

RELEASE_ASSERT_NOT_REACHED();

case TypeKind::RefNull:

case TypeKind::Ref:

case TypeKind::Externref:

case TypeKind::Funcref:

case TypeKind::Exnref:

case TypeKind::I32: {

if (wasmParam.isStack()) {

uint64_t raw = *access.operator()<UCPURegister>(callFrame, wasmParam.offsetFromFP());

if (argType.isI32())

*access.operator()<uint64_t>(calleeFrame, dst) = static_cast<uint32_t>(raw) | JSValue::NumberTag;

else

*access.operator()<uint64_t>(calleeFrame, dst) = raw;

} else {

auto raw = *access.operator()<UCPURegister>(argumentRegisters, GPRInfo::toArgumentIndex(wasmParam.gpr()) * sizeof(UCPURegister));

if (argType.isI32())

*access.operator()<uint64_t>(calleeFrame, dst) = static_cast<uint32_t>(raw) | JSValue::NumberTag;

else

*access.operator()<uint64_t>(calleeFrame, dst) = raw;

}

break;

}

case TypeKind::I64: {

if (wasmParam.isStack()) {

auto result = JSBigInt::makeHeapBigIntOrBigInt32(globalObject, *access.operator()<int64_t>(callFrame, wasmParam.offsetFromFP()));

OPERATION_RETURN_IF_EXCEPTION(scope);

*access.operator()<uint64_t>(calleeFrame, dst) = JSValue::encode(result);

} else {

auto result = JSBigInt::makeHeapBigIntOrBigInt32(globalObject, *access.operator()<int64_t>(argumentRegisters, GPRInfo::toArgumentIndex(wasmParam.gpr()) * sizeof(UCPURegister)));

OPERATION_RETURN_IF_EXCEPTION(scope);

*access.operator()<uint64_t>(calleeFrame, dst) = JSValue::encode(result);

}

break;

}

case TypeKind::F32: {

float val;

if (wasmParam.isStack())

val = *access.operator()<float>(callFrame, wasmParam.offsetFromFP());

else

val = *access.operator()<float>(argumentRegisters, GPRInfo::numberOfArgumentRegisters * sizeof(UCPURegister) + FPRInfo::toArgumentIndex(wasmParam.fpr()) * bytesForWidth(Width::Width64));

double marshalled = purifyNaN(val);

uint64_t raw = std::bit_cast<uint64_t>(marshalled);

raw += JSValue::DoubleEncodeOffset;

*access.operator()<uint64_t>(calleeFrame, dst) = raw;

break;

}

case TypeKind::F64:

double val;

if (wasmParam.isStack())

val = *access.operator()<double>(callFrame, wasmParam.offsetFromFP());

else

val = *access.operator()<double>(argumentRegisters, GPRInfo::numberOfArgumentRegisters * sizeof(UCPURegister) + FPRInfo::toArgumentIndex(wasmParam.fpr()) * bytesForWidth(Width::Width64));

double marshalled = purifyNaN(val);

uint64_t raw = std::bit_cast<uint64_t>(marshalled);

raw += JSValue::DoubleEncodeOffset;

*access.operator()<uint64_t>(calleeFrame, dst) = raw;

break;

}

}

// store this argument

*access.operator()<uint64_t>(calleeFrame, CallFrameSlot::thisArgument * static_cast<int>(sizeof(Register))) = JSValue::encode(jsUndefined());

// materializeImportJSCell and store

*access.operator()<uint64_t>(calleeFrame, CallFrameSlot::callee * sizeof(Register)) = JSValue::encode(importableFunction->importFunction.get());

*access.operator()<uint32_t>(calleeFrame, CallFrameSlot::argumentCountIncludingThis * static_cast<int>(sizeof(Register)) + LowWordOffset) = argCount + 1; // including this = +1

OPERATION_RETURN(scope);

{

#if ASSERT_ENABLED

if (!frame->callee().isNativeCallee())

return;

CalleeGroup& calleeGroup = *instance->calleeGroup();

Wasm::Callee* callee = uncheckedDowncast<Wasm::Callee>(frame->callee().asNativeCallee());

TriState status;

{

Locker locker { calleeGroup.m_lock };

status = calleeGroup.calleeIsReferenced(locker, callee);

}

if (status == TriState::Indeterminate)

ASSERT(instance->vm().heap.isWasmCalleePendingDestruction(*callee));

else

ASSERT(status == TriState::True);

#else

UNUSED_PARAM(frame);

UNUSED_PARAM(instance);

#endif

{

auto access = []<typename V>(auto* arr, int i) -> V* {

return &reinterpret_cast<V*>(arr)[i / sizeof(V)];

};

void* registerSpace = sp;

assertCalleeIsReferenced(callFrame, instance);

WasmOperationPrologueCallFrameTracer tracer(instance->vm(), callFrame, OUR_RETURN_ADDRESS);

auto scope = DECLARE_THROW_SCOPE(instance->vm());

auto* importableFunction = *access.operator()<WasmOrJSImportableFunctionCallLinkInfo*>(callFrame, WasmToJSCallableFunctionSlot);

const RTT& signature = *importableFunction->rtt;

auto* globalObject = instance->realm();

auto wasmCC = wasmCallingConvention().callInformationFor(signature, CallRole::Callee);

JSValue returned = *(reinterpret_cast<JSValue*>(registerSpace));

if (!signature.returnCount())

OPERATION_RETURN(scope);

if (signature.returnCount() == 1) {

const auto& returnType = signature.returnType(0);

switch (returnType.kind()) {

case TypeKind::I32: {

if (!returned.isNumber() || !returned.isInt32()) {

// slow path

uint32_t result = JSValue::decode(std::bit_cast<EncodedJSValue>(returned)).toInt32(globalObject);

OPERATION_RETURN_IF_EXCEPTION(scope);

*access.operator()<uint64_t>(registerSpace, 0) = static_cast<uint64_t>(result);

} else {

uint64_t result = static_cast<uint64_t>(*access.operator()<uint32_t>(registerSpace, 0));

*access.operator()<uint64_t>(registerSpace, 0) = result;

}

break;

}

case TypeKind::I64: {

uint64_t result = JSValue::decode(std::bit_cast<EncodedJSValue>(returned)).toBigInt64(globalObject);

OPERATION_RETURN_IF_EXCEPTION(scope);

*access.operator()<uint64_t>(registerSpace, 0) = result;

break;

}

case TypeKind::F32: {

FPRReg dest = wasmCC.results[0].location.fpr();

auto offset = GPRInfo::numberOfArgumentRegisters * sizeof(UCPURegister) + FPRInfo::toArgumentIndex(dest) * bytesForWidth(Width::Width64);

if (returned.isNumber()) {

if (returned.isInt32()) {

float result = static_cast<float>(*access.operator()<int32_t>(registerSpace, 0));

*access.operator()<float>(registerSpace, offset) = result;

} else {

uint64_t intermediate = *access.operator()<uint64_t>(registerSpace, 0) + JSValue::NumberTag;

double d = std::bit_cast<double>(intermediate);

*access.operator()<uint64_t>(registerSpace, offset) = static_cast<uint64_t>(std::bit_cast<uint32_t>(static_cast<float>(d)));

}

} else {

float result = static_cast<float>(JSValue::decode(std::bit_cast<EncodedJSValue>(returned)).toNumber(globalObject));

OPERATION_RETURN_IF_EXCEPTION(scope);

*access.operator()<uint64_t>(registerSpace, offset) = static_cast<uint64_t>(std::bit_cast<uint32_t>(result));

}

break;

}

case TypeKind::F64: {

FPRReg dest = wasmCC.results[0].location.fpr();

auto offset = GPRInfo::numberOfArgumentRegisters * sizeof(UCPURegister) + FPRInfo::toArgumentIndex(dest) * bytesForWidth(Width::Width64);

if (returned.isNumber()) {

if (returned.isInt32()) {

double result = static_cast<double>(*access.operator()<int32_t>(registerSpace, 0));

*access.operator()<double>(registerSpace, offset) = result;

} else {

uint64_t intermediate = *access.operator()<uint64_t>(registerSpace, 0) + JSValue::NumberTag;

double d = std::bit_cast<double>(intermediate);

*access.operator()<double>(registerSpace, offset) = d;

}

} else {

double result = static_cast<double>(JSValue::decode(std::bit_cast<EncodedJSValue>(returned)).toNumber(globalObject));

OPERATION_RETURN_IF_EXCEPTION(scope);

*access.operator()<double>(registerSpace, offset) = result;

}

break;

}

default: {

if (Wasm::isRefType(returnType)) {

JSValue value = JSValue::decode(std::bit_cast<EncodedJSValue>(returned));

if (value.isNull() && !returnType.isNullable()) [[unlikely]] {

throwTypeError(globalObject, scope, "Host function incorrectly returned null for a nonnullable reference type"_s);

OPERATION_RETURN(scope);

}

if (Wasm::isExternref(returnType)) {

// Do nothing.

} else if (Wasm::isFuncref(returnType)) {

// operationConvertToFuncref

WebAssemblyFunction* wasmFunction = nullptr;

WebAssemblyWrapperFunction* wasmWrapperFunction = nullptr;

if (!isWebAssemblyHostFunction(value, wasmFunction, wasmWrapperFunction) && !value.isNull()) [[unlikely]] {

throwTypeError(globalObject, scope, "Funcref value is not a function"_s);

OPERATION_RETURN(scope);

}

if (isRefWithTypeIndex(returnType) && !value.isNull()) {

Wasm::TypeIndex paramIndex = returnType.index();

Wasm::TypeIndex argIndex = (wasmFunction ? wasmFunction->rtt() : wasmWrapperFunction->rtt())->asTypeIndex();

if (paramIndex != argIndex) {

throwVMTypeError(globalObject, scope, "Argument function did not match the reference type"_s);

OPERATION_RETURN(scope);

}

}

// Validation only: value not modified so write back not needed.

} else {

// operationConvertToAnyref

value = Wasm::internalizeExternref(value);

if (!Wasm::TypeInformation::isReferenceValueAssignable(value, returnType.isNullable(), returnType.index())) [[unlikely]] {

throwTypeError(globalObject, scope, "Argument value did not match the reference type"_s);

OPERATION_RETURN(scope);

}

*access.operator()<uint64_t>(registerSpace, 0) = std::bit_cast<uint64_t>(value);

}

} else

RELEASE_ASSERT_NOT_REACHED();

}

}

OPERATION_RETURN(scope);

}

unsigned iterationCount = 0;

MarkedArgumentBuffer buffer;

buffer.ensureCapacity(signature.returnCount());

forEachInIterable(globalObject, returned, [&](VM&, JSGlobalObject*, JSValue value) -> void {

if (buffer.size() < signature.returnCount()) {

buffer.append(value);

if (buffer.hasOverflowed()) [[unlikely]]

throwOutOfMemoryError(globalObject, scope);

}

++iterationCount;

});

OPERATION_RETURN_IF_EXCEPTION(scope);

if (buffer.hasOverflowed()) {

throwOutOfMemoryError(globalObject, scope, "JS results to Wasm are too large"_s);

OPERATION_RETURN(scope);

}

if (iterationCount != signature.returnCount()) {

throwVMTypeError(globalObject, scope, "Incorrect number of values returned to Wasm from JS"_s);

OPERATION_RETURN(scope);

}

for (unsigned index = 0; index < buffer.size(); ++index) {

JSValue value = buffer.at(index);

uint64_t unboxedValue = 0;

const auto& returnType = signature.returnType(index);

switch (returnType.kind()) {

case TypeKind::I32:

unboxedValue = static_cast<uint32_t>(value.toInt32(globalObject));

OPERATION_RETURN_IF_EXCEPTION(scope);

break;

case TypeKind::I64:

unboxedValue = value.toBigInt64(globalObject);

OPERATION_RETURN_IF_EXCEPTION(scope);

break;

case TypeKind::F32:

unboxedValue = std::bit_cast<uint32_t>(value.toFloat(globalObject));

OPERATION_RETURN_IF_EXCEPTION(scope);

break;

case TypeKind::F64:

unboxedValue = std::bit_cast<uint64_t>(value.toNumber(globalObject));

OPERATION_RETURN_IF_EXCEPTION(scope);

break;

default: {

if (Wasm::isRefType(returnType)) {

if (value.isNull() && !returnType.isNullable()) [[unlikely]] {

throwTypeError(globalObject, scope, "Host function incorrectly returned null for a nonnullable reference type"_s);

OPERATION_RETURN(scope);

}

if (isExternref(returnType)) {

// Do nothing.

} else if (isFuncref(returnType)) {

WebAssemblyFunction* wasmFunction = nullptr;

WebAssemblyWrapperFunction* wasmWrapperFunction = nullptr;

if (!isWebAssemblyHostFunction(value, wasmFunction, wasmWrapperFunction) && !value.isNull()) [[unlikely]] {

throwTypeError(globalObject, scope, "Argument value did not match the reference type"_s);

OPERATION_RETURN(scope);

}

if (Wasm::isRefWithTypeIndex(returnType) && !value.isNull()) {

Wasm::TypeIndex paramIndex = returnType.index();

Wasm::TypeIndex argIndex = (wasmFunction ? wasmFunction->rtt() : wasmWrapperFunction->rtt())->asTypeIndex();

if (paramIndex != argIndex) {

throwTypeError(globalObject, scope, "Argument value did not match the reference type"_s);

OPERATION_RETURN(scope);

}

}

} else {

value = Wasm::internalizeExternref(value);

if (!Wasm::TypeInformation::isReferenceValueAssignable(value, returnType.isNullable(), returnType.index())) [[unlikely]] {

throwTypeError(globalObject, scope, "Argument value did not match the reference type"_s);

OPERATION_RETURN(scope);

}

}

} else

RELEASE_ASSERT_NOT_REACHED();

unboxedValue = std::bit_cast<uint64_t>(value);

}

}

auto rep = wasmCC.results[index];

if (rep.location.isGPR()) {

auto offset = GPRInfo::toArgumentIndex(rep.location.gpr()) * sizeof(UCPURegister);

*access.operator()<uint64_t>(registerSpace, offset) = unboxedValue;

} else if (rep.location.isFPR()) {

auto offset = GPRInfo::numberOfArgumentRegisters * sizeof(UCPURegister) + FPRInfo::toArgumentIndex(rep.location.fpr()) * bytesForWidth(Width::Width64);

*access.operator()<uint64_t>(registerSpace, offset) = unboxedValue;

} else

*access.operator()<uint64_t>(callFrame, rep.location.offsetFromFP()) = unboxedValue;

}

OPERATION_RETURN(scope);

{

if (!OMGPlan::ensureGlobalOMGAllowlist().containsWasmFunction(functionIndex)) {

dataLogLnIf(Options::verboseOSR(), "\tNot optimizing ", functionIndex, " as it's not in the allow list.");

tierUp.deferIndefinitely();

return false;

}

if (!replacement && !tierUp.checkIfOptimizationThresholdReached()) {

dataLogLnIf(Options::verboseOSR(), "\tdelayOMGCompile counter = ", tierUp, " for ", functionIndex);

dataLogLnIf(Options::verboseOSR(), "\tChoosing not to OMG-optimize ", functionIndex, " yet.");

return false;

}

return true;

{

MemoryMode memoryMode = instance->memory0Mode();

bool compile = false;

{

Locker locker { tierUp.getLock() };

switch (tierUp.compilationStatusForOMG(memoryMode)) {

case TierUpCount::CompilationStatus::StartCompilation:

tierUp.setOptimizationThresholdBasedOnCompilationResult(functionIndex, CompilationResult::CompilationDeferred);

return;

case TierUpCount::CompilationStatus::NotCompiled:

compile = true;

tierUp.setCompilationStatusForOMG(memoryMode, TierUpCount::CompilationStatus::StartCompilation);

break;

case TierUpCount::CompilationStatus::Compiled:

tierUp.optimizeSoon(functionIndex);

return;

default:

break;

}

}

if (compile) {

dataLogLnIf(Options::verboseOSR(), "\ttriggerOMGReplacement for ", functionIndex);

// We need to compile the code.

Ref<Plan> plan = adoptRef(*new OMGPlan(instance->vm(), Ref<Wasm::Module>(instance->module()), functionIndex, calleeGroup.mode(), Plan::dontFinalize()));

ensureWorklist().enqueue(plan.copyRef());

if (!Options::useConcurrentJIT()) [[unlikely]]

plan->waitForCompletion();

else

tierUp.setOptimizationThresholdBasedOnCompilationResult(functionIndex, CompilationResult::CompilationDeferred);

}

{

constexpr bool verbose = false || WasmOperationsInternal::verbose;

dataLogLnIf(verbose, "loadValuesIntoBuffer: values.size() = ", values.size());

for (unsigned index = 0; index < values.size(); ++index) {

const OSREntryValue& value = values[index];

dataLogLnIf(Options::verboseOSR() || verbose, "OMG OSR entry values[", index, "] ", value.type(), " ", value);

if (value.isGPR()) {

switch (value.type().kind()) {

case B3::Float:

case B3::Double:

RELEASE_ASSERT_NOT_REACHED();

default:

*std::bit_cast<uint64_t*>(buffer + index) = context.gpr(value.gpr());

}

dataLogLnIf(verbose, "GPR for value ", index, " ", value.gpr(), " = ", context.gpr(value.gpr()));

} else if (value.isFPR()) {

switch (value.type().kind()) {

case B3::Float:

case B3::Double:

dataLogLnIf(verbose, "FPR for value ", index, " ", value.fpr(), " = ", context.fpr(value.fpr()));

*std::bit_cast<double*>(buffer + index) = context.fpr(value.fpr());

break;

case B3::V128:

#if CPU(X86_64) || CPU(ARM64)

dataLogLnIf(verbose, "Vector FPR for value ", index, " ", value.fpr(), " = ", context.vector(value.fpr()));

*std::bit_cast<v128_t*>(buffer + index) = context.vector(value.fpr());

#else

UNREACHABLE_FOR_PLATFORM();

#endif

break;

default:

RELEASE_ASSERT_NOT_REACHED();

}

} else if (value.isConstant()) {

switch (value.type().kind()) {

case B3::Float:

*std::bit_cast<float*>(buffer + index) = value.floatValue();

break;

case B3::Double:

*std::bit_cast<double*>(buffer + index) = value.doubleValue();

break;

case B3::V128:

RELEASE_ASSERT_NOT_REACHED();

break;

default:

*std::bit_cast<uint64_t*>(buffer + index) = value.value();

}

} else if (value.isStack()) {

auto* baseLoad = std::bit_cast<uint8_t*>(context.fp()) + value.offsetFromFP();

auto* baseStore = std::bit_cast<uint8_t*>(buffer + index);

if (value.type().isFloat() || value.type().isVector()) {

dataLogLnIf(verbose, "Stack float or vector for value ", index, " fp offset ", value.offsetFromFP(), " = ",

*std::bit_cast<v128_t*>(baseLoad),

" or double ", *std::bit_cast<double*>(baseLoad));

} else

dataLogLnIf(verbose, "Stack for value ", index, " fp[", value.offsetFromFP(), "] = ", RawHex(*std::bit_cast<uint64_t*>(baseLoad)), " ", static_cast<int32_t>(*std::bit_cast<uint64_t*>(baseLoad)), " ", *std::bit_cast<int64_t*>(baseLoad));

switch (value.type().kind()) {

case B3::Float:

*std::bit_cast<float*>(baseStore) = *std::bit_cast<float*>(baseLoad);

break;

case B3::Double:

*std::bit_cast<double*>(baseStore) = *std::bit_cast<double*>(baseLoad);

break;

case B3::V128:

*std::bit_cast<v128_t*>(baseStore) = *std::bit_cast<v128_t*>(baseLoad);

break;

default:

*std::bit_cast<uint64_t*>(baseStore) = *std::bit_cast<uint64_t*>(baseLoad);

break;

}

} else

RELEASE_ASSERT_NOT_REACHED();

}

{

auto returnWithoutOSREntry = [&] {

context.gpr(GPRInfo::nonPreservedNonArgumentGPR0) = 0;

};

RELEASE_ASSERT(osrEntryCallee.osrEntryScratchBufferSize() == osrEntryData.values().size());

auto* buffer = instance->vm().wasmContext.scratchBufferForSize(osrEntryCallee.osrEntryScratchBufferSize());

if (!buffer)

return returnWithoutOSREntry();

dataLogLnIf(Options::verboseOSR(), callee, ": OMG OSR entry: functionCodeIndex=", osrEntryData.functionIndex(), " got entry callee ", RawPointer(&osrEntryCallee));

// 1. Place required values in scratch buffer.

loadValuesIntoBuffer(context, osrEntryData.values(), buffer);

// 2. Restore callee saves.

auto dontRestoreRegisters = RegisterSet::stackRegisters();

for (const RegisterAtOffset& entry : *callee.calleeSaveRegisters()) {

if (dontRestoreRegisters.contains(entry.reg(), IgnoreVectors))

continue;

if (entry.reg().isGPR())

context.gpr(entry.reg().gpr()) = *std::bit_cast<UCPURegister*>(std::bit_cast<uint8_t*>(context.fp()) + entry.offset());

else

context.fpr(entry.reg().fpr()) = *std::bit_cast<double*>(std::bit_cast<uint8_t*>(context.fp()) + entry.offset());

}

// 3. Function epilogue, like a tail-call.

UCPURegister* framePointer = std::bit_cast<UCPURegister*>(context.fp());

#if CPU(X86_64)

// move(framePointerRegister, stackPointerRegister);

// pop(framePointerRegister);

context.fp() = std::bit_cast<UCPURegister*>(*framePointer);

context.sp() = framePointer + 1;

static_assert(prologueStackPointerDelta() == sizeof(void*) * 1);

#elif CPU(ARM64)

// move(framePointerRegister, stackPointerRegister);

// popPair(framePointerRegister, linkRegister);

context.fp() = std::bit_cast<UCPURegister*>(*framePointer);

context.gpr(ARM64Registers::lr) = std::bit_cast<UCPURegister>(*(framePointer + 1));

context.sp() = framePointer + 2;

static_assert(prologueStackPointerDelta() == sizeof(void*) * 2);

#if CPU(ARM64E)

// LR needs to be untagged since OSR entry function prologue will tag it with SP. This is similar to tail-call.

context.gpr(ARM64Registers::lr) = std::bit_cast<UCPURegister>(untagCodePtrWithStackPointerForJITCall(context.gpr<void*>(ARM64Registers::lr), context.sp()));

#endif

#elif CPU(RISCV64)

// move(framePointerRegister, stackPointerRegister);

// popPair(framePointerRegister, linkRegister);

context.fp() = std::bit_cast<UCPURegister*>(*framePointer);

context.gpr(RISCV64Registers::ra) = std::bit_cast<UCPURegister>(*(framePointer + 1));

context.sp() = framePointer + 2;

static_assert(prologueStackPointerDelta() == sizeof(void*) * 2);

#else

#error Unsupported architecture.

#endif

// 4. Configure argument registers to jump to OSR entry from the caller of this runtime function.

context.gpr(GPRInfo::argumentGPR0) = std::bit_cast<UCPURegister>(buffer); // Modify this only when we definitely tier up.

context.gpr(GPRInfo::nonPreservedNonArgumentGPR0) = std::bit_cast<UCPURegister>(osrEntryCallee.entrypoint().taggedPtr<>());

{

auto& info = instance->module().moduleInformation();

if (info.functions[functionIndex].data.size() > Options::maximumOMGCandidateCost())

return false;

if (!Options::wasmFunctionIndexRangeToCompile().isInRange(functionIndex))

return false;

return true;

{

BBQCallee& callee = uncheckedDowncast<BBQCallee>(uncheckedDowncast<Wasm::Callee>(*callFrame->callee().asNativeCallee()));

ASSERT(callee.compilationMode() == CompilationMode::BBQMode);

Wasm::CalleeGroup& calleeGroup = *instance->calleeGroup();

ASSERT(instance->memory0Mode() == calleeGroup.mode());

FunctionSpaceIndex functionIndexInSpace = callee.index();

FunctionCodeIndex functionIndex = calleeGroup.toCodeIndex(functionIndexInSpace);

TierUpCount& tierUp = callee.tierUpCounter();

if (!shouldOMGJIT(instance, functionIndex)) {

tierUp.deferIndefinitely();

return;

}

if (shouldTriggerOMGCompile(tierUp, nullptr, functionIndex))

triggerOMGReplacementCompile(tierUp, instance, calleeGroup, functionIndex);

{

OSREntryData& osrEntryData = *context.arg<OSREntryData*>();

auto functionIndex = osrEntryData.functionIndex();

uint32_t loopIndex = osrEntryData.loopIndex();

JSWebAssemblyInstance* instance = context.gpr<JSWebAssemblyInstance*>(GPRInfo::wasmContextInstancePointer);

BBQCallee& callee = uncheckedDowncast<BBQCallee>(uncheckedDowncast<Wasm::Callee>(*context.gpr<CallFrame*>(MacroAssembler::framePointerRegister)->callee().asNativeCallee()));

ASSERT(callee.compilationMode() == Wasm::CompilationMode::BBQMode);

ASSERT(callee.refCount());

Wasm::CalleeGroup& calleeGroup = *instance->calleeGroup();

auto returnWithoutOSREntry = [&] {

context.gpr(GPRInfo::nonPreservedNonArgumentGPR0) = 0;

};

auto doStackCheck = [instance](OMGOSREntryCallee* callee) -> bool {

uintptr_t stackPointer = reinterpret_cast<uintptr_t>(currentStackPointer());

ASSERT(callee->stackCheckSize());

if (callee->stackCheckSize() == stackCheckNotNeeded)

return true;

uintptr_t stackExtent = stackPointer - callee->stackCheckSize();

uintptr_t stackLimit = reinterpret_cast<uintptr_t>(instance->softStackLimit());

if (stackExtent >= stackPointer || stackExtent <= stackLimit) [[unlikely]] {

dataLogLnIf(Options::verboseOSR(), "\tSkipping OMG loop tier up due to stack check; ", RawHex(stackPointer), " -> ", RawHex(stackExtent), " is past soft limit ", RawHex(stackLimit));

return false;

}

return true;

};

MemoryMode memoryMode = instance->memory0Mode();

ASSERT(memoryMode == calleeGroup.mode());

TierUpCount& tierUp = callee.tierUpCounter();

if (!shouldOMGJIT(instance, functionIndex)) {

tierUp.deferIndefinitely();

return returnWithoutOSREntry();

}

RefPtr<OMGCallee> replacement = calleeGroup.tryGetOMGCalleeConcurrently(functionIndex);

dataLogLnIf(Options::verboseOSR(), callee, ": Consider OSREntryPlan for functionCodeIndex=", osrEntryData.functionIndex(), " loopIndex#", loopIndex, " with executeCounter = ", tierUp, " ", RawPointer(replacement.get()));

if (!Options::useWasmOSR()) {

if (shouldTriggerOMGCompile(tierUp, replacement.get(), functionIndex))

triggerOMGReplacementCompile(tierUp, instance, calleeGroup, functionIndex);

// We already have an OMG replacement.

if (replacement) {

// No OSR entry points. Just defer indefinitely.

if (tierUp.osrEntryTriggers().isEmpty()) {

tierUp.dontOptimizeAnytimeSoon(functionIndex);

return;

}

// FIXME: Is this ever taken?

// Found one OSR entry point. Since we do not have a way to jettison Wasm::Callee right now, this means that tierUp function is now meaningless.

// Not call it as much as possible.

if (callee.osrEntryCallee()) {

tierUp.dontOptimizeAnytimeSoon(functionIndex);

return;

}

}

return returnWithoutOSREntry();

}

TierUpCount::CompilationStatus compilationStatus = TierUpCount::CompilationStatus::NotCompiled;

{

Locker locker { tierUp.getLock() };

compilationStatus = tierUp.compilationStatusForOMGForOSREntry(memoryMode);

}

bool triggeredSlowPathToStartCompilation = false;

switch (tierUp.osrEntryTriggers()[loopIndex]) {

case TierUpCount::TriggerReason::DontTrigger:

// The trigger isn't set, we entered because the counter reached its

// threshold.

break;

case TierUpCount::TriggerReason::CompilationDone:

// The trigger was set because compilation completed. Don't unset it

// so that further BBQ executions OSR enter as well.

break;

case TierUpCount::TriggerReason::StartCompilation: {

// We were asked to enter as soon as possible and start compiling an

// entry for the current loopIndex. Unset this trigger so we

// don't continually enter.

Locker locker { tierUp.getLock() };

TierUpCount::TriggerReason reason = tierUp.osrEntryTriggers()[loopIndex];

if (reason == TierUpCount::TriggerReason::StartCompilation) {

tierUp.osrEntryTriggers()[loopIndex] = TierUpCount::TriggerReason::DontTrigger;

triggeredSlowPathToStartCompilation = true;

}

break;

}

}

if (compilationStatus == TierUpCount::CompilationStatus::StartCompilation) {

dataLogLnIf(Options::verboseOSR(), "\tdelayOMGCompile still compiling for ", functionIndex);

tierUp.setOptimizationThresholdBasedOnCompilationResult(functionIndex, CompilationResult::CompilationDeferred);

return returnWithoutOSREntry();

}

if (OMGOSREntryCallee* osrEntryCallee = callee.osrEntryCallee()) {

if (osrEntryCallee->loopIndex() == loopIndex) {

if (!doStackCheck(osrEntryCallee))

return returnWithoutOSREntry();

return doOSREntry(instance, context, callee, *osrEntryCallee, osrEntryData);

}

}