Skip to content

Commit 861ea3b

Browse files
ConstellationConstellation
committed
[JSC] Add linkThunk mechanism
https://bugs.webkit.org/show_bug.cgi?id=265854 rdar://119171337 Reviewed by Mark Lam. This patch adds linkThunk mechanism to Jump and Call. Previously, we were using LinkBuffer::link when jumping to a thunk or calling a thunk. But the implementation of this is really costly and it was not so good. (1) it uses performJITMemcpy to rewrite code, which does JIT memory permission switch every time. And (2) it does not leverage branch compaction. So even though it is thunk address, we never do branch compaction and typically we have several nops around it. Instead, we have linkThunk mechanism which can just link Jump and Call to a thunk. On ARM64, this is fully integrated into our branch compaction mechanism so that we can repatch it through LinkBuffer's branch compaction. On the other architectures, it is the same to the current behavior. * Source/JavaScriptCore/assembler/ARM64Assembler.h: (JSC::ARM64Assembler::LinkRecord::LinkRecord): (JSC::ARM64Assembler::LinkRecord::isThunk const): * Source/JavaScriptCore/assembler/ARMv7Assembler.h: (JSC::ARMv7Assembler::LinkRecord::setFrom): (JSC::ARMv7Assembler::LinkRecord::to const): * Source/JavaScriptCore/assembler/AbstractMacroAssembler.h: (JSC::AbstractMacroAssembler::Jump::linkThunk const): (JSC::AbstractMacroAssembler::JumpList::linkTo const): (JSC::AbstractMacroAssembler::JumpList::linkThunk const): * Source/JavaScriptCore/assembler/LinkBuffer.cpp: (JSC::LinkBuffer::copyCompactAndLinkCode): * Source/JavaScriptCore/assembler/MacroAssemblerARM64.h: (JSC::MacroAssemblerARM64::callOperation): (JSC::MacroAssemblerARM64::callThunk): (JSC::MacroAssemblerARM64::jumpThunk): * Source/JavaScriptCore/assembler/MacroAssemblerARM64E.h: (JSC::MacroAssemblerARM64E::callOperation): * Source/JavaScriptCore/assembler/MacroAssemblerARMv7.h: (JSC::MacroAssemblerARMv7::callOperation): (JSC::MacroAssemblerARMv7::callThunk): (JSC::MacroAssemblerARMv7::jumpThunk): * Source/JavaScriptCore/assembler/MacroAssemblerRISCV64.h: (JSC::MacroAssemblerRISCV64::callOperation): (JSC::MacroAssemblerRISCV64::callThunk): (JSC::MacroAssemblerRISCV64::jumpThunk): * Source/JavaScriptCore/assembler/MacroAssemblerX86_64.h: (JSC::MacroAssemblerX86_64::callOperation): (JSC::MacroAssemblerX86_64::callThunk): (JSC::MacroAssemblerX86_64::jumpThunk): * Source/JavaScriptCore/bytecode/AccessCaseSnippetParams.cpp: * Source/JavaScriptCore/bytecode/InlineCacheCompiler.cpp: (JSC::InlineCacheCompiler::emitExplicitExceptionHandler): (JSC::getByIdSlowPathCodeGenerator): (JSC::getByIdWithThisSlowPathCodeGenerator): (JSC::getByValSlowPathCodeGenerator): (JSC::getPrivateNameSlowPathCodeGenerator): (JSC::getByValWithThisSlowPathCodeGenerator): (JSC::putByIdSlowPathCodeGenerator): (JSC::putByValSlowPathCodeGenerator): (JSC::instanceOfSlowPathCodeGenerator): (JSC::delByIdSlowPathCodeGenerator): (JSC::delByValSlowPathCodeGenerator): (JSC::InlineCacheCompiler::generateImpl): (JSC::InlineCacheCompiler::regenerate): * Source/JavaScriptCore/bytecode/Repatch.cpp: (JSC::linkPolymorphicCall): * Source/JavaScriptCore/ftl/FTLCompile.cpp: (JSC::FTL::compile): * Source/JavaScriptCore/ftl/FTLLink.cpp: (JSC::FTL::link): * Source/JavaScriptCore/jit/JIT.cpp: (JSC::JIT::compileAndLinkWithoutFinalizing): (JSC::JIT::link): * Source/JavaScriptCore/jit/JITMathIC.h: (JSC::JITMathIC::generateOutOfLine): * Source/JavaScriptCore/jit/JITOpcodes.cpp: (JSC::JIT::op_check_traps_handlerGenerator): * Source/JavaScriptCore/jit/JITPropertyAccess.cpp: (JSC::JIT::slow_op_resolve_scopeGenerator): (JSC::JIT::slow_op_put_to_scopeGenerator): * Source/JavaScriptCore/jit/SlowPathCall.cpp: (JSC::JITSlowPathCall::generateThunk): * Source/JavaScriptCore/jit/SpecializedThunkJIT.h: (JSC::SpecializedThunkJIT::finalize): * Source/JavaScriptCore/jit/ThunkGenerators.cpp: (JSC::popThunkStackPreservesAndHandleExceptionGenerator): (JSC::checkExceptionGenerator): (JSC::nativeForGenerator): * Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp: (JSC::Wasm::B3IRGenerator::B3IRGenerator): (JSC::Wasm::B3IRGenerator::emitExceptionCheck): * Source/JavaScriptCore/wasm/WasmBBQJIT.cpp: (JSC::Wasm::BBQJIT::emitWriteBarrier): (JSC::Wasm::BBQJIT::emitThrowException): (JSC::Wasm::BBQJIT::addTopLevel): (JSC::Wasm::BBQJIT::addLoopOSREntrypoint): * Source/JavaScriptCore/wasm/WasmIRGeneratorHelpers.h: (JSC::Wasm::emitRethrowImpl): (JSC::Wasm::emitThrowImpl): * Source/JavaScriptCore/wasm/WasmThunks.cpp: (JSC::Wasm::catchInWasmThunkGenerator): * Source/JavaScriptCore/wasm/js/JSToWasm.cpp: (JSC::Wasm::marshallJSResult): * Source/JavaScriptCore/wasm/js/WasmToJS.cpp: (JSC::Wasm::wasmToJS): (JSC::Wasm::emitThrowWasmToJSException): Canonical link: https://commits.webkit.org/271801@main
1 parent e9e0f31 commit 861ea3b

41 files changed

Lines changed: 387 additions & 496 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Source/JavaScriptCore/assembler/ARM64Assembler.h‎

Lines changed: 88 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -334,9 +334,33 @@ class ARM64Assembler {
334334
LinkJumpTestBitDirect = JUMP_ENUM_WITH_SIZE(7, 1 * sizeof(uint32_t)),
335335
};
336336

337+
enum BranchType : uint8_t {
338+
BranchType_JMP,
339+
BranchType_CALL,
340+
BranchType_RET
341+
};
342+
343+
enum class ThunkOrNot : uint8_t {
344+
NotThunk = false,
345+
Thunk = true,
346+
};
347+
337348
class LinkRecord {
338349
public:
339-
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition)
350+
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, ThunkOrNot isThunk)
351+
{
352+
data.realTypes.m_from = from;
353+
#if CPU(ARM64E)
354+
data.realTypes.m_to = tagInt(to, static_cast<PtrTag>(from ^ bitwise_cast<intptr_t>(assembler)));
355+
#else
356+
UNUSED_PARAM(assembler);
357+
data.realTypes.m_to = to;
358+
#endif
359+
data.realTypes.m_isThunk = isThunk;
360+
data.realTypes.m_branchType = BranchType_CALL;
361+
}
362+
363+
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, ThunkOrNot isThunk)
340364
{
341365
data.realTypes.m_from = from;
342366
#if CPU(ARM64E)
@@ -346,10 +370,10 @@ class ARM64Assembler {
346370
data.realTypes.m_to = to;
347371
#endif
348372
data.realTypes.m_type = type;
349-
data.realTypes.m_linkType = LinkInvalid;
350373
data.realTypes.m_condition = condition;
374+
data.realTypes.m_isThunk = isThunk;
351375
}
352-
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister)
376+
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister, ThunkOrNot isThunk)
353377
{
354378
data.realTypes.m_from = from;
355379
#if CPU(ARM64E)
@@ -359,12 +383,12 @@ class ARM64Assembler {
359383
data.realTypes.m_to = to;
360384
#endif
361385
data.realTypes.m_type = type;
362-
data.realTypes.m_linkType = LinkInvalid;
363386
data.realTypes.m_condition = condition;
364387
data.realTypes.m_is64Bit = is64Bit;
388+
data.realTypes.m_isThunk = isThunk;
365389
data.realTypes.m_compareRegister = compareRegister;
366390
}
367-
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister)
391+
LinkRecord(const ARM64Assembler* assembler, intptr_t from, intptr_t to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister, ThunkOrNot isThunk)
368392
{
369393
data.realTypes.m_from = from;
370394
#if CPU(ARM64E)
@@ -374,9 +398,9 @@ class ARM64Assembler {
374398
data.realTypes.m_to = to;
375399
#endif
376400
data.realTypes.m_type = type;
377-
data.realTypes.m_linkType = LinkInvalid;
378401
data.realTypes.m_condition = condition;
379402
data.realTypes.m_bitNumber = bitNumber;
403+
data.realTypes.m_isThunk = isThunk;
380404
data.realTypes.m_compareRegister = compareRegister;
381405
}
382406
// We are defining a copy constructor and assignment operator
@@ -412,24 +436,28 @@ class ARM64Assembler {
412436
}
413437
JumpType type() const { return data.realTypes.m_type; }
414438
JumpLinkType linkType() const { return data.realTypes.m_linkType; }
439+
BranchType branchType() const { return data.realTypes.m_branchType; }
415440
void setLinkType(JumpLinkType linkType) { ASSERT(data.realTypes.m_linkType == LinkInvalid); data.realTypes.m_linkType = linkType; }
416441
Condition condition() const { return data.realTypes.m_condition; }
417442
bool is64Bit() const { return data.realTypes.m_is64Bit; }
443+
bool isThunk() const { return data.realTypes.m_isThunk == ThunkOrNot::Thunk; }
418444
unsigned bitNumber() const { return data.realTypes.m_bitNumber; }
419445
RegisterID compareRegister() const { return data.realTypes.m_compareRegister; }
420446

421447
private:
422448
union {
423449
struct RealTypes {
424-
int64_t m_from;
425-
int64_t m_to;
426-
RegisterID m_compareRegister;
427-
JumpType m_type : 8;
428-
JumpLinkType m_linkType : 8;
429-
Condition m_condition : 4;
430-
unsigned m_bitNumber : 6;
431-
bool m_is64Bit : 1;
432-
} realTypes;
450+
int64_t m_from { 0 };
451+
int64_t m_to { 0 };
452+
RegisterID m_compareRegister { ARM64Registers::InvalidGPRReg };
453+
JumpType m_type : 8 { JumpNoCondition };
454+
JumpLinkType m_linkType : 8 { LinkInvalid };
455+
Condition m_condition : 4 { ConditionInvalid };
456+
unsigned m_bitNumber : 6 { 0 };
457+
bool m_is64Bit : 1 { false };
458+
ThunkOrNot m_isThunk : 1 { ThunkOrNot::NotThunk };
459+
BranchType m_branchType : 2 { BranchType_JMP };
460+
} realTypes { };
433461
struct CopyTypes {
434462
uint64_t content[3];
435463
} copyTypes;
@@ -529,12 +557,6 @@ class ARM64Assembler {
529557
MemOpSize_64 = 3,
530558
};
531559

532-
enum BranchType {
533-
BranchType_JMP,
534-
BranchType_CALL,
535-
BranchType_RET
536-
};
537-
538560
enum AddOp {
539561
AddOp_ADD,
540562
AddOp_SUB
@@ -3412,21 +3434,49 @@ class ARM64Assembler {
34123434
{
34133435
ASSERT(to.isSet());
34143436
ASSERT(from.isSet());
3415-
m_jumpsToLink.append(LinkRecord(this, from.offset(), to.offset(), type, condition));
3437+
m_jumpsToLink.append(LinkRecord(this, from.offset(), to.offset(), type, condition, ThunkOrNot::NotThunk));
34163438
}
34173439

34183440
void linkJump(AssemblerLabel from, AssemblerLabel to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister)
34193441
{
34203442
ASSERT(to.isSet());
34213443
ASSERT(from.isSet());
3422-
m_jumpsToLink.append(LinkRecord(this, from.offset(), to.offset(), type, condition, is64Bit, compareRegister));
3444+
m_jumpsToLink.append(LinkRecord(this, from.offset(), to.offset(), type, condition, is64Bit, compareRegister, ThunkOrNot::NotThunk));
34233445
}
34243446

34253447
void linkJump(AssemblerLabel from, AssemblerLabel to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister)
34263448
{
34273449
ASSERT(to.isSet());
34283450
ASSERT(from.isSet());
3429-
m_jumpsToLink.append(LinkRecord(this, from.offset(), to.offset(), type, condition, bitNumber, compareRegister));
3451+
m_jumpsToLink.append(LinkRecord(this, from.offset(), to.offset(), type, condition, bitNumber, compareRegister, ThunkOrNot::NotThunk));
3452+
}
3453+
3454+
void linkJumpThunk(AssemblerLabel from, void* to, JumpType type, Condition condition)
3455+
{
3456+
ASSERT(to);
3457+
ASSERT(from.isSet());
3458+
m_jumpsToLink.append(LinkRecord(this, from.offset(), bitwise_cast<intptr_t>(to), type, condition, ThunkOrNot::Thunk));
3459+
}
3460+
3461+
void linkJumpThunk(AssemblerLabel from, void* to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister)
3462+
{
3463+
ASSERT(to);
3464+
ASSERT(from.isSet());
3465+
m_jumpsToLink.append(LinkRecord(this, from.offset(), bitwise_cast<intptr_t>(to), type, condition, is64Bit, compareRegister, ThunkOrNot::Thunk));
3466+
}
3467+
3468+
void linkJumpThunk(AssemblerLabel from, void* to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister)
3469+
{
3470+
ASSERT(to);
3471+
ASSERT(from.isSet());
3472+
m_jumpsToLink.append(LinkRecord(this, from.offset(), bitwise_cast<intptr_t>(to), type, condition, bitNumber, compareRegister, ThunkOrNot::Thunk));
3473+
}
3474+
3475+
void linkNearCallThunk(AssemblerLabel from, void* to)
3476+
{
3477+
ASSERT(to);
3478+
ASSERT(from.isSet());
3479+
m_jumpsToLink.append(LinkRecord(this, from.offset() - sizeof(int), bitwise_cast<intptr_t>(to), ThunkOrNot::Thunk));
34303480
}
34313481

34323482
static void linkJump(void* code, AssemblerLabel from, void* to)
@@ -3636,6 +3686,7 @@ class ARM64Assembler {
36363686
static bool canCompact(JumpType jumpType)
36373687
{
36383688
// Fixed jumps cannot be compacted
3689+
// Keep in mind that nearCall and tailCall are encoded as JumpNoCondition.
36393690
return (jumpType == JumpNoCondition) || (jumpType == JumpCondition) || (jumpType == JumpCompareAndBranch) || (jumpType == JumpTestBit);
36403691
}
36413692

@@ -3711,9 +3762,20 @@ class ARM64Assembler {
37113762
{
37123763
const int* fromInstruction = reinterpret_cast<const int*>(fromInstruction8);
37133764
switch (record.linkType()) {
3714-
case LinkJumpNoCondition:
3715-
linkJumpOrCall<BranchType_JMP, copy>(reinterpret_cast<int*>(from), fromInstruction, to);
3765+
case LinkJumpNoCondition: {
3766+
switch (record.branchType()) {
3767+
case BranchType_JMP:
3768+
linkJumpOrCall<BranchType_JMP, copy>(reinterpret_cast<int*>(from), fromInstruction, to);
3769+
break;
3770+
case BranchType_CALL:
3771+
linkJumpOrCall<BranchType_CALL, copy>(reinterpret_cast<int*>(from), fromInstruction, to);
3772+
break;
3773+
case BranchType_RET:
3774+
ASSERT_NOT_REACHED();
3775+
break;
3776+
}
37163777
break;
3778+
}
37173779
case LinkJumpConditionDirect:
37183780
linkConditionalBranch<DirectBranch, copy>(record.condition(), reinterpret_cast<int*>(from), fromInstruction, to);
37193781
break;

‎Source/JavaScriptCore/assembler/ARMv7Assembler.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -459,12 +459,13 @@ class ARMv7Assembler {
459459
return *this;
460460
}
461461
intptr_t from() const { return data.realTypes.m_from; }
462-
void setFrom(intptr_t from) { data.realTypes.m_from = from; }
463-
intptr_t to() const { return data.realTypes.m_to; }
462+
void setFrom(const ARMv7Assembler*, intptr_t from) { data.realTypes.m_from = from; }
463+
intptr_t to(const ARMv7Assembler*) const { return data.realTypes.m_to; }
464464
JumpType type() const { return data.realTypes.m_type; }
465465
JumpLinkType linkType() const { return data.realTypes.m_linkType; }
466466
void setLinkType(JumpLinkType linkType) { ASSERT(data.realTypes.m_linkType == LinkInvalid); data.realTypes.m_linkType = linkType; }
467467
Condition condition() const { return data.realTypes.m_condition; }
468+
bool isThunk() const { return false; }
468469
private:
469470
union {
470471
struct RealTypes {

‎Source/JavaScriptCore/assembler/AbstractMacroAssembler.h‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -575,7 +575,7 @@ class AbstractMacroAssembler : public AbstractMacroAssemblerBase {
575575
{
576576
}
577577

578-
bool isFlagSet(Flags flag)
578+
bool isFlagSet(Flags flag) const
579579
{
580580
return m_flags & flag;
581581
}
@@ -585,6 +585,24 @@ class AbstractMacroAssembler : public AbstractMacroAssemblerBase {
585585
return Call(jump.m_label, Linkable);
586586
}
587587

588+
template<PtrTag tag>
589+
void linkThunk(CodeLocationLabel<tag> label, AbstractMacroAssemblerType* masm) const
590+
{
591+
ASSERT(isFlagSet(Near));
592+
ASSERT(isFlagSet(Linkable));
593+
#if CPU(ARM64)
594+
if (isFlagSet(Tail))
595+
masm->m_assembler.linkJumpThunk(m_label, label.dataLocation(), ARM64Assembler::JumpNoCondition, ARM64Assembler::ConditionInvalid);
596+
else
597+
masm->m_assembler.linkNearCallThunk(m_label, label.dataLocation());
598+
#else
599+
Call class="pl-c1">this;
600+
masm->addLinkTask([=](auto& linkBuffer) {
601+
linkBuffer.link(target, label);
602+
});
603+
#endif
604+
}
605+
588606
AssemblerLabel m_label;
589607
private:
590608
Flags m_flags;
@@ -695,6 +713,24 @@ class AbstractMacroAssembler : public AbstractMacroAssemblerBase {
695713
#endif
696714
}
697715

716+
template<PtrTag tag>
717+
void linkThunk(CodeLocationLabel<tag> label, AbstractMacroAssemblerType* masm) const
718+
{
719+
#if CPU(ARM64)
720+
if ((m_type == ARM64Assembler::JumpCompareAndBranch) || (m_type == ARM64Assembler::JumpCompareAndBranchFixedSize))
721+
masm->m_assembler.linkJumpThunk(m_label, label.dataLocation(), m_type, m_condition, m_is64Bit, m_compareRegister);
722+
else if ((m_type == ARM64Assembler::JumpTestBit) || (m_type == ARM64Assembler::JumpTestBitFixedSize))
723+
masm->m_assembler.linkJumpThunk(m_label, label.dataLocation(), m_type, m_condition, m_bitNumber, m_compareRegister);
724+
else
725+
masm->m_assembler.linkJumpThunk(m_label, label.dataLocation(), m_type, m_condition);
726+
#else
727+
Jump class="pl-c1">this;
728+
masm->addLinkTask([=](auto& linkBuffer) {
729+
linkBuffer.link(target, label);
730+
});
731+
#endif
732+
}
733+
698734
bool isSet() const { return m_label.isSet(); }
699735

700736
private:
@@ -723,6 +759,12 @@ class AbstractMacroAssembler : public AbstractMacroAssemblerBase {
723759

724760
operator Jump&() { return m_jump; }
725761

762+
template<PtrTag tag>
763+
void linkThunk(CodeLocationLabel<tag> label, AbstractMacroAssemblerType* masm) const
764+
{
765+
m_jump.linkThunk(label, masm);
766+
}
767+
726768
Jump m_jump;
727769
};
728770

@@ -751,9 +793,15 @@ class AbstractMacroAssembler : public AbstractMacroAssemblerBase {
751793

752794
void linkTo(Label label, AbstractMacroAssemblerType* masm) const
753795
{
754-
size_t size = m_jumps.size();
755-
for (size_t i = 0; i < size; ++i)
756-
m_jumps[i].linkTo(label, masm);
796+
for (auto& jump : m_jumps)
797+
jump.linkTo(label, masm);
798+
}
799+
800+
template<PtrTag tag>
801+
void linkThunk(CodeLocationLabel<tag> label, AbstractMacroAssemblerType* masm) const
802+
{
803+
for (auto& jump : m_jumps)
804+
jump.linkThunk(label, masm);
757805
}
758806

759807
void append(Jump jump)

0 commit comments

Comments
 (0)