#include "config.h"

#include "ExecutableAllocator.h"

#if ENABLE(JIT)

#include "ExecutableAllocationFuzz.h"

#include "JITOperationValidation.h"

#include "LinkBuffer.h"

#include <bit>

#include <wtf/CryptographicallyRandomNumber.h>

#include <wtf/FastBitVector.h>

#include <wtf/FileSystem.h>

#include <wtf/FixedVector.h>

#include <wtf/IterationStatus.h>

#include <wtf/MallocSpan.h>

#include <wtf/PageReservation.h>

#include <wtf/ProcessID.h>

#include <wtf/RedBlackTree.h>

#include <wtf/Scope.h>

#include <wtf/SequesteredMalloc.h>

#include <wtf/SystemTracing.h>

#include <wtf/TZoneMallocInlines.h>

#include <wtf/UUID.h>

#include <wtf/WorkQueue.h>

#include <wtf/posix/POSIXExtras.h>

#if ENABLE(LIBPAS_JIT_HEAP)

#include <bmalloc/jit_heap.h>

#include <bmalloc/jit_heap_config.h>

#else

#include <wtf/MetaAllocator.h>

#endif

#if HAVE(IOS_JIT_RESTRICTIONS) || HAVE(MAC_JIT_RESTRICTIONS)

#include <wtf/cocoa/Entitlements.h>

#endif

#if OS(DARWIN)

#include <fcntl.h>

#include <mach/mach.h>

#include <mach/mach_time.h>

#include <wtf/spi/cocoa/MachVMSPI.h>

#endif

#if USE(INLINE_JIT_PERMISSIONS_API)

#include <wtf/darwin/WeakLinking.h>

WTF_WEAK_LINK_FORCE_IMPORT(be_memory_inline_jit_restrict_with_witness_supported);

#endif

namespace JSC {

using namespace WTF;

#if OS(DARWIN) && CPU(ARM64)

static inline constexpr size_t LEGACY_NODELETE executablePageSize() { return CeilingOnPageSize; }

#else

static inline size_t executablePageSize() { return WTF::pageSize(); }

#endif

#if ENABLE(LIBPAS_JIT_HEAP)

static constexpr size_t minimumPoolSizeForSegregatedHeap = 256 * MB;

#endif

#if defined(FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB) && FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB > 0

static constexpr size_t fixedExecutableMemoryPoolSize = FIXED_EXECUTABLE_MEMORY_POOL_SIZE_IN_MB * MB;

#elif CPU(ARM64)

#if ENABLE(JUMP_ISLANDS)

static constexpr size_t fixedExecutableMemoryPoolSize = 512 * MB;

#else

static constexpr size_t fixedExecutableMemoryPoolSize = 128 * MB;

#endif

#elif CPU(X86_64)

static constexpr size_t fixedExecutableMemoryPoolSize = 1 * GB;

#else

static constexpr size_t fixedExecutableMemoryPoolSize = 32 * MB;

#endif

#if ENABLE(JUMP_ISLANDS)

#if CPU(ARM64)

static constexpr double islandRegionSizeFraction = 0.125;

static constexpr size_t islandSizeInBytes = 4;

#endif

#endif

#if !ENABLE(JUMP_ISLANDS)

static_assert(fixedExecutableMemoryPoolSize <= MacroAssembler::nearJumpRange, "Executable pool size is too large for near jump/call without JUMP_ISLANDS");

#endif

static constexpr double executablePoolReservationFraction = 0.25;

#if ENABLE(LIBPAS_JIT_HEAP)

static constexpr size_t minimumExecutablePoolReservationSize = 256 * KB;

static_assert(fixedExecutableMemoryPoolSize * executablePoolReservationFraction >= minimumExecutablePoolReservationSize);

static_assert(fixedExecutableMemoryPoolSize < 4 * GB, "ExecutableMemoryHandle assumes it is less than 4GB");

#endif

#if HAVE(KDEBUG_H)

static constexpr WTF::UUID jscJITNamespace = WTF::UUID::createConstant(0x325696c8e7cc11eeULL, 0x9f4e325096b39f47ULL);

#endif

static bool LEGACY_NODELETE isJITEnabled()

}

void ExecutableAllocator::disableJIT()

}

#if OS(DARWIN) && HAVE(REMAP_JIT)

#if USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)

static ALWAYS_INLINE MacroAssemblerCodeRef<JITThunkPtrTag> jitWriteThunkGenerator(void* writableAddr, void* stubBase, size_t stubSize)

{

auto exitScope = makeScopeExit([] {

RELEASE_ASSERT(!g_jscConfig.useFastJITPermissions);

});

using namespace ARM64Registers;

using TrustedImm32 = MacroAssembler::TrustedImm32;

MacroAssembler jit;

jit.tagReturnAddress();

jit.move(MacroAssembler::TrustedImmPtr(writableAddr), x7);

jit.addPtr(x7, x0);

jit.move(x0, x3);

MacroAssembler::Jump smallCopy = jit.branch64(MacroAssembler::Below, x2, MacroAssembler::TrustedImm64(64));

jit.add64(TrustedImm32(32), x3);

jit.and64(TrustedImm32(-32), x3);

jit.loadPair64(x1, x12, x13);

jit.loadPair64(x1, TrustedImm32(16), x14, x15);

jit.sub64(x3, x0, x5);

jit.addPtr(x5, x1);

jit.loadPair64(x1, x8, x9);

jit.loadPair64(x1, TrustedImm32(16), x10, x11);

jit.add64(TrustedImm32(32), x1);

jit.sub64(x5, x2);

jit.storePair64(x12, x13, x0);

jit.storePair64(x14, x15, x0, TrustedImm32(16));

MacroAssembler::Jump cleanup = jit.branchSub64(MacroAssembler::BelowOrEqual, TrustedImm32(64), x2);

MacroAssembler::Label copyLoop = jit.label();

jit.storePair64WithNonTemporalAccess(x8, x9, x3);

jit.storePair64WithNonTemporalAccess(x10, x11, x3, TrustedImm32(16));

jit.add64(TrustedImm32(32), x3);

jit.loadPair64WithNonTemporalAccess(x1, x8, x9);

jit.loadPair64WithNonTemporalAccess(x1, TrustedImm32(16), x10, x11);

jit.add64(TrustedImm32(32), x1);

jit.branchSub64(MacroAssembler::Above, TrustedImm32(32), x2).linkTo(copyLoop, &jit);

cleanup.link(&jit);

jit.add64(x2, x1);

jit.loadPair64(x1, x12, x13);

jit.loadPair64(x1, TrustedImm32(16), x14, x15);

jit.storePair64(x8, x9, x3);

jit.storePair64(x10, x11, x3, TrustedImm32(16));

jit.addPtr(x2, x3);

jit.storePair64(x12, x13, x3, TrustedImm32(32));

jit.storePair64(x14, x15, x3, TrustedImm32(48));

jit.ret();

MacroAssembler::Label local0 = jit.label();

jit.load64(MacroAssembler::PostIndexAddress(x1, 8), x6);

jit.store64(x6, MacroAssembler::PostIndexAddress(x3, 8));

smallCopy.link(&jit);

jit.branchSub64(MacroAssembler::AboveOrEqual, TrustedImm32(8), x2).linkTo(local0, &jit);

MacroAssembler::Jump local2 = jit.branchAdd64(MacroAssembler::Equal, TrustedImm32(8), x2);

MacroAssembler::Label local1 = jit.label();

jit.load8(x1, PostIndex(1), x6);

jit.store8(x6, x3, PostIndex(1));

jit.branchSub64(MacroAssembler::NotEqual, TrustedImm32(1), x2).linkTo(local1, &jit);

local2.link(&jit);

jit.ret();

auto stubBaseCodePtr = CodePtr<LinkBufferPtrTag>(tagCodePtr<LinkBufferPtrTag>(stubBase));

LinkBuffer linkBuffer(jit, stubBaseCodePtr, stubSize, LinkBuffer::Profile::Thunk);

// We don't use FINALIZE_CODE() because we don't want the writeable address, as

// disassembled instructions, to appear in the console or anywhere in memory, via

// the PrintStream buffer.

return linkBuffer.finalizeCodeWithoutDisassembly<JITThunkPtrTag>(nullptr);

}

#else // not USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)

static void genericWriteToJITRegion(off_t offset, const void* data, size_t dataSize)

}

static MacroAssemblerCodeRef<JITThunkPtrTag> ALWAYS_INLINE jitWriteThunkGenerator(void* address, void*, size_t)

}

#endif // USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)

static ALWAYS_INLINE void initializeSeparatedWXHeaps(void* stubBase, size_t stubSize, void* jitBase, size_t jitSize)

}

#endif // OS(DARWIN) && HAVE(REMAP_JIT)

struct JITReservation {

};

static ALWAYS_INLINE JITReservation initializeJITPageReservation()

}

class FixedVMPoolExecutableAllocator final {

// This does not need to be TZONE_ALLOCATED because it's only used as a singleton

// and is only allocated once long before any scripts are executed.

WTF_MAKE_SEQUESTERED_IMMORTAL_ALLOCATED(FixedVMPoolExecutableAllocator);

#if ENABLE(JUMP_ISLANDS)

class Islands;

class RegionAllocator;

#endif

FixedVMPoolExecutableAllocator()

#if !ENABLE(JUMP_ISLANDS)

: m_allocator(*this)

#endif

{

JITReservation reservation = initializeJITPageReservation();

m_reservation = WTF::move(reservation.pageReservation);

if (m_reservation) {

#if ENABLE(JUMP_ISLANDS)

// Consider this scenario:

//

// <------------- nearJumpRange ------------->

// <------------- nearJumpRange -------------->

// [ island 0 ] [ JIT region 1 ] [ island 1 ] [ JIT region 2 ] [ island 2 ] [ JIT region3 ]

//

// C1 ---jump---> I1 --------------jump---------> I2 ---jump---> C3

//

// In order to jump across a distance that spans multiple nearJumpRanges, we currently

// use chaining near jumps. Hence, a near jump in a jump island also needs to be able

// to reach its neighboring jump islands in order to form this chain.

//

// For example, let say we have code in JIT region 1 that needs to jump to code in JIT region 3 in

// the illustration above. That jump will be implemented as:

// 1. Code C1 in JIT region 1 near jumps to island I1 in island 1.

// 2. Island I1 near jumps to island I2 in island 2.

// 3. Island I2 near jumps to code C3 in JIT region 3.

//

// Each of these near jumps need to be within the range of MacroAssembler::nearJumpRange.

//

// As a result, the maximum size of each JIT region is:

// MacroAssembler::nearJumpRange - 2 * islandRegionSize

//

// This is why each RegionAllocator tracks a range of m_regionSize instead of

// MacroAssembler::nearJumpRange.

//

// Note: the illustration above shows a jump chain in the forward direction. The jump island

// scheme also allows for a jump chain in the backward direction e.g. from C3 to C1.

const size_t islandRegionSize = roundUpToMultipleOf(executablePageSize(), static_cast<size_t>(MacroAssembler::nearJumpRange * islandRegionSizeFraction));

m_regionSize = MacroAssembler::nearJumpRange - islandRegionSize;

RELEASE_ASSERT(isPageAligned(executablePageSize(), islandRegionSize));

RELEASE_ASSERT(isPageAligned(executablePageSize(), m_regionSize));

const unsigned numAllocators = (reservation.size + m_regionSize - 1) / m_regionSize;

m_allocators = FixedVector<RegionAllocator>::createWithSizeAndConstructorArguments(numAllocators, *this);

uintptr_t start = std::bit_cast<uintptr_t>(memoryStart());

uintptr_t reservationEnd = std::bit_cast<uintptr_t>(memoryEnd());

for (size_t i = 0; i < numAllocators; ++i) {

uintptr_t end = start + m_regionSize;

uintptr_t islandBegin = end - islandRegionSize;

// The island in the very last region is never actually used (everything goes backwards), but we

// can't put code there in case they do need to use a backward jump island, so set up accordingly.

if (i == numAllocators - 1)

islandBegin = end = std::min(islandBegin, reservationEnd);

RELEASE_ASSERT(end <= reservationEnd);

m_allocators[i].configure(start, islandBegin, end);

m_bytesReserved += m_allocators[i].allocatorSize();

start += m_regionSize;

}

#else

m_allocator.addFreshFreeSpace(reservation.base, reservation.size);

m_bytesReserved += reservation.size;

#endif

#if ENABLE(MPROTECT_RX_TO_RWX)

ptrdiff_t pagesInReservation = (std::bit_cast<uint8_t*>(g_jscConfig.endExecutableMemory) - std::bit_cast<uint8_t*>(g_jscConfig.startExecutableMemory)) / executablePageSize();

m_pageWriterCounts = std::bit_cast<uint8_t*>(WTF::fastZeroedMalloc(pagesInReservation));

#endif

}

}

~FixedVMPoolExecutableAllocator()

{

m_reservation.deallocate();

}

void* LEGACY_NODELETE memoryStart() { return g_jscConfig.startExecutableMemory; }

void* LEGACY_NODELETE memoryEnd() { return g_jscConfig.endExecutableMemory; }

bool isValid() { return !!m_reservation; }

RefPtr<ExecutableMemoryHandle> allocate(size_t sizeInBytes)

}

Lock& getLock() WTF_RETURNS_LOCK(m_lock) { return m_lock; }

#if ENABLE(LIBPAS_JIT_HEAP)

void LEGACY_NODELETE shrinkBytesAllocated(size_t oldSizeInBytes, size_t newSizeInBytes)

{

m_bytesAllocated.fetch_add(newSizeInBytes - oldSizeInBytes, std::memory_order_relaxed);

}

#endif

// Non atomic

size_t LEGACY_NODELETE bytesAllocated()

}

size_t LEGACY_NODELETE bytesReserved() const

{

return m_bytesReserved;

}

size_t LEGACY_NODELETE bytesAvailable()

}

#if !ENABLE(LIBPAS_JIT_HEAP)

size_t bytesCommitted()

}

#endif

bool isInAllocatedMemory(const AbstractLocker& locker, void* address)

}

#if ENABLE(META_ALLOCATOR_PROFILE)

void dumpProfile()

}

#endif

#if ENABLE(MPROTECT_RX_TO_RWX)

static std::pair<size_t, size_t> pageRangeForWrittenRegion(const void* start, size_t sizeInBytes, size_t pageSize)

}

void startWriting(const void* start, size_t sizeInBytes)

}

void finishWriting(const void* start, size_t sizeInBytes)

}

#endif

#if !ENABLE(LIBPAS_JIT_HEAP)

MetaAllocator::Statistics currentStatistics()

}

#endif // !ENABLE(LIBPAS_JIT_HEAP)

#if ENABLE(LIBPAS_JIT_HEAP)

void handleWillBeReleased(ExecutableMemoryHandle& handle, size_t sizeInBytes)

}

#endif // ENABLE(LIBPAS_JIT_HEAP)

#if ENABLE(JUMP_ISLANDS)

void handleWillBeReleased(const Locker<Lock>& locker, ExecutableMemoryHandle& handle)

}

void* makeIsland(uintptr_t jumpLocation, uintptr_t newTarget, bool concurrently, bool useMemcpy)

{

Locker locker { getLock() };

return islandForJumpLocation(locker, jumpLocation, newTarget, concurrently, useMemcpy);

}

RegionAllocator* findRegion(uintptr_t ptr)

{

RegionAllocator* result = nullptr;

forEachAllocator([&] (RegionAllocator& allocator) {

if (allocator.start() <= ptr && ptr < allocator.end()) {

result = &allocator;

return IterationStatus::Done;

}

return IterationStatus::Continue;

});

return result;

}

void freeJumpIslands(const Locker<Lock>&, Islands* islands)

}

void freeIslands(const Locker<Lock>& locker, Islands* islands)

}

void* islandForJumpLocation(const Locker<Lock>& locker, uintptr_t jumpLocation, uintptr_t target, bool concurrently, bool useMemcpy)

{

CheckedPtr islands = m_islandsForJumpSourceLocation.findExact(std::bit_cast<void*>(jumpLocation));

if (islands) {

// FIXME: We could create some method of reusing already allocated islands here, but it's

// unlikely to matter in practice.

if (!concurrently)

freeJumpIslands(locker, islands.get());

} else {

islands = new Islands;

islands->jumpSourceLocation = CodeLocationLabel<ExecutableMemoryPtrTag>(tagCodePtr<ExecutableMemoryPtrTag>(std::bit_cast<void*>(jumpLocation)));

m_islandsForJumpSourceLocation.insert(islands.get());

}

RegionAllocator* allocator = findRegion(jumpLocation > target ? jumpLocation - m_regionSize : jumpLocation);

RELEASE_ASSERT(allocator);

void* result = allocator->allocateIsland();

void* currentIsland = result;

jumpLocation = std::bit_cast<uintptr_t>(currentIsland);

while (true) {

islands->jumpIslands.append(CodeLocationLabel<ExecutableMemoryPtrTag>(tagCodePtr<ExecutableMemoryPtrTag>(currentIsland)));

auto emitJumpTo = [&] (void* target) {

RELEASE_ASSERT(Assembler::canEmitJump(std::bit_cast<void*>(jumpLocation), target));

if (useMemcpy)

Assembler::fillNearTailCall<memcpyRepatchFlush>(currentIsland, target);

else

Assembler::fillNearTailCall<jitMemcpyRepatchFlush>(currentIsland, target);

};

if (Assembler::canEmitJump(std::bit_cast<void*>(jumpLocation), std::bit_cast<void*>(target))) {

emitJumpTo(std::bit_cast<void*>(target));

break;

}

uintptr_t nextIslandRegion;

if (jumpLocation > target)

nextIslandRegion = jumpLocation - m_regionSize;

else

nextIslandRegion = jumpLocation + m_regionSize;

RegionAllocator* allocator = findRegion(nextIslandRegion);

RELEASE_ASSERT(allocator);

void* nextIsland = allocator->allocateIsland();

emitJumpTo(nextIsland);

jumpLocation = std::bit_cast<uintptr_t>(nextIsland);

currentIsland = nextIsland;

}

return result;

}

#endif // ENABLE(JUMP_ISLANDS)

class Allocator

#if !ENABLE(LIBPAS_JIT_HEAP)

: public MetaAllocator

#endif

{

#if !ENABLE(LIBPAS_JIT_HEAP)

using Base = MetaAllocator;

#endif

public:

Allocator(FixedVMPoolExecutableAllocator& allocator)

#if !ENABLE(LIBPAS_JIT_HEAP)

: Base(allocator.getLock(), jitAllocationGranule, executablePageSize()) // round up all allocations to 32 bytes

,

#else

:

#endif

m_fixedAllocator(allocator)

{

}

#if ENABLE(LIBPAS_JIT_HEAP)

void addFreshFreeSpace(void* start, size_t sizeInBytes)

}

bool LEGACY_NODELETE isInAllocatedMemory(const AbstractLocker&, void* address)

}

#endif // ENABLE(LIBPAS_JIT_HEAP)

#if !ENABLE(LIBPAS_JIT_HEAP)

FreeSpacePtr allocateNewSpace(size_t&) override

}

void notifyNeedPage(void* page, size_t count) override

{

m_fixedAllocator.m_reservation.commit(page, executablePageSize() * count);

}

void notifyPageIsFree(void* page, size_t count) override

{

m_fixedAllocator.m_reservation.decommit(page, executablePageSize() * count);

}

#endif // !ENABLE(LIBPAS_JIT_HEAP)

FixedVMPoolExecutableAllocator& m_fixedAllocator;

#if ENABLE(LIBPAS_JIT_HEAP)

uintptr_t m_start { 0 };

uintptr_t m_end { 0 };

#endif // ENABLE(LIBPAS_JIT_HEAP)

};

#if ENABLE(JUMP_ISLANDS)

class RegionAllocator final : public Allocator {