1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmSlowPaths.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "ExceptionFuzz.h"
32#include "FrameTracers.h"
33#include "LLIntData.h"
34#include "WasmBBQPlan.h"
35#include "WasmCallee.h"
36#include "WasmCompilationMode.h"
37#include "WasmContextInlines.h"
38#include "WasmFunctionCodeBlock.h"
39#include "WasmInstance.h"
40#include "WasmModuleInformation.h"
41#include "WasmOMGForOSREntryPlan.h"
42#include "WasmOMGPlan.h"
43#include "WasmOperations.h"
44#include "WasmThunks.h"
45#include "WasmWorklist.h"
46
47namespace JSC { namespace LLInt {
48
49#define WASM_RETURN_TWO(first, second) do { \
50 return encodeResult(first, second); \
51 } while (false)
52
53#define WASM_END_IMPL() WASM_RETURN_TWO(pc, 0)
54
55#define WASM_THROW(exceptionType) do { \
56 callFrame->setArgumentCountIncludingThis(static_cast<int>(exceptionType)); \
57 WASM_RETURN_TWO(LLInt::wasmExceptionInstructions(), 0); \
58 } while (false)
59
60#define WASM_END() do { \
61 WASM_END_IMPL(); \
62 } while (false)
63
64#define WASM_RETURN(value) do { \
65 callFrame->uncheckedR(instruction.m_dst) = static_cast<EncodedJSValue>(value); \
66 WASM_END_IMPL(); \
67 } while (false)
68
69#define WASM_CALL_RETURN(targetInstance, callTarget, callTargetTag) do { \
70 WASM_RETURN_TWO(retagCodePtr(callTarget, callTargetTag, SlowPathPtrTag), targetInstance); \
71 } while (false)
72
73#define CODE_BLOCK() \
74 bitwise_cast<Wasm::FunctionCodeBlock*>(callFrame->codeBlock())
75
76#define READ(virtualRegister) \
77 (virtualRegister.isConstant() \
78 ? JSValue::decode(CODE_BLOCK()->getConstant(virtualRegister.offset())) \
79 : callFrame->r(virtualRegister))
80
81inline bool jitCompileAndSetHeuristics(Wasm::LLIntCallee* callee, Wasm::FunctionCodeBlock* codeBlock, Wasm::Instance* instance)
82{
83 Wasm::LLIntTierUpCounter& tierUpCounter = codeBlock->tierUpCounter();
84 if (!tierUpCounter.checkIfOptimizationThresholdReached()) {
85 dataLogLnIf(Options::verboseOSR(), " JIT threshold should be lifted.");
86 return false;
87 }
88
89 if (callee->replacement()) {
90 dataLogLnIf(Options::verboseOSR(), " Code was already compiled.");
91 tierUpCounter.optimizeSoon();
92 return true;
93 }
94
95 bool compile = false;
96 {
97 auto locker = holdLock(tierUpCounter.m_lock);
98 switch (tierUpCounter.m_compilationStatus) {
99 case Wasm::LLIntTierUpCounter::CompilationStatus::NotCompiled:
100 compile = true;
101 tierUpCounter.m_compilationStatus = Wasm::LLIntTierUpCounter::CompilationStatus::Compiling;
102 break;
103 case Wasm::LLIntTierUpCounter::CompilationStatus::Compiling:
104 tierUpCounter.optimizeAfterWarmUp();
105 break;
106 case Wasm::LLIntTierUpCounter::CompilationStatus::Compiled:
107 break;
108 }
109 }
110
111 if (compile) {
112 uint32_t functionIndex = codeBlock->functionIndex();
113 RefPtr<Wasm::Plan> plan;
114 if (Options::wasmLLIntTiersUpToBBQ())
115 plan = adoptRef(*new Wasm::BBQPlan(instance->context(), makeRef(const_cast<Wasm::ModuleInformation&>(instance->module().moduleInformation())), functionIndex, instance->codeBlock(), Wasm::Plan::dontFinalize()));
116 else
117 plan = adoptRef(*new Wasm::OMGPlan(instance->context(), Ref<Wasm::Module>(instance->module()), functionIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize()));
118
119 Wasm::ensureWorklist().enqueue(makeRef(*plan));
120 if (UNLIKELY(!Options::useConcurrentJIT()))
121 plan->waitForCompletion();
122 else
123 tierUpCounter.optimizeAfterWarmUp();
124 }
125
126 return !!callee->replacement();
127}
128
129WASM_SLOW_PATH_DECL(prologue_osr)
130{
131 UNUSED_PARAM(pc);
132
133 Wasm::LLIntCallee* callee = static_cast<Wasm::LLIntCallee*>(callFrame->callee().asWasmCallee());
134 Wasm::FunctionCodeBlock* codeBlock = CODE_BLOCK();
135
136 dataLogLnIf(Options::verboseOSR(), *callee, ": Entered epilogue_osr with tierUpCounter = ", codeBlock->tierUpCounter());
137
138 if (!jitCompileAndSetHeuristics(callee, codeBlock, instance))
139 WASM_RETURN_TWO(0, 0);
140
141 WASM_RETURN_TWO(callee->replacement()->entrypoint().executableAddress(), 0);
142}
143
144WASM_SLOW_PATH_DECL(loop_osr)
145{
146 if (!Options::useWebAssemblyOSR()) {
147 slow_path_wasm_prologue_osr(callFrame, pc, instance);
148 WASM_RETURN_TWO(0, 0);
149 }
150
151 Wasm::LLIntCallee* callee = static_cast<Wasm::LLIntCallee*>(callFrame->callee().asWasmCallee());
152 Wasm::FunctionCodeBlock* codeBlock = CODE_BLOCK();
153 Wasm::LLIntTierUpCounter& tierUpCounter = codeBlock->tierUpCounter();
154
155 dataLogLnIf(Options::verboseOSR(), *callee, ": Entered loop_osr with tierUpCounter = ", codeBlock->tierUpCounter());
156
157 unsigned loopOSREntryBytecodeOffset = codeBlock->bytecodeOffset(pc);
158 const auto& osrEntryData = tierUpCounter.osrEntryDataForLoop(loopOSREntryBytecodeOffset);
159
160 if (!tierUpCounter.checkIfOptimizationThresholdReached()) {
161 dataLogLnIf(Options::verboseOSR(), " JIT threshold should be lifted.");
162 WASM_RETURN_TWO(0, 0);
163 }
164
165 const auto doOSREntry = [&] {
166 Wasm::OMGForOSREntryCallee* osrEntryCallee = callee->osrEntryCallee();
167 if (osrEntryCallee->loopIndex() != osrEntryData.loopIndex)
168 WASM_RETURN_TWO(0, 0);
169
170 size_t osrEntryScratchBufferSize = osrEntryCallee->osrEntryScratchBufferSize();
171 RELEASE_ASSERT(osrEntryScratchBufferSize == osrEntryData.values.size());
172 uint64_t* buffer = instance->context()->scratchBufferForSize(osrEntryScratchBufferSize);
173 if (!buffer)
174 WASM_RETURN_TWO(0, 0);
175
176 uint32_t index = 0;
177 for (VirtualRegister reg : osrEntryData.values)
178 buffer[index++] = READ(reg).encodedJSValue();
179
180 WASM_RETURN_TWO(buffer, osrEntryCallee->entrypoint().executableAddress());
181 };
182
183 if (callee->osrEntryCallee())
184 return doOSREntry();
185
186 bool compile = false;
187 {
188 auto locker = holdLock(tierUpCounter.m_lock);
189 switch (tierUpCounter.m_loopCompilationStatus) {
190 case Wasm::LLIntTierUpCounter::CompilationStatus::NotCompiled:
191 compile = true;
192 tierUpCounter.m_loopCompilationStatus = Wasm::LLIntTierUpCounter::CompilationStatus::Compiling;
193 break;
194 case Wasm::LLIntTierUpCounter::CompilationStatus::Compiling:
195 tierUpCounter.optimizeAfterWarmUp();
196 break;
197 case Wasm::LLIntTierUpCounter::CompilationStatus::Compiled:
198 break;
199 }
200 }
201
202 if (compile) {
203 Ref<Wasm::Plan> plan = adoptRef(*static_cast<Wasm::Plan*>(new Wasm::OMGForOSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::Callee>(*callee), codeBlock->functionIndex(), osrEntryData.loopIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize())));
204 Wasm::ensureWorklist().enqueue(plan.copyRef());
205 if (UNLIKELY(!Options::useConcurrentJIT()))
206 plan->waitForCompletion();
207 else
208 tierUpCounter.optimizeAfterWarmUp();
209 }
210
211 if (callee->osrEntryCallee())
212 return doOSREntry();
213
214 WASM_RETURN_TWO(0, 0);
215}
216
217WASM_SLOW_PATH_DECL(epilogue_osr)
218{
219 Wasm::LLIntCallee* callee = static_cast<Wasm::LLIntCallee*>(callFrame->callee().asWasmCallee());
220 Wasm::FunctionCodeBlock* codeBlock = CODE_BLOCK();
221
222 dataLogLnIf(Options::verboseOSR(), *callee, ": Entered epilogue_osr with tierUpCounter = ", codeBlock->tierUpCounter());
223
224 jitCompileAndSetHeuristics(callee, codeBlock, instance);
225 WASM_END_IMPL();
226}
227
228
229WASM_SLOW_PATH_DECL(trace)
230{
231 UNUSED_PARAM(instance);
232
233 if (!Options::traceLLIntExecution())
234 WASM_END_IMPL();
235
236 WasmOpcodeID opcodeID = pc->opcodeID<WasmOpcodeTraits>();
237 dataLogF("<%p> %p / %p: executing bc#%zu, %s, pc = %p\n",
238 &Thread::current(),
239 callFrame->codeBlock(),
240 callFrame,
241 static_cast<intptr_t>(CODE_BLOCK()->bytecodeOffset(pc)),
242 pc->name<WasmOpcodeTraits>(),
243 pc);
244 if (opcodeID == wasm_enter) {
245 dataLogF("Frame will eventually return to %p\n", callFrame->returnPC().value());
246 *removeCodePtrTag<volatile char*>(callFrame->returnPC().value());
247 }
248 if (opcodeID == wasm_ret) {
249 dataLogF("Will be returning to %p\n", callFrame->returnPC().value());
250 dataLogF("The new cfr will be %p\n", callFrame->callerFrame());
251 }
252 WASM_END_IMPL();
253}
254
255WASM_SLOW_PATH_DECL(out_of_line_jump_target)
256{
257 UNUSED_PARAM(instance);
258
259 pc = CODE_BLOCK()->outOfLineJumpTarget(pc);
260 WASM_END_IMPL();
261}
262
263WASM_SLOW_PATH_DECL(ref_func)
264{
265 auto instruction = pc->as<WasmRefFunc, WasmOpcodeTraits>();
266 WASM_RETURN(Wasm::operationWasmRefFunc(instance, instruction.m_functionIndex));
267}
268
269WASM_SLOW_PATH_DECL(table_get)
270{
271 auto instruction = pc->as<WasmTableGet, WasmOpcodeTraits>();
272 int32_t index = READ(instruction.m_index).unboxedInt32();
273 EncodedJSValue result = Wasm::operationGetWasmTableElement(instance, instruction.m_tableIndex, index);
274 if (!result)
275 WASM_THROW(Wasm::ExceptionType::OutOfBoundsTableAccess);
276 WASM_RETURN(result);
277}
278
279WASM_SLOW_PATH_DECL(table_set)
280{
281 auto instruction = pc->as<WasmTableSet, WasmOpcodeTraits>();
282 int32_t index = READ(instruction.m_index).unboxedInt32();
283 EncodedJSValue value = READ(instruction.m_value).encodedJSValue();
284 if (!Wasm::operationSetWasmTableElement(instance, instruction.m_tableIndex, index, value))
285 WASM_THROW(Wasm::ExceptionType::OutOfBoundsTableAccess);
286 WASM_END();
287}
288
289WASM_SLOW_PATH_DECL(table_size)
290{
291 auto instruction = pc->as<WasmTableSize, WasmOpcodeTraits>();
292 WASM_RETURN(Wasm::operationGetWasmTableSize(instance, instruction.m_tableIndex));
293}
294
295WASM_SLOW_PATH_DECL(table_fill)
296{
297 auto instruction = pc->as<WasmTableFill, WasmOpcodeTraits>();
298 int32_t offset = READ(instruction.m_offset).unboxedInt32();
299 EncodedJSValue fill = READ(instruction.m_fill).encodedJSValue();
300 int32_t size = READ(instruction.m_size).unboxedInt32();
301 if (!Wasm::operationWasmTableFill(instance, instruction.m_tableIndex, offset, fill, size))
302 WASM_THROW(Wasm::ExceptionType::OutOfBoundsTableAccess);
303 WASM_END();
304}
305
306WASM_SLOW_PATH_DECL(table_grow)
307{
308 auto instruction = pc->as<WasmTableGrow, WasmOpcodeTraits>();
309 EncodedJSValue fill = READ(instruction.m_fill).encodedJSValue();
310 int32_t size = READ(instruction.m_size).unboxedInt32();
311 WASM_RETURN(Wasm::operationWasmTableGrow(instance, instruction.m_tableIndex, fill, size));
312}
313
314WASM_SLOW_PATH_DECL(grow_memory)
315{
316 auto instruction = pc->as<WasmGrowMemory, WasmOpcodeTraits>();
317 int32_t delta = READ(instruction.m_delta).unboxedInt32();
318 WASM_RETURN(Wasm::operationGrowMemory(callFrame, instance, delta));
319}
320
321inline SlowPathReturnType doWasmCall(Wasm::Instance* instance, unsigned functionIndex)
322{
323 uint32_t importFunctionCount = instance->module().moduleInformation().importFunctionCount();
324
325 MacroAssemblerCodePtr<WasmEntryPtrTag> codePtr;
326
327 if (functionIndex < importFunctionCount) {
328 Wasm::Instance::ImportFunctionInfo* functionInfo = instance->importFunctionInfo(functionIndex);
329 if (functionInfo->targetInstance) {
330 // target is a wasm function from a different instance
331 codePtr = instance->codeBlock()->wasmToWasmExitStub(functionIndex);
332 } else {
333 // target is JS
334 codePtr = functionInfo->wasmToEmbedderStub;
335 }
336 } else {
337 // Target is a wasm function within the same instance
338 codePtr = *instance->codeBlock()->entrypointLoadLocationFromFunctionIndexSpace(functionIndex);
339 }
340
341 WASM_CALL_RETURN(instance, codePtr.executableAddress(), WasmEntryPtrTag);
342}
343
344WASM_SLOW_PATH_DECL(call)
345{
346 UNUSED_PARAM(callFrame);
347
348 auto instruction = pc->as<WasmCall, WasmOpcodeTraits>();
349 return doWasmCall(instance, instruction.m_functionIndex);
350}
351
352WASM_SLOW_PATH_DECL(call_no_tls)
353{
354 UNUSED_PARAM(callFrame);
355
356 auto instruction = pc->as<WasmCallNoTls, WasmOpcodeTraits>();
357 return doWasmCall(instance, instruction.m_functionIndex);
358}
359
360inline SlowPathReturnType doWasmCallIndirect(CallFrame* callFrame, Wasm::Instance* instance, unsigned functionIndex, unsigned tableIndex, unsigned signatureIndex)
361{
362 Wasm::FuncRefTable* table = instance->table(tableIndex)->asFuncrefTable();
363
364 if (functionIndex >= table->length())
365 WASM_THROW(Wasm::ExceptionType::OutOfBoundsCallIndirect);
366
367 Wasm::Instance* targetInstance = table->instance(functionIndex);
368 const Wasm::WasmToWasmImportableFunction& function = table->function(functionIndex);
369
370 if (function.signatureIndex == Wasm::Signature::invalidIndex)
371 WASM_THROW(Wasm::ExceptionType::NullTableEntry);
372
373 const Wasm::Signature& callSignature = CODE_BLOCK()->signature(signatureIndex);
374 if (function.signatureIndex != Wasm::SignatureInformation::get(callSignature))
375 WASM_THROW(Wasm::ExceptionType::BadSignature);
376
377 if (targetInstance != instance)
378 targetInstance->setCachedStackLimit(instance->cachedStackLimit());
379
380 WASM_CALL_RETURN(targetInstance, function.entrypointLoadLocation->executableAddress(), WasmEntryPtrTag);
381}
382
383WASM_SLOW_PATH_DECL(call_indirect)
384{
385 auto instruction = pc->as<WasmCallIndirect, WasmOpcodeTraits>();
386 unsigned functionIndex = READ(instruction.m_functionIndex).unboxedInt32();
387 return doWasmCallIndirect(callFrame, instance, functionIndex, instruction.m_tableIndex, instruction.m_signatureIndex);
388}
389
390WASM_SLOW_PATH_DECL(call_indirect_no_tls)
391{
392 auto instruction = pc->as<WasmCallIndirectNoTls, WasmOpcodeTraits>();
393 unsigned functionIndex = READ(instruction.m_functionIndex).unboxedInt32();
394 return doWasmCallIndirect(callFrame, instance, functionIndex, instruction.m_tableIndex, instruction.m_signatureIndex);
395}
396
397WASM_SLOW_PATH_DECL(set_global_ref)
398{
399 auto instruction = pc->as<WasmSetGlobalRef, WasmOpcodeTraits>();
400 instance->setGlobal(instruction.m_globalIndex, READ(instruction.m_value).jsValue());
401 WASM_END_IMPL();
402}
403
404extern "C" SlowPathReturnType slow_path_wasm_throw_exception(CallFrame* callFrame, const Instruction* pc, Wasm::Instance* instance, Wasm::ExceptionType exceptionType)
405{
406 UNUSED_PARAM(pc);
407 WASM_RETURN_TWO(Wasm::Thunks::singleton().throwWasmException()(callFrame, exceptionType, instance), 0);
408}
409
410extern "C" SlowPathReturnType slow_path_wasm_popcount(const Instruction* pc, uint32_t x)
411{
412 void* result = bitwise_cast<void*>(static_cast<uint64_t>(__builtin_popcount(x)));
413 WASM_RETURN_TWO(pc, result);
414}
415
416extern "C" SlowPathReturnType slow_path_wasm_popcountll(const Instruction* pc, uint64_t x)
417{
418 void* result = bitwise_cast<void*>(static_cast<uint64_t>(__builtin_popcountll(x)));
419 WASM_RETURN_TWO(pc, result);
420}
421
422} } // namespace JSC::LLInt
423
424#endif // ENABLE(WEBASSEMBLY)
425