1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmLLIntGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "BytecodeGeneratorBaseInlines.h"
32#include "BytecodeStructs.h"
33#include "InstructionStream.h"
34#include "Label.h"
35#include "RegisterID.h"
36#include "WasmCallingConvention.h"
37#include "WasmContextInlines.h"
38#include "WasmFunctionCodeBlock.h"
39#include "WasmFunctionParser.h"
40#include "WasmGeneratorTraits.h"
41#include "WasmThunks.h"
42#include <wtf/RefPtr.h>
43#include <wtf/StdUnorderedMap.h>
44#include <wtf/Variant.h>
45
46namespace JSC { namespace Wasm {
47
48class LLIntGenerator : public BytecodeGeneratorBase<GeneratorTraits> {
49public:
50 using ExpressionType = RefPtr<RegisterID>;
51 using ExpressionList = Vector<ExpressionType, 1>;
52 using Stack = ExpressionList;
53
54 struct ControlLoop {
55 Ref<Label> m_body;
56 };
57
58 struct ControlTopLevel {
59 };
60
61 struct ControlBlock {
62 };
63
64 struct ControlIf {
65 Ref<Label> m_alternate;
66 };
67
68 struct ControlType : public Variant<ControlLoop, ControlTopLevel, ControlBlock, ControlIf> {
69 using Base = Variant<ControlLoop, ControlTopLevel, ControlBlock, ControlIf>;
70
71 ControlType()
72 : Base(ControlBlock { })
73 {
74 }
75
76 static ControlType loop(BlockSignature signature, const ExpressionList& results, Ref<Label> body, RefPtr<Label> continuation)
77 {
78 return ControlType(signature, results, WTFMove(continuation), ControlLoop { WTFMove(body) });
79 }
80
81 static ControlType topLevel(BlockSignature signature, const ExpressionList& results, RefPtr<Label> continuation)
82 {
83 return ControlType(signature, results, WTFMove(continuation), ControlTopLevel { });
84 }
85
86 static ControlType block(BlockSignature signature, const ExpressionList& results, RefPtr<Label> continuation)
87 {
88 return ControlType(signature, results, WTFMove(continuation), ControlBlock { });
89 }
90
91 static ControlType if_(BlockSignature signature, const ExpressionList& results, Ref<Label> alternate, RefPtr<Label> continuation)
92 {
93 return ControlType(signature, results, WTFMove(continuation), ControlIf { WTFMove(alternate) });
94 }
95
96 RefPtr<Label> targetLabelForBranch() const
97 {
98 if (WTF::holds_alternative<ControlLoop>(*this))
99 return WTF::get<ControlLoop>(*this).m_body.ptr();
100 return m_continuation;
101 }
102
103 BlockSignature m_signature;
104 ExpressionList m_results;
105 RefPtr<Label> m_continuation;
106
107 private:
108 template<typename T>
109 ControlType(BlockSignature signature, const ExpressionList& results, RefPtr<Label> continuation, T t)
110 : Base(WTFMove(t))
111 , m_signature(signature)
112 , m_results(results)
113 , m_continuation(WTFMove(continuation))
114 {
115 }
116 };
117
118 using ErrorType = String;
119 using PartialResult = Expected<void, ErrorType>;
120 using ResultList = ExpressionList;
121 using UnexpectedResult = Unexpected<ErrorType>;
122
123 using ControlEntry = FunctionParser<LLIntGenerator>::ControlEntry;
124
125 LLIntGenerator(const ModuleInformation&, unsigned functionIndex, ThrowWasmException, const Signature&);
126
127 std::unique_ptr<FunctionCodeBlock> finalize();
128
129 template <typename ...Args>
130 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
131 {
132 using namespace FailureHelper; // See ADL comment in WasmParser.h.
133 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
134 }
135
136 template<typename ExpressionListA, typename ExpressionListB>
137 void unifyValuesWithBlock(const ExpressionListA& destinations, const ExpressionListB& values)
138 {
139 ASSERT(destinations.size() <= values.size());
140 for (size_t i = 0; i < destinations.size(); ++i)
141 WasmMov::emit(this, destinations[destinations.size() - i - 1], values[values.size() - i - 1]);
142 }
143
144
145 static ExpressionType emptyExpression() { return nullptr; };
146 Stack createStack() { return Stack(); }
147 bool isControlTypeIf(const ControlType& control) { return WTF::holds_alternative<ControlIf>(control); }
148
149 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
150 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
151 ExpressionType addConstant(Type, uint64_t);
152
153 // References
154 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType value, ExpressionType& result);
155 PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);
156
157 // Tables
158 PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType index, ExpressionType& result);
159 PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType index, ExpressionType value);
160 PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);
161 PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType fill, ExpressionType delta, ExpressionType& result);
162 PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType offset, ExpressionType fill, ExpressionType count);
163
164 // Locals
165 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
166 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
167
168 // Globals
169 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
170 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
171
172 // Memory
173 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
174 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
175 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
176 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
177
178 // Basic operators
179 template<OpType>
180 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
181 template<OpType>
182 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
183 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
184
185 // Control flow
186 ControlType WARN_UNUSED_RETURN addTopLevel(BlockSignature);
187 PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);
188 PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);
189 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);
190 PartialResult WARN_UNUSED_RETURN addElse(ControlType&, const ExpressionList&);
191 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlType&);
192
193 PartialResult WARN_UNUSED_RETURN addReturn(const ControlType&, const ExpressionList& returnValues);
194 PartialResult WARN_UNUSED_RETURN addBranch(ControlType&, ExpressionType condition, const ExpressionList& returnValues);
195 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTargets, const ExpressionList& expressionStack);
196 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
197 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& expressionStack = { });
198
199 // Calls
200 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionList& results);
201 PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, ExpressionList& results);
202 PartialResult WARN_UNUSED_RETURN addUnreachable();
203
204 void didFinishParsingLocals();
205
206 void setParser(FunctionParser<LLIntGenerator>* parser) { m_parser = parser; };
207
208 void dump(const Vector<ControlEntry>&, const ExpressionList*) { }
209
210private:
211 friend GenericLabel<Wasm::GeneratorTraits>;
212
213 struct LLIntCallInformation {
214 unsigned stackOffset;
215 unsigned numberOfStackArguments;
216 ExpressionList arguments;
217 ExpressionList results;
218 };
219
220 LLIntCallInformation callInformationFor(const Signature&, CallRole = CallRole::Caller);
221
222 VirtualRegister virtualRegisterForLocal(uint32_t index)
223 {
224 if (index < m_codeBlock->m_numArguments)
225 return m_normalizedArguments[index];
226
227 const auto& callingConvention = wasmCallingConvention();
228 const uint32_t gprCount = callingConvention.gprArgs.size();
229 const uint32_t fprCount = callingConvention.fprArgs.size();
230 return ::JSC::virtualRegisterForLocal(index - m_codeBlock->m_numArguments + gprCount + fprCount + numberOfLLIntCalleeSaveRegisters);
231 }
232
233 ExpressionList tmpsForSignature(BlockSignature signature)
234 {
235 ExpressionList result(signature->returnCount());
236 for (unsigned i = 0; i < signature->returnCount(); ++i)
237 result[i] = newTemporary();
238 return result;
239 }
240
241 ExpressionType jsNullConstant()
242 {
243 if (!m_jsNullConstant)
244 m_jsNullConstant = addConstant(Type::Anyref, JSValue::encode(jsNull()));
245 return m_jsNullConstant;
246 }
247
248 struct SwitchEntry {
249 InstructionStream::Offset offset;
250 InstructionStream::Offset* jumpTarget;
251 };
252
253 FunctionParser<LLIntGenerator>* m_parser { nullptr };
254 const ModuleInformation& m_info;
255 const unsigned m_functionIndex { UINT_MAX };
256 Vector<VirtualRegister> m_normalizedArguments;
257 HashMap<Label*, Vector<SwitchEntry>> m_switches;
258 ExpressionType m_jsNullConstant;
259 ExpressionList m_unitializedLocals;
260 StdUnorderedMap<uint64_t, VirtualRegister> m_constantMap;
261};
262
263Expected<std::unique_ptr<FunctionCodeBlock>, String> parseAndCompileBytecode(const uint8_t* functionStart, size_t functionLength, const Signature& signature, const ModuleInformation& info, uint32_t functionIndex, ThrowWasmException throwWasmException)
264{
265 LLIntGenerator llintGenerator(info, functionIndex, throwWasmException, signature);
266 FunctionParser<LLIntGenerator> parser(llintGenerator, functionStart, functionLength, signature, info);
267 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
268
269 return llintGenerator.finalize();
270}
271
272LLIntGenerator::LLIntGenerator(const ModuleInformation& info, unsigned functionIndex, ThrowWasmException throwWasmException, const Signature&)
273 : BytecodeGeneratorBase(makeUnique<FunctionCodeBlock>(functionIndex), numberOfLLIntCalleeSaveRegisters)
274 , m_info(info)
275 , m_functionIndex(functionIndex)
276{
277 if (throwWasmException)
278 Thunks::singleton().setThrowWasmException(throwWasmException);
279
280 WasmEnter::emit(this);
281}
282
283std::unique_ptr<FunctionCodeBlock> LLIntGenerator::finalize()
284{
285 RELEASE_ASSERT(m_codeBlock);
286 m_codeBlock->setInstructions(m_writer.finalize());
287 return WTFMove(m_codeBlock);
288}
289
290// Generated from wasm.json
291#include "WasmLLIntGeneratorInlines.h"
292
293auto LLIntGenerator::callInformationFor(const Signature& signature, CallRole role) -> LLIntCallInformation
294{
295 const auto& callingConvention = wasmCallingConvention();
296 const uint32_t gprCount = callingConvention.gprArgs.size();
297 const uint32_t fprCount = callingConvention.fprArgs.size();
298
299 uint32_t stackCount = 0;
300 uint32_t gprIndex = 0;
301 uint32_t fprIndex = 0;
302
303 Vector<RefPtr<RegisterID>, 16> registers;
304
305 ExpressionList arguments(signature.argumentCount());
306 ExpressionList results(signature.returnCount());
307
308 auto allocateStackRegister = [&](Type type) {
309 ASSERT(role == CallRole::Caller);
310 switch (type) {
311 case Type::I32:
312 case Type::I64:
313 case Type::Anyref:
314 case Type::Funcref:
315 if (gprIndex < gprCount)
316 ++gprIndex;
317 else {
318 registers.append(newTemporary());
319 ++stackCount;
320 }
321 break;
322 case Type::F32:
323 case Type::F64:
324 if (fprIndex < fprCount)
325 ++fprIndex;
326 else {
327 registers.append(newTemporary());
328 ++stackCount;
329 }
330 break;
331 case Void:
332 case Func:
333 RELEASE_ASSERT_NOT_REACHED();
334 }
335 };
336
337
338 if (role == CallRole::Callee) {
339 // Reuse the slots we allocated to spill the registers in addArguments
340 for (uint32_t i = gprCount + fprCount; i--;)
341 registers.append(new RegisterID(::JSC::virtualRegisterForLocal(numberOfLLIntCalleeSaveRegisters + i)));
342 } else {
343 for (uint32_t i = 0; i < gprCount; i++)
344 registers.append(newTemporary());
345 for (uint32_t i = 0; i < fprCount; i++)
346 registers.append(newTemporary());
347
348 for (uint32_t i = 0; i < signature.argumentCount(); i++)
349 allocateStackRegister(signature.argument(i));
350 gprIndex = 0;
351 fprIndex = 0;
352 for (uint32_t i = 0; i < signature.returnCount(); i++)
353 allocateStackRegister(signature.returnType(i));
354 }
355
356 unsigned stackOffset;
357 if (role == CallRole::Callee)
358 stackOffset = static_cast<unsigned>(-registers.last()->index());
359 else {
360 // Align the stack
361 auto computeStackOffset = [&] {
362 return static_cast<unsigned>(-registers.last()->index()) + CallFrame::headerSizeInRegisters;
363 };
364 while (computeStackOffset() % stackAlignmentRegisters())
365 registers.append(newTemporary());
366 stackOffset = computeStackOffset();
367 }
368
369 ASSERT(role == CallRole::Caller || !stackCount);
370 const uint32_t maxGPRIndex = stackCount + gprCount;
371 const uint32_t maxFPRIndex = maxGPRIndex + fprCount;
372 uint32_t stackIndex = 0;
373 auto appendForType = [&](Type type, unsigned index, auto& vector) {
374 switch (type) {
375 case Type::I32:
376 case Type::I64:
377 case Type::Anyref:
378 case Type::Funcref:
379 if (gprIndex < maxGPRIndex)
380 vector[index] = registers[registers.size() - gprIndex++ - 1];
381 else {
382 if (role == CallRole::Caller)
383 vector[index] = registers[registers.size() - stackIndex++ - 1];
384 else
385 vector[index] = new RegisterID(virtualRegisterForArgument(stackIndex++));
386 }
387 break;
388 case Type::F32:
389 case Type::F64:
390 if (fprIndex < maxFPRIndex)
391 vector[index] = registers[registers.size() - fprIndex++ - 1];
392 else {
393 if (role == CallRole::Caller)
394 vector[index] = registers[registers.size() - stackIndex++ - 1];
395 else
396 vector[index] = new RegisterID(virtualRegisterForArgument(stackIndex++));
397 }
398 break;
399 case Void:
400 case Func:
401 RELEASE_ASSERT_NOT_REACHED();
402 }
403 };
404
405 gprIndex = stackCount;
406 fprIndex = maxGPRIndex;
407 for (uint32_t i = 0; i < signature.argumentCount(); i++)
408 appendForType(signature.argument(i), i, arguments);
409 gprIndex = stackCount;
410 fprIndex = maxGPRIndex;
411 for (uint32_t i = 0; i < signature.returnCount(); i++)
412 appendForType(signature.returnType(i), i, results);
413
414 if (role == CallRole::Caller) {
415 // Reserve space for call frame.
416 Vector<RefPtr<RegisterID>, CallFrame::headerSizeInRegisters + 2, UnsafeVectorOverflow> callFrame;
417 for (int i = 0; i < CallFrame::headerSizeInRegisters; ++i)
418 callFrame.append(newTemporary());
419 }
420
421 return LLIntCallInformation { stackOffset, stackCount, WTFMove(arguments), WTFMove(results) };
422}
423
424auto LLIntGenerator::addArguments(const Signature& signature) -> PartialResult
425{
426 m_codeBlock->m_numArguments = signature.argumentCount();
427 m_normalizedArguments.resize(m_codeBlock->m_numArguments);
428
429 const auto& callingConvention = wasmCallingConvention();
430 const uint32_t gprCount = callingConvention.gprArgs.size();
431 const uint32_t fprCount = callingConvention.fprArgs.size();
432 const uint32_t maxGPRIndex = gprCount;
433 const uint32_t maxFPRIndex = gprCount + fprCount;
434 uint32_t gprIndex = 0;
435 uint32_t fprIndex = maxGPRIndex;
436 uint32_t stackIndex = 0;
437
438 Vector<RefPtr<RegisterID>> registerArguments(gprCount + fprCount);
439 for (uint32_t i = 0; i < gprCount + fprCount; i++)
440 registerArguments[i] = addVar();
441
442 const auto addArgument = [&](uint32_t index, uint32_t& count, uint32_t max) {
443 if (count < max)
444 m_normalizedArguments[index] = registerArguments[count++];
445 else
446 m_normalizedArguments[index] = virtualRegisterForArgument(stackIndex++);
447 };
448
449 for (uint32_t i = 0; i < signature.argumentCount(); i++) {
450 switch (signature.argument(i)) {
451 case Type::I32:
452 case Type::I64:
453 case Type::Anyref:
454 case Type::Funcref:
455 addArgument(i, gprIndex, maxGPRIndex);
456 break;
457 case Type::F32:
458 case Type::F64:
459 addArgument(i, fprIndex, maxFPRIndex);
460 break;
461 case Void:
462 case Func:
463 RELEASE_ASSERT_NOT_REACHED();
464 }
465 }
466
467 return { };
468}
469
470auto LLIntGenerator::addLocal(Type type, uint32_t count) -> PartialResult
471{
472 while (count--) {
473 auto local = addVar();
474 switch (type) {
475 case Type::Anyref:
476 case Type::Funcref:
477 m_unitializedLocals.append(local);
478 break;
479 default:
480 break;
481 }
482 }
483 return { };
484}
485
486void LLIntGenerator::didFinishParsingLocals()
487{
488 auto null = jsNullConstant();
489 for (auto local : m_unitializedLocals)
490 WasmMov::emit(this, local, null);
491 m_unitializedLocals.clear();
492}
493
494auto LLIntGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
495{
496 VirtualRegister source(FirstConstantRegisterIndex + m_codeBlock->m_constants.size());
497 auto result = m_constantMap.emplace(value, source);
498 if (result.second) {
499 m_codeBlock->m_constants.append(value);
500 if (UNLIKELY(Options::dumpGeneratedWasmBytecodes()))
501 m_codeBlock->m_constantTypes.append(type);
502 } else
503 source = result.first->second;
504 auto target = newTemporary();
505 WasmMov::emit(this, target, source);
506 return target;
507}
508
509auto LLIntGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
510{
511 // FIXME: Remove unnecessary moves
512 // https://bugs.webkit.org/show_bug.cgi?id=203657
513 result = newTemporary();
514 WasmMov::emit(this, result, virtualRegisterForLocal(index));
515 return { };
516}
517
518auto LLIntGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
519{
520 WasmMov::emit(this, virtualRegisterForLocal(index), value);
521 return { };
522}
523
524auto LLIntGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
525{
526 result = newTemporary();
527 WasmGetGlobal::emit(this, result, index);
528 return { };
529}
530
531auto LLIntGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
532{
533 Type type = m_info.globals[index].type;
534 if (isSubtype(type, Anyref))
535 WasmSetGlobalRef::emit(this, index, value);
536 else
537 WasmSetGlobal::emit(this, index, value);
538 return { };
539}
540
541auto LLIntGenerator::addLoop(BlockSignature signature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex) -> PartialResult
542{
543 Ref<Label> body = newEmittedLabel();
544 Ref<Label> continuation = newLabel();
545
546 newStack = splitStack(signature, enclosingStack);
547 block = ControlType::loop(signature, newStack, WTFMove(body), WTFMove(continuation));
548
549 Vector<VirtualRegister> osrEntryData;
550 for (uint32_t i = 0; i < m_codeBlock->m_numArguments; i++)
551 osrEntryData.append(m_normalizedArguments[i]);
552
553 const auto& callingConvention = wasmCallingConvention();
554 const uint32_t gprCount = callingConvention.gprArgs.size();
555 const uint32_t fprCount = callingConvention.fprArgs.size();
556 for (int32_t i = gprCount + fprCount + numberOfLLIntCalleeSaveRegisters; i < m_codeBlock->m_numVars; i++)
557 osrEntryData.append(::JSC::virtualRegisterForLocal(i));
558 for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) {
559 ExpressionList& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack;
560 for (auto& expression : expressionStack)
561 osrEntryData.append(expression->virtualRegister());
562 }
563
564 WasmLoopHint::emit(this);
565
566 m_codeBlock->tierUpCounter().addOSREntryDataForLoop(m_lastInstruction.offset(), { loopIndex, WTFMove(osrEntryData) });
567
568 return { };
569}
570
571auto LLIntGenerator::addTopLevel(BlockSignature signature) -> ControlType
572{
573 return ControlType::topLevel(signature, tmpsForSignature(signature), newLabel());
574}
575
576auto LLIntGenerator::addBlock(BlockSignature signature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack) -> PartialResult
577{
578 newStack = splitStack(signature, enclosingStack);
579 newBlock = ControlType::block(signature, tmpsForSignature(signature), newLabel());
580 return { };
581}
582
583auto LLIntGenerator::addIf(ExpressionType condition, BlockSignature signature, Stack& enclosingStack, ControlType& result, Stack& newStack) -> PartialResult
584{
585 Ref<Label> alternate = newLabel();
586 Ref<Label> continuation = newLabel();
587
588 WasmJfalse::emit(this, condition, alternate->bind(this));
589
590 newStack = splitStack(signature, enclosingStack);
591 result = ControlType::if_(signature, tmpsForSignature(signature), WTFMove(alternate), WTFMove(continuation));
592 return { };
593}
594
595auto LLIntGenerator::addElse(ControlType& data, const ExpressionList& currentStack) -> PartialResult
596{
597 ASSERT(WTF::holds_alternative<ControlIf>(data));
598 unifyValuesWithBlock(data.m_results, currentStack);
599 WasmJmp::emit(this, data.m_continuation->bind(this));
600 return addElseToUnreachable(data);
601}
602
603auto LLIntGenerator::addElseToUnreachable(ControlType& data) -> PartialResult
604{
605 ControlIf& control = WTF::get<ControlIf>(data);
606 emitLabel(control.m_alternate.get());
607 data = ControlType::block(data.m_signature, WTFMove(data.m_results), WTFMove(data.m_continuation));
608 return { };
609}
610
611auto LLIntGenerator::addReturn(const ControlType& data, const ExpressionList& returnValues) -> PartialResult
612{
613 if (!data.m_signature->returnCount()) {
614 WasmRetVoid::emit(this);
615 return { };
616 }
617
618 LLIntCallInformation info = callInformationFor(*data.m_signature, CallRole::Callee);
619 unifyValuesWithBlock(info.results, returnValues);
620 WasmRet::emit(this);
621
622 return { };
623}
624
625auto LLIntGenerator::addBranch(ControlType& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
626{
627 unifyValuesWithBlock(data.m_results, returnValues);
628
629 RefPtr<Label> target = data.targetLabelForBranch();
630 if (condition)
631 WasmJtrue::emit(this, condition, target->bind(this));
632 else
633 WasmJmp::emit(this, target->bind(this));
634
635 return { };
636}
637
638auto LLIntGenerator::addSwitch(ExpressionType condition, const Vector<ControlType*>& targets, ControlType& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
639{
640 unsigned tableIndex = m_codeBlock->numberOfJumpTables();
641 FunctionCodeBlock::JumpTable& jumpTable = m_codeBlock->addJumpTable(targets.size());
642
643 for (const auto& target : targets)
644 unifyValuesWithBlock(target->m_results, expressionStack);
645 unifyValuesWithBlock(defaultTarget.m_results, expressionStack);
646
647 WasmSwitch::emit(this, condition, tableIndex, defaultTarget.targetLabelForBranch()->bind(this));
648
649 unsigned index = 0;
650 InstructionStream::Offset offset = m_lastInstruction.offset();
651 for (const auto& target : targets) {
652 RefPtr<Label> targetLabel = target->targetLabelForBranch();
653 if (targetLabel->isForward()) {
654 auto result = m_switches.add(targetLabel.get(), Vector<SwitchEntry>());
655 ASSERT(!jumpTable[index]);
656 result.iterator->value.append({ offset, &jumpTable[index++] });
657 } else {
658 int jumpTarget = targetLabel->bind(this).target();
659 ASSERT(jumpTarget);
660 jumpTable[index++] = jumpTarget;
661 }
662 }
663
664
665 return { };
666}
667
668auto LLIntGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
669{
670 ControlType& data = entry.controlData;
671
672 if (!WTF::holds_alternative<ControlLoop>(data))
673 unifyValuesWithBlock(data.m_results, expressionStack);
674
675 return addEndToUnreachable(entry, expressionStack);
676}
677
678
679auto LLIntGenerator::addEndToUnreachable(ControlEntry& entry, const Stack& expressionStack) -> PartialResult
680{
681 ControlType& data = entry.controlData;
682
683 emitLabel(*data.m_continuation);
684
685 if (!WTF::holds_alternative<ControlLoop>(data))
686 entry.enclosedExpressionStack.appendVector(data.m_results);
687 else {
688 for (unsigned i = 0; i < data.m_signature->returnCount(); ++i) {
689 if (i < expressionStack.size())
690 entry.enclosedExpressionStack.append(expressionStack[i]);
691 else
692 entry.enclosedExpressionStack.append(newTemporary());
693 }
694 }
695
696 // TopLevel does not have any code after this so we need to make sure we emit a return here.
697 if (WTF::holds_alternative<ControlTopLevel>(data))
698 return addReturn(data, entry.enclosedExpressionStack);
699
700 return { };
701}
702
703auto LLIntGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionList& results) -> PartialResult
704{
705 ASSERT(signature.argumentCount() == args.size());
706 LLIntCallInformation info = callInformationFor(signature);
707 unifyValuesWithBlock(info.arguments, args);
708 results = WTFMove(info.results);
709 if (Context::useFastTLS())
710 WasmCall::emit(this, functionIndex, info.stackOffset, info.numberOfStackArguments);
711 else
712 WasmCallNoTls::emit(this, functionIndex, info.stackOffset, info.numberOfStackArguments);
713
714 return { };
715}
716
717auto LLIntGenerator::addCallIndirect(unsigned tableIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionList& results) -> PartialResult
718{
719 ExpressionType calleeIndex = args.takeLast();
720
721 ASSERT(signature.argumentCount() == args.size());
722 ASSERT(m_info.tableCount() > tableIndex);
723 ASSERT(m_info.tables[tableIndex].type() == TableElementType::Funcref);
724
725 LLIntCallInformation info = callInformationFor(signature);
726 unifyValuesWithBlock(info.arguments, args);
727 results = WTFMove(info.results);
728 if (Context::useFastTLS())
729 WasmCallIndirect::emit(this, calleeIndex, m_codeBlock->addSignature(signature), info.stackOffset, info.numberOfStackArguments, tableIndex);
730 else
731 WasmCallIndirectNoTls::emit(this, calleeIndex, m_codeBlock->addSignature(signature), info.stackOffset, info.numberOfStackArguments, tableIndex);
732
733 return { };
734}
735
736auto LLIntGenerator::addRefIsNull(ExpressionType value, ExpressionType& result) -> PartialResult
737{
738 result = value;
739 WasmRefIsNull::emit(this, result, value);
740
741 return { };
742}
743
744auto LLIntGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
745{
746 result = newTemporary();
747 WasmRefFunc::emit(this, result, index);
748
749 return { };
750}
751
752auto LLIntGenerator::addTableGet(unsigned tableIndex, ExpressionType index, ExpressionType& result) -> PartialResult
753{
754 result = index;
755 WasmTableGet::emit(this, result, index, tableIndex);
756
757 return { };
758}
759
760auto LLIntGenerator::addTableSet(unsigned tableIndex, ExpressionType index, ExpressionType value) -> PartialResult
761{
762 WasmTableSet::emit(this, index, value, tableIndex);
763
764 return { };
765}
766
767auto LLIntGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
768{
769 result = newTemporary();
770 WasmTableSize::emit(this, result, tableIndex);
771
772 return { };
773}
774
775auto LLIntGenerator::addTableGrow(unsigned tableIndex, ExpressionType fill, ExpressionType delta, ExpressionType& result) -> PartialResult
776{
777 result = fill;
778 WasmTableGrow::emit(this, result, fill, delta, tableIndex);
779
780 return { };
781}
782
783auto LLIntGenerator::addTableFill(unsigned tableIndex, ExpressionType offset, ExpressionType fill, ExpressionType count) -> PartialResult
784{
785 WasmTableFill::emit(this, offset, fill, count, tableIndex);
786
787 return { };
788}
789
790auto LLIntGenerator::addUnreachable() -> PartialResult
791{
792 WasmUnreachable::emit(this);
793
794 return { };
795}
796
797auto LLIntGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
798{
799 result = newTemporary();
800 WasmCurrentMemory::emit(this, result);
801
802 return { };
803}
804
805auto LLIntGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
806{
807 result = delta;
808 WasmGrowMemory::emit(this, result, delta);
809
810 return { };
811}
812
813auto LLIntGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
814{
815 result = condition;
816 WasmSelect::emit(this, result, condition, nonZero, zero);
817
818 return { };
819}
820
821auto LLIntGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
822{
823 result = pointer;
824 switch (op) {
825 case LoadOpType::I32Load8S:
826 WasmI32Load8S::emit(this, result, pointer, offset);
827 break;
828
829 case LoadOpType::I64Load8S:
830 WasmI64Load8S::emit(this, result, pointer, offset);
831 break;
832
833 case LoadOpType::I32Load8U:
834 case LoadOpType::I64Load8U:
835 WasmLoad8U::emit(this, result, pointer, offset);
836 break;
837
838 case LoadOpType::I32Load16S:
839 WasmI32Load16S::emit(this, result, pointer, offset);
840 break;
841
842 case LoadOpType::I64Load16S:
843 WasmI64Load16S::emit(this, result, pointer, offset);
844 break;
845
846 case LoadOpType::I32Load16U:
847 case LoadOpType::I64Load16U:
848 WasmLoad16U::emit(this, result, pointer, offset);
849 break;
850
851 case LoadOpType::I32Load:
852 case LoadOpType::F32Load:
853 case LoadOpType::I64Load32U:
854 WasmLoad32U::emit(this, result, pointer, offset);
855 break;
856
857 case LoadOpType::I64Load32S:
858 WasmI64Load32S::emit(this, result, pointer, offset);
859 break;
860
861 case LoadOpType::I64Load:
862 case LoadOpType::F64Load:
863 WasmLoad64U::emit(this, result, pointer, offset);
864 break;
865 }
866
867 return { };
868}
869
870auto LLIntGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
871{
872 switch (op) {
873 case StoreOpType::I64Store8:
874 case StoreOpType::I32Store8:
875 WasmStore8::emit(this, pointer, value, offset);
876 break;
877
878 case StoreOpType::I64Store16:
879 case StoreOpType::I32Store16:
880 WasmStore16::emit(this, pointer, value, offset);
881 break;
882
883 case StoreOpType::I64Store32:
884 case StoreOpType::I32Store:
885 case StoreOpType::F32Store:
886 WasmStore32::emit(this, pointer, value, offset);
887 break;
888
889 case StoreOpType::I64Store:
890 case StoreOpType::F64Store:
891 WasmStore64::emit(this, pointer, value, offset);
892 break;
893 }
894
895 return { };
896}
897
898}
899
900template<>
901void GenericLabel<Wasm::GeneratorTraits>::setLocation(BytecodeGeneratorBase<Wasm::GeneratorTraits>& generator, unsigned location)
902{
903 RELEASE_ASSERT(isForward());
904
905 m_location = location;
906
907 Wasm::LLIntGenerator* llintGenerator = static_cast<Wasm::LLIntGenerator*>(&generator);
908
909 auto it = llintGenerator->m_switches.find(this);
910 if (it != llintGenerator->m_switches.end()) {
911 for (const auto& entry : it->value) {
912 ASSERT(!*entry.jumpTarget);
913 *entry.jumpTarget = m_location - entry.offset;
914 }
915 llintGenerator->m_switches.remove(it);
916 }
917
918
919 for (auto offset : m_unresolvedJumps) {
920 auto instruction = generator.m_writer.ref(offset);
921 int target = m_location - offset;
922
923#define CASE(__op) \
924 case __op::opcodeID: \
925 instruction->cast<__op, WasmOpcodeTraits>()->setTargetLabel(BoundLabel(target), [&]() { \
926 generator.m_codeBlock->addOutOfLineJumpTarget(instruction.offset(), target); \
927 return BoundLabel(); \
928 }); \
929 break;
930
931 switch (instruction->opcodeID<WasmOpcodeTraits>()) {
932 CASE(WasmJmp)
933 CASE(WasmJtrue)
934 CASE(WasmJfalse)
935 case WasmSwitch::opcodeID: {
936 ASSERT((!instruction->as<WasmSwitch, WasmOpcodeTraits>().m_defaultTarget));
937 instruction->cast<WasmSwitch, WasmOpcodeTraits>()->setDefaultTarget(BoundLabel(target), [&]() {
938 generator.m_codeBlock->addOutOfLineJumpTarget(instruction.offset(), target);
939 return BoundLabel();
940 });
941 break;
942 }
943 default:
944 ASSERT_NOT_REACHED();
945 }
946#undef CASE
947 }
948}
949
950} // namespace JSC::Wasm
951
952#endif // ENABLE(WEBASSEMBLY)
953