1/*
2 * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmB3IRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AllowMacroScratchRegisterUsageIf.h"
32#include "B3BasicBlockInlines.h"
33#include "B3CCallValue.h"
34#include "B3Compile.h"
35#include "B3ConstPtrValue.h"
36#include "B3FixSSA.h"
37#include "B3Generate.h"
38#include "B3InsertionSet.h"
39#include "B3SlotBaseValue.h"
40#include "B3StackmapGenerationParams.h"
41#include "B3SwitchValue.h"
42#include "B3UpsilonValue.h"
43#include "B3Validate.h"
44#include "B3ValueInlines.h"
45#include "B3ValueKey.h"
46#include "B3Variable.h"
47#include "B3VariableValue.h"
48#include "B3WasmAddressValue.h"
49#include "B3WasmBoundsCheckValue.h"
50#include "DisallowMacroScratchRegisterUsage.h"
51#include "JSCInlines.h"
52#include "JSWebAssemblyInstance.h"
53#include "ScratchRegisterAllocator.h"
54#include "VirtualRegister.h"
55#include "WasmCallingConvention.h"
56#include "WasmContextInlines.h"
57#include "WasmExceptionType.h"
58#include "WasmFunctionParser.h"
59#include "WasmInstance.h"
60#include "WasmMemory.h"
61#include "WasmOMGPlan.h"
62#include "WasmOpcodeOrigin.h"
63#include "WasmSignatureInlines.h"
64#include "WasmThunks.h"
65#include <limits>
66#include <wtf/Optional.h>
67#include <wtf/StdLibExtras.h>
68
69void dumpProcedure(void* ptr)
70{
71 JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
72 proc->dump(WTF::dataFile());
73}
74
75namespace JSC { namespace Wasm {
76
77using namespace B3;
78
79namespace {
80namespace WasmB3IRGeneratorInternal {
81static const bool verbose = false;
82}
83}
84
85class B3IRGenerator {
86public:
87 struct ControlData {
88 ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
89 : blockType(type)
90 , continuation(continuation)
91 , special(special)
92 {
93 if (signature != Void)
94 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
95 }
96
97 ControlData()
98 {
99 }
100
101 void dump(PrintStream& out) const
102 {
103 switch (type()) {
104 case BlockType::If:
105 out.print("If: ");
106 break;
107 case BlockType::Block:
108 out.print("Block: ");
109 break;
110 case BlockType::Loop:
111 out.print("Loop: ");
112 break;
113 case BlockType::TopLevel:
114 out.print("TopLevel: ");
115 break;
116 }
117 out.print("Continuation: ", *continuation, ", Special: ");
118 if (special)
119 out.print(*special);
120 else
121 out.print("None");
122 }
123
124 BlockType type() const { return blockType; }
125
126 bool hasNonVoidSignature() const { return result.size(); }
127
128 BasicBlock* targetBlockForBranch()
129 {
130 if (type() == BlockType::Loop)
131 return special;
132 return continuation;
133 }
134
135 void convertIfToBlock()
136 {
137 ASSERT(type() == BlockType::If);
138 blockType = BlockType::Block;
139 special = nullptr;
140 }
141
142 using ResultList = Vector<Value*, 1>; // Value must be a Phi
143
144 ResultList resultForBranch() const
145 {
146 if (type() == BlockType::Loop)
147 return ResultList();
148 return result;
149 }
150
151 private:
152 friend class B3IRGenerator;
153 BlockType blockType;
154 BasicBlock* continuation;
155 BasicBlock* special;
156 ResultList result;
157 };
158
159 typedef Value* ExpressionType;
160 typedef ControlData ControlType;
161 typedef Vector<ExpressionType, 1> ExpressionList;
162 typedef ControlData::ResultList ResultList;
163 typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
164
165 static constexpr ExpressionType emptyExpression() { return nullptr; }
166
167 typedef String ErrorType;
168 typedef Unexpected<ErrorType> UnexpectedResult;
169 typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
170 typedef Expected<void, ErrorType> PartialResult;
171 template <typename ...Args>
172 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
173 {
174 using namespace FailureHelper; // See ADL comment in WasmParser.h.
175 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
176 }
177#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
178 if (UNLIKELY(condition)) \
179 return fail(__VA_ARGS__); \
180 } while (0)
181
182 B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
183
184 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
185 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
186 ExpressionType addConstant(Type, uint64_t);
187
188 // References
189 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
190 PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);
191
192 // Tables
193 PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType& index, ExpressionType& result);
194 PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType& index, ExpressionType& value);
195 PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);
196 PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType& fill, ExpressionType& delta, ExpressionType& result);
197 PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType& offset, ExpressionType& fill, ExpressionType& count);
198 // Locals
199 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
200 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
201
202 // Globals
203 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
204 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
205
206 // Memory
207 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
208 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
209 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
210 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
211
212 // Basic operators
213 template<OpType>
214 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
215 template<OpType>
216 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
217 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
218
219 // Control flow
220 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
221 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
222 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
223 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
224 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
225 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
226
227 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
228 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
229 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
230 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
231 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
232
233 // Calls
234 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
235 PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
236 PartialResult WARN_UNUSED_RETURN addUnreachable();
237
238 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
239 void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
240
241 Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
242 void insertConstants();
243
244 ALWAYS_INLINE void didKill(ExpressionType) { }
245
246private:
247 void emitExceptionCheck(CCallHelpers&, ExceptionType);
248
249 void emitTierUpCheck(uint32_t decrementCount, Origin);
250
251 void emitWriteBarrierForJSWrapper();
252 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
253 B3::Kind memoryKind(B3::Opcode memoryOp);
254 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
255 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
256
257 void unify(const ExpressionType phi, const ExpressionType source);
258 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
259
260 void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
261
262 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
263
264 void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
265 enum class RestoreCachedStackLimit { No, Yes };
266 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
267
268 Origin origin();
269
270 FunctionParser<B3IRGenerator>* m_parser { nullptr };
271 const ModuleInformation& m_info;
272 const MemoryMode m_mode { MemoryMode::BoundsChecking };
273 const CompilationMode m_compilationMode { CompilationMode::BBQMode };
274 const unsigned m_functionIndex { UINT_MAX };
275 const TierUpCount* m_tierUp { nullptr };
276
277 Procedure& m_proc;
278 BasicBlock* m_currentBlock { nullptr };
279 Vector<Variable*> m_locals;
280 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
281 HashMap<ValueKey, Value*> m_constantPool;
282 InsertionSet m_constantInsertionValues;
283 GPRReg m_memoryBaseGPR { InvalidGPRReg };
284 GPRReg m_memorySizeGPR { InvalidGPRReg };
285 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
286 bool m_makesCalls { false };
287
288 Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
289 bool m_usesInstanceValue { false };
290 Value* instanceValue()
291 {
292 m_usesInstanceValue = true;
293 return m_instanceValue;
294 }
295
296 uint32_t m_maxNumJSCallArguments { 0 };
297 unsigned m_numImportFunctions;
298};
299
300// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
301int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
302{
303 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
304 ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
305 return 0;
306 }
307 return offset;
308}
309
310void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
311{
312 if (Context::useFastTLS()) {
313 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
314 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
315 patchpoint->clobber(RegisterSet::macroScratchRegisters());
316 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
317 patchpoint->setGenerator(
318 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
319 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
320 jit.storeWasmContextInstance(params[0].gpr());
321 });
322 return;
323 }
324
325 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
326 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
327 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
328 Effects effects = Effects::none();
329 effects.writesPinned = true;
330 effects.reads = B3::HeapRange::top();
331 patchpoint->effects = effects;
332 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
333 patchpoint->append(arg, ValueRep::SomeRegister);
334 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
335 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
336 jit.move(param[0].gpr(), wasmContextInstanceGPR);
337 });
338}
339
340B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
341 : m_info(info)
342 , m_mode(mode)
343 , m_compilationMode(compilationMode)
344 , m_functionIndex(functionIndex)
345 , m_tierUp(tierUp)
346 , m_proc(procedure)
347 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
348 , m_constantInsertionValues(m_proc)
349 , m_numImportFunctions(info.importFunctionCount())
350{
351 m_currentBlock = m_proc.addBlock();
352
353 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
354 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
355
356 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
357 m_proc.pinRegister(m_memoryBaseGPR);
358
359 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
360 if (!Context::useFastTLS())
361 m_proc.pinRegister(m_wasmContextInstanceGPR);
362
363 if (mode != MemoryMode::Signaling) {
364 m_memorySizeGPR = pinnedRegs.sizeRegister;
365 m_proc.pinRegister(m_memorySizeGPR);
366 }
367
368 if (throwWasmException)
369 Thunks::singleton().setThrowWasmException(throwWasmException);
370
371 if (info.memory) {
372 m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
373 AllowMacroScratchRegisterUsage allowScratch(jit);
374 switch (m_mode) {
375 case MemoryMode::BoundsChecking:
376 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
377 break;
378 case MemoryMode::Signaling:
379 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
380 break;
381 }
382 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
383 });
384
385 switch (m_mode) {
386 case MemoryMode::BoundsChecking:
387 break;
388 case MemoryMode::Signaling:
389 // Most memory accesses in signaling mode don't do an explicit
390 // exception check because they can rely on fault handling to detect
391 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
392 // thunk to exist so that it can jump to that thunk.
393 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
394 CRASH();
395 break;
396 }
397 }
398
399 wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
400
401 {
402 B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
403 B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
404 m_instanceValue = stackOverflowCheck;
405 stackOverflowCheck->appendSomeRegister(framePointer);
406 stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
407 if (!Context::useFastTLS()) {
408 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
409 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
410 stackOverflowCheck->effects.writesPinned = false;
411 stackOverflowCheck->effects.readsPinned = true;
412 stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
413 }
414 stackOverflowCheck->numGPScratchRegisters = 2;
415 stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
416 const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
417 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
418 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
419 // This allows us to elide stack checks for functions that are terminal nodes in the call
420 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
421 // having any such terminal node have its parent caller include some extra size in its
422 // own check for it. The goal here is twofold:
423 // 1. Emit less code.
424 // 2. Try to speed things up by skipping stack checks.
425 minimumParentCheckSize,
426 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
427 // spill all arguments to the stack, we ensure that a stack check here covers the
428 // stack that such a stub would use.
429 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
430 ));
431 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
432 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
433 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
434
435 GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
436
437 // This allows leaf functions to not do stack checks if their frame size is within
438 // certain limits since their caller would have already done the check.
439 if (needsOverflowCheck) {
440 AllowMacroScratchRegisterUsage allowScratch(jit);
441 GPRReg fp = params[1].gpr();
442 GPRReg scratch1 = params.gpScratch(0);
443 GPRReg scratch2 = params.gpScratch(1);
444
445 if (Context::useFastTLS())
446 jit.loadWasmContextInstance(contextInstance);
447
448 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
449 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
450 MacroAssembler::JumpList overflow;
451 if (UNLIKELY(needUnderflowCheck))
452 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
453 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
454 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
455 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
456 });
457 } else if (m_usesInstanceValue && Context::useFastTLS()) {
458 // No overflow check is needed, but the instance values still needs to be correct.
459 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
460 jit.loadWasmContextInstance(contextInstance);
461 } else {
462 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
463 }
464 });
465 }
466
467 emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
468}
469
470void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
471{
472 restoreWasmContextInstance(proc, block, instance);
473
474 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
475 // The Instance caches the stack limit, but also knows where its canonical location is.
476 Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
477 Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
478 block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
479 }
480
481 if (!!memory) {
482 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
483 RegisterSet clobbers;
484 clobbers.set(pinnedRegs->baseMemoryPointer);
485 clobbers.set(pinnedRegs->sizeRegister);
486 if (!isARM64())
487 clobbers.set(RegisterSet::macroScratchRegisters());
488
489 B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
490 Effects effects = Effects::none();
491 effects.writesPinned = true;
492 effects.reads = B3::HeapRange::top();
493 patchpoint->effects = effects;
494 patchpoint->clobber(clobbers);
495 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
496
497 patchpoint->append(instance, ValueRep::SomeRegister);
498 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
499 AllowMacroScratchRegisterUsage allowScratch(jit);
500 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
501 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
502
503 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
504 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
505
506 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->sizeRegister, scratchOrSize);
507 });
508 }
509}
510
511void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
512{
513 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
514 auto jumpToExceptionStub = jit.jump();
515
516 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
517 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
518 });
519}
520
521Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
522{
523 auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
524 Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
525 m_constantInsertionValues.insertValue(0, result);
526 return result;
527 });
528 return result.iterator->value;
529}
530
531void B3IRGenerator::insertConstants()
532{
533 m_constantInsertionValues.execute(m_proc.at(0));
534}
535
536auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
537{
538 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
539 totalBytesChecked += m_locals.size();
540 uint32_t totalBytes;
541 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
542
543 for (uint32_t i = 0; i < count; ++i) {
544 Variable* local = m_proc.addVariable(toB3Type(type));
545 m_locals.uncheckedAppend(local);
546 auto val = isSubtype(type, Anyref) ? JSValue::encode(jsNull()) : 0;
547 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), val, Origin()));
548 }
549 return { };
550}
551
552auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
553{
554 ASSERT(!m_locals.size());
555 WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
556
557 m_locals.grow(signature.argumentCount());
558 wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
559 [=] (ExpressionType argument, unsigned i) {
560 Variable* argumentVariable = m_proc.addVariable(argument->type());
561 m_locals[i] = argumentVariable;
562 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
563 });
564 return { };
565}
566
567auto B3IRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
568{
569 result = m_currentBlock->appendNew<Value>(m_proc, B3::Equal, origin(), value, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), JSValue::encode(jsNull())));
570 return { };
571}
572
573auto B3IRGenerator::addTableGet(unsigned tableIndex, ExpressionType& index, ExpressionType& result) -> PartialResult
574{
575 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
576 result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(Anyref), origin(),
577 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&getWasmTableElement, B3CCallPtrTag)),
578 instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), index);
579
580 {
581 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
582 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0)));
583
584 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
585 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
586 });
587 }
588
589 return { };
590}
591
592auto B3IRGenerator::addTableSet(unsigned tableIndex, ExpressionType& index, ExpressionType& value) -> PartialResult
593{
594 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
595 auto shouldThrow = m_currentBlock->appendNew<CCallValue>(m_proc, B3::Int32, origin(),
596 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&setWasmTableElement, B3CCallPtrTag)),
597 instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), index, value);
598
599 {
600 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
601 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), shouldThrow, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
602
603 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
604 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
605 });
606 }
607
608 return { };
609}
610
611auto B3IRGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
612{
613 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
614
615 result = m_currentBlock->appendNew<CCallValue>(m_proc, B3::Int64, origin(),
616 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&doWasmRefFunc, B3CCallPtrTag)),
617 instanceValue(), addConstant(Type::I32, index));
618
619 return { };
620}
621
622auto B3IRGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
623{
624 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
625 uint32_t (*doSize)(Instance*, unsigned) = [] (Instance* instance, unsigned tableIndex) -> uint32_t {
626 return instance->table(tableIndex)->length();
627 };
628
629 result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
630 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(doSize, B3CCallPtrTag)),
631 instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex));
632
633 return { };
634}
635
636auto B3IRGenerator::addTableGrow(unsigned tableIndex, ExpressionType& fill, ExpressionType& delta, ExpressionType& result) -> PartialResult
637{
638 result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
639 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&doWasmTableGrow, B3CCallPtrTag)),
640 instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), fill, delta);
641
642 return { };
643}
644
645auto B3IRGenerator::addTableFill(unsigned tableIndex, ExpressionType& offset, ExpressionType& fill, ExpressionType& count) -> PartialResult
646{
647 auto result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(I32), origin(),
648 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(&doWasmTableFill, B3CCallPtrTag)),
649 instanceValue(), m_currentBlock->appendNew<Const32Value>(m_proc, origin(), tableIndex), offset, fill, count);
650
651 {
652 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
653 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), result, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), 0)));
654
655 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
656 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTableAccess);
657 });
658 }
659
660 return { };
661}
662
663auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
664{
665 ASSERT(m_locals[index]);
666 result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
667 return { };
668}
669
670auto B3IRGenerator::addUnreachable() -> PartialResult
671{
672 B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
673 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
674 this->emitExceptionCheck(jit, ExceptionType::Unreachable);
675 });
676 unreachable->effects.terminal = true;
677 return { };
678}
679
680auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
681{
682 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
683 instance->storeTopCallFrame(callFrame);
684
685 if (delta < 0)
686 return -1;
687
688 auto grown = instance->memory()->grow(PageCount(delta));
689 if (!grown) {
690 switch (grown.error()) {
691 case Memory::GrowFailReason::InvalidDelta:
692 case Memory::GrowFailReason::InvalidGrowSize:
693 case Memory::GrowFailReason::WouldExceedMaximum:
694 case Memory::GrowFailReason::OutOfMemory:
695 return -1;
696 }
697 RELEASE_ASSERT_NOT_REACHED();
698 }
699
700 return grown.value().pageCount();
701 };
702
703 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
704 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
705 m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
706
707 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
708
709 return { };
710}
711
712auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
713{
714 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
715 Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
716
717 constexpr uint32_t shiftValue = 16;
718 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
719 Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
720 size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
721
722 result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
723
724 return { };
725}
726
727auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
728{
729 ASSERT(m_locals[index]);
730 m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
731 return { };
732}
733
734auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
735{
736 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
737 result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
738 return { };
739}
740
741auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
742{
743 ASSERT(toB3Type(m_info.globals[index].type) == value->type());
744 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
745 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
746
747 if (isSubtype(m_info.globals[index].type, Anyref))
748 emitWriteBarrierForJSWrapper();
749
750 return { };
751}
752
753inline void B3IRGenerator::emitWriteBarrierForJSWrapper()
754{
755 Value* cell = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfOwner()));
756 Value* cellState = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
757 Value* vm = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), cell, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfVM()));
758 Value* threshold = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapBarrierThreshold()));
759
760 BasicBlock* fenceCheckPath = m_proc.addBlock();
761 BasicBlock* fencePath = m_proc.addBlock();
762 BasicBlock* doSlowPath = m_proc.addBlock();
763 BasicBlock* continuation = m_proc.addBlock();
764
765 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
766 m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellState, threshold),
767 FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare));
768 fenceCheckPath->addPredecessor(m_currentBlock);
769 continuation->addPredecessor(m_currentBlock);
770 m_currentBlock = fenceCheckPath;
771
772 Value* shouldFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapMutatorShouldBeFenced()));
773 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
774 shouldFence,
775 FrequentedBlock(fencePath), FrequentedBlock(doSlowPath));
776 fencePath->addPredecessor(m_currentBlock);
777 doSlowPath->addPredecessor(m_currentBlock);
778 m_currentBlock = fencePath;
779
780 B3::PatchpointValue* doFence = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
781 doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
782 jit.memoryFence();
783 });
784
785 Value* cellStateLoadAfterFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
786 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
787 m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), blackThreshold)),
788 FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare));
789 doSlowPath->addPredecessor(m_currentBlock);
790 continuation->addPredecessor(m_currentBlock);
791 m_currentBlock = doSlowPath;
792
793 void (*writeBarrier)(JSWebAssemblyInstance*, VM*) = [] (JSWebAssemblyInstance* cell, VM* vm) -> void {
794 vm->heap.writeBarrierSlowPath(cell);
795 };
796
797 Value* writeBarrierAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(writeBarrier, B3CCallPtrTag));
798 m_currentBlock->appendNew<CCallValue>(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm);
799 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
800
801 continuation->addPredecessor(m_currentBlock);
802 m_currentBlock = continuation;
803}
804
805inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
806{
807 ASSERT(m_memoryBaseGPR);
808
809 switch (m_mode) {
810 case MemoryMode::BoundsChecking: {
811 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
812 ASSERT(m_memorySizeGPR);
813 ASSERT(sizeOfOperation + offset > offset);
814 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
815 break;
816 }
817
818 case MemoryMode::Signaling: {
819 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
820 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
821 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
822 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
823 // register + large offset access. We don't think this will be generated frequently.
824 //
825 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
826 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
827 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
828 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
829 if (offset >= Memory::fastMappedRedzoneBytes()) {
830 size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
831 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
832 }
833 break;
834 }
835 }
836
837 pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
838 return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
839}
840
841inline uint32_t sizeOfLoadOp(LoadOpType op)
842{
843 switch (op) {
844 case LoadOpType::I32Load8S:
845 case LoadOpType::I32Load8U:
846 case LoadOpType::I64Load8S:
847 case LoadOpType::I64Load8U:
848 return 1;
849 case LoadOpType::I32Load16S:
850 case LoadOpType::I64Load16S:
851 case LoadOpType::I32Load16U:
852 case LoadOpType::I64Load16U:
853 return 2;
854 case LoadOpType::I32Load:
855 case LoadOpType::I64Load32S:
856 case LoadOpType::I64Load32U:
857 case LoadOpType::F32Load:
858 return 4;
859 case LoadOpType::I64Load:
860 case LoadOpType::F64Load:
861 return 8;
862 }
863 RELEASE_ASSERT_NOT_REACHED();
864}
865
866inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
867{
868 if (m_mode == MemoryMode::Signaling)
869 return trapping(memoryOp);
870 return memoryOp;
871}
872
873inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
874{
875 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
876
877 switch (op) {
878 case LoadOpType::I32Load8S: {
879 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
880 }
881
882 case LoadOpType::I64Load8S: {
883 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
884 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
885 }
886
887 case LoadOpType::I32Load8U: {
888 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
889 }
890
891 case LoadOpType::I64Load8U: {
892 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
893 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
894 }
895
896 case LoadOpType::I32Load16S: {
897 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
898 }
899
900 case LoadOpType::I64Load16S: {
901 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
902 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
903 }
904
905 case LoadOpType::I32Load16U: {
906 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
907 }
908
909 case LoadOpType::I64Load16U: {
910 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
911 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
912 }
913
914 case LoadOpType::I32Load: {
915 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
916 }
917
918 case LoadOpType::I64Load32U: {
919 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
920 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
921 }
922
923 case LoadOpType::I64Load32S: {
924 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
925 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
926 }
927
928 case LoadOpType::I64Load: {
929 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
930 }
931
932 case LoadOpType::F32Load: {
933 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
934 }
935
936 case LoadOpType::F64Load: {
937 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
938 }
939 }
940 RELEASE_ASSERT_NOT_REACHED();
941}
942
943auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
944{
945 ASSERT(pointer->type() == Int32);
946
947 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
948 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
949 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
950 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
951 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
952 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
953 });
954
955 switch (op) {
956 case LoadOpType::I32Load8S:
957 case LoadOpType::I32Load16S:
958 case LoadOpType::I32Load:
959 case LoadOpType::I32Load16U:
960 case LoadOpType::I32Load8U:
961 result = constant(Int32, 0);
962 break;
963 case LoadOpType::I64Load8S:
964 case LoadOpType::I64Load8U:
965 case LoadOpType::I64Load16S:
966 case LoadOpType::I64Load32U:
967 case LoadOpType::I64Load32S:
968 case LoadOpType::I64Load:
969 case LoadOpType::I64Load16U:
970 result = constant(Int64, 0);
971 break;
972 case LoadOpType::F32Load:
973 result = constant(Float, 0);
974 break;
975 case LoadOpType::F64Load:
976 result = constant(Double, 0);
977 break;
978 }
979
980 } else
981 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
982
983 return { };
984}
985
986inline uint32_t sizeOfStoreOp(StoreOpType op)
987{
988 switch (op) {
989 case StoreOpType::I32Store8:
990 case StoreOpType::I64Store8:
991 return 1;
992 case StoreOpType::I32Store16:
993 case StoreOpType::I64Store16:
994 return 2;
995 case StoreOpType::I32Store:
996 case StoreOpType::I64Store32:
997 case StoreOpType::F32Store:
998 return 4;
999 case StoreOpType::I64Store:
1000 case StoreOpType::F64Store:
1001 return 8;
1002 }
1003 RELEASE_ASSERT_NOT_REACHED();
1004}
1005
1006
1007inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1008{
1009 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1010
1011 switch (op) {
1012 case StoreOpType::I64Store8:
1013 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1014 FALLTHROUGH;
1015
1016 case StoreOpType::I32Store8:
1017 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
1018 return;
1019
1020 case StoreOpType::I64Store16:
1021 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1022 FALLTHROUGH;
1023
1024 case StoreOpType::I32Store16:
1025 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
1026 return;
1027
1028 case StoreOpType::I64Store32:
1029 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
1030 FALLTHROUGH;
1031
1032 case StoreOpType::I64Store:
1033 case StoreOpType::I32Store:
1034 case StoreOpType::F32Store:
1035 case StoreOpType::F64Store:
1036 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
1037 return;
1038 }
1039 RELEASE_ASSERT_NOT_REACHED();
1040}
1041
1042auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1043{
1044 ASSERT(pointer->type() == Int32);
1045
1046 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1047 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1048 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1049 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
1050 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1051 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
1052 });
1053 } else
1054 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1055
1056 return { };
1057}
1058
1059auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1060{
1061 result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
1062 return { };
1063}
1064
1065B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
1066{
1067 return constant(toB3Type(type), value);
1068}
1069
1070void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
1071{
1072 if (!m_tierUp)
1073 return;
1074
1075 ASSERT(m_tierUp);
1076 Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
1077 Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
1078 Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
1079 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
1080
1081 PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
1082 Effects effects = Effects::none();
1083 // FIXME: we should have a more precise heap range for the tier up count.
1084 effects.reads = B3::HeapRange::top();
1085 effects.writes = B3::HeapRange::top();
1086 patch->effects = effects;
1087
1088 patch->append(newCountDown, ValueRep::SomeRegister);
1089 patch->append(oldCountDown, ValueRep::SomeRegister);
1090 patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1091 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1092 MacroAssembler::Label tierUpResume = jit.label();
1093
1094 params.addLatePath([=] (CCallHelpers& jit) {
1095 tierUp.link(&jit);
1096
1097 const unsigned extraPaddingBytes = 0;
1098 RegisterSet registersToSpill = { };
1099 registersToSpill.add(GPRInfo::argumentGPR1);
1100 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1101
1102 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1103 MacroAssembler::Call call = jit.nearCall();
1104
1105 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1106 jit.jump(tierUpResume);
1107
1108 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1109 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1110
1111 });
1112 });
1113 });
1114}
1115
1116B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
1117{
1118 BasicBlock* body = m_proc.addBlock();
1119 BasicBlock* continuation = m_proc.addBlock();
1120
1121 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
1122
1123 m_currentBlock = body;
1124 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1125
1126 return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
1127}
1128
1129B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
1130{
1131 return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
1132}
1133
1134B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
1135{
1136 return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
1137}
1138
1139auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1140{
1141 // FIXME: This needs to do some kind of stack passing.
1142
1143 BasicBlock* taken = m_proc.addBlock();
1144 BasicBlock* notTaken = m_proc.addBlock();
1145 BasicBlock* continuation = m_proc.addBlock();
1146
1147 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1148 m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
1149 taken->addPredecessor(m_currentBlock);
1150 notTaken->addPredecessor(m_currentBlock);
1151
1152 m_currentBlock = taken;
1153 result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
1154 return { };
1155}
1156
1157auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1158{
1159 unifyValuesWithBlock(currentStack, data.result);
1160 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1161 return addElseToUnreachable(data);
1162}
1163
1164auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1165{
1166 ASSERT(data.type() == BlockType::If);
1167 m_currentBlock = data.special;
1168 data.convertIfToBlock();
1169 return { };
1170}
1171
1172auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
1173{
1174 ASSERT(returnValues.size() <= 1);
1175 if (returnValues.size())
1176 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1177 else
1178 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1179 return { };
1180}
1181
1182auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1183{
1184 unifyValuesWithBlock(returnValues, data.resultForBranch());
1185
1186 BasicBlock* target = data.targetBlockForBranch();
1187 if (condition) {
1188 BasicBlock* continuation = m_proc.addBlock();
1189 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1190 m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1191 target->addPredecessor(m_currentBlock);
1192 continuation->addPredecessor(m_currentBlock);
1193 m_currentBlock = continuation;
1194 } else {
1195 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1196 target->addPredecessor(m_currentBlock);
1197 }
1198
1199 return { };
1200}
1201
1202auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1203{
1204 for (size_t i = 0; i < targets.size(); ++i)
1205 unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1206 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1207
1208 SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1209 switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1210 for (size_t i = 0; i < targets.size(); ++i)
1211 switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1212
1213 return { };
1214}
1215
1216auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1217{
1218 ControlData& data = entry.controlData;
1219
1220 unifyValuesWithBlock(expressionStack, data.result);
1221 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1222 data.continuation->addPredecessor(m_currentBlock);
1223
1224 return addEndToUnreachable(entry);
1225}
1226
1227
1228auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1229{
1230 ControlData& data = entry.controlData;
1231 m_currentBlock = data.continuation;
1232
1233 if (data.type() == BlockType::If) {
1234 data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1235 m_currentBlock->addPredecessor(data.special);
1236 }
1237
1238 for (Value* result : data.result) {
1239 m_currentBlock->append(result);
1240 entry.enclosedExpressionStack.append(result);
1241 }
1242
1243 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1244 if (data.type() == BlockType::TopLevel)
1245 return addReturn(entry.controlData, entry.enclosedExpressionStack);
1246
1247 return { };
1248}
1249
1250auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1251{
1252 ASSERT(signature.argumentCount() == args.size());
1253
1254 m_makesCalls = true;
1255
1256 Type returnType = signature.returnType();
1257 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1258
1259 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1260 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1261
1262 // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1263 Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1264 // The target instance is 0 unless the call is wasm->wasm.
1265 Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1266
1267 BasicBlock* isWasmBlock = m_proc.addBlock();
1268 BasicBlock* isEmbedderBlock = m_proc.addBlock();
1269 BasicBlock* continuation = m_proc.addBlock();
1270 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1271
1272 Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1273 [=] (PatchpointValue* patchpoint) {
1274 patchpoint->effects.writesPinned = true;
1275 patchpoint->effects.readsPinned = true;
1276 // We need to clobber all potential pinned registers since we might be leaving the instance.
1277 // We pessimistically assume we could be calling to something that is bounds checking.
1278 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1279 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1280 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1281 AllowMacroScratchRegisterUsage allowScratch(jit);
1282 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1283 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1284 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1285 });
1286 });
1287 });
1288 UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1289 isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1290
1291 // FIXME: Let's remove this indirection by creating a PIC friendly IC
1292 // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1293 // implement the IC to be over Context*.
1294 // https://bugs.webkit.org/show_bug.cgi?id=170375
1295 Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1296 Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1297
1298 Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1299 [=] (PatchpointValue* patchpoint) {
1300 patchpoint->effects.writesPinned = true;
1301 patchpoint->effects.readsPinned = true;
1302 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1303 // We need to clobber all potential pinned registers since we might be leaving the instance.
1304 // We pessimistically assume we could be calling to something that is bounds checking.
1305 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1306 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1307 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1308 AllowMacroScratchRegisterUsage allowScratch(jit);
1309 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1310 });
1311 });
1312 UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1313 isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1314
1315 m_currentBlock = continuation;
1316
1317 if (returnType == Void)
1318 result = nullptr;
1319 else {
1320 result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1321 wasmCallResultUpsilon->setPhi(result);
1322 embedderCallResultUpsilon->setPhi(result);
1323 }
1324
1325 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1326 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1327 } else {
1328 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1329 [=] (PatchpointValue* patchpoint) {
1330 patchpoint->effects.writesPinned = true;
1331 patchpoint->effects.readsPinned = true;
1332
1333 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1334 AllowMacroScratchRegisterUsage allowScratch(jit);
1335 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1336 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1337 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1338 });
1339 });
1340 });
1341 }
1342
1343 return { };
1344}
1345
1346auto B3IRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1347{
1348 ExpressionType calleeIndex = args.takeLast();
1349 ASSERT(signature.argumentCount() == args.size());
1350
1351 m_makesCalls = true;
1352 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1353 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1354 // can be to the embedder for our stack check calculation.
1355 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1356
1357 ExpressionType callableFunctionBuffer;
1358 ExpressionType instancesBuffer;
1359 ExpressionType callableFunctionBufferLength;
1360 ExpressionType mask;
1361 {
1362 ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1363 instanceValue(), safeCast<int32_t>(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)));
1364 callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1365 table, safeCast<int32_t>(FuncRefTable::offsetOfFunctions()));
1366 instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1367 table, safeCast<int32_t>(FuncRefTable::offsetOfInstances()));
1368 callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1369 table, safeCast<int32_t>(Table::offsetOfLength()));
1370 mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1371 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1372 table, safeCast<int32_t>(Table::offsetOfMask())));
1373 }
1374
1375 // Check the index we are looking for is valid.
1376 {
1377 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1378 m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1379
1380 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1381 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1382 });
1383 }
1384
1385 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1386
1387 if (Options::enableSpectreMitigations())
1388 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1389
1390 ExpressionType callableFunction;
1391 {
1392 // Compute the offset in the table index space we are looking for.
1393 ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1394 calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1395 callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1396
1397 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1398 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1399 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1400 ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1401 {
1402 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1403 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1404 calleeSignatureIndex,
1405 m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1406
1407 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1408 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1409 });
1410 }
1411
1412 // Check the signature matches the value we expect.
1413 {
1414 ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1415 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1416 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1417
1418 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1419 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1420 });
1421 }
1422 }
1423
1424 // Do a context switch if needed.
1425 {
1426 Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1427 calleeIndex, constant(pointerType(), sizeof(Instance*)));
1428 Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1429 m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1430
1431 BasicBlock* continuation = m_proc.addBlock();
1432 BasicBlock* doContextSwitch = m_proc.addBlock();
1433
1434 Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1435 newContextInstance, instanceValue());
1436 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1437 isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1438
1439 PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1440 patchpoint->effects.writesPinned = true;
1441 // We pessimistically assume we're calling something with BoundsChecking memory.
1442 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1443 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1444 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1445 patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1446 patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1447 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
1448
1449 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1450 AllowMacroScratchRegisterUsage allowScratch(jit);
1451 GPRReg newContextInstance = params[0].gpr();
1452 GPRReg oldContextInstance = params[1].gpr();
1453 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1454 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1455 ASSERT(newContextInstance != baseMemory);
1456 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1457 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1458 jit.storeWasmContextInstance(newContextInstance);
1459 ASSERT(pinnedRegs.sizeRegister != baseMemory);
1460 // FIXME: We should support more than one memory size register
1461 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1462 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1463 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
1464
1465 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1466 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1467
1468 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.sizeRegister, scratchOrSize);
1469 });
1470 doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1471
1472 m_currentBlock = continuation;
1473 }
1474
1475 ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1476 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1477 safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1478
1479 Type returnType = signature.returnType();
1480 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1481 [=] (PatchpointValue* patchpoint) {
1482 patchpoint->effects.writesPinned = true;
1483 patchpoint->effects.readsPinned = true;
1484 // We need to clobber all potential pinned registers since we might be leaving the instance.
1485 // We pessimistically assume we're always calling something that is bounds checking so
1486 // because the wasm->wasm thunk unconditionally overrides the size registers.
1487 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1488 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1489 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1490
1491 patchpoint->append(calleeCode, ValueRep::SomeRegister);
1492 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1493 AllowMacroScratchRegisterUsage allowScratch(jit);
1494 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1495 });
1496 });
1497
1498 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1499 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1500
1501 return { };
1502}
1503
1504void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1505{
1506 m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1507}
1508
1509void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1510{
1511 ASSERT(result.size() <= resultStack.size());
1512
1513 for (size_t i = 0; i < result.size(); ++i)
1514 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1515}
1516
1517static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1518{
1519 dataLog(comma, "ExpressionStack:");
1520 for (const auto& expression : expressionStack)
1521 dataLog(comma, *expression);
1522}
1523
1524void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1525{
1526 dataLogLn("Constants:");
1527 for (const auto& constant : m_constantPool)
1528 dataLogLn(deepDump(m_proc, constant.value));
1529
1530 dataLogLn("Processing Graph:");
1531 dataLog(m_proc);
1532 dataLogLn("With current block:", *m_currentBlock);
1533 dataLogLn("Control stack:");
1534 ASSERT(controlStack.size());
1535 for (size_t i = controlStack.size(); i--;) {
1536 dataLog(" ", controlStack[i].controlData, ": ");
1537 CommaPrinter comma(", ", "");
1538 dumpExpressionStack(comma, *expressionStack);
1539 expressionStack = &controlStack[i].enclosedExpressionStack;
1540 dataLogLn();
1541 }
1542 dataLogLn();
1543}
1544
1545auto B3IRGenerator::origin() -> Origin
1546{
1547 OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1548 ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1549 return bitwise_cast<Origin>(origin);
1550}
1551
1552Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1553{
1554 auto result = std::make_unique<InternalFunction>();
1555
1556 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1557 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1558
1559 Procedure procedure;
1560
1561 procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1562 if (origin.data())
1563 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1564 });
1565
1566 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1567 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1568 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1569 // optLevel=1.
1570 procedure.setNeedsUsedRegisters(false);
1571
1572 procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1573 ? Options::webAssemblyBBQOptimizationLevel()
1574 : Options::webAssemblyOMGOptimizationLevel());
1575
1576 B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1577 FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1578 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1579
1580 irGenerator.insertConstants();
1581
1582 procedure.resetReachability();
1583 if (!ASSERT_DISABLED)
1584 validate(procedure, "After parsing:\n");
1585
1586 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1587 fixSSA(procedure);
1588 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1589
1590 {
1591 B3::prepareForGeneration(procedure);
1592 B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1593 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1594 result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1595 }
1596
1597 return result;
1598}
1599
1600// Custom wasm ops. These are the ones too messy to do in wasm.json.
1601
1602void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1603{
1604 ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1605 const B3::Type type = left->type();
1606
1607 {
1608 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1609 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1610
1611 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1612 this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1613 });
1614 }
1615
1616 if (operation == Div) {
1617 int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1618
1619 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1620 m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1621 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1622 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1623
1624 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1625 this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1626 });
1627 }
1628}
1629
1630template<>
1631auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1632{
1633 const B3::Opcode op = Div;
1634 emitChecksForModOrDiv(op, left, right);
1635 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1636 return { };
1637}
1638
1639template<>
1640auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1641{
1642 const B3::Opcode op = Mod;
1643 emitChecksForModOrDiv(op, left, right);
1644 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1645 return { };
1646}
1647
1648template<>
1649auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1650{
1651 const B3::Opcode op = UDiv;
1652 emitChecksForModOrDiv(op, left, right);
1653 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1654 return { };
1655}
1656
1657template<>
1658auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1659{
1660 const B3::Opcode op = UMod;
1661 emitChecksForModOrDiv(op, left, right);
1662 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1663 return { };
1664}
1665
1666template<>
1667auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1668{
1669 const B3::Opcode op = Div;
1670 emitChecksForModOrDiv(op, left, right);
1671 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1672 return { };
1673}
1674
1675template<>
1676auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1677{
1678 const B3::Opcode op = Mod;
1679 emitChecksForModOrDiv(op, left, right);
1680 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1681 return { };
1682}
1683
1684template<>
1685auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1686{
1687 const B3::Opcode op = UDiv;
1688 emitChecksForModOrDiv(op, left, right);
1689 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1690 return { };
1691}
1692
1693template<>
1694auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1695{
1696 const B3::Opcode op = UMod;
1697 emitChecksForModOrDiv(op, left, right);
1698 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1699 return { };
1700}
1701
1702template<>
1703auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1704{
1705 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1706 patchpoint->append(arg, ValueRep::SomeRegister);
1707 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1708 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1709 });
1710 patchpoint->effects = Effects::none();
1711 result = patchpoint;
1712 return { };
1713}
1714
1715template<>
1716auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1717{
1718 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1719 patchpoint->append(arg, ValueRep::SomeRegister);
1720 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1721 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1722 });
1723 patchpoint->effects = Effects::none();
1724 result = patchpoint;
1725 return { };
1726}
1727
1728template<>
1729auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1730{
1731#if CPU(X86_64)
1732 if (MacroAssembler::supportsCountPopulation()) {
1733 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1734 patchpoint->append(arg, ValueRep::SomeRegister);
1735 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1736 jit.countPopulation32(params[1].gpr(), params[0].gpr());
1737 });
1738 patchpoint->effects = Effects::none();
1739 result = patchpoint;
1740 return { };
1741 }
1742#endif
1743
1744 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1745 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1746 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1747 return { };
1748}
1749
1750template<>
1751auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1752{
1753#if CPU(X86_64)
1754 if (MacroAssembler::supportsCountPopulation()) {
1755 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1756 patchpoint->append(arg, ValueRep::SomeRegister);
1757 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1758 jit.countPopulation64(params[1].gpr(), params[0].gpr());
1759 });
1760 patchpoint->effects = Effects::none();
1761 result = patchpoint;
1762 return { };
1763 }
1764#endif
1765
1766 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1767 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1768 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1769 return { };
1770}
1771
1772template<>
1773auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1774{
1775 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1776 if (isX86())
1777 patchpoint->numGPScratchRegisters = 1;
1778 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1779 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1780 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1781 AllowMacroScratchRegisterUsage allowScratch(jit);
1782#if CPU(X86_64)
1783 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1784#else
1785 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1786#endif
1787 });
1788 patchpoint->effects = Effects::none();
1789 result = patchpoint;
1790 return { };
1791}
1792
1793template<>
1794auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1795{
1796 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1797 if (isX86())
1798 patchpoint->numGPScratchRegisters = 1;
1799 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1800 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1801 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1802 AllowMacroScratchRegisterUsage allowScratch(jit);
1803#if CPU(X86_64)
1804 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1805#else
1806 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1807#endif
1808 });
1809 patchpoint->effects = Effects::none();
1810 result = patchpoint;
1811 return { };
1812}
1813
1814template<>
1815auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1816{
1817 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1818 patchpoint->append(arg, ValueRep::SomeRegister);
1819 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1820 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1821 });
1822 patchpoint->effects = Effects::none();
1823 result = patchpoint;
1824 return { };
1825}
1826
1827template<>
1828auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1829{
1830 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1831 patchpoint->append(arg, ValueRep::SomeRegister);
1832 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1833 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1834 });
1835 patchpoint->effects = Effects::none();
1836 result = patchpoint;
1837 return { };
1838}
1839
1840template<>
1841auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1842{
1843 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1844 patchpoint->append(arg, ValueRep::SomeRegister);
1845 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1846 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1847 });
1848 patchpoint->effects = Effects::none();
1849 result = patchpoint;
1850 return { };
1851}
1852
1853template<>
1854auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1855{
1856 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1857 patchpoint->append(arg, ValueRep::SomeRegister);
1858 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1859 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1860 });
1861 patchpoint->effects = Effects::none();
1862 result = patchpoint;
1863 return { };
1864}
1865
1866template<>
1867auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1868{
1869 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1870 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1871 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1872 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1873 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1874 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1875 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1876 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1877 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1878 });
1879 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1880 patchpoint->append(arg, ValueRep::SomeRegister);
1881 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1882 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1883 });
1884 patchpoint->effects = Effects::none();
1885 result = patchpoint;
1886 return { };
1887}
1888
1889template<>
1890auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1891{
1892 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1893 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1894 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1895 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1896 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1897 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1898 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1899 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1900 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1901 });
1902 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1903 patchpoint->append(arg, ValueRep::SomeRegister);
1904 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1905 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1906 });
1907 patchpoint->effects = Effects::none();
1908 result = patchpoint;
1909 return { };
1910}
1911
1912
1913template<>
1914auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1915{
1916 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1917 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1918 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1919 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1920 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1921 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1922 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1923 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1924 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1925 });
1926 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1927 patchpoint->append(arg, ValueRep::SomeRegister);
1928 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1929 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1930 });
1931 patchpoint->effects = Effects::none();
1932 result = patchpoint;
1933 return { };
1934}
1935
1936template<>
1937auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1938{
1939 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1940 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1941 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1942 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1943 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1944 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1945 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1946 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1947 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1948 });
1949 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1950 patchpoint->append(arg, ValueRep::SomeRegister);
1951 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1952 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1953 });
1954 patchpoint->effects = Effects::none();
1955 result = patchpoint;
1956 return { };
1957}
1958
1959template<>
1960auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1961{
1962 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1963 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1964 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1965 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1966 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1967 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1968 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1969 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1970 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1971 });
1972 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1973 patchpoint->append(arg, ValueRep::SomeRegister);
1974 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1975 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1976 });
1977 patchpoint->effects = Effects::none();
1978 result = patchpoint;
1979 return { };
1980}
1981
1982template<>
1983auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1984{
1985 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1986 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1987 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1988 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1989 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1990 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1991 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1992 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1993 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1994 });
1995
1996 Value* signBitConstant;
1997 if (isX86()) {
1998 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1999 // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
2000 // so we can pool them if needed.
2001 signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2002 }
2003 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2004 patchpoint->append(arg, ValueRep::SomeRegister);
2005 if (isX86()) {
2006 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
2007 patchpoint->numFPScratchRegisters = 1;
2008 }
2009 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2010 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2011 AllowMacroScratchRegisterUsage allowScratch(jit);
2012 FPRReg scratch = InvalidFPRReg;
2013 FPRReg constant = InvalidFPRReg;
2014 if (isX86()) {
2015 scratch = params.fpScratch(0);
2016 constant = params[2].fpr();
2017 }
2018 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2019 });
2020 patchpoint->effects = Effects::none();
2021 result = patchpoint;
2022 return { };
2023}
2024
2025template<>
2026auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2027{
2028 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2029 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2030 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2031 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2032 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
2033 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2034 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2035 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2036 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2037 });
2038 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2039 patchpoint->append(arg, ValueRep::SomeRegister);
2040 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2041 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2042 });
2043 patchpoint->effects = Effects::none();
2044 result = patchpoint;
2045 return { };
2046}
2047
2048template<>
2049auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2050{
2051 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2052 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2053 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
2054 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
2055 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
2056 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
2057 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
2058 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
2059 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
2060 });
2061
2062 Value* signBitConstant;
2063 if (isX86()) {
2064 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
2065 // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
2066 // so we can pool them if needed.
2067 signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2068 }
2069 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2070 patchpoint->append(arg, ValueRep::SomeRegister);
2071 if (isX86()) {
2072 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
2073 patchpoint->numFPScratchRegisters = 1;
2074 }
2075 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2076 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2077 AllowMacroScratchRegisterUsage allowScratch(jit);
2078 FPRReg scratch = InvalidFPRReg;
2079 FPRReg constant = InvalidFPRReg;
2080 if (isX86()) {
2081 scratch = params.fpScratch(0);
2082 constant = params[2].fpr();
2083 }
2084 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2085 });
2086 patchpoint->effects = Effects::none();
2087 result = patchpoint;
2088 return { };
2089}
2090
2091} } // namespace JSC::Wasm
2092
2093#include "WasmB3IRGeneratorInlines.h"
2094
2095#endif // ENABLE(WEBASSEMBLY)
2096