1/*
2 * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmB3IRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AllowMacroScratchRegisterUsageIf.h"
32#include "B3BasicBlockInlines.h"
33#include "B3CCallValue.h"
34#include "B3Compile.h"
35#include "B3ConstPtrValue.h"
36#include "B3FixSSA.h"
37#include "B3Generate.h"
38#include "B3InsertionSet.h"
39#include "B3SlotBaseValue.h"
40#include "B3StackmapGenerationParams.h"
41#include "B3SwitchValue.h"
42#include "B3UpsilonValue.h"
43#include "B3Validate.h"
44#include "B3ValueInlines.h"
45#include "B3ValueKey.h"
46#include "B3Variable.h"
47#include "B3VariableValue.h"
48#include "B3WasmAddressValue.h"
49#include "B3WasmBoundsCheckValue.h"
50#include "DisallowMacroScratchRegisterUsage.h"
51#include "JSCInlines.h"
52#include "JSWebAssemblyInstance.h"
53#include "ScratchRegisterAllocator.h"
54#include "VirtualRegister.h"
55#include "WasmCallingConvention.h"
56#include "WasmContextInlines.h"
57#include "WasmExceptionType.h"
58#include "WasmFunctionParser.h"
59#include "WasmInstance.h"
60#include "WasmMemory.h"
61#include "WasmOMGPlan.h"
62#include "WasmOpcodeOrigin.h"
63#include "WasmSignatureInlines.h"
64#include "WasmThunks.h"
65#include <limits>
66#include <wtf/Optional.h>
67#include <wtf/StdLibExtras.h>
68
69void dumpProcedure(void* ptr)
70{
71 JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
72 proc->dump(WTF::dataFile());
73}
74
75namespace JSC { namespace Wasm {
76
77using namespace B3;
78
79namespace {
80namespace WasmB3IRGeneratorInternal {
81static const bool verbose = false;
82}
83}
84
85class B3IRGenerator {
86public:
87 struct ControlData {
88 ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
89 : blockType(type)
90 , continuation(continuation)
91 , special(special)
92 {
93 if (signature != Void)
94 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
95 }
96
97 ControlData()
98 {
99 }
100
101 void dump(PrintStream& out) const
102 {
103 switch (type()) {
104 case BlockType::If:
105 out.print("If: ");
106 break;
107 case BlockType::Block:
108 out.print("Block: ");
109 break;
110 case BlockType::Loop:
111 out.print("Loop: ");
112 break;
113 case BlockType::TopLevel:
114 out.print("TopLevel: ");
115 break;
116 }
117 out.print("Continuation: ", *continuation, ", Special: ");
118 if (special)
119 out.print(*special);
120 else
121 out.print("None");
122 }
123
124 BlockType type() const { return blockType; }
125
126 bool hasNonVoidSignature() const { return result.size(); }
127
128 BasicBlock* targetBlockForBranch()
129 {
130 if (type() == BlockType::Loop)
131 return special;
132 return continuation;
133 }
134
135 void convertIfToBlock()
136 {
137 ASSERT(type() == BlockType::If);
138 blockType = BlockType::Block;
139 special = nullptr;
140 }
141
142 using ResultList = Vector<Value*, 1>; // Value must be a Phi
143
144 ResultList resultForBranch() const
145 {
146 if (type() == BlockType::Loop)
147 return ResultList();
148 return result;
149 }
150
151 private:
152 friend class B3IRGenerator;
153 BlockType blockType;
154 BasicBlock* continuation;
155 BasicBlock* special;
156 ResultList result;
157 };
158
159 typedef Value* ExpressionType;
160 typedef ControlData ControlType;
161 typedef Vector<ExpressionType, 1> ExpressionList;
162 typedef ControlData::ResultList ResultList;
163 typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
164
165 static constexpr ExpressionType emptyExpression() { return nullptr; }
166
167 typedef String ErrorType;
168 typedef Unexpected<ErrorType> UnexpectedResult;
169 typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
170 typedef Expected<void, ErrorType> PartialResult;
171 template <typename ...Args>
172 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
173 {
174 using namespace FailureHelper; // See ADL comment in WasmParser.h.
175 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
176 }
177#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
178 if (UNLIKELY(condition)) \
179 return fail(__VA_ARGS__); \
180 } while (0)
181
182 B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
183
184 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
185 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
186 ExpressionType addConstant(Type, uint64_t);
187
188 // References
189 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
190
191 // Tables
192 PartialResult WARN_UNUSED_RETURN addTableGet(ExpressionType& idx, ExpressionType& result);
193 PartialResult WARN_UNUSED_RETURN addTableSet(ExpressionType& idx, ExpressionType& value);
194
195 // Locals
196 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
197 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
198
199 // Globals
200 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
201 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
202
203 // Memory
204 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
205 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
206 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
207 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
208
209 // Basic operators
210 template<OpType>
211 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
212 template<OpType>
213 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
214 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
215
216 // Control flow
217 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
218 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
219 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
220 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
221 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
222 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
223
224 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
225 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
226 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
227 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
228 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
229
230 // Calls
231 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
232 PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
233 PartialResult WARN_UNUSED_RETURN addUnreachable();
234
235 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
236 void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
237
238 Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
239 void insertConstants();
240
241 ALWAYS_INLINE void didKill(ExpressionType) { }
242
243private:
244 void emitExceptionCheck(CCallHelpers&, ExceptionType);
245
246 void emitTierUpCheck(uint32_t decrementCount, Origin);
247
248 void emitWriteBarrierForJSWrapper();
249 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
250 B3::Kind memoryKind(B3::Opcode memoryOp);
251 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
252 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
253
254 void unify(const ExpressionType phi, const ExpressionType source);
255 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
256
257 void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
258
259 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
260
261 void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
262 enum class RestoreCachedStackLimit { No, Yes };
263 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
264
265 Origin origin();
266
267 FunctionParser<B3IRGenerator>* m_parser { nullptr };
268 const ModuleInformation& m_info;
269 const MemoryMode m_mode { MemoryMode::BoundsChecking };
270 const CompilationMode m_compilationMode { CompilationMode::BBQMode };
271 const unsigned m_functionIndex { UINT_MAX };
272 const TierUpCount* m_tierUp { nullptr };
273
274 Procedure& m_proc;
275 BasicBlock* m_currentBlock { nullptr };
276 Vector<Variable*> m_locals;
277 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
278 HashMap<ValueKey, Value*> m_constantPool;
279 InsertionSet m_constantInsertionValues;
280 GPRReg m_memoryBaseGPR { InvalidGPRReg };
281 GPRReg m_memorySizeGPR { InvalidGPRReg };
282 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
283 bool m_makesCalls { false };
284
285 Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
286 bool m_usesInstanceValue { false };
287 Value* instanceValue()
288 {
289 m_usesInstanceValue = true;
290 return m_instanceValue;
291 }
292
293 uint32_t m_maxNumJSCallArguments { 0 };
294};
295
296// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
297int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
298{
299 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
300 ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
301 return 0;
302 }
303 return offset;
304}
305
306void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
307{
308 if (Context::useFastTLS()) {
309 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
310 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
311 patchpoint->clobber(RegisterSet::macroScratchRegisters());
312 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
313 patchpoint->setGenerator(
314 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
315 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
316 jit.storeWasmContextInstance(params[0].gpr());
317 });
318 return;
319 }
320
321 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
322 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
323 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
324 Effects effects = Effects::none();
325 effects.writesPinned = true;
326 effects.reads = B3::HeapRange::top();
327 patchpoint->effects = effects;
328 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
329 patchpoint->append(arg, ValueRep::SomeRegister);
330 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
331 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
332 jit.move(param[0].gpr(), wasmContextInstanceGPR);
333 });
334}
335
336B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
337 : m_info(info)
338 , m_mode(mode)
339 , m_compilationMode(compilationMode)
340 , m_functionIndex(functionIndex)
341 , m_tierUp(tierUp)
342 , m_proc(procedure)
343 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
344 , m_constantInsertionValues(m_proc)
345{
346 m_currentBlock = m_proc.addBlock();
347
348 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
349 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
350
351 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
352 m_proc.pinRegister(m_memoryBaseGPR);
353
354 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
355 if (!Context::useFastTLS())
356 m_proc.pinRegister(m_wasmContextInstanceGPR);
357
358 if (mode != MemoryMode::Signaling) {
359 m_memorySizeGPR = pinnedRegs.sizeRegister;
360 m_proc.pinRegister(m_memorySizeGPR);
361 }
362
363 if (throwWasmException)
364 Thunks::singleton().setThrowWasmException(throwWasmException);
365
366 if (info.memory) {
367 m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
368 AllowMacroScratchRegisterUsage allowScratch(jit);
369 switch (m_mode) {
370 case MemoryMode::BoundsChecking:
371 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
372 break;
373 case MemoryMode::Signaling:
374 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
375 break;
376 }
377 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
378 });
379
380 switch (m_mode) {
381 case MemoryMode::BoundsChecking:
382 break;
383 case MemoryMode::Signaling:
384 // Most memory accesses in signaling mode don't do an explicit
385 // exception check because they can rely on fault handling to detect
386 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
387 // thunk to exist so that it can jump to that thunk.
388 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
389 CRASH();
390 break;
391 }
392 }
393
394 wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
395
396 {
397 B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
398 B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
399 m_instanceValue = stackOverflowCheck;
400 stackOverflowCheck->appendSomeRegister(framePointer);
401 stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
402 if (!Context::useFastTLS()) {
403 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
404 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
405 stackOverflowCheck->effects.writesPinned = false;
406 stackOverflowCheck->effects.readsPinned = true;
407 stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
408 }
409 stackOverflowCheck->numGPScratchRegisters = 2;
410 stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
411 const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
412 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
413 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
414 // This allows us to elide stack checks for functions that are terminal nodes in the call
415 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
416 // having any such terminal node have its parent caller include some extra size in its
417 // own check for it. The goal here is twofold:
418 // 1. Emit less code.
419 // 2. Try to speed things up by skipping stack checks.
420 minimumParentCheckSize,
421 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
422 // spill all arguments to the stack, we ensure that a stack check here covers the
423 // stack that such a stub would use.
424 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
425 ));
426 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
427 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
428 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
429
430 GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
431
432 // This allows leaf functions to not do stack checks if their frame size is within
433 // certain limits since their caller would have already done the check.
434 if (needsOverflowCheck) {
435 AllowMacroScratchRegisterUsage allowScratch(jit);
436 GPRReg fp = params[1].gpr();
437 GPRReg scratch1 = params.gpScratch(0);
438 GPRReg scratch2 = params.gpScratch(1);
439
440 if (Context::useFastTLS())
441 jit.loadWasmContextInstance(contextInstance);
442
443 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
444 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
445 MacroAssembler::JumpList overflow;
446 if (UNLIKELY(needUnderflowCheck))
447 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
448 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
449 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
450 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
451 });
452 } else if (m_usesInstanceValue && Context::useFastTLS()) {
453 // No overflow check is needed, but the instance values still needs to be correct.
454 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
455 jit.loadWasmContextInstance(contextInstance);
456 } else {
457 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
458 }
459 });
460 }
461
462 emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
463}
464
465void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
466{
467 restoreWasmContextInstance(proc, block, instance);
468
469 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
470 // The Instance caches the stack limit, but also knows where its canonical location is.
471 Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
472 Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
473 block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
474 }
475
476 if (!!memory) {
477 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
478 RegisterSet clobbers;
479 clobbers.set(pinnedRegs->baseMemoryPointer);
480 clobbers.set(pinnedRegs->sizeRegister);
481 if (!isARM64())
482 clobbers.set(RegisterSet::macroScratchRegisters());
483
484 B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
485 Effects effects = Effects::none();
486 effects.writesPinned = true;
487 effects.reads = B3::HeapRange::top();
488 patchpoint->effects = effects;
489 patchpoint->clobber(clobbers);
490 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
491
492 patchpoint->append(instance, ValueRep::SomeRegister);
493 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
494 AllowMacroScratchRegisterUsage allowScratch(jit);
495 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
496 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
497
498 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
499 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
500
501 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->sizeRegister, scratchOrSize);
502 });
503 }
504}
505
506void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
507{
508 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
509 auto jumpToExceptionStub = jit.jump();
510
511 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
512 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
513 });
514}
515
516Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
517{
518 auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
519 Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
520 m_constantInsertionValues.insertValue(0, result);
521 return result;
522 });
523 return result.iterator->value;
524}
525
526void B3IRGenerator::insertConstants()
527{
528 m_constantInsertionValues.execute(m_proc.at(0));
529}
530
531auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
532{
533 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
534 totalBytesChecked += m_locals.size();
535 uint32_t totalBytes;
536 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
537
538 for (uint32_t i = 0; i < count; ++i) {
539 Variable* local = m_proc.addVariable(toB3Type(type));
540 m_locals.uncheckedAppend(local);
541 auto val = type == Anyref ? JSValue::encode(jsNull()) : 0;
542 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), val, Origin()));
543 }
544 return { };
545}
546
547auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
548{
549 ASSERT(!m_locals.size());
550 WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
551
552 m_locals.grow(signature.argumentCount());
553 wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
554 [=] (ExpressionType argument, unsigned i) {
555 Variable* argumentVariable = m_proc.addVariable(argument->type());
556 m_locals[i] = argumentVariable;
557 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
558 });
559 return { };
560}
561
562auto B3IRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
563{
564 result = m_currentBlock->appendNew<Value>(m_proc, B3::Equal, origin(), value, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), JSValue::encode(jsNull())));
565 return { };
566}
567
568auto B3IRGenerator::addTableGet(ExpressionType& idx, ExpressionType& result) -> PartialResult
569{
570 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
571 uint64_t (*doGet)(Instance*, int32_t) = [] (Instance* instance, int32_t idx) -> uint64_t {
572 return JSValue::encode(instance->table()->get(idx));
573 };
574
575 result = m_currentBlock->appendNew<CCallValue>(m_proc, toB3Type(Anyref), origin(),
576 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(doGet, B3CCallPtrTag)),
577 instanceValue(), idx);
578
579 return { };
580}
581
582auto B3IRGenerator::addTableSet(ExpressionType& idx, ExpressionType& value) -> PartialResult
583{
584 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
585 void (*doSet)(Instance*, int32_t, uint64_t value) = [] (Instance* instance, int32_t idx, uint64_t value) -> void {
586 // FIXME: We need to box wasm Funcrefs once they are supported here.
587 // <https://bugs.webkit.org/show_bug.cgi?id=198157>
588 instance->table()->set(idx, JSValue::decode(value));
589 };
590
591 m_currentBlock->appendNew<CCallValue>(m_proc, B3::Void, origin(),
592 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(doSet, B3CCallPtrTag)),
593 instanceValue(), idx, value);
594
595 return { };
596}
597
598auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
599{
600 ASSERT(m_locals[index]);
601 result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
602 return { };
603}
604
605auto B3IRGenerator::addUnreachable() -> PartialResult
606{
607 B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
608 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
609 this->emitExceptionCheck(jit, ExceptionType::Unreachable);
610 });
611 unreachable->effects.terminal = true;
612 return { };
613}
614
615auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
616{
617 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
618 instance->storeTopCallFrame(callFrame);
619
620 if (delta < 0)
621 return -1;
622
623 auto grown = instance->memory()->grow(PageCount(delta));
624 if (!grown) {
625 switch (grown.error()) {
626 case Memory::GrowFailReason::InvalidDelta:
627 case Memory::GrowFailReason::InvalidGrowSize:
628 case Memory::GrowFailReason::WouldExceedMaximum:
629 case Memory::GrowFailReason::OutOfMemory:
630 return -1;
631 }
632 RELEASE_ASSERT_NOT_REACHED();
633 }
634
635 return grown.value().pageCount();
636 };
637
638 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
639 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
640 m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
641
642 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
643
644 return { };
645}
646
647auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
648{
649 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
650 Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
651
652 constexpr uint32_t shiftValue = 16;
653 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
654 Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
655 size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
656
657 result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
658
659 return { };
660}
661
662auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
663{
664 ASSERT(m_locals[index]);
665 m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
666 return { };
667}
668
669auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
670{
671 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
672 result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
673 return { };
674}
675
676auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
677{
678 ASSERT(toB3Type(m_info.globals[index].type) == value->type());
679 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
680 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
681
682 if (m_info.globals[index].type == Anyref)
683 emitWriteBarrierForJSWrapper();
684
685 return { };
686}
687
688inline void B3IRGenerator::emitWriteBarrierForJSWrapper()
689{
690 Value* cell = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfOwner()));
691 Value* cellState = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
692 Value* vm = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), cell, safeCast<int32_t>(JSWebAssemblyInstance::offsetOfVM()));
693 Value* threshold = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapBarrierThreshold()));
694
695 BasicBlock* fenceCheckPath = m_proc.addBlock();
696 BasicBlock* fencePath = m_proc.addBlock();
697 BasicBlock* doSlowPath = m_proc.addBlock();
698 BasicBlock* continuation = m_proc.addBlock();
699
700 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
701 m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellState, threshold),
702 FrequentedBlock(continuation), FrequentedBlock(fenceCheckPath, FrequencyClass::Rare));
703 fenceCheckPath->addPredecessor(m_currentBlock);
704 continuation->addPredecessor(m_currentBlock);
705 m_currentBlock = fenceCheckPath;
706
707 Value* shouldFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), vm, safeCast<int32_t>(VM::offsetOfHeapMutatorShouldBeFenced()));
708 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
709 shouldFence,
710 FrequentedBlock(fencePath), FrequentedBlock(doSlowPath));
711 fencePath->addPredecessor(m_currentBlock);
712 doSlowPath->addPredecessor(m_currentBlock);
713 m_currentBlock = fencePath;
714
715 B3::PatchpointValue* doFence = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
716 doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
717 jit.memoryFence();
718 });
719
720 Value* cellStateLoadAfterFence = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, Int32, origin(), cell, safeCast<int32_t>(JSCell::cellStateOffset()));
721 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
722 m_currentBlock->appendNew<Value>(m_proc, Above, origin(), cellStateLoadAfterFence, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), blackThreshold)),
723 FrequentedBlock(continuation), FrequentedBlock(doSlowPath, FrequencyClass::Rare));
724 doSlowPath->addPredecessor(m_currentBlock);
725 continuation->addPredecessor(m_currentBlock);
726 m_currentBlock = doSlowPath;
727
728 void (*writeBarrier)(JSWebAssemblyInstance*, VM*) = [] (JSWebAssemblyInstance* cell, VM* vm) -> void {
729 vm->heap.writeBarrierSlowPath(cell);
730 };
731
732 Value* writeBarrierAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(writeBarrier, B3CCallPtrTag));
733 m_currentBlock->appendNew<CCallValue>(m_proc, B3::Void, origin(), writeBarrierAddress, cell, vm);
734 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
735
736 continuation->addPredecessor(m_currentBlock);
737 m_currentBlock = continuation;
738}
739
740inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
741{
742 ASSERT(m_memoryBaseGPR);
743
744 switch (m_mode) {
745 case MemoryMode::BoundsChecking: {
746 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
747 ASSERT(m_memorySizeGPR);
748 ASSERT(sizeOfOperation + offset > offset);
749 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
750 break;
751 }
752
753 case MemoryMode::Signaling: {
754 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
755 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
756 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
757 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
758 // register + large offset access. We don't think this will be generated frequently.
759 //
760 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
761 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
762 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
763 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
764 if (offset >= Memory::fastMappedRedzoneBytes()) {
765 size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
766 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
767 }
768 break;
769 }
770 }
771
772 pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
773 return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
774}
775
776inline uint32_t sizeOfLoadOp(LoadOpType op)
777{
778 switch (op) {
779 case LoadOpType::I32Load8S:
780 case LoadOpType::I32Load8U:
781 case LoadOpType::I64Load8S:
782 case LoadOpType::I64Load8U:
783 return 1;
784 case LoadOpType::I32Load16S:
785 case LoadOpType::I64Load16S:
786 case LoadOpType::I32Load16U:
787 case LoadOpType::I64Load16U:
788 return 2;
789 case LoadOpType::I32Load:
790 case LoadOpType::I64Load32S:
791 case LoadOpType::I64Load32U:
792 case LoadOpType::F32Load:
793 return 4;
794 case LoadOpType::I64Load:
795 case LoadOpType::F64Load:
796 return 8;
797 }
798 RELEASE_ASSERT_NOT_REACHED();
799}
800
801inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
802{
803 if (m_mode == MemoryMode::Signaling)
804 return trapping(memoryOp);
805 return memoryOp;
806}
807
808inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
809{
810 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
811
812 switch (op) {
813 case LoadOpType::I32Load8S: {
814 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
815 }
816
817 case LoadOpType::I64Load8S: {
818 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
819 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
820 }
821
822 case LoadOpType::I32Load8U: {
823 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
824 }
825
826 case LoadOpType::I64Load8U: {
827 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
828 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
829 }
830
831 case LoadOpType::I32Load16S: {
832 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
833 }
834
835 case LoadOpType::I64Load16S: {
836 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
837 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
838 }
839
840 case LoadOpType::I32Load16U: {
841 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
842 }
843
844 case LoadOpType::I64Load16U: {
845 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
846 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
847 }
848
849 case LoadOpType::I32Load: {
850 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
851 }
852
853 case LoadOpType::I64Load32U: {
854 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
855 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
856 }
857
858 case LoadOpType::I64Load32S: {
859 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
860 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
861 }
862
863 case LoadOpType::I64Load: {
864 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
865 }
866
867 case LoadOpType::F32Load: {
868 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
869 }
870
871 case LoadOpType::F64Load: {
872 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
873 }
874 }
875 RELEASE_ASSERT_NOT_REACHED();
876}
877
878auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
879{
880 ASSERT(pointer->type() == Int32);
881
882 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
883 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
884 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
885 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
886 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
887 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
888 });
889
890 switch (op) {
891 case LoadOpType::I32Load8S:
892 case LoadOpType::I32Load16S:
893 case LoadOpType::I32Load:
894 case LoadOpType::I32Load16U:
895 case LoadOpType::I32Load8U:
896 result = constant(Int32, 0);
897 break;
898 case LoadOpType::I64Load8S:
899 case LoadOpType::I64Load8U:
900 case LoadOpType::I64Load16S:
901 case LoadOpType::I64Load32U:
902 case LoadOpType::I64Load32S:
903 case LoadOpType::I64Load:
904 case LoadOpType::I64Load16U:
905 result = constant(Int64, 0);
906 break;
907 case LoadOpType::F32Load:
908 result = constant(Float, 0);
909 break;
910 case LoadOpType::F64Load:
911 result = constant(Double, 0);
912 break;
913 }
914
915 } else
916 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
917
918 return { };
919}
920
921inline uint32_t sizeOfStoreOp(StoreOpType op)
922{
923 switch (op) {
924 case StoreOpType::I32Store8:
925 case StoreOpType::I64Store8:
926 return 1;
927 case StoreOpType::I32Store16:
928 case StoreOpType::I64Store16:
929 return 2;
930 case StoreOpType::I32Store:
931 case StoreOpType::I64Store32:
932 case StoreOpType::F32Store:
933 return 4;
934 case StoreOpType::I64Store:
935 case StoreOpType::F64Store:
936 return 8;
937 }
938 RELEASE_ASSERT_NOT_REACHED();
939}
940
941
942inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
943{
944 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
945
946 switch (op) {
947 case StoreOpType::I64Store8:
948 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
949 FALLTHROUGH;
950
951 case StoreOpType::I32Store8:
952 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
953 return;
954
955 case StoreOpType::I64Store16:
956 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
957 FALLTHROUGH;
958
959 case StoreOpType::I32Store16:
960 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
961 return;
962
963 case StoreOpType::I64Store32:
964 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
965 FALLTHROUGH;
966
967 case StoreOpType::I64Store:
968 case StoreOpType::I32Store:
969 case StoreOpType::F32Store:
970 case StoreOpType::F64Store:
971 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
972 return;
973 }
974 RELEASE_ASSERT_NOT_REACHED();
975}
976
977auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
978{
979 ASSERT(pointer->type() == Int32);
980
981 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
982 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
983 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
984 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
985 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
986 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
987 });
988 } else
989 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
990
991 return { };
992}
993
994auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
995{
996 result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
997 return { };
998}
999
1000B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
1001{
1002 return constant(toB3Type(type), value);
1003}
1004
1005void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
1006{
1007 if (!m_tierUp)
1008 return;
1009
1010 ASSERT(m_tierUp);
1011 Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
1012 Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
1013 Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
1014 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
1015
1016 PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
1017 Effects effects = Effects::none();
1018 // FIXME: we should have a more precise heap range for the tier up count.
1019 effects.reads = B3::HeapRange::top();
1020 effects.writes = B3::HeapRange::top();
1021 patch->effects = effects;
1022
1023 patch->append(newCountDown, ValueRep::SomeRegister);
1024 patch->append(oldCountDown, ValueRep::SomeRegister);
1025 patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1026 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1027 MacroAssembler::Label tierUpResume = jit.label();
1028
1029 params.addLatePath([=] (CCallHelpers& jit) {
1030 tierUp.link(&jit);
1031
1032 const unsigned extraPaddingBytes = 0;
1033 RegisterSet registersToSpill = { };
1034 registersToSpill.add(GPRInfo::argumentGPR1);
1035 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1036
1037 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1038 MacroAssembler::Call call = jit.nearCall();
1039
1040 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1041 jit.jump(tierUpResume);
1042
1043 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1044 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1045
1046 });
1047 });
1048 });
1049}
1050
1051B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
1052{
1053 BasicBlock* body = m_proc.addBlock();
1054 BasicBlock* continuation = m_proc.addBlock();
1055
1056 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
1057
1058 m_currentBlock = body;
1059 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1060
1061 return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
1062}
1063
1064B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
1065{
1066 return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
1067}
1068
1069B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
1070{
1071 return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
1072}
1073
1074auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1075{
1076 // FIXME: This needs to do some kind of stack passing.
1077
1078 BasicBlock* taken = m_proc.addBlock();
1079 BasicBlock* notTaken = m_proc.addBlock();
1080 BasicBlock* continuation = m_proc.addBlock();
1081
1082 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1083 m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
1084 taken->addPredecessor(m_currentBlock);
1085 notTaken->addPredecessor(m_currentBlock);
1086
1087 m_currentBlock = taken;
1088 result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
1089 return { };
1090}
1091
1092auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1093{
1094 unifyValuesWithBlock(currentStack, data.result);
1095 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1096 return addElseToUnreachable(data);
1097}
1098
1099auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1100{
1101 ASSERT(data.type() == BlockType::If);
1102 m_currentBlock = data.special;
1103 data.convertIfToBlock();
1104 return { };
1105}
1106
1107auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
1108{
1109 ASSERT(returnValues.size() <= 1);
1110 if (returnValues.size())
1111 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1112 else
1113 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1114 return { };
1115}
1116
1117auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1118{
1119 unifyValuesWithBlock(returnValues, data.resultForBranch());
1120
1121 BasicBlock* target = data.targetBlockForBranch();
1122 if (condition) {
1123 BasicBlock* continuation = m_proc.addBlock();
1124 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1125 m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1126 target->addPredecessor(m_currentBlock);
1127 continuation->addPredecessor(m_currentBlock);
1128 m_currentBlock = continuation;
1129 } else {
1130 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1131 target->addPredecessor(m_currentBlock);
1132 }
1133
1134 return { };
1135}
1136
1137auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1138{
1139 for (size_t i = 0; i < targets.size(); ++i)
1140 unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1141 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1142
1143 SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1144 switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1145 for (size_t i = 0; i < targets.size(); ++i)
1146 switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1147
1148 return { };
1149}
1150
1151auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1152{
1153 ControlData& data = entry.controlData;
1154
1155 unifyValuesWithBlock(expressionStack, data.result);
1156 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1157 data.continuation->addPredecessor(m_currentBlock);
1158
1159 return addEndToUnreachable(entry);
1160}
1161
1162
1163auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1164{
1165 ControlData& data = entry.controlData;
1166 m_currentBlock = data.continuation;
1167
1168 if (data.type() == BlockType::If) {
1169 data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1170 m_currentBlock->addPredecessor(data.special);
1171 }
1172
1173 for (Value* result : data.result) {
1174 m_currentBlock->append(result);
1175 entry.enclosedExpressionStack.append(result);
1176 }
1177
1178 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1179 if (data.type() == BlockType::TopLevel)
1180 return addReturn(entry.controlData, entry.enclosedExpressionStack);
1181
1182 return { };
1183}
1184
1185auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1186{
1187 ASSERT(signature.argumentCount() == args.size());
1188
1189 m_makesCalls = true;
1190
1191 Type returnType = signature.returnType();
1192 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1193
1194 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1195 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1196
1197 // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1198 Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1199 // The target instance is 0 unless the call is wasm->wasm.
1200 Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1201
1202 BasicBlock* isWasmBlock = m_proc.addBlock();
1203 BasicBlock* isEmbedderBlock = m_proc.addBlock();
1204 BasicBlock* continuation = m_proc.addBlock();
1205 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1206
1207 Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1208 [=] (PatchpointValue* patchpoint) {
1209 patchpoint->effects.writesPinned = true;
1210 patchpoint->effects.readsPinned = true;
1211 // We need to clobber all potential pinned registers since we might be leaving the instance.
1212 // We pessimistically assume we could be calling to something that is bounds checking.
1213 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1214 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1215 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1216 AllowMacroScratchRegisterUsage allowScratch(jit);
1217 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1218 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1219 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1220 });
1221 });
1222 });
1223 UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1224 isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1225
1226 // FIXME: Let's remove this indirection by creating a PIC friendly IC
1227 // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1228 // implement the IC to be over Context*.
1229 // https://bugs.webkit.org/show_bug.cgi?id=170375
1230 Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1231 Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1232
1233 Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1234 [=] (PatchpointValue* patchpoint) {
1235 patchpoint->effects.writesPinned = true;
1236 patchpoint->effects.readsPinned = true;
1237 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1238 // We need to clobber all potential pinned registers since we might be leaving the instance.
1239 // We pessimistically assume we could be calling to something that is bounds checking.
1240 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1241 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1242 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1243 AllowMacroScratchRegisterUsage allowScratch(jit);
1244 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1245 });
1246 });
1247 UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1248 isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1249
1250 m_currentBlock = continuation;
1251
1252 if (returnType == Void)
1253 result = nullptr;
1254 else {
1255 result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1256 wasmCallResultUpsilon->setPhi(result);
1257 embedderCallResultUpsilon->setPhi(result);
1258 }
1259
1260 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1261 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1262 } else {
1263 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1264 [=] (PatchpointValue* patchpoint) {
1265 patchpoint->effects.writesPinned = true;
1266 patchpoint->effects.readsPinned = true;
1267
1268 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1269 AllowMacroScratchRegisterUsage allowScratch(jit);
1270 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1271 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1272 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1273 });
1274 });
1275 });
1276 }
1277
1278 return { };
1279}
1280
1281auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1282{
1283 ExpressionType calleeIndex = args.takeLast();
1284 ASSERT(signature.argumentCount() == args.size());
1285
1286 m_makesCalls = true;
1287 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1288 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1289 // can be to the embedder for our stack check calculation.
1290 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1291
1292 ExpressionType callableFunctionBuffer;
1293 ExpressionType instancesBuffer;
1294 ExpressionType callableFunctionBufferLength;
1295 ExpressionType mask;
1296 {
1297 ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1298 instanceValue(), safeCast<int32_t>(Instance::offsetOfTable()));
1299 callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1300 table, safeCast<int32_t>(FuncRefTable::offsetOfFunctions()));
1301 instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1302 table, safeCast<int32_t>(FuncRefTable::offsetOfInstances()));
1303 callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1304 table, safeCast<int32_t>(Table::offsetOfLength()));
1305 mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1306 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1307 table, safeCast<int32_t>(Table::offsetOfMask())));
1308 }
1309
1310 // Check the index we are looking for is valid.
1311 {
1312 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1313 m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1314
1315 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1316 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1317 });
1318 }
1319
1320 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1321
1322 if (Options::enableSpectreMitigations())
1323 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1324
1325 ExpressionType callableFunction;
1326 {
1327 // Compute the offset in the table index space we are looking for.
1328 ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1329 calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1330 callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1331
1332 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1333 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1334 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1335 ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1336 {
1337 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1338 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1339 calleeSignatureIndex,
1340 m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1341
1342 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1343 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1344 });
1345 }
1346
1347 // Check the signature matches the value we expect.
1348 {
1349 ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1350 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1351 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1352
1353 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1354 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1355 });
1356 }
1357 }
1358
1359 // Do a context switch if needed.
1360 {
1361 Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1362 calleeIndex, constant(pointerType(), sizeof(Instance*)));
1363 Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1364 m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1365
1366 BasicBlock* continuation = m_proc.addBlock();
1367 BasicBlock* doContextSwitch = m_proc.addBlock();
1368
1369 Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1370 newContextInstance, instanceValue());
1371 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1372 isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1373
1374 PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1375 patchpoint->effects.writesPinned = true;
1376 // We pessimistically assume we're calling something with BoundsChecking memory.
1377 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1378 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1379 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1380 patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1381 patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1382 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
1383
1384 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1385 AllowMacroScratchRegisterUsage allowScratch(jit);
1386 GPRReg newContextInstance = params[0].gpr();
1387 GPRReg oldContextInstance = params[1].gpr();
1388 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1389 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1390 ASSERT(newContextInstance != baseMemory);
1391 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1392 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1393 jit.storeWasmContextInstance(newContextInstance);
1394 ASSERT(pinnedRegs.sizeRegister != baseMemory);
1395 // FIXME: We should support more than one memory size register
1396 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1397 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1398 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
1399
1400 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1401 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1402
1403 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.sizeRegister, scratchOrSize);
1404 });
1405 doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1406
1407 m_currentBlock = continuation;
1408 }
1409
1410 ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1411 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1412 safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1413
1414 Type returnType = signature.returnType();
1415 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1416 [=] (PatchpointValue* patchpoint) {
1417 patchpoint->effects.writesPinned = true;
1418 patchpoint->effects.readsPinned = true;
1419 // We need to clobber all potential pinned registers since we might be leaving the instance.
1420 // We pessimistically assume we're always calling something that is bounds checking so
1421 // because the wasm->wasm thunk unconditionally overrides the size registers.
1422 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1423 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1424 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1425
1426 patchpoint->append(calleeCode, ValueRep::SomeRegister);
1427 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1428 AllowMacroScratchRegisterUsage allowScratch(jit);
1429 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1430 });
1431 });
1432
1433 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1434 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1435
1436 return { };
1437}
1438
1439void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1440{
1441 m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1442}
1443
1444void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1445{
1446 ASSERT(result.size() <= resultStack.size());
1447
1448 for (size_t i = 0; i < result.size(); ++i)
1449 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1450}
1451
1452static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1453{
1454 dataLog(comma, "ExpressionStack:");
1455 for (const auto& expression : expressionStack)
1456 dataLog(comma, *expression);
1457}
1458
1459void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1460{
1461 dataLogLn("Constants:");
1462 for (const auto& constant : m_constantPool)
1463 dataLogLn(deepDump(m_proc, constant.value));
1464
1465 dataLogLn("Processing Graph:");
1466 dataLog(m_proc);
1467 dataLogLn("With current block:", *m_currentBlock);
1468 dataLogLn("Control stack:");
1469 ASSERT(controlStack.size());
1470 for (size_t i = controlStack.size(); i--;) {
1471 dataLog(" ", controlStack[i].controlData, ": ");
1472 CommaPrinter comma(", ", "");
1473 dumpExpressionStack(comma, *expressionStack);
1474 expressionStack = &controlStack[i].enclosedExpressionStack;
1475 dataLogLn();
1476 }
1477 dataLogLn();
1478}
1479
1480auto B3IRGenerator::origin() -> Origin
1481{
1482 OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1483 ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1484 return bitwise_cast<Origin>(origin);
1485}
1486
1487Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1488{
1489 auto result = std::make_unique<InternalFunction>();
1490
1491 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1492 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1493
1494 Procedure procedure;
1495
1496 procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1497 if (origin.data())
1498 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1499 });
1500
1501 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1502 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1503 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1504 // optLevel=1.
1505 procedure.setNeedsUsedRegisters(false);
1506
1507 procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1508 ? Options::webAssemblyBBQOptimizationLevel()
1509 : Options::webAssemblyOMGOptimizationLevel());
1510
1511 B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1512 FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1513 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1514
1515 irGenerator.insertConstants();
1516
1517 procedure.resetReachability();
1518 if (!ASSERT_DISABLED)
1519 validate(procedure, "After parsing:\n");
1520
1521 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1522 fixSSA(procedure);
1523 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1524
1525 {
1526 B3::prepareForGeneration(procedure);
1527 B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1528 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1529 result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1530 }
1531
1532 return result;
1533}
1534
1535// Custom wasm ops. These are the ones too messy to do in wasm.json.
1536
1537void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1538{
1539 ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1540 const B3::Type type = left->type();
1541
1542 {
1543 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1544 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1545
1546 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1547 this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1548 });
1549 }
1550
1551 if (operation == Div) {
1552 int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1553
1554 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1555 m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1556 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1557 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1558
1559 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1560 this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1561 });
1562 }
1563}
1564
1565template<>
1566auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1567{
1568 const B3::Opcode op = Div;
1569 emitChecksForModOrDiv(op, left, right);
1570 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1571 return { };
1572}
1573
1574template<>
1575auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1576{
1577 const B3::Opcode op = Mod;
1578 emitChecksForModOrDiv(op, left, right);
1579 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1580 return { };
1581}
1582
1583template<>
1584auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1585{
1586 const B3::Opcode op = UDiv;
1587 emitChecksForModOrDiv(op, left, right);
1588 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1589 return { };
1590}
1591
1592template<>
1593auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1594{
1595 const B3::Opcode op = UMod;
1596 emitChecksForModOrDiv(op, left, right);
1597 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1598 return { };
1599}
1600
1601template<>
1602auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1603{
1604 const B3::Opcode op = Div;
1605 emitChecksForModOrDiv(op, left, right);
1606 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1607 return { };
1608}
1609
1610template<>
1611auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1612{
1613 const B3::Opcode op = Mod;
1614 emitChecksForModOrDiv(op, left, right);
1615 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1616 return { };
1617}
1618
1619template<>
1620auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1621{
1622 const B3::Opcode op = UDiv;
1623 emitChecksForModOrDiv(op, left, right);
1624 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1625 return { };
1626}
1627
1628template<>
1629auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1630{
1631 const B3::Opcode op = UMod;
1632 emitChecksForModOrDiv(op, left, right);
1633 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1634 return { };
1635}
1636
1637template<>
1638auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1639{
1640 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1641 patchpoint->append(arg, ValueRep::SomeRegister);
1642 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1643 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1644 });
1645 patchpoint->effects = Effects::none();
1646 result = patchpoint;
1647 return { };
1648}
1649
1650template<>
1651auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1652{
1653 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1654 patchpoint->append(arg, ValueRep::SomeRegister);
1655 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1656 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1657 });
1658 patchpoint->effects = Effects::none();
1659 result = patchpoint;
1660 return { };
1661}
1662
1663template<>
1664auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1665{
1666#if CPU(X86_64)
1667 if (MacroAssembler::supportsCountPopulation()) {
1668 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1669 patchpoint->append(arg, ValueRep::SomeRegister);
1670 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1671 jit.countPopulation32(params[1].gpr(), params[0].gpr());
1672 });
1673 patchpoint->effects = Effects::none();
1674 result = patchpoint;
1675 return { };
1676 }
1677#endif
1678
1679 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1680 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1681 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1682 return { };
1683}
1684
1685template<>
1686auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1687{
1688#if CPU(X86_64)
1689 if (MacroAssembler::supportsCountPopulation()) {
1690 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1691 patchpoint->append(arg, ValueRep::SomeRegister);
1692 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1693 jit.countPopulation64(params[1].gpr(), params[0].gpr());
1694 });
1695 patchpoint->effects = Effects::none();
1696 result = patchpoint;
1697 return { };
1698 }
1699#endif
1700
1701 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1702 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1703 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1704 return { };
1705}
1706
1707template<>
1708auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1709{
1710 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1711 if (isX86())
1712 patchpoint->numGPScratchRegisters = 1;
1713 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1714 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1715 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1716 AllowMacroScratchRegisterUsage allowScratch(jit);
1717#if CPU(X86_64)
1718 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1719#else
1720 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1721#endif
1722 });
1723 patchpoint->effects = Effects::none();
1724 result = patchpoint;
1725 return { };
1726}
1727
1728template<>
1729auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1730{
1731 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1732 if (isX86())
1733 patchpoint->numGPScratchRegisters = 1;
1734 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1735 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1736 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1737 AllowMacroScratchRegisterUsage allowScratch(jit);
1738#if CPU(X86_64)
1739 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1740#else
1741 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1742#endif
1743 });
1744 patchpoint->effects = Effects::none();
1745 result = patchpoint;
1746 return { };
1747}
1748
1749template<>
1750auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1751{
1752 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1753 patchpoint->append(arg, ValueRep::SomeRegister);
1754 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1755 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1756 });
1757 patchpoint->effects = Effects::none();
1758 result = patchpoint;
1759 return { };
1760}
1761
1762template<>
1763auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1764{
1765 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1766 patchpoint->append(arg, ValueRep::SomeRegister);
1767 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1768 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1769 });
1770 patchpoint->effects = Effects::none();
1771 result = patchpoint;
1772 return { };
1773}
1774
1775template<>
1776auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1777{
1778 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1779 patchpoint->append(arg, ValueRep::SomeRegister);
1780 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1781 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1782 });
1783 patchpoint->effects = Effects::none();
1784 result = patchpoint;
1785 return { };
1786}
1787
1788template<>
1789auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1790{
1791 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1792 patchpoint->append(arg, ValueRep::SomeRegister);
1793 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1794 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1795 });
1796 patchpoint->effects = Effects::none();
1797 result = patchpoint;
1798 return { };
1799}
1800
1801template<>
1802auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1803{
1804 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1805 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1806 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1807 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1808 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1809 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1810 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1811 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1812 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1813 });
1814 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1815 patchpoint->append(arg, ValueRep::SomeRegister);
1816 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1817 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1818 });
1819 patchpoint->effects = Effects::none();
1820 result = patchpoint;
1821 return { };
1822}
1823
1824template<>
1825auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1826{
1827 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1828 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1829 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1830 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1831 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1832 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1833 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1834 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1835 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1836 });
1837 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1838 patchpoint->append(arg, ValueRep::SomeRegister);
1839 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1840 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1841 });
1842 patchpoint->effects = Effects::none();
1843 result = patchpoint;
1844 return { };
1845}
1846
1847
1848template<>
1849auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1850{
1851 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1852 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1853 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1854 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1855 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1856 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1857 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1858 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1859 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1860 });
1861 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1862 patchpoint->append(arg, ValueRep::SomeRegister);
1863 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1864 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1865 });
1866 patchpoint->effects = Effects::none();
1867 result = patchpoint;
1868 return { };
1869}
1870
1871template<>
1872auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1873{
1874 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1875 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1876 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1877 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1878 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1879 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1880 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1881 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1882 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1883 });
1884 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1885 patchpoint->append(arg, ValueRep::SomeRegister);
1886 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1887 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1888 });
1889 patchpoint->effects = Effects::none();
1890 result = patchpoint;
1891 return { };
1892}
1893
1894template<>
1895auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1896{
1897 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1898 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1899 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1900 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1901 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1902 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1903 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1904 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1905 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1906 });
1907 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1908 patchpoint->append(arg, ValueRep::SomeRegister);
1909 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1910 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1911 });
1912 patchpoint->effects = Effects::none();
1913 result = patchpoint;
1914 return { };
1915}
1916
1917template<>
1918auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1919{
1920 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1921 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1922 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1923 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1924 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1925 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1926 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1927 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1928 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1929 });
1930
1931 Value* signBitConstant;
1932 if (isX86()) {
1933 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1934 // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1935 // so we can pool them if needed.
1936 signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1937 }
1938 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1939 patchpoint->append(arg, ValueRep::SomeRegister);
1940 if (isX86()) {
1941 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1942 patchpoint->numFPScratchRegisters = 1;
1943 }
1944 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1945 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1946 AllowMacroScratchRegisterUsage allowScratch(jit);
1947 FPRReg scratch = InvalidFPRReg;
1948 FPRReg constant = InvalidFPRReg;
1949 if (isX86()) {
1950 scratch = params.fpScratch(0);
1951 constant = params[2].fpr();
1952 }
1953 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1954 });
1955 patchpoint->effects = Effects::none();
1956 result = patchpoint;
1957 return { };
1958}
1959
1960template<>
1961auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1962{
1963 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1964 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1965 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1966 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1967 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1968 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1969 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1970 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1971 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1972 });
1973 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1974 patchpoint->append(arg, ValueRep::SomeRegister);
1975 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1976 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1977 });
1978 patchpoint->effects = Effects::none();
1979 result = patchpoint;
1980 return { };
1981}
1982
1983template<>
1984auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1985{
1986 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1987 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1988 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1989 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1990 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1991 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1992 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1993 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1994 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1995 });
1996
1997 Value* signBitConstant;
1998 if (isX86()) {
1999 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
2000 // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
2001 // so we can pool them if needed.
2002 signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2003 }
2004 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
2005 patchpoint->append(arg, ValueRep::SomeRegister);
2006 if (isX86()) {
2007 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
2008 patchpoint->numFPScratchRegisters = 1;
2009 }
2010 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2011 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2012 AllowMacroScratchRegisterUsage allowScratch(jit);
2013 FPRReg scratch = InvalidFPRReg;
2014 FPRReg constant = InvalidFPRReg;
2015 if (isX86()) {
2016 scratch = params.fpScratch(0);
2017 constant = params[2].fpr();
2018 }
2019 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2020 });
2021 patchpoint->effects = Effects::none();
2022 result = patchpoint;
2023 return { };
2024}
2025
2026} } // namespace JSC::Wasm
2027
2028#include "WasmB3IRGeneratorInlines.h"
2029
2030#endif // ENABLE(WEBASSEMBLY)
2031