1/*
2 * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmB3IRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AllowMacroScratchRegisterUsageIf.h"
32#include "B3BasicBlockInlines.h"
33#include "B3CCallValue.h"
34#include "B3Compile.h"
35#include "B3ConstPtrValue.h"
36#include "B3FixSSA.h"
37#include "B3Generate.h"
38#include "B3InsertionSet.h"
39#include "B3SlotBaseValue.h"
40#include "B3StackmapGenerationParams.h"
41#include "B3SwitchValue.h"
42#include "B3UpsilonValue.h"
43#include "B3Validate.h"
44#include "B3ValueInlines.h"
45#include "B3ValueKey.h"
46#include "B3Variable.h"
47#include "B3VariableValue.h"
48#include "B3WasmAddressValue.h"
49#include "B3WasmBoundsCheckValue.h"
50#include "JSCInlines.h"
51#include "ScratchRegisterAllocator.h"
52#include "VirtualRegister.h"
53#include "WasmCallingConvention.h"
54#include "WasmContextInlines.h"
55#include "WasmExceptionType.h"
56#include "WasmFunctionParser.h"
57#include "WasmInstance.h"
58#include "WasmMemory.h"
59#include "WasmOMGPlan.h"
60#include "WasmOpcodeOrigin.h"
61#include "WasmSignatureInlines.h"
62#include "WasmThunks.h"
63#include <limits>
64#include <wtf/Optional.h>
65#include <wtf/StdLibExtras.h>
66
67void dumpProcedure(void* ptr)
68{
69 JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
70 proc->dump(WTF::dataFile());
71}
72
73namespace JSC { namespace Wasm {
74
75using namespace B3;
76
77namespace {
78namespace WasmB3IRGeneratorInternal {
79static const bool verbose = false;
80}
81}
82
83class B3IRGenerator {
84public:
85 struct ControlData {
86 ControlData(Procedure& proc, Origin origin, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
87 : blockType(type)
88 , continuation(continuation)
89 , special(special)
90 {
91 if (signature != Void)
92 result.append(proc.add<Value>(Phi, toB3Type(signature), origin));
93 }
94
95 ControlData()
96 {
97 }
98
99 void dump(PrintStream& out) const
100 {
101 switch (type()) {
102 case BlockType::If:
103 out.print("If: ");
104 break;
105 case BlockType::Block:
106 out.print("Block: ");
107 break;
108 case BlockType::Loop:
109 out.print("Loop: ");
110 break;
111 case BlockType::TopLevel:
112 out.print("TopLevel: ");
113 break;
114 }
115 out.print("Continuation: ", *continuation, ", Special: ");
116 if (special)
117 out.print(*special);
118 else
119 out.print("None");
120 }
121
122 BlockType type() const { return blockType; }
123
124 bool hasNonVoidSignature() const { return result.size(); }
125
126 BasicBlock* targetBlockForBranch()
127 {
128 if (type() == BlockType::Loop)
129 return special;
130 return continuation;
131 }
132
133 void convertIfToBlock()
134 {
135 ASSERT(type() == BlockType::If);
136 blockType = BlockType::Block;
137 special = nullptr;
138 }
139
140 using ResultList = Vector<Value*, 1>; // Value must be a Phi
141
142 ResultList resultForBranch() const
143 {
144 if (type() == BlockType::Loop)
145 return ResultList();
146 return result;
147 }
148
149 private:
150 friend class B3IRGenerator;
151 BlockType blockType;
152 BasicBlock* continuation;
153 BasicBlock* special;
154 ResultList result;
155 };
156
157 typedef Value* ExpressionType;
158 typedef ControlData ControlType;
159 typedef Vector<ExpressionType, 1> ExpressionList;
160 typedef ControlData::ResultList ResultList;
161 typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
162
163 static constexpr ExpressionType emptyExpression() { return nullptr; }
164
165 typedef String ErrorType;
166 typedef Unexpected<ErrorType> UnexpectedResult;
167 typedef Expected<std::unique_ptr<InternalFunction>, ErrorType> Result;
168 typedef Expected<void, ErrorType> PartialResult;
169 template <typename ...Args>
170 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
171 {
172 using namespace FailureHelper; // See ADL comment in WasmParser.h.
173 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
174 }
175#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
176 if (UNLIKELY(condition)) \
177 return fail(__VA_ARGS__); \
178 } while (0)
179
180 B3IRGenerator(const ModuleInformation&, Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, CompilationMode, unsigned functionIndex, TierUpCount*, ThrowWasmException);
181
182 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
183 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
184 ExpressionType addConstant(Type, uint64_t);
185
186 // Locals
187 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
188 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
189
190 // Globals
191 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
192 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
193
194 // Memory
195 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
196 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
197 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
198 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
199
200 // Basic operators
201 template<OpType>
202 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
203 template<OpType>
204 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
205 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
206
207 // Control flow
208 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
209 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
210 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
211 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
212 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
213 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
214
215 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
216 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
217 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
218 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
219 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
220
221 // Calls
222 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
223 PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
224 PartialResult WARN_UNUSED_RETURN addUnreachable();
225
226 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
227 void setParser(FunctionParser<B3IRGenerator>* parser) { m_parser = parser; };
228
229 Value* constant(B3::Type, uint64_t bits, Optional<Origin> = WTF::nullopt);
230 void insertConstants();
231
232 ALWAYS_INLINE void didKill(ExpressionType) { }
233
234private:
235 void emitExceptionCheck(CCallHelpers&, ExceptionType);
236
237 void emitTierUpCheck(uint32_t decrementCount, Origin);
238
239 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
240 B3::Kind memoryKind(B3::Opcode memoryOp);
241 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
242 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
243
244 void unify(const ExpressionType phi, const ExpressionType source);
245 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
246
247 void emitChecksForModOrDiv(B3::Opcode, ExpressionType left, ExpressionType right);
248
249 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
250
251 void restoreWasmContextInstance(Procedure&, BasicBlock*, Value*);
252 enum class RestoreCachedStackLimit { No, Yes };
253 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, Value* instance, Procedure&, BasicBlock*);
254
255 Origin origin();
256
257 FunctionParser<B3IRGenerator>* m_parser { nullptr };
258 const ModuleInformation& m_info;
259 const MemoryMode m_mode { MemoryMode::BoundsChecking };
260 const CompilationMode m_compilationMode { CompilationMode::BBQMode };
261 const unsigned m_functionIndex { UINT_MAX };
262 const TierUpCount* m_tierUp { nullptr };
263
264 Procedure& m_proc;
265 BasicBlock* m_currentBlock { nullptr };
266 Vector<Variable*> m_locals;
267 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
268 HashMap<ValueKey, Value*> m_constantPool;
269 InsertionSet m_constantInsertionValues;
270 GPRReg m_memoryBaseGPR { InvalidGPRReg };
271 GPRReg m_memorySizeGPR { InvalidGPRReg };
272 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
273 bool m_makesCalls { false };
274
275 Value* m_instanceValue { nullptr }; // Always use the accessor below to ensure the instance value is materialized when used.
276 bool m_usesInstanceValue { false };
277 Value* instanceValue()
278 {
279 m_usesInstanceValue = true;
280 return m_instanceValue;
281 }
282
283 uint32_t m_maxNumJSCallArguments { 0 };
284};
285
286// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
287int32_t B3IRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
288{
289 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
290 ptr = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), ptr, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), offset));
291 return 0;
292 }
293 return offset;
294}
295
296void B3IRGenerator::restoreWasmContextInstance(Procedure& proc, BasicBlock* block, Value* arg)
297{
298 if (Context::useFastTLS()) {
299 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
300 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
301 patchpoint->clobber(RegisterSet::macroScratchRegisters());
302 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
303 patchpoint->setGenerator(
304 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
305 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
306 jit.storeWasmContextInstance(params[0].gpr());
307 });
308 return;
309 }
310
311 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
312 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
313 PatchpointValue* patchpoint = block->appendNew<PatchpointValue>(proc, B3::Void, Origin());
314 Effects effects = Effects::none();
315 effects.writesPinned = true;
316 effects.reads = B3::HeapRange::top();
317 patchpoint->effects = effects;
318 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
319 patchpoint->append(arg, ValueRep::SomeRegister);
320 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
321 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& param) {
322 jit.move(param[0].gpr(), wasmContextInstanceGPR);
323 });
324}
325
326B3IRGenerator::B3IRGenerator(const ModuleInformation& info, Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, CompilationMode compilationMode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
327 : m_info(info)
328 , m_mode(mode)
329 , m_compilationMode(compilationMode)
330 , m_functionIndex(functionIndex)
331 , m_tierUp(tierUp)
332 , m_proc(procedure)
333 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
334 , m_constantInsertionValues(m_proc)
335{
336 m_currentBlock = m_proc.addBlock();
337
338 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
339 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
340
341 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
342 m_proc.pinRegister(m_memoryBaseGPR);
343
344 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
345 if (!Context::useFastTLS())
346 m_proc.pinRegister(m_wasmContextInstanceGPR);
347
348 if (mode != MemoryMode::Signaling) {
349 m_memorySizeGPR = pinnedRegs.sizeRegister;
350 m_proc.pinRegister(m_memorySizeGPR);
351 }
352
353 if (throwWasmException)
354 Thunks::singleton().setThrowWasmException(throwWasmException);
355
356 if (info.memory) {
357 m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR) {
358 AllowMacroScratchRegisterUsage allowScratch(jit);
359 switch (m_mode) {
360 case MemoryMode::BoundsChecking:
361 ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
362 break;
363 case MemoryMode::Signaling:
364 ASSERT_UNUSED(pinnedGPR, InvalidGPRReg == pinnedGPR);
365 break;
366 }
367 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
368 });
369
370 switch (m_mode) {
371 case MemoryMode::BoundsChecking:
372 break;
373 case MemoryMode::Signaling:
374 // Most memory accesses in signaling mode don't do an explicit
375 // exception check because they can rely on fault handling to detect
376 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
377 // thunk to exist so that it can jump to that thunk.
378 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
379 CRASH();
380 break;
381 }
382 }
383
384 wasmCallingConvention().setupFrameInPrologue(&compilation->calleeMoveLocation, m_proc, Origin(), m_currentBlock);
385
386 {
387 B3::Value* framePointer = m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, Origin());
388 B3::PatchpointValue* stackOverflowCheck = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, pointerType(), Origin());
389 m_instanceValue = stackOverflowCheck;
390 stackOverflowCheck->appendSomeRegister(framePointer);
391 stackOverflowCheck->clobber(RegisterSet::macroScratchRegisters());
392 if (!Context::useFastTLS()) {
393 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
394 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
395 stackOverflowCheck->effects.writesPinned = false;
396 stackOverflowCheck->effects.readsPinned = true;
397 stackOverflowCheck->resultConstraint = ValueRep::reg(m_wasmContextInstanceGPR);
398 }
399 stackOverflowCheck->numGPScratchRegisters = 2;
400 stackOverflowCheck->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
401 const Checked<int32_t> wasmFrameSize = params.proc().frameSize();
402 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
403 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
404 // This allows us to elide stack checks for functions that are terminal nodes in the call
405 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
406 // having any such terminal node have its parent caller include some extra size in its
407 // own check for it. The goal here is twofold:
408 // 1. Emit less code.
409 // 2. Try to speed things up by skipping stack checks.
410 minimumParentCheckSize,
411 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
412 // spill all arguments to the stack, we ensure that a stack check here covers the
413 // stack that such a stub would use.
414 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
415 ));
416 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
417 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
418 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
419
420 GPRReg contextInstance = Context::useFastTLS() ? params[0].gpr() : m_wasmContextInstanceGPR;
421
422 // This allows leaf functions to not do stack checks if their frame size is within
423 // certain limits since their caller would have already done the check.
424 if (needsOverflowCheck) {
425 AllowMacroScratchRegisterUsage allowScratch(jit);
426 GPRReg fp = params[1].gpr();
427 GPRReg scratch1 = params.gpScratch(0);
428 GPRReg scratch2 = params.gpScratch(1);
429
430 if (Context::useFastTLS())
431 jit.loadWasmContextInstance(contextInstance);
432
433 jit.loadPtr(CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit()), scratch2);
434 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), fp, scratch1);
435 MacroAssembler::JumpList overflow;
436 if (UNLIKELY(needUnderflowCheck))
437 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch1, fp));
438 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch1, scratch2));
439 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
440 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
441 });
442 } else if (m_usesInstanceValue && Context::useFastTLS()) {
443 // No overflow check is needed, but the instance values still needs to be correct.
444 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::loadWasmContextInstanceNeedsMacroScratchRegister());
445 jit.loadWasmContextInstance(contextInstance);
446 } else {
447 // We said we'd return a pointer. We don't actually need to because it isn't used, but the patchpoint conservatively said it had effects (potential stack check) which prevent it from getting removed.
448 }
449 });
450 }
451
452 emitTierUpCheck(TierUpCount::functionEntryDecrement(), Origin());
453}
454
455void B3IRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, Value* instance, Procedure& proc, BasicBlock* block)
456{
457 restoreWasmContextInstance(proc, block, instance);
458
459 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
460 // The Instance caches the stack limit, but also knows where its canonical location is.
461 Value* pointerToActualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfPointerToActualStackLimit()));
462 Value* actualStackLimit = block->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), pointerToActualStackLimit);
463 block->appendNew<MemoryValue>(m_proc, Store, origin(), actualStackLimit, instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedStackLimit()));
464 }
465
466 if (!!memory) {
467 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
468 RegisterSet clobbers;
469 clobbers.set(pinnedRegs->baseMemoryPointer);
470 clobbers.set(pinnedRegs->sizeRegister);
471
472 B3::PatchpointValue* patchpoint = block->appendNew<B3::PatchpointValue>(proc, B3::Void, origin());
473 Effects effects = Effects::none();
474 effects.writesPinned = true;
475 effects.reads = B3::HeapRange::top();
476 patchpoint->effects = effects;
477 patchpoint->clobber(clobbers);
478
479 patchpoint->append(instance, ValueRep::SomeRegister);
480
481 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
482 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
483 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
484 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
485 });
486 }
487}
488
489void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
490{
491 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
492 auto jumpToExceptionStub = jit.jump();
493
494 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
495 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
496 });
497}
498
499Value* B3IRGenerator::constant(B3::Type type, uint64_t bits, Optional<Origin> maybeOrigin)
500{
501 auto result = m_constantPool.ensure(ValueKey(opcodeForConstant(type), type, static_cast<int64_t>(bits)), [&] {
502 Value* result = m_proc.addConstant(maybeOrigin ? *maybeOrigin : origin(), type, bits);
503 m_constantInsertionValues.insertValue(0, result);
504 return result;
505 });
506 return result.iterator->value;
507}
508
509void B3IRGenerator::insertConstants()
510{
511 m_constantInsertionValues.execute(m_proc.at(0));
512}
513
514auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
515{
516 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
517 totalBytesChecked += m_locals.size();
518 uint32_t totalBytes;
519 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
520
521 for (uint32_t i = 0; i < count; ++i) {
522 Variable* local = m_proc.addVariable(toB3Type(type));
523 m_locals.uncheckedAppend(local);
524 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, constant(toB3Type(type), 0, Origin()));
525 }
526 return { };
527}
528
529auto B3IRGenerator::addArguments(const Signature& signature) -> PartialResult
530{
531 ASSERT(!m_locals.size());
532 WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(signature.argumentCount()), "can't allocate memory for ", signature.argumentCount(), " arguments");
533
534 m_locals.grow(signature.argumentCount());
535 wasmCallingConvention().loadArguments(signature, m_proc, m_currentBlock, Origin(),
536 [=] (ExpressionType argument, unsigned i) {
537 Variable* argumentVariable = m_proc.addVariable(argument->type());
538 m_locals[i] = argumentVariable;
539 m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
540 });
541 return { };
542}
543
544auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
545{
546 ASSERT(m_locals[index]);
547 result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, origin(), m_locals[index]);
548 return { };
549}
550
551auto B3IRGenerator::addUnreachable() -> PartialResult
552{
553 B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
554 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
555 this->emitExceptionCheck(jit, ExceptionType::Unreachable);
556 });
557 unreachable->effects.terminal = true;
558 return { };
559}
560
561auto B3IRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
562{
563 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
564 instance->storeTopCallFrame(callFrame);
565
566 if (delta < 0)
567 return -1;
568
569 auto grown = instance->memory()->grow(PageCount(delta));
570 if (!grown) {
571 switch (grown.error()) {
572 case Memory::GrowFailReason::InvalidDelta:
573 case Memory::GrowFailReason::InvalidGrowSize:
574 case Memory::GrowFailReason::WouldExceedMaximum:
575 case Memory::GrowFailReason::OutOfMemory:
576 return -1;
577 }
578 RELEASE_ASSERT_NOT_REACHED();
579 }
580
581 return grown.value().pageCount();
582 };
583
584 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(),
585 m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(growMemory, B3CCallPtrTag)),
586 m_currentBlock->appendNew<B3::Value>(m_proc, B3::FramePointer, origin()), instanceValue(), delta);
587
588 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_proc, m_currentBlock);
589
590 return { };
591}
592
593auto B3IRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
594{
595 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
596 Value* size = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfCachedMemorySize()));
597
598 constexpr uint32_t shiftValue = 16;
599 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
600 Value* numPages = m_currentBlock->appendNew<Value>(m_proc, ZShr, origin(),
601 size, m_currentBlock->appendNew<Const32Value>(m_proc, origin(), shiftValue));
602
603 result = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), numPages);
604
605 return { };
606}
607
608auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
609{
610 ASSERT(m_locals[index]);
611 m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, origin(), m_locals[index], value);
612 return { };
613}
614
615auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
616{
617 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
618 result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), origin(), globalsArray, safeCast<int32_t>(index * sizeof(Register)));
619 return { };
620}
621
622auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
623{
624 ASSERT(toB3Type(m_info.globals[index].type) == value->type());
625 Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfGlobals()));
626 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin(), value, globalsArray, safeCast<int32_t>(index * sizeof(Register)));
627 return { };
628}
629
630inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
631{
632 ASSERT(m_memoryBaseGPR);
633
634 switch (m_mode) {
635 case MemoryMode::BoundsChecking: {
636 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
637 ASSERT(m_memorySizeGPR);
638 ASSERT(sizeOfOperation + offset > offset);
639 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), m_memorySizeGPR, pointer, sizeOfOperation + offset - 1);
640 break;
641 }
642
643 case MemoryMode::Signaling: {
644 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
645 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
646 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
647 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
648 // register + large offset access. We don't think this will be generated frequently.
649 //
650 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
651 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
652 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
653 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
654 if (offset >= Memory::fastMappedRedzoneBytes()) {
655 size_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
656 m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, origin(), pointer, sizeOfOperation + offset - 1, maximum);
657 }
658 break;
659 }
660 }
661
662 pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), pointer);
663 return m_currentBlock->appendNew<WasmAddressValue>(m_proc, origin(), pointer, m_memoryBaseGPR);
664}
665
666inline uint32_t sizeOfLoadOp(LoadOpType op)
667{
668 switch (op) {
669 case LoadOpType::I32Load8S:
670 case LoadOpType::I32Load8U:
671 case LoadOpType::I64Load8S:
672 case LoadOpType::I64Load8U:
673 return 1;
674 case LoadOpType::I32Load16S:
675 case LoadOpType::I64Load16S:
676 case LoadOpType::I32Load16U:
677 case LoadOpType::I64Load16U:
678 return 2;
679 case LoadOpType::I32Load:
680 case LoadOpType::I64Load32S:
681 case LoadOpType::I64Load32U:
682 case LoadOpType::F32Load:
683 return 4;
684 case LoadOpType::I64Load:
685 case LoadOpType::F64Load:
686 return 8;
687 }
688 RELEASE_ASSERT_NOT_REACHED();
689}
690
691inline B3::Kind B3IRGenerator::memoryKind(B3::Opcode memoryOp)
692{
693 if (m_mode == MemoryMode::Signaling)
694 return trapping(memoryOp);
695 return memoryOp;
696}
697
698inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
699{
700 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
701
702 switch (op) {
703 case LoadOpType::I32Load8S: {
704 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
705 }
706
707 case LoadOpType::I64Load8S: {
708 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8S), origin(), pointer, offset);
709 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
710 }
711
712 case LoadOpType::I32Load8U: {
713 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
714 }
715
716 case LoadOpType::I64Load8U: {
717 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load8Z), origin(), pointer, offset);
718 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
719 }
720
721 case LoadOpType::I32Load16S: {
722 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
723 }
724
725 case LoadOpType::I64Load16S: {
726 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16S), origin(), pointer, offset);
727 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
728 }
729
730 case LoadOpType::I32Load16U: {
731 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
732 }
733
734 case LoadOpType::I64Load16U: {
735 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load16Z), origin(), pointer, offset);
736 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
737 }
738
739 case LoadOpType::I32Load: {
740 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
741 }
742
743 case LoadOpType::I64Load32U: {
744 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
745 return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), value);
746 }
747
748 case LoadOpType::I64Load32S: {
749 Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int32, origin(), pointer, offset);
750 return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin(), value);
751 }
752
753 case LoadOpType::I64Load: {
754 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Int64, origin(), pointer, offset);
755 }
756
757 case LoadOpType::F32Load: {
758 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Float, origin(), pointer, offset);
759 }
760
761 case LoadOpType::F64Load: {
762 return m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Load), Double, origin(), pointer, offset);
763 }
764 }
765 RELEASE_ASSERT_NOT_REACHED();
766}
767
768auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
769{
770 ASSERT(pointer->type() == Int32);
771
772 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
773 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
774 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
775 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
776 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
777 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
778 });
779
780 switch (op) {
781 case LoadOpType::I32Load8S:
782 case LoadOpType::I32Load16S:
783 case LoadOpType::I32Load:
784 case LoadOpType::I32Load16U:
785 case LoadOpType::I32Load8U:
786 result = constant(Int32, 0);
787 break;
788 case LoadOpType::I64Load8S:
789 case LoadOpType::I64Load8U:
790 case LoadOpType::I64Load16S:
791 case LoadOpType::I64Load32U:
792 case LoadOpType::I64Load32S:
793 case LoadOpType::I64Load:
794 case LoadOpType::I64Load16U:
795 result = constant(Int64, 0);
796 break;
797 case LoadOpType::F32Load:
798 result = constant(Float, 0);
799 break;
800 case LoadOpType::F64Load:
801 result = constant(Double, 0);
802 break;
803 }
804
805 } else
806 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
807
808 return { };
809}
810
811inline uint32_t sizeOfStoreOp(StoreOpType op)
812{
813 switch (op) {
814 case StoreOpType::I32Store8:
815 case StoreOpType::I64Store8:
816 return 1;
817 case StoreOpType::I32Store16:
818 case StoreOpType::I64Store16:
819 return 2;
820 case StoreOpType::I32Store:
821 case StoreOpType::I64Store32:
822 case StoreOpType::F32Store:
823 return 4;
824 case StoreOpType::I64Store:
825 case StoreOpType::F64Store:
826 return 8;
827 }
828 RELEASE_ASSERT_NOT_REACHED();
829}
830
831
832inline void B3IRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
833{
834 int32_t offset = fixupPointerPlusOffset(pointer, uoffset);
835
836 switch (op) {
837 case StoreOpType::I64Store8:
838 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
839 FALLTHROUGH;
840
841 case StoreOpType::I32Store8:
842 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store8), origin(), value, pointer, offset);
843 return;
844
845 case StoreOpType::I64Store16:
846 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
847 FALLTHROUGH;
848
849 case StoreOpType::I32Store16:
850 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store16), origin(), value, pointer, offset);
851 return;
852
853 case StoreOpType::I64Store32:
854 value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin(), value);
855 FALLTHROUGH;
856
857 case StoreOpType::I64Store:
858 case StoreOpType::I32Store:
859 case StoreOpType::F32Store:
860 case StoreOpType::F64Store:
861 m_currentBlock->appendNew<MemoryValue>(m_proc, memoryKind(Store), origin(), value, pointer, offset);
862 return;
863 }
864 RELEASE_ASSERT_NOT_REACHED();
865}
866
867auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
868{
869 ASSERT(pointer->type() == Int32);
870
871 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
872 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
873 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
874 B3::PatchpointValue* throwException = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, origin());
875 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
876 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
877 });
878 } else
879 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
880
881 return { };
882}
883
884auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
885{
886 result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, origin(), condition, nonZero, zero);
887 return { };
888}
889
890B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
891{
892 return constant(toB3Type(type), value);
893}
894
895void B3IRGenerator::emitTierUpCheck(uint32_t decrementCount, Origin origin)
896{
897 if (!m_tierUp)
898 return;
899
900 ASSERT(m_tierUp);
901 Value* countDownLocation = constant(pointerType(), reinterpret_cast<uint64_t>(m_tierUp), origin);
902 Value* oldCountDown = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, countDownLocation);
903 Value* newCountDown = m_currentBlock->appendNew<Value>(m_proc, Sub, origin, oldCountDown, constant(Int32, decrementCount, origin));
904 m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, newCountDown, countDownLocation);
905
906 PatchpointValue* patch = m_currentBlock->appendNew<PatchpointValue>(m_proc, B3::Void, origin);
907 Effects effects = Effects::none();
908 // FIXME: we should have a more precise heap range for the tier up count.
909 effects.reads = B3::HeapRange::top();
910 effects.writes = B3::HeapRange::top();
911 patch->effects = effects;
912
913 patch->append(newCountDown, ValueRep::SomeRegister);
914 patch->append(oldCountDown, ValueRep::SomeRegister);
915 patch->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
916 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
917 MacroAssembler::Label tierUpResume = jit.label();
918
919 params.addLatePath([=] (CCallHelpers& jit) {
920 tierUp.link(&jit);
921
922 const unsigned extraPaddingBytes = 0;
923 RegisterSet registersToSpill = { };
924 registersToSpill.add(GPRInfo::argumentGPR1);
925 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
926
927 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
928 MacroAssembler::Call call = jit.nearCall();
929
930 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
931 jit.jump(tierUpResume);
932
933 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
934 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
935
936 });
937 });
938 });
939}
940
941B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
942{
943 BasicBlock* body = m_proc.addBlock();
944 BasicBlock* continuation = m_proc.addBlock();
945
946 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), body);
947
948 m_currentBlock = body;
949 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
950
951 return ControlData(m_proc, origin(), signature, BlockType::Loop, continuation, body);
952}
953
954B3IRGenerator::ControlData B3IRGenerator::addTopLevel(Type signature)
955{
956 return ControlData(m_proc, Origin(), signature, BlockType::TopLevel, m_proc.addBlock());
957}
958
959B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
960{
961 return ControlData(m_proc, origin(), signature, BlockType::Block, m_proc.addBlock());
962}
963
964auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
965{
966 // FIXME: This needs to do some kind of stack passing.
967
968 BasicBlock* taken = m_proc.addBlock();
969 BasicBlock* notTaken = m_proc.addBlock();
970 BasicBlock* continuation = m_proc.addBlock();
971
972 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
973 m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
974 taken->addPredecessor(m_currentBlock);
975 notTaken->addPredecessor(m_currentBlock);
976
977 m_currentBlock = taken;
978 result = ControlData(m_proc, origin(), signature, BlockType::If, continuation, notTaken);
979 return { };
980}
981
982auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
983{
984 unifyValuesWithBlock(currentStack, data.result);
985 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
986 return addElseToUnreachable(data);
987}
988
989auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
990{
991 ASSERT(data.type() == BlockType::If);
992 m_currentBlock = data.special;
993 data.convertIfToBlock();
994 return { };
995}
996
997auto B3IRGenerator::addReturn(const ControlData&, const ExpressionList& returnValues) -> PartialResult
998{
999 ASSERT(returnValues.size() <= 1);
1000 if (returnValues.size())
1001 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin(), returnValues[0]);
1002 else
1003 m_currentBlock->appendNewControlValue(m_proc, B3::Return, origin());
1004 return { };
1005}
1006
1007auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1008{
1009 unifyValuesWithBlock(returnValues, data.resultForBranch());
1010
1011 BasicBlock* target = data.targetBlockForBranch();
1012 if (condition) {
1013 BasicBlock* continuation = m_proc.addBlock();
1014 m_currentBlock->appendNew<Value>(m_proc, B3::Branch, origin(), condition);
1015 m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
1016 target->addPredecessor(m_currentBlock);
1017 continuation->addPredecessor(m_currentBlock);
1018 m_currentBlock = continuation;
1019 } else {
1020 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), FrequentedBlock(target));
1021 target->addPredecessor(m_currentBlock);
1022 }
1023
1024 return { };
1025}
1026
1027auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1028{
1029 for (size_t i = 0; i < targets.size(); ++i)
1030 unifyValuesWithBlock(expressionStack, targets[i]->resultForBranch());
1031 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1032
1033 SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, origin(), condition);
1034 switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
1035 for (size_t i = 0; i < targets.size(); ++i)
1036 switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
1037
1038 return { };
1039}
1040
1041auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1042{
1043 ControlData& data = entry.controlData;
1044
1045 unifyValuesWithBlock(expressionStack, data.result);
1046 m_currentBlock->appendNewControlValue(m_proc, Jump, origin(), data.continuation);
1047 data.continuation->addPredecessor(m_currentBlock);
1048
1049 return addEndToUnreachable(entry);
1050}
1051
1052
1053auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1054{
1055 ControlData& data = entry.controlData;
1056 m_currentBlock = data.continuation;
1057
1058 if (data.type() == BlockType::If) {
1059 data.special->appendNewControlValue(m_proc, Jump, origin(), m_currentBlock);
1060 m_currentBlock->addPredecessor(data.special);
1061 }
1062
1063 for (Value* result : data.result) {
1064 m_currentBlock->append(result);
1065 entry.enclosedExpressionStack.append(result);
1066 }
1067
1068 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1069 if (data.type() == BlockType::TopLevel)
1070 return addReturn(entry.controlData, entry.enclosedExpressionStack);
1071
1072 return { };
1073}
1074
1075auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1076{
1077 ASSERT(signature.argumentCount() == args.size());
1078
1079 m_makesCalls = true;
1080
1081 Type returnType = signature.returnType();
1082 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1083
1084 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1085 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1086
1087 // FIXME imports can be linked here, instead of generating a patchpoint, because all import stubs are generated before B3 compilation starts. https://bugs.webkit.org/show_bug.cgi?id=166462
1088 Value* targetInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfTargetInstance(functionIndex)));
1089 // The target instance is 0 unless the call is wasm->wasm.
1090 Value* isWasmCall = m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), targetInstance, m_currentBlock->appendNew<Const64Value>(m_proc, origin(), 0));
1091
1092 BasicBlock* isWasmBlock = m_proc.addBlock();
1093 BasicBlock* isEmbedderBlock = m_proc.addBlock();
1094 BasicBlock* continuation = m_proc.addBlock();
1095 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(), isWasmCall, FrequentedBlock(isWasmBlock), FrequentedBlock(isEmbedderBlock));
1096
1097 Value* wasmCallResult = wasmCallingConvention().setupCall(m_proc, isWasmBlock, origin(), args, toB3Type(returnType),
1098 [=] (PatchpointValue* patchpoint) {
1099 patchpoint->effects.writesPinned = true;
1100 patchpoint->effects.readsPinned = true;
1101 // We need to clobber all potential pinned registers since we might be leaving the instance.
1102 // We pessimistically assume we could be calling to something that is bounds checking.
1103 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1104 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1105 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1106 AllowMacroScratchRegisterUsage allowScratch(jit);
1107 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1108 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1109 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1110 });
1111 });
1112 });
1113 UpsilonValue* wasmCallResultUpsilon = returnType == Void ? nullptr : isWasmBlock->appendNew<UpsilonValue>(m_proc, origin(), wasmCallResult);
1114 isWasmBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1115
1116 // FIXME: Let's remove this indirection by creating a PIC friendly IC
1117 // for calls out to the embedder. This shouldn't be that hard to do. We could probably
1118 // implement the IC to be over Context*.
1119 // https://bugs.webkit.org/show_bug.cgi?id=170375
1120 Value* jumpDestination = isEmbedderBlock->appendNew<MemoryValue>(m_proc,
1121 Load, pointerType(), origin(), instanceValue(), safeCast<int32_t>(Instance::offsetOfWasmToEmbedderStub(functionIndex)));
1122
1123 Value* embedderCallResult = wasmCallingConvention().setupCall(m_proc, isEmbedderBlock, origin(), args, toB3Type(returnType),
1124 [=] (PatchpointValue* patchpoint) {
1125 patchpoint->effects.writesPinned = true;
1126 patchpoint->effects.readsPinned = true;
1127 patchpoint->append(jumpDestination, ValueRep::SomeRegister);
1128 // We need to clobber all potential pinned registers since we might be leaving the instance.
1129 // We pessimistically assume we could be calling to something that is bounds checking.
1130 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1131 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1132 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1133 AllowMacroScratchRegisterUsage allowScratch(jit);
1134 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1135 });
1136 });
1137 UpsilonValue* embedderCallResultUpsilon = returnType == Void ? nullptr : isEmbedderBlock->appendNew<UpsilonValue>(m_proc, origin(), embedderCallResult);
1138 isEmbedderBlock->appendNewControlValue(m_proc, Jump, origin(), continuation);
1139
1140 m_currentBlock = continuation;
1141
1142 if (returnType == Void)
1143 result = nullptr;
1144 else {
1145 result = continuation->appendNew<Value>(m_proc, Phi, toB3Type(returnType), origin());
1146 wasmCallResultUpsilon->setPhi(result);
1147 embedderCallResultUpsilon->setPhi(result);
1148 }
1149
1150 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1151 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, continuation);
1152 } else {
1153 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1154 [=] (PatchpointValue* patchpoint) {
1155 patchpoint->effects.writesPinned = true;
1156 patchpoint->effects.readsPinned = true;
1157
1158 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1159 AllowMacroScratchRegisterUsage allowScratch(jit);
1160 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1161 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1162 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1163 });
1164 });
1165 });
1166 }
1167
1168 return { };
1169}
1170
1171auto B3IRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1172{
1173 ExpressionType calleeIndex = args.takeLast();
1174 ASSERT(signature.argumentCount() == args.size());
1175
1176 m_makesCalls = true;
1177 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1178 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1179 // can be to the embedder for our stack check calculation.
1180 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1181
1182 ExpressionType callableFunctionBuffer;
1183 ExpressionType instancesBuffer;
1184 ExpressionType callableFunctionBufferLength;
1185 ExpressionType mask;
1186 {
1187 ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1188 instanceValue(), safeCast<int32_t>(Instance::offsetOfTable()));
1189 callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1190 table, safeCast<int32_t>(Table::offsetOfFunctions()));
1191 instancesBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1192 table, safeCast<int32_t>(Table::offsetOfInstances()));
1193 callableFunctionBufferLength = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1194 table, safeCast<int32_t>(Table::offsetOfLength()));
1195 mask = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(),
1196 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin(),
1197 table, safeCast<int32_t>(Table::offsetOfMask())));
1198 }
1199
1200 // Check the index we are looking for is valid.
1201 {
1202 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1203 m_currentBlock->appendNew<Value>(m_proc, AboveEqual, origin(), calleeIndex, callableFunctionBufferLength));
1204
1205 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1206 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
1207 });
1208 }
1209
1210 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin(), calleeIndex);
1211
1212 if (Options::enableSpectreMitigations())
1213 calleeIndex = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(), mask, calleeIndex);
1214
1215 ExpressionType callableFunction;
1216 {
1217 // Compute the offset in the table index space we are looking for.
1218 ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1219 calleeIndex, constant(pointerType(), sizeof(WasmToWasmImportableFunction)));
1220 callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, origin(), callableFunctionBuffer, offset);
1221
1222 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1223 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1224 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1225 ExpressionType calleeSignatureIndex = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin(), callableFunction, safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfSignatureIndex()));
1226 {
1227 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1228 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1229 calleeSignatureIndex,
1230 m_currentBlock->appendNew<Const64Value>(m_proc, origin(), Signature::invalidIndex)));
1231
1232 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1233 this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
1234 });
1235 }
1236
1237 // Check the signature matches the value we expect.
1238 {
1239 ExpressionType expectedSignatureIndex = m_currentBlock->appendNew<Const64Value>(m_proc, origin(), SignatureInformation::get(signature));
1240 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1241 m_currentBlock->appendNew<Value>(m_proc, NotEqual, origin(), calleeSignatureIndex, expectedSignatureIndex));
1242
1243 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1244 this->emitExceptionCheck(jit, ExceptionType::BadSignature);
1245 });
1246 }
1247 }
1248
1249 // Do a context switch if needed.
1250 {
1251 Value* offset = m_currentBlock->appendNew<Value>(m_proc, Mul, origin(),
1252 calleeIndex, constant(pointerType(), sizeof(Instance*)));
1253 Value* newContextInstance = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1254 m_currentBlock->appendNew<Value>(m_proc, Add, origin(), instancesBuffer, offset));
1255
1256 BasicBlock* continuation = m_proc.addBlock();
1257 BasicBlock* doContextSwitch = m_proc.addBlock();
1258
1259 Value* isSameContextInstance = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(),
1260 newContextInstance, instanceValue());
1261 m_currentBlock->appendNewControlValue(m_proc, B3::Branch, origin(),
1262 isSameContextInstance, FrequentedBlock(continuation), FrequentedBlock(doContextSwitch));
1263
1264 PatchpointValue* patchpoint = doContextSwitch->appendNew<PatchpointValue>(m_proc, B3::Void, origin());
1265 patchpoint->effects.writesPinned = true;
1266 // We pessimistically assume we're calling something with BoundsChecking memory.
1267 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1268 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1269 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1270 patchpoint->append(newContextInstance, ValueRep::SomeRegister);
1271 patchpoint->append(instanceValue(), ValueRep::SomeRegister);
1272 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1273 AllowMacroScratchRegisterUsage allowScratch(jit);
1274 GPRReg newContextInstance = params[0].gpr();
1275 GPRReg oldContextInstance = params[1].gpr();
1276 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1277 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1278 ASSERT(newContextInstance != baseMemory);
1279 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1280 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1281 jit.storeWasmContextInstance(newContextInstance);
1282 ASSERT(pinnedRegs.sizeRegister != baseMemory);
1283 // FIXME: We should support more than one memory size register
1284 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1285 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1286 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1287 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1288 });
1289 doContextSwitch->appendNewControlValue(m_proc, Jump, origin(), continuation);
1290
1291 m_currentBlock = continuation;
1292 }
1293
1294 ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(),
1295 m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), origin(), callableFunction,
1296 safeCast<int32_t>(WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation())));
1297
1298 Type returnType = signature.returnType();
1299 result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, origin(), args, toB3Type(returnType),
1300 [=] (PatchpointValue* patchpoint) {
1301 patchpoint->effects.writesPinned = true;
1302 patchpoint->effects.readsPinned = true;
1303 // We need to clobber all potential pinned registers since we might be leaving the instance.
1304 // We pessimistically assume we're always calling something that is bounds checking so
1305 // because the wasm->wasm thunk unconditionally overrides the size registers.
1306 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1307 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1308 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1309
1310 patchpoint->append(calleeCode, ValueRep::SomeRegister);
1311 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1312 AllowMacroScratchRegisterUsage allowScratch(jit);
1313 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1314 });
1315 });
1316
1317 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1318 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, instanceValue(), m_proc, m_currentBlock);
1319
1320 return { };
1321}
1322
1323void B3IRGenerator::unify(const ExpressionType phi, const ExpressionType source)
1324{
1325 m_currentBlock->appendNew<UpsilonValue>(m_proc, origin(), source, phi);
1326}
1327
1328void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1329{
1330 ASSERT(result.size() <= resultStack.size());
1331
1332 for (size_t i = 0; i < result.size(); ++i)
1333 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1334}
1335
1336static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
1337{
1338 dataLog(comma, "ExpressionStack:");
1339 for (const auto& expression : expressionStack)
1340 dataLog(comma, *expression);
1341}
1342
1343void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack)
1344{
1345 dataLogLn("Constants:");
1346 for (const auto& constant : m_constantPool)
1347 dataLogLn(deepDump(m_proc, constant.value));
1348
1349 dataLogLn("Processing Graph:");
1350 dataLog(m_proc);
1351 dataLogLn("With current block:", *m_currentBlock);
1352 dataLogLn("Control stack:");
1353 ASSERT(controlStack.size());
1354 for (size_t i = controlStack.size(); i--;) {
1355 dataLog(" ", controlStack[i].controlData, ": ");
1356 CommaPrinter comma(", ", "");
1357 dumpExpressionStack(comma, *expressionStack);
1358 expressionStack = &controlStack[i].enclosedExpressionStack;
1359 dataLogLn();
1360 }
1361 dataLogLn();
1362}
1363
1364auto B3IRGenerator::origin() -> Origin
1365{
1366 OpcodeOrigin origin(m_parser->currentOpcode(), m_parser->currentOpcodeStartingOffset());
1367 ASSERT(isValidOpType(static_cast<uint8_t>(origin.opcode())));
1368 return bitwise_cast<Origin>(origin);
1369}
1370
1371Expected<std::unique_ptr<InternalFunction>, String> parseAndCompile(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, CompilationMode compilationMode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1372{
1373 auto result = std::make_unique<InternalFunction>();
1374
1375 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1376 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1377
1378 Procedure procedure;
1379
1380 procedure.setOriginPrinter([] (PrintStream& out, Origin origin) {
1381 if (origin.data())
1382 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1383 });
1384
1385 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1386 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1387 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1388 // optLevel=1.
1389 procedure.setNeedsUsedRegisters(false);
1390
1391 procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
1392 ? Options::webAssemblyBBQOptimizationLevel()
1393 : Options::webAssemblyOMGOptimizationLevel());
1394
1395 B3IRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, compilationMode, functionIndex, tierUp, throwWasmException);
1396 FunctionParser<B3IRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1397 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1398
1399 irGenerator.insertConstants();
1400
1401 procedure.resetReachability();
1402 if (!ASSERT_DISABLED)
1403 validate(procedure, "After parsing:\n");
1404
1405 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Pre SSA: ", procedure);
1406 fixSSA(procedure);
1407 dataLogIf(WasmB3IRGeneratorInternal::verbose, "Post SSA: ", procedure);
1408
1409 {
1410 B3::prepareForGeneration(procedure);
1411 B3::generate(procedure, *compilationContext.wasmEntrypointJIT);
1412 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1413 result->entrypoint.calleeSaveRegisters = procedure.calleeSaveRegisterAtOffsetList();
1414 }
1415
1416 return result;
1417}
1418
1419// Custom wasm ops. These are the ones too messy to do in wasm.json.
1420
1421void B3IRGenerator::emitChecksForModOrDiv(B3::Opcode operation, ExpressionType left, ExpressionType right)
1422{
1423 ASSERT(operation == Div || operation == Mod || operation == UDiv || operation == UMod);
1424 const B3::Type type = left->type();
1425
1426 {
1427 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1428 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, 0)));
1429
1430 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1431 this->emitExceptionCheck(jit, ExceptionType::DivisionByZero);
1432 });
1433 }
1434
1435 if (operation == Div) {
1436 int64_t min = type == Int32 ? std::numeric_limits<int32_t>::min() : std::numeric_limits<int64_t>::min();
1437
1438 CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(),
1439 m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1440 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), left, constant(type, min)),
1441 m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), right, constant(type, -1))));
1442
1443 check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1444 this->emitExceptionCheck(jit, ExceptionType::IntegerOverflow);
1445 });
1446 }
1447}
1448
1449template<>
1450auto B3IRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1451{
1452 const B3::Opcode op = Div;
1453 emitChecksForModOrDiv(op, left, right);
1454 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1455 return { };
1456}
1457
1458template<>
1459auto B3IRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1460{
1461 const B3::Opcode op = Mod;
1462 emitChecksForModOrDiv(op, left, right);
1463 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1464 return { };
1465}
1466
1467template<>
1468auto B3IRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1469{
1470 const B3::Opcode op = UDiv;
1471 emitChecksForModOrDiv(op, left, right);
1472 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1473 return { };
1474}
1475
1476template<>
1477auto B3IRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1478{
1479 const B3::Opcode op = UMod;
1480 emitChecksForModOrDiv(op, left, right);
1481 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1482 return { };
1483}
1484
1485template<>
1486auto B3IRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1487{
1488 const B3::Opcode op = Div;
1489 emitChecksForModOrDiv(op, left, right);
1490 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1491 return { };
1492}
1493
1494template<>
1495auto B3IRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1496{
1497 const B3::Opcode op = Mod;
1498 emitChecksForModOrDiv(op, left, right);
1499 result = m_currentBlock->appendNew<Value>(m_proc, chill(op), origin(), left, right);
1500 return { };
1501}
1502
1503template<>
1504auto B3IRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1505{
1506 const B3::Opcode op = UDiv;
1507 emitChecksForModOrDiv(op, left, right);
1508 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1509 return { };
1510}
1511
1512template<>
1513auto B3IRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
1514{
1515 const B3::Opcode op = UMod;
1516 emitChecksForModOrDiv(op, left, right);
1517 result = m_currentBlock->appendNew<Value>(m_proc, op, origin(), left, right);
1518 return { };
1519}
1520
1521template<>
1522auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1523{
1524 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1525 patchpoint->append(arg, ValueRep::SomeRegister);
1526 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1527 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
1528 });
1529 patchpoint->effects = Effects::none();
1530 result = patchpoint;
1531 return { };
1532}
1533
1534template<>
1535auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
1536{
1537 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1538 patchpoint->append(arg, ValueRep::SomeRegister);
1539 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1540 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
1541 });
1542 patchpoint->effects = Effects::none();
1543 result = patchpoint;
1544 return { };
1545}
1546
1547template<>
1548auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1549{
1550#if CPU(X86_64)
1551 if (MacroAssembler::supportsCountPopulation()) {
1552 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1553 patchpoint->append(arg, ValueRep::SomeRegister);
1554 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1555 jit.countPopulation32(params[1].gpr(), params[0].gpr());
1556 });
1557 patchpoint->effects = Effects::none();
1558 result = patchpoint;
1559 return { };
1560 }
1561#endif
1562
1563 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
1564 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1565 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, origin(), Effects::none(), funcAddress, arg);
1566 return { };
1567}
1568
1569template<>
1570auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
1571{
1572#if CPU(X86_64)
1573 if (MacroAssembler::supportsCountPopulation()) {
1574 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1575 patchpoint->append(arg, ValueRep::SomeRegister);
1576 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1577 jit.countPopulation64(params[1].gpr(), params[0].gpr());
1578 });
1579 patchpoint->effects = Effects::none();
1580 result = patchpoint;
1581 return { };
1582 }
1583#endif
1584
1585 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
1586 Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, origin(), tagCFunctionPtr<void*>(popcount, B3CCallPtrTag));
1587 result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, origin(), Effects::none(), funcAddress, arg);
1588 return { };
1589}
1590
1591template<>
1592auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1593{
1594 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1595 if (isX86())
1596 patchpoint->numGPScratchRegisters = 1;
1597 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1598 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1599 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1600 AllowMacroScratchRegisterUsage allowScratch(jit);
1601#if CPU(X86_64)
1602 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1603#else
1604 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1605#endif
1606 });
1607 patchpoint->effects = Effects::none();
1608 result = patchpoint;
1609 return { };
1610}
1611
1612template<>
1613auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1614{
1615 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1616 if (isX86())
1617 patchpoint->numGPScratchRegisters = 1;
1618 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1619 patchpoint->append(ConstrainedValue(arg, ValueRep::SomeRegister));
1620 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1621 AllowMacroScratchRegisterUsage allowScratch(jit);
1622#if CPU(X86_64)
1623 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1624#else
1625 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1626#endif
1627 });
1628 patchpoint->effects = Effects::none();
1629 result = patchpoint;
1630 return { };
1631}
1632
1633template<>
1634auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1635{
1636 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1637 patchpoint->append(arg, ValueRep::SomeRegister);
1638 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1639 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1640 });
1641 patchpoint->effects = Effects::none();
1642 result = patchpoint;
1643 return { };
1644}
1645
1646template<>
1647auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1648{
1649 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1650 patchpoint->append(arg, ValueRep::SomeRegister);
1651 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1652 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1653 });
1654 patchpoint->effects = Effects::none();
1655 result = patchpoint;
1656 return { };
1657}
1658
1659template<>
1660auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1661{
1662 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, origin());
1663 patchpoint->append(arg, ValueRep::SomeRegister);
1664 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1665 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1666 });
1667 patchpoint->effects = Effects::none();
1668 result = patchpoint;
1669 return { };
1670}
1671
1672template<>
1673auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1674{
1675 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, origin());
1676 patchpoint->append(arg, ValueRep::SomeRegister);
1677 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1678 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1679 });
1680 patchpoint->effects = Effects::none();
1681 result = patchpoint;
1682 return { };
1683}
1684
1685template<>
1686auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1687{
1688 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
1689 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
1690 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1691 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1692 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1693 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1694 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1695 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1696 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1697 });
1698 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1699 patchpoint->append(arg, ValueRep::SomeRegister);
1700 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1701 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1702 });
1703 patchpoint->effects = Effects::none();
1704 result = patchpoint;
1705 return { };
1706}
1707
1708template<>
1709auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1710{
1711 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
1712 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
1713 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1714 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1715 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1716 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1717 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1718 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1719 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1720 });
1721 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1722 patchpoint->append(arg, ValueRep::SomeRegister);
1723 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1724 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1725 });
1726 patchpoint->effects = Effects::none();
1727 result = patchpoint;
1728 return { };
1729}
1730
1731
1732template<>
1733auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1734{
1735 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
1736 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1737 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1738 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1739 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1740 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1741 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1742 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1743 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1744 });
1745 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1746 patchpoint->append(arg, ValueRep::SomeRegister);
1747 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1748 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1749 });
1750 patchpoint->effects = Effects::none();
1751 result = patchpoint;
1752 return { };
1753}
1754
1755template<>
1756auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1757{
1758 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
1759 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1760 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1761 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1762 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1763 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1764 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1765 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1766 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1767 });
1768 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, origin());
1769 patchpoint->append(arg, ValueRep::SomeRegister);
1770 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1771 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1772 });
1773 patchpoint->effects = Effects::none();
1774 result = patchpoint;
1775 return { };
1776}
1777
1778template<>
1779auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1780{
1781 Value* max = constant(Double, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
1782 Value* min = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
1783 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1784 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1785 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1786 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1787 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1788 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1789 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1790 });
1791 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1792 patchpoint->append(arg, ValueRep::SomeRegister);
1793 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1794 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1795 });
1796 patchpoint->effects = Effects::none();
1797 result = patchpoint;
1798 return { };
1799}
1800
1801template<>
1802auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1803{
1804 Value* max = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
1805 Value* min = constant(Double, bitwise_cast<uint64_t>(-1.0));
1806 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1807 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1808 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1809 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1810 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1811 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1812 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1813 });
1814
1815 Value* signBitConstant;
1816 if (isX86()) {
1817 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1818 // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1819 // so we can pool them if needed.
1820 signBitConstant = constant(Double, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1821 }
1822 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1823 patchpoint->append(arg, ValueRep::SomeRegister);
1824 if (isX86()) {
1825 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1826 patchpoint->numFPScratchRegisters = 1;
1827 }
1828 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1829 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1830 AllowMacroScratchRegisterUsage allowScratch(jit);
1831 FPRReg scratch = InvalidFPRReg;
1832 FPRReg constant = InvalidFPRReg;
1833 if (isX86()) {
1834 scratch = params.fpScratch(0);
1835 constant = params[2].fpr();
1836 }
1837 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1838 });
1839 patchpoint->effects = Effects::none();
1840 result = patchpoint;
1841 return { };
1842}
1843
1844template<>
1845auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1846{
1847 Value* max = constant(Float, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
1848 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
1849 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1850 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1851 m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, origin(), arg, min));
1852 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1853 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1854 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1855 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1856 });
1857 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1858 patchpoint->append(arg, ValueRep::SomeRegister);
1859 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1860 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1861 });
1862 patchpoint->effects = Effects::none();
1863 result = patchpoint;
1864 return { };
1865}
1866
1867template<>
1868auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1869{
1870 Value* max = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
1871 Value* min = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
1872 Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, origin(),
1873 m_currentBlock->appendNew<Value>(m_proc, LessThan, origin(), arg, max),
1874 m_currentBlock->appendNew<Value>(m_proc, GreaterThan, origin(), arg, min));
1875 outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, origin(), outOfBounds, constant(Int32, 0));
1876 CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, origin(), outOfBounds);
1877 trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1878 this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1879 });
1880
1881 Value* signBitConstant;
1882 if (isX86()) {
1883 // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1884 // the numbers would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1885 // so we can pool them if needed.
1886 signBitConstant = constant(Float, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
1887 }
1888 PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, origin());
1889 patchpoint->append(arg, ValueRep::SomeRegister);
1890 if (isX86()) {
1891 patchpoint->append(signBitConstant, ValueRep::SomeRegister);
1892 patchpoint->numFPScratchRegisters = 1;
1893 }
1894 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1895 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1896 AllowMacroScratchRegisterUsage allowScratch(jit);
1897 FPRReg scratch = InvalidFPRReg;
1898 FPRReg constant = InvalidFPRReg;
1899 if (isX86()) {
1900 scratch = params.fpScratch(0);
1901 constant = params[2].fpr();
1902 }
1903 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1904 });
1905 patchpoint->effects = Effects::none();
1906 result = patchpoint;
1907 return { };
1908}
1909
1910} } // namespace JSC::Wasm
1911
1912#include "WasmB3IRGeneratorInlines.h"
1913
1914#endif // ENABLE(WEBASSEMBLY)
1915