1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmAirIRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AirCode.h"
32#include "AirGenerate.h"
33#include "AirOpcodeUtils.h"
34#include "AirValidate.h"
35#include "AllowMacroScratchRegisterUsageIf.h"
36#include "B3CCallValue.h"
37#include "B3CheckSpecial.h"
38#include "B3CheckValue.h"
39#include "B3PatchpointSpecial.h"
40#include "B3Procedure.h"
41#include "B3ProcedureInlines.h"
42#include "BinarySwitch.h"
43#include "ScratchRegisterAllocator.h"
44#include "VirtualRegister.h"
45#include "WasmCallingConvention.h"
46#include "WasmContextInlines.h"
47#include "WasmExceptionType.h"
48#include "WasmFunctionParser.h"
49#include "WasmInstance.h"
50#include "WasmMemory.h"
51#include "WasmOMGPlan.h"
52#include "WasmOpcodeOrigin.h"
53#include "WasmSignatureInlines.h"
54#include "WasmThunks.h"
55#include <limits>
56#include <wtf/Box.h>
57#include <wtf/Optional.h>
58#include <wtf/StdLibExtras.h>
59
60namespace JSC { namespace Wasm {
61
62using namespace B3::Air;
63
64struct ConstrainedTmp {
65 ConstrainedTmp(Tmp tmp)
66 : ConstrainedTmp(tmp, tmp.isReg() ? B3::ValueRep::reg(tmp.reg()) : B3::ValueRep::SomeRegister)
67 { }
68
69 ConstrainedTmp(Tmp tmp, B3::ValueRep rep)
70 : tmp(tmp)
71 , rep(rep)
72 {
73 }
74
75 Tmp tmp;
76 B3::ValueRep rep;
77};
78
79class TypedTmp {
80public:
81 constexpr TypedTmp()
82 : m_tmp()
83 , m_type(Type::Void)
84 { }
85
86 TypedTmp(Tmp tmp, Type type)
87 : m_tmp(tmp)
88 , m_type(type)
89 { }
90
91 TypedTmp(const TypedTmp&) = default;
92 TypedTmp(TypedTmp&&) = default;
93 TypedTmp& operator=(TypedTmp&&) = default;
94 TypedTmp& operator=(const TypedTmp&) = default;
95
96 bool operator==(const TypedTmp& other) const
97 {
98 return m_tmp == other.m_tmp && m_type == other.m_type;
99 }
100 bool operator!=(const TypedTmp& other) const
101 {
102 return !(*this == other);
103 }
104
105 explicit operator bool() const { return !!tmp(); }
106
107 operator Tmp() const { return tmp(); }
108 operator Arg() const { return Arg(tmp()); }
109 Tmp tmp() const { return m_tmp; }
110 Type type() const { return m_type; }
111
112private:
113
114 Tmp m_tmp;
115 Type m_type;
116};
117
118class AirIRGenerator {
119public:
120 struct ControlData {
121 ControlData(B3::Origin origin, Type returnType, TypedTmp resultTmp, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
122 : blockType(type)
123 , continuation(continuation)
124 , special(special)
125 , returnType(returnType)
126 {
127 UNUSED_PARAM(origin); // FIXME: Use origin.
128 if (resultTmp) {
129 ASSERT(returnType != Type::Void);
130 result.append(resultTmp);
131 } else
132 ASSERT(returnType == Type::Void);
133 }
134
135 ControlData()
136 {
137 }
138
139 void dump(PrintStream& out) const
140 {
141 switch (type()) {
142 case BlockType::If:
143 out.print("If: ");
144 break;
145 case BlockType::Block:
146 out.print("Block: ");
147 break;
148 case BlockType::Loop:
149 out.print("Loop: ");
150 break;
151 case BlockType::TopLevel:
152 out.print("TopLevel: ");
153 break;
154 }
155 out.print("Continuation: ", *continuation, ", Special: ");
156 if (special)
157 out.print(*special);
158 else
159 out.print("None");
160 }
161
162 BlockType type() const { return blockType; }
163
164 Type signature() const { return returnType; }
165
166 bool hasNonVoidSignature() const { return result.size(); }
167
168 BasicBlock* targetBlockForBranch()
169 {
170 if (type() == BlockType::Loop)
171 return special;
172 return continuation;
173 }
174
175 void convertIfToBlock()
176 {
177 ASSERT(type() == BlockType::If);
178 blockType = BlockType::Block;
179 special = nullptr;
180 }
181
182 using ResultList = Vector<TypedTmp, 1>;
183
184 ResultList resultForBranch() const
185 {
186 if (type() == BlockType::Loop)
187 return ResultList();
188 return result;
189 }
190
191 private:
192 friend class AirIRGenerator;
193 BlockType blockType;
194 BasicBlock* continuation;
195 BasicBlock* special;
196 ResultList result;
197 Type returnType;
198 };
199
200 using ExpressionType = TypedTmp;
201 using ControlType = ControlData;
202 using ExpressionList = Vector<ExpressionType, 1>;
203 using ResultList = ControlData::ResultList;
204 using ControlEntry = FunctionParser<AirIRGenerator>::ControlEntry;
205
206 static ExpressionType emptyExpression() { return { }; };
207
208 using ErrorType = String;
209 using UnexpectedResult = Unexpected<ErrorType>;
210 using Result = Expected<std::unique_ptr<InternalFunction>, ErrorType>;
211 using PartialResult = Expected<void, ErrorType>;
212
213 template <typename ...Args>
214 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
215 {
216 using namespace FailureHelper; // See ADL comment in WasmParser.h.
217 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
218 }
219
220#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
221 if (UNLIKELY(condition)) \
222 return fail(__VA_ARGS__); \
223 } while (0)
224
225 AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
226
227 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
228 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
229 ExpressionType addConstant(Type, uint64_t);
230 ExpressionType addConstant(BasicBlock*, Type, uint64_t);
231
232 // Locals
233 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
234 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
235
236 // Globals
237 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
238 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
239
240 // Memory
241 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
242 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
243 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
244 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
245
246 // Basic operators
247 template<OpType>
248 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
249 template<OpType>
250 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
251 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
252
253 // Control flow
254 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
255 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
256 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
257 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
258 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
259 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
260
261 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
262 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
263 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
264 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
265 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
266
267 // Calls
268 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
269 PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
270 PartialResult WARN_UNUSED_RETURN addUnreachable();
271
272 PartialResult addShift(Type, B3::Air::Opcode, ExpressionType value, ExpressionType shift, ExpressionType& result);
273 PartialResult addIntegerSub(B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
274 PartialResult addFloatingPointAbs(B3::Air::Opcode, ExpressionType value, ExpressionType& result);
275 PartialResult addFloatingPointBinOp(Type, B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
276
277 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
278 void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };
279
280 static Vector<Tmp> toTmpVector(const Vector<TypedTmp>& vector)
281 {
282 Vector<Tmp> result;
283 for (const auto& item : vector)
284 result.append(item.tmp());
285 return result;
286 }
287
288 ALWAYS_INLINE void didKill(const ExpressionType& typedTmp)
289 {
290 Tmp tmp = typedTmp.tmp();
291 if (!tmp)
292 return;
293 if (tmp.isGP())
294 m_freeGPs.append(tmp);
295 else
296 m_freeFPs.append(tmp);
297 }
298
299private:
300 ALWAYS_INLINE void validateInst(Inst& inst)
301 {
302 if (!ASSERT_DISABLED) {
303 if (!inst.isValidForm()) {
304 dataLogLn(inst);
305 CRASH();
306 }
307 }
308 }
309
310 static Arg extractArg(const TypedTmp& tmp) { return tmp.tmp(); }
311 static Arg extractArg(const Tmp& tmp) { return Arg(tmp); }
312 static Arg extractArg(const Arg& arg) { return arg; }
313
314 template<typename... Arguments>
315 void append(BasicBlock* block, Kind kind, Arguments&&... arguments)
316 {
317 // FIXME: Find a way to use origin here.
318 auto& inst = block->append(kind, nullptr, extractArg(arguments)...);
319 validateInst(inst);
320 }
321
322 template<typename... Arguments>
323 void append(Kind kind, Arguments&&... arguments)
324 {
325 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
326 }
327
328 template<typename... Arguments>
329 void appendEffectful(B3::Air::Opcode op, Arguments&&... arguments)
330 {
331 Kind kind = op;
332 kind.effects = true;
333 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
334 }
335
336 Tmp newTmp(B3::Bank bank)
337 {
338 switch (bank) {
339 case B3::GP:
340 if (m_freeGPs.size())
341 return m_freeGPs.takeLast();
342 break;
343 case B3::FP:
344 if (m_freeFPs.size())
345 return m_freeFPs.takeLast();
346 break;
347 }
348 return m_code.newTmp(bank);
349 }
350
351 TypedTmp g32() { return { newTmp(B3::GP), Type::I32 }; }
352 TypedTmp g64() { return { newTmp(B3::GP), Type::I64 }; }
353 TypedTmp f32() { return { newTmp(B3::FP), Type::F32 }; }
354 TypedTmp f64() { return { newTmp(B3::FP), Type::F64 }; }
355
356 TypedTmp tmpForType(Type type)
357 {
358 switch (type) {
359 case Type::I32:
360 return g32();
361 case Type::I64:
362 return g64();
363 case Type::F32:
364 return f32();
365 case Type::F64:
366 return f64();
367 case Type::Void:
368 return { };
369 default:
370 RELEASE_ASSERT_NOT_REACHED();
371 }
372 }
373
374 B3::PatchpointValue* addPatchpoint(B3::Type type)
375 {
376 return m_proc.add<B3::PatchpointValue>(type, B3::Origin());
377 }
378
379 template <typename ...Args>
380 void emitPatchpoint(B3::PatchpointValue* patch, Tmp result, Args... theArgs)
381 {
382 emitPatchpoint(m_currentBlock, patch, result, std::forward<Args>(theArgs)...);
383 }
384
385 template <typename ...Args>
386 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Args... theArgs)
387 {
388 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp, sizeof...(Args)>::from(theArgs...));
389 }
390
391 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result)
392 {
393 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp>());
394 }
395
396 template <size_t inlineSize>
397 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Vector<ConstrainedTmp, inlineSize>&& args)
398 {
399 if (!m_patchpointSpecial)
400 m_patchpointSpecial = static_cast<B3::PatchpointSpecial*>(m_code.addSpecial(std::make_unique<B3::PatchpointSpecial>()));
401
402 Inst inst(Patch, patch, Arg::special(m_patchpointSpecial));
403 Inst resultMov;
404 if (result) {
405 ASSERT(patch->type() != B3::Void);
406 switch (patch->resultConstraint.kind()) {
407 case B3::ValueRep::Register:
408 inst.args.append(Tmp(patch->resultConstraint.reg()));
409 resultMov = Inst(result.isGP() ? Move : MoveDouble, nullptr, Tmp(patch->resultConstraint.reg()), result);
410 break;
411 case B3::ValueRep::SomeRegister:
412 inst.args.append(result);
413 break;
414 default:
415 RELEASE_ASSERT_NOT_REACHED();
416 }
417 } else
418 ASSERT(patch->type() == B3::Void);
419
420 for (ConstrainedTmp& tmp : args) {
421 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
422 // validation. We should abstrcat Patch enough so ValueRep's don't need to be
423 // backed by Values.
424 // https://bugs.webkit.org/show_bug.cgi?id=194040
425 B3::Value* dummyValue = m_proc.addConstant(B3::Origin(), tmp.tmp.isGP() ? B3::Int64 : B3::Double, 0);
426 patch->append(dummyValue, tmp.rep);
427 switch (tmp.rep.kind()) {
428 case B3::ValueRep::SomeRegister:
429 inst.args.append(tmp.tmp);
430 break;
431 case B3::ValueRep::Register:
432 patch->earlyClobbered().clear(tmp.rep.reg());
433 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, tmp.rep.reg());
434 inst.args.append(Tmp(tmp.rep.reg()));
435 break;
436 case B3::ValueRep::StackArgument: {
437 auto arg = Arg::callArg(tmp.rep.offsetFromSP());
438 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, arg);
439 inst.args.append(arg);
440 break;
441 }
442 default:
443 RELEASE_ASSERT_NOT_REACHED();
444 }
445 }
446
447 if (patch->resultConstraint.isReg())
448 patch->lateClobbered().clear(patch->resultConstraint.reg());
449 for (unsigned i = patch->numGPScratchRegisters; i--;)
450 inst.args.append(g64().tmp());
451 for (unsigned i = patch->numFPScratchRegisters; i--;)
452 inst.args.append(f64().tmp());
453
454 validateInst(inst);
455 basicBlock->append(WTFMove(inst));
456 if (resultMov) {
457 validateInst(resultMov);
458 basicBlock->append(WTFMove(resultMov));
459 }
460 }
461
462 template <typename Branch, typename Generator>
463 void emitCheck(const Branch& makeBranch, const Generator& generator)
464 {
465 // We fail along the truthy edge of 'branch'.
466 Inst branch = makeBranch();
467
468 // FIXME: Make a hashmap of these.
469 B3::CheckSpecial::Key key(branch);
470 B3::CheckSpecial* special = static_cast<B3::CheckSpecial*>(m_code.addSpecial(std::make_unique<B3::CheckSpecial>(key)));
471
472 // FIXME: Remove the need for dummy values
473 // https://bugs.webkit.org/show_bug.cgi?id=194040
474 B3::Value* dummyPredicate = m_proc.addConstant(B3::Origin(), B3::Int32, 42);
475 B3::CheckValue* checkValue = m_proc.add<B3::CheckValue>(B3::Check, B3::Origin(), dummyPredicate);
476 checkValue->setGenerator(generator);
477
478 Inst inst(Patch, checkValue, Arg::special(special));
479 inst.args.appendVector(branch.args);
480 m_currentBlock->append(WTFMove(inst));
481 }
482
483 template <typename Func, typename ...Args>
484 void emitCCall(Func func, TypedTmp result, Args... args)
485 {
486 emitCCall(m_currentBlock, func, result, std::forward<Args>(args)...);
487 }
488 template <typename Func, typename ...Args>
489 void emitCCall(BasicBlock* block, Func func, TypedTmp result, Args... theArgs)
490 {
491 B3::Type resultType = B3::Void;
492 if (result) {
493 switch (result.type()) {
494 case Type::I32:
495 resultType = B3::Int32;
496 break;
497 case Type::I64:
498 resultType = B3::Int64;
499 break;
500 case Type::F32:
501 resultType = B3::Float;
502 break;
503 case Type::F64:
504 resultType = B3::Double;
505 break;
506 default:
507 RELEASE_ASSERT_NOT_REACHED();
508 }
509 }
510
511 auto makeDummyValue = [&] (Tmp tmp) {
512 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
513 // validation. We should abstrcat CCall enough so we're not reliant on arguments
514 // to the B3::CCallValue.
515 // https://bugs.webkit.org/show_bug.cgi?id=194040
516 if (tmp.isGP())
517 return m_proc.addConstant(B3::Origin(), B3::Int64, 0);
518 return m_proc.addConstant(B3::Origin(), B3::Double, 0);
519 };
520
521 B3::Value* dummyFunc = m_proc.addConstant(B3::Origin(), B3::Int64, bitwise_cast<uintptr_t>(func));
522 B3::Value* origin = m_proc.add<B3::CCallValue>(resultType, B3::Origin(), B3::Effects::none(), dummyFunc, makeDummyValue(theArgs)...);
523
524 Inst inst(CCall, origin);
525
526 Tmp callee = g64();
527 append(Move, Arg::immPtr(tagCFunctionPtr<void*>(func, B3CCallPtrTag)), callee);
528 inst.args.append(callee);
529
530 if (result)
531 inst.args.append(result.tmp());
532
533 for (Tmp tmp : Vector<Tmp, sizeof...(Args)>::from(theArgs.tmp()...))
534 inst.args.append(tmp);
535
536 block->append(WTFMove(inst));
537 }
538
539 static B3::Air::Opcode moveOpForValueType(Type type)
540 {
541 switch (type) {
542 case Type::I32:
543 return Move32;
544 case Type::I64:
545 return Move;
546 case Type::F32:
547 return MoveFloat;
548 case Type::F64:
549 return MoveDouble;
550 default:
551 RELEASE_ASSERT_NOT_REACHED();
552 }
553 }
554
555 void emitThrowException(CCallHelpers&, ExceptionType);
556
557 void emitTierUpCheck(uint32_t decrementCount, B3::Origin);
558
559 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
560 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
561 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
562
563 void unify(const ExpressionType& dst, const ExpressionType& source);
564 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
565
566 template <typename IntType>
567 void emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right);
568
569 template <typename IntType>
570 void emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
571
572 enum class MinOrMax { Min, Max };
573
574 PartialResult addFloatingPointMinOrMax(Type, MinOrMax, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
575
576 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
577
578 void restoreWasmContextInstance(BasicBlock*, TypedTmp);
579 enum class RestoreCachedStackLimit { No, Yes };
580 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, TypedTmp instance, BasicBlock*);
581
582 B3::Origin origin();
583
584 FunctionParser<AirIRGenerator>* m_parser { nullptr };
585 const ModuleInformation& m_info;
586 const MemoryMode m_mode { MemoryMode::BoundsChecking };
587 const unsigned m_functionIndex { UINT_MAX };
588 const TierUpCount* m_tierUp { nullptr };
589
590 B3::Procedure& m_proc;
591 Code& m_code;
592 BasicBlock* m_currentBlock { nullptr };
593 BasicBlock* m_rootBlock { nullptr };
594 Vector<TypedTmp> m_locals;
595 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
596 GPRReg m_memoryBaseGPR { InvalidGPRReg };
597 GPRReg m_memorySizeGPR { InvalidGPRReg };
598 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
599 bool m_makesCalls { false };
600
601 Vector<Tmp, 8> m_freeGPs;
602 Vector<Tmp, 8> m_freeFPs;
603
604 TypedTmp m_instanceValue; // Always use the accessor below to ensure the instance value is materialized when used.
605 bool m_usesInstanceValue { false };
606 TypedTmp instanceValue()
607 {
608 m_usesInstanceValue = true;
609 return m_instanceValue;
610 }
611
612 uint32_t m_maxNumJSCallArguments { 0 };
613
614 B3::PatchpointSpecial* m_patchpointSpecial { nullptr };
615};
616
617// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
618int32_t AirIRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
619{
620 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
621 auto previousPtr = ptr;
622 ptr = g64();
623 auto constant = g64();
624 append(Move, Arg::bigImm(offset), constant);
625 append(Add64, constant, previousPtr, ptr);
626 return 0;
627 }
628 return offset;
629}
630
631void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp instance)
632{
633 if (Context::useFastTLS()) {
634 auto* patchpoint = addPatchpoint(B3::Void);
635 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
636 patchpoint->clobber(RegisterSet::macroScratchRegisters());
637 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
638 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
639 jit.storeWasmContextInstance(params[0].gpr());
640 });
641 emitPatchpoint(block, patchpoint, Tmp(), instance);
642 return;
643 }
644
645 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
646 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
647 auto* patchpoint = addPatchpoint(B3::Void);
648 B3::Effects effects = B3::Effects::none();
649 effects.writesPinned = true;
650 effects.reads = B3::HeapRange::top();
651 patchpoint->effects = effects;
652 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
653 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
654 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& param) {
655 jit.move(param[0].gpr(), wasmContextInstanceGPR);
656 });
657 emitPatchpoint(block, patchpoint, Tmp(), instance);
658}
659
660AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
661 : m_info(info)
662 , m_mode(mode)
663 , m_functionIndex(functionIndex)
664 , m_tierUp(tierUp)
665 , m_proc(procedure)
666 , m_code(m_proc.code())
667 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
668{
669 m_currentBlock = m_code.addBlock();
670 m_rootBlock = m_currentBlock;
671
672 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
673 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
674
675 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
676 m_code.pinRegister(m_memoryBaseGPR);
677
678 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
679 if (!Context::useFastTLS())
680 m_code.pinRegister(m_wasmContextInstanceGPR);
681
682 if (mode != MemoryMode::Signaling) {
683 m_memorySizeGPR = pinnedRegs.sizeRegister;
684 m_code.pinRegister(m_memorySizeGPR);
685 }
686
687 if (throwWasmException)
688 Thunks::singleton().setThrowWasmException(throwWasmException);
689
690 if (info.memory) {
691 switch (m_mode) {
692 case MemoryMode::BoundsChecking:
693 break;
694 case MemoryMode::Signaling:
695 // Most memory accesses in signaling mode don't do an explicit
696 // exception check because they can rely on fault handling to detect
697 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
698 // thunk to exist so that it can jump to that thunk.
699 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
700 CRASH();
701 break;
702 }
703 }
704
705 m_code.setNumEntrypoints(1);
706
707 GPRReg contextInstance = Context::useFastTLS() ? wasmCallingConventionAir().prologueScratch(1) : m_wasmContextInstanceGPR;
708
709 Ref<B3::Air::PrologueGenerator> prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=] (CCallHelpers& jit, B3::Air::Code& code) {
710 AllowMacroScratchRegisterUsage allowScratch(jit);
711 code.emitDefaultPrologue(jit);
712
713 {
714 GPRReg calleeGPR = wasmCallingConventionAir().prologueScratch(0);
715 auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
716 jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
717 compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
718 });
719 jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
720 jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
721 }
722
723 {
724 const Checked<int32_t> wasmFrameSize = m_code.frameSize();
725 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
726 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
727 // This allows us to elide stack checks for functions that are terminal nodes in the call
728 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
729 // having any such terminal node have its parent caller include some extra size in its
730 // own check for it. The goal here is twofold:
731 // 1. Emit less code.
732 // 2. Try to speed things up by skipping stack checks.
733 minimumParentCheckSize,
734 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
735 // spill all arguments to the stack, we ensure that a stack check here covers the
736 // stack that such a stub would use.
737 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
738 ));
739 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
740 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
741 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
742
743 // This allows leaf functions to not do stack checks if their frame size is within
744 // certain limits since their caller would have already done the check.
745 if (needsOverflowCheck) {
746 GPRReg scratch = wasmCallingConventionAir().prologueScratch(0);
747
748 if (Context::useFastTLS())
749 jit.loadWasmContextInstance(contextInstance);
750
751 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), GPRInfo::callFrameRegister, scratch);
752 MacroAssembler::JumpList overflow;
753 if (UNLIKELY(needUnderflowCheck))
754 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch, GPRInfo::callFrameRegister));
755 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch, CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit())));
756 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
757 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
758 });
759 } else if (m_usesInstanceValue && Context::useFastTLS()) {
760 // No overflow check is needed, but the instance values still needs to be correct.
761 jit.loadWasmContextInstance(contextInstance);
762 }
763 }
764 });
765
766 m_code.setPrologueForEntrypoint(0, WTFMove(prologueGenerator));
767
768 if (Context::useFastTLS()) {
769 m_instanceValue = g64();
770 // FIXME: Would be nice to only do this if we use instance value.
771 append(Move, Tmp(contextInstance), m_instanceValue);
772 } else
773 m_instanceValue = { Tmp(contextInstance), Type::I64 };
774
775 ASSERT(!m_locals.size());
776 m_locals.grow(signature.argumentCount());
777 for (unsigned i = 0; i < signature.argumentCount(); ++i) {
778 Type type = signature.argument(i);
779 m_locals[i] = tmpForType(type);
780 }
781
782 wasmCallingConventionAir().loadArguments(signature, [&] (const Arg& arg, unsigned i) {
783 switch (signature.argument(i)) {
784 case Type::I32:
785 append(Move32, arg, m_locals[i]);
786 break;
787 case Type::I64:
788 append(Move, arg, m_locals[i]);
789 break;
790 case Type::F32:
791 append(MoveFloat, arg, m_locals[i]);
792 break;
793 case Type::F64:
794 append(MoveDouble, arg, m_locals[i]);
795 break;
796 default:
797 RELEASE_ASSERT_NOT_REACHED();
798 }
799 });
800
801 emitTierUpCheck(TierUpCount::functionEntryDecrement(), B3::Origin());
802}
803
804void AirIRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, TypedTmp instance, BasicBlock* block)
805{
806 restoreWasmContextInstance(block, instance);
807
808 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
809 // The Instance caches the stack limit, but also knows where its canonical location is.
810 static_assert(sizeof(decltype(static_cast<Instance*>(nullptr)->cachedStackLimit())) == sizeof(uint64_t), "");
811
812 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfPointerToActualStackLimit(), B3::Width64));
813 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedStackLimit(), B3::Width64));
814 auto temp = g64();
815 append(block, Move, Arg::addr(instanceValue(), Instance::offsetOfPointerToActualStackLimit()), temp);
816 append(block, Move, Arg::addr(temp), temp);
817 append(block, Move, temp, Arg::addr(instanceValue(), Instance::offsetOfCachedStackLimit()));
818 }
819
820 if (!!memory) {
821 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
822 RegisterSet clobbers;
823 clobbers.set(pinnedRegs->baseMemoryPointer);
824 clobbers.set(pinnedRegs->sizeRegister);
825
826 auto* patchpoint = addPatchpoint(B3::Void);
827 B3::Effects effects = B3::Effects::none();
828 effects.writesPinned = true;
829 effects.reads = B3::HeapRange::top();
830 patchpoint->effects = effects;
831 patchpoint->clobber(clobbers);
832
833 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
834 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
835 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), pinnedRegs->baseMemoryPointer);
836 });
837
838 emitPatchpoint(block, patchpoint, Tmp(), instance);
839 }
840}
841
842void AirIRGenerator::emitThrowException(CCallHelpers& jit, ExceptionType type)
843{
844 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
845 auto jumpToExceptionStub = jit.jump();
846
847 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
848 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
849 });
850}
851
852auto AirIRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
853{
854 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
855 totalBytesChecked += m_locals.size();
856 uint32_t totalBytes;
857 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
858
859 for (uint32_t i = 0; i < count; ++i) {
860 auto local = tmpForType(type);
861 m_locals.uncheckedAppend(local);
862 switch (type) {
863 case Type::I32:
864 case Type::I64: {
865 append(Xor64, local, local);
866 break;
867 }
868 case Type::F32:
869 case Type::F64: {
870 auto temp = g64();
871 // IEEE 754 "0" is just int32/64 zero.
872 append(Xor64, temp, temp);
873 append(type == Type::F32 ? Move32ToFloat : Move64ToDouble, temp, local);
874 break;
875 }
876 default:
877 RELEASE_ASSERT_NOT_REACHED();
878 }
879 }
880 return { };
881}
882
883auto AirIRGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
884{
885 return addConstant(m_currentBlock, type, value);
886}
887
888auto AirIRGenerator::addConstant(BasicBlock* block, Type type, uint64_t value) -> ExpressionType
889{
890 auto result = tmpForType(type);
891 switch (type) {
892 case Type::I32:
893 case Type::I64:
894 append(block, Move, Arg::bigImm(value), result);
895 break;
896 case Type::F32:
897 case Type::F64: {
898 auto tmp = g64();
899 append(block, Move, Arg::bigImm(value), tmp);
900 append(block, type == Type::F32 ? Move32ToFloat : Move64ToDouble, tmp, result);
901 break;
902 }
903
904 default:
905 RELEASE_ASSERT_NOT_REACHED();
906 }
907
908 return result;
909}
910
911auto AirIRGenerator::addArguments(const Signature& signature) -> PartialResult
912{
913 RELEASE_ASSERT(m_locals.size() == signature.argumentCount()); // We handle arguments in the prologue
914 return { };
915}
916
917auto AirIRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
918{
919 ASSERT(m_locals[index].tmp());
920 result = tmpForType(m_locals[index].type());
921 append(moveOpForValueType(m_locals[index].type()), m_locals[index].tmp(), result);
922 return { };
923}
924
925auto AirIRGenerator::addUnreachable() -> PartialResult
926{
927 B3::PatchpointValue* unreachable = addPatchpoint(B3::Void);
928 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
929 this->emitThrowException(jit, ExceptionType::Unreachable);
930 });
931 unreachable->effects.terminal = true;
932 emitPatchpoint(unreachable, Tmp());
933 return { };
934}
935
936auto AirIRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
937{
938 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
939 instance->storeTopCallFrame(callFrame);
940
941 if (delta < 0)
942 return -1;
943
944 auto grown = instance->memory()->grow(PageCount(delta));
945 if (!grown) {
946 switch (grown.error()) {
947 case Memory::GrowFailReason::InvalidDelta:
948 case Memory::GrowFailReason::InvalidGrowSize:
949 case Memory::GrowFailReason::WouldExceedMaximum:
950 case Memory::GrowFailReason::OutOfMemory:
951 return -1;
952 }
953 RELEASE_ASSERT_NOT_REACHED();
954 }
955
956 return grown.value().pageCount();
957 };
958
959 result = g32();
960 emitCCall(growMemory, result, TypedTmp { Tmp(GPRInfo::callFrameRegister), Type::I64 }, instanceValue(), delta);
961 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_currentBlock);
962
963 return { };
964}
965
966auto AirIRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
967{
968 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
969
970 auto temp1 = g64();
971 auto temp2 = g64();
972
973 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedMemorySize(), B3::Width64));
974 append(Move, Arg::addr(instanceValue(), Instance::offsetOfCachedMemorySize()), temp1);
975 constexpr uint32_t shiftValue = 16;
976 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
977 append(Move, Arg::imm(16), temp2);
978 addShift(Type::I32, Urshift64, temp1, temp2, result);
979 append(Move32, result, result);
980
981 return { };
982}
983
984auto AirIRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
985{
986 ASSERT(m_locals[index].tmp());
987 append(moveOpForValueType(m_locals[index].type()), value, m_locals[index].tmp());
988 return { };
989}
990
991auto AirIRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
992{
993 Type type = m_info.globals[index].type;
994
995 result = tmpForType(type);
996
997 auto temp = g64();
998
999 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1000 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1001
1002 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1003 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1004 append(moveOpForValueType(type), Arg::addr(temp, offset), result);
1005 else {
1006 auto temp2 = g64();
1007 append(Move, Arg::bigImm(offset), temp2);
1008 append(Add64, temp2, temp, temp);
1009 append(moveOpForValueType(type), Arg::addr(temp), result);
1010 }
1011 return { };
1012}
1013
1014auto AirIRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
1015{
1016 auto temp = g64();
1017
1018 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1019 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1020
1021 Type type = m_info.globals[index].type;
1022
1023 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1024 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1025 append(moveOpForValueType(type), value, Arg::addr(temp, offset));
1026 else {
1027 auto temp2 = g64();
1028 append(Move, Arg::bigImm(offset), temp2);
1029 append(Add64, temp2, temp, temp);
1030 append(moveOpForValueType(type), value, Arg::addr(temp));
1031 }
1032
1033 return { };
1034}
1035
1036inline AirIRGenerator::ExpressionType AirIRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
1037{
1038 ASSERT(m_memoryBaseGPR);
1039
1040 auto result = g64();
1041 append(Move32, pointer, result);
1042
1043 switch (m_mode) {
1044 case MemoryMode::BoundsChecking: {
1045 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
1046 ASSERT(m_memorySizeGPR);
1047 ASSERT(sizeOfOperation + offset > offset);
1048 auto temp = g64();
1049 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1050 append(Add64, result, temp);
1051
1052 emitCheck([&] {
1053 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, Tmp(m_memorySizeGPR));
1054 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1055 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1056 });
1057 break;
1058 }
1059
1060 case MemoryMode::Signaling: {
1061 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
1062 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
1063 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
1064 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
1065 // register + large offset access. We don't think this will be generated frequently.
1066 //
1067 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
1068 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
1069 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
1070 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
1071 if (offset >= Memory::fastMappedRedzoneBytes()) {
1072 uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
1073 auto temp = g64();
1074 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1075 append(Add64, result, temp);
1076 auto sizeMax = addConstant(Type::I64, maximum);
1077
1078 emitCheck([&] {
1079 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, sizeMax);
1080 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1081 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1082 });
1083 }
1084 break;
1085 }
1086 }
1087
1088 append(Add64, Tmp(m_memoryBaseGPR), result);
1089 return result;
1090}
1091
1092inline uint32_t sizeOfLoadOp(LoadOpType op)
1093{
1094 switch (op) {
1095 case LoadOpType::I32Load8S:
1096 case LoadOpType::I32Load8U:
1097 case LoadOpType::I64Load8S:
1098 case LoadOpType::I64Load8U:
1099 return 1;
1100 case LoadOpType::I32Load16S:
1101 case LoadOpType::I64Load16S:
1102 case LoadOpType::I32Load16U:
1103 case LoadOpType::I64Load16U:
1104 return 2;
1105 case LoadOpType::I32Load:
1106 case LoadOpType::I64Load32S:
1107 case LoadOpType::I64Load32U:
1108 case LoadOpType::F32Load:
1109 return 4;
1110 case LoadOpType::I64Load:
1111 case LoadOpType::F64Load:
1112 return 8;
1113 }
1114 RELEASE_ASSERT_NOT_REACHED();
1115}
1116
1117inline TypedTmp AirIRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
1118{
1119 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1120
1121 TypedTmp immTmp;
1122 TypedTmp newPtr;
1123 TypedTmp result;
1124
1125 Arg addrArg;
1126 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfLoadOp(op))))
1127 addrArg = Arg::addr(pointer, offset);
1128 else {
1129 immTmp = g64();
1130 newPtr = g64();
1131 append(Move, Arg::bigImm(offset), immTmp);
1132 append(Add64, immTmp, pointer, newPtr);
1133 addrArg = Arg::addr(newPtr);
1134 }
1135
1136 switch (op) {
1137 case LoadOpType::I32Load8S: {
1138 result = g32();
1139 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1140 break;
1141 }
1142
1143 case LoadOpType::I64Load8S: {
1144 result = g64();
1145 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1146 append(SignExtend32ToPtr, result, result);
1147 break;
1148 }
1149
1150 case LoadOpType::I32Load8U: {
1151 result = g32();
1152 appendEffectful(Load8, addrArg, result);
1153 break;
1154 }
1155
1156 case LoadOpType::I64Load8U: {
1157 result = g64();
1158 appendEffectful(Load8, addrArg, result);
1159 break;
1160 }
1161
1162 case LoadOpType::I32Load16S: {
1163 result = g32();
1164 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1165 break;
1166 }
1167
1168 case LoadOpType::I64Load16S: {
1169 result = g64();
1170 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1171 append(SignExtend32ToPtr, result, result);
1172 break;
1173 }
1174
1175 case LoadOpType::I32Load16U: {
1176 result = g32();
1177 appendEffectful(Load16, addrArg, result);
1178 break;
1179 }
1180
1181 case LoadOpType::I64Load16U: {
1182 result = g64();
1183 appendEffectful(Load16, addrArg, result);
1184 break;
1185 }
1186
1187 case LoadOpType::I32Load:
1188 result = g32();
1189 appendEffectful(Move32, addrArg, result);
1190 break;
1191
1192 case LoadOpType::I64Load32U: {
1193 result = g64();
1194 appendEffectful(Move32, addrArg, result);
1195 break;
1196 }
1197
1198 case LoadOpType::I64Load32S: {
1199 result = g64();
1200 appendEffectful(Move32, addrArg, result);
1201 append(SignExtend32ToPtr, result, result);
1202 break;
1203 }
1204
1205 case LoadOpType::I64Load: {
1206 result = g64();
1207 appendEffectful(Move, addrArg, result);
1208 break;
1209 }
1210
1211 case LoadOpType::F32Load: {
1212 result = f32();
1213 appendEffectful(MoveFloat, addrArg, result);
1214 break;
1215 }
1216
1217 case LoadOpType::F64Load: {
1218 result = f64();
1219 appendEffectful(MoveDouble, addrArg, result);
1220 break;
1221 }
1222 }
1223
1224 return result;
1225}
1226
1227auto AirIRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1228{
1229 ASSERT(pointer.tmp().isGP());
1230
1231 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
1232 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1233 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1234 auto* patch = addPatchpoint(B3::Void);
1235 patch->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1236 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1237 });
1238 emitPatchpoint(patch, Tmp());
1239
1240 // We won't reach here, so we just pick a random reg.
1241 switch (op) {
1242 case LoadOpType::I32Load8S:
1243 case LoadOpType::I32Load16S:
1244 case LoadOpType::I32Load:
1245 case LoadOpType::I32Load16U:
1246 case LoadOpType::I32Load8U:
1247 result = g32();
1248 break;
1249 case LoadOpType::I64Load8S:
1250 case LoadOpType::I64Load8U:
1251 case LoadOpType::I64Load16S:
1252 case LoadOpType::I64Load32U:
1253 case LoadOpType::I64Load32S:
1254 case LoadOpType::I64Load:
1255 case LoadOpType::I64Load16U:
1256 result = g64();
1257 break;
1258 case LoadOpType::F32Load:
1259 result = f32();
1260 break;
1261 case LoadOpType::F64Load:
1262 result = f64();
1263 break;
1264 }
1265 } else
1266 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
1267
1268 return { };
1269}
1270
1271inline uint32_t sizeOfStoreOp(StoreOpType op)
1272{
1273 switch (op) {
1274 case StoreOpType::I32Store8:
1275 case StoreOpType::I64Store8:
1276 return 1;
1277 case StoreOpType::I32Store16:
1278 case StoreOpType::I64Store16:
1279 return 2;
1280 case StoreOpType::I32Store:
1281 case StoreOpType::I64Store32:
1282 case StoreOpType::F32Store:
1283 return 4;
1284 case StoreOpType::I64Store:
1285 case StoreOpType::F64Store:
1286 return 8;
1287 }
1288 RELEASE_ASSERT_NOT_REACHED();
1289}
1290
1291
1292inline void AirIRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1293{
1294 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1295
1296 TypedTmp immTmp;
1297 TypedTmp newPtr;
1298
1299 Arg addrArg;
1300 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfStoreOp(op))))
1301 addrArg = Arg::addr(pointer, offset);
1302 else {
1303 immTmp = g64();
1304 newPtr = g64();
1305 append(Move, Arg::bigImm(offset), immTmp);
1306 append(Add64, immTmp, pointer, newPtr);
1307 addrArg = Arg::addr(newPtr);
1308 }
1309
1310 switch (op) {
1311 case StoreOpType::I64Store8:
1312 case StoreOpType::I32Store8:
1313 append(Store8, value, addrArg);
1314 return;
1315
1316 case StoreOpType::I64Store16:
1317 case StoreOpType::I32Store16:
1318 append(Store16, value, addrArg);
1319 return;
1320
1321 case StoreOpType::I64Store32:
1322 case StoreOpType::I32Store:
1323 append(Move32, value, addrArg);
1324 return;
1325
1326 case StoreOpType::I64Store:
1327 append(Move, value, addrArg);
1328 return;
1329
1330 case StoreOpType::F32Store:
1331 append(MoveFloat, value, addrArg);
1332 return;
1333
1334 case StoreOpType::F64Store:
1335 append(MoveDouble, value, addrArg);
1336 return;
1337 }
1338
1339 RELEASE_ASSERT_NOT_REACHED();
1340}
1341
1342auto AirIRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1343{
1344 ASSERT(pointer.tmp().isGP());
1345
1346 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1347 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1348 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1349 auto* throwException = addPatchpoint(B3::Void);
1350 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1351 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1352 });
1353 emitPatchpoint(throwException, Tmp());
1354 } else
1355 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1356
1357 return { };
1358}
1359
1360auto AirIRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1361{
1362 ASSERT(nonZero.type() == zero.type());
1363 result = tmpForType(nonZero.type());
1364 append(moveOpForValueType(nonZero.type()), nonZero, result);
1365
1366 BasicBlock* isZero = m_code.addBlock();
1367 BasicBlock* continuation = m_code.addBlock();
1368
1369 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), condition, condition);
1370 m_currentBlock->setSuccessors(isZero, continuation);
1371
1372 append(isZero, moveOpForValueType(zero.type()), zero, result);
1373 append(isZero, Jump);
1374 isZero->setSuccessors(continuation);
1375
1376 m_currentBlock = continuation;
1377
1378 return { };
1379}
1380
1381void AirIRGenerator::emitTierUpCheck(uint32_t decrementCount, B3::Origin origin)
1382{
1383 UNUSED_PARAM(origin);
1384
1385 if (!m_tierUp)
1386 return;
1387
1388 auto countdownPtr = g64();
1389 auto oldCountdown = g64();
1390 auto newCountdown = g64();
1391
1392 append(Move, Arg::bigImm(reinterpret_cast<uint64_t>(m_tierUp)), countdownPtr);
1393 append(Move32, Arg::addr(countdownPtr), oldCountdown);
1394
1395 RELEASE_ASSERT(Arg::isValidImmForm(decrementCount));
1396 append(Move32, oldCountdown, newCountdown);
1397 append(Sub32, Arg::imm(decrementCount), newCountdown);
1398 append(Move32, newCountdown, Arg::addr(countdownPtr));
1399
1400 auto* patch = addPatchpoint(B3::Void);
1401 B3::Effects effects = B3::Effects::none();
1402 effects.reads = B3::HeapRange::top();
1403 effects.writes = B3::HeapRange::top();
1404 patch->effects = effects;
1405
1406 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1407 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1408 MacroAssembler::Label tierUpResume = jit.label();
1409
1410 params.addLatePath([=] (CCallHelpers& jit) {
1411 tierUp.link(&jit);
1412
1413 const unsigned extraPaddingBytes = 0;
1414 RegisterSet registersToSpill = { };
1415 registersToSpill.add(GPRInfo::argumentGPR1);
1416 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1417
1418 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1419 MacroAssembler::Call call = jit.nearCall();
1420
1421 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1422 jit.jump(tierUpResume);
1423
1424 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1425 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1426
1427 });
1428 });
1429 });
1430
1431 emitPatchpoint(patch, Tmp(), newCountdown, oldCountdown);
1432}
1433
1434AirIRGenerator::ControlData AirIRGenerator::addLoop(Type signature)
1435{
1436 BasicBlock* body = m_code.addBlock();
1437 BasicBlock* continuation = m_code.addBlock();
1438
1439 append(Jump);
1440 m_currentBlock->setSuccessors(body);
1441
1442 m_currentBlock = body;
1443 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1444
1445 return ControlData(origin(), signature, tmpForType(signature), BlockType::Loop, continuation, body);
1446}
1447
1448AirIRGenerator::ControlData AirIRGenerator::addTopLevel(Type signature)
1449{
1450 return ControlData(B3::Origin(), signature, tmpForType(signature), BlockType::TopLevel, m_code.addBlock());
1451}
1452
1453AirIRGenerator::ControlData AirIRGenerator::addBlock(Type signature)
1454{
1455 return ControlData(origin(), signature, tmpForType(signature), BlockType::Block, m_code.addBlock());
1456}
1457
1458auto AirIRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1459{
1460 BasicBlock* taken = m_code.addBlock();
1461 BasicBlock* notTaken = m_code.addBlock();
1462 BasicBlock* continuation = m_code.addBlock();
1463
1464 // Wasm bools are i32.
1465 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1466 m_currentBlock->setSuccessors(taken, notTaken);
1467
1468 m_currentBlock = taken;
1469 result = ControlData(origin(), signature, tmpForType(signature), BlockType::If, continuation, notTaken);
1470 return { };
1471}
1472
1473auto AirIRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1474{
1475 unifyValuesWithBlock(currentStack, data.result);
1476 append(Jump);
1477 m_currentBlock->setSuccessors(data.continuation);
1478 return addElseToUnreachable(data);
1479}
1480
1481auto AirIRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1482{
1483 ASSERT(data.type() == BlockType::If);
1484 m_currentBlock = data.special;
1485 data.convertIfToBlock();
1486 return { };
1487}
1488
1489auto AirIRGenerator::addReturn(const ControlData& data, const ExpressionList& returnValues) -> PartialResult
1490{
1491 ASSERT(returnValues.size() <= 1);
1492 if (returnValues.size()) {
1493 Tmp returnValueGPR = Tmp(GPRInfo::returnValueGPR);
1494 Tmp returnValueFPR = Tmp(FPRInfo::returnValueFPR);
1495 switch (data.signature()) {
1496 case Type::I32:
1497 append(Move32, returnValues[0], returnValueGPR);
1498 append(Ret32, returnValueGPR);
1499 break;
1500 case Type::I64:
1501 append(Move, returnValues[0], returnValueGPR);
1502 append(Ret64, returnValueGPR);
1503 break;
1504 case Type::F32:
1505 append(MoveFloat, returnValues[0], returnValueFPR);
1506 append(RetFloat, returnValueFPR);
1507 break;
1508 case Type::F64:
1509 append(MoveDouble, returnValues[0], returnValueFPR);
1510 append(RetFloat, returnValueFPR);
1511 break;
1512 default:
1513 RELEASE_ASSERT_NOT_REACHED();
1514 }
1515 } else
1516 append(RetVoid);
1517 return { };
1518}
1519
1520// NOTE: All branches in Wasm are on 32-bit ints
1521
1522auto AirIRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1523{
1524 unifyValuesWithBlock(returnValues, data.resultForBranch());
1525
1526 BasicBlock* target = data.targetBlockForBranch();
1527 if (condition) {
1528 BasicBlock* continuation = m_code.addBlock();
1529 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1530 m_currentBlock->setSuccessors(target, continuation);
1531 m_currentBlock = continuation;
1532 } else {
1533 append(Jump);
1534 m_currentBlock->setSuccessors(target);
1535 }
1536
1537 return { };
1538}
1539
1540auto AirIRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1541{
1542 auto& successors = m_currentBlock->successors();
1543 ASSERT(successors.isEmpty());
1544 for (const auto& target : targets) {
1545 unifyValuesWithBlock(expressionStack, target->resultForBranch());
1546 successors.append(target->targetBlockForBranch());
1547 }
1548 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1549 successors.append(defaultTarget.targetBlockForBranch());
1550
1551 ASSERT(condition.type() == Type::I32);
1552
1553 // FIXME: We should consider dynamically switching between a jump table
1554 // and a binary switch depending on the number of successors.
1555 // https://bugs.webkit.org/show_bug.cgi?id=194477
1556
1557 size_t numTargets = targets.size();
1558
1559 auto* patchpoint = addPatchpoint(B3::Void);
1560 patchpoint->effects = B3::Effects::none();
1561 patchpoint->effects.terminal = true;
1562 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1563
1564 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1565 AllowMacroScratchRegisterUsage allowScratch(jit);
1566
1567 Vector<int64_t> cases;
1568 cases.reserveInitialCapacity(numTargets);
1569 for (size_t i = 0; i < numTargets; ++i)
1570 cases.uncheckedAppend(i);
1571
1572 GPRReg valueReg = params[0].gpr();
1573 BinarySwitch binarySwitch(valueReg, cases, BinarySwitch::Int32);
1574
1575 Vector<CCallHelpers::Jump> caseJumps;
1576 caseJumps.resize(numTargets);
1577
1578 while (binarySwitch.advance(jit)) {
1579 unsigned value = binarySwitch.caseValue();
1580 unsigned index = binarySwitch.caseIndex();
1581 ASSERT_UNUSED(value, value == index);
1582 ASSERT(index < numTargets);
1583 caseJumps[index] = jit.jump();
1584 }
1585
1586 CCallHelpers::JumpList fallThrough = binarySwitch.fallThrough();
1587
1588 Vector<Box<CCallHelpers::Label>> successorLabels = params.successorLabels();
1589 ASSERT(successorLabels.size() == caseJumps.size() + 1);
1590
1591 params.addLatePath([=, caseJumps = WTFMove(caseJumps), successorLabels = WTFMove(successorLabels)] (CCallHelpers& jit) {
1592 for (size_t i = 0; i < numTargets; ++i)
1593 caseJumps[i].linkTo(*successorLabels[i], &jit);
1594 fallThrough.linkTo(*successorLabels[numTargets], &jit);
1595 });
1596 });
1597
1598 emitPatchpoint(patchpoint, TypedTmp(), condition);
1599
1600 return { };
1601}
1602
1603auto AirIRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1604{
1605 ControlData& data = entry.controlData;
1606
1607 unifyValuesWithBlock(expressionStack, data.result);
1608 append(Jump);
1609 m_currentBlock->setSuccessors(data.continuation);
1610
1611 return addEndToUnreachable(entry);
1612}
1613
1614
1615auto AirIRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1616{
1617 ControlData& data = entry.controlData;
1618 m_currentBlock = data.continuation;
1619
1620 if (data.type() == BlockType::If) {
1621 append(data.special, Jump);
1622 data.special->setSuccessors(m_currentBlock);
1623 }
1624
1625 for (const auto& result : data.result)
1626 entry.enclosedExpressionStack.append(result);
1627
1628 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1629 if (data.type() == BlockType::TopLevel)
1630 return addReturn(data, entry.enclosedExpressionStack);
1631
1632 return { };
1633}
1634
1635auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1636{
1637 ASSERT(signature.argumentCount() == args.size());
1638
1639 m_makesCalls = true;
1640
1641 Type returnType = signature.returnType();
1642 if (returnType != Type::Void)
1643 result = tmpForType(returnType);
1644
1645 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1646
1647 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1648 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1649
1650 auto currentInstance = g64();
1651 append(Move, instanceValue(), currentInstance);
1652
1653 auto targetInstance = g64();
1654
1655 // FIXME: We should have better isel here.
1656 // https://bugs.webkit.org/show_bug.cgi?id=193999
1657 append(Move, Arg::bigImm(Instance::offsetOfTargetInstance(functionIndex)), targetInstance);
1658 append(Add64, instanceValue(), targetInstance);
1659 append(Move, Arg::addr(targetInstance), targetInstance);
1660
1661 BasicBlock* isWasmBlock = m_code.addBlock();
1662 BasicBlock* isEmbedderBlock = m_code.addBlock();
1663 BasicBlock* continuation = m_code.addBlock();
1664
1665 append(BranchTest64, Arg::resCond(MacroAssembler::NonZero), targetInstance, targetInstance);
1666 m_currentBlock->setSuccessors(isWasmBlock, isEmbedderBlock);
1667
1668 {
1669 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1670 patchpoint->effects.writesPinned = true;
1671 patchpoint->effects.readsPinned = true;
1672 // We need to clobber all potential pinned registers since we might be leaving the instance.
1673 // We pessimistically assume we could be calling to something that is bounds checking.
1674 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1675 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1676
1677 Vector<ConstrainedTmp> patchArgs;
1678 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1679 patchArgs.append({ tmp, rep });
1680 });
1681
1682 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1683 AllowMacroScratchRegisterUsage allowScratch(jit);
1684 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1685 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1686 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1687 });
1688 });
1689
1690 emitPatchpoint(isWasmBlock, patchpoint, result, WTFMove(patchArgs));
1691 append(isWasmBlock, Jump);
1692 isWasmBlock->setSuccessors(continuation);
1693 }
1694
1695 {
1696 auto jumpDestination = g64();
1697 append(isEmbedderBlock, Move, Arg::bigImm(Instance::offsetOfWasmToEmbedderStub(functionIndex)), jumpDestination);
1698 append(isEmbedderBlock, Add64, instanceValue(), jumpDestination);
1699 append(isEmbedderBlock, Move, Arg::addr(jumpDestination), jumpDestination);
1700
1701 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1702 patchpoint->effects.writesPinned = true;
1703 patchpoint->effects.readsPinned = true;
1704 // We need to clobber all potential pinned registers since we might be leaving the instance.
1705 // We pessimistically assume we could be calling to something that is bounds checking.
1706 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1707 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1708
1709 Vector<ConstrainedTmp> patchArgs;
1710 patchArgs.append(jumpDestination);
1711
1712 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1713 patchArgs.append({ tmp, rep });
1714 });
1715
1716 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1717 AllowMacroScratchRegisterUsage allowScratch(jit);
1718 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1719 });
1720
1721 emitPatchpoint(isEmbedderBlock, patchpoint, result, WTFMove(patchArgs));
1722 append(isEmbedderBlock, Jump);
1723 isEmbedderBlock->setSuccessors(continuation);
1724 }
1725
1726 m_currentBlock = continuation;
1727 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1728 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, continuation);
1729 } else {
1730 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1731 patchpoint->effects.writesPinned = true;
1732 patchpoint->effects.readsPinned = true;
1733
1734 Vector<ConstrainedTmp> patchArgs;
1735 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1736 patchArgs.append({ tmp, rep });
1737 });
1738
1739 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1740 AllowMacroScratchRegisterUsage allowScratch(jit);
1741 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1742 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1743 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1744 });
1745 });
1746
1747 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(patchArgs));
1748 }
1749
1750 return { };
1751}
1752
1753auto AirIRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1754{
1755 ExpressionType calleeIndex = args.takeLast();
1756 ASSERT(signature.argumentCount() == args.size());
1757
1758 m_makesCalls = true;
1759 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1760 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1761 // can be to the embedder for our stack check calculation.
1762 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1763
1764 auto currentInstance = g64();
1765 append(Move, instanceValue(), currentInstance);
1766
1767 ExpressionType callableFunctionBuffer = g64();
1768 ExpressionType instancesBuffer = g64();
1769 ExpressionType callableFunctionBufferLength = g64();
1770 {
1771 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfTable(), B3::Width64));
1772 RELEASE_ASSERT(Arg::isValidAddrForm(Table::offsetOfFunctions(), B3::Width64));
1773 RELEASE_ASSERT(Arg::isValidAddrForm(Table::offsetOfInstances(), B3::Width64));
1774 RELEASE_ASSERT(Arg::isValidAddrForm(Table::offsetOfLength(), B3::Width64));
1775
1776 append(Move, Arg::addr(instanceValue(), Instance::offsetOfTable()), callableFunctionBufferLength);
1777 append(Move, Arg::addr(callableFunctionBufferLength, Table::offsetOfFunctions()), callableFunctionBuffer);
1778 append(Move, Arg::addr(callableFunctionBufferLength, Table::offsetOfInstances()), instancesBuffer);
1779 append(Move32, Arg::addr(callableFunctionBufferLength, Table::offsetOfLength()), callableFunctionBufferLength);
1780 }
1781
1782 append(Move32, calleeIndex, calleeIndex);
1783
1784 // Check the index we are looking for is valid.
1785 emitCheck([&] {
1786 return Inst(Branch32, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), calleeIndex, callableFunctionBufferLength);
1787 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1788 this->emitThrowException(jit, ExceptionType::OutOfBoundsCallIndirect);
1789 });
1790
1791 ExpressionType calleeCode = g64();
1792 {
1793 ExpressionType calleeSignatureIndex = g64();
1794 // Compute the offset in the table index space we are looking for.
1795 append(Move, Arg::imm(sizeof(WasmToWasmImportableFunction)), calleeSignatureIndex);
1796 append(Mul64, calleeIndex, calleeSignatureIndex);
1797 append(Add64, callableFunctionBuffer, calleeSignatureIndex);
1798
1799 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()), calleeCode); // Pointer to callee code.
1800
1801 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1802 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1803 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1804
1805 // FIXME: This seems dumb to do two checks just for a nicer error message.
1806 // We should move just to use a single branch and then figure out what
1807 // error to use in the exception handler.
1808
1809 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfSignatureIndex()), calleeSignatureIndex);
1810
1811 emitCheck([&] {
1812 static_assert(Signature::invalidIndex == 0, "");
1813 return Inst(BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), calleeSignatureIndex, calleeSignatureIndex);
1814 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1815 this->emitThrowException(jit, ExceptionType::NullTableEntry);
1816 });
1817
1818 ExpressionType expectedSignatureIndex = g64();
1819 append(Move, Arg::bigImm(SignatureInformation::get(signature)), expectedSignatureIndex);
1820 emitCheck([&] {
1821 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::NotEqual), calleeSignatureIndex, expectedSignatureIndex);
1822 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1823 this->emitThrowException(jit, ExceptionType::BadSignature);
1824 });
1825 }
1826
1827 // Do a context switch if needed.
1828 {
1829 auto newContextInstance = g64();
1830 append(Move, Arg::index(instancesBuffer, calleeIndex, 8, 0), newContextInstance);
1831
1832 BasicBlock* doContextSwitch = m_code.addBlock();
1833 BasicBlock* continuation = m_code.addBlock();
1834
1835 append(Branch64, Arg::relCond(MacroAssembler::Equal), newContextInstance, instanceValue());
1836 m_currentBlock->setSuccessors(continuation, doContextSwitch);
1837
1838 auto* patchpoint = addPatchpoint(B3::Void);
1839 patchpoint->effects.writesPinned = true;
1840 // We pessimistically assume we're calling something with BoundsChecking memory.
1841 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1842 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1843 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1844 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1845 AllowMacroScratchRegisterUsage allowScratch(jit);
1846 GPRReg newContextInstance = params[0].gpr();
1847 GPRReg oldContextInstance = params[1].gpr();
1848 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1849 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1850 ASSERT(newContextInstance != baseMemory);
1851 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1852 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1853 jit.storeWasmContextInstance(newContextInstance);
1854 // FIXME: We should support more than one memory size register
1855 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1856 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1857 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1858 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1859 });
1860
1861 emitPatchpoint(doContextSwitch, patchpoint, Tmp(), newContextInstance, instanceValue());
1862 append(doContextSwitch, Jump);
1863 doContextSwitch->setSuccessors(continuation);
1864
1865 m_currentBlock = continuation;
1866 }
1867
1868 append(Move, Arg::addr(calleeCode), calleeCode);
1869
1870 Type returnType = signature.returnType();
1871 if (returnType != Type::Void)
1872 result = tmpForType(returnType);
1873
1874 auto* patch = addPatchpoint(toB3Type(returnType));
1875 patch->effects.writesPinned = true;
1876 patch->effects.readsPinned = true;
1877 // We need to clobber all potential pinned registers since we might be leaving the instance.
1878 // We pessimistically assume we're always calling something that is bounds checking so
1879 // because the wasm->wasm thunk unconditionally overrides the size registers.
1880 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
1881 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
1882 patch->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1883
1884 Vector<ConstrainedTmp> emitArgs;
1885 emitArgs.append(calleeCode);
1886 wasmCallingConventionAir().setupCall(m_code, returnType, patch, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1887 emitArgs.append({ tmp, rep });
1888 });
1889 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1890 AllowMacroScratchRegisterUsage allowScratch(jit);
1891 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1892 });
1893
1894 emitPatchpoint(m_currentBlock, patch, result, WTFMove(emitArgs));
1895
1896 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1897 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, m_currentBlock);
1898
1899 return { };
1900}
1901
1902void AirIRGenerator::unify(const ExpressionType& dst, const ExpressionType& source)
1903{
1904 ASSERT(dst.type() == source.type());
1905 append(moveOpForValueType(dst.type()), source, dst);
1906}
1907
1908void AirIRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
1909{
1910 ASSERT(result.size() <= resultStack.size());
1911
1912 for (size_t i = 0; i < result.size(); ++i)
1913 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
1914}
1915
1916void AirIRGenerator::dump(const Vector<ControlEntry>&, const ExpressionList*)
1917{
1918}
1919
1920auto AirIRGenerator::origin() -> B3::Origin
1921{
1922 // FIXME: We should implement a way to give Inst's an origin.
1923 return B3::Origin();
1924}
1925
1926Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
1927{
1928 auto result = std::make_unique<InternalFunction>();
1929
1930 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
1931 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
1932
1933 B3::Procedure procedure;
1934 Code& code = procedure.code();
1935
1936 procedure.setOriginPrinter([] (PrintStream& out, B3::Origin origin) {
1937 if (origin.data())
1938 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
1939 });
1940
1941 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
1942 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
1943 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
1944 // optLevel=1.
1945 procedure.setNeedsUsedRegisters(false);
1946
1947 procedure.setOptLevel(Options::webAssemblyBBQOptimizationLevel());
1948
1949 AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, throwWasmException, signature);
1950 FunctionParser<AirIRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
1951 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
1952
1953
1954 for (BasicBlock* block : code) {
1955 for (size_t i = 0; i < block->numSuccessors(); ++i)
1956 block->successorBlock(i)->addPredecessor(block);
1957 }
1958
1959 {
1960 B3::Air::prepareForGeneration(code);
1961 B3::Air::generate(code, *compilationContext.wasmEntrypointJIT);
1962 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
1963 result->entrypoint.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
1964 }
1965
1966 return result;
1967}
1968
1969template <typename IntType>
1970void AirIRGenerator::emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right)
1971{
1972 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
1973
1974 emitCheck([&] {
1975 return Inst(sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), right, right);
1976 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1977 this->emitThrowException(jit, ExceptionType::DivisionByZero);
1978 });
1979
1980 if (isSignedDiv) {
1981 ASSERT(std::is_signed<IntType>::value);
1982 IntType min = std::numeric_limits<IntType>::min();
1983
1984 // FIXME: Better isel for compare with imms here.
1985 // https://bugs.webkit.org/show_bug.cgi?id=193999
1986 auto minTmp = sizeof(IntType) == 4 ? g32() : g64();
1987 auto negOne = sizeof(IntType) == 4 ? g32() : g64();
1988
1989 B3::Air::Opcode op = sizeof(IntType) == 4 ? Compare32 : Compare64;
1990 append(Move, Arg::bigImm(static_cast<uint64_t>(min)), minTmp);
1991 append(op, Arg::relCond(MacroAssembler::Equal), left, minTmp, minTmp);
1992
1993 append(Move, Arg::imm(-1), negOne);
1994 append(op, Arg::relCond(MacroAssembler::Equal), right, negOne, negOne);
1995
1996 emitCheck([&] {
1997 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), minTmp, negOne);
1998 },
1999 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2000 this->emitThrowException(jit, ExceptionType::IntegerOverflow);
2001 });
2002 }
2003}
2004
2005template <typename IntType>
2006void AirIRGenerator::emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result)
2007{
2008 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2009
2010 result = sizeof(IntType) == 4 ? g32() : g64();
2011
2012 bool isSigned = std::is_signed<IntType>::value;
2013
2014 if (isARM64()) {
2015 B3::Air::Opcode div;
2016 switch (sizeof(IntType)) {
2017 case 4:
2018 div = isSigned ? Div32 : UDiv32;
2019 break;
2020 case 8:
2021 div = isSigned ? Div64 : UDiv64;
2022 break;
2023 }
2024
2025 append(div, lhs, rhs, result);
2026
2027 if (!isDiv) {
2028 append(sizeof(IntType) == 4 ? Mul32 : Mul64, result, rhs, result);
2029 append(sizeof(IntType) == 4 ? Sub32 : Sub64, lhs, result, result);
2030 }
2031
2032 return;
2033 }
2034
2035#if CPU(X86) || CPU(X86_64)
2036 Tmp eax(X86Registers::eax);
2037 Tmp edx(X86Registers::edx);
2038
2039 if (isSigned) {
2040 B3::Air::Opcode convertToDoubleWord;
2041 B3::Air::Opcode div;
2042 switch (sizeof(IntType)) {
2043 case 4:
2044 convertToDoubleWord = X86ConvertToDoubleWord32;
2045 div = X86Div32;
2046 break;
2047 case 8:
2048 convertToDoubleWord = X86ConvertToQuadWord64;
2049 div = X86Div64;
2050 break;
2051 default:
2052 RELEASE_ASSERT_NOT_REACHED();
2053 }
2054
2055 // We implement "res = Div<Chill>/Mod<Chill>(num, den)" as follows:
2056 //
2057 // if (den + 1 <=_unsigned 1) {
2058 // if (!den) {
2059 // res = 0;
2060 // goto done;
2061 // }
2062 // if (num == -2147483648) {
2063 // res = isDiv ? num : 0;
2064 // goto done;
2065 // }
2066 // }
2067 // res = num (/ or %) dev;
2068 // done:
2069
2070 BasicBlock* denIsGood = m_code.addBlock();
2071 BasicBlock* denMayBeBad = m_code.addBlock();
2072 BasicBlock* denNotZero = m_code.addBlock();
2073 BasicBlock* continuation = m_code.addBlock();
2074
2075 auto temp = sizeof(IntType) == 4 ? g32() : g64();
2076 auto one = addConstant(sizeof(IntType) == 4 ? Type::I32 : Type::I64, 1);
2077
2078 append(sizeof(IntType) == 4 ? Add32 : Add64, rhs, one, temp);
2079 append(sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Above), temp, one);
2080 m_currentBlock->setSuccessors(denIsGood, denMayBeBad);
2081
2082 append(denMayBeBad, Xor64, result, result);
2083 append(denMayBeBad, sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, Arg::resCond(MacroAssembler::Zero), rhs, rhs);
2084 denMayBeBad->setSuccessors(continuation, denNotZero);
2085
2086 auto min = addConstant(denNotZero, sizeof(IntType) == 4 ? Type::I32 : Type::I64, std::numeric_limits<IntType>::min());
2087 if (isDiv)
2088 append(denNotZero, sizeof(IntType) == 4 ? Move32 : Move, min, result);
2089 else {
2090 // Result is zero, as set above...
2091 }
2092 append(denNotZero, sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Equal), lhs, min);
2093 denNotZero->setSuccessors(continuation, denIsGood);
2094
2095 auto divResult = isDiv ? eax : edx;
2096 append(denIsGood, Move, lhs, eax);
2097 append(denIsGood, convertToDoubleWord, eax, edx);
2098 append(denIsGood, div, eax, edx, rhs);
2099 append(denIsGood, sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2100 append(denIsGood, Jump);
2101 denIsGood->setSuccessors(continuation);
2102
2103 m_currentBlock = continuation;
2104 return;
2105 }
2106
2107 B3::Air::Opcode div = sizeof(IntType) == 4 ? X86UDiv32 : X86UDiv64;
2108
2109 Tmp divResult = isDiv ? eax : edx;
2110
2111 append(Move, lhs, eax);
2112 append(Xor64, edx, edx);
2113 append(div, eax, edx, rhs);
2114 append(sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2115#else
2116 RELEASE_ASSERT_NOT_REACHED();
2117#endif
2118}
2119
2120template<>
2121auto AirIRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2122{
2123 emitChecksForModOrDiv<int32_t>(true, left, right);
2124 emitModOrDiv<int32_t>(true, left, right, result);
2125 return { };
2126}
2127
2128template<>
2129auto AirIRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2130{
2131 emitChecksForModOrDiv<int32_t>(false, left, right);
2132 emitModOrDiv<int32_t>(false, left, right, result);
2133 return { };
2134}
2135
2136template<>
2137auto AirIRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2138{
2139 emitChecksForModOrDiv<uint32_t>(false, left, right);
2140 emitModOrDiv<uint32_t>(true, left, right, result);
2141 return { };
2142}
2143
2144template<>
2145auto AirIRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2146{
2147 emitChecksForModOrDiv<uint32_t>(false, left, right);
2148 emitModOrDiv<uint32_t>(false, left, right, result);
2149 return { };
2150}
2151
2152template<>
2153auto AirIRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2154{
2155 emitChecksForModOrDiv<int64_t>(true, left, right);
2156 emitModOrDiv<int64_t>(true, left, right, result);
2157 return { };
2158}
2159
2160template<>
2161auto AirIRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2162{
2163 emitChecksForModOrDiv<int64_t>(false, left, right);
2164 emitModOrDiv<int64_t>(false, left, right, result);
2165 return { };
2166}
2167
2168template<>
2169auto AirIRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2170{
2171 emitChecksForModOrDiv<uint64_t>(false, left, right);
2172 emitModOrDiv<uint64_t>(true, left, right, result);
2173 return { };
2174}
2175
2176template<>
2177auto AirIRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2178{
2179 emitChecksForModOrDiv<uint64_t>(false, left, right);
2180 emitModOrDiv<uint64_t>(false, left, right, result);
2181 return { };
2182}
2183
2184template<>
2185auto AirIRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2186{
2187 auto* patchpoint = addPatchpoint(B3::Int32);
2188 patchpoint->effects = B3::Effects::none();
2189 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2190 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
2191 });
2192 result = g32();
2193 emitPatchpoint(patchpoint, result, arg);
2194 return { };
2195}
2196
2197template<>
2198auto AirIRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2199{
2200 auto* patchpoint = addPatchpoint(B3::Int64);
2201 patchpoint->effects = B3::Effects::none();
2202 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2203 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
2204 });
2205 result = g64();
2206 emitPatchpoint(patchpoint, result, arg);
2207 return { };
2208}
2209
2210template<>
2211auto AirIRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2212{
2213 result = g32();
2214
2215#if CPU(X86_64)
2216 if (MacroAssembler::supportsCountPopulation()) {
2217 auto* patchpoint = addPatchpoint(B3::Int32);
2218 patchpoint->effects = B3::Effects::none();
2219 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2220 jit.countPopulation32(params[1].gpr(), params[0].gpr());
2221 });
2222 emitPatchpoint(patchpoint, result, arg);
2223 return { };
2224 }
2225#endif
2226
2227 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
2228 emitCCall(popcount, result, arg);
2229 return { };
2230}
2231
2232template<>
2233auto AirIRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2234{
2235 result = g64();
2236
2237#if CPU(X86_64)
2238 if (MacroAssembler::supportsCountPopulation()) {
2239 auto* patchpoint = addPatchpoint(B3::Int64);
2240 patchpoint->effects = B3::Effects::none();
2241 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2242 jit.countPopulation64(params[1].gpr(), params[0].gpr());
2243 });
2244 emitPatchpoint(patchpoint, result, arg);
2245 return { };
2246 }
2247#endif
2248
2249 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
2250 emitCCall(popcount, result, arg);
2251 return { };
2252}
2253
2254template<>
2255auto AirIRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2256{
2257 auto* patchpoint = addPatchpoint(B3::Double);
2258 patchpoint->effects = B3::Effects::none();
2259 if (isX86())
2260 patchpoint->numGPScratchRegisters = 1;
2261 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2262 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2263 AllowMacroScratchRegisterUsage allowScratch(jit);
2264#if CPU(X86_64)
2265 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2266#else
2267 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
2268#endif
2269 });
2270 result = f64();
2271 emitPatchpoint(patchpoint, result, arg);
2272 return { };
2273}
2274
2275template<>
2276auto AirIRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2277{
2278 auto* patchpoint = addPatchpoint(B3::Float);
2279 patchpoint->effects = B3::Effects::none();
2280 if (isX86())
2281 patchpoint->numGPScratchRegisters = 1;
2282 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2283 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2284 AllowMacroScratchRegisterUsage allowScratch(jit);
2285#if CPU(X86_64)
2286 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2287#else
2288 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
2289#endif
2290 });
2291 result = f32();
2292 emitPatchpoint(patchpoint, result, arg);
2293 return { };
2294}
2295
2296template<>
2297auto AirIRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2298{
2299 auto* patchpoint = addPatchpoint(B3::Double);
2300 patchpoint->effects = B3::Effects::none();
2301 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2302 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
2303 });
2304 result = f64();
2305 emitPatchpoint(patchpoint, result, arg);
2306 return { };
2307}
2308
2309template<>
2310auto AirIRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2311{
2312 auto* patchpoint = addPatchpoint(B3::Float);
2313 patchpoint->effects = B3::Effects::none();
2314 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2315 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
2316 });
2317 result = f32();
2318 emitPatchpoint(patchpoint, result, arg);
2319 return { };
2320}
2321
2322template<>
2323auto AirIRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2324{
2325 auto* patchpoint = addPatchpoint(B3::Double);
2326 patchpoint->effects = B3::Effects::none();
2327 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2328 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
2329 });
2330 result = f64();
2331 emitPatchpoint(patchpoint, result, arg);
2332 return { };
2333}
2334
2335template<>
2336auto AirIRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2337{
2338 auto* patchpoint = addPatchpoint(B3::Float);
2339 patchpoint->effects = B3::Effects::none();
2340 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2341 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
2342 });
2343 result = f32();
2344 emitPatchpoint(patchpoint, result, arg);
2345 return { };
2346}
2347
2348template<>
2349auto AirIRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2350{
2351 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
2352 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
2353
2354 auto temp1 = g32();
2355 auto temp2 = g32();
2356 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2357 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2358 append(Or32, temp1, temp2);
2359
2360 emitCheck([&] {
2361 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2362 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2363 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2364 });
2365
2366 auto* patchpoint = addPatchpoint(B3::Int32);
2367 patchpoint->effects = B3::Effects::none();
2368 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2369 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
2370 });
2371 result = g32();
2372 emitPatchpoint(patchpoint, result, arg);
2373
2374 return { };
2375}
2376
2377template<>
2378auto AirIRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2379{
2380 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
2381 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
2382
2383 auto temp1 = g32();
2384 auto temp2 = g32();
2385 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2386 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2387 append(Or32, temp1, temp2);
2388
2389 emitCheck([&] {
2390 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2391 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2392 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2393 });
2394
2395 auto* patchpoint = addPatchpoint(B3::Int32);
2396 patchpoint->effects = B3::Effects::none();
2397 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2398 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
2399 });
2400 result = g32();
2401 emitPatchpoint(patchpoint, result, arg);
2402 return { };
2403}
2404
2405
2406template<>
2407auto AirIRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2408{
2409 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
2410 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2411
2412 auto temp1 = g32();
2413 auto temp2 = g32();
2414 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2415 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2416 append(Or32, temp1, temp2);
2417
2418 emitCheck([&] {
2419 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2420 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2421 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2422 });
2423
2424 auto* patchpoint = addPatchpoint(B3::Int32);
2425 patchpoint->effects = B3::Effects::none();
2426 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2427 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
2428 });
2429 result = g32();
2430 emitPatchpoint(patchpoint, result, arg);
2431 return { };
2432}
2433
2434template<>
2435auto AirIRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2436{
2437 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
2438 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2439
2440 auto temp1 = g32();
2441 auto temp2 = g32();
2442 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2443 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2444 append(Or32, temp1, temp2);
2445
2446 emitCheck([&] {
2447 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2448 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2449 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2450 });
2451
2452 auto* patchpoint = addPatchpoint(B3::Int32);
2453 patchpoint->effects = B3::Effects::none();
2454 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2455 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
2456 });
2457 result = g32();
2458 emitPatchpoint(patchpoint, result, arg);
2459 return { };
2460}
2461
2462template<>
2463auto AirIRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2464{
2465 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
2466 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
2467
2468 auto temp1 = g32();
2469 auto temp2 = g32();
2470 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2471 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2472 append(Or32, temp1, temp2);
2473
2474 emitCheck([&] {
2475 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2476 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2477 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2478 });
2479
2480 auto* patchpoint = addPatchpoint(B3::Int64);
2481 patchpoint->effects = B3::Effects::none();
2482 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2483 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
2484 });
2485
2486 result = g64();
2487 emitPatchpoint(patchpoint, result, arg);
2488 return { };
2489}
2490
2491template<>
2492auto AirIRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2493{
2494 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
2495 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2496
2497 auto temp1 = g32();
2498 auto temp2 = g32();
2499 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2500 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2501 append(Or32, temp1, temp2);
2502
2503 emitCheck([&] {
2504 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2505 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2506 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2507 });
2508
2509 TypedTmp signBitConstant;
2510 if (isX86())
2511 signBitConstant = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2512
2513 Vector<ConstrainedTmp> args;
2514 auto* patchpoint = addPatchpoint(B3::Int64);
2515 patchpoint->effects = B3::Effects::none();
2516 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2517 args.append(arg);
2518 if (isX86()) {
2519 args.append(signBitConstant);
2520 patchpoint->numFPScratchRegisters = 1;
2521 }
2522 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2523 AllowMacroScratchRegisterUsage allowScratch(jit);
2524 FPRReg scratch = InvalidFPRReg;
2525 FPRReg constant = InvalidFPRReg;
2526 if (isX86()) {
2527 scratch = params.fpScratch(0);
2528 constant = params[2].fpr();
2529 }
2530 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2531 });
2532
2533 result = g64();
2534 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2535 return { };
2536}
2537
2538template<>
2539auto AirIRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2540{
2541 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2542 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2543
2544 auto temp1 = g32();
2545 auto temp2 = g32();
2546 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2547 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2548 append(Or32, temp1, temp2);
2549
2550 emitCheck([&] {
2551 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2552 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2553 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2554 });
2555
2556 auto* patchpoint = addPatchpoint(B3::Int64);
2557 patchpoint->effects = B3::Effects::none();
2558 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2559 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2560 });
2561 result = g64();
2562 emitPatchpoint(patchpoint, result, arg);
2563 return { };
2564}
2565
2566template<>
2567auto AirIRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2568{
2569 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2570 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2571
2572 auto temp1 = g32();
2573 auto temp2 = g32();
2574 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2575 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2576 append(Or32, temp1, temp2);
2577
2578 emitCheck([&] {
2579 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2580 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2581 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2582 });
2583
2584 TypedTmp signBitConstant;
2585 if (isX86())
2586 signBitConstant = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2587
2588 auto* patchpoint = addPatchpoint(B3::Int64);
2589 patchpoint->effects = B3::Effects::none();
2590 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2591 Vector<ConstrainedTmp> args;
2592 args.append(arg);
2593 if (isX86()) {
2594 args.append(signBitConstant);
2595 patchpoint->numFPScratchRegisters = 1;
2596 }
2597 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2598 AllowMacroScratchRegisterUsage allowScratch(jit);
2599 FPRReg scratch = InvalidFPRReg;
2600 FPRReg constant = InvalidFPRReg;
2601 if (isX86()) {
2602 scratch = params.fpScratch(0);
2603 constant = params[2].fpr();
2604 }
2605 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2606 });
2607
2608 result = g64();
2609 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2610
2611 return { };
2612}
2613
2614auto AirIRGenerator::addShift(Type type, B3::Air::Opcode op, ExpressionType value, ExpressionType shift, ExpressionType& result) -> PartialResult
2615{
2616 ASSERT(type == Type::I64 || type == Type::I32);
2617 result = tmpForType(type);
2618
2619 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2620 append(op, value, shift, result);
2621 return { };
2622 }
2623
2624#if CPU(X86_64)
2625 Tmp ecx = Tmp(X86Registers::ecx);
2626 append(Move, value, result);
2627 append(Move, shift, ecx);
2628 append(op, ecx, result);
2629#else
2630 RELEASE_ASSERT_NOT_REACHED();
2631#endif
2632 return { };
2633}
2634
2635auto AirIRGenerator::addIntegerSub(B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2636{
2637 ASSERT(op == Sub32 || op == Sub64);
2638
2639 result = op == Sub32 ? g32() : g64();
2640
2641 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2642 append(op, lhs, rhs, result);
2643 return { };
2644 }
2645
2646 RELEASE_ASSERT(isX86());
2647 // Sub a, b
2648 // means
2649 // b = b Sub a
2650 append(Move, lhs, result);
2651 append(op, rhs, result);
2652 return { };
2653}
2654
2655auto AirIRGenerator::addFloatingPointAbs(B3::Air::Opcode op, ExpressionType value, ExpressionType& result) -> PartialResult
2656{
2657 RELEASE_ASSERT(op == AbsFloat || op == AbsDouble);
2658
2659 result = op == AbsFloat ? f32() : f64();
2660
2661 if (isValidForm(op, Arg::Tmp, Arg::Tmp)) {
2662 append(op, value, result);
2663 return { };
2664 }
2665
2666 RELEASE_ASSERT(isX86());
2667
2668 if (op == AbsFloat) {
2669 auto constant = g32();
2670 append(Move, Arg::imm(static_cast<uint32_t>(~(1ull << 31))), constant);
2671 append(Move32ToFloat, constant, result);
2672 append(AndFloat, value, result);
2673 } else {
2674 auto constant = g64();
2675 append(Move, Arg::bigImm(~(1ull << 63)), constant);
2676 append(Move64ToDouble, constant, result);
2677 append(AndDouble, value, result);
2678 }
2679 return { };
2680}
2681
2682auto AirIRGenerator::addFloatingPointBinOp(Type type, B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2683{
2684 ASSERT(type == Type::F32 || type == Type::F64);
2685 result = tmpForType(type);
2686
2687 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2688 append(op, lhs, rhs, result);
2689 return { };
2690 }
2691
2692 RELEASE_ASSERT(isX86());
2693
2694 // Op a, b
2695 // means
2696 // b = b Op a
2697 append(moveOpForValueType(type), lhs, result);
2698 append(op, rhs, result);
2699 return { };
2700}
2701
2702template<> auto AirIRGenerator::addOp<OpType::F32Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2703{
2704 result = f32();
2705 append(CeilFloat, arg0, result);
2706 return { };
2707}
2708
2709template<> auto AirIRGenerator::addOp<OpType::I32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2710{
2711 result = g32();
2712 append(Mul32, arg0, arg1, result);
2713 return { };
2714}
2715
2716template<> auto AirIRGenerator::addOp<OpType::I32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2717{
2718 return addIntegerSub(Sub32, arg0, arg1, result);
2719}
2720
2721template<> auto AirIRGenerator::addOp<OpType::F64Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2722{
2723 result = g32();
2724 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
2725 return { };
2726}
2727
2728template<> auto AirIRGenerator::addOp<OpType::F32DemoteF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2729{
2730 result = f32();
2731 append(ConvertDoubleToFloat, arg0, result);
2732 return { };
2733}
2734
2735template<> auto AirIRGenerator::addOp<OpType::F32Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2736{
2737 return addFloatingPointMinOrMax(F32, MinOrMax::Min, arg0, arg1, result);
2738}
2739
2740template<> auto AirIRGenerator::addOp<OpType::F64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2741{
2742 result = g32();
2743 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
2744 return { };
2745}
2746
2747template<> auto AirIRGenerator::addOp<OpType::F64Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2748{
2749 result = g32();
2750 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
2751 return { };
2752}
2753
2754auto AirIRGenerator::addFloatingPointMinOrMax(Type floatType, MinOrMax minOrMax, ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2755{
2756 ASSERT(floatType == F32 || floatType == F64);
2757 result = tmpForType(floatType);
2758
2759 BasicBlock* isEqual = m_code.addBlock();
2760 BasicBlock* notEqual = m_code.addBlock();
2761 BasicBlock* isLessThan = m_code.addBlock();
2762 BasicBlock* notLessThan = m_code.addBlock();
2763 BasicBlock* isGreaterThan = m_code.addBlock();
2764 BasicBlock* isNaN = m_code.addBlock();
2765 BasicBlock* continuation = m_code.addBlock();
2766
2767 auto branchOp = floatType == F32 ? BranchFloat : BranchDouble;
2768 append(m_currentBlock, branchOp, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1);
2769 m_currentBlock->setSuccessors(isEqual, notEqual);
2770
2771 append(notEqual, branchOp, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1);
2772 notEqual->setSuccessors(isLessThan, notLessThan);
2773
2774 append(notLessThan, branchOp, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1);
2775 notLessThan->setSuccessors(isGreaterThan, isNaN);
2776
2777 auto andOp = floatType == F32 ? AndFloat : AndDouble;
2778 auto orOp = floatType == F32 ? OrFloat : OrDouble;
2779 append(isEqual, minOrMax == MinOrMax::Max ? andOp : orOp, arg0, arg1, result);
2780 append(isEqual, Jump);
2781 isEqual->setSuccessors(continuation);
2782
2783 auto isLessThanResult = minOrMax == MinOrMax::Max ? arg1 : arg0;
2784 append(isLessThan, moveOpForValueType(floatType), isLessThanResult, result);
2785 append(isLessThan, Jump);
2786 isLessThan->setSuccessors(continuation);
2787
2788 auto isGreaterThanResult = minOrMax == MinOrMax::Max ? arg0 : arg1;
2789 append(isGreaterThan, moveOpForValueType(floatType), isGreaterThanResult, result);
2790 append(isGreaterThan, Jump);
2791 isGreaterThan->setSuccessors(continuation);
2792
2793 auto addOp = floatType == F32 ? AddFloat : AddDouble;
2794 append(isNaN, addOp, arg0, arg1, result);
2795 append(isNaN, Jump);
2796 isNaN->setSuccessors(continuation);
2797
2798 m_currentBlock = continuation;
2799
2800 return { };
2801}
2802
2803template<> auto AirIRGenerator::addOp<OpType::F32Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2804{
2805 return addFloatingPointMinOrMax(F32, MinOrMax::Max, arg0, arg1, result);
2806}
2807
2808template<> auto AirIRGenerator::addOp<OpType::F64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2809{
2810 return addFloatingPointBinOp(Type::F64, MulDouble, arg0, arg1, result);
2811}
2812
2813template<> auto AirIRGenerator::addOp<OpType::F32Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2814{
2815 return addFloatingPointBinOp(Type::F32, DivFloat, arg0, arg1, result);
2816}
2817
2818template<> auto AirIRGenerator::addOp<OpType::I32Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2819{
2820 result = g32();
2821 append(CountLeadingZeros32, arg0, result);
2822 return { };
2823}
2824
2825template<> auto AirIRGenerator::addOp<OpType::F32Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2826{
2827 // FIXME: We can have better codegen here for the imms and two operand forms on x86
2828 // https://bugs.webkit.org/show_bug.cgi?id=193999
2829 result = f32();
2830 auto temp1 = g32();
2831 auto sign = g32();
2832 auto value = g32();
2833
2834 // FIXME: Try to use Imm where possible:
2835 // https://bugs.webkit.org/show_bug.cgi?id=193999
2836 append(MoveFloatTo32, arg1, temp1);
2837 append(Move, Arg::bigImm(0x80000000), sign);
2838 append(And32, temp1, sign, sign);
2839
2840 append(MoveDoubleTo64, arg0, temp1);
2841 append(Move, Arg::bigImm(0x7fffffff), value);
2842 append(And32, temp1, value, value);
2843
2844 append(Or32, sign, value, value);
2845 append(Move32ToFloat, value, result);
2846
2847 return { };
2848}
2849
2850template<> auto AirIRGenerator::addOp<OpType::F64ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2851{
2852 result = f64();
2853 auto temp = g64();
2854 append(Move32, arg0, temp);
2855 append(ConvertInt64ToDouble, temp, result);
2856 return { };
2857}
2858
2859template<> auto AirIRGenerator::addOp<OpType::F32ReinterpretI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2860{
2861 result = f32();
2862 append(Move32ToFloat, arg0, result);
2863 return { };
2864}
2865
2866template<> auto AirIRGenerator::addOp<OpType::I64And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2867{
2868 result = g64();
2869 append(And64, arg0, arg1, result);
2870 return { };
2871}
2872
2873template<> auto AirIRGenerator::addOp<OpType::F32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2874{
2875 result = g32();
2876 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
2877 return { };
2878}
2879
2880template<> auto AirIRGenerator::addOp<OpType::F64Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2881{
2882 result = g32();
2883 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
2884 return { };
2885}
2886
2887template<> auto AirIRGenerator::addOp<OpType::F32Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2888{
2889 result = f32();
2890 append(SqrtFloat, arg0, result);
2891 return { };
2892}
2893
2894template<> auto AirIRGenerator::addOp<OpType::F64Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2895{
2896 result = g32();
2897 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
2898 return { };
2899}
2900
2901template<> auto AirIRGenerator::addOp<OpType::I64GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2902{
2903 result = g32();
2904 append(Compare64, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
2905 return { };
2906}
2907
2908template<> auto AirIRGenerator::addOp<OpType::I64GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2909{
2910 result = g32();
2911 append(Compare64, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
2912 return { };
2913}
2914
2915template<> auto AirIRGenerator::addOp<OpType::I64Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2916{
2917 result = g32();
2918 append(Test64, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
2919 return { };
2920}
2921
2922template<> auto AirIRGenerator::addOp<OpType::F64Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2923{
2924 return addFloatingPointBinOp(Type::F64, DivDouble, arg0, arg1, result);
2925}
2926
2927template<> auto AirIRGenerator::addOp<OpType::F32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2928{
2929 result = f32();
2930 append(AddFloat, arg0, arg1, result);
2931 return { };
2932}
2933
2934template<> auto AirIRGenerator::addOp<OpType::I64Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2935{
2936 result = g64();
2937 append(Or64, arg0, arg1, result);
2938 return { };
2939}
2940
2941template<> auto AirIRGenerator::addOp<OpType::I32LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2942{
2943 result = g32();
2944 append(Compare32, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
2945 return { };
2946}
2947
2948template<> auto AirIRGenerator::addOp<OpType::I32LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2949{
2950 result = g32();
2951 append(Compare32, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
2952 return { };
2953}
2954
2955template<> auto AirIRGenerator::addOp<OpType::I64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2956{
2957 result = g32();
2958 append(Compare64, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
2959 return { };
2960}
2961
2962template<> auto AirIRGenerator::addOp<OpType::I64Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2963{
2964 result = g64();
2965 append(CountLeadingZeros64, arg0, result);
2966 return { };
2967}
2968
2969template<> auto AirIRGenerator::addOp<OpType::F32Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2970{
2971 result = f32();
2972 if (isValidForm(NegateFloat, Arg::Tmp, Arg::Tmp))
2973 append(NegateFloat, arg0, result);
2974 else {
2975 auto constant = addConstant(Type::I32, bitwise_cast<uint32_t>(static_cast<float>(-0.0)));
2976 auto temp = g32();
2977 append(MoveFloatTo32, arg0, temp);
2978 append(Xor32, constant, temp);
2979 append(Move32ToFloat, temp, result);
2980 }
2981 return { };
2982}
2983
2984template<> auto AirIRGenerator::addOp<OpType::I32And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2985{
2986 result = g32();
2987 append(And32, arg0, arg1, result);
2988 return { };
2989}
2990
2991template<> auto AirIRGenerator::addOp<OpType::I32LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2992{
2993 result = g32();
2994 append(Compare32, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
2995 return { };
2996}
2997
2998template<> auto AirIRGenerator::addOp<OpType::I64Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2999{
3000 return addShift(Type::I64, RotateRight64, arg0, arg1, result);
3001}
3002
3003template<> auto AirIRGenerator::addOp<OpType::F64Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3004{
3005 return addFloatingPointAbs(AbsDouble, arg0, result);
3006}
3007
3008template<> auto AirIRGenerator::addOp<OpType::I32LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3009{
3010 result = g32();
3011 append(Compare32, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3012 return { };
3013}
3014
3015template<> auto AirIRGenerator::addOp<OpType::I32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3016{
3017 result = g32();
3018 append(Compare32, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3019 return { };
3020}
3021
3022template<> auto AirIRGenerator::addOp<OpType::F64Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3023{
3024 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3025 // https://bugs.webkit.org/show_bug.cgi?id=193999
3026 result = f64();
3027 auto temp1 = g64();
3028 auto sign = g64();
3029 auto value = g64();
3030
3031 append(MoveDoubleTo64, arg1, temp1);
3032 append(Move, Arg::bigImm(0x8000000000000000), sign);
3033 append(And64, temp1, sign, sign);
3034
3035 append(MoveDoubleTo64, arg0, temp1);
3036 append(Move, Arg::bigImm(0x7fffffffffffffff), value);
3037 append(And64, temp1, value, value);
3038
3039 append(Or64, sign, value, value);
3040 append(Move64ToDouble, value, result);
3041
3042 return { };
3043}
3044
3045template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3046{
3047 result = f32();
3048 append(ConvertInt64ToFloat, arg0, result);
3049 return { };
3050}
3051
3052template<> auto AirIRGenerator::addOp<OpType::I64Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3053{
3054 if (isARM64()) {
3055 // ARM64 doesn't have a rotate left.
3056 auto newShift = g64();
3057 append(Move, arg1, newShift);
3058 append(Neg64, newShift);
3059 return addShift(Type::I64, RotateRight64, arg0, newShift, result);
3060 } else
3061 return addShift(Type::I64, RotateLeft64, arg0, arg1, result);
3062}
3063
3064template<> auto AirIRGenerator::addOp<OpType::F32Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3065{
3066 result = g32();
3067 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
3068 return { };
3069}
3070
3071template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3072{
3073 result = f64();
3074 append(ConvertInt32ToDouble, arg0, result);
3075 return { };
3076}
3077
3078template<> auto AirIRGenerator::addOp<OpType::F64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3079{
3080 result = g32();
3081 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3082 return { };
3083}
3084
3085template<> auto AirIRGenerator::addOp<OpType::F32Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3086{
3087 result = g32();
3088 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
3089 return { };
3090}
3091
3092template<> auto AirIRGenerator::addOp<OpType::F32Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3093{
3094 result = g32();
3095 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3096 return { };
3097}
3098
3099template<> auto AirIRGenerator::addOp<OpType::I32ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3100{
3101 return addShift(Type::I32, Urshift32, arg0, arg1, result);
3102}
3103
3104template<> auto AirIRGenerator::addOp<OpType::F32ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3105{
3106 result = f32();
3107 auto temp = g64();
3108 append(Move32, arg0, temp);
3109 append(ConvertInt64ToFloat, temp, result);
3110 return { };
3111}
3112
3113template<> auto AirIRGenerator::addOp<OpType::I32ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3114{
3115 return addShift(Type::I32, Rshift32, arg0, arg1, result);
3116}
3117
3118template<> auto AirIRGenerator::addOp<OpType::I32GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3119{
3120 result = g32();
3121 append(Compare32, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3122 return { };
3123}
3124
3125template<> auto AirIRGenerator::addOp<OpType::F64Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3126{
3127 result = f64();
3128 append(CeilDouble, arg0, result);
3129 return { };
3130}
3131
3132template<> auto AirIRGenerator::addOp<OpType::I32GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3133{
3134 result = g32();
3135 append(Compare32, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3136 return { };
3137}
3138
3139template<> auto AirIRGenerator::addOp<OpType::I32Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3140{
3141 return addShift(Type::I32, Lshift32, arg0, arg1, result);
3142}
3143
3144template<> auto AirIRGenerator::addOp<OpType::F64Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3145{
3146 result = f64();
3147 append(FloorDouble, arg0, result);
3148 return { };
3149}
3150
3151template<> auto AirIRGenerator::addOp<OpType::I32Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3152{
3153 result = g32();
3154 append(Xor32, arg0, arg1, result);
3155 return { };
3156}
3157
3158template<> auto AirIRGenerator::addOp<OpType::F32Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3159{
3160 return addFloatingPointAbs(AbsFloat, arg0, result);
3161}
3162
3163template<> auto AirIRGenerator::addOp<OpType::F64Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3164{
3165 return addFloatingPointMinOrMax(F64, MinOrMax::Min, arg0, arg1, result);
3166}
3167
3168template<> auto AirIRGenerator::addOp<OpType::F32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3169{
3170 result = f32();
3171 append(MulFloat, arg0, arg1, result);
3172 return { };
3173}
3174
3175template<> auto AirIRGenerator::addOp<OpType::I64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3176{
3177 return addIntegerSub(Sub64, arg0, arg1, result);
3178}
3179
3180template<> auto AirIRGenerator::addOp<OpType::I32ReinterpretF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3181{
3182 result = g32();
3183 append(MoveFloatTo32, arg0, result);
3184 return { };
3185}
3186
3187template<> auto AirIRGenerator::addOp<OpType::I32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3188{
3189 result = g32();
3190 append(Add32, arg0, arg1, result);
3191 return { };
3192}
3193
3194template<> auto AirIRGenerator::addOp<OpType::F64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3195{
3196 return addFloatingPointBinOp(Type::F64, SubDouble, arg0, arg1, result);
3197}
3198
3199template<> auto AirIRGenerator::addOp<OpType::I32Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3200{
3201 result = g32();
3202 append(Or32, arg0, arg1, result);
3203 return { };
3204}
3205
3206template<> auto AirIRGenerator::addOp<OpType::I64LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3207{
3208 result = g32();
3209 append(Compare64, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3210 return { };
3211}
3212
3213template<> auto AirIRGenerator::addOp<OpType::I64LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3214{
3215 result = g32();
3216 append(Compare64, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3217 return { };
3218}
3219
3220template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3221{
3222 result = f64();
3223 append(ConvertInt64ToDouble, arg0, result);
3224 return { };
3225}
3226
3227template<> auto AirIRGenerator::addOp<OpType::I64Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3228{
3229 result = g64();
3230 append(Xor64, arg0, arg1, result);
3231 return { };
3232}
3233
3234template<> auto AirIRGenerator::addOp<OpType::I64GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3235{
3236 result = g32();
3237 append(Compare64, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3238 return { };
3239}
3240
3241template<> auto AirIRGenerator::addOp<OpType::I64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3242{
3243 result = g64();
3244 append(Mul64, arg0, arg1, result);
3245 return { };
3246}
3247
3248template<> auto AirIRGenerator::addOp<OpType::F32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3249{
3250 result = f32();
3251 if (isValidForm(SubFloat, Arg::Tmp, Arg::Tmp, Arg::Tmp))
3252 append(SubFloat, arg0, arg1, result);
3253 else {
3254 RELEASE_ASSERT(isX86());
3255 append(MoveFloat, arg0, result);
3256 append(SubFloat, arg1, result);
3257 }
3258 return { };
3259}
3260
3261template<> auto AirIRGenerator::addOp<OpType::F64PromoteF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3262{
3263 result = f64();
3264 append(ConvertFloatToDouble, arg0, result);
3265 return { };
3266}
3267
3268template<> auto AirIRGenerator::addOp<OpType::F64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3269{
3270 result = f64();
3271 append(AddDouble, arg0, arg1, result);
3272 return { };
3273}
3274
3275template<> auto AirIRGenerator::addOp<OpType::I64GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3276{
3277 result = g32();
3278 append(Compare64, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3279 return { };
3280}
3281
3282template<> auto AirIRGenerator::addOp<OpType::I64ExtendUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3283{
3284 result = g64();
3285 append(Move32, arg0, result);
3286 return { };
3287}
3288
3289template<> auto AirIRGenerator::addOp<OpType::I32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3290{
3291 result = g32();
3292 RELEASE_ASSERT(arg0 && arg1);
3293 append(Compare32, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3294 return { };
3295}
3296
3297template<> auto AirIRGenerator::addOp<OpType::F64ReinterpretI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3298{
3299 result = f64();
3300 append(Move64ToDouble, arg0, result);
3301 return { };
3302}
3303
3304template<> auto AirIRGenerator::addOp<OpType::F32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3305{
3306 result = g32();
3307 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3308 return { };
3309}
3310
3311template<> auto AirIRGenerator::addOp<OpType::I64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3312{
3313 result = g32();
3314 append(Compare64, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3315 return { };
3316}
3317
3318template<> auto AirIRGenerator::addOp<OpType::F32Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3319{
3320 result = f32();
3321 append(FloorFloat, arg0, result);
3322 return { };
3323}
3324
3325template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3326{
3327 result = f32();
3328 append(ConvertInt32ToFloat, arg0, result);
3329 return { };
3330}
3331
3332template<> auto AirIRGenerator::addOp<OpType::I32Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3333{
3334 result = g32();
3335 append(Test32, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3336 return { };
3337}
3338
3339template<> auto AirIRGenerator::addOp<OpType::I64ReinterpretF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3340{
3341 result = g64();
3342 append(MoveDoubleTo64, arg0, result);
3343 return { };
3344}
3345
3346template<> auto AirIRGenerator::addOp<OpType::I64ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3347{
3348 return addShift(Type::I64, Rshift64, arg0, arg1, result);
3349}
3350
3351template<> auto AirIRGenerator::addOp<OpType::I64ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3352{
3353 return addShift(Type::I64, Urshift64, arg0, arg1, result);
3354}
3355
3356template<> auto AirIRGenerator::addOp<OpType::F64Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3357{
3358 result = f64();
3359 append(SqrtDouble, arg0, result);
3360 return { };
3361}
3362
3363template<> auto AirIRGenerator::addOp<OpType::I64Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3364{
3365 return addShift(Type::I64, Lshift64, arg0, arg1, result);
3366}
3367
3368template<> auto AirIRGenerator::addOp<OpType::F32Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3369{
3370 result = g32();
3371 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3372 return { };
3373}
3374
3375template<> auto AirIRGenerator::addOp<OpType::I32WrapI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3376{
3377 result = g32();
3378 append(Move32, arg0, result);
3379 return { };
3380}
3381
3382template<> auto AirIRGenerator::addOp<OpType::I32Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3383{
3384 if (isARM64()) {
3385 // ARM64 doesn't have a rotate left.
3386 auto newShift = g64();
3387 append(Move, arg1, newShift);
3388 append(Neg64, newShift);
3389 return addShift(Type::I32, RotateRight32, arg0, newShift, result);
3390 } else
3391 return addShift(Type::I32, RotateLeft32, arg0, arg1, result);
3392}
3393
3394template<> auto AirIRGenerator::addOp<OpType::I32Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3395{
3396 return addShift(Type::I32, RotateRight32, arg0, arg1, result);
3397}
3398
3399template<> auto AirIRGenerator::addOp<OpType::I32GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3400{
3401 result = g32();
3402 append(Compare32, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3403 return { };
3404}
3405
3406template<> auto AirIRGenerator::addOp<OpType::I64ExtendSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3407{
3408 result = g64();
3409 append(SignExtend32ToPtr, arg0, result);
3410 return { };
3411}
3412
3413template<> auto AirIRGenerator::addOp<OpType::I32GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3414{
3415 result = g32();
3416 append(Compare32, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3417 return { };
3418}
3419
3420template<> auto AirIRGenerator::addOp<OpType::F64Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3421{
3422 result = f64();
3423 if (isValidForm(NegateDouble, Arg::Tmp, Arg::Tmp))
3424 append(NegateDouble, arg0, result);
3425 else {
3426 auto constant = addConstant(Type::I64, bitwise_cast<uint64_t>(static_cast<double>(-0.0)));
3427 auto temp = g64();
3428 append(MoveDoubleTo64, arg0, temp);
3429 append(Xor64, constant, temp);
3430 append(Move64ToDouble, temp, result);
3431 }
3432 return { };
3433}
3434
3435template<> auto AirIRGenerator::addOp<OpType::F64Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3436{
3437 return addFloatingPointMinOrMax(F64, MinOrMax::Max, arg0, arg1, result);
3438}
3439
3440template<> auto AirIRGenerator::addOp<OpType::I64LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3441{
3442 result = g32();
3443 append(Compare64, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3444 return { };
3445}
3446
3447template<> auto AirIRGenerator::addOp<OpType::I64LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3448{
3449 result = g32();
3450 append(Compare64, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3451 return { };
3452}
3453
3454template<> auto AirIRGenerator::addOp<OpType::I64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3455{
3456 result = g64();
3457 append(Add64, arg0, arg1, result);
3458 return { };
3459}
3460
3461} } // namespace JSC::Wasm
3462
3463#endif // ENABLE(WEBASSEMBLY)
3464