1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmAirIRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AirCode.h"
32#include "AirGenerate.h"
33#include "AirOpcodeUtils.h"
34#include "AirValidate.h"
35#include "AllowMacroScratchRegisterUsageIf.h"
36#include "B3CCallValue.h"
37#include "B3CheckSpecial.h"
38#include "B3CheckValue.h"
39#include "B3PatchpointSpecial.h"
40#include "B3Procedure.h"
41#include "B3ProcedureInlines.h"
42#include "BinarySwitch.h"
43#include "DisallowMacroScratchRegisterUsage.h"
44#include "JSCInlines.h"
45#include "JSWebAssemblyInstance.h"
46#include "ScratchRegisterAllocator.h"
47#include "VirtualRegister.h"
48#include "WasmCallingConvention.h"
49#include "WasmContextInlines.h"
50#include "WasmExceptionType.h"
51#include "WasmFunctionParser.h"
52#include "WasmInstance.h"
53#include "WasmMemory.h"
54#include "WasmOMGPlan.h"
55#include "WasmOpcodeOrigin.h"
56#include "WasmSignatureInlines.h"
57#include "WasmThunks.h"
58#include <limits>
59#include <wtf/Box.h>
60#include <wtf/Optional.h>
61#include <wtf/StdLibExtras.h>
62
63namespace JSC { namespace Wasm {
64
65using namespace B3::Air;
66
67struct ConstrainedTmp {
68 ConstrainedTmp(Tmp tmp)
69 : ConstrainedTmp(tmp, tmp.isReg() ? B3::ValueRep::reg(tmp.reg()) : B3::ValueRep::SomeRegister)
70 { }
71
72 ConstrainedTmp(Tmp tmp, B3::ValueRep rep)
73 : tmp(tmp)
74 , rep(rep)
75 {
76 }
77
78 Tmp tmp;
79 B3::ValueRep rep;
80};
81
82class TypedTmp {
83public:
84 constexpr TypedTmp()
85 : m_tmp()
86 , m_type(Type::Void)
87 { }
88
89 TypedTmp(Tmp tmp, Type type)
90 : m_tmp(tmp)
91 , m_type(type)
92 { }
93
94 TypedTmp(const TypedTmp&) = default;
95 TypedTmp(TypedTmp&&) = default;
96 TypedTmp& operator=(TypedTmp&&) = default;
97 TypedTmp& operator=(const TypedTmp&) = default;
98
99 bool operator==(const TypedTmp& other) const
100 {
101 return m_tmp == other.m_tmp && m_type == other.m_type;
102 }
103 bool operator!=(const TypedTmp& other) const
104 {
105 return !(*this == other);
106 }
107
108 explicit operator bool() const { return !!tmp(); }
109
110 operator Tmp() const { return tmp(); }
111 operator Arg() const { return Arg(tmp()); }
112 Tmp tmp() const { return m_tmp; }
113 Type type() const { return m_type; }
114
115private:
116
117 Tmp m_tmp;
118 Type m_type;
119};
120
121class AirIRGenerator {
122public:
123 struct ControlData {
124 ControlData(B3::Origin origin, Type returnType, TypedTmp resultTmp, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
125 : blockType(type)
126 , continuation(continuation)
127 , special(special)
128 , returnType(returnType)
129 {
130 UNUSED_PARAM(origin); // FIXME: Use origin.
131 if (resultTmp) {
132 ASSERT(returnType != Type::Void);
133 result.append(resultTmp);
134 } else
135 ASSERT(returnType == Type::Void);
136 }
137
138 ControlData()
139 {
140 }
141
142 void dump(PrintStream& out) const
143 {
144 switch (type()) {
145 case BlockType::If:
146 out.print("If: ");
147 break;
148 case BlockType::Block:
149 out.print("Block: ");
150 break;
151 case BlockType::Loop:
152 out.print("Loop: ");
153 break;
154 case BlockType::TopLevel:
155 out.print("TopLevel: ");
156 break;
157 }
158 out.print("Continuation: ", *continuation, ", Special: ");
159 if (special)
160 out.print(*special);
161 else
162 out.print("None");
163 }
164
165 BlockType type() const { return blockType; }
166
167 Type signature() const { return returnType; }
168
169 bool hasNonVoidSignature() const { return result.size(); }
170
171 BasicBlock* targetBlockForBranch()
172 {
173 if (type() == BlockType::Loop)
174 return special;
175 return continuation;
176 }
177
178 void convertIfToBlock()
179 {
180 ASSERT(type() == BlockType::If);
181 blockType = BlockType::Block;
182 special = nullptr;
183 }
184
185 using ResultList = Vector<TypedTmp, 1>;
186
187 ResultList resultForBranch() const
188 {
189 if (type() == BlockType::Loop)
190 return ResultList();
191 return result;
192 }
193
194 private:
195 friend class AirIRGenerator;
196 BlockType blockType;
197 BasicBlock* continuation;
198 BasicBlock* special;
199 ResultList result;
200 Type returnType;
201 };
202
203 using ExpressionType = TypedTmp;
204 using ControlType = ControlData;
205 using ExpressionList = Vector<ExpressionType, 1>;
206 using ResultList = ControlData::ResultList;
207 using ControlEntry = FunctionParser<AirIRGenerator>::ControlEntry;
208
209 static ExpressionType emptyExpression() { return { }; };
210
211 using ErrorType = String;
212 using UnexpectedResult = Unexpected<ErrorType>;
213 using Result = Expected<std::unique_ptr<InternalFunction>, ErrorType>;
214 using PartialResult = Expected<void, ErrorType>;
215
216 template <typename ...Args>
217 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
218 {
219 using namespace FailureHelper; // See ADL comment in WasmParser.h.
220 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
221 }
222
223#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
224 if (UNLIKELY(condition)) \
225 return fail(__VA_ARGS__); \
226 } while (0)
227
228 AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
229
230 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
231 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
232 ExpressionType addConstant(Type, uint64_t);
233 ExpressionType addConstant(BasicBlock*, Type, uint64_t);
234
235 // References
236 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
237
238 // Tables
239 PartialResult WARN_UNUSED_RETURN addTableGet(ExpressionType& idx, ExpressionType& result);
240 PartialResult WARN_UNUSED_RETURN addTableSet(ExpressionType& idx, ExpressionType& value);
241
242 // Locals
243 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
244 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
245
246 // Globals
247 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
248 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
249
250 // Memory
251 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
252 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
253 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
254 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
255
256 // Basic operators
257 template<OpType>
258 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
259 template<OpType>
260 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
261 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
262
263 // Control flow
264 ControlData WARN_UNUSED_RETURN addTopLevel(Type signature);
265 ControlData WARN_UNUSED_RETURN addBlock(Type signature);
266 ControlData WARN_UNUSED_RETURN addLoop(Type signature);
267 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
268 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
269 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
270
271 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
272 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
273 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
274 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
275 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
276
277 // Calls
278 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
279 PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature&, Vector<ExpressionType>& args, ExpressionType& result);
280 PartialResult WARN_UNUSED_RETURN addUnreachable();
281
282 PartialResult addShift(Type, B3::Air::Opcode, ExpressionType value, ExpressionType shift, ExpressionType& result);
283 PartialResult addIntegerSub(B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
284 PartialResult addFloatingPointAbs(B3::Air::Opcode, ExpressionType value, ExpressionType& result);
285 PartialResult addFloatingPointBinOp(Type, B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
286
287 void dump(const Vector<ControlEntry>& controlStack, const ExpressionList* expressionStack);
288 void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };
289
290 static Vector<Tmp> toTmpVector(const Vector<TypedTmp>& vector)
291 {
292 Vector<Tmp> result;
293 for (const auto& item : vector)
294 result.append(item.tmp());
295 return result;
296 }
297
298 ALWAYS_INLINE void didKill(const ExpressionType& typedTmp)
299 {
300 Tmp tmp = typedTmp.tmp();
301 if (!tmp)
302 return;
303 if (tmp.isGP())
304 m_freeGPs.append(tmp);
305 else
306 m_freeFPs.append(tmp);
307 }
308
309private:
310 ALWAYS_INLINE void validateInst(Inst& inst)
311 {
312 if (!ASSERT_DISABLED) {
313 if (!inst.isValidForm()) {
314 dataLogLn(inst);
315 CRASH();
316 }
317 }
318 }
319
320 static Arg extractArg(const TypedTmp& tmp) { return tmp.tmp(); }
321 static Arg extractArg(const Tmp& tmp) { return Arg(tmp); }
322 static Arg extractArg(const Arg& arg) { return arg; }
323
324 template<typename... Arguments>
325 void append(BasicBlock* block, Kind kind, Arguments&&... arguments)
326 {
327 // FIXME: Find a way to use origin here.
328 auto& inst = block->append(kind, nullptr, extractArg(arguments)...);
329 validateInst(inst);
330 }
331
332 template<typename... Arguments>
333 void append(Kind kind, Arguments&&... arguments)
334 {
335 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
336 }
337
338 template<typename... Arguments>
339 void appendEffectful(B3::Air::Opcode op, Arguments&&... arguments)
340 {
341 Kind kind = op;
342 kind.effects = true;
343 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
344 }
345
346 Tmp newTmp(B3::Bank bank)
347 {
348 switch (bank) {
349 case B3::GP:
350 if (m_freeGPs.size())
351 return m_freeGPs.takeLast();
352 break;
353 case B3::FP:
354 if (m_freeFPs.size())
355 return m_freeFPs.takeLast();
356 break;
357 }
358 return m_code.newTmp(bank);
359 }
360
361 TypedTmp g32() { return { newTmp(B3::GP), Type::I32 }; }
362 TypedTmp g64() { return { newTmp(B3::GP), Type::I64 }; }
363 TypedTmp f32() { return { newTmp(B3::FP), Type::F32 }; }
364 TypedTmp f64() { return { newTmp(B3::FP), Type::F64 }; }
365
366 TypedTmp tmpForType(Type type)
367 {
368 switch (type) {
369 case Type::I32:
370 return g32();
371 case Type::I64:
372 case Type::Anyref:
373 return g64();
374 case Type::F32:
375 return f32();
376 case Type::F64:
377 return f64();
378 case Type::Void:
379 return { };
380 default:
381 RELEASE_ASSERT_NOT_REACHED();
382 }
383 }
384
385 B3::PatchpointValue* addPatchpoint(B3::Type type)
386 {
387 return m_proc.add<B3::PatchpointValue>(type, B3::Origin());
388 }
389
390 template <typename ...Args>
391 void emitPatchpoint(B3::PatchpointValue* patch, Tmp result, Args... theArgs)
392 {
393 emitPatchpoint(m_currentBlock, patch, result, std::forward<Args>(theArgs)...);
394 }
395
396 template <typename ...Args>
397 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Args... theArgs)
398 {
399 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp, sizeof...(Args)>::from(theArgs...));
400 }
401
402 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result)
403 {
404 emitPatchpoint(basicBlock, patch, result, Vector<ConstrainedTmp>());
405 }
406
407 template <size_t inlineSize>
408 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Vector<ConstrainedTmp, inlineSize>&& args)
409 {
410 if (!m_patchpointSpecial)
411 m_patchpointSpecial = static_cast<B3::PatchpointSpecial*>(m_code.addSpecial(std::make_unique<B3::PatchpointSpecial>()));
412
413 Inst inst(Patch, patch, Arg::special(m_patchpointSpecial));
414 Inst resultMov;
415 if (result) {
416 ASSERT(patch->type() != B3::Void);
417 switch (patch->resultConstraint.kind()) {
418 case B3::ValueRep::Register:
419 inst.args.append(Tmp(patch->resultConstraint.reg()));
420 resultMov = Inst(result.isGP() ? Move : MoveDouble, nullptr, Tmp(patch->resultConstraint.reg()), result);
421 break;
422 case B3::ValueRep::SomeRegister:
423 inst.args.append(result);
424 break;
425 default:
426 RELEASE_ASSERT_NOT_REACHED();
427 }
428 } else
429 ASSERT(patch->type() == B3::Void);
430
431 for (ConstrainedTmp& tmp : args) {
432 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
433 // validation. We should abstrcat Patch enough so ValueRep's don't need to be
434 // backed by Values.
435 // https://bugs.webkit.org/show_bug.cgi?id=194040
436 B3::Value* dummyValue = m_proc.addConstant(B3::Origin(), tmp.tmp.isGP() ? B3::Int64 : B3::Double, 0);
437 patch->append(dummyValue, tmp.rep);
438 switch (tmp.rep.kind()) {
439 case B3::ValueRep::SomeRegister:
440 inst.args.append(tmp.tmp);
441 break;
442 case B3::ValueRep::Register:
443 patch->earlyClobbered().clear(tmp.rep.reg());
444 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, tmp.rep.reg());
445 inst.args.append(Tmp(tmp.rep.reg()));
446 break;
447 case B3::ValueRep::StackArgument: {
448 auto arg = Arg::callArg(tmp.rep.offsetFromSP());
449 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, arg);
450 inst.args.append(arg);
451 break;
452 }
453 default:
454 RELEASE_ASSERT_NOT_REACHED();
455 }
456 }
457
458 if (patch->resultConstraint.isReg())
459 patch->lateClobbered().clear(patch->resultConstraint.reg());
460 for (unsigned i = patch->numGPScratchRegisters; i--;)
461 inst.args.append(g64().tmp());
462 for (unsigned i = patch->numFPScratchRegisters; i--;)
463 inst.args.append(f64().tmp());
464
465 validateInst(inst);
466 basicBlock->append(WTFMove(inst));
467 if (resultMov) {
468 validateInst(resultMov);
469 basicBlock->append(WTFMove(resultMov));
470 }
471 }
472
473 template <typename Branch, typename Generator>
474 void emitCheck(const Branch& makeBranch, const Generator& generator)
475 {
476 // We fail along the truthy edge of 'branch'.
477 Inst branch = makeBranch();
478
479 // FIXME: Make a hashmap of these.
480 B3::CheckSpecial::Key key(branch);
481 B3::CheckSpecial* special = static_cast<B3::CheckSpecial*>(m_code.addSpecial(std::make_unique<B3::CheckSpecial>(key)));
482
483 // FIXME: Remove the need for dummy values
484 // https://bugs.webkit.org/show_bug.cgi?id=194040
485 B3::Value* dummyPredicate = m_proc.addConstant(B3::Origin(), B3::Int32, 42);
486 B3::CheckValue* checkValue = m_proc.add<B3::CheckValue>(B3::Check, B3::Origin(), dummyPredicate);
487 checkValue->setGenerator(generator);
488
489 Inst inst(Patch, checkValue, Arg::special(special));
490 inst.args.appendVector(branch.args);
491 m_currentBlock->append(WTFMove(inst));
492 }
493
494 template <typename Func, typename ...Args>
495 void emitCCall(Func func, TypedTmp result, Args... args)
496 {
497 emitCCall(m_currentBlock, func, result, std::forward<Args>(args)...);
498 }
499 template <typename Func, typename ...Args>
500 void emitCCall(BasicBlock* block, Func func, TypedTmp result, Args... theArgs)
501 {
502 B3::Type resultType = B3::Void;
503 if (result) {
504 switch (result.type()) {
505 case Type::I32:
506 resultType = B3::Int32;
507 break;
508 case Type::I64:
509 resultType = B3::Int64;
510 break;
511 case Type::F32:
512 resultType = B3::Float;
513 break;
514 case Type::F64:
515 resultType = B3::Double;
516 break;
517 default:
518 RELEASE_ASSERT_NOT_REACHED();
519 }
520 }
521
522 auto makeDummyValue = [&] (Tmp tmp) {
523 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
524 // validation. We should abstrcat CCall enough so we're not reliant on arguments
525 // to the B3::CCallValue.
526 // https://bugs.webkit.org/show_bug.cgi?id=194040
527 if (tmp.isGP())
528 return m_proc.addConstant(B3::Origin(), B3::Int64, 0);
529 return m_proc.addConstant(B3::Origin(), B3::Double, 0);
530 };
531
532 B3::Value* dummyFunc = m_proc.addConstant(B3::Origin(), B3::Int64, bitwise_cast<uintptr_t>(func));
533 B3::Value* origin = m_proc.add<B3::CCallValue>(resultType, B3::Origin(), B3::Effects::none(), dummyFunc, makeDummyValue(theArgs)...);
534
535 Inst inst(CCall, origin);
536
537 Tmp callee = g64();
538 append(block, Move, Arg::immPtr(tagCFunctionPtr<void*>(func, B3CCallPtrTag)), callee);
539 inst.args.append(callee);
540
541 if (result)
542 inst.args.append(result.tmp());
543
544 for (Tmp tmp : Vector<Tmp, sizeof...(Args)>::from(theArgs.tmp()...))
545 inst.args.append(tmp);
546
547 block->append(WTFMove(inst));
548 }
549
550 static B3::Air::Opcode moveOpForValueType(Type type)
551 {
552 switch (type) {
553 case Type::I32:
554 return Move32;
555 case Type::I64:
556 case Type::Anyref:
557 return Move;
558 case Type::F32:
559 return MoveFloat;
560 case Type::F64:
561 return MoveDouble;
562 default:
563 RELEASE_ASSERT_NOT_REACHED();
564 }
565 }
566
567 void emitThrowException(CCallHelpers&, ExceptionType);
568
569 void emitTierUpCheck(uint32_t decrementCount, B3::Origin);
570
571 void emitWriteBarrierForJSWrapper();
572 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
573 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
574 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
575
576 void unify(const ExpressionType& dst, const ExpressionType& source);
577 void unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& stack);
578
579 template <typename IntType>
580 void emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right);
581
582 template <typename IntType>
583 void emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
584
585 enum class MinOrMax { Min, Max };
586
587 PartialResult addFloatingPointMinOrMax(Type, MinOrMax, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
588
589 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
590
591 void restoreWasmContextInstance(BasicBlock*, TypedTmp);
592 enum class RestoreCachedStackLimit { No, Yes };
593 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, TypedTmp instance, BasicBlock*);
594
595 B3::Origin origin();
596
597 FunctionParser<AirIRGenerator>* m_parser { nullptr };
598 const ModuleInformation& m_info;
599 const MemoryMode m_mode { MemoryMode::BoundsChecking };
600 const unsigned m_functionIndex { UINT_MAX };
601 const TierUpCount* m_tierUp { nullptr };
602
603 B3::Procedure& m_proc;
604 Code& m_code;
605 BasicBlock* m_currentBlock { nullptr };
606 BasicBlock* m_rootBlock { nullptr };
607 Vector<TypedTmp> m_locals;
608 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
609 GPRReg m_memoryBaseGPR { InvalidGPRReg };
610 GPRReg m_memorySizeGPR { InvalidGPRReg };
611 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
612 bool m_makesCalls { false };
613
614 Vector<Tmp, 8> m_freeGPs;
615 Vector<Tmp, 8> m_freeFPs;
616
617 TypedTmp m_instanceValue; // Always use the accessor below to ensure the instance value is materialized when used.
618 bool m_usesInstanceValue { false };
619 TypedTmp instanceValue()
620 {
621 m_usesInstanceValue = true;
622 return m_instanceValue;
623 }
624
625 uint32_t m_maxNumJSCallArguments { 0 };
626
627 B3::PatchpointSpecial* m_patchpointSpecial { nullptr };
628};
629
630// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
631int32_t AirIRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
632{
633 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
634 auto previousPtr = ptr;
635 ptr = g64();
636 auto constant = g64();
637 append(Move, Arg::bigImm(offset), constant);
638 append(Add64, constant, previousPtr, ptr);
639 return 0;
640 }
641 return offset;
642}
643
644void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp instance)
645{
646 if (Context::useFastTLS()) {
647 auto* patchpoint = addPatchpoint(B3::Void);
648 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
649 patchpoint->clobber(RegisterSet::macroScratchRegisters());
650 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
651 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
652 jit.storeWasmContextInstance(params[0].gpr());
653 });
654 emitPatchpoint(block, patchpoint, Tmp(), instance);
655 return;
656 }
657
658 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
659 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
660 auto* patchpoint = addPatchpoint(B3::Void);
661 B3::Effects effects = B3::Effects::none();
662 effects.writesPinned = true;
663 effects.reads = B3::HeapRange::top();
664 patchpoint->effects = effects;
665 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
666 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
667 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& param) {
668 jit.move(param[0].gpr(), wasmContextInstanceGPR);
669 });
670 emitPatchpoint(block, patchpoint, Tmp(), instance);
671}
672
673AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
674 : m_info(info)
675 , m_mode(mode)
676 , m_functionIndex(functionIndex)
677 , m_tierUp(tierUp)
678 , m_proc(procedure)
679 , m_code(m_proc.code())
680 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
681{
682 m_currentBlock = m_code.addBlock();
683 m_rootBlock = m_currentBlock;
684
685 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
686 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
687
688 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
689 m_code.pinRegister(m_memoryBaseGPR);
690
691 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
692 if (!Context::useFastTLS())
693 m_code.pinRegister(m_wasmContextInstanceGPR);
694
695 if (mode != MemoryMode::Signaling) {
696 m_memorySizeGPR = pinnedRegs.sizeRegister;
697 m_code.pinRegister(m_memorySizeGPR);
698 }
699
700 if (throwWasmException)
701 Thunks::singleton().setThrowWasmException(throwWasmException);
702
703 if (info.memory) {
704 switch (m_mode) {
705 case MemoryMode::BoundsChecking:
706 break;
707 case MemoryMode::Signaling:
708 // Most memory accesses in signaling mode don't do an explicit
709 // exception check because they can rely on fault handling to detect
710 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
711 // thunk to exist so that it can jump to that thunk.
712 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
713 CRASH();
714 break;
715 }
716 }
717
718 m_code.setNumEntrypoints(1);
719
720 GPRReg contextInstance = Context::useFastTLS() ? wasmCallingConventionAir().prologueScratch(1) : m_wasmContextInstanceGPR;
721
722 Ref<B3::Air::PrologueGenerator> prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=] (CCallHelpers& jit, B3::Air::Code& code) {
723 AllowMacroScratchRegisterUsage allowScratch(jit);
724 code.emitDefaultPrologue(jit);
725
726 {
727 GPRReg calleeGPR = wasmCallingConventionAir().prologueScratch(0);
728 auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
729 jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
730 compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
731 });
732 jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
733 jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
734 }
735
736 {
737 const Checked<int32_t> wasmFrameSize = m_code.frameSize();
738 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
739 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
740 // This allows us to elide stack checks for functions that are terminal nodes in the call
741 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
742 // having any such terminal node have its parent caller include some extra size in its
743 // own check for it. The goal here is twofold:
744 // 1. Emit less code.
745 // 2. Try to speed things up by skipping stack checks.
746 minimumParentCheckSize,
747 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
748 // spill all arguments to the stack, we ensure that a stack check here covers the
749 // stack that such a stub would use.
750 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jscCallingConvention().headerSizeInBytes()).unsafeGet()
751 ));
752 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
753 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
754 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
755
756 // This allows leaf functions to not do stack checks if their frame size is within
757 // certain limits since their caller would have already done the check.
758 if (needsOverflowCheck) {
759 GPRReg scratch = wasmCallingConventionAir().prologueScratch(0);
760
761 if (Context::useFastTLS())
762 jit.loadWasmContextInstance(contextInstance);
763
764 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), GPRInfo::callFrameRegister, scratch);
765 MacroAssembler::JumpList overflow;
766 if (UNLIKELY(needUnderflowCheck))
767 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch, GPRInfo::callFrameRegister));
768 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch, CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit())));
769 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
770 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
771 });
772 } else if (m_usesInstanceValue && Context::useFastTLS()) {
773 // No overflow check is needed, but the instance values still needs to be correct.
774 jit.loadWasmContextInstance(contextInstance);
775 }
776 }
777 });
778
779 m_code.setPrologueForEntrypoint(0, WTFMove(prologueGenerator));
780
781 if (Context::useFastTLS()) {
782 m_instanceValue = g64();
783 // FIXME: Would be nice to only do this if we use instance value.
784 append(Move, Tmp(contextInstance), m_instanceValue);
785 } else
786 m_instanceValue = { Tmp(contextInstance), Type::I64 };
787
788 ASSERT(!m_locals.size());
789 m_locals.grow(signature.argumentCount());
790 for (unsigned i = 0; i < signature.argumentCount(); ++i) {
791 Type type = signature.argument(i);
792 m_locals[i] = tmpForType(type);
793 }
794
795 wasmCallingConventionAir().loadArguments(signature, [&] (const Arg& arg, unsigned i) {
796 switch (signature.argument(i)) {
797 case Type::I32:
798 append(Move32, arg, m_locals[i]);
799 break;
800 case Type::I64:
801 case Type::Anyref:
802 append(Move, arg, m_locals[i]);
803 break;
804 case Type::F32:
805 append(MoveFloat, arg, m_locals[i]);
806 break;
807 case Type::F64:
808 append(MoveDouble, arg, m_locals[i]);
809 break;
810 default:
811 RELEASE_ASSERT_NOT_REACHED();
812 }
813 });
814
815 emitTierUpCheck(TierUpCount::functionEntryDecrement(), B3::Origin());
816}
817
818void AirIRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, TypedTmp instance, BasicBlock* block)
819{
820 restoreWasmContextInstance(block, instance);
821
822 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
823 // The Instance caches the stack limit, but also knows where its canonical location is.
824 static_assert(sizeof(decltype(static_cast<Instance*>(nullptr)->cachedStackLimit())) == sizeof(uint64_t), "");
825
826 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfPointerToActualStackLimit(), B3::Width64));
827 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedStackLimit(), B3::Width64));
828 auto temp = g64();
829 append(block, Move, Arg::addr(instanceValue(), Instance::offsetOfPointerToActualStackLimit()), temp);
830 append(block, Move, Arg::addr(temp), temp);
831 append(block, Move, temp, Arg::addr(instanceValue(), Instance::offsetOfCachedStackLimit()));
832 }
833
834 if (!!memory) {
835 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
836 RegisterSet clobbers;
837 clobbers.set(pinnedRegs->baseMemoryPointer);
838 clobbers.set(pinnedRegs->sizeRegister);
839 if (!isARM64())
840 clobbers.set(RegisterSet::macroScratchRegisters());
841
842 auto* patchpoint = addPatchpoint(B3::Void);
843 B3::Effects effects = B3::Effects::none();
844 effects.writesPinned = true;
845 effects.reads = B3::HeapRange::top();
846 patchpoint->effects = effects;
847 patchpoint->clobber(clobbers);
848 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
849
850 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
851 AllowMacroScratchRegisterUsage allowScratch(jit);
852 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
853 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
854
855 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
856 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
857
858 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->sizeRegister, scratchOrSize);
859 });
860
861 emitPatchpoint(block, patchpoint, Tmp(), instance);
862 }
863}
864
865void AirIRGenerator::emitThrowException(CCallHelpers& jit, ExceptionType type)
866{
867 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
868 auto jumpToExceptionStub = jit.jump();
869
870 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
871 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
872 });
873}
874
875auto AirIRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
876{
877 Checked<uint32_t, RecordOverflow> totalBytesChecked = count;
878 totalBytesChecked += m_locals.size();
879 uint32_t totalBytes;
880 WASM_COMPILE_FAIL_IF((totalBytesChecked.safeGet(totalBytes) == CheckedState::DidOverflow) || !m_locals.tryReserveCapacity(totalBytes), "can't allocate memory for ", totalBytes, " locals");
881
882 for (uint32_t i = 0; i < count; ++i) {
883 auto local = tmpForType(type);
884 m_locals.uncheckedAppend(local);
885 switch (type) {
886 case Type::Anyref:
887 append(Move, Arg::imm(JSValue::encode(jsNull())), local);
888 break;
889 case Type::I32:
890 case Type::I64: {
891 append(Xor64, local, local);
892 break;
893 }
894 case Type::F32:
895 case Type::F64: {
896 auto temp = g64();
897 // IEEE 754 "0" is just int32/64 zero.
898 append(Xor64, temp, temp);
899 append(type == Type::F32 ? Move32ToFloat : Move64ToDouble, temp, local);
900 break;
901 }
902 default:
903 RELEASE_ASSERT_NOT_REACHED();
904 }
905 }
906 return { };
907}
908
909auto AirIRGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
910{
911 return addConstant(m_currentBlock, type, value);
912}
913
914auto AirIRGenerator::addConstant(BasicBlock* block, Type type, uint64_t value) -> ExpressionType
915{
916 auto result = tmpForType(type);
917 switch (type) {
918 case Type::I32:
919 case Type::I64:
920 case Type::Anyref:
921 append(block, Move, Arg::bigImm(value), result);
922 break;
923 case Type::F32:
924 case Type::F64: {
925 auto tmp = g64();
926 append(block, Move, Arg::bigImm(value), tmp);
927 append(block, type == Type::F32 ? Move32ToFloat : Move64ToDouble, tmp, result);
928 break;
929 }
930
931 default:
932 RELEASE_ASSERT_NOT_REACHED();
933 }
934
935 return result;
936}
937
938auto AirIRGenerator::addArguments(const Signature& signature) -> PartialResult
939{
940 RELEASE_ASSERT(m_locals.size() == signature.argumentCount()); // We handle arguments in the prologue
941 return { };
942}
943
944auto AirIRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
945{
946 ASSERT(value.tmp());
947 result = tmpForType(Type::I32);
948 auto tmp = g64();
949
950 append(Move, Arg::bigImm(JSValue::encode(jsNull())), tmp);
951 append(Compare64, Arg::relCond(MacroAssembler::Equal), value, tmp, result);
952
953 return { };
954}
955
956auto AirIRGenerator::addTableGet(ExpressionType& idx, ExpressionType& result) -> PartialResult
957{
958 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
959 ASSERT(idx.tmp());
960 ASSERT(idx.type() == Type::I32);
961 result = tmpForType(Type::Anyref);
962
963 uint64_t (*doGet)(Instance*, int32_t) = [] (Instance* instance, int32_t idx) -> uint64_t {
964 return JSValue::encode(instance->table()->get(idx));
965 };
966
967 emitCCall(doGet, result, instanceValue(), idx);
968
969 return { };
970}
971
972auto AirIRGenerator::addTableSet(ExpressionType& idx, ExpressionType& value) -> PartialResult
973{
974 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
975 ASSERT(idx.tmp());
976 ASSERT(idx.type() == Type::I32);
977 ASSERT(value.tmp());
978
979 void (*doSet)(Instance*, int32_t, uint64_t value) = [] (Instance* instance, int32_t idx, uint64_t value) -> void {
980 // FIXME: We need to box wasm Funcrefs once they are supported here.
981 // <https://bugs.webkit.org/show_bug.cgi?id=198157>
982 instance->table()->set(idx, JSValue::decode(value));
983 };
984
985 emitCCall(doSet, TypedTmp(), instanceValue(), idx, value);
986
987 return { };
988}
989
990auto AirIRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
991{
992 ASSERT(m_locals[index].tmp());
993 result = tmpForType(m_locals[index].type());
994 append(moveOpForValueType(m_locals[index].type()), m_locals[index].tmp(), result);
995 return { };
996}
997
998auto AirIRGenerator::addUnreachable() -> PartialResult
999{
1000 B3::PatchpointValue* unreachable = addPatchpoint(B3::Void);
1001 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1002 this->emitThrowException(jit, ExceptionType::Unreachable);
1003 });
1004 unreachable->effects.terminal = true;
1005 emitPatchpoint(unreachable, Tmp());
1006 return { };
1007}
1008
1009auto AirIRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
1010{
1011 int32_t (*growMemory)(void*, Instance*, int32_t) = [] (void* callFrame, Instance* instance, int32_t delta) -> int32_t {
1012 instance->storeTopCallFrame(callFrame);
1013
1014 if (delta < 0)
1015 return -1;
1016
1017 auto grown = instance->memory()->grow(PageCount(delta));
1018 if (!grown) {
1019 switch (grown.error()) {
1020 case Memory::GrowFailReason::InvalidDelta:
1021 case Memory::GrowFailReason::InvalidGrowSize:
1022 case Memory::GrowFailReason::WouldExceedMaximum:
1023 case Memory::GrowFailReason::OutOfMemory:
1024 return -1;
1025 }
1026 RELEASE_ASSERT_NOT_REACHED();
1027 }
1028
1029 return grown.value().pageCount();
1030 };
1031
1032 result = g32();
1033 emitCCall(growMemory, result, TypedTmp { Tmp(GPRInfo::callFrameRegister), Type::I64 }, instanceValue(), delta);
1034 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_currentBlock);
1035
1036 return { };
1037}
1038
1039auto AirIRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
1040{
1041 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
1042
1043 auto temp1 = g64();
1044 auto temp2 = g64();
1045
1046 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedMemorySize(), B3::Width64));
1047 append(Move, Arg::addr(instanceValue(), Instance::offsetOfCachedMemorySize()), temp1);
1048 constexpr uint32_t shiftValue = 16;
1049 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
1050 append(Move, Arg::imm(16), temp2);
1051 addShift(Type::I32, Urshift64, temp1, temp2, result);
1052 append(Move32, result, result);
1053
1054 return { };
1055}
1056
1057auto AirIRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
1058{
1059 ASSERT(m_locals[index].tmp());
1060 append(moveOpForValueType(m_locals[index].type()), value, m_locals[index].tmp());
1061 return { };
1062}
1063
1064auto AirIRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
1065{
1066 Type type = m_info.globals[index].type;
1067
1068 result = tmpForType(type);
1069
1070 auto temp = g64();
1071
1072 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1073 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1074
1075 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1076 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1077 append(moveOpForValueType(type), Arg::addr(temp, offset), result);
1078 else {
1079 auto temp2 = g64();
1080 append(Move, Arg::bigImm(offset), temp2);
1081 append(Add64, temp2, temp, temp);
1082 append(moveOpForValueType(type), Arg::addr(temp), result);
1083 }
1084 return { };
1085}
1086
1087auto AirIRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
1088{
1089 auto temp = g64();
1090
1091 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1092 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1093
1094 Type type = m_info.globals[index].type;
1095
1096 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1097 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1098 append(moveOpForValueType(type), value, Arg::addr(temp, offset));
1099 else {
1100 auto temp2 = g64();
1101 append(Move, Arg::bigImm(offset), temp2);
1102 append(Add64, temp2, temp, temp);
1103 append(moveOpForValueType(type), value, Arg::addr(temp));
1104 }
1105
1106 if (type == Anyref)
1107 emitWriteBarrierForJSWrapper();
1108
1109 return { };
1110}
1111
1112inline void AirIRGenerator::emitWriteBarrierForJSWrapper()
1113{
1114 auto cell = g64();
1115 auto vm = g64();
1116 auto cellState = g32();
1117 auto threshold = g32();
1118
1119 BasicBlock* fenceCheckPath = m_code.addBlock();
1120 BasicBlock* fencePath = m_code.addBlock();
1121 BasicBlock* doSlowPath = m_code.addBlock();
1122 BasicBlock* continuation = m_code.addBlock();
1123
1124 append(Move, Arg::addr(instanceValue(), Instance::offsetOfOwner()), cell);
1125 append(Move, Arg::addr(cell, JSWebAssemblyInstance::offsetOfVM()), vm);
1126 append(Load8, Arg::addr(cell, JSCell::cellStateOffset()), cellState);
1127 append(Move32, Arg::addr(vm, VM::offsetOfHeapBarrierThreshold()), threshold);
1128
1129 append(Branch32, Arg::relCond(MacroAssembler::Above), cellState, threshold);
1130 m_currentBlock->setSuccessors(continuation, fenceCheckPath);
1131 m_currentBlock = fenceCheckPath;
1132
1133 append(Load8, Arg::addr(vm, VM::offsetOfHeapMutatorShouldBeFenced()), threshold);
1134 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), threshold, threshold);
1135 m_currentBlock->setSuccessors(doSlowPath, fencePath);
1136 m_currentBlock = fencePath;
1137
1138 auto* doFence = addPatchpoint(B3::Void);
1139 doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1140 jit.memoryFence();
1141 });
1142 emitPatchpoint(doFence, Tmp());
1143
1144 append(Load8, Arg::addr(cell, JSCell::cellStateOffset()), cellState);
1145 append(Branch32, Arg::relCond(MacroAssembler::Above), cellState, Arg::imm(blackThreshold));
1146 m_currentBlock->setSuccessors(continuation, doSlowPath);
1147 m_currentBlock = doSlowPath;
1148
1149 void (*writeBarrier)(JSWebAssemblyInstance*, VM*) = [] (JSWebAssemblyInstance* cell, VM* vm) -> void {
1150 ASSERT(cell);
1151 ASSERT(vm);
1152 vm->heap.writeBarrierSlowPath(cell);
1153 };
1154 emitCCall(writeBarrier, TypedTmp(), cell, vm);
1155 append(Jump);
1156 m_currentBlock->setSuccessors(continuation);
1157 m_currentBlock = continuation;
1158}
1159
1160inline AirIRGenerator::ExpressionType AirIRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
1161{
1162 ASSERT(m_memoryBaseGPR);
1163
1164 auto result = g64();
1165 append(Move32, pointer, result);
1166
1167 switch (m_mode) {
1168 case MemoryMode::BoundsChecking: {
1169 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
1170 ASSERT(m_memorySizeGPR);
1171 ASSERT(sizeOfOperation + offset > offset);
1172 auto temp = g64();
1173 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1174 append(Add64, result, temp);
1175
1176 emitCheck([&] {
1177 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, Tmp(m_memorySizeGPR));
1178 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1179 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1180 });
1181 break;
1182 }
1183
1184 case MemoryMode::Signaling: {
1185 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
1186 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
1187 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
1188 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
1189 // register + large offset access. We don't think this will be generated frequently.
1190 //
1191 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
1192 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
1193 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
1194 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
1195 if (offset >= Memory::fastMappedRedzoneBytes()) {
1196 uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
1197 auto temp = g64();
1198 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1199 append(Add64, result, temp);
1200 auto sizeMax = addConstant(Type::I64, maximum);
1201
1202 emitCheck([&] {
1203 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, sizeMax);
1204 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1205 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1206 });
1207 }
1208 break;
1209 }
1210 }
1211
1212 append(Add64, Tmp(m_memoryBaseGPR), result);
1213 return result;
1214}
1215
1216inline uint32_t sizeOfLoadOp(LoadOpType op)
1217{
1218 switch (op) {
1219 case LoadOpType::I32Load8S:
1220 case LoadOpType::I32Load8U:
1221 case LoadOpType::I64Load8S:
1222 case LoadOpType::I64Load8U:
1223 return 1;
1224 case LoadOpType::I32Load16S:
1225 case LoadOpType::I64Load16S:
1226 case LoadOpType::I32Load16U:
1227 case LoadOpType::I64Load16U:
1228 return 2;
1229 case LoadOpType::I32Load:
1230 case LoadOpType::I64Load32S:
1231 case LoadOpType::I64Load32U:
1232 case LoadOpType::F32Load:
1233 return 4;
1234 case LoadOpType::I64Load:
1235 case LoadOpType::F64Load:
1236 return 8;
1237 }
1238 RELEASE_ASSERT_NOT_REACHED();
1239}
1240
1241inline TypedTmp AirIRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
1242{
1243 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1244
1245 TypedTmp immTmp;
1246 TypedTmp newPtr;
1247 TypedTmp result;
1248
1249 Arg addrArg;
1250 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfLoadOp(op))))
1251 addrArg = Arg::addr(pointer, offset);
1252 else {
1253 immTmp = g64();
1254 newPtr = g64();
1255 append(Move, Arg::bigImm(offset), immTmp);
1256 append(Add64, immTmp, pointer, newPtr);
1257 addrArg = Arg::addr(newPtr);
1258 }
1259
1260 switch (op) {
1261 case LoadOpType::I32Load8S: {
1262 result = g32();
1263 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1264 break;
1265 }
1266
1267 case LoadOpType::I64Load8S: {
1268 result = g64();
1269 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1270 append(SignExtend32ToPtr, result, result);
1271 break;
1272 }
1273
1274 case LoadOpType::I32Load8U: {
1275 result = g32();
1276 appendEffectful(Load8, addrArg, result);
1277 break;
1278 }
1279
1280 case LoadOpType::I64Load8U: {
1281 result = g64();
1282 appendEffectful(Load8, addrArg, result);
1283 break;
1284 }
1285
1286 case LoadOpType::I32Load16S: {
1287 result = g32();
1288 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1289 break;
1290 }
1291
1292 case LoadOpType::I64Load16S: {
1293 result = g64();
1294 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1295 append(SignExtend32ToPtr, result, result);
1296 break;
1297 }
1298
1299 case LoadOpType::I32Load16U: {
1300 result = g32();
1301 appendEffectful(Load16, addrArg, result);
1302 break;
1303 }
1304
1305 case LoadOpType::I64Load16U: {
1306 result = g64();
1307 appendEffectful(Load16, addrArg, result);
1308 break;
1309 }
1310
1311 case LoadOpType::I32Load:
1312 result = g32();
1313 appendEffectful(Move32, addrArg, result);
1314 break;
1315
1316 case LoadOpType::I64Load32U: {
1317 result = g64();
1318 appendEffectful(Move32, addrArg, result);
1319 break;
1320 }
1321
1322 case LoadOpType::I64Load32S: {
1323 result = g64();
1324 appendEffectful(Move32, addrArg, result);
1325 append(SignExtend32ToPtr, result, result);
1326 break;
1327 }
1328
1329 case LoadOpType::I64Load: {
1330 result = g64();
1331 appendEffectful(Move, addrArg, result);
1332 break;
1333 }
1334
1335 case LoadOpType::F32Load: {
1336 result = f32();
1337 appendEffectful(MoveFloat, addrArg, result);
1338 break;
1339 }
1340
1341 case LoadOpType::F64Load: {
1342 result = f64();
1343 appendEffectful(MoveDouble, addrArg, result);
1344 break;
1345 }
1346 }
1347
1348 return result;
1349}
1350
1351auto AirIRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1352{
1353 ASSERT(pointer.tmp().isGP());
1354
1355 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
1356 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1357 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1358 auto* patch = addPatchpoint(B3::Void);
1359 patch->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1360 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1361 });
1362 emitPatchpoint(patch, Tmp());
1363
1364 // We won't reach here, so we just pick a random reg.
1365 switch (op) {
1366 case LoadOpType::I32Load8S:
1367 case LoadOpType::I32Load16S:
1368 case LoadOpType::I32Load:
1369 case LoadOpType::I32Load16U:
1370 case LoadOpType::I32Load8U:
1371 result = g32();
1372 break;
1373 case LoadOpType::I64Load8S:
1374 case LoadOpType::I64Load8U:
1375 case LoadOpType::I64Load16S:
1376 case LoadOpType::I64Load32U:
1377 case LoadOpType::I64Load32S:
1378 case LoadOpType::I64Load:
1379 case LoadOpType::I64Load16U:
1380 result = g64();
1381 break;
1382 case LoadOpType::F32Load:
1383 result = f32();
1384 break;
1385 case LoadOpType::F64Load:
1386 result = f64();
1387 break;
1388 }
1389 } else
1390 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
1391
1392 return { };
1393}
1394
1395inline uint32_t sizeOfStoreOp(StoreOpType op)
1396{
1397 switch (op) {
1398 case StoreOpType::I32Store8:
1399 case StoreOpType::I64Store8:
1400 return 1;
1401 case StoreOpType::I32Store16:
1402 case StoreOpType::I64Store16:
1403 return 2;
1404 case StoreOpType::I32Store:
1405 case StoreOpType::I64Store32:
1406 case StoreOpType::F32Store:
1407 return 4;
1408 case StoreOpType::I64Store:
1409 case StoreOpType::F64Store:
1410 return 8;
1411 }
1412 RELEASE_ASSERT_NOT_REACHED();
1413}
1414
1415
1416inline void AirIRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1417{
1418 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1419
1420 TypedTmp immTmp;
1421 TypedTmp newPtr;
1422
1423 Arg addrArg;
1424 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfStoreOp(op))))
1425 addrArg = Arg::addr(pointer, offset);
1426 else {
1427 immTmp = g64();
1428 newPtr = g64();
1429 append(Move, Arg::bigImm(offset), immTmp);
1430 append(Add64, immTmp, pointer, newPtr);
1431 addrArg = Arg::addr(newPtr);
1432 }
1433
1434 switch (op) {
1435 case StoreOpType::I64Store8:
1436 case StoreOpType::I32Store8:
1437 append(Store8, value, addrArg);
1438 return;
1439
1440 case StoreOpType::I64Store16:
1441 case StoreOpType::I32Store16:
1442 append(Store16, value, addrArg);
1443 return;
1444
1445 case StoreOpType::I64Store32:
1446 case StoreOpType::I32Store:
1447 append(Move32, value, addrArg);
1448 return;
1449
1450 case StoreOpType::I64Store:
1451 append(Move, value, addrArg);
1452 return;
1453
1454 case StoreOpType::F32Store:
1455 append(MoveFloat, value, addrArg);
1456 return;
1457
1458 case StoreOpType::F64Store:
1459 append(MoveDouble, value, addrArg);
1460 return;
1461 }
1462
1463 RELEASE_ASSERT_NOT_REACHED();
1464}
1465
1466auto AirIRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1467{
1468 ASSERT(pointer.tmp().isGP());
1469
1470 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1471 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1472 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1473 auto* throwException = addPatchpoint(B3::Void);
1474 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1475 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1476 });
1477 emitPatchpoint(throwException, Tmp());
1478 } else
1479 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1480
1481 return { };
1482}
1483
1484auto AirIRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1485{
1486 ASSERT(nonZero.type() == zero.type());
1487 result = tmpForType(nonZero.type());
1488 append(moveOpForValueType(nonZero.type()), nonZero, result);
1489
1490 BasicBlock* isZero = m_code.addBlock();
1491 BasicBlock* continuation = m_code.addBlock();
1492
1493 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), condition, condition);
1494 m_currentBlock->setSuccessors(isZero, continuation);
1495
1496 append(isZero, moveOpForValueType(zero.type()), zero, result);
1497 append(isZero, Jump);
1498 isZero->setSuccessors(continuation);
1499
1500 m_currentBlock = continuation;
1501
1502 return { };
1503}
1504
1505void AirIRGenerator::emitTierUpCheck(uint32_t decrementCount, B3::Origin origin)
1506{
1507 UNUSED_PARAM(origin);
1508
1509 if (!m_tierUp)
1510 return;
1511
1512 auto countdownPtr = g64();
1513 auto oldCountdown = g64();
1514 auto newCountdown = g64();
1515
1516 append(Move, Arg::bigImm(reinterpret_cast<uint64_t>(m_tierUp)), countdownPtr);
1517 append(Move32, Arg::addr(countdownPtr), oldCountdown);
1518
1519 RELEASE_ASSERT(Arg::isValidImmForm(decrementCount));
1520 append(Move32, oldCountdown, newCountdown);
1521 append(Sub32, Arg::imm(decrementCount), newCountdown);
1522 append(Move32, newCountdown, Arg::addr(countdownPtr));
1523
1524 auto* patch = addPatchpoint(B3::Void);
1525 B3::Effects effects = B3::Effects::none();
1526 effects.reads = B3::HeapRange::top();
1527 effects.writes = B3::HeapRange::top();
1528 patch->effects = effects;
1529
1530 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1531 MacroAssembler::Jump tierUp = jit.branch32(MacroAssembler::Above, params[0].gpr(), params[1].gpr());
1532 MacroAssembler::Label tierUpResume = jit.label();
1533
1534 params.addLatePath([=] (CCallHelpers& jit) {
1535 tierUp.link(&jit);
1536
1537 const unsigned extraPaddingBytes = 0;
1538 RegisterSet registersToSpill = { };
1539 registersToSpill.add(GPRInfo::argumentGPR1);
1540 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1541
1542 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1543 MacroAssembler::Call call = jit.nearCall();
1544
1545 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1546 jit.jump(tierUpResume);
1547
1548 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1549 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGTierUpThunkGenerator).code()));
1550
1551 });
1552 });
1553 });
1554
1555 emitPatchpoint(patch, Tmp(), newCountdown, oldCountdown);
1556}
1557
1558AirIRGenerator::ControlData AirIRGenerator::addLoop(Type signature)
1559{
1560 BasicBlock* body = m_code.addBlock();
1561 BasicBlock* continuation = m_code.addBlock();
1562
1563 append(Jump);
1564 m_currentBlock->setSuccessors(body);
1565
1566 m_currentBlock = body;
1567 emitTierUpCheck(TierUpCount::loopDecrement(), origin());
1568
1569 return ControlData(origin(), signature, tmpForType(signature), BlockType::Loop, continuation, body);
1570}
1571
1572AirIRGenerator::ControlData AirIRGenerator::addTopLevel(Type signature)
1573{
1574 return ControlData(B3::Origin(), signature, tmpForType(signature), BlockType::TopLevel, m_code.addBlock());
1575}
1576
1577AirIRGenerator::ControlData AirIRGenerator::addBlock(Type signature)
1578{
1579 return ControlData(origin(), signature, tmpForType(signature), BlockType::Block, m_code.addBlock());
1580}
1581
1582auto AirIRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
1583{
1584 BasicBlock* taken = m_code.addBlock();
1585 BasicBlock* notTaken = m_code.addBlock();
1586 BasicBlock* continuation = m_code.addBlock();
1587
1588 // Wasm bools are i32.
1589 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1590 m_currentBlock->setSuccessors(taken, notTaken);
1591
1592 m_currentBlock = taken;
1593 result = ControlData(origin(), signature, tmpForType(signature), BlockType::If, continuation, notTaken);
1594 return { };
1595}
1596
1597auto AirIRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
1598{
1599 unifyValuesWithBlock(currentStack, data.result);
1600 append(Jump);
1601 m_currentBlock->setSuccessors(data.continuation);
1602 return addElseToUnreachable(data);
1603}
1604
1605auto AirIRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1606{
1607 ASSERT(data.type() == BlockType::If);
1608 m_currentBlock = data.special;
1609 data.convertIfToBlock();
1610 return { };
1611}
1612
1613auto AirIRGenerator::addReturn(const ControlData& data, const ExpressionList& returnValues) -> PartialResult
1614{
1615 ASSERT(returnValues.size() <= 1);
1616 if (returnValues.size()) {
1617 Tmp returnValueGPR = Tmp(GPRInfo::returnValueGPR);
1618 Tmp returnValueFPR = Tmp(FPRInfo::returnValueFPR);
1619 switch (data.signature()) {
1620 case Type::I32:
1621 append(Move32, returnValues[0], returnValueGPR);
1622 append(Ret32, returnValueGPR);
1623 break;
1624 case Type::I64:
1625 case Type::Anyref:
1626 append(Move, returnValues[0], returnValueGPR);
1627 append(Ret64, returnValueGPR);
1628 break;
1629 case Type::F32:
1630 append(MoveFloat, returnValues[0], returnValueFPR);
1631 append(RetFloat, returnValueFPR);
1632 break;
1633 case Type::F64:
1634 append(MoveDouble, returnValues[0], returnValueFPR);
1635 append(RetFloat, returnValueFPR);
1636 break;
1637 default:
1638 RELEASE_ASSERT_NOT_REACHED();
1639 }
1640 } else
1641 append(RetVoid);
1642 return { };
1643}
1644
1645// NOTE: All branches in Wasm are on 32-bit ints
1646
1647auto AirIRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
1648{
1649 unifyValuesWithBlock(returnValues, data.resultForBranch());
1650
1651 BasicBlock* target = data.targetBlockForBranch();
1652 if (condition) {
1653 BasicBlock* continuation = m_code.addBlock();
1654 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1655 m_currentBlock->setSuccessors(target, continuation);
1656 m_currentBlock = continuation;
1657 } else {
1658 append(Jump);
1659 m_currentBlock->setSuccessors(target);
1660 }
1661
1662 return { };
1663}
1664
1665auto AirIRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
1666{
1667 auto& successors = m_currentBlock->successors();
1668 ASSERT(successors.isEmpty());
1669 for (const auto& target : targets) {
1670 unifyValuesWithBlock(expressionStack, target->resultForBranch());
1671 successors.append(target->targetBlockForBranch());
1672 }
1673 unifyValuesWithBlock(expressionStack, defaultTarget.resultForBranch());
1674 successors.append(defaultTarget.targetBlockForBranch());
1675
1676 ASSERT(condition.type() == Type::I32);
1677
1678 // FIXME: We should consider dynamically switching between a jump table
1679 // and a binary switch depending on the number of successors.
1680 // https://bugs.webkit.org/show_bug.cgi?id=194477
1681
1682 size_t numTargets = targets.size();
1683
1684 auto* patchpoint = addPatchpoint(B3::Void);
1685 patchpoint->effects = B3::Effects::none();
1686 patchpoint->effects.terminal = true;
1687 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1688
1689 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1690 AllowMacroScratchRegisterUsage allowScratch(jit);
1691
1692 Vector<int64_t> cases;
1693 cases.reserveInitialCapacity(numTargets);
1694 for (size_t i = 0; i < numTargets; ++i)
1695 cases.uncheckedAppend(i);
1696
1697 GPRReg valueReg = params[0].gpr();
1698 BinarySwitch binarySwitch(valueReg, cases, BinarySwitch::Int32);
1699
1700 Vector<CCallHelpers::Jump> caseJumps;
1701 caseJumps.resize(numTargets);
1702
1703 while (binarySwitch.advance(jit)) {
1704 unsigned value = binarySwitch.caseValue();
1705 unsigned index = binarySwitch.caseIndex();
1706 ASSERT_UNUSED(value, value == index);
1707 ASSERT(index < numTargets);
1708 caseJumps[index] = jit.jump();
1709 }
1710
1711 CCallHelpers::JumpList fallThrough = binarySwitch.fallThrough();
1712
1713 Vector<Box<CCallHelpers::Label>> successorLabels = params.successorLabels();
1714 ASSERT(successorLabels.size() == caseJumps.size() + 1);
1715
1716 params.addLatePath([=, caseJumps = WTFMove(caseJumps), successorLabels = WTFMove(successorLabels)] (CCallHelpers& jit) {
1717 for (size_t i = 0; i < numTargets; ++i)
1718 caseJumps[i].linkTo(*successorLabels[i], &jit);
1719 fallThrough.linkTo(*successorLabels[numTargets], &jit);
1720 });
1721 });
1722
1723 emitPatchpoint(patchpoint, TypedTmp(), condition);
1724
1725 return { };
1726}
1727
1728auto AirIRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
1729{
1730 ControlData& data = entry.controlData;
1731
1732 unifyValuesWithBlock(expressionStack, data.result);
1733 append(Jump);
1734 m_currentBlock->setSuccessors(data.continuation);
1735
1736 return addEndToUnreachable(entry);
1737}
1738
1739
1740auto AirIRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
1741{
1742 ControlData& data = entry.controlData;
1743 m_currentBlock = data.continuation;
1744
1745 if (data.type() == BlockType::If) {
1746 append(data.special, Jump);
1747 data.special->setSuccessors(m_currentBlock);
1748 }
1749
1750 for (const auto& result : data.result)
1751 entry.enclosedExpressionStack.append(result);
1752
1753 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1754 if (data.type() == BlockType::TopLevel)
1755 return addReturn(data, entry.enclosedExpressionStack);
1756
1757 return { };
1758}
1759
1760auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1761{
1762 ASSERT(signature.argumentCount() == args.size());
1763
1764 m_makesCalls = true;
1765
1766 Type returnType = signature.returnType();
1767 if (returnType != Type::Void)
1768 result = tmpForType(returnType);
1769
1770 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
1771
1772 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
1773 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1774
1775 auto currentInstance = g64();
1776 append(Move, instanceValue(), currentInstance);
1777
1778 auto targetInstance = g64();
1779
1780 // FIXME: We should have better isel here.
1781 // https://bugs.webkit.org/show_bug.cgi?id=193999
1782 append(Move, Arg::bigImm(Instance::offsetOfTargetInstance(functionIndex)), targetInstance);
1783 append(Add64, instanceValue(), targetInstance);
1784 append(Move, Arg::addr(targetInstance), targetInstance);
1785
1786 BasicBlock* isWasmBlock = m_code.addBlock();
1787 BasicBlock* isEmbedderBlock = m_code.addBlock();
1788 BasicBlock* continuation = m_code.addBlock();
1789
1790 append(BranchTest64, Arg::resCond(MacroAssembler::NonZero), targetInstance, targetInstance);
1791 m_currentBlock->setSuccessors(isWasmBlock, isEmbedderBlock);
1792
1793 {
1794 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1795 patchpoint->effects.writesPinned = true;
1796 patchpoint->effects.readsPinned = true;
1797 // We need to clobber all potential pinned registers since we might be leaving the instance.
1798 // We pessimistically assume we could be calling to something that is bounds checking.
1799 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1800 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1801
1802 Vector<ConstrainedTmp> patchArgs;
1803 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1804 patchArgs.append({ tmp, rep });
1805 });
1806
1807 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1808 AllowMacroScratchRegisterUsage allowScratch(jit);
1809 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1810 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1811 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1812 });
1813 });
1814
1815 emitPatchpoint(isWasmBlock, patchpoint, result, WTFMove(patchArgs));
1816 append(isWasmBlock, Jump);
1817 isWasmBlock->setSuccessors(continuation);
1818 }
1819
1820 {
1821 auto jumpDestination = g64();
1822 append(isEmbedderBlock, Move, Arg::bigImm(Instance::offsetOfWasmToEmbedderStub(functionIndex)), jumpDestination);
1823 append(isEmbedderBlock, Add64, instanceValue(), jumpDestination);
1824 append(isEmbedderBlock, Move, Arg::addr(jumpDestination), jumpDestination);
1825
1826 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1827 patchpoint->effects.writesPinned = true;
1828 patchpoint->effects.readsPinned = true;
1829 // We need to clobber all potential pinned registers since we might be leaving the instance.
1830 // We pessimistically assume we could be calling to something that is bounds checking.
1831 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1832 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1833
1834 Vector<ConstrainedTmp> patchArgs;
1835 patchArgs.append(jumpDestination);
1836
1837 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1838 patchArgs.append({ tmp, rep });
1839 });
1840
1841 patchpoint->setGenerator([returnType] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1842 AllowMacroScratchRegisterUsage allowScratch(jit);
1843 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
1844 });
1845
1846 emitPatchpoint(isEmbedderBlock, patchpoint, result, WTFMove(patchArgs));
1847 append(isEmbedderBlock, Jump);
1848 isEmbedderBlock->setSuccessors(continuation);
1849 }
1850
1851 m_currentBlock = continuation;
1852 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
1853 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, continuation);
1854 } else {
1855 auto* patchpoint = addPatchpoint(toB3Type(returnType));
1856 patchpoint->effects.writesPinned = true;
1857 patchpoint->effects.readsPinned = true;
1858
1859 Vector<ConstrainedTmp> patchArgs;
1860 wasmCallingConventionAir().setupCall(m_code, returnType, patchpoint, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
1861 patchArgs.append({ tmp, rep });
1862 });
1863
1864 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1865 AllowMacroScratchRegisterUsage allowScratch(jit);
1866 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
1867 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
1868 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
1869 });
1870 });
1871
1872 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(patchArgs));
1873 }
1874
1875 return { };
1876}
1877
1878auto AirIRGenerator::addCallIndirect(const Signature& signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
1879{
1880 ExpressionType calleeIndex = args.takeLast();
1881 ASSERT(signature.argumentCount() == args.size());
1882 ASSERT(m_info.tableInformation.type() == TableElementType::Funcref);
1883
1884 m_makesCalls = true;
1885 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
1886 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
1887 // can be to the embedder for our stack check calculation.
1888 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
1889
1890 auto currentInstance = g64();
1891 append(Move, instanceValue(), currentInstance);
1892
1893 ExpressionType callableFunctionBuffer = g64();
1894 ExpressionType instancesBuffer = g64();
1895 ExpressionType callableFunctionBufferLength = g64();
1896 {
1897 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfTable(), B3::Width64));
1898 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfFunctions(), B3::Width64));
1899 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfInstances(), B3::Width64));
1900 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfLength(), B3::Width64));
1901
1902 append(Move, Arg::addr(instanceValue(), Instance::offsetOfTable()), callableFunctionBufferLength);
1903 append(Move, Arg::addr(callableFunctionBufferLength, FuncRefTable::offsetOfFunctions()), callableFunctionBuffer);
1904 append(Move, Arg::addr(callableFunctionBufferLength, FuncRefTable::offsetOfInstances()), instancesBuffer);
1905 append(Move32, Arg::addr(callableFunctionBufferLength, Table::offsetOfLength()), callableFunctionBufferLength);
1906 }
1907
1908 append(Move32, calleeIndex, calleeIndex);
1909
1910 // Check the index we are looking for is valid.
1911 emitCheck([&] {
1912 return Inst(Branch32, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), calleeIndex, callableFunctionBufferLength);
1913 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1914 this->emitThrowException(jit, ExceptionType::OutOfBoundsCallIndirect);
1915 });
1916
1917 ExpressionType calleeCode = g64();
1918 {
1919 ExpressionType calleeSignatureIndex = g64();
1920 // Compute the offset in the table index space we are looking for.
1921 append(Move, Arg::imm(sizeof(WasmToWasmImportableFunction)), calleeSignatureIndex);
1922 append(Mul64, calleeIndex, calleeSignatureIndex);
1923 append(Add64, callableFunctionBuffer, calleeSignatureIndex);
1924
1925 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()), calleeCode); // Pointer to callee code.
1926
1927 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
1928 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
1929 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
1930
1931 // FIXME: This seems dumb to do two checks just for a nicer error message.
1932 // We should move just to use a single branch and then figure out what
1933 // error to use in the exception handler.
1934
1935 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfSignatureIndex()), calleeSignatureIndex);
1936
1937 emitCheck([&] {
1938 static_assert(Signature::invalidIndex == 0, "");
1939 return Inst(BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), calleeSignatureIndex, calleeSignatureIndex);
1940 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1941 this->emitThrowException(jit, ExceptionType::NullTableEntry);
1942 });
1943
1944 ExpressionType expectedSignatureIndex = g64();
1945 append(Move, Arg::bigImm(SignatureInformation::get(signature)), expectedSignatureIndex);
1946 emitCheck([&] {
1947 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::NotEqual), calleeSignatureIndex, expectedSignatureIndex);
1948 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1949 this->emitThrowException(jit, ExceptionType::BadSignature);
1950 });
1951 }
1952
1953 // Do a context switch if needed.
1954 {
1955 auto newContextInstance = g64();
1956 append(Move, Arg::index(instancesBuffer, calleeIndex, 8, 0), newContextInstance);
1957
1958 BasicBlock* doContextSwitch = m_code.addBlock();
1959 BasicBlock* continuation = m_code.addBlock();
1960
1961 append(Branch64, Arg::relCond(MacroAssembler::Equal), newContextInstance, instanceValue());
1962 m_currentBlock->setSuccessors(continuation, doContextSwitch);
1963
1964 auto* patchpoint = addPatchpoint(B3::Void);
1965 patchpoint->effects.writesPinned = true;
1966 // We pessimistically assume we're calling something with BoundsChecking memory.
1967 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
1968 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
1969 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1970 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
1971
1972 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1973 AllowMacroScratchRegisterUsage allowScratch(jit);
1974 GPRReg newContextInstance = params[0].gpr();
1975 GPRReg oldContextInstance = params[1].gpr();
1976 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
1977 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
1978 ASSERT(newContextInstance != baseMemory);
1979 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
1980 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
1981 jit.storeWasmContextInstance(newContextInstance);
1982 // FIXME: We should support more than one memory size register
1983 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
1984 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
1985 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
1986
1987 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
1988 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
1989
1990 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.sizeRegister, scratchOrSize);
1991 });
1992
1993 emitPatchpoint(doContextSwitch, patchpoint, Tmp(), newContextInstance, instanceValue());
1994 append(doContextSwitch, Jump);
1995 doContextSwitch->setSuccessors(continuation);
1996
1997 m_currentBlock = continuation;
1998 }
1999
2000 append(Move, Arg::addr(calleeCode), calleeCode);
2001
2002 Type returnType = signature.returnType();
2003 if (returnType != Type::Void)
2004 result = tmpForType(returnType);
2005
2006 auto* patch = addPatchpoint(toB3Type(returnType));
2007 patch->effects.writesPinned = true;
2008 patch->effects.readsPinned = true;
2009 // We need to clobber all potential pinned registers since we might be leaving the instance.
2010 // We pessimistically assume we're always calling something that is bounds checking so
2011 // because the wasm->wasm thunk unconditionally overrides the size registers.
2012 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
2013 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
2014 patch->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2015
2016 Vector<ConstrainedTmp> emitArgs;
2017 emitArgs.append(calleeCode);
2018 wasmCallingConventionAir().setupCall(m_code, returnType, patch, toTmpVector(args), [&] (Tmp tmp, B3::ValueRep rep) {
2019 emitArgs.append({ tmp, rep });
2020 });
2021 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2022 AllowMacroScratchRegisterUsage allowScratch(jit);
2023 jit.call(params[returnType == Void ? 0 : 1].gpr(), WasmEntryPtrTag);
2024 });
2025
2026 emitPatchpoint(m_currentBlock, patch, result, WTFMove(emitArgs));
2027
2028 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
2029 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, m_currentBlock);
2030
2031 return { };
2032}
2033
2034void AirIRGenerator::unify(const ExpressionType& dst, const ExpressionType& source)
2035{
2036 ASSERT(dst.type() == source.type());
2037 append(moveOpForValueType(dst.type()), source, dst);
2038}
2039
2040void AirIRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, const ResultList& result)
2041{
2042 ASSERT(result.size() <= resultStack.size());
2043
2044 for (size_t i = 0; i < result.size(); ++i)
2045 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
2046}
2047
2048void AirIRGenerator::dump(const Vector<ControlEntry>&, const ExpressionList*)
2049{
2050}
2051
2052auto AirIRGenerator::origin() -> B3::Origin
2053{
2054 // FIXME: We should implement a way to give Inst's an origin.
2055 return B3::Origin();
2056}
2057
2058Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext& compilationContext, const uint8_t* functionStart, size_t functionLength, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
2059{
2060 auto result = std::make_unique<InternalFunction>();
2061
2062 compilationContext.embedderEntrypointJIT = std::make_unique<CCallHelpers>();
2063 compilationContext.wasmEntrypointJIT = std::make_unique<CCallHelpers>();
2064
2065 B3::Procedure procedure;
2066 Code& code = procedure.code();
2067
2068 procedure.setOriginPrinter([] (PrintStream& out, B3::Origin origin) {
2069 if (origin.data())
2070 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
2071 });
2072
2073 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
2074 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
2075 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
2076 // optLevel=1.
2077 procedure.setNeedsUsedRegisters(false);
2078
2079 procedure.setOptLevel(Options::webAssemblyBBQOptimizationLevel());
2080
2081 AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, throwWasmException, signature);
2082 FunctionParser<AirIRGenerator> parser(irGenerator, functionStart, functionLength, signature, info);
2083 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
2084
2085
2086 for (BasicBlock* block : code) {
2087 for (size_t i = 0; i < block->numSuccessors(); ++i)
2088 block->successorBlock(i)->addPredecessor(block);
2089 }
2090
2091 {
2092 B3::Air::prepareForGeneration(code);
2093 B3::Air::generate(code, *compilationContext.wasmEntrypointJIT);
2094 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
2095 result->entrypoint.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
2096 }
2097
2098 return result;
2099}
2100
2101template <typename IntType>
2102void AirIRGenerator::emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right)
2103{
2104 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2105
2106 emitCheck([&] {
2107 return Inst(sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), right, right);
2108 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2109 this->emitThrowException(jit, ExceptionType::DivisionByZero);
2110 });
2111
2112 if (isSignedDiv) {
2113 ASSERT(std::is_signed<IntType>::value);
2114 IntType min = std::numeric_limits<IntType>::min();
2115
2116 // FIXME: Better isel for compare with imms here.
2117 // https://bugs.webkit.org/show_bug.cgi?id=193999
2118 auto minTmp = sizeof(IntType) == 4 ? g32() : g64();
2119 auto negOne = sizeof(IntType) == 4 ? g32() : g64();
2120
2121 B3::Air::Opcode op = sizeof(IntType) == 4 ? Compare32 : Compare64;
2122 append(Move, Arg::bigImm(static_cast<uint64_t>(min)), minTmp);
2123 append(op, Arg::relCond(MacroAssembler::Equal), left, minTmp, minTmp);
2124
2125 append(Move, Arg::imm(-1), negOne);
2126 append(op, Arg::relCond(MacroAssembler::Equal), right, negOne, negOne);
2127
2128 emitCheck([&] {
2129 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), minTmp, negOne);
2130 },
2131 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2132 this->emitThrowException(jit, ExceptionType::IntegerOverflow);
2133 });
2134 }
2135}
2136
2137template <typename IntType>
2138void AirIRGenerator::emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result)
2139{
2140 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2141
2142 result = sizeof(IntType) == 4 ? g32() : g64();
2143
2144 bool isSigned = std::is_signed<IntType>::value;
2145
2146 if (isARM64()) {
2147 B3::Air::Opcode div;
2148 switch (sizeof(IntType)) {
2149 case 4:
2150 div = isSigned ? Div32 : UDiv32;
2151 break;
2152 case 8:
2153 div = isSigned ? Div64 : UDiv64;
2154 break;
2155 }
2156
2157 append(div, lhs, rhs, result);
2158
2159 if (!isDiv) {
2160 append(sizeof(IntType) == 4 ? Mul32 : Mul64, result, rhs, result);
2161 append(sizeof(IntType) == 4 ? Sub32 : Sub64, lhs, result, result);
2162 }
2163
2164 return;
2165 }
2166
2167#if CPU(X86) || CPU(X86_64)
2168 Tmp eax(X86Registers::eax);
2169 Tmp edx(X86Registers::edx);
2170
2171 if (isSigned) {
2172 B3::Air::Opcode convertToDoubleWord;
2173 B3::Air::Opcode div;
2174 switch (sizeof(IntType)) {
2175 case 4:
2176 convertToDoubleWord = X86ConvertToDoubleWord32;
2177 div = X86Div32;
2178 break;
2179 case 8:
2180 convertToDoubleWord = X86ConvertToQuadWord64;
2181 div = X86Div64;
2182 break;
2183 default:
2184 RELEASE_ASSERT_NOT_REACHED();
2185 }
2186
2187 // We implement "res = Div<Chill>/Mod<Chill>(num, den)" as follows:
2188 //
2189 // if (den + 1 <=_unsigned 1) {
2190 // if (!den) {
2191 // res = 0;
2192 // goto done;
2193 // }
2194 // if (num == -2147483648) {
2195 // res = isDiv ? num : 0;
2196 // goto done;
2197 // }
2198 // }
2199 // res = num (/ or %) dev;
2200 // done:
2201
2202 BasicBlock* denIsGood = m_code.addBlock();
2203 BasicBlock* denMayBeBad = m_code.addBlock();
2204 BasicBlock* denNotZero = m_code.addBlock();
2205 BasicBlock* continuation = m_code.addBlock();
2206
2207 auto temp = sizeof(IntType) == 4 ? g32() : g64();
2208 auto one = addConstant(sizeof(IntType) == 4 ? Type::I32 : Type::I64, 1);
2209
2210 append(sizeof(IntType) == 4 ? Add32 : Add64, rhs, one, temp);
2211 append(sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Above), temp, one);
2212 m_currentBlock->setSuccessors(denIsGood, denMayBeBad);
2213
2214 append(denMayBeBad, Xor64, result, result);
2215 append(denMayBeBad, sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, Arg::resCond(MacroAssembler::Zero), rhs, rhs);
2216 denMayBeBad->setSuccessors(continuation, denNotZero);
2217
2218 auto min = addConstant(denNotZero, sizeof(IntType) == 4 ? Type::I32 : Type::I64, std::numeric_limits<IntType>::min());
2219 if (isDiv)
2220 append(denNotZero, sizeof(IntType) == 4 ? Move32 : Move, min, result);
2221 else {
2222 // Result is zero, as set above...
2223 }
2224 append(denNotZero, sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Equal), lhs, min);
2225 denNotZero->setSuccessors(continuation, denIsGood);
2226
2227 auto divResult = isDiv ? eax : edx;
2228 append(denIsGood, Move, lhs, eax);
2229 append(denIsGood, convertToDoubleWord, eax, edx);
2230 append(denIsGood, div, eax, edx, rhs);
2231 append(denIsGood, sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2232 append(denIsGood, Jump);
2233 denIsGood->setSuccessors(continuation);
2234
2235 m_currentBlock = continuation;
2236 return;
2237 }
2238
2239 B3::Air::Opcode div = sizeof(IntType) == 4 ? X86UDiv32 : X86UDiv64;
2240
2241 Tmp divResult = isDiv ? eax : edx;
2242
2243 append(Move, lhs, eax);
2244 append(Xor64, edx, edx);
2245 append(div, eax, edx, rhs);
2246 append(sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2247#else
2248 RELEASE_ASSERT_NOT_REACHED();
2249#endif
2250}
2251
2252template<>
2253auto AirIRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2254{
2255 emitChecksForModOrDiv<int32_t>(true, left, right);
2256 emitModOrDiv<int32_t>(true, left, right, result);
2257 return { };
2258}
2259
2260template<>
2261auto AirIRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2262{
2263 emitChecksForModOrDiv<int32_t>(false, left, right);
2264 emitModOrDiv<int32_t>(false, left, right, result);
2265 return { };
2266}
2267
2268template<>
2269auto AirIRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2270{
2271 emitChecksForModOrDiv<uint32_t>(false, left, right);
2272 emitModOrDiv<uint32_t>(true, left, right, result);
2273 return { };
2274}
2275
2276template<>
2277auto AirIRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2278{
2279 emitChecksForModOrDiv<uint32_t>(false, left, right);
2280 emitModOrDiv<uint32_t>(false, left, right, result);
2281 return { };
2282}
2283
2284template<>
2285auto AirIRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2286{
2287 emitChecksForModOrDiv<int64_t>(true, left, right);
2288 emitModOrDiv<int64_t>(true, left, right, result);
2289 return { };
2290}
2291
2292template<>
2293auto AirIRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2294{
2295 emitChecksForModOrDiv<int64_t>(false, left, right);
2296 emitModOrDiv<int64_t>(false, left, right, result);
2297 return { };
2298}
2299
2300template<>
2301auto AirIRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2302{
2303 emitChecksForModOrDiv<uint64_t>(false, left, right);
2304 emitModOrDiv<uint64_t>(true, left, right, result);
2305 return { };
2306}
2307
2308template<>
2309auto AirIRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2310{
2311 emitChecksForModOrDiv<uint64_t>(false, left, right);
2312 emitModOrDiv<uint64_t>(false, left, right, result);
2313 return { };
2314}
2315
2316template<>
2317auto AirIRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2318{
2319 auto* patchpoint = addPatchpoint(B3::Int32);
2320 patchpoint->effects = B3::Effects::none();
2321 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2322 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
2323 });
2324 result = g32();
2325 emitPatchpoint(patchpoint, result, arg);
2326 return { };
2327}
2328
2329template<>
2330auto AirIRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2331{
2332 auto* patchpoint = addPatchpoint(B3::Int64);
2333 patchpoint->effects = B3::Effects::none();
2334 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2335 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
2336 });
2337 result = g64();
2338 emitPatchpoint(patchpoint, result, arg);
2339 return { };
2340}
2341
2342template<>
2343auto AirIRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2344{
2345 result = g32();
2346
2347#if CPU(X86_64)
2348 if (MacroAssembler::supportsCountPopulation()) {
2349 auto* patchpoint = addPatchpoint(B3::Int32);
2350 patchpoint->effects = B3::Effects::none();
2351 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2352 jit.countPopulation32(params[1].gpr(), params[0].gpr());
2353 });
2354 emitPatchpoint(patchpoint, result, arg);
2355 return { };
2356 }
2357#endif
2358
2359 uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
2360 emitCCall(popcount, result, arg);
2361 return { };
2362}
2363
2364template<>
2365auto AirIRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2366{
2367 result = g64();
2368
2369#if CPU(X86_64)
2370 if (MacroAssembler::supportsCountPopulation()) {
2371 auto* patchpoint = addPatchpoint(B3::Int64);
2372 patchpoint->effects = B3::Effects::none();
2373 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2374 jit.countPopulation64(params[1].gpr(), params[0].gpr());
2375 });
2376 emitPatchpoint(patchpoint, result, arg);
2377 return { };
2378 }
2379#endif
2380
2381 uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
2382 emitCCall(popcount, result, arg);
2383 return { };
2384}
2385
2386template<>
2387auto AirIRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2388{
2389 auto* patchpoint = addPatchpoint(B3::Double);
2390 patchpoint->effects = B3::Effects::none();
2391 if (isX86())
2392 patchpoint->numGPScratchRegisters = 1;
2393 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2394 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2395 AllowMacroScratchRegisterUsage allowScratch(jit);
2396#if CPU(X86_64)
2397 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2398#else
2399 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
2400#endif
2401 });
2402 result = f64();
2403 emitPatchpoint(patchpoint, result, arg);
2404 return { };
2405}
2406
2407template<>
2408auto AirIRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2409{
2410 auto* patchpoint = addPatchpoint(B3::Float);
2411 patchpoint->effects = B3::Effects::none();
2412 if (isX86())
2413 patchpoint->numGPScratchRegisters = 1;
2414 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2415 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2416 AllowMacroScratchRegisterUsage allowScratch(jit);
2417#if CPU(X86_64)
2418 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2419#else
2420 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
2421#endif
2422 });
2423 result = f32();
2424 emitPatchpoint(patchpoint, result, arg);
2425 return { };
2426}
2427
2428template<>
2429auto AirIRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2430{
2431 auto* patchpoint = addPatchpoint(B3::Double);
2432 patchpoint->effects = B3::Effects::none();
2433 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2434 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
2435 });
2436 result = f64();
2437 emitPatchpoint(patchpoint, result, arg);
2438 return { };
2439}
2440
2441template<>
2442auto AirIRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2443{
2444 auto* patchpoint = addPatchpoint(B3::Float);
2445 patchpoint->effects = B3::Effects::none();
2446 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2447 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
2448 });
2449 result = f32();
2450 emitPatchpoint(patchpoint, result, arg);
2451 return { };
2452}
2453
2454template<>
2455auto AirIRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2456{
2457 auto* patchpoint = addPatchpoint(B3::Double);
2458 patchpoint->effects = B3::Effects::none();
2459 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2460 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
2461 });
2462 result = f64();
2463 emitPatchpoint(patchpoint, result, arg);
2464 return { };
2465}
2466
2467template<>
2468auto AirIRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2469{
2470 auto* patchpoint = addPatchpoint(B3::Float);
2471 patchpoint->effects = B3::Effects::none();
2472 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2473 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
2474 });
2475 result = f32();
2476 emitPatchpoint(patchpoint, result, arg);
2477 return { };
2478}
2479
2480template<>
2481auto AirIRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2482{
2483 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
2484 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
2485
2486 auto temp1 = g32();
2487 auto temp2 = g32();
2488 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2489 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2490 append(Or32, temp1, temp2);
2491
2492 emitCheck([&] {
2493 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2494 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2495 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2496 });
2497
2498 auto* patchpoint = addPatchpoint(B3::Int32);
2499 patchpoint->effects = B3::Effects::none();
2500 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2501 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
2502 });
2503 result = g32();
2504 emitPatchpoint(patchpoint, result, arg);
2505
2506 return { };
2507}
2508
2509template<>
2510auto AirIRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2511{
2512 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
2513 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
2514
2515 auto temp1 = g32();
2516 auto temp2 = g32();
2517 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2518 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2519 append(Or32, temp1, temp2);
2520
2521 emitCheck([&] {
2522 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2523 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2524 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2525 });
2526
2527 auto* patchpoint = addPatchpoint(B3::Int32);
2528 patchpoint->effects = B3::Effects::none();
2529 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2530 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
2531 });
2532 result = g32();
2533 emitPatchpoint(patchpoint, result, arg);
2534 return { };
2535}
2536
2537
2538template<>
2539auto AirIRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2540{
2541 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
2542 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2543
2544 auto temp1 = g32();
2545 auto temp2 = g32();
2546 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2547 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2548 append(Or32, temp1, temp2);
2549
2550 emitCheck([&] {
2551 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2552 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2553 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2554 });
2555
2556 auto* patchpoint = addPatchpoint(B3::Int32);
2557 patchpoint->effects = B3::Effects::none();
2558 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2559 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
2560 });
2561 result = g32();
2562 emitPatchpoint(patchpoint, result, arg);
2563 return { };
2564}
2565
2566template<>
2567auto AirIRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2568{
2569 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
2570 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2571
2572 auto temp1 = g32();
2573 auto temp2 = g32();
2574 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2575 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2576 append(Or32, temp1, temp2);
2577
2578 emitCheck([&] {
2579 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2580 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2581 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2582 });
2583
2584 auto* patchpoint = addPatchpoint(B3::Int32);
2585 patchpoint->effects = B3::Effects::none();
2586 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2587 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
2588 });
2589 result = g32();
2590 emitPatchpoint(patchpoint, result, arg);
2591 return { };
2592}
2593
2594template<>
2595auto AirIRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2596{
2597 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
2598 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
2599
2600 auto temp1 = g32();
2601 auto temp2 = g32();
2602 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2603 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2604 append(Or32, temp1, temp2);
2605
2606 emitCheck([&] {
2607 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2608 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2609 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2610 });
2611
2612 auto* patchpoint = addPatchpoint(B3::Int64);
2613 patchpoint->effects = B3::Effects::none();
2614 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2615 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
2616 });
2617
2618 result = g64();
2619 emitPatchpoint(patchpoint, result, arg);
2620 return { };
2621}
2622
2623template<>
2624auto AirIRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2625{
2626 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
2627 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2628
2629 auto temp1 = g32();
2630 auto temp2 = g32();
2631 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2632 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2633 append(Or32, temp1, temp2);
2634
2635 emitCheck([&] {
2636 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2637 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2638 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2639 });
2640
2641 TypedTmp signBitConstant;
2642 if (isX86())
2643 signBitConstant = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2644
2645 Vector<ConstrainedTmp> args;
2646 auto* patchpoint = addPatchpoint(B3::Int64);
2647 patchpoint->effects = B3::Effects::none();
2648 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2649 args.append(arg);
2650 if (isX86()) {
2651 args.append(signBitConstant);
2652 patchpoint->numFPScratchRegisters = 1;
2653 }
2654 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2655 AllowMacroScratchRegisterUsage allowScratch(jit);
2656 FPRReg scratch = InvalidFPRReg;
2657 FPRReg constant = InvalidFPRReg;
2658 if (isX86()) {
2659 scratch = params.fpScratch(0);
2660 constant = params[2].fpr();
2661 }
2662 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2663 });
2664
2665 result = g64();
2666 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2667 return { };
2668}
2669
2670template<>
2671auto AirIRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2672{
2673 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2674 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2675
2676 auto temp1 = g32();
2677 auto temp2 = g32();
2678 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2679 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2680 append(Or32, temp1, temp2);
2681
2682 emitCheck([&] {
2683 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2684 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2685 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2686 });
2687
2688 auto* patchpoint = addPatchpoint(B3::Int64);
2689 patchpoint->effects = B3::Effects::none();
2690 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2691 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2692 });
2693 result = g64();
2694 emitPatchpoint(patchpoint, result, arg);
2695 return { };
2696}
2697
2698template<>
2699auto AirIRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2700{
2701 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2702 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2703
2704 auto temp1 = g32();
2705 auto temp2 = g32();
2706 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2707 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2708 append(Or32, temp1, temp2);
2709
2710 emitCheck([&] {
2711 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2712 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2713 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2714 });
2715
2716 TypedTmp signBitConstant;
2717 if (isX86())
2718 signBitConstant = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2719
2720 auto* patchpoint = addPatchpoint(B3::Int64);
2721 patchpoint->effects = B3::Effects::none();
2722 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2723 Vector<ConstrainedTmp> args;
2724 args.append(arg);
2725 if (isX86()) {
2726 args.append(signBitConstant);
2727 patchpoint->numFPScratchRegisters = 1;
2728 }
2729 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2730 AllowMacroScratchRegisterUsage allowScratch(jit);
2731 FPRReg scratch = InvalidFPRReg;
2732 FPRReg constant = InvalidFPRReg;
2733 if (isX86()) {
2734 scratch = params.fpScratch(0);
2735 constant = params[2].fpr();
2736 }
2737 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2738 });
2739
2740 result = g64();
2741 emitPatchpoint(m_currentBlock, patchpoint, result, WTFMove(args));
2742
2743 return { };
2744}
2745
2746auto AirIRGenerator::addShift(Type type, B3::Air::Opcode op, ExpressionType value, ExpressionType shift, ExpressionType& result) -> PartialResult
2747{
2748 ASSERT(type == Type::I64 || type == Type::I32);
2749 result = tmpForType(type);
2750
2751 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2752 append(op, value, shift, result);
2753 return { };
2754 }
2755
2756#if CPU(X86_64)
2757 Tmp ecx = Tmp(X86Registers::ecx);
2758 append(Move, value, result);
2759 append(Move, shift, ecx);
2760 append(op, ecx, result);
2761#else
2762 RELEASE_ASSERT_NOT_REACHED();
2763#endif
2764 return { };
2765}
2766
2767auto AirIRGenerator::addIntegerSub(B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2768{
2769 ASSERT(op == Sub32 || op == Sub64);
2770
2771 result = op == Sub32 ? g32() : g64();
2772
2773 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2774 append(op, lhs, rhs, result);
2775 return { };
2776 }
2777
2778 RELEASE_ASSERT(isX86());
2779 // Sub a, b
2780 // means
2781 // b = b Sub a
2782 append(Move, lhs, result);
2783 append(op, rhs, result);
2784 return { };
2785}
2786
2787auto AirIRGenerator::addFloatingPointAbs(B3::Air::Opcode op, ExpressionType value, ExpressionType& result) -> PartialResult
2788{
2789 RELEASE_ASSERT(op == AbsFloat || op == AbsDouble);
2790
2791 result = op == AbsFloat ? f32() : f64();
2792
2793 if (isValidForm(op, Arg::Tmp, Arg::Tmp)) {
2794 append(op, value, result);
2795 return { };
2796 }
2797
2798 RELEASE_ASSERT(isX86());
2799
2800 if (op == AbsFloat) {
2801 auto constant = g32();
2802 append(Move, Arg::imm(static_cast<uint32_t>(~(1ull << 31))), constant);
2803 append(Move32ToFloat, constant, result);
2804 append(AndFloat, value, result);
2805 } else {
2806 auto constant = g64();
2807 append(Move, Arg::bigImm(~(1ull << 63)), constant);
2808 append(Move64ToDouble, constant, result);
2809 append(AndDouble, value, result);
2810 }
2811 return { };
2812}
2813
2814auto AirIRGenerator::addFloatingPointBinOp(Type type, B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
2815{
2816 ASSERT(type == Type::F32 || type == Type::F64);
2817 result = tmpForType(type);
2818
2819 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2820 append(op, lhs, rhs, result);
2821 return { };
2822 }
2823
2824 RELEASE_ASSERT(isX86());
2825
2826 // Op a, b
2827 // means
2828 // b = b Op a
2829 append(moveOpForValueType(type), lhs, result);
2830 append(op, rhs, result);
2831 return { };
2832}
2833
2834template<> auto AirIRGenerator::addOp<OpType::F32Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2835{
2836 result = f32();
2837 append(CeilFloat, arg0, result);
2838 return { };
2839}
2840
2841template<> auto AirIRGenerator::addOp<OpType::I32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2842{
2843 result = g32();
2844 append(Mul32, arg0, arg1, result);
2845 return { };
2846}
2847
2848template<> auto AirIRGenerator::addOp<OpType::I32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2849{
2850 return addIntegerSub(Sub32, arg0, arg1, result);
2851}
2852
2853template<> auto AirIRGenerator::addOp<OpType::F64Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2854{
2855 result = g32();
2856 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
2857 return { };
2858}
2859
2860template<> auto AirIRGenerator::addOp<OpType::F32DemoteF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2861{
2862 result = f32();
2863 append(ConvertDoubleToFloat, arg0, result);
2864 return { };
2865}
2866
2867template<> auto AirIRGenerator::addOp<OpType::F32Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2868{
2869 return addFloatingPointMinOrMax(F32, MinOrMax::Min, arg0, arg1, result);
2870}
2871
2872template<> auto AirIRGenerator::addOp<OpType::F64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2873{
2874 result = g32();
2875 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
2876 return { };
2877}
2878
2879template<> auto AirIRGenerator::addOp<OpType::F64Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2880{
2881 result = g32();
2882 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
2883 return { };
2884}
2885
2886auto AirIRGenerator::addFloatingPointMinOrMax(Type floatType, MinOrMax minOrMax, ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2887{
2888 ASSERT(floatType == F32 || floatType == F64);
2889 result = tmpForType(floatType);
2890
2891 BasicBlock* isEqual = m_code.addBlock();
2892 BasicBlock* notEqual = m_code.addBlock();
2893 BasicBlock* isLessThan = m_code.addBlock();
2894 BasicBlock* notLessThan = m_code.addBlock();
2895 BasicBlock* isGreaterThan = m_code.addBlock();
2896 BasicBlock* isNaN = m_code.addBlock();
2897 BasicBlock* continuation = m_code.addBlock();
2898
2899 auto branchOp = floatType == F32 ? BranchFloat : BranchDouble;
2900 append(m_currentBlock, branchOp, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1);
2901 m_currentBlock->setSuccessors(isEqual, notEqual);
2902
2903 append(notEqual, branchOp, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1);
2904 notEqual->setSuccessors(isLessThan, notLessThan);
2905
2906 append(notLessThan, branchOp, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1);
2907 notLessThan->setSuccessors(isGreaterThan, isNaN);
2908
2909 auto andOp = floatType == F32 ? AndFloat : AndDouble;
2910 auto orOp = floatType == F32 ? OrFloat : OrDouble;
2911 append(isEqual, minOrMax == MinOrMax::Max ? andOp : orOp, arg0, arg1, result);
2912 append(isEqual, Jump);
2913 isEqual->setSuccessors(continuation);
2914
2915 auto isLessThanResult = minOrMax == MinOrMax::Max ? arg1 : arg0;
2916 append(isLessThan, moveOpForValueType(floatType), isLessThanResult, result);
2917 append(isLessThan, Jump);
2918 isLessThan->setSuccessors(continuation);
2919
2920 auto isGreaterThanResult = minOrMax == MinOrMax::Max ? arg0 : arg1;
2921 append(isGreaterThan, moveOpForValueType(floatType), isGreaterThanResult, result);
2922 append(isGreaterThan, Jump);
2923 isGreaterThan->setSuccessors(continuation);
2924
2925 auto addOp = floatType == F32 ? AddFloat : AddDouble;
2926 append(isNaN, addOp, arg0, arg1, result);
2927 append(isNaN, Jump);
2928 isNaN->setSuccessors(continuation);
2929
2930 m_currentBlock = continuation;
2931
2932 return { };
2933}
2934
2935template<> auto AirIRGenerator::addOp<OpType::F32Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2936{
2937 return addFloatingPointMinOrMax(F32, MinOrMax::Max, arg0, arg1, result);
2938}
2939
2940template<> auto AirIRGenerator::addOp<OpType::F64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2941{
2942 return addFloatingPointBinOp(Type::F64, MulDouble, arg0, arg1, result);
2943}
2944
2945template<> auto AirIRGenerator::addOp<OpType::F32Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2946{
2947 return addFloatingPointBinOp(Type::F32, DivFloat, arg0, arg1, result);
2948}
2949
2950template<> auto AirIRGenerator::addOp<OpType::I32Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2951{
2952 result = g32();
2953 append(CountLeadingZeros32, arg0, result);
2954 return { };
2955}
2956
2957template<> auto AirIRGenerator::addOp<OpType::F32Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2958{
2959 // FIXME: We can have better codegen here for the imms and two operand forms on x86
2960 // https://bugs.webkit.org/show_bug.cgi?id=193999
2961 result = f32();
2962 auto temp1 = g32();
2963 auto sign = g32();
2964 auto value = g32();
2965
2966 // FIXME: Try to use Imm where possible:
2967 // https://bugs.webkit.org/show_bug.cgi?id=193999
2968 append(MoveFloatTo32, arg1, temp1);
2969 append(Move, Arg::bigImm(0x80000000), sign);
2970 append(And32, temp1, sign, sign);
2971
2972 append(MoveDoubleTo64, arg0, temp1);
2973 append(Move, Arg::bigImm(0x7fffffff), value);
2974 append(And32, temp1, value, value);
2975
2976 append(Or32, sign, value, value);
2977 append(Move32ToFloat, value, result);
2978
2979 return { };
2980}
2981
2982template<> auto AirIRGenerator::addOp<OpType::F64ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2983{
2984 result = f64();
2985 auto temp = g64();
2986 append(Move32, arg0, temp);
2987 append(ConvertInt64ToDouble, temp, result);
2988 return { };
2989}
2990
2991template<> auto AirIRGenerator::addOp<OpType::F32ReinterpretI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
2992{
2993 result = f32();
2994 append(Move32ToFloat, arg0, result);
2995 return { };
2996}
2997
2998template<> auto AirIRGenerator::addOp<OpType::I64And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
2999{
3000 result = g64();
3001 append(And64, arg0, arg1, result);
3002 return { };
3003}
3004
3005template<> auto AirIRGenerator::addOp<OpType::F32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3006{
3007 result = g32();
3008 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
3009 return { };
3010}
3011
3012template<> auto AirIRGenerator::addOp<OpType::F64Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3013{
3014 result = g32();
3015 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3016 return { };
3017}
3018
3019template<> auto AirIRGenerator::addOp<OpType::F32Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3020{
3021 result = f32();
3022 append(SqrtFloat, arg0, result);
3023 return { };
3024}
3025
3026template<> auto AirIRGenerator::addOp<OpType::F64Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3027{
3028 result = g32();
3029 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3030 return { };
3031}
3032
3033template<> auto AirIRGenerator::addOp<OpType::I64GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3034{
3035 result = g32();
3036 append(Compare64, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3037 return { };
3038}
3039
3040template<> auto AirIRGenerator::addOp<OpType::I64GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3041{
3042 result = g32();
3043 append(Compare64, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3044 return { };
3045}
3046
3047template<> auto AirIRGenerator::addOp<OpType::I64Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3048{
3049 result = g32();
3050 append(Test64, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3051 return { };
3052}
3053
3054template<> auto AirIRGenerator::addOp<OpType::F64Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3055{
3056 return addFloatingPointBinOp(Type::F64, DivDouble, arg0, arg1, result);
3057}
3058
3059template<> auto AirIRGenerator::addOp<OpType::F32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3060{
3061 result = f32();
3062 append(AddFloat, arg0, arg1, result);
3063 return { };
3064}
3065
3066template<> auto AirIRGenerator::addOp<OpType::I64Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3067{
3068 result = g64();
3069 append(Or64, arg0, arg1, result);
3070 return { };
3071}
3072
3073template<> auto AirIRGenerator::addOp<OpType::I32LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3074{
3075 result = g32();
3076 append(Compare32, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3077 return { };
3078}
3079
3080template<> auto AirIRGenerator::addOp<OpType::I32LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3081{
3082 result = g32();
3083 append(Compare32, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3084 return { };
3085}
3086
3087template<> auto AirIRGenerator::addOp<OpType::I64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3088{
3089 result = g32();
3090 append(Compare64, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3091 return { };
3092}
3093
3094template<> auto AirIRGenerator::addOp<OpType::I64Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3095{
3096 result = g64();
3097 append(CountLeadingZeros64, arg0, result);
3098 return { };
3099}
3100
3101template<> auto AirIRGenerator::addOp<OpType::F32Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3102{
3103 result = f32();
3104 if (isValidForm(NegateFloat, Arg::Tmp, Arg::Tmp))
3105 append(NegateFloat, arg0, result);
3106 else {
3107 auto constant = addConstant(Type::I32, bitwise_cast<uint32_t>(static_cast<float>(-0.0)));
3108 auto temp = g32();
3109 append(MoveFloatTo32, arg0, temp);
3110 append(Xor32, constant, temp);
3111 append(Move32ToFloat, temp, result);
3112 }
3113 return { };
3114}
3115
3116template<> auto AirIRGenerator::addOp<OpType::I32And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3117{
3118 result = g32();
3119 append(And32, arg0, arg1, result);
3120 return { };
3121}
3122
3123template<> auto AirIRGenerator::addOp<OpType::I32LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3124{
3125 result = g32();
3126 append(Compare32, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3127 return { };
3128}
3129
3130template<> auto AirIRGenerator::addOp<OpType::I64Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3131{
3132 return addShift(Type::I64, RotateRight64, arg0, arg1, result);
3133}
3134
3135template<> auto AirIRGenerator::addOp<OpType::F64Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3136{
3137 return addFloatingPointAbs(AbsDouble, arg0, result);
3138}
3139
3140template<> auto AirIRGenerator::addOp<OpType::I32LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3141{
3142 result = g32();
3143 append(Compare32, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3144 return { };
3145}
3146
3147template<> auto AirIRGenerator::addOp<OpType::I32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3148{
3149 result = g32();
3150 append(Compare32, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3151 return { };
3152}
3153
3154template<> auto AirIRGenerator::addOp<OpType::F64Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3155{
3156 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3157 // https://bugs.webkit.org/show_bug.cgi?id=193999
3158 result = f64();
3159 auto temp1 = g64();
3160 auto sign = g64();
3161 auto value = g64();
3162
3163 append(MoveDoubleTo64, arg1, temp1);
3164 append(Move, Arg::bigImm(0x8000000000000000), sign);
3165 append(And64, temp1, sign, sign);
3166
3167 append(MoveDoubleTo64, arg0, temp1);
3168 append(Move, Arg::bigImm(0x7fffffffffffffff), value);
3169 append(And64, temp1, value, value);
3170
3171 append(Or64, sign, value, value);
3172 append(Move64ToDouble, value, result);
3173
3174 return { };
3175}
3176
3177template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3178{
3179 result = f32();
3180 append(ConvertInt64ToFloat, arg0, result);
3181 return { };
3182}
3183
3184template<> auto AirIRGenerator::addOp<OpType::I64Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3185{
3186 if (isARM64()) {
3187 // ARM64 doesn't have a rotate left.
3188 auto newShift = g64();
3189 append(Move, arg1, newShift);
3190 append(Neg64, newShift);
3191 return addShift(Type::I64, RotateRight64, arg0, newShift, result);
3192 } else
3193 return addShift(Type::I64, RotateLeft64, arg0, arg1, result);
3194}
3195
3196template<> auto AirIRGenerator::addOp<OpType::F32Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3197{
3198 result = g32();
3199 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
3200 return { };
3201}
3202
3203template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3204{
3205 result = f64();
3206 append(ConvertInt32ToDouble, arg0, result);
3207 return { };
3208}
3209
3210template<> auto AirIRGenerator::addOp<OpType::F64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3211{
3212 result = g32();
3213 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3214 return { };
3215}
3216
3217template<> auto AirIRGenerator::addOp<OpType::F32Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3218{
3219 result = g32();
3220 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
3221 return { };
3222}
3223
3224template<> auto AirIRGenerator::addOp<OpType::F32Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3225{
3226 result = g32();
3227 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3228 return { };
3229}
3230
3231template<> auto AirIRGenerator::addOp<OpType::I32ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3232{
3233 return addShift(Type::I32, Urshift32, arg0, arg1, result);
3234}
3235
3236template<> auto AirIRGenerator::addOp<OpType::F32ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3237{
3238 result = f32();
3239 auto temp = g64();
3240 append(Move32, arg0, temp);
3241 append(ConvertInt64ToFloat, temp, result);
3242 return { };
3243}
3244
3245template<> auto AirIRGenerator::addOp<OpType::I32ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3246{
3247 return addShift(Type::I32, Rshift32, arg0, arg1, result);
3248}
3249
3250template<> auto AirIRGenerator::addOp<OpType::I32GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3251{
3252 result = g32();
3253 append(Compare32, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3254 return { };
3255}
3256
3257template<> auto AirIRGenerator::addOp<OpType::F64Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3258{
3259 result = f64();
3260 append(CeilDouble, arg0, result);
3261 return { };
3262}
3263
3264template<> auto AirIRGenerator::addOp<OpType::I32GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3265{
3266 result = g32();
3267 append(Compare32, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3268 return { };
3269}
3270
3271template<> auto AirIRGenerator::addOp<OpType::I32Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3272{
3273 return addShift(Type::I32, Lshift32, arg0, arg1, result);
3274}
3275
3276template<> auto AirIRGenerator::addOp<OpType::F64Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3277{
3278 result = f64();
3279 append(FloorDouble, arg0, result);
3280 return { };
3281}
3282
3283template<> auto AirIRGenerator::addOp<OpType::I32Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3284{
3285 result = g32();
3286 append(Xor32, arg0, arg1, result);
3287 return { };
3288}
3289
3290template<> auto AirIRGenerator::addOp<OpType::F32Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3291{
3292 return addFloatingPointAbs(AbsFloat, arg0, result);
3293}
3294
3295template<> auto AirIRGenerator::addOp<OpType::F64Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3296{
3297 return addFloatingPointMinOrMax(F64, MinOrMax::Min, arg0, arg1, result);
3298}
3299
3300template<> auto AirIRGenerator::addOp<OpType::F32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3301{
3302 result = f32();
3303 append(MulFloat, arg0, arg1, result);
3304 return { };
3305}
3306
3307template<> auto AirIRGenerator::addOp<OpType::I64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3308{
3309 return addIntegerSub(Sub64, arg0, arg1, result);
3310}
3311
3312template<> auto AirIRGenerator::addOp<OpType::I32ReinterpretF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3313{
3314 result = g32();
3315 append(MoveFloatTo32, arg0, result);
3316 return { };
3317}
3318
3319template<> auto AirIRGenerator::addOp<OpType::I32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3320{
3321 result = g32();
3322 append(Add32, arg0, arg1, result);
3323 return { };
3324}
3325
3326template<> auto AirIRGenerator::addOp<OpType::F64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3327{
3328 return addFloatingPointBinOp(Type::F64, SubDouble, arg0, arg1, result);
3329}
3330
3331template<> auto AirIRGenerator::addOp<OpType::I32Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3332{
3333 result = g32();
3334 append(Or32, arg0, arg1, result);
3335 return { };
3336}
3337
3338template<> auto AirIRGenerator::addOp<OpType::I64LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3339{
3340 result = g32();
3341 append(Compare64, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3342 return { };
3343}
3344
3345template<> auto AirIRGenerator::addOp<OpType::I64LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3346{
3347 result = g32();
3348 append(Compare64, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3349 return { };
3350}
3351
3352template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3353{
3354 result = f64();
3355 append(ConvertInt64ToDouble, arg0, result);
3356 return { };
3357}
3358
3359template<> auto AirIRGenerator::addOp<OpType::I64Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3360{
3361 result = g64();
3362 append(Xor64, arg0, arg1, result);
3363 return { };
3364}
3365
3366template<> auto AirIRGenerator::addOp<OpType::I64GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3367{
3368 result = g32();
3369 append(Compare64, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3370 return { };
3371}
3372
3373template<> auto AirIRGenerator::addOp<OpType::I64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3374{
3375 result = g64();
3376 append(Mul64, arg0, arg1, result);
3377 return { };
3378}
3379
3380template<> auto AirIRGenerator::addOp<OpType::F32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3381{
3382 result = f32();
3383 if (isValidForm(SubFloat, Arg::Tmp, Arg::Tmp, Arg::Tmp))
3384 append(SubFloat, arg0, arg1, result);
3385 else {
3386 RELEASE_ASSERT(isX86());
3387 append(MoveFloat, arg0, result);
3388 append(SubFloat, arg1, result);
3389 }
3390 return { };
3391}
3392
3393template<> auto AirIRGenerator::addOp<OpType::F64PromoteF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3394{
3395 result = f64();
3396 append(ConvertFloatToDouble, arg0, result);
3397 return { };
3398}
3399
3400template<> auto AirIRGenerator::addOp<OpType::F64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3401{
3402 result = f64();
3403 append(AddDouble, arg0, arg1, result);
3404 return { };
3405}
3406
3407template<> auto AirIRGenerator::addOp<OpType::I64GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3408{
3409 result = g32();
3410 append(Compare64, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3411 return { };
3412}
3413
3414template<> auto AirIRGenerator::addOp<OpType::I64ExtendUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3415{
3416 result = g64();
3417 append(Move32, arg0, result);
3418 return { };
3419}
3420
3421template<> auto AirIRGenerator::addOp<OpType::I32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3422{
3423 result = g32();
3424 RELEASE_ASSERT(arg0 && arg1);
3425 append(Compare32, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3426 return { };
3427}
3428
3429template<> auto AirIRGenerator::addOp<OpType::F64ReinterpretI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3430{
3431 result = f64();
3432 append(Move64ToDouble, arg0, result);
3433 return { };
3434}
3435
3436template<> auto AirIRGenerator::addOp<OpType::F32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3437{
3438 result = g32();
3439 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3440 return { };
3441}
3442
3443template<> auto AirIRGenerator::addOp<OpType::I64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3444{
3445 result = g32();
3446 append(Compare64, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3447 return { };
3448}
3449
3450template<> auto AirIRGenerator::addOp<OpType::F32Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3451{
3452 result = f32();
3453 append(FloorFloat, arg0, result);
3454 return { };
3455}
3456
3457template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3458{
3459 result = f32();
3460 append(ConvertInt32ToFloat, arg0, result);
3461 return { };
3462}
3463
3464template<> auto AirIRGenerator::addOp<OpType::I32Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3465{
3466 result = g32();
3467 append(Test32, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3468 return { };
3469}
3470
3471template<> auto AirIRGenerator::addOp<OpType::I64ReinterpretF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3472{
3473 result = g64();
3474 append(MoveDoubleTo64, arg0, result);
3475 return { };
3476}
3477
3478template<> auto AirIRGenerator::addOp<OpType::I64ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3479{
3480 return addShift(Type::I64, Rshift64, arg0, arg1, result);
3481}
3482
3483template<> auto AirIRGenerator::addOp<OpType::I64ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3484{
3485 return addShift(Type::I64, Urshift64, arg0, arg1, result);
3486}
3487
3488template<> auto AirIRGenerator::addOp<OpType::F64Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3489{
3490 result = f64();
3491 append(SqrtDouble, arg0, result);
3492 return { };
3493}
3494
3495template<> auto AirIRGenerator::addOp<OpType::I64Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3496{
3497 return addShift(Type::I64, Lshift64, arg0, arg1, result);
3498}
3499
3500template<> auto AirIRGenerator::addOp<OpType::F32Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3501{
3502 result = g32();
3503 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3504 return { };
3505}
3506
3507template<> auto AirIRGenerator::addOp<OpType::I32WrapI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3508{
3509 result = g32();
3510 append(Move32, arg0, result);
3511 return { };
3512}
3513
3514template<> auto AirIRGenerator::addOp<OpType::I32Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3515{
3516 if (isARM64()) {
3517 // ARM64 doesn't have a rotate left.
3518 auto newShift = g64();
3519 append(Move, arg1, newShift);
3520 append(Neg64, newShift);
3521 return addShift(Type::I32, RotateRight32, arg0, newShift, result);
3522 } else
3523 return addShift(Type::I32, RotateLeft32, arg0, arg1, result);
3524}
3525
3526template<> auto AirIRGenerator::addOp<OpType::I32Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3527{
3528 return addShift(Type::I32, RotateRight32, arg0, arg1, result);
3529}
3530
3531template<> auto AirIRGenerator::addOp<OpType::I32GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3532{
3533 result = g32();
3534 append(Compare32, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3535 return { };
3536}
3537
3538template<> auto AirIRGenerator::addOp<OpType::I64ExtendSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3539{
3540 result = g64();
3541 append(SignExtend32ToPtr, arg0, result);
3542 return { };
3543}
3544
3545template<> auto AirIRGenerator::addOp<OpType::I32GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3546{
3547 result = g32();
3548 append(Compare32, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3549 return { };
3550}
3551
3552template<> auto AirIRGenerator::addOp<OpType::F64Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3553{
3554 result = f64();
3555 if (isValidForm(NegateDouble, Arg::Tmp, Arg::Tmp))
3556 append(NegateDouble, arg0, result);
3557 else {
3558 auto constant = addConstant(Type::I64, bitwise_cast<uint64_t>(static_cast<double>(-0.0)));
3559 auto temp = g64();
3560 append(MoveDoubleTo64, arg0, temp);
3561 append(Xor64, constant, temp);
3562 append(Move64ToDouble, temp, result);
3563 }
3564 return { };
3565}
3566
3567template<> auto AirIRGenerator::addOp<OpType::F64Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3568{
3569 return addFloatingPointMinOrMax(F64, MinOrMax::Max, arg0, arg1, result);
3570}
3571
3572template<> auto AirIRGenerator::addOp<OpType::I64LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3573{
3574 result = g32();
3575 append(Compare64, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3576 return { };
3577}
3578
3579template<> auto AirIRGenerator::addOp<OpType::I64LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3580{
3581 result = g32();
3582 append(Compare64, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3583 return { };
3584}
3585
3586template<> auto AirIRGenerator::addOp<OpType::I64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3587{
3588 result = g64();
3589 append(Add64, arg0, arg1, result);
3590 return { };
3591}
3592
3593} } // namespace JSC::Wasm
3594
3595#endif // ENABLE(WEBASSEMBLY)
3596