1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmAirIRGenerator.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "AirCode.h"
32#include "AirGenerate.h"
33#include "AirHelpers.h"
34#include "AirOpcodeUtils.h"
35#include "AirValidate.h"
36#include "AllowMacroScratchRegisterUsageIf.h"
37#include "B3CCallValue.h"
38#include "B3CheckSpecial.h"
39#include "B3CheckValue.h"
40#include "B3PatchpointSpecial.h"
41#include "B3Procedure.h"
42#include "B3ProcedureInlines.h"
43#include "BinarySwitch.h"
44#include "DisallowMacroScratchRegisterUsage.h"
45#include "JSCInlines.h"
46#include "JSWebAssemblyInstance.h"
47#include "ScratchRegisterAllocator.h"
48#include "VirtualRegister.h"
49#include "WasmCallingConvention.h"
50#include "WasmContextInlines.h"
51#include "WasmExceptionType.h"
52#include "WasmFunctionParser.h"
53#include "WasmInstance.h"
54#include "WasmMemory.h"
55#include "WasmOMGPlan.h"
56#include "WasmOSREntryData.h"
57#include "WasmOpcodeOrigin.h"
58#include "WasmOperations.h"
59#include "WasmSignatureInlines.h"
60#include "WasmThunks.h"
61#include <limits>
62#include <wtf/Box.h>
63#include <wtf/Optional.h>
64#include <wtf/StdLibExtras.h>
65
66namespace JSC { namespace Wasm {
67
68using namespace B3::Air;
69
70struct ConstrainedTmp {
71 ConstrainedTmp() = default;
72 ConstrainedTmp(Tmp tmp)
73 : ConstrainedTmp(tmp, tmp.isReg() ? B3::ValueRep::reg(tmp.reg()) : B3::ValueRep::SomeRegister)
74 { }
75
76 ConstrainedTmp(Tmp tmp, B3::ValueRep rep)
77 : tmp(tmp)
78 , rep(rep)
79 {
80 }
81
82 explicit operator bool() const { return !!tmp; }
83
84 Tmp tmp;
85 B3::ValueRep rep;
86};
87
88class TypedTmp {
89public:
90 constexpr TypedTmp()
91 : m_tmp()
92 , m_type(Type::Void)
93 { }
94
95 TypedTmp(Tmp tmp, Type type)
96 : m_tmp(tmp)
97 , m_type(type)
98 { }
99
100 TypedTmp(const TypedTmp&) = default;
101 TypedTmp(TypedTmp&&) = default;
102 TypedTmp& operator=(TypedTmp&&) = default;
103 TypedTmp& operator=(const TypedTmp&) = default;
104
105 bool operator==(const TypedTmp& other) const
106 {
107 return m_tmp == other.m_tmp && m_type == other.m_type;
108 }
109 bool operator!=(const TypedTmp& other) const
110 {
111 return !(*this == other);
112 }
113
114 explicit operator bool() const { return !!tmp(); }
115
116 operator Tmp() const { return tmp(); }
117 operator Arg() const { return Arg(tmp()); }
118 Tmp tmp() const { return m_tmp; }
119 Type type() const { return m_type; }
120
121 void dump(PrintStream& out) const
122 {
123 out.print("(", m_tmp, ", ", m_type, ")");
124 }
125
126private:
127
128 Tmp m_tmp;
129 Type m_type;
130};
131
132using TypedTmps = Vector<TypedTmp, 1>;
133
134class AirIRGenerator {
135public:
136 struct ControlData {
137 ControlData(B3::Origin origin, BlockSignature result, TypedTmps resultTmps, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
138 : controlBlockType(type)
139 , continuation(continuation)
140 , special(special)
141 , results(resultTmps)
142 , returnType(result)
143 {
144 UNUSED_PARAM(origin);
145 }
146
147 ControlData()
148 {
149 }
150
151 void dump(PrintStream& out) const
152 {
153 switch (blockType()) {
154 case BlockType::If:
155 out.print("If: ");
156 break;
157 case BlockType::Block:
158 out.print("Block: ");
159 break;
160 case BlockType::Loop:
161 out.print("Loop: ");
162 break;
163 case BlockType::TopLevel:
164 out.print("TopLevel: ");
165 break;
166 }
167 out.print("Continuation: ", *continuation, ", Special: ");
168 if (special)
169 out.print(*special);
170 else
171 out.print("None");
172
173 CommaPrinter comma(", ", " Result Tmps: [");
174 for (const auto& tmp : results)
175 out.print(comma, tmp);
176 if (comma.didPrint())
177 out.print("]");
178 }
179
180 BlockType blockType() const { return controlBlockType; }
181 BlockSignature signature() const { return returnType; }
182
183 BasicBlock* targetBlockForBranch()
184 {
185 if (blockType() == BlockType::Loop)
186 return special;
187 return continuation;
188 }
189
190 void convertIfToBlock()
191 {
192 ASSERT(blockType() == BlockType::If);
193 controlBlockType = BlockType::Block;
194 special = nullptr;
195 }
196
197 private:
198 friend class AirIRGenerator;
199 BlockType controlBlockType;
200 BasicBlock* continuation;
201 BasicBlock* special;
202 TypedTmps results;
203 BlockSignature returnType;
204 };
205
206 using ExpressionType = TypedTmp;
207 using ControlType = ControlData;
208 using ExpressionList = Vector<ExpressionType, 1>;
209 using Stack = ExpressionList;
210 using ResultList = TypedTmps;
211 using ControlEntry = FunctionParser<AirIRGenerator>::ControlEntry;
212
213 static ExpressionType emptyExpression() { return { }; };
214 Stack createStack() { return Stack(); }
215 bool isControlTypeIf(const ControlType& control) { return control.blockType() == BlockType::If; }
216
217 using ErrorType = String;
218 using UnexpectedResult = Unexpected<ErrorType>;
219 using Result = Expected<std::unique_ptr<InternalFunction>, ErrorType>;
220 using PartialResult = Expected<void, ErrorType>;
221
222 template <typename ...Args>
223 NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
224 {
225 using namespace FailureHelper; // See ADL comment in WasmParser.h.
226 return UnexpectedResult(makeString("WebAssembly.Module failed compiling: "_s, makeString(args)...));
227 }
228
229#define WASM_COMPILE_FAIL_IF(condition, ...) do { \
230 if (UNLIKELY(condition)) \
231 return fail(__VA_ARGS__); \
232 } while (0)
233
234 AirIRGenerator(const ModuleInformation&, B3::Procedure&, InternalFunction*, Vector<UnlinkedWasmToWasmCall>&, MemoryMode, unsigned functionIndex, TierUpCount*, ThrowWasmException, const Signature&);
235
236 PartialResult WARN_UNUSED_RETURN addArguments(const Signature&);
237 PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
238 ExpressionType addConstant(Type, uint64_t);
239 ExpressionType addConstant(BasicBlock*, Type, uint64_t);
240 ExpressionType addBottom(BasicBlock*, Type);
241
242 // References
243 PartialResult WARN_UNUSED_RETURN addRefIsNull(ExpressionType& value, ExpressionType& result);
244 PartialResult WARN_UNUSED_RETURN addRefFunc(uint32_t index, ExpressionType& result);
245
246 // Tables
247 PartialResult WARN_UNUSED_RETURN addTableGet(unsigned, ExpressionType& index, ExpressionType& result);
248 PartialResult WARN_UNUSED_RETURN addTableSet(unsigned, ExpressionType& index, ExpressionType& value);
249 PartialResult WARN_UNUSED_RETURN addTableSize(unsigned, ExpressionType& result);
250 PartialResult WARN_UNUSED_RETURN addTableGrow(unsigned, ExpressionType& fill, ExpressionType& delta, ExpressionType& result);
251 PartialResult WARN_UNUSED_RETURN addTableFill(unsigned, ExpressionType& offset, ExpressionType& fill, ExpressionType& count);
252
253 // Locals
254 PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
255 PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
256
257 // Globals
258 PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
259 PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
260
261 // Memory
262 PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
263 PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
264 PartialResult WARN_UNUSED_RETURN addGrowMemory(ExpressionType delta, ExpressionType& result);
265 PartialResult WARN_UNUSED_RETURN addCurrentMemory(ExpressionType& result);
266
267 // Basic operators
268 template<OpType>
269 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
270 template<OpType>
271 PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
272 PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
273
274 // Control flow
275 ControlData WARN_UNUSED_RETURN addTopLevel(BlockSignature);
276 PartialResult WARN_UNUSED_RETURN addBlock(BlockSignature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack);
277 PartialResult WARN_UNUSED_RETURN addLoop(BlockSignature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex);
278 PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, BlockSignature, Stack& enclosingStack, ControlType& result, Stack& newStack);
279 PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const Stack&);
280 PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
281
282 PartialResult WARN_UNUSED_RETURN addReturn(const ControlData&, const ExpressionList& returnValues);
283 PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const Stack& returnValues);
284 PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const Stack& expressionStack);
285 PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, Stack& expressionStack);
286 PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&, const Stack& expressionStack = { });
287
288 // Calls
289 PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature&, Vector<ExpressionType>& args, Vector<ExpressionType, 1>& results);
290 PartialResult WARN_UNUSED_RETURN addCallIndirect(unsigned tableIndex, const Signature&, Vector<ExpressionType>& args, Vector<ExpressionType, 1>& results);
291 PartialResult WARN_UNUSED_RETURN addUnreachable();
292 B3::PatchpointValue* WARN_UNUSED_RETURN emitCallPatchpoint(BasicBlock*, const Signature&, const Vector<ExpressionType, 1>& results, const Vector<TypedTmp>& args, Vector<ConstrainedTmp>&& extraArgs = { });
293
294 PartialResult addShift(Type, B3::Air::Opcode, ExpressionType value, ExpressionType shift, ExpressionType& result);
295 PartialResult addIntegerSub(B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
296 PartialResult addFloatingPointAbs(B3::Air::Opcode, ExpressionType value, ExpressionType& result);
297 PartialResult addFloatingPointBinOp(Type, B3::Air::Opcode, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
298
299 void dump(const Vector<ControlEntry>& controlStack, const Stack* expressionStack);
300 void setParser(FunctionParser<AirIRGenerator>* parser) { m_parser = parser; };
301 void didFinishParsingLocals() { }
302
303 static Vector<Tmp> toTmpVector(const Vector<TypedTmp>& vector)
304 {
305 Vector<Tmp> result;
306 for (const auto& item : vector)
307 result.append(item.tmp());
308 return result;
309 }
310
311 const Bag<B3::PatchpointValue*>& patchpoints() const
312 {
313 return m_patchpoints;
314 }
315
316private:
317 B3::Type toB3ResultType(BlockSignature returnType);
318 ALWAYS_INLINE void validateInst(Inst& inst)
319 {
320 if (!ASSERT_DISABLED) {
321 if (!inst.isValidForm()) {
322 dataLogLn("Inst validation failed:");
323 dataLogLn(inst, "\n");
324 if (inst.origin)
325 dataLogLn(deepDump(inst.origin), "\n");
326 CRASH();
327 }
328 }
329 }
330
331 static Arg extractArg(const TypedTmp& tmp) { return tmp.tmp(); }
332 static Arg extractArg(const Tmp& tmp) { return Arg(tmp); }
333 static Arg extractArg(const Arg& arg) { return arg; }
334
335 template<typename... Arguments>
336 void append(BasicBlock* block, Kind kind, Arguments&&... arguments)
337 {
338 // FIXME: Find a way to use origin here.
339 auto& inst = block->append(kind, nullptr, extractArg(arguments)...);
340 validateInst(inst);
341 }
342
343 template<typename... Arguments>
344 void append(Kind kind, Arguments&&... arguments)
345 {
346 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
347 }
348
349 template<typename... Arguments>
350 void appendEffectful(B3::Air::Opcode op, Arguments&&... arguments)
351 {
352 Kind kind = op;
353 kind.effects = true;
354 append(m_currentBlock, kind, std::forward<Arguments>(arguments)...);
355 }
356
357 Tmp newTmp(B3::Bank bank)
358 {
359 switch (bank) {
360 case B3::GP:
361 if (m_freeGPs.size())
362 return m_freeGPs.takeLast();
363 break;
364 case B3::FP:
365 if (m_freeFPs.size())
366 return m_freeFPs.takeLast();
367 break;
368 }
369 return m_code.newTmp(bank);
370 }
371
372 TypedTmp g32() { return { newTmp(B3::GP), Type::I32 }; }
373 TypedTmp g64() { return { newTmp(B3::GP), Type::I64 }; }
374 TypedTmp gAnyref() { return { newTmp(B3::GP), Type::Anyref }; }
375 TypedTmp gFuncref() { return { newTmp(B3::GP), Type::Funcref }; }
376 TypedTmp f32() { return { newTmp(B3::FP), Type::F32 }; }
377 TypedTmp f64() { return { newTmp(B3::FP), Type::F64 }; }
378
379 TypedTmp tmpForType(Type type)
380 {
381 switch (type) {
382 case Type::I32:
383 return g32();
384 case Type::I64:
385 return g64();
386 case Type::Funcref:
387 return gFuncref();
388 case Type::Anyref:
389 return gAnyref();
390 case Type::F32:
391 return f32();
392 case Type::F64:
393 return f64();
394 case Type::Void:
395 return { };
396 default:
397 RELEASE_ASSERT_NOT_REACHED();
398 }
399 }
400
401 TypedTmps tmpsForSignature(BlockSignature signature)
402 {
403 TypedTmps result(signature->returnCount());
404 for (unsigned i = 0; i < signature->returnCount(); ++i)
405 result[i] = tmpForType(signature->returnType(i));
406 return result;
407 }
408
409 B3::PatchpointValue* addPatchpoint(B3::Type type)
410 {
411 auto* result = m_proc.add<B3::PatchpointValue>(type, B3::Origin());
412 if (UNLIKELY(shouldDumpIRAtEachPhase(B3::AirMode)))
413 m_patchpoints.add(result);
414 return result;
415 }
416
417 template <typename ...Args>
418 void emitPatchpoint(B3::PatchpointValue* patch, Tmp result, Args... theArgs)
419 {
420 emitPatchpoint(m_currentBlock, patch, result, std::forward<Args>(theArgs)...);
421 }
422
423 template <typename ...Args>
424 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result, Args... theArgs)
425 {
426 emitPatchpoint(basicBlock, patch, Vector<Tmp, 1> { result }, Vector<ConstrainedTmp, sizeof...(Args)>::from(theArgs...));
427 }
428
429 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, Tmp result)
430 {
431 emitPatchpoint(basicBlock, patch, Vector<Tmp, 1> { result }, Vector<ConstrainedTmp>());
432 }
433
434 template <typename ResultTmpType, size_t inlineSize>
435 void emitPatchpoint(BasicBlock* basicBlock, B3::PatchpointValue* patch, const Vector<ResultTmpType, 1>& results, Vector<ConstrainedTmp, inlineSize>&& args)
436 {
437 if (!m_patchpointSpecial)
438 m_patchpointSpecial = static_cast<B3::PatchpointSpecial*>(m_code.addSpecial(makeUnique<B3::PatchpointSpecial>()));
439
440 auto toTmp = [&] (ResultTmpType tmp) {
441 if constexpr (std::is_same_v<ResultTmpType, Tmp>)
442 return tmp;
443 else
444 return tmp.tmp();
445 };
446
447 Inst inst(Patch, patch, Arg::special(m_patchpointSpecial));
448 Vector<Inst, 1> resultMovs;
449 switch (patch->type().kind()) {
450 case B3::Void:
451 break;
452 default: {
453 ASSERT(results.size());
454 for (unsigned i = 0; i < results.size(); ++i) {
455 switch (patch->resultConstraints[i].kind()) {
456 case B3::ValueRep::StackArgument: {
457 Arg arg = Arg::callArg(patch->resultConstraints[i].offsetFromSP());
458 inst.args.append(arg);
459 resultMovs.append(Inst(B3::Air::moveForType(m_proc.typeAtOffset(patch->type(), i)), nullptr, arg, toTmp(results[i])));
460 break;
461 }
462 case B3::ValueRep::Register: {
463 inst.args.append(Tmp(patch->resultConstraints[i].reg()));
464 resultMovs.append(Inst(B3::Air::relaxedMoveForType(m_proc.typeAtOffset(patch->type(), i)), nullptr, Tmp(patch->resultConstraints[i].reg()), toTmp(results[i])));
465 break;
466 }
467 case B3::ValueRep::SomeRegister: {
468 inst.args.append(toTmp(results[i]));
469 break;
470 }
471 default:
472 RELEASE_ASSERT_NOT_REACHED();
473 }
474 }
475 }
476 }
477
478 for (unsigned i = 0; i < args.size(); ++i) {
479 ConstrainedTmp& tmp = args[i];
480 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
481 // validation. We should abstrcat Patch enough so ValueRep's don't need to be
482 // backed by Values.
483 // https://bugs.webkit.org/show_bug.cgi?id=194040
484 B3::Value* dummyValue = m_proc.addConstant(B3::Origin(), tmp.tmp.isGP() ? B3::Int64 : B3::Double, 0);
485 patch->append(dummyValue, tmp.rep);
486 switch (tmp.rep.kind()) {
487 case B3::ValueRep::ColdAny: // B3::Value propagates ColdAny information and later Air will allocate appropriate stack.
488 case B3::ValueRep::SomeRegister:
489 inst.args.append(tmp.tmp);
490 break;
491 case B3::ValueRep::Register:
492 patch->earlyClobbered().clear(tmp.rep.reg());
493 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, tmp.rep.reg());
494 inst.args.append(Tmp(tmp.rep.reg()));
495 break;
496 case B3::ValueRep::StackArgument: {
497 ASSERT(!patch->effects.terminal);
498 Arg arg = Arg::callArg(tmp.rep.offsetFromSP());
499 append(basicBlock, tmp.tmp.isGP() ? Move : MoveDouble, tmp.tmp, arg);
500 ASSERT(arg.canRepresent(patch->child(i)->type()));
501 inst.args.append(arg);
502 break;
503 }
504 default:
505 RELEASE_ASSERT_NOT_REACHED();
506 }
507 }
508
509 for (auto valueRep : patch->resultConstraints) {
510 if (valueRep.isReg())
511 patch->lateClobbered().clear(valueRep.reg());
512 }
513 for (unsigned i = patch->numGPScratchRegisters; i--;)
514 inst.args.append(g64().tmp());
515 for (unsigned i = patch->numFPScratchRegisters; i--;)
516 inst.args.append(f64().tmp());
517
518 validateInst(inst);
519 basicBlock->append(WTFMove(inst));
520 for (Inst result : resultMovs) {
521 validateInst(result);
522 basicBlock->append(WTFMove(result));
523 }
524 }
525
526 template <typename Branch, typename Generator>
527 void emitCheck(const Branch& makeBranch, const Generator& generator)
528 {
529 // We fail along the truthy edge of 'branch'.
530 Inst branch = makeBranch();
531
532 // FIXME: Make a hashmap of these.
533 B3::CheckSpecial::Key key(branch);
534 B3::CheckSpecial* special = static_cast<B3::CheckSpecial*>(m_code.addSpecial(makeUnique<B3::CheckSpecial>(key)));
535
536 // FIXME: Remove the need for dummy values
537 // https://bugs.webkit.org/show_bug.cgi?id=194040
538 B3::Value* dummyPredicate = m_proc.addConstant(B3::Origin(), B3::Int32, 42);
539 B3::CheckValue* checkValue = m_proc.add<B3::CheckValue>(B3::Check, B3::Origin(), dummyPredicate);
540 checkValue->setGenerator(generator);
541
542 Inst inst(Patch, checkValue, Arg::special(special));
543 inst.args.appendVector(branch.args);
544 m_currentBlock->append(WTFMove(inst));
545 }
546
547 template <typename Func, typename ...Args>
548 void emitCCall(Func func, TypedTmp result, Args... args)
549 {
550 emitCCall(m_currentBlock, func, result, std::forward<Args>(args)...);
551 }
552 template <typename Func, typename ...Args>
553 void emitCCall(BasicBlock* block, Func func, TypedTmp result, Args... theArgs)
554 {
555 B3::Type resultType = B3::Void;
556 if (result) {
557 switch (result.type()) {
558 case Type::I32:
559 resultType = B3::Int32;
560 break;
561 case Type::I64:
562 case Type::Anyref:
563 case Type::Funcref:
564 resultType = B3::Int64;
565 break;
566 case Type::F32:
567 resultType = B3::Float;
568 break;
569 case Type::F64:
570 resultType = B3::Double;
571 break;
572 default:
573 RELEASE_ASSERT_NOT_REACHED();
574 }
575 }
576
577 auto makeDummyValue = [&] (Tmp tmp) {
578 // FIXME: This is less than ideal to create dummy values just to satisfy Air's
579 // validation. We should abstrcat CCall enough so we're not reliant on arguments
580 // to the B3::CCallValue.
581 // https://bugs.webkit.org/show_bug.cgi?id=194040
582 if (tmp.isGP())
583 return m_proc.addConstant(B3::Origin(), B3::Int64, 0);
584 return m_proc.addConstant(B3::Origin(), B3::Double, 0);
585 };
586
587 B3::Value* dummyFunc = m_proc.addConstant(B3::Origin(), B3::Int64, bitwise_cast<uintptr_t>(func));
588 B3::Value* origin = m_proc.add<B3::CCallValue>(resultType, B3::Origin(), B3::Effects::none(), dummyFunc, makeDummyValue(theArgs)...);
589
590 Inst inst(CCall, origin);
591
592 Tmp callee = g64();
593 append(block, Move, Arg::immPtr(tagCFunctionPtr<void*>(func, B3CCallPtrTag)), callee);
594 inst.args.append(callee);
595
596 if (result)
597 inst.args.append(result.tmp());
598
599 for (Tmp tmp : Vector<Tmp, sizeof...(Args)>::from(theArgs.tmp()...))
600 inst.args.append(tmp);
601
602 block->append(WTFMove(inst));
603 }
604
605 static B3::Air::Opcode moveOpForValueType(Type type)
606 {
607 switch (type) {
608 case Type::I32:
609 return Move32;
610 case Type::I64:
611 case Type::Anyref:
612 case Type::Funcref:
613 return Move;
614 case Type::F32:
615 return MoveFloat;
616 case Type::F64:
617 return MoveDouble;
618 default:
619 RELEASE_ASSERT_NOT_REACHED();
620 }
621 }
622
623 void emitThrowException(CCallHelpers&, ExceptionType);
624
625 void emitEntryTierUpCheck();
626 void emitLoopTierUpCheck(uint32_t loopIndex);
627
628 void emitWriteBarrierForJSWrapper();
629 ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
630 ExpressionType emitLoadOp(LoadOpType, ExpressionType pointer, uint32_t offset);
631 void emitStoreOp(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
632
633 void unify(const ExpressionType& dst, const ExpressionType& source);
634 void unifyValuesWithBlock(const Stack& resultStack, const ResultList& stack);
635
636 template <typename IntType>
637 void emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right);
638
639 template <typename IntType>
640 void emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
641
642 enum class MinOrMax { Min, Max };
643
644 PartialResult addFloatingPointMinOrMax(Type, MinOrMax, ExpressionType lhs, ExpressionType rhs, ExpressionType& result);
645
646 int32_t WARN_UNUSED_RETURN fixupPointerPlusOffset(ExpressionType&, uint32_t);
647
648 void restoreWasmContextInstance(BasicBlock*, TypedTmp);
649 enum class RestoreCachedStackLimit { No, Yes };
650 void restoreWebAssemblyGlobalState(RestoreCachedStackLimit, const MemoryInformation&, TypedTmp instance, BasicBlock*);
651
652 B3::Origin origin();
653
654 uint32_t outerLoopIndex() const
655 {
656 if (m_outerLoops.isEmpty())
657 return UINT32_MAX;
658 return m_outerLoops.last();
659 }
660
661 FunctionParser<AirIRGenerator>* m_parser { nullptr };
662 const ModuleInformation& m_info;
663 const MemoryMode m_mode { MemoryMode::BoundsChecking };
664 const unsigned m_functionIndex { UINT_MAX };
665 TierUpCount* m_tierUp { nullptr };
666
667 B3::Procedure& m_proc;
668 Code& m_code;
669 Vector<uint32_t> m_outerLoops;
670 BasicBlock* m_currentBlock { nullptr };
671 BasicBlock* m_rootBlock { nullptr };
672 Vector<TypedTmp> m_locals;
673 Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
674 GPRReg m_memoryBaseGPR { InvalidGPRReg };
675 GPRReg m_memorySizeGPR { InvalidGPRReg };
676 GPRReg m_wasmContextInstanceGPR { InvalidGPRReg };
677 bool m_makesCalls { false };
678
679 Vector<Tmp, 8> m_freeGPs;
680 Vector<Tmp, 8> m_freeFPs;
681
682 HashMap<BlockSignature, B3::Type> m_tupleMap;
683 // This is only filled if we are dumping IR.
684 Bag<B3::PatchpointValue*> m_patchpoints;
685
686 TypedTmp m_instanceValue; // Always use the accessor below to ensure the instance value is materialized when used.
687 bool m_usesInstanceValue { false };
688 TypedTmp instanceValue()
689 {
690 m_usesInstanceValue = true;
691 return m_instanceValue;
692 }
693
694 uint32_t m_maxNumJSCallArguments { 0 };
695 unsigned m_numImportFunctions;
696
697 B3::PatchpointSpecial* m_patchpointSpecial { nullptr };
698};
699
700// Memory accesses in WebAssembly have unsigned 32-bit offsets, whereas they have signed 32-bit offsets in B3.
701int32_t AirIRGenerator::fixupPointerPlusOffset(ExpressionType& ptr, uint32_t offset)
702{
703 if (static_cast<uint64_t>(offset) > static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
704 auto previousPtr = ptr;
705 ptr = g64();
706 auto constant = g64();
707 append(Move, Arg::bigImm(offset), constant);
708 append(Add64, constant, previousPtr, ptr);
709 return 0;
710 }
711 return offset;
712}
713
714void AirIRGenerator::restoreWasmContextInstance(BasicBlock* block, TypedTmp instance)
715{
716 if (Context::useFastTLS()) {
717 auto* patchpoint = addPatchpoint(B3::Void);
718 if (CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister())
719 patchpoint->clobber(RegisterSet::macroScratchRegisters());
720 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
721 AllowMacroScratchRegisterUsageIf allowScratch(jit, CCallHelpers::storeWasmContextInstanceNeedsMacroScratchRegister());
722 jit.storeWasmContextInstance(params[0].gpr());
723 });
724 emitPatchpoint(block, patchpoint, Tmp(), instance);
725 return;
726 }
727
728 // FIXME: Because WasmToWasm call clobbers wasmContextInstance register and does not restore it, we need to restore it in the caller side.
729 // This prevents us from using ArgumentReg to this (logically) immutable pinned register.
730 auto* patchpoint = addPatchpoint(B3::Void);
731 B3::Effects effects = B3::Effects::none();
732 effects.writesPinned = true;
733 effects.reads = B3::HeapRange::top();
734 patchpoint->effects = effects;
735 patchpoint->clobberLate(RegisterSet(m_wasmContextInstanceGPR));
736 GPRReg wasmContextInstanceGPR = m_wasmContextInstanceGPR;
737 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& param) {
738 jit.move(param[0].gpr(), wasmContextInstanceGPR);
739 });
740 emitPatchpoint(block, patchpoint, Tmp(), instance);
741}
742
743AirIRGenerator::AirIRGenerator(const ModuleInformation& info, B3::Procedure& procedure, InternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, MemoryMode mode, unsigned functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException, const Signature& signature)
744 : m_info(info)
745 , m_mode(mode)
746 , m_functionIndex(functionIndex)
747 , m_tierUp(tierUp)
748 , m_proc(procedure)
749 , m_code(m_proc.code())
750 , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
751 , m_numImportFunctions(info.importFunctionCount())
752{
753 m_currentBlock = m_code.addBlock();
754 m_rootBlock = m_currentBlock;
755
756 // FIXME we don't really need to pin registers here if there's no memory. It makes wasm -> wasm thunks simpler for now. https://bugs.webkit.org/show_bug.cgi?id=166623
757 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
758
759 m_memoryBaseGPR = pinnedRegs.baseMemoryPointer;
760 m_code.pinRegister(m_memoryBaseGPR);
761
762 m_wasmContextInstanceGPR = pinnedRegs.wasmContextInstancePointer;
763 if (!Context::useFastTLS())
764 m_code.pinRegister(m_wasmContextInstanceGPR);
765
766 if (mode != MemoryMode::Signaling) {
767 m_memorySizeGPR = pinnedRegs.sizeRegister;
768 m_code.pinRegister(m_memorySizeGPR);
769 }
770
771 if (throwWasmException)
772 Thunks::singleton().setThrowWasmException(throwWasmException);
773
774 if (info.memory) {
775 switch (m_mode) {
776 case MemoryMode::BoundsChecking:
777 break;
778 case MemoryMode::Signaling:
779 // Most memory accesses in signaling mode don't do an explicit
780 // exception check because they can rely on fault handling to detect
781 // out-of-bounds accesses. FaultSignalHandler nonetheless needs the
782 // thunk to exist so that it can jump to that thunk.
783 if (UNLIKELY(!Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator)))
784 CRASH();
785 break;
786 }
787 }
788
789 m_code.setNumEntrypoints(1);
790
791 GPRReg contextInstance = Context::useFastTLS() ? wasmCallingConvention().prologueScratchGPRs[1] : m_wasmContextInstanceGPR;
792
793 Ref<B3::Air::PrologueGenerator> prologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>([=] (CCallHelpers& jit, B3::Air::Code& code) {
794 AllowMacroScratchRegisterUsage allowScratch(jit);
795 code.emitDefaultPrologue(jit);
796
797 {
798 GPRReg calleeGPR = wasmCallingConvention().prologueScratchGPRs[0];
799 auto moveLocation = jit.moveWithPatch(MacroAssembler::TrustedImmPtr(nullptr), calleeGPR);
800 jit.addLinkTask([compilation, moveLocation] (LinkBuffer& linkBuffer) {
801 compilation->calleeMoveLocation = linkBuffer.locationOf<WasmEntryPtrTag>(moveLocation);
802 });
803 jit.emitPutToCallFrameHeader(calleeGPR, CallFrameSlot::callee);
804 jit.emitPutToCallFrameHeader(nullptr, CallFrameSlot::codeBlock);
805 }
806
807 {
808 const Checked<int32_t> wasmFrameSize = m_code.frameSize();
809 const unsigned minimumParentCheckSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), 1024);
810 const unsigned extraFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), std::max<uint32_t>(
811 // This allows us to elide stack checks for functions that are terminal nodes in the call
812 // tree, (e.g they don't make any calls) and have a small enough frame size. This works by
813 // having any such terminal node have its parent caller include some extra size in its
814 // own check for it. The goal here is twofold:
815 // 1. Emit less code.
816 // 2. Try to speed things up by skipping stack checks.
817 minimumParentCheckSize,
818 // This allows us to elide stack checks in the Wasm -> Embedder call IC stub. Since these will
819 // spill all arguments to the stack, we ensure that a stack check here covers the
820 // stack that such a stub would use.
821 (Checked<uint32_t>(m_maxNumJSCallArguments) * sizeof(Register) + jsCallingConvention().headerSizeInBytes).unsafeGet()
822 ));
823 const int32_t checkSize = m_makesCalls ? (wasmFrameSize + extraFrameSize).unsafeGet() : wasmFrameSize.unsafeGet();
824 bool needUnderflowCheck = static_cast<unsigned>(checkSize) > Options::reservedZoneSize();
825 bool needsOverflowCheck = m_makesCalls || wasmFrameSize >= minimumParentCheckSize || needUnderflowCheck;
826
827 // This allows leaf functions to not do stack checks if their frame size is within
828 // certain limits since their caller would have already done the check.
829 if (needsOverflowCheck) {
830 GPRReg scratch = wasmCallingConvention().prologueScratchGPRs[0];
831
832 if (Context::useFastTLS())
833 jit.loadWasmContextInstance(contextInstance);
834
835 jit.addPtr(CCallHelpers::TrustedImm32(-checkSize), GPRInfo::callFrameRegister, scratch);
836 MacroAssembler::JumpList overflow;
837 if (UNLIKELY(needUnderflowCheck))
838 overflow.append(jit.branchPtr(CCallHelpers::Above, scratch, GPRInfo::callFrameRegister));
839 overflow.append(jit.branchPtr(CCallHelpers::Below, scratch, CCallHelpers::Address(contextInstance, Instance::offsetOfCachedStackLimit())));
840 jit.addLinkTask([overflow] (LinkBuffer& linkBuffer) {
841 linkBuffer.link(overflow, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwStackOverflowFromWasmThunkGenerator).code()));
842 });
843 } else if (m_usesInstanceValue && Context::useFastTLS()) {
844 // No overflow check is needed, but the instance values still needs to be correct.
845 jit.loadWasmContextInstance(contextInstance);
846 }
847 }
848 });
849
850 m_code.setPrologueForEntrypoint(0, WTFMove(prologueGenerator));
851
852 if (Context::useFastTLS()) {
853 m_instanceValue = g64();
854 // FIXME: Would be nice to only do this if we use instance value.
855 append(Move, Tmp(contextInstance), m_instanceValue);
856 } else
857 m_instanceValue = { Tmp(contextInstance), Type::I64 };
858
859 ASSERT(!m_locals.size());
860 m_locals.grow(signature.argumentCount());
861 for (unsigned i = 0; i < signature.argumentCount(); ++i) {
862 Type type = signature.argument(i);
863 m_locals[i] = tmpForType(type);
864 }
865
866 CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(signature, CallRole::Callee);
867
868 for (unsigned i = 0; i < wasmCallInfo.params.size(); ++i) {
869 B3::ValueRep location = wasmCallInfo.params[i];
870 Arg arg = location.isReg() ? Arg(Tmp(location.reg())) : Arg::addr(Tmp(GPRInfo::callFrameRegister), location.offsetFromFP());
871 switch (signature.argument(i)) {
872 case Type::I32:
873 append(Move32, arg, m_locals[i]);
874 break;
875 case Type::I64:
876 case Type::Anyref:
877 case Type::Funcref:
878 append(Move, arg, m_locals[i]);
879 break;
880 case Type::F32:
881 append(MoveFloat, arg, m_locals[i]);
882 break;
883 case Type::F64:
884 append(MoveDouble, arg, m_locals[i]);
885 break;
886 default:
887 RELEASE_ASSERT_NOT_REACHED();
888 }
889 }
890
891 emitEntryTierUpCheck();
892}
893
894B3::Type AirIRGenerator::toB3ResultType(BlockSignature returnType)
895{
896 if (returnType->returnsVoid())
897 return B3::Void;
898
899 if (returnType->returnCount() == 1)
900 return toB3Type(returnType->returnType(0));
901
902 auto result = m_tupleMap.ensure(returnType, [&] {
903 Vector<B3::Type> result;
904 for (unsigned i = 0; i < returnType->returnCount(); ++i)
905 result.append(toB3Type(returnType->returnType(i)));
906 return m_proc.addTuple(WTFMove(result));
907 });
908 return result.iterator->value;
909}
910
911void AirIRGenerator::restoreWebAssemblyGlobalState(RestoreCachedStackLimit restoreCachedStackLimit, const MemoryInformation& memory, TypedTmp instance, BasicBlock* block)
912{
913 restoreWasmContextInstance(block, instance);
914
915 if (restoreCachedStackLimit == RestoreCachedStackLimit::Yes) {
916 // The Instance caches the stack limit, but also knows where its canonical location is.
917 static_assert(sizeof(decltype(static_cast<Instance*>(nullptr)->cachedStackLimit())) == sizeof(uint64_t), "");
918
919 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfPointerToActualStackLimit(), B3::Width64));
920 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedStackLimit(), B3::Width64));
921 auto temp = g64();
922 append(block, Move, Arg::addr(instanceValue(), Instance::offsetOfPointerToActualStackLimit()), temp);
923 append(block, Move, Arg::addr(temp), temp);
924 append(block, Move, temp, Arg::addr(instanceValue(), Instance::offsetOfCachedStackLimit()));
925 }
926
927 if (!!memory) {
928 const PinnedRegisterInfo* pinnedRegs = &PinnedRegisterInfo::get();
929 RegisterSet clobbers;
930 clobbers.set(pinnedRegs->baseMemoryPointer);
931 clobbers.set(pinnedRegs->sizeRegister);
932 if (!isARM64())
933 clobbers.set(RegisterSet::macroScratchRegisters());
934
935 auto* patchpoint = addPatchpoint(B3::Void);
936 B3::Effects effects = B3::Effects::none();
937 effects.writesPinned = true;
938 effects.reads = B3::HeapRange::top();
939 patchpoint->effects = effects;
940 patchpoint->clobber(clobbers);
941 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
942
943 patchpoint->setGenerator([pinnedRegs] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
944 AllowMacroScratchRegisterUsage allowScratch(jit);
945 GPRReg baseMemory = pinnedRegs->baseMemoryPointer;
946 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs->sizeRegister;
947
948 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemorySize()), pinnedRegs->sizeRegister);
949 jit.loadPtr(CCallHelpers::Address(params[0].gpr(), Instance::offsetOfCachedMemory()), baseMemory);
950
951 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs->sizeRegister, scratchOrSize);
952 });
953
954 emitPatchpoint(block, patchpoint, Tmp(), instance);
955 }
956}
957
958void AirIRGenerator::emitThrowException(CCallHelpers& jit, ExceptionType type)
959{
960 jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
961 auto jumpToExceptionStub = jit.jump();
962
963 jit.addLinkTask([jumpToExceptionStub] (LinkBuffer& linkBuffer) {
964 linkBuffer.link(jumpToExceptionStub, CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(throwExceptionFromWasmThunkGenerator).code()));
965 });
966}
967
968auto AirIRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
969{
970 size_t newSize = m_locals.size() + count;
971 ASSERT(!(CheckedUint32(count) + m_locals.size()).hasOverflowed());
972 ASSERT(newSize <= maxFunctionLocals);
973 WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(newSize), "can't allocate memory for ", newSize, " locals");
974
975 for (uint32_t i = 0; i < count; ++i) {
976 auto local = tmpForType(type);
977 m_locals.uncheckedAppend(local);
978 switch (type) {
979 case Type::Anyref:
980 case Type::Funcref:
981 append(Move, Arg::imm(JSValue::encode(jsNull())), local);
982 break;
983 case Type::I32:
984 case Type::I64: {
985 append(Xor64, local, local);
986 break;
987 }
988 case Type::F32:
989 case Type::F64: {
990 auto temp = g64();
991 // IEEE 754 "0" is just int32/64 zero.
992 append(Xor64, temp, temp);
993 append(type == Type::F32 ? Move32ToFloat : Move64ToDouble, temp, local);
994 break;
995 }
996 default:
997 RELEASE_ASSERT_NOT_REACHED();
998 }
999 }
1000 return { };
1001}
1002
1003auto AirIRGenerator::addConstant(Type type, uint64_t value) -> ExpressionType
1004{
1005 return addConstant(m_currentBlock, type, value);
1006}
1007
1008auto AirIRGenerator::addConstant(BasicBlock* block, Type type, uint64_t value) -> ExpressionType
1009{
1010 auto result = tmpForType(type);
1011 switch (type) {
1012 case Type::I32:
1013 case Type::I64:
1014 case Type::Anyref:
1015 case Type::Funcref:
1016 append(block, Move, Arg::bigImm(value), result);
1017 break;
1018 case Type::F32:
1019 case Type::F64: {
1020 auto tmp = g64();
1021 append(block, Move, Arg::bigImm(value), tmp);
1022 append(block, type == Type::F32 ? Move32ToFloat : Move64ToDouble, tmp, result);
1023 break;
1024 }
1025
1026 default:
1027 RELEASE_ASSERT_NOT_REACHED();
1028 }
1029
1030 return result;
1031}
1032
1033auto AirIRGenerator::addBottom(BasicBlock* block, Type type) -> ExpressionType
1034{
1035 append(block, B3::Air::Oops);
1036 return addConstant(type, 0);
1037}
1038
1039auto AirIRGenerator::addArguments(const Signature& signature) -> PartialResult
1040{
1041 RELEASE_ASSERT(m_locals.size() == signature.argumentCount()); // We handle arguments in the prologue
1042 return { };
1043}
1044
1045auto AirIRGenerator::addRefIsNull(ExpressionType& value, ExpressionType& result) -> PartialResult
1046{
1047 ASSERT(value.tmp());
1048 result = tmpForType(Type::I32);
1049 auto tmp = g64();
1050
1051 append(Move, Arg::bigImm(JSValue::encode(jsNull())), tmp);
1052 append(Compare64, Arg::relCond(MacroAssembler::Equal), value, tmp, result);
1053
1054 return { };
1055}
1056
1057auto AirIRGenerator::addRefFunc(uint32_t index, ExpressionType& result) -> PartialResult
1058{
1059 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
1060 result = tmpForType(Type::Funcref);
1061 emitCCall(&operationWasmRefFunc, result, instanceValue(), addConstant(Type::I32, index));
1062
1063 return { };
1064}
1065
1066auto AirIRGenerator::addTableGet(unsigned tableIndex, ExpressionType& index, ExpressionType& result) -> PartialResult
1067{
1068 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
1069 ASSERT(index.tmp());
1070 ASSERT(index.type() == Type::I32);
1071 result = tmpForType(m_info.tables[tableIndex].wasmType());
1072
1073 emitCCall(&operationGetWasmTableElement, result, instanceValue(), addConstant(Type::I32, tableIndex), index);
1074 emitCheck([&] {
1075 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::Zero), result, result);
1076 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1077 this->emitThrowException(jit, ExceptionType::OutOfBoundsTableAccess);
1078 });
1079
1080 return { };
1081}
1082
1083auto AirIRGenerator::addTableSet(unsigned tableIndex, ExpressionType& index, ExpressionType& value) -> PartialResult
1084{
1085 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
1086 ASSERT(index.tmp());
1087 ASSERT(index.type() == Type::I32);
1088 ASSERT(value.tmp());
1089
1090 auto shouldThrow = g32();
1091 emitCCall(&operationSetWasmTableElement, shouldThrow, instanceValue(), addConstant(Type::I32, tableIndex), index, value);
1092
1093 emitCheck([&] {
1094 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::Zero), shouldThrow, shouldThrow);
1095 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1096 this->emitThrowException(jit, ExceptionType::OutOfBoundsTableAccess);
1097 });
1098
1099 return { };
1100}
1101
1102auto AirIRGenerator::addTableSize(unsigned tableIndex, ExpressionType& result) -> PartialResult
1103{
1104 // FIXME: Emit this inline <https://bugs.webkit.org/show_bug.cgi?id=198506>.
1105 result = tmpForType(Type::I32);
1106
1107 emitCCall(&operationGetWasmTableSize, result, instanceValue(), addConstant(Type::I32, tableIndex));
1108
1109 return { };
1110}
1111
1112auto AirIRGenerator::addTableGrow(unsigned tableIndex, ExpressionType& fill, ExpressionType& delta, ExpressionType& result) -> PartialResult
1113{
1114 ASSERT(fill.tmp());
1115 ASSERT(isSubtype(fill.type(), m_info.tables[tableIndex].wasmType()));
1116 ASSERT(delta.tmp());
1117 ASSERT(delta.type() == Type::I32);
1118 result = tmpForType(Type::I32);
1119
1120 emitCCall(&operationWasmTableGrow, result, instanceValue(), addConstant(Type::I32, tableIndex), fill, delta);
1121
1122 return { };
1123}
1124
1125auto AirIRGenerator::addTableFill(unsigned tableIndex, ExpressionType& offset, ExpressionType& fill, ExpressionType& count) -> PartialResult
1126{
1127 ASSERT(fill.tmp());
1128 ASSERT(isSubtype(fill.type(), m_info.tables[tableIndex].wasmType()));
1129 ASSERT(offset.tmp());
1130 ASSERT(offset.type() == Type::I32);
1131 ASSERT(count.tmp());
1132 ASSERT(count.type() == Type::I32);
1133
1134 auto result = tmpForType(Type::I32);
1135 emitCCall(&operationWasmTableFill, result, instanceValue(), addConstant(Type::I32, tableIndex), offset, fill, count);
1136
1137 emitCheck([&] {
1138 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::Zero), result, result);
1139 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1140 this->emitThrowException(jit, ExceptionType::OutOfBoundsTableAccess);
1141 });
1142
1143 return { };
1144}
1145
1146auto AirIRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
1147{
1148 ASSERT(m_locals[index].tmp());
1149 result = tmpForType(m_locals[index].type());
1150 append(moveOpForValueType(m_locals[index].type()), m_locals[index].tmp(), result);
1151 return { };
1152}
1153
1154auto AirIRGenerator::addUnreachable() -> PartialResult
1155{
1156 B3::PatchpointValue* unreachable = addPatchpoint(B3::Void);
1157 unreachable->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1158 this->emitThrowException(jit, ExceptionType::Unreachable);
1159 });
1160 unreachable->effects.terminal = true;
1161 emitPatchpoint(unreachable, Tmp());
1162 return { };
1163}
1164
1165auto AirIRGenerator::addGrowMemory(ExpressionType delta, ExpressionType& result) -> PartialResult
1166{
1167 result = g32();
1168 emitCCall(&operationGrowMemory, result, TypedTmp { Tmp(GPRInfo::callFrameRegister), Type::I64 }, instanceValue(), delta);
1169 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::No, m_info.memory, instanceValue(), m_currentBlock);
1170
1171 return { };
1172}
1173
1174auto AirIRGenerator::addCurrentMemory(ExpressionType& result) -> PartialResult
1175{
1176 static_assert(sizeof(decltype(static_cast<Memory*>(nullptr)->size())) == sizeof(uint64_t), "codegen relies on this size");
1177
1178 auto temp1 = g64();
1179 auto temp2 = g64();
1180
1181 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfCachedMemorySize(), B3::Width64));
1182 append(Move, Arg::addr(instanceValue(), Instance::offsetOfCachedMemorySize()), temp1);
1183 constexpr uint32_t shiftValue = 16;
1184 static_assert(PageCount::pageSize == 1ull << shiftValue, "This must hold for the code below to be correct.");
1185 append(Move, Arg::imm(16), temp2);
1186 addShift(Type::I32, Urshift64, temp1, temp2, result);
1187 append(Move32, result, result);
1188
1189 return { };
1190}
1191
1192auto AirIRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
1193{
1194 ASSERT(m_locals[index].tmp());
1195 append(moveOpForValueType(m_locals[index].type()), value, m_locals[index].tmp());
1196 return { };
1197}
1198
1199auto AirIRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
1200{
1201 Type type = m_info.globals[index].type;
1202
1203 result = tmpForType(type);
1204
1205 auto temp = g64();
1206
1207 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1208 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1209
1210 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1211 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1212 append(moveOpForValueType(type), Arg::addr(temp, offset), result);
1213 else {
1214 auto temp2 = g64();
1215 append(Move, Arg::bigImm(offset), temp2);
1216 append(Add64, temp2, temp, temp);
1217 append(moveOpForValueType(type), Arg::addr(temp), result);
1218 }
1219 return { };
1220}
1221
1222auto AirIRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
1223{
1224 auto temp = g64();
1225
1226 RELEASE_ASSERT(Arg::isValidAddrForm(Instance::offsetOfGlobals(), B3::Width64));
1227 append(Move, Arg::addr(instanceValue(), Instance::offsetOfGlobals()), temp);
1228
1229 Type type = m_info.globals[index].type;
1230
1231 int32_t offset = safeCast<int32_t>(index * sizeof(Register));
1232 if (Arg::isValidAddrForm(offset, B3::widthForType(toB3Type(type))))
1233 append(moveOpForValueType(type), value, Arg::addr(temp, offset));
1234 else {
1235 auto temp2 = g64();
1236 append(Move, Arg::bigImm(offset), temp2);
1237 append(Add64, temp2, temp, temp);
1238 append(moveOpForValueType(type), value, Arg::addr(temp));
1239 }
1240
1241 if (isSubtype(type, Anyref))
1242 emitWriteBarrierForJSWrapper();
1243
1244 return { };
1245}
1246
1247inline void AirIRGenerator::emitWriteBarrierForJSWrapper()
1248{
1249 auto cell = g64();
1250 auto vm = g64();
1251 auto cellState = g32();
1252 auto threshold = g32();
1253
1254 BasicBlock* fenceCheckPath = m_code.addBlock();
1255 BasicBlock* fencePath = m_code.addBlock();
1256 BasicBlock* doSlowPath = m_code.addBlock();
1257 BasicBlock* continuation = m_code.addBlock();
1258
1259 append(Move, Arg::addr(instanceValue(), Instance::offsetOfOwner()), cell);
1260 append(Move, Arg::addr(cell, JSWebAssemblyInstance::offsetOfVM()), vm);
1261 append(Load8, Arg::addr(cell, JSCell::cellStateOffset()), cellState);
1262 append(Move32, Arg::addr(vm, VM::offsetOfHeapBarrierThreshold()), threshold);
1263
1264 append(Branch32, Arg::relCond(MacroAssembler::Above), cellState, threshold);
1265 m_currentBlock->setSuccessors(continuation, fenceCheckPath);
1266 m_currentBlock = fenceCheckPath;
1267
1268 append(Load8, Arg::addr(vm, VM::offsetOfHeapMutatorShouldBeFenced()), threshold);
1269 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), threshold, threshold);
1270 m_currentBlock->setSuccessors(doSlowPath, fencePath);
1271 m_currentBlock = fencePath;
1272
1273 auto* doFence = addPatchpoint(B3::Void);
1274 doFence->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1275 jit.memoryFence();
1276 });
1277 emitPatchpoint(doFence, Tmp());
1278
1279 append(Load8, Arg::addr(cell, JSCell::cellStateOffset()), cellState);
1280 append(Branch32, Arg::relCond(MacroAssembler::Above), cellState, Arg::imm(blackThreshold));
1281 m_currentBlock->setSuccessors(continuation, doSlowPath);
1282 m_currentBlock = doSlowPath;
1283
1284 emitCCall(&operationWasmWriteBarrierSlowPath, TypedTmp(), cell, vm);
1285 append(Jump);
1286 m_currentBlock->setSuccessors(continuation);
1287 m_currentBlock = continuation;
1288}
1289
1290inline AirIRGenerator::ExpressionType AirIRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
1291{
1292 ASSERT(m_memoryBaseGPR);
1293
1294 auto result = g64();
1295 append(Move32, pointer, result);
1296
1297 switch (m_mode) {
1298 case MemoryMode::BoundsChecking: {
1299 // We're not using signal handling at all, we must therefore check that no memory access exceeds the current memory size.
1300 ASSERT(m_memorySizeGPR);
1301 ASSERT(sizeOfOperation + offset > offset);
1302 auto temp = g64();
1303 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1304 append(Add64, result, temp);
1305
1306 emitCheck([&] {
1307 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, Tmp(m_memorySizeGPR));
1308 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1309 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1310 });
1311 break;
1312 }
1313
1314 case MemoryMode::Signaling: {
1315 // We've virtually mapped 4GiB+redzone for this memory. Only the user-allocated pages are addressable, contiguously in range [0, current],
1316 // and everything above is mapped PROT_NONE. We don't need to perform any explicit bounds check in the 4GiB range because WebAssembly register
1317 // memory accesses are 32-bit. However WebAssembly register + offset accesses perform the addition in 64-bit which can push an access above
1318 // the 32-bit limit (the offset is unsigned 32-bit). The redzone will catch most small offsets, and we'll explicitly bounds check any
1319 // register + large offset access. We don't think this will be generated frequently.
1320 //
1321 // We could check that register + large offset doesn't exceed 4GiB+redzone since that's technically the limit we need to avoid overflowing the
1322 // PROT_NONE region, but it's better if we use a smaller immediate because it can codegens better. We know that anything equal to or greater
1323 // than the declared 'maximum' will trap, so we can compare against that number. If there was no declared 'maximum' then we still know that
1324 // any access equal to or greater than 4GiB will trap, no need to add the redzone.
1325 if (offset >= Memory::fastMappedRedzoneBytes()) {
1326 uint64_t maximum = m_info.memory.maximum() ? m_info.memory.maximum().bytes() : std::numeric_limits<uint32_t>::max();
1327 auto temp = g64();
1328 append(Move, Arg::bigImm(static_cast<uint64_t>(sizeOfOperation) + offset - 1), temp);
1329 append(Add64, result, temp);
1330 auto sizeMax = addConstant(Type::I64, maximum);
1331
1332 emitCheck([&] {
1333 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), temp, sizeMax);
1334 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1335 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1336 });
1337 }
1338 break;
1339 }
1340 }
1341
1342 append(Add64, Tmp(m_memoryBaseGPR), result);
1343 return result;
1344}
1345
1346inline uint32_t sizeOfLoadOp(LoadOpType op)
1347{
1348 switch (op) {
1349 case LoadOpType::I32Load8S:
1350 case LoadOpType::I32Load8U:
1351 case LoadOpType::I64Load8S:
1352 case LoadOpType::I64Load8U:
1353 return 1;
1354 case LoadOpType::I32Load16S:
1355 case LoadOpType::I64Load16S:
1356 case LoadOpType::I32Load16U:
1357 case LoadOpType::I64Load16U:
1358 return 2;
1359 case LoadOpType::I32Load:
1360 case LoadOpType::I64Load32S:
1361 case LoadOpType::I64Load32U:
1362 case LoadOpType::F32Load:
1363 return 4;
1364 case LoadOpType::I64Load:
1365 case LoadOpType::F64Load:
1366 return 8;
1367 }
1368 RELEASE_ASSERT_NOT_REACHED();
1369}
1370
1371inline TypedTmp AirIRGenerator::emitLoadOp(LoadOpType op, ExpressionType pointer, uint32_t uoffset)
1372{
1373 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1374
1375 TypedTmp immTmp;
1376 TypedTmp newPtr;
1377 TypedTmp result;
1378
1379 Arg addrArg;
1380 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfLoadOp(op))))
1381 addrArg = Arg::addr(pointer, offset);
1382 else {
1383 immTmp = g64();
1384 newPtr = g64();
1385 append(Move, Arg::bigImm(offset), immTmp);
1386 append(Add64, immTmp, pointer, newPtr);
1387 addrArg = Arg::addr(newPtr);
1388 }
1389
1390 switch (op) {
1391 case LoadOpType::I32Load8S: {
1392 result = g32();
1393 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1394 break;
1395 }
1396
1397 case LoadOpType::I64Load8S: {
1398 result = g64();
1399 appendEffectful(Load8SignedExtendTo32, addrArg, result);
1400 append(SignExtend32ToPtr, result, result);
1401 break;
1402 }
1403
1404 case LoadOpType::I32Load8U: {
1405 result = g32();
1406 appendEffectful(Load8, addrArg, result);
1407 break;
1408 }
1409
1410 case LoadOpType::I64Load8U: {
1411 result = g64();
1412 appendEffectful(Load8, addrArg, result);
1413 break;
1414 }
1415
1416 case LoadOpType::I32Load16S: {
1417 result = g32();
1418 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1419 break;
1420 }
1421
1422 case LoadOpType::I64Load16S: {
1423 result = g64();
1424 appendEffectful(Load16SignedExtendTo32, addrArg, result);
1425 append(SignExtend32ToPtr, result, result);
1426 break;
1427 }
1428
1429 case LoadOpType::I32Load16U: {
1430 result = g32();
1431 appendEffectful(Load16, addrArg, result);
1432 break;
1433 }
1434
1435 case LoadOpType::I64Load16U: {
1436 result = g64();
1437 appendEffectful(Load16, addrArg, result);
1438 break;
1439 }
1440
1441 case LoadOpType::I32Load:
1442 result = g32();
1443 appendEffectful(Move32, addrArg, result);
1444 break;
1445
1446 case LoadOpType::I64Load32U: {
1447 result = g64();
1448 appendEffectful(Move32, addrArg, result);
1449 break;
1450 }
1451
1452 case LoadOpType::I64Load32S: {
1453 result = g64();
1454 appendEffectful(Move32, addrArg, result);
1455 append(SignExtend32ToPtr, result, result);
1456 break;
1457 }
1458
1459 case LoadOpType::I64Load: {
1460 result = g64();
1461 appendEffectful(Move, addrArg, result);
1462 break;
1463 }
1464
1465 case LoadOpType::F32Load: {
1466 result = f32();
1467 appendEffectful(MoveFloat, addrArg, result);
1468 break;
1469 }
1470
1471 case LoadOpType::F64Load: {
1472 result = f64();
1473 appendEffectful(MoveDouble, addrArg, result);
1474 break;
1475 }
1476 }
1477
1478 return result;
1479}
1480
1481auto AirIRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
1482{
1483 ASSERT(pointer.tmp().isGP());
1484
1485 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfLoadOp(op)))) {
1486 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1487 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1488 auto* patch = addPatchpoint(B3::Void);
1489 patch->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1490 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1491 });
1492 emitPatchpoint(patch, Tmp());
1493
1494 // We won't reach here, so we just pick a random reg.
1495 switch (op) {
1496 case LoadOpType::I32Load8S:
1497 case LoadOpType::I32Load16S:
1498 case LoadOpType::I32Load:
1499 case LoadOpType::I32Load16U:
1500 case LoadOpType::I32Load8U:
1501 result = g32();
1502 break;
1503 case LoadOpType::I64Load8S:
1504 case LoadOpType::I64Load8U:
1505 case LoadOpType::I64Load16S:
1506 case LoadOpType::I64Load32U:
1507 case LoadOpType::I64Load32S:
1508 case LoadOpType::I64Load:
1509 case LoadOpType::I64Load16U:
1510 result = g64();
1511 break;
1512 case LoadOpType::F32Load:
1513 result = f32();
1514 break;
1515 case LoadOpType::F64Load:
1516 result = f64();
1517 break;
1518 }
1519 } else
1520 result = emitLoadOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
1521
1522 return { };
1523}
1524
1525inline uint32_t sizeOfStoreOp(StoreOpType op)
1526{
1527 switch (op) {
1528 case StoreOpType::I32Store8:
1529 case StoreOpType::I64Store8:
1530 return 1;
1531 case StoreOpType::I32Store16:
1532 case StoreOpType::I64Store16:
1533 return 2;
1534 case StoreOpType::I32Store:
1535 case StoreOpType::I64Store32:
1536 case StoreOpType::F32Store:
1537 return 4;
1538 case StoreOpType::I64Store:
1539 case StoreOpType::F64Store:
1540 return 8;
1541 }
1542 RELEASE_ASSERT_NOT_REACHED();
1543}
1544
1545
1546inline void AirIRGenerator::emitStoreOp(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t uoffset)
1547{
1548 uint32_t offset = fixupPointerPlusOffset(pointer, uoffset);
1549
1550 TypedTmp immTmp;
1551 TypedTmp newPtr;
1552
1553 Arg addrArg;
1554 if (Arg::isValidAddrForm(offset, B3::widthForBytes(sizeOfStoreOp(op))))
1555 addrArg = Arg::addr(pointer, offset);
1556 else {
1557 immTmp = g64();
1558 newPtr = g64();
1559 append(Move, Arg::bigImm(offset), immTmp);
1560 append(Add64, immTmp, pointer, newPtr);
1561 addrArg = Arg::addr(newPtr);
1562 }
1563
1564 switch (op) {
1565 case StoreOpType::I64Store8:
1566 case StoreOpType::I32Store8:
1567 append(Store8, value, addrArg);
1568 return;
1569
1570 case StoreOpType::I64Store16:
1571 case StoreOpType::I32Store16:
1572 append(Store16, value, addrArg);
1573 return;
1574
1575 case StoreOpType::I64Store32:
1576 case StoreOpType::I32Store:
1577 append(Move32, value, addrArg);
1578 return;
1579
1580 case StoreOpType::I64Store:
1581 append(Move, value, addrArg);
1582 return;
1583
1584 case StoreOpType::F32Store:
1585 append(MoveFloat, value, addrArg);
1586 return;
1587
1588 case StoreOpType::F64Store:
1589 append(MoveDouble, value, addrArg);
1590 return;
1591 }
1592
1593 RELEASE_ASSERT_NOT_REACHED();
1594}
1595
1596auto AirIRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
1597{
1598 ASSERT(pointer.tmp().isGP());
1599
1600 if (UNLIKELY(sumOverflows<uint32_t>(offset, sizeOfStoreOp(op)))) {
1601 // FIXME: Even though this is provably out of bounds, it's not a validation error, so we have to handle it
1602 // as a runtime exception. However, this may change: https://bugs.webkit.org/show_bug.cgi?id=166435
1603 auto* throwException = addPatchpoint(B3::Void);
1604 throwException->setGenerator([this] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
1605 this->emitThrowException(jit, ExceptionType::OutOfBoundsMemoryAccess);
1606 });
1607 emitPatchpoint(throwException, Tmp());
1608 } else
1609 emitStoreOp(op, emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
1610
1611 return { };
1612}
1613
1614auto AirIRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
1615{
1616 ASSERT(nonZero.type() == zero.type());
1617 result = tmpForType(nonZero.type());
1618 append(moveOpForValueType(nonZero.type()), nonZero, result);
1619
1620 BasicBlock* isZero = m_code.addBlock();
1621 BasicBlock* continuation = m_code.addBlock();
1622
1623 append(BranchTest32, Arg::resCond(MacroAssembler::Zero), condition, condition);
1624 m_currentBlock->setSuccessors(isZero, continuation);
1625
1626 append(isZero, moveOpForValueType(zero.type()), zero, result);
1627 append(isZero, Jump);
1628 isZero->setSuccessors(continuation);
1629
1630 m_currentBlock = continuation;
1631
1632 return { };
1633}
1634
1635void AirIRGenerator::emitEntryTierUpCheck()
1636{
1637 if (!m_tierUp)
1638 return;
1639
1640 auto countdownPtr = g64();
1641
1642 append(Move, Arg::bigImm(reinterpret_cast<uint64_t>(&m_tierUp->m_counter)), countdownPtr);
1643
1644 auto* patch = addPatchpoint(B3::Void);
1645 B3::Effects effects = B3::Effects::none();
1646 effects.reads = B3::HeapRange::top();
1647 effects.writes = B3::HeapRange::top();
1648 patch->effects = effects;
1649 patch->clobber(RegisterSet::macroScratchRegisters());
1650
1651 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1652 AllowMacroScratchRegisterUsage allowScratch(jit);
1653
1654 CCallHelpers::Jump tierUp = jit.branchAdd32(CCallHelpers::PositiveOrZero, CCallHelpers::TrustedImm32(TierUpCount::functionEntryIncrement()), CCallHelpers::Address(params[0].gpr()));
1655 CCallHelpers::Label tierUpResume = jit.label();
1656
1657 params.addLatePath([=] (CCallHelpers& jit) {
1658 tierUp.link(&jit);
1659
1660 const unsigned extraPaddingBytes = 0;
1661 RegisterSet registersToSpill = { };
1662 registersToSpill.add(GPRInfo::argumentGPR1);
1663 unsigned numberOfStackBytesUsedForRegisterPreservation = ScratchRegisterAllocator::preserveRegistersToStackForCall(jit, registersToSpill, extraPaddingBytes);
1664
1665 jit.move(MacroAssembler::TrustedImm32(m_functionIndex), GPRInfo::argumentGPR1);
1666 MacroAssembler::Call call = jit.nearCall();
1667
1668 ScratchRegisterAllocator::restoreRegistersFromStackForCall(jit, registersToSpill, RegisterSet(), numberOfStackBytesUsedForRegisterPreservation, extraPaddingBytes);
1669 jit.jump(tierUpResume);
1670
1671 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
1672 MacroAssembler::repatchNearCall(linkBuffer.locationOfNearCall<NoPtrTag>(call), CodeLocationLabel<JITThunkPtrTag>(Thunks::singleton().stub(triggerOMGEntryTierUpThunkGenerator).code()));
1673 });
1674 });
1675 });
1676
1677 emitPatchpoint(patch, Tmp(), countdownPtr);
1678}
1679
1680void AirIRGenerator::emitLoopTierUpCheck(uint32_t loopIndex)
1681{
1682 uint32_t outerLoopIndex = this->outerLoopIndex();
1683 m_outerLoops.append(loopIndex);
1684
1685 if (!m_tierUp)
1686 return;
1687
1688 ASSERT(m_tierUp->osrEntryTriggers().size() == loopIndex);
1689 m_tierUp->osrEntryTriggers().append(TierUpCount::TriggerReason::DontTrigger);
1690 m_tierUp->outerLoops().append(outerLoopIndex);
1691
1692 auto countdownPtr = g64();
1693
1694 append(Move, Arg::bigImm(reinterpret_cast<uint64_t>(&m_tierUp->m_counter)), countdownPtr);
1695
1696 auto* patch = addPatchpoint(B3::Void);
1697 B3::Effects effects = B3::Effects::none();
1698 effects.reads = B3::HeapRange::top();
1699 effects.writes = B3::HeapRange::top();
1700 effects.exitsSideways = true;
1701 patch->effects = effects;
1702
1703 patch->clobber(RegisterSet::macroScratchRegisters());
1704 RegisterSet clobberLate;
1705 clobberLate.add(GPRInfo::argumentGPR0);
1706 patch->clobberLate(clobberLate);
1707
1708 Vector<ConstrainedTmp> patchArgs;
1709 patchArgs.append(countdownPtr);
1710
1711 for (auto& local : m_locals)
1712 patchArgs.append(ConstrainedTmp(local, B3::ValueRep::ColdAny));
1713 for (unsigned controlIndex = 0; controlIndex < m_parser->controlStack().size(); ++controlIndex) {
1714 ExpressionList& expressionStack = m_parser->controlStack()[controlIndex].enclosedExpressionStack;
1715 for (auto& value : expressionStack)
1716 patchArgs.append(ConstrainedTmp(value, B3::ValueRep::ColdAny));
1717 }
1718
1719 TierUpCount::TriggerReason* forceEntryTrigger = &(m_tierUp->osrEntryTriggers().last());
1720 static_assert(!static_cast<uint8_t>(TierUpCount::TriggerReason::DontTrigger), "the JIT code assumes non-zero means 'enter'");
1721 static_assert(sizeof(TierUpCount::TriggerReason) == 1, "branchTest8 assumes this size");
1722 patch->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1723 AllowMacroScratchRegisterUsage allowScratch(jit);
1724 CCallHelpers::Jump forceOSREntry = jit.branchTest8(CCallHelpers::NonZero, CCallHelpers::AbsoluteAddress(forceEntryTrigger));
1725 CCallHelpers::Jump tierUp = jit.branchAdd32(CCallHelpers::PositiveOrZero, CCallHelpers::TrustedImm32(TierUpCount::loopIncrement()), CCallHelpers::Address(params[0].gpr()));
1726 MacroAssembler::Label tierUpResume = jit.label();
1727
1728 OSREntryData& osrEntryData = m_tierUp->addOSREntryData(m_functionIndex, loopIndex);
1729 // First argument is the countdown location.
1730 for (unsigned index = 1; index < params.value()->numChildren(); ++index)
1731 osrEntryData.values().constructAndAppend(params[index], params.value()->child(index)->type());
1732 OSREntryData* osrEntryDataPtr = &osrEntryData;
1733
1734 params.addLatePath([=] (CCallHelpers& jit) {
1735 AllowMacroScratchRegisterUsage allowScratch(jit);
1736 forceOSREntry.link(&jit);
1737 tierUp.link(&jit);
1738
1739 jit.probe(operationWasmTriggerOSREntryNow, osrEntryDataPtr);
1740 jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::argumentGPR0).linkTo(tierUpResume, &jit);
1741 jit.farJump(GPRInfo::argumentGPR1, WasmEntryPtrTag);
1742 });
1743 });
1744
1745 emitPatchpoint(m_currentBlock, patch, Vector<Tmp, 1>(), WTFMove(patchArgs));
1746}
1747
1748AirIRGenerator::ControlData AirIRGenerator::addTopLevel(BlockSignature signature)
1749{
1750 return ControlData(B3::Origin(), signature, tmpsForSignature(signature), BlockType::TopLevel, m_code.addBlock());
1751}
1752
1753auto AirIRGenerator::addLoop(BlockSignature signature, Stack& enclosingStack, ControlType& block, Stack& newStack, uint32_t loopIndex) -> PartialResult
1754{
1755 BasicBlock* body = m_code.addBlock();
1756 BasicBlock* continuation = m_code.addBlock();
1757
1758 newStack = splitStack(signature, enclosingStack);
1759 block = ControlData(origin(), signature, newStack, BlockType::Loop, continuation, body);
1760
1761 append(Jump);
1762 m_currentBlock->setSuccessors(body);
1763
1764 m_currentBlock = body;
1765 emitLoopTierUpCheck(loopIndex);
1766
1767 return { };
1768}
1769
1770auto AirIRGenerator::addBlock(BlockSignature signature, Stack& enclosingStack, ControlType& newBlock, Stack& newStack) -> PartialResult
1771{
1772 newStack = splitStack(signature, enclosingStack);
1773 newBlock = ControlData(origin(), signature, tmpsForSignature(signature), BlockType::Block, m_code.addBlock());
1774 return { };
1775}
1776
1777auto AirIRGenerator::addIf(ExpressionType condition, BlockSignature signature, Stack& enclosingStack, ControlType& result, Stack& newStack) -> PartialResult
1778{
1779 BasicBlock* taken = m_code.addBlock();
1780 BasicBlock* notTaken = m_code.addBlock();
1781 BasicBlock* continuation = m_code.addBlock();
1782
1783 // Wasm bools are i32.
1784 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1785 m_currentBlock->setSuccessors(taken, notTaken);
1786
1787 m_currentBlock = taken;
1788 newStack = splitStack(signature, enclosingStack);
1789 result = ControlData(origin(), signature, tmpsForSignature(signature), BlockType::If, continuation, notTaken);
1790 return { };
1791}
1792
1793auto AirIRGenerator::addElse(ControlData& data, const Stack& currentStack) -> PartialResult
1794{
1795 unifyValuesWithBlock(currentStack, data.results);
1796 append(Jump);
1797 m_currentBlock->setSuccessors(data.continuation);
1798 return addElseToUnreachable(data);
1799}
1800
1801auto AirIRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
1802{
1803 ASSERT(data.blockType() == BlockType::If);
1804 m_currentBlock = data.special;
1805 data.convertIfToBlock();
1806 return { };
1807}
1808
1809auto AirIRGenerator::addReturn(const ControlData& data, const ExpressionList& returnValues) -> PartialResult
1810{
1811 CallInformation wasmCallInfo = wasmCallingConvention().callInformationFor(*data.signature(), CallRole::Callee);
1812 if (!wasmCallInfo.results.size()) {
1813 append(RetVoid);
1814 return { };
1815 }
1816
1817 B3::PatchpointValue* patch = addPatchpoint(B3::Void);
1818 patch->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1819 auto calleeSaves = params.code().calleeSaveRegisterAtOffsetList();
1820
1821 for (RegisterAtOffset calleeSave : calleeSaves)
1822 jit.load64ToReg(CCallHelpers::Address(GPRInfo::callFrameRegister, calleeSave.offset()), calleeSave.reg());
1823
1824 jit.emitFunctionEpilogue();
1825 jit.ret();
1826 });
1827 patch->effects.terminal = true;
1828
1829 ASSERT(returnValues.size() >= wasmCallInfo.results.size());
1830 unsigned offset = returnValues.size() - wasmCallInfo.results.size();
1831 Vector<ConstrainedTmp, 8> returnConstraints;
1832 for (unsigned i = 0; i < wasmCallInfo.results.size(); ++i) {
1833 B3::ValueRep rep = wasmCallInfo.results[i];
1834 TypedTmp tmp = returnValues[offset + i];
1835
1836 if (rep.isStack()) {
1837 append(moveForType(toB3Type(tmp.type())), tmp, Arg::addr(Tmp(GPRInfo::callFrameRegister), rep.offsetFromFP()));
1838 continue;
1839 }
1840
1841 ASSERT(rep.isReg());
1842 if (data.signature()->returnType(i) == I32)
1843 append(Move32, tmp, tmp);
1844 returnConstraints.append(ConstrainedTmp(tmp, wasmCallInfo.results[i]));
1845 }
1846
1847 emitPatchpoint(m_currentBlock, patch, Vector<Tmp, 1>(), WTFMove(returnConstraints));
1848 return { };
1849}
1850
1851// NOTE: All branches in Wasm are on 32-bit ints
1852
1853auto AirIRGenerator::addBranch(ControlData& data, ExpressionType condition, const Stack& returnValues) -> PartialResult
1854{
1855 unifyValuesWithBlock(returnValues, data.results);
1856
1857 BasicBlock* target = data.targetBlockForBranch();
1858 if (condition) {
1859 BasicBlock* continuation = m_code.addBlock();
1860 append(BranchTest32, Arg::resCond(MacroAssembler::NonZero), condition, condition);
1861 m_currentBlock->setSuccessors(target, continuation);
1862 m_currentBlock = continuation;
1863 } else {
1864 append(Jump);
1865 m_currentBlock->setSuccessors(target);
1866 }
1867
1868 return { };
1869}
1870
1871auto AirIRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const Stack& expressionStack) -> PartialResult
1872{
1873 auto& successors = m_currentBlock->successors();
1874 ASSERT(successors.isEmpty());
1875 for (const auto& target : targets) {
1876 unifyValuesWithBlock(expressionStack, target->results);
1877 successors.append(target->targetBlockForBranch());
1878 }
1879 unifyValuesWithBlock(expressionStack, defaultTarget.results);
1880 successors.append(defaultTarget.targetBlockForBranch());
1881
1882 ASSERT(condition.type() == Type::I32);
1883
1884 // FIXME: We should consider dynamically switching between a jump table
1885 // and a binary switch depending on the number of successors.
1886 // https://bugs.webkit.org/show_bug.cgi?id=194477
1887
1888 size_t numTargets = targets.size();
1889
1890 auto* patchpoint = addPatchpoint(B3::Void);
1891 patchpoint->effects = B3::Effects::none();
1892 patchpoint->effects.terminal = true;
1893 patchpoint->clobber(RegisterSet::macroScratchRegisters());
1894
1895 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
1896 AllowMacroScratchRegisterUsage allowScratch(jit);
1897
1898 Vector<int64_t> cases;
1899 cases.reserveInitialCapacity(numTargets);
1900 for (size_t i = 0; i < numTargets; ++i)
1901 cases.uncheckedAppend(i);
1902
1903 GPRReg valueReg = params[0].gpr();
1904 BinarySwitch binarySwitch(valueReg, cases, BinarySwitch::Int32);
1905
1906 Vector<CCallHelpers::Jump> caseJumps;
1907 caseJumps.resize(numTargets);
1908
1909 while (binarySwitch.advance(jit)) {
1910 unsigned value = binarySwitch.caseValue();
1911 unsigned index = binarySwitch.caseIndex();
1912 ASSERT_UNUSED(value, value == index);
1913 ASSERT(index < numTargets);
1914 caseJumps[index] = jit.jump();
1915 }
1916
1917 CCallHelpers::JumpList fallThrough = binarySwitch.fallThrough();
1918
1919 Vector<Box<CCallHelpers::Label>> successorLabels = params.successorLabels();
1920 ASSERT(successorLabels.size() == caseJumps.size() + 1);
1921
1922 params.addLatePath([=, caseJumps = WTFMove(caseJumps), successorLabels = WTFMove(successorLabels)] (CCallHelpers& jit) {
1923 for (size_t i = 0; i < numTargets; ++i)
1924 caseJumps[i].linkTo(*successorLabels[i], &jit);
1925 fallThrough.linkTo(*successorLabels[numTargets], &jit);
1926 });
1927 });
1928
1929 emitPatchpoint(patchpoint, TypedTmp(), condition);
1930
1931 return { };
1932}
1933
1934auto AirIRGenerator::endBlock(ControlEntry& entry, Stack& expressionStack) -> PartialResult
1935{
1936 ControlData& data = entry.controlData;
1937
1938 if (data.blockType() != BlockType::Loop)
1939 unifyValuesWithBlock(expressionStack, data.results);
1940 append(Jump);
1941 m_currentBlock->setSuccessors(data.continuation);
1942
1943 return addEndToUnreachable(entry, expressionStack);
1944}
1945
1946
1947auto AirIRGenerator::addEndToUnreachable(ControlEntry& entry, const Stack& expressionStack) -> PartialResult
1948{
1949 ControlData& data = entry.controlData;
1950 m_currentBlock = data.continuation;
1951
1952 if (data.blockType() == BlockType::If) {
1953 append(data.special, Jump);
1954 data.special->setSuccessors(m_currentBlock);
1955 }
1956
1957 if (data.blockType() == BlockType::Loop) {
1958 m_outerLoops.removeLast();
1959 for (unsigned i = 0; i < data.signature()->returnCount(); ++i) {
1960 if (i < expressionStack.size())
1961 entry.enclosedExpressionStack.append(expressionStack[i]);
1962 else
1963 entry.enclosedExpressionStack.append(addBottom(m_currentBlock, data.signature()->returnType(i)));
1964 }
1965 } else {
1966 for (const auto& result : data.results)
1967 entry.enclosedExpressionStack.append(result);
1968 }
1969
1970 // TopLevel does not have any code after this so we need to make sure we emit a return here.
1971 if (data.blockType() == BlockType::TopLevel)
1972 return addReturn(data, entry.enclosedExpressionStack);
1973
1974 return { };
1975}
1976
1977B3::PatchpointValue* AirIRGenerator::emitCallPatchpoint(BasicBlock* block, const Signature& signature, const Vector<ExpressionType, 1>& results, const Vector<TypedTmp>& args, Vector<ConstrainedTmp>&& patchArgs)
1978{
1979 auto* patchpoint = addPatchpoint(toB3ResultType(&signature));
1980 patchpoint->effects.writesPinned = true;
1981 patchpoint->effects.readsPinned = true;
1982 patchpoint->clobberEarly(RegisterSet::macroScratchRegisters());
1983 patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
1984
1985 CallInformation locations = wasmCallingConvention().callInformationFor(signature);
1986 m_code.requestCallArgAreaSizeInBytes(WTF::roundUpToMultipleOf(stackAlignmentBytes(), locations.headerAndArgumentStackSizeInBytes));
1987
1988 size_t offset = patchArgs.size();
1989 Checked<size_t> newSize = checkedSum<size_t>(patchArgs.size(), args.size());
1990 RELEASE_ASSERT(!newSize.hasOverflowed());
1991
1992 patchArgs.grow(newSize.unsafeGet());
1993 for (unsigned i = 0; i < args.size(); ++i)
1994 patchArgs[i + offset] = ConstrainedTmp(args[i], locations.params[i]);
1995
1996 if (patchpoint->type() != B3::Void)
1997 patchpoint->resultConstraints = WTFMove(locations.results);
1998 emitPatchpoint(block, patchpoint, results, WTFMove(patchArgs));
1999 return patchpoint;
2000}
2001
2002auto AirIRGenerator::addCall(uint32_t functionIndex, const Signature& signature, Vector<ExpressionType>& args, Vector<ExpressionType, 1>& results) -> PartialResult
2003{
2004 ASSERT(signature.argumentCount() == args.size());
2005
2006 m_makesCalls = true;
2007
2008 for (unsigned i = 0; i < signature.returnCount(); ++i)
2009 results.append(tmpForType(signature.returnType(i)));
2010
2011 Vector<UnlinkedWasmToWasmCall>* unlinkedWasmToWasmCalls = &m_unlinkedWasmToWasmCalls;
2012
2013 if (m_info.isImportedFunctionFromFunctionIndexSpace(functionIndex)) {
2014 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
2015
2016 auto currentInstance = g64();
2017 append(Move, instanceValue(), currentInstance);
2018
2019 auto targetInstance = g64();
2020
2021 // FIXME: We should have better isel here.
2022 // https://bugs.webkit.org/show_bug.cgi?id=193999
2023 append(Move, Arg::bigImm(Instance::offsetOfTargetInstance(functionIndex)), targetInstance);
2024 append(Add64, instanceValue(), targetInstance);
2025 append(Move, Arg::addr(targetInstance), targetInstance);
2026
2027 BasicBlock* isWasmBlock = m_code.addBlock();
2028 BasicBlock* isEmbedderBlock = m_code.addBlock();
2029 BasicBlock* continuation = m_code.addBlock();
2030
2031 append(BranchTest64, Arg::resCond(MacroAssembler::NonZero), targetInstance, targetInstance);
2032 m_currentBlock->setSuccessors(isWasmBlock, isEmbedderBlock);
2033
2034 {
2035 auto* patchpoint = emitCallPatchpoint(isWasmBlock, signature, results, args);
2036 // We need to clobber all potential pinned registers since we might be leaving the instance.
2037 // We pessimistically assume we could be calling to something that is bounds checking.
2038 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2039 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2040
2041 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2042 AllowMacroScratchRegisterUsage allowScratch(jit);
2043 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
2044 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
2045 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
2046 });
2047 });
2048
2049 append(isWasmBlock, Jump);
2050 isWasmBlock->setSuccessors(continuation);
2051 }
2052
2053 {
2054 auto jumpDestination = g64();
2055 append(isEmbedderBlock, Move, Arg::bigImm(Instance::offsetOfWasmToEmbedderStub(functionIndex)), jumpDestination);
2056 append(isEmbedderBlock, Add64, instanceValue(), jumpDestination);
2057 append(isEmbedderBlock, Move, Arg::addr(jumpDestination), jumpDestination);
2058
2059 Vector<ConstrainedTmp> jumpArgs;
2060 jumpArgs.append({ jumpDestination, B3::ValueRep::SomeRegister });
2061 auto* patchpoint = emitCallPatchpoint(isEmbedderBlock, signature, results, args, WTFMove(jumpArgs));
2062 // We need to clobber all potential pinned registers since we might be leaving the instance.
2063 // We pessimistically assume we could be calling to something that is bounds checking.
2064 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2065 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2066 patchpoint->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2067 AllowMacroScratchRegisterUsage allowScratch(jit);
2068 jit.call(params[params.proc().resultCount(params.value()->type())].gpr(), WasmEntryPtrTag);
2069 });
2070
2071 append(isEmbedderBlock, Jump);
2072 isEmbedderBlock->setSuccessors(continuation);
2073 }
2074
2075 m_currentBlock = continuation;
2076 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
2077 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, continuation);
2078 } else {
2079 auto* patchpoint = emitCallPatchpoint(m_currentBlock, signature, results, args);
2080 patchpoint->setGenerator([unlinkedWasmToWasmCalls, functionIndex] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2081 AllowMacroScratchRegisterUsage allowScratch(jit);
2082 CCallHelpers::Call call = jit.threadSafePatchableNearCall();
2083 jit.addLinkTask([unlinkedWasmToWasmCalls, call, functionIndex] (LinkBuffer& linkBuffer) {
2084 unlinkedWasmToWasmCalls->append({ linkBuffer.locationOfNearCall<WasmEntryPtrTag>(call), functionIndex });
2085 });
2086 });
2087 }
2088
2089 return { };
2090}
2091
2092auto AirIRGenerator::addCallIndirect(unsigned tableIndex, const Signature& signature, Vector<ExpressionType>& args, Vector<ExpressionType, 1>& results) -> PartialResult
2093{
2094 ExpressionType calleeIndex = args.takeLast();
2095 ASSERT(signature.argumentCount() == args.size());
2096 ASSERT(m_info.tableCount() > tableIndex);
2097 ASSERT(m_info.tables[tableIndex].type() == TableElementType::Funcref);
2098
2099 m_makesCalls = true;
2100 // Note: call indirect can call either WebAssemblyFunction or WebAssemblyWrapperFunction. Because
2101 // WebAssemblyWrapperFunction is like calling into the embedder, we conservatively assume all call indirects
2102 // can be to the embedder for our stack check calculation.
2103 m_maxNumJSCallArguments = std::max(m_maxNumJSCallArguments, static_cast<uint32_t>(args.size()));
2104
2105 auto currentInstance = g64();
2106 append(Move, instanceValue(), currentInstance);
2107
2108 ExpressionType callableFunctionBuffer = g64();
2109 ExpressionType instancesBuffer = g64();
2110 ExpressionType callableFunctionBufferLength = g64();
2111 {
2112 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfFunctions(), B3::Width64));
2113 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfInstances(), B3::Width64));
2114 RELEASE_ASSERT(Arg::isValidAddrForm(FuncRefTable::offsetOfLength(), B3::Width64));
2115
2116 if (UNLIKELY(!Arg::isValidAddrForm(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex), B3::Width64))) {
2117 append(Move, Arg::bigImm(Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)), callableFunctionBufferLength);
2118 append(Add64, instanceValue(), callableFunctionBufferLength);
2119 append(Move, Arg::addr(callableFunctionBufferLength), callableFunctionBufferLength);
2120 } else
2121 append(Move, Arg::addr(instanceValue(), Instance::offsetOfTablePtr(m_numImportFunctions, tableIndex)), callableFunctionBufferLength);
2122 append(Move, Arg::addr(callableFunctionBufferLength, FuncRefTable::offsetOfFunctions()), callableFunctionBuffer);
2123 append(Move, Arg::addr(callableFunctionBufferLength, FuncRefTable::offsetOfInstances()), instancesBuffer);
2124 append(Move32, Arg::addr(callableFunctionBufferLength, Table::offsetOfLength()), callableFunctionBufferLength);
2125 }
2126
2127 append(Move32, calleeIndex, calleeIndex);
2128
2129 // Check the index we are looking for is valid.
2130 emitCheck([&] {
2131 return Inst(Branch32, nullptr, Arg::relCond(MacroAssembler::AboveOrEqual), calleeIndex, callableFunctionBufferLength);
2132 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2133 this->emitThrowException(jit, ExceptionType::OutOfBoundsCallIndirect);
2134 });
2135
2136 ExpressionType calleeCode = g64();
2137 {
2138 ExpressionType calleeSignatureIndex = g64();
2139 // Compute the offset in the table index space we are looking for.
2140 append(Move, Arg::imm(sizeof(WasmToWasmImportableFunction)), calleeSignatureIndex);
2141 append(Mul64, calleeIndex, calleeSignatureIndex);
2142 append(Add64, callableFunctionBuffer, calleeSignatureIndex);
2143
2144 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfEntrypointLoadLocation()), calleeCode); // Pointer to callee code.
2145
2146 // Check that the WasmToWasmImportableFunction is initialized. We trap if it isn't. An "invalid" SignatureIndex indicates it's not initialized.
2147 // FIXME: when we have trap handlers, we can just let the call fail because Signature::invalidIndex is 0. https://bugs.webkit.org/show_bug.cgi?id=177210
2148 static_assert(sizeof(WasmToWasmImportableFunction::signatureIndex) == sizeof(uint64_t), "Load codegen assumes i64");
2149
2150 // FIXME: This seems dumb to do two checks just for a nicer error message.
2151 // We should move just to use a single branch and then figure out what
2152 // error to use in the exception handler.
2153
2154 append(Move, Arg::addr(calleeSignatureIndex, WasmToWasmImportableFunction::offsetOfSignatureIndex()), calleeSignatureIndex);
2155
2156 emitCheck([&] {
2157 static_assert(Signature::invalidIndex == 0, "");
2158 return Inst(BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), calleeSignatureIndex, calleeSignatureIndex);
2159 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2160 this->emitThrowException(jit, ExceptionType::NullTableEntry);
2161 });
2162
2163 ExpressionType expectedSignatureIndex = g64();
2164 append(Move, Arg::bigImm(SignatureInformation::get(signature)), expectedSignatureIndex);
2165 emitCheck([&] {
2166 return Inst(Branch64, nullptr, Arg::relCond(MacroAssembler::NotEqual), calleeSignatureIndex, expectedSignatureIndex);
2167 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2168 this->emitThrowException(jit, ExceptionType::BadSignature);
2169 });
2170 }
2171
2172 // Do a context switch if needed.
2173 {
2174 auto newContextInstance = g64();
2175 append(Move, Arg::index(instancesBuffer, calleeIndex, 8, 0), newContextInstance);
2176
2177 BasicBlock* doContextSwitch = m_code.addBlock();
2178 BasicBlock* continuation = m_code.addBlock();
2179
2180 append(Branch64, Arg::relCond(MacroAssembler::Equal), newContextInstance, instanceValue());
2181 m_currentBlock->setSuccessors(continuation, doContextSwitch);
2182
2183 auto* patchpoint = addPatchpoint(B3::Void);
2184 patchpoint->effects.writesPinned = true;
2185 // We pessimistically assume we're calling something with BoundsChecking memory.
2186 // FIXME: We shouldn't have to do this: https://bugs.webkit.org/show_bug.cgi?id=172181
2187 patchpoint->clobber(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2188 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2189 patchpoint->numGPScratchRegisters = Gigacage::isEnabled(Gigacage::Primitive) ? 1 : 0;
2190
2191 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2192 AllowMacroScratchRegisterUsage allowScratch(jit);
2193 GPRReg newContextInstance = params[0].gpr();
2194 GPRReg oldContextInstance = params[1].gpr();
2195 const PinnedRegisterInfo& pinnedRegs = PinnedRegisterInfo::get();
2196 GPRReg baseMemory = pinnedRegs.baseMemoryPointer;
2197 ASSERT(newContextInstance != baseMemory);
2198 jit.loadPtr(CCallHelpers::Address(oldContextInstance, Instance::offsetOfCachedStackLimit()), baseMemory);
2199 jit.storePtr(baseMemory, CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedStackLimit()));
2200 jit.storeWasmContextInstance(newContextInstance);
2201 // FIXME: We should support more than one memory size register
2202 // see: https://bugs.webkit.org/show_bug.cgi?id=162952
2203 ASSERT(pinnedRegs.sizeRegister != newContextInstance);
2204 GPRReg scratchOrSize = Gigacage::isEnabled(Gigacage::Primitive) ? params.gpScratch(0) : pinnedRegs.sizeRegister;
2205
2206 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemorySize()), pinnedRegs.sizeRegister); // Memory size.
2207 jit.loadPtr(CCallHelpers::Address(newContextInstance, Instance::offsetOfCachedMemory()), baseMemory); // Memory::void*.
2208
2209 jit.cageConditionally(Gigacage::Primitive, baseMemory, pinnedRegs.sizeRegister, scratchOrSize);
2210 });
2211
2212 emitPatchpoint(doContextSwitch, patchpoint, Tmp(), newContextInstance, instanceValue());
2213 append(doContextSwitch, Jump);
2214 doContextSwitch->setSuccessors(continuation);
2215
2216 m_currentBlock = continuation;
2217 }
2218
2219 append(Move, Arg::addr(calleeCode), calleeCode);
2220
2221 Vector<ConstrainedTmp> extraArgs;
2222 extraArgs.append(calleeCode);
2223
2224 for (unsigned i = 0; i < signature.returnCount(); ++i)
2225 results.append(tmpForType(signature.returnType(i)));
2226
2227 auto* patchpoint = emitCallPatchpoint(m_currentBlock, signature, results, args, WTFMove(extraArgs));
2228
2229 // We need to clobber all potential pinned registers since we might be leaving the instance.
2230 // We pessimistically assume we're always calling something that is bounds checking so
2231 // because the wasm->wasm thunk unconditionally overrides the size registers.
2232 // FIXME: We should not have to do this, but the wasm->wasm stub assumes it can
2233 // use all the pinned registers as scratch: https://bugs.webkit.org/show_bug.cgi?id=172181
2234
2235 patchpoint->clobberLate(PinnedRegisterInfo::get().toSave(MemoryMode::BoundsChecking));
2236
2237 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2238 AllowMacroScratchRegisterUsage allowScratch(jit);
2239 jit.call(params[params.proc().resultCount(params.value()->type())].gpr(), WasmEntryPtrTag);
2240 });
2241
2242 // The call could have been to another WebAssembly instance, and / or could have modified our Memory.
2243 restoreWebAssemblyGlobalState(RestoreCachedStackLimit::Yes, m_info.memory, currentInstance, m_currentBlock);
2244
2245 return { };
2246}
2247
2248void AirIRGenerator::unify(const ExpressionType& dst, const ExpressionType& source)
2249{
2250 ASSERT(isSubtype(source.type(), dst.type()));
2251 append(moveOpForValueType(dst.type()), source, dst);
2252}
2253
2254void AirIRGenerator::unifyValuesWithBlock(const Stack& resultStack, const ResultList& result)
2255{
2256 ASSERT(result.size() <= resultStack.size());
2257
2258 for (size_t i = 0; i < result.size(); ++i)
2259 unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
2260}
2261
2262static void dumpExpressionStack(const CommaPrinter& comma, const AirIRGenerator::Stack& expressionStack)
2263{
2264 dataLog(comma, "ExpressionStack:");
2265 for (const auto& expression : expressionStack)
2266 dataLog(comma, expression);
2267}
2268
2269void AirIRGenerator::dump(const Vector<ControlEntry>& controlStack, const Stack* stack)
2270{
2271 dataLogLn("Processing Graph:");
2272 dataLog(m_code);
2273 dataLogLn("With current block:", *m_currentBlock);
2274 dataLogLn("Control stack:");
2275 for (size_t i = controlStack.size(); i--;) {
2276 dataLog(" ", controlStack[i].controlData, ": ");
2277 CommaPrinter comma(", ", "");
2278 dumpExpressionStack(comma, *stack);
2279 stack = &controlStack[i].enclosedExpressionStack;
2280 dataLogLn();
2281 }
2282 dataLogLn("\n");
2283}
2284
2285auto AirIRGenerator::origin() -> B3::Origin
2286{
2287 // FIXME: We should implement a way to give Inst's an origin.
2288 return B3::Origin();
2289}
2290
2291Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileAir(CompilationContext& compilationContext, const FunctionData& function, const Signature& signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ModuleInformation& info, MemoryMode mode, uint32_t functionIndex, TierUpCount* tierUp, ThrowWasmException throwWasmException)
2292{
2293 auto result = makeUnique<InternalFunction>();
2294
2295 compilationContext.embedderEntrypointJIT = makeUnique<CCallHelpers>();
2296 compilationContext.wasmEntrypointJIT = makeUnique<CCallHelpers>();
2297
2298 B3::Procedure procedure;
2299 Code& code = procedure.code();
2300
2301 procedure.setOriginPrinter([] (PrintStream& out, B3::Origin origin) {
2302 if (origin.data())
2303 out.print("Wasm: ", bitwise_cast<OpcodeOrigin>(origin));
2304 });
2305
2306 // This means we cannot use either StackmapGenerationParams::usedRegisters() or
2307 // StackmapGenerationParams::unavailableRegisters(). In exchange for this concession, we
2308 // don't strictly need to run Air::reportUsedRegisters(), which saves a bit of CPU time at
2309 // optLevel=1.
2310 procedure.setNeedsUsedRegisters(false);
2311
2312 procedure.setOptLevel(Options::webAssemblyBBQAirOptimizationLevel());
2313
2314 AirIRGenerator irGenerator(info, procedure, result.get(), unlinkedWasmToWasmCalls, mode, functionIndex, tierUp, throwWasmException, signature);
2315 FunctionParser<AirIRGenerator> parser(irGenerator, function.data.data(), function.data.size(), signature, info);
2316 WASM_FAIL_IF_HELPER_FAILS(parser.parse());
2317
2318
2319 for (BasicBlock* block : code) {
2320 for (size_t i = 0; i < block->numSuccessors(); ++i)
2321 block->successorBlock(i)->addPredecessor(block);
2322 }
2323
2324 {
2325 if (UNLIKELY(shouldDumpIRAtEachPhase(B3::AirMode))) {
2326 dataLogLn("Generated patchpoints");
2327 for (B3::PatchpointValue** patch : irGenerator.patchpoints())
2328 dataLogLn(deepDump(procedure, *patch));
2329 }
2330
2331 B3::Air::prepareForGeneration(code);
2332 B3::Air::generate(code, *compilationContext.wasmEntrypointJIT);
2333 compilationContext.wasmEntrypointByproducts = procedure.releaseByproducts();
2334 result->entrypoint.calleeSaveRegisters = code.calleeSaveRegisterAtOffsetList();
2335 }
2336
2337 return result;
2338}
2339
2340template <typename IntType>
2341void AirIRGenerator::emitChecksForModOrDiv(bool isSignedDiv, ExpressionType left, ExpressionType right)
2342{
2343 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2344
2345 emitCheck([&] {
2346 return Inst(sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, nullptr, Arg::resCond(MacroAssembler::Zero), right, right);
2347 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2348 this->emitThrowException(jit, ExceptionType::DivisionByZero);
2349 });
2350
2351 if (isSignedDiv) {
2352 ASSERT(std::is_signed<IntType>::value);
2353 IntType min = std::numeric_limits<IntType>::min();
2354
2355 // FIXME: Better isel for compare with imms here.
2356 // https://bugs.webkit.org/show_bug.cgi?id=193999
2357 auto minTmp = sizeof(IntType) == 4 ? g32() : g64();
2358 auto negOne = sizeof(IntType) == 4 ? g32() : g64();
2359
2360 B3::Air::Opcode op = sizeof(IntType) == 4 ? Compare32 : Compare64;
2361 append(Move, Arg::bigImm(static_cast<uint64_t>(min)), minTmp);
2362 append(op, Arg::relCond(MacroAssembler::Equal), left, minTmp, minTmp);
2363
2364 append(Move, Arg::imm(-1), negOne);
2365 append(op, Arg::relCond(MacroAssembler::Equal), right, negOne, negOne);
2366
2367 emitCheck([&] {
2368 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), minTmp, negOne);
2369 },
2370 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2371 this->emitThrowException(jit, ExceptionType::IntegerOverflow);
2372 });
2373 }
2374}
2375
2376template <typename IntType>
2377void AirIRGenerator::emitModOrDiv(bool isDiv, ExpressionType lhs, ExpressionType rhs, ExpressionType& result)
2378{
2379 static_assert(sizeof(IntType) == 4 || sizeof(IntType) == 8, "");
2380
2381 result = sizeof(IntType) == 4 ? g32() : g64();
2382
2383 bool isSigned = std::is_signed<IntType>::value;
2384
2385 if (isARM64()) {
2386 B3::Air::Opcode div;
2387 switch (sizeof(IntType)) {
2388 case 4:
2389 div = isSigned ? Div32 : UDiv32;
2390 break;
2391 case 8:
2392 div = isSigned ? Div64 : UDiv64;
2393 break;
2394 }
2395
2396 append(div, lhs, rhs, result);
2397
2398 if (!isDiv) {
2399 append(sizeof(IntType) == 4 ? Mul32 : Mul64, result, rhs, result);
2400 append(sizeof(IntType) == 4 ? Sub32 : Sub64, lhs, result, result);
2401 }
2402
2403 return;
2404 }
2405
2406#if CPU(X86_64)
2407 Tmp eax(X86Registers::eax);
2408 Tmp edx(X86Registers::edx);
2409
2410 if (isSigned) {
2411 B3::Air::Opcode convertToDoubleWord;
2412 B3::Air::Opcode div;
2413 switch (sizeof(IntType)) {
2414 case 4:
2415 convertToDoubleWord = X86ConvertToDoubleWord32;
2416 div = X86Div32;
2417 break;
2418 case 8:
2419 convertToDoubleWord = X86ConvertToQuadWord64;
2420 div = X86Div64;
2421 break;
2422 default:
2423 RELEASE_ASSERT_NOT_REACHED();
2424 }
2425
2426 // We implement "res = Div<Chill>/Mod<Chill>(num, den)" as follows:
2427 //
2428 // if (den + 1 <=_unsigned 1) {
2429 // if (!den) {
2430 // res = 0;
2431 // goto done;
2432 // }
2433 // if (num == -2147483648) {
2434 // res = isDiv ? num : 0;
2435 // goto done;
2436 // }
2437 // }
2438 // res = num (/ or %) dev;
2439 // done:
2440
2441 BasicBlock* denIsGood = m_code.addBlock();
2442 BasicBlock* denMayBeBad = m_code.addBlock();
2443 BasicBlock* denNotZero = m_code.addBlock();
2444 BasicBlock* continuation = m_code.addBlock();
2445
2446 auto temp = sizeof(IntType) == 4 ? g32() : g64();
2447 auto one = addConstant(sizeof(IntType) == 4 ? Type::I32 : Type::I64, 1);
2448
2449 append(sizeof(IntType) == 4 ? Add32 : Add64, rhs, one, temp);
2450 append(sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Above), temp, one);
2451 m_currentBlock->setSuccessors(denIsGood, denMayBeBad);
2452
2453 append(denMayBeBad, Xor64, result, result);
2454 append(denMayBeBad, sizeof(IntType) == 4 ? BranchTest32 : BranchTest64, Arg::resCond(MacroAssembler::Zero), rhs, rhs);
2455 denMayBeBad->setSuccessors(continuation, denNotZero);
2456
2457 auto min = addConstant(denNotZero, sizeof(IntType) == 4 ? Type::I32 : Type::I64, std::numeric_limits<IntType>::min());
2458 if (isDiv)
2459 append(denNotZero, sizeof(IntType) == 4 ? Move32 : Move, min, result);
2460 else {
2461 // Result is zero, as set above...
2462 }
2463 append(denNotZero, sizeof(IntType) == 4 ? Branch32 : Branch64, Arg::relCond(MacroAssembler::Equal), lhs, min);
2464 denNotZero->setSuccessors(continuation, denIsGood);
2465
2466 auto divResult = isDiv ? eax : edx;
2467 append(denIsGood, Move, lhs, eax);
2468 append(denIsGood, convertToDoubleWord, eax, edx);
2469 append(denIsGood, div, eax, edx, rhs);
2470 append(denIsGood, sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2471 append(denIsGood, Jump);
2472 denIsGood->setSuccessors(continuation);
2473
2474 m_currentBlock = continuation;
2475 return;
2476 }
2477
2478 B3::Air::Opcode div = sizeof(IntType) == 4 ? X86UDiv32 : X86UDiv64;
2479
2480 Tmp divResult = isDiv ? eax : edx;
2481
2482 append(Move, lhs, eax);
2483 append(Xor64, edx, edx);
2484 append(div, eax, edx, rhs);
2485 append(sizeof(IntType) == 4 ? Move32 : Move, divResult, result);
2486#else
2487 RELEASE_ASSERT_NOT_REACHED();
2488#endif
2489}
2490
2491template<>
2492auto AirIRGenerator::addOp<OpType::I32DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2493{
2494 emitChecksForModOrDiv<int32_t>(true, left, right);
2495 emitModOrDiv<int32_t>(true, left, right, result);
2496 return { };
2497}
2498
2499template<>
2500auto AirIRGenerator::addOp<OpType::I32RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2501{
2502 emitChecksForModOrDiv<int32_t>(false, left, right);
2503 emitModOrDiv<int32_t>(false, left, right, result);
2504 return { };
2505}
2506
2507template<>
2508auto AirIRGenerator::addOp<OpType::I32DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2509{
2510 emitChecksForModOrDiv<uint32_t>(false, left, right);
2511 emitModOrDiv<uint32_t>(true, left, right, result);
2512 return { };
2513}
2514
2515template<>
2516auto AirIRGenerator::addOp<OpType::I32RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2517{
2518 emitChecksForModOrDiv<uint32_t>(false, left, right);
2519 emitModOrDiv<uint32_t>(false, left, right, result);
2520 return { };
2521}
2522
2523template<>
2524auto AirIRGenerator::addOp<OpType::I64DivS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2525{
2526 emitChecksForModOrDiv<int64_t>(true, left, right);
2527 emitModOrDiv<int64_t>(true, left, right, result);
2528 return { };
2529}
2530
2531template<>
2532auto AirIRGenerator::addOp<OpType::I64RemS>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2533{
2534 emitChecksForModOrDiv<int64_t>(false, left, right);
2535 emitModOrDiv<int64_t>(false, left, right, result);
2536 return { };
2537}
2538
2539template<>
2540auto AirIRGenerator::addOp<OpType::I64DivU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2541{
2542 emitChecksForModOrDiv<uint64_t>(false, left, right);
2543 emitModOrDiv<uint64_t>(true, left, right, result);
2544 return { };
2545}
2546
2547template<>
2548auto AirIRGenerator::addOp<OpType::I64RemU>(ExpressionType left, ExpressionType right, ExpressionType& result) -> PartialResult
2549{
2550 emitChecksForModOrDiv<uint64_t>(false, left, right);
2551 emitModOrDiv<uint64_t>(false, left, right, result);
2552 return { };
2553}
2554
2555template<>
2556auto AirIRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2557{
2558 auto* patchpoint = addPatchpoint(B3::Int32);
2559 patchpoint->effects = B3::Effects::none();
2560 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2561 jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
2562 });
2563 result = g32();
2564 emitPatchpoint(patchpoint, result, arg);
2565 return { };
2566}
2567
2568template<>
2569auto AirIRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
2570{
2571 auto* patchpoint = addPatchpoint(B3::Int64);
2572 patchpoint->effects = B3::Effects::none();
2573 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2574 jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
2575 });
2576 result = g64();
2577 emitPatchpoint(patchpoint, result, arg);
2578 return { };
2579}
2580
2581template<>
2582auto AirIRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2583{
2584 result = g32();
2585
2586#if CPU(X86_64)
2587 if (MacroAssembler::supportsCountPopulation()) {
2588 auto* patchpoint = addPatchpoint(B3::Int32);
2589 patchpoint->effects = B3::Effects::none();
2590 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2591 jit.countPopulation32(params[1].gpr(), params[0].gpr());
2592 });
2593 emitPatchpoint(patchpoint, result, arg);
2594 return { };
2595 }
2596#endif
2597
2598 emitCCall(&operationPopcount32, result, arg);
2599 return { };
2600}
2601
2602template<>
2603auto AirIRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
2604{
2605 result = g64();
2606
2607#if CPU(X86_64)
2608 if (MacroAssembler::supportsCountPopulation()) {
2609 auto* patchpoint = addPatchpoint(B3::Int64);
2610 patchpoint->effects = B3::Effects::none();
2611 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2612 jit.countPopulation64(params[1].gpr(), params[0].gpr());
2613 });
2614 emitPatchpoint(patchpoint, result, arg);
2615 return { };
2616 }
2617#endif
2618
2619 emitCCall(&operationPopcount64, result, arg);
2620 return { };
2621}
2622
2623template<>
2624auto AirIRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2625{
2626 auto* patchpoint = addPatchpoint(B3::Double);
2627 patchpoint->effects = B3::Effects::none();
2628 if (isX86())
2629 patchpoint->numGPScratchRegisters = 1;
2630 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2631 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2632 AllowMacroScratchRegisterUsage allowScratch(jit);
2633#if CPU(X86_64)
2634 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2635#else
2636 jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
2637#endif
2638 });
2639 result = f64();
2640 emitPatchpoint(patchpoint, result, arg);
2641 return { };
2642}
2643
2644template<>
2645auto AirIRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2646{
2647 auto* patchpoint = addPatchpoint(B3::Float);
2648 patchpoint->effects = B3::Effects::none();
2649 if (isX86())
2650 patchpoint->numGPScratchRegisters = 1;
2651 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2652 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2653 AllowMacroScratchRegisterUsage allowScratch(jit);
2654#if CPU(X86_64)
2655 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
2656#else
2657 jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
2658#endif
2659 });
2660 result = f32();
2661 emitPatchpoint(patchpoint, result, arg);
2662 return { };
2663}
2664
2665template<>
2666auto AirIRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2667{
2668 auto* patchpoint = addPatchpoint(B3::Double);
2669 patchpoint->effects = B3::Effects::none();
2670 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2671 jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
2672 });
2673 result = f64();
2674 emitPatchpoint(patchpoint, result, arg);
2675 return { };
2676}
2677
2678template<>
2679auto AirIRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
2680{
2681 auto* patchpoint = addPatchpoint(B3::Float);
2682 patchpoint->effects = B3::Effects::none();
2683 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2684 jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
2685 });
2686 result = f32();
2687 emitPatchpoint(patchpoint, result, arg);
2688 return { };
2689}
2690
2691template<>
2692auto AirIRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2693{
2694 auto* patchpoint = addPatchpoint(B3::Double);
2695 patchpoint->effects = B3::Effects::none();
2696 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2697 jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
2698 });
2699 result = f64();
2700 emitPatchpoint(patchpoint, result, arg);
2701 return { };
2702}
2703
2704template<>
2705auto AirIRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
2706{
2707 auto* patchpoint = addPatchpoint(B3::Float);
2708 patchpoint->effects = B3::Effects::none();
2709 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2710 jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
2711 });
2712 result = f32();
2713 emitPatchpoint(patchpoint, result, arg);
2714 return { };
2715}
2716
2717template<>
2718auto AirIRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2719{
2720 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int32_t>::min())));
2721 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min())));
2722
2723 auto temp1 = g32();
2724 auto temp2 = g32();
2725 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2726 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2727 append(Or32, temp1, temp2);
2728
2729 emitCheck([&] {
2730 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2731 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2732 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2733 });
2734
2735 auto* patchpoint = addPatchpoint(B3::Int32);
2736 patchpoint->effects = B3::Effects::none();
2737 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2738 jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
2739 });
2740 result = g32();
2741 emitPatchpoint(patchpoint, result, arg);
2742
2743 return { };
2744}
2745
2746template<>
2747auto AirIRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2748{
2749 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int32_t>::min())));
2750 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min())));
2751
2752 auto temp1 = g32();
2753 auto temp2 = g32();
2754 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2755 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2756 append(Or32, temp1, temp2);
2757
2758 emitCheck([&] {
2759 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2760 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2761 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2762 });
2763
2764 auto* patchpoint = addPatchpoint(B3::Int32);
2765 patchpoint->effects = B3::Effects::none();
2766 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2767 jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
2768 });
2769 result = g32();
2770 emitPatchpoint(patchpoint, result, arg);
2771 return { };
2772}
2773
2774
2775template<>
2776auto AirIRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2777{
2778 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0));
2779 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2780
2781 auto temp1 = g32();
2782 auto temp2 = g32();
2783 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2784 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2785 append(Or32, temp1, temp2);
2786
2787 emitCheck([&] {
2788 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2789 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2790 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2791 });
2792
2793 auto* patchpoint = addPatchpoint(B3::Int32);
2794 patchpoint->effects = B3::Effects::none();
2795 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2796 jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
2797 });
2798 result = g32();
2799 emitPatchpoint(patchpoint, result, arg);
2800 return { };
2801}
2802
2803template<>
2804auto AirIRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2805{
2806 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int32_t>::min()) * static_cast<float>(-2.0)));
2807 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2808
2809 auto temp1 = g32();
2810 auto temp2 = g32();
2811 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2812 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2813 append(Or32, temp1, temp2);
2814
2815 emitCheck([&] {
2816 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2817 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2818 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2819 });
2820
2821 auto* patchpoint = addPatchpoint(B3::Int32);
2822 patchpoint->effects = B3::Effects::none();
2823 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2824 jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
2825 });
2826 result = g32();
2827 emitPatchpoint(patchpoint, result, arg);
2828 return { };
2829}
2830
2831template<>
2832auto AirIRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2833{
2834 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(-static_cast<double>(std::numeric_limits<int64_t>::min())));
2835 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min())));
2836
2837 auto temp1 = g32();
2838 auto temp2 = g32();
2839 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2840 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2841 append(Or32, temp1, temp2);
2842
2843 emitCheck([&] {
2844 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2845 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2846 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2847 });
2848
2849 auto* patchpoint = addPatchpoint(B3::Int64);
2850 patchpoint->effects = B3::Effects::none();
2851 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2852 jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
2853 });
2854
2855 result = g64();
2856 emitPatchpoint(patchpoint, result, arg);
2857 return { };
2858}
2859
2860template<>
2861auto AirIRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
2862{
2863 auto max = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0));
2864 auto min = addConstant(Type::F64, bitwise_cast<uint64_t>(-1.0));
2865
2866 auto temp1 = g32();
2867 auto temp2 = g32();
2868 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2869 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2870 append(Or32, temp1, temp2);
2871
2872 emitCheck([&] {
2873 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2874 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2875 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2876 });
2877
2878 TypedTmp signBitConstant;
2879 if (isX86())
2880 signBitConstant = addConstant(Type::F64, bitwise_cast<uint64_t>(static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2881
2882 Vector<ConstrainedTmp> args;
2883 auto* patchpoint = addPatchpoint(B3::Int64);
2884 patchpoint->effects = B3::Effects::none();
2885 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2886 args.append(arg);
2887 if (isX86()) {
2888 args.append(signBitConstant);
2889 patchpoint->numFPScratchRegisters = 1;
2890 }
2891 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2892 AllowMacroScratchRegisterUsage allowScratch(jit);
2893 FPRReg scratch = InvalidFPRReg;
2894 FPRReg constant = InvalidFPRReg;
2895 if (isX86()) {
2896 scratch = params.fpScratch(0);
2897 constant = params[2].fpr();
2898 }
2899 jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2900 });
2901
2902 result = g64();
2903 emitPatchpoint(m_currentBlock, patchpoint, Vector<Tmp, 1> { result }, WTFMove(args));
2904 return { };
2905}
2906
2907template<>
2908auto AirIRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2909{
2910 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(-static_cast<float>(std::numeric_limits<int64_t>::min())));
2911 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min())));
2912
2913 auto temp1 = g32();
2914 auto temp2 = g32();
2915 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrUnordered), arg, min, temp1);
2916 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2917 append(Or32, temp1, temp2);
2918
2919 emitCheck([&] {
2920 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2921 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2922 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2923 });
2924
2925 auto* patchpoint = addPatchpoint(B3::Int64);
2926 patchpoint->effects = B3::Effects::none();
2927 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2928 jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
2929 });
2930 result = g64();
2931 emitPatchpoint(patchpoint, result, arg);
2932 return { };
2933}
2934
2935template<>
2936auto AirIRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
2937{
2938 auto max = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<int64_t>::min()) * static_cast<float>(-2.0)));
2939 auto min = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(-1.0)));
2940
2941 auto temp1 = g32();
2942 auto temp2 = g32();
2943 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqualOrUnordered), arg, min, temp1);
2944 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqualOrUnordered), arg, max, temp2);
2945 append(Or32, temp1, temp2);
2946
2947 emitCheck([&] {
2948 return Inst(BranchTest32, nullptr, Arg::resCond(MacroAssembler::NonZero), temp2, temp2);
2949 }, [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
2950 this->emitThrowException(jit, ExceptionType::OutOfBoundsTrunc);
2951 });
2952
2953 TypedTmp signBitConstant;
2954 if (isX86())
2955 signBitConstant = addConstant(Type::F32, bitwise_cast<uint32_t>(static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max())));
2956
2957 auto* patchpoint = addPatchpoint(B3::Int64);
2958 patchpoint->effects = B3::Effects::none();
2959 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2960 Vector<ConstrainedTmp, 2> args;
2961 args.append(arg);
2962 if (isX86()) {
2963 args.append(signBitConstant);
2964 patchpoint->numFPScratchRegisters = 1;
2965 }
2966 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
2967 AllowMacroScratchRegisterUsage allowScratch(jit);
2968 FPRReg scratch = InvalidFPRReg;
2969 FPRReg constant = InvalidFPRReg;
2970 if (isX86()) {
2971 scratch = params.fpScratch(0);
2972 constant = params[2].fpr();
2973 }
2974 jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
2975 });
2976
2977 result = g64();
2978 emitPatchpoint(m_currentBlock, patchpoint, Vector<Tmp, 1> { result }, WTFMove(args));
2979
2980 return { };
2981}
2982
2983auto AirIRGenerator::addShift(Type type, B3::Air::Opcode op, ExpressionType value, ExpressionType shift, ExpressionType& result) -> PartialResult
2984{
2985 ASSERT(type == Type::I64 || type == Type::I32);
2986 result = tmpForType(type);
2987
2988 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
2989 append(op, value, shift, result);
2990 return { };
2991 }
2992
2993#if CPU(X86_64)
2994 Tmp ecx = Tmp(X86Registers::ecx);
2995 append(Move, value, result);
2996 append(Move, shift, ecx);
2997 append(op, ecx, result);
2998#else
2999 RELEASE_ASSERT_NOT_REACHED();
3000#endif
3001 return { };
3002}
3003
3004auto AirIRGenerator::addIntegerSub(B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
3005{
3006 ASSERT(op == Sub32 || op == Sub64);
3007
3008 result = op == Sub32 ? g32() : g64();
3009
3010 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
3011 append(op, lhs, rhs, result);
3012 return { };
3013 }
3014
3015 RELEASE_ASSERT(isX86());
3016 // Sub a, b
3017 // means
3018 // b = b Sub a
3019 append(Move, lhs, result);
3020 append(op, rhs, result);
3021 return { };
3022}
3023
3024auto AirIRGenerator::addFloatingPointAbs(B3::Air::Opcode op, ExpressionType value, ExpressionType& result) -> PartialResult
3025{
3026 RELEASE_ASSERT(op == AbsFloat || op == AbsDouble);
3027
3028 result = op == AbsFloat ? f32() : f64();
3029
3030 if (isValidForm(op, Arg::Tmp, Arg::Tmp)) {
3031 append(op, value, result);
3032 return { };
3033 }
3034
3035 RELEASE_ASSERT(isX86());
3036
3037 if (op == AbsFloat) {
3038 auto constant = g32();
3039 append(Move, Arg::imm(static_cast<uint32_t>(~(1ull << 31))), constant);
3040 append(Move32ToFloat, constant, result);
3041 append(AndFloat, value, result);
3042 } else {
3043 auto constant = g64();
3044 append(Move, Arg::bigImm(~(1ull << 63)), constant);
3045 append(Move64ToDouble, constant, result);
3046 append(AndDouble, value, result);
3047 }
3048 return { };
3049}
3050
3051auto AirIRGenerator::addFloatingPointBinOp(Type type, B3::Air::Opcode op, ExpressionType lhs, ExpressionType rhs, ExpressionType& result) -> PartialResult
3052{
3053 ASSERT(type == Type::F32 || type == Type::F64);
3054 result = tmpForType(type);
3055
3056 if (isValidForm(op, Arg::Tmp, Arg::Tmp, Arg::Tmp)) {
3057 append(op, lhs, rhs, result);
3058 return { };
3059 }
3060
3061 RELEASE_ASSERT(isX86());
3062
3063 // Op a, b
3064 // means
3065 // b = b Op a
3066 append(moveOpForValueType(type), lhs, result);
3067 append(op, rhs, result);
3068 return { };
3069}
3070
3071template<> auto AirIRGenerator::addOp<OpType::F32Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3072{
3073 result = f32();
3074 append(CeilFloat, arg0, result);
3075 return { };
3076}
3077
3078template<> auto AirIRGenerator::addOp<OpType::I32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3079{
3080 result = g32();
3081 append(Mul32, arg0, arg1, result);
3082 return { };
3083}
3084
3085template<> auto AirIRGenerator::addOp<OpType::I32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3086{
3087 return addIntegerSub(Sub32, arg0, arg1, result);
3088}
3089
3090template<> auto AirIRGenerator::addOp<OpType::F64Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3091{
3092 result = g32();
3093 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
3094 return { };
3095}
3096
3097template<> auto AirIRGenerator::addOp<OpType::F32DemoteF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3098{
3099 result = f32();
3100 append(ConvertDoubleToFloat, arg0, result);
3101 return { };
3102}
3103
3104template<> auto AirIRGenerator::addOp<OpType::F32Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3105{
3106 return addFloatingPointMinOrMax(F32, MinOrMax::Min, arg0, arg1, result);
3107}
3108
3109template<> auto AirIRGenerator::addOp<OpType::F64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3110{
3111 result = g32();
3112 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
3113 return { };
3114}
3115
3116template<> auto AirIRGenerator::addOp<OpType::F64Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3117{
3118 result = g32();
3119 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
3120 return { };
3121}
3122
3123auto AirIRGenerator::addFloatingPointMinOrMax(Type floatType, MinOrMax minOrMax, ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3124{
3125 ASSERT(floatType == F32 || floatType == F64);
3126 result = tmpForType(floatType);
3127
3128 BasicBlock* isEqual = m_code.addBlock();
3129 BasicBlock* notEqual = m_code.addBlock();
3130 BasicBlock* isLessThan = m_code.addBlock();
3131 BasicBlock* notLessThan = m_code.addBlock();
3132 BasicBlock* isGreaterThan = m_code.addBlock();
3133 BasicBlock* isNaN = m_code.addBlock();
3134 BasicBlock* continuation = m_code.addBlock();
3135
3136 auto branchOp = floatType == F32 ? BranchFloat : BranchDouble;
3137 append(m_currentBlock, branchOp, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1);
3138 m_currentBlock->setSuccessors(isEqual, notEqual);
3139
3140 append(notEqual, branchOp, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1);
3141 notEqual->setSuccessors(isLessThan, notLessThan);
3142
3143 append(notLessThan, branchOp, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1);
3144 notLessThan->setSuccessors(isGreaterThan, isNaN);
3145
3146 auto andOp = floatType == F32 ? AndFloat : AndDouble;
3147 auto orOp = floatType == F32 ? OrFloat : OrDouble;
3148 append(isEqual, minOrMax == MinOrMax::Max ? andOp : orOp, arg0, arg1, result);
3149 append(isEqual, Jump);
3150 isEqual->setSuccessors(continuation);
3151
3152 auto isLessThanResult = minOrMax == MinOrMax::Max ? arg1 : arg0;
3153 append(isLessThan, moveOpForValueType(floatType), isLessThanResult, result);
3154 append(isLessThan, Jump);
3155 isLessThan->setSuccessors(continuation);
3156
3157 auto isGreaterThanResult = minOrMax == MinOrMax::Max ? arg0 : arg1;
3158 append(isGreaterThan, moveOpForValueType(floatType), isGreaterThanResult, result);
3159 append(isGreaterThan, Jump);
3160 isGreaterThan->setSuccessors(continuation);
3161
3162 auto addOp = floatType == F32 ? AddFloat : AddDouble;
3163 append(isNaN, addOp, arg0, arg1, result);
3164 append(isNaN, Jump);
3165 isNaN->setSuccessors(continuation);
3166
3167 m_currentBlock = continuation;
3168
3169 return { };
3170}
3171
3172template<> auto AirIRGenerator::addOp<OpType::F32Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3173{
3174 return addFloatingPointMinOrMax(F32, MinOrMax::Max, arg0, arg1, result);
3175}
3176
3177template<> auto AirIRGenerator::addOp<OpType::F64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3178{
3179 return addFloatingPointBinOp(Type::F64, MulDouble, arg0, arg1, result);
3180}
3181
3182template<> auto AirIRGenerator::addOp<OpType::F32Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3183{
3184 return addFloatingPointBinOp(Type::F32, DivFloat, arg0, arg1, result);
3185}
3186
3187template<> auto AirIRGenerator::addOp<OpType::I32Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3188{
3189 result = g32();
3190 append(CountLeadingZeros32, arg0, result);
3191 return { };
3192}
3193
3194template<> auto AirIRGenerator::addOp<OpType::F32Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3195{
3196 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3197 // https://bugs.webkit.org/show_bug.cgi?id=193999
3198 result = f32();
3199 auto temp1 = g32();
3200 auto sign = g32();
3201 auto value = g32();
3202
3203 // FIXME: Try to use Imm where possible:
3204 // https://bugs.webkit.org/show_bug.cgi?id=193999
3205 append(MoveFloatTo32, arg1, temp1);
3206 append(Move, Arg::bigImm(0x80000000), sign);
3207 append(And32, temp1, sign, sign);
3208
3209 append(MoveDoubleTo64, arg0, temp1);
3210 append(Move, Arg::bigImm(0x7fffffff), value);
3211 append(And32, temp1, value, value);
3212
3213 append(Or32, sign, value, value);
3214 append(Move32ToFloat, value, result);
3215
3216 return { };
3217}
3218
3219template<> auto AirIRGenerator::addOp<OpType::F64ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3220{
3221 result = f64();
3222 auto temp = g64();
3223 append(Move32, arg0, temp);
3224 append(ConvertInt64ToDouble, temp, result);
3225 return { };
3226}
3227
3228template<> auto AirIRGenerator::addOp<OpType::F32ReinterpretI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3229{
3230 result = f32();
3231 append(Move32ToFloat, arg0, result);
3232 return { };
3233}
3234
3235template<> auto AirIRGenerator::addOp<OpType::I64And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3236{
3237 result = g64();
3238 append(And64, arg0, arg1, result);
3239 return { };
3240}
3241
3242template<> auto AirIRGenerator::addOp<OpType::F32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3243{
3244 result = g32();
3245 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleNotEqualOrUnordered), arg0, arg1, result);
3246 return { };
3247}
3248
3249template<> auto AirIRGenerator::addOp<OpType::F64Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3250{
3251 result = g32();
3252 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3253 return { };
3254}
3255
3256template<> auto AirIRGenerator::addOp<OpType::F32Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3257{
3258 result = f32();
3259 append(SqrtFloat, arg0, result);
3260 return { };
3261}
3262
3263template<> auto AirIRGenerator::addOp<OpType::F64Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3264{
3265 result = g32();
3266 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3267 return { };
3268}
3269
3270template<> auto AirIRGenerator::addOp<OpType::I64GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3271{
3272 result = g32();
3273 append(Compare64, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3274 return { };
3275}
3276
3277template<> auto AirIRGenerator::addOp<OpType::I64GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3278{
3279 result = g32();
3280 append(Compare64, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3281 return { };
3282}
3283
3284template<> auto AirIRGenerator::addOp<OpType::I64Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3285{
3286 result = g32();
3287 append(Test64, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3288 return { };
3289}
3290
3291template<> auto AirIRGenerator::addOp<OpType::F64Div>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3292{
3293 return addFloatingPointBinOp(Type::F64, DivDouble, arg0, arg1, result);
3294}
3295
3296template<> auto AirIRGenerator::addOp<OpType::F32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3297{
3298 result = f32();
3299 append(AddFloat, arg0, arg1, result);
3300 return { };
3301}
3302
3303template<> auto AirIRGenerator::addOp<OpType::I64Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3304{
3305 result = g64();
3306 append(Or64, arg0, arg1, result);
3307 return { };
3308}
3309
3310template<> auto AirIRGenerator::addOp<OpType::I32LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3311{
3312 result = g32();
3313 append(Compare32, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3314 return { };
3315}
3316
3317template<> auto AirIRGenerator::addOp<OpType::I32LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3318{
3319 result = g32();
3320 append(Compare32, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3321 return { };
3322}
3323
3324template<> auto AirIRGenerator::addOp<OpType::I64Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3325{
3326 result = g32();
3327 append(Compare64, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3328 return { };
3329}
3330
3331template<> auto AirIRGenerator::addOp<OpType::I64Clz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3332{
3333 result = g64();
3334 append(CountLeadingZeros64, arg0, result);
3335 return { };
3336}
3337
3338template<> auto AirIRGenerator::addOp<OpType::F32Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3339{
3340 result = f32();
3341 if (isValidForm(NegateFloat, Arg::Tmp, Arg::Tmp))
3342 append(NegateFloat, arg0, result);
3343 else {
3344 auto constant = addConstant(Type::I32, bitwise_cast<uint32_t>(static_cast<float>(-0.0)));
3345 auto temp = g32();
3346 append(MoveFloatTo32, arg0, temp);
3347 append(Xor32, constant, temp);
3348 append(Move32ToFloat, temp, result);
3349 }
3350 return { };
3351}
3352
3353template<> auto AirIRGenerator::addOp<OpType::I32And>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3354{
3355 result = g32();
3356 append(And32, arg0, arg1, result);
3357 return { };
3358}
3359
3360template<> auto AirIRGenerator::addOp<OpType::I32LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3361{
3362 result = g32();
3363 append(Compare32, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3364 return { };
3365}
3366
3367template<> auto AirIRGenerator::addOp<OpType::I64Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3368{
3369 return addShift(Type::I64, RotateRight64, arg0, arg1, result);
3370}
3371
3372template<> auto AirIRGenerator::addOp<OpType::F64Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3373{
3374 return addFloatingPointAbs(AbsDouble, arg0, result);
3375}
3376
3377template<> auto AirIRGenerator::addOp<OpType::I32LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3378{
3379 result = g32();
3380 append(Compare32, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3381 return { };
3382}
3383
3384template<> auto AirIRGenerator::addOp<OpType::I32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3385{
3386 result = g32();
3387 append(Compare32, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3388 return { };
3389}
3390
3391template<> auto AirIRGenerator::addOp<OpType::F64Copysign>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3392{
3393 // FIXME: We can have better codegen here for the imms and two operand forms on x86
3394 // https://bugs.webkit.org/show_bug.cgi?id=193999
3395 result = f64();
3396 auto temp1 = g64();
3397 auto sign = g64();
3398 auto value = g64();
3399
3400 append(MoveDoubleTo64, arg1, temp1);
3401 append(Move, Arg::bigImm(0x8000000000000000), sign);
3402 append(And64, temp1, sign, sign);
3403
3404 append(MoveDoubleTo64, arg0, temp1);
3405 append(Move, Arg::bigImm(0x7fffffffffffffff), value);
3406 append(And64, temp1, value, value);
3407
3408 append(Or64, sign, value, value);
3409 append(Move64ToDouble, value, result);
3410
3411 return { };
3412}
3413
3414template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3415{
3416 result = f32();
3417 append(ConvertInt64ToFloat, arg0, result);
3418 return { };
3419}
3420
3421template<> auto AirIRGenerator::addOp<OpType::I64Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3422{
3423 if (isARM64()) {
3424 // ARM64 doesn't have a rotate left.
3425 auto newShift = g64();
3426 append(Move, arg1, newShift);
3427 append(Neg64, newShift);
3428 return addShift(Type::I64, RotateRight64, arg0, newShift, result);
3429 } else
3430 return addShift(Type::I64, RotateLeft64, arg0, arg1, result);
3431}
3432
3433template<> auto AirIRGenerator::addOp<OpType::F32Lt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3434{
3435 result = g32();
3436 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThan), arg0, arg1, result);
3437 return { };
3438}
3439
3440template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3441{
3442 result = f64();
3443 append(ConvertInt32ToDouble, arg0, result);
3444 return { };
3445}
3446
3447template<> auto AirIRGenerator::addOp<OpType::F64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3448{
3449 result = g32();
3450 append(CompareDouble, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3451 return { };
3452}
3453
3454template<> auto AirIRGenerator::addOp<OpType::F32Le>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3455{
3456 result = g32();
3457 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleLessThanOrEqual), arg0, arg1, result);
3458 return { };
3459}
3460
3461template<> auto AirIRGenerator::addOp<OpType::F32Ge>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3462{
3463 result = g32();
3464 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThanOrEqual), arg0, arg1, result);
3465 return { };
3466}
3467
3468template<> auto AirIRGenerator::addOp<OpType::I32ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3469{
3470 return addShift(Type::I32, Urshift32, arg0, arg1, result);
3471}
3472
3473template<> auto AirIRGenerator::addOp<OpType::F32ConvertUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3474{
3475 result = f32();
3476 auto temp = g64();
3477 append(Move32, arg0, temp);
3478 append(ConvertInt64ToFloat, temp, result);
3479 return { };
3480}
3481
3482template<> auto AirIRGenerator::addOp<OpType::I32ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3483{
3484 return addShift(Type::I32, Rshift32, arg0, arg1, result);
3485}
3486
3487template<> auto AirIRGenerator::addOp<OpType::I32GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3488{
3489 result = g32();
3490 append(Compare32, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3491 return { };
3492}
3493
3494template<> auto AirIRGenerator::addOp<OpType::F64Ceil>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3495{
3496 result = f64();
3497 append(CeilDouble, arg0, result);
3498 return { };
3499}
3500
3501template<> auto AirIRGenerator::addOp<OpType::I32GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3502{
3503 result = g32();
3504 append(Compare32, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3505 return { };
3506}
3507
3508template<> auto AirIRGenerator::addOp<OpType::I32Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3509{
3510 return addShift(Type::I32, Lshift32, arg0, arg1, result);
3511}
3512
3513template<> auto AirIRGenerator::addOp<OpType::F64Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3514{
3515 result = f64();
3516 append(FloorDouble, arg0, result);
3517 return { };
3518}
3519
3520template<> auto AirIRGenerator::addOp<OpType::I32Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3521{
3522 result = g32();
3523 append(Xor32, arg0, arg1, result);
3524 return { };
3525}
3526
3527template<> auto AirIRGenerator::addOp<OpType::F32Abs>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3528{
3529 return addFloatingPointAbs(AbsFloat, arg0, result);
3530}
3531
3532template<> auto AirIRGenerator::addOp<OpType::F64Min>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3533{
3534 return addFloatingPointMinOrMax(F64, MinOrMax::Min, arg0, arg1, result);
3535}
3536
3537template<> auto AirIRGenerator::addOp<OpType::F32Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3538{
3539 result = f32();
3540 append(MulFloat, arg0, arg1, result);
3541 return { };
3542}
3543
3544template<> auto AirIRGenerator::addOp<OpType::I64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3545{
3546 return addIntegerSub(Sub64, arg0, arg1, result);
3547}
3548
3549template<> auto AirIRGenerator::addOp<OpType::I32ReinterpretF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3550{
3551 result = g32();
3552 append(MoveFloatTo32, arg0, result);
3553 return { };
3554}
3555
3556template<> auto AirIRGenerator::addOp<OpType::I32Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3557{
3558 result = g32();
3559 append(Add32, arg0, arg1, result);
3560 return { };
3561}
3562
3563template<> auto AirIRGenerator::addOp<OpType::F64Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3564{
3565 return addFloatingPointBinOp(Type::F64, SubDouble, arg0, arg1, result);
3566}
3567
3568template<> auto AirIRGenerator::addOp<OpType::I32Or>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3569{
3570 result = g32();
3571 append(Or32, arg0, arg1, result);
3572 return { };
3573}
3574
3575template<> auto AirIRGenerator::addOp<OpType::I64LtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3576{
3577 result = g32();
3578 append(Compare64, Arg::relCond(MacroAssembler::Below), arg0, arg1, result);
3579 return { };
3580}
3581
3582template<> auto AirIRGenerator::addOp<OpType::I64LtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3583{
3584 result = g32();
3585 append(Compare64, Arg::relCond(MacroAssembler::LessThan), arg0, arg1, result);
3586 return { };
3587}
3588
3589template<> auto AirIRGenerator::addOp<OpType::F64ConvertSI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3590{
3591 result = f64();
3592 append(ConvertInt64ToDouble, arg0, result);
3593 return { };
3594}
3595
3596template<> auto AirIRGenerator::addOp<OpType::I64Xor>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3597{
3598 result = g64();
3599 append(Xor64, arg0, arg1, result);
3600 return { };
3601}
3602
3603template<> auto AirIRGenerator::addOp<OpType::I64GeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3604{
3605 result = g32();
3606 append(Compare64, Arg::relCond(MacroAssembler::AboveOrEqual), arg0, arg1, result);
3607 return { };
3608}
3609
3610template<> auto AirIRGenerator::addOp<OpType::I64Mul>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3611{
3612 result = g64();
3613 append(Mul64, arg0, arg1, result);
3614 return { };
3615}
3616
3617template<> auto AirIRGenerator::addOp<OpType::F32Sub>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3618{
3619 result = f32();
3620 if (isValidForm(SubFloat, Arg::Tmp, Arg::Tmp, Arg::Tmp))
3621 append(SubFloat, arg0, arg1, result);
3622 else {
3623 RELEASE_ASSERT(isX86());
3624 append(MoveFloat, arg0, result);
3625 append(SubFloat, arg1, result);
3626 }
3627 return { };
3628}
3629
3630template<> auto AirIRGenerator::addOp<OpType::F64PromoteF32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3631{
3632 result = f64();
3633 append(ConvertFloatToDouble, arg0, result);
3634 return { };
3635}
3636
3637template<> auto AirIRGenerator::addOp<OpType::F64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3638{
3639 result = f64();
3640 append(AddDouble, arg0, arg1, result);
3641 return { };
3642}
3643
3644template<> auto AirIRGenerator::addOp<OpType::I64GeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3645{
3646 result = g32();
3647 append(Compare64, Arg::relCond(MacroAssembler::GreaterThanOrEqual), arg0, arg1, result);
3648 return { };
3649}
3650
3651template<> auto AirIRGenerator::addOp<OpType::I64ExtendUI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3652{
3653 result = g64();
3654 append(Move32, arg0, result);
3655 return { };
3656}
3657
3658template<> auto AirIRGenerator::addOp<OpType::I32Ne>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3659{
3660 result = g32();
3661 RELEASE_ASSERT(arg0 && arg1);
3662 append(Compare32, Arg::relCond(MacroAssembler::NotEqual), arg0, arg1, result);
3663 return { };
3664}
3665
3666template<> auto AirIRGenerator::addOp<OpType::F64ReinterpretI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3667{
3668 result = f64();
3669 append(Move64ToDouble, arg0, result);
3670 return { };
3671}
3672
3673template<> auto AirIRGenerator::addOp<OpType::F32Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3674{
3675 result = g32();
3676 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleEqual), arg0, arg1, result);
3677 return { };
3678}
3679
3680template<> auto AirIRGenerator::addOp<OpType::I64Eq>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3681{
3682 result = g32();
3683 append(Compare64, Arg::relCond(MacroAssembler::Equal), arg0, arg1, result);
3684 return { };
3685}
3686
3687template<> auto AirIRGenerator::addOp<OpType::F32Floor>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3688{
3689 result = f32();
3690 append(FloorFloat, arg0, result);
3691 return { };
3692}
3693
3694template<> auto AirIRGenerator::addOp<OpType::F32ConvertSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3695{
3696 result = f32();
3697 append(ConvertInt32ToFloat, arg0, result);
3698 return { };
3699}
3700
3701template<> auto AirIRGenerator::addOp<OpType::I32Eqz>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3702{
3703 result = g32();
3704 append(Test32, Arg::resCond(MacroAssembler::Zero), arg0, arg0, result);
3705 return { };
3706}
3707
3708template<> auto AirIRGenerator::addOp<OpType::I64ReinterpretF64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3709{
3710 result = g64();
3711 append(MoveDoubleTo64, arg0, result);
3712 return { };
3713}
3714
3715template<> auto AirIRGenerator::addOp<OpType::I64ShrS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3716{
3717 return addShift(Type::I64, Rshift64, arg0, arg1, result);
3718}
3719
3720template<> auto AirIRGenerator::addOp<OpType::I64ShrU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3721{
3722 return addShift(Type::I64, Urshift64, arg0, arg1, result);
3723}
3724
3725template<> auto AirIRGenerator::addOp<OpType::F64Sqrt>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3726{
3727 result = f64();
3728 append(SqrtDouble, arg0, result);
3729 return { };
3730}
3731
3732template<> auto AirIRGenerator::addOp<OpType::I64Shl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3733{
3734 return addShift(Type::I64, Lshift64, arg0, arg1, result);
3735}
3736
3737template<> auto AirIRGenerator::addOp<OpType::F32Gt>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3738{
3739 result = g32();
3740 append(CompareFloat, Arg::doubleCond(MacroAssembler::DoubleGreaterThan), arg0, arg1, result);
3741 return { };
3742}
3743
3744template<> auto AirIRGenerator::addOp<OpType::I32WrapI64>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3745{
3746 result = g32();
3747 append(Move32, arg0, result);
3748 return { };
3749}
3750
3751template<> auto AirIRGenerator::addOp<OpType::I32Rotl>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3752{
3753 if (isARM64()) {
3754 // ARM64 doesn't have a rotate left.
3755 auto newShift = g64();
3756 append(Move, arg1, newShift);
3757 append(Neg64, newShift);
3758 return addShift(Type::I32, RotateRight32, arg0, newShift, result);
3759 } else
3760 return addShift(Type::I32, RotateLeft32, arg0, arg1, result);
3761}
3762
3763template<> auto AirIRGenerator::addOp<OpType::I32Rotr>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3764{
3765 return addShift(Type::I32, RotateRight32, arg0, arg1, result);
3766}
3767
3768template<> auto AirIRGenerator::addOp<OpType::I32GtU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3769{
3770 result = g32();
3771 append(Compare32, Arg::relCond(MacroAssembler::Above), arg0, arg1, result);
3772 return { };
3773}
3774
3775template<> auto AirIRGenerator::addOp<OpType::I64ExtendSI32>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3776{
3777 result = g64();
3778 append(SignExtend32ToPtr, arg0, result);
3779 return { };
3780}
3781
3782template<> auto AirIRGenerator::addOp<OpType::I32GtS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3783{
3784 result = g32();
3785 append(Compare32, Arg::relCond(MacroAssembler::GreaterThan), arg0, arg1, result);
3786 return { };
3787}
3788
3789template<> auto AirIRGenerator::addOp<OpType::F64Neg>(ExpressionType arg0, ExpressionType& result) -> PartialResult
3790{
3791 result = f64();
3792 if (isValidForm(NegateDouble, Arg::Tmp, Arg::Tmp))
3793 append(NegateDouble, arg0, result);
3794 else {
3795 auto constant = addConstant(Type::I64, bitwise_cast<uint64_t>(static_cast<double>(-0.0)));
3796 auto temp = g64();
3797 append(MoveDoubleTo64, arg0, temp);
3798 append(Xor64, constant, temp);
3799 append(Move64ToDouble, temp, result);
3800 }
3801 return { };
3802}
3803
3804template<> auto AirIRGenerator::addOp<OpType::F64Max>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3805{
3806 return addFloatingPointMinOrMax(F64, MinOrMax::Max, arg0, arg1, result);
3807}
3808
3809template<> auto AirIRGenerator::addOp<OpType::I64LeU>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3810{
3811 result = g32();
3812 append(Compare64, Arg::relCond(MacroAssembler::BelowOrEqual), arg0, arg1, result);
3813 return { };
3814}
3815
3816template<> auto AirIRGenerator::addOp<OpType::I64LeS>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3817{
3818 result = g32();
3819 append(Compare64, Arg::relCond(MacroAssembler::LessThanOrEqual), arg0, arg1, result);
3820 return { };
3821}
3822
3823template<> auto AirIRGenerator::addOp<OpType::I64Add>(ExpressionType arg0, ExpressionType arg1, ExpressionType& result) -> PartialResult
3824{
3825 result = g64();
3826 append(Add64, arg0, arg1, result);
3827 return { };
3828}
3829
3830} } // namespace JSC::Wasm
3831
3832#endif // ENABLE(WEBASSEMBLY)
3833