1// Copyright 2017 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
6#define V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
7
8#include "src/wasm/baseline/liftoff-assembler.h"
9
10#include "src/assembler.h"
11#include "src/wasm/value-type.h"
12
13namespace v8 {
14namespace internal {
15namespace wasm {
16
17#define REQUIRE_CPU_FEATURE(name, ...) \
18 if (!CpuFeatures::IsSupported(name)) { \
19 bailout("no " #name); \
20 return __VA_ARGS__; \
21 } \
22 CpuFeatureScope feature(this, name);
23
24namespace liftoff {
25
26constexpr Register kScratchRegister2 = r11;
27static_assert(kScratchRegister != kScratchRegister2, "collision");
28static_assert((kLiftoffAssemblerGpCacheRegs &
29 Register::ListOf<kScratchRegister, kScratchRegister2>()) == 0,
30 "scratch registers must not be used as cache registers");
31
32constexpr DoubleRegister kScratchDoubleReg2 = xmm14;
33static_assert(kScratchDoubleReg != kScratchDoubleReg2, "collision");
34static_assert(
35 (kLiftoffAssemblerFpCacheRegs &
36 DoubleRegister::ListOf<kScratchDoubleReg, kScratchDoubleReg2>()) == 0,
37 "scratch registers must not be used as cache registers");
38
39// rbp-8 holds the stack marker, rbp-16 is the instance parameter, first stack
40// slot is located at rbp-24.
41constexpr int32_t kConstantStackSpace = 16;
42constexpr int32_t kFirstStackSlotOffset =
43 kConstantStackSpace + LiftoffAssembler::kStackSlotSize;
44
45inline Operand GetStackSlot(uint32_t index) {
46 int32_t offset = index * LiftoffAssembler::kStackSlotSize;
47 return Operand(rbp, -kFirstStackSlotOffset - offset);
48}
49
50// TODO(clemensh): Make this a constexpr variable once Operand is constexpr.
51inline Operand GetInstanceOperand() { return Operand(rbp, -16); }
52
53inline Operand GetMemOp(LiftoffAssembler* assm, Register addr, Register offset,
54 uint32_t offset_imm) {
55 if (is_uint31(offset_imm)) {
56 if (offset == no_reg) return Operand(addr, offset_imm);
57 return Operand(addr, offset, times_1, offset_imm);
58 }
59 // Offset immediate does not fit in 31 bits.
60 Register scratch = kScratchRegister;
61 assm->movl(scratch, Immediate(offset_imm));
62 if (offset != no_reg) {
63 assm->addq(scratch, offset);
64 }
65 return Operand(addr, scratch, times_1, 0);
66}
67
68inline void Load(LiftoffAssembler* assm, LiftoffRegister dst, Operand src,
69 ValueType type) {
70 switch (type) {
71 case kWasmI32:
72 assm->movl(dst.gp(), src);
73 break;
74 case kWasmI64:
75 assm->movq(dst.gp(), src);
76 break;
77 case kWasmF32:
78 assm->Movss(dst.fp(), src);
79 break;
80 case kWasmF64:
81 assm->Movsd(dst.fp(), src);
82 break;
83 default:
84 UNREACHABLE();
85 }
86}
87
88inline void Store(LiftoffAssembler* assm, Operand dst, LiftoffRegister src,
89 ValueType type) {
90 switch (type) {
91 case kWasmI32:
92 assm->movl(dst, src.gp());
93 break;
94 case kWasmI64:
95 assm->movq(dst, src.gp());
96 break;
97 case kWasmF32:
98 assm->Movss(dst, src.fp());
99 break;
100 case kWasmF64:
101 assm->Movsd(dst, src.fp());
102 break;
103 default:
104 UNREACHABLE();
105 }
106}
107
108inline void push(LiftoffAssembler* assm, LiftoffRegister reg, ValueType type) {
109 switch (type) {
110 case kWasmI32:
111 case kWasmI64:
112 assm->pushq(reg.gp());
113 break;
114 case kWasmF32:
115 assm->subq(rsp, Immediate(kSystemPointerSize));
116 assm->Movss(Operand(rsp, 0), reg.fp());
117 break;
118 case kWasmF64:
119 assm->subq(rsp, Immediate(kSystemPointerSize));
120 assm->Movsd(Operand(rsp, 0), reg.fp());
121 break;
122 default:
123 UNREACHABLE();
124 }
125}
126
127template <typename... Regs>
128inline void SpillRegisters(LiftoffAssembler* assm, Regs... regs) {
129 for (LiftoffRegister r : {LiftoffRegister(regs)...}) {
130 if (assm->cache_state()->is_used(r)) assm->SpillRegister(r);
131 }
132}
133
134} // namespace liftoff
135
136int LiftoffAssembler::PrepareStackFrame() {
137 int offset = pc_offset();
138 sub_sp_32(0);
139 return offset;
140}
141
142void LiftoffAssembler::PatchPrepareStackFrame(int offset,
143 uint32_t stack_slots) {
144 uint32_t bytes = liftoff::kConstantStackSpace + kStackSlotSize * stack_slots;
145 DCHECK_LE(bytes, kMaxInt);
146 // We can't run out of space, just pass anything big enough to not cause the
147 // assembler to try to grow the buffer.
148 constexpr int kAvailableSpace = 64;
149 Assembler patching_assembler(
150 AssemblerOptions{},
151 ExternalAssemblerBuffer(buffer_start_ + offset, kAvailableSpace));
152 patching_assembler.sub_sp_32(bytes);
153}
154
155void LiftoffAssembler::FinishCode() {}
156
157void LiftoffAssembler::AbortCompilation() {}
158
159void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
160 RelocInfo::Mode rmode) {
161 switch (value.type()) {
162 case kWasmI32:
163 if (value.to_i32() == 0 && RelocInfo::IsNone(rmode)) {
164 xorl(reg.gp(), reg.gp());
165 } else {
166 movl(reg.gp(), Immediate(value.to_i32(), rmode));
167 }
168 break;
169 case kWasmI64:
170 if (RelocInfo::IsNone(rmode)) {
171 TurboAssembler::Set(reg.gp(), value.to_i64());
172 } else {
173 movq(reg.gp(), Immediate64(value.to_i64(), rmode));
174 }
175 break;
176 case kWasmF32:
177 TurboAssembler::Move(reg.fp(), value.to_f32_boxed().get_bits());
178 break;
179 case kWasmF64:
180 TurboAssembler::Move(reg.fp(), value.to_f64_boxed().get_bits());
181 break;
182 default:
183 UNREACHABLE();
184 }
185}
186
187void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset,
188 int size) {
189 DCHECK_LE(offset, kMaxInt);
190 movq(dst, liftoff::GetInstanceOperand());
191 DCHECK(size == 4 || size == 8);
192 if (size == 4) {
193 movl(dst, Operand(dst, offset));
194 } else {
195 movq(dst, Operand(dst, offset));
196 }
197}
198
199void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst,
200 uint32_t offset) {
201 DCHECK_LE(offset, kMaxInt);
202 movq(dst, liftoff::GetInstanceOperand());
203 LoadTaggedPointerField(dst, Operand(dst, offset));
204}
205
206void LiftoffAssembler::SpillInstance(Register instance) {
207 movq(liftoff::GetInstanceOperand(), instance);
208}
209
210void LiftoffAssembler::FillInstanceInto(Register dst) {
211 movq(dst, liftoff::GetInstanceOperand());
212}
213
214void LiftoffAssembler::LoadTaggedPointer(Register dst, Register src_addr,
215 Register offset_reg,
216 uint32_t offset_imm,
217 LiftoffRegList pinned) {
218 if (emit_debug_code() && offset_reg != no_reg) {
219 AssertZeroExtended(offset_reg);
220 }
221 Operand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
222 LoadTaggedPointerField(dst, src_op);
223}
224
225void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr,
226 Register offset_reg, uint32_t offset_imm,
227 LoadType type, LiftoffRegList pinned,
228 uint32_t* protected_load_pc, bool is_load_mem) {
229 if (emit_debug_code() && offset_reg != no_reg) {
230 AssertZeroExtended(offset_reg);
231 }
232 Operand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
233 if (protected_load_pc) *protected_load_pc = pc_offset();
234 switch (type.value()) {
235 case LoadType::kI32Load8U:
236 case LoadType::kI64Load8U:
237 movzxbl(dst.gp(), src_op);
238 break;
239 case LoadType::kI32Load8S:
240 movsxbl(dst.gp(), src_op);
241 break;
242 case LoadType::kI64Load8S:
243 movsxbq(dst.gp(), src_op);
244 break;
245 case LoadType::kI32Load16U:
246 case LoadType::kI64Load16U:
247 movzxwl(dst.gp(), src_op);
248 break;
249 case LoadType::kI32Load16S:
250 movsxwl(dst.gp(), src_op);
251 break;
252 case LoadType::kI64Load16S:
253 movsxwq(dst.gp(), src_op);
254 break;
255 case LoadType::kI32Load:
256 case LoadType::kI64Load32U:
257 movl(dst.gp(), src_op);
258 break;
259 case LoadType::kI64Load32S:
260 movsxlq(dst.gp(), src_op);
261 break;
262 case LoadType::kI64Load:
263 movq(dst.gp(), src_op);
264 break;
265 case LoadType::kF32Load:
266 Movss(dst.fp(), src_op);
267 break;
268 case LoadType::kF64Load:
269 Movsd(dst.fp(), src_op);
270 break;
271 default:
272 UNREACHABLE();
273 }
274}
275
276void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
277 uint32_t offset_imm, LiftoffRegister src,
278 StoreType type, LiftoffRegList /* pinned */,
279 uint32_t* protected_store_pc, bool is_store_mem) {
280 if (emit_debug_code() && offset_reg != no_reg) {
281 AssertZeroExtended(offset_reg);
282 }
283 Operand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
284 if (protected_store_pc) *protected_store_pc = pc_offset();
285 switch (type.value()) {
286 case StoreType::kI32Store8:
287 case StoreType::kI64Store8:
288 movb(dst_op, src.gp());
289 break;
290 case StoreType::kI32Store16:
291 case StoreType::kI64Store16:
292 movw(dst_op, src.gp());
293 break;
294 case StoreType::kI32Store:
295 case StoreType::kI64Store32:
296 movl(dst_op, src.gp());
297 break;
298 case StoreType::kI64Store:
299 movq(dst_op, src.gp());
300 break;
301 case StoreType::kF32Store:
302 Movss(dst_op, src.fp());
303 break;
304 case StoreType::kF64Store:
305 Movsd(dst_op, src.fp());
306 break;
307 default:
308 UNREACHABLE();
309 }
310}
311
312void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst,
313 uint32_t caller_slot_idx,
314 ValueType type) {
315 Operand src(rbp, kSystemPointerSize * (caller_slot_idx + 1));
316 liftoff::Load(this, dst, src, type);
317}
318
319void LiftoffAssembler::MoveStackValue(uint32_t dst_index, uint32_t src_index,
320 ValueType type) {
321 DCHECK_NE(dst_index, src_index);
322 Operand src = liftoff::GetStackSlot(src_index);
323 Operand dst = liftoff::GetStackSlot(dst_index);
324 if (ValueTypes::ElementSizeLog2Of(type) == 2) {
325 movl(kScratchRegister, src);
326 movl(dst, kScratchRegister);
327 } else {
328 DCHECK_EQ(3, ValueTypes::ElementSizeLog2Of(type));
329 movq(kScratchRegister, src);
330 movq(dst, kScratchRegister);
331 }
332}
333
334void LiftoffAssembler::Move(Register dst, Register src, ValueType type) {
335 DCHECK_NE(dst, src);
336 if (type == kWasmI32) {
337 movl(dst, src);
338 } else {
339 DCHECK_EQ(kWasmI64, type);
340 movq(dst, src);
341 }
342}
343
344void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src,
345 ValueType type) {
346 DCHECK_NE(dst, src);
347 if (type == kWasmF32) {
348 Movss(dst, src);
349 } else {
350 DCHECK_EQ(kWasmF64, type);
351 Movsd(dst, src);
352 }
353}
354
355void LiftoffAssembler::Spill(uint32_t index, LiftoffRegister reg,
356 ValueType type) {
357 RecordUsedSpillSlot(index);
358 Operand dst = liftoff::GetStackSlot(index);
359 switch (type) {
360 case kWasmI32:
361 movl(dst, reg.gp());
362 break;
363 case kWasmI64:
364 movq(dst, reg.gp());
365 break;
366 case kWasmF32:
367 Movss(dst, reg.fp());
368 break;
369 case kWasmF64:
370 Movsd(dst, reg.fp());
371 break;
372 default:
373 UNREACHABLE();
374 }
375}
376
377void LiftoffAssembler::Spill(uint32_t index, WasmValue value) {
378 RecordUsedSpillSlot(index);
379 Operand dst = liftoff::GetStackSlot(index);
380 switch (value.type()) {
381 case kWasmI32:
382 movl(dst, Immediate(value.to_i32()));
383 break;
384 case kWasmI64: {
385 if (is_int32(value.to_i64())) {
386 // Sign extend low word.
387 movq(dst, Immediate(static_cast<int32_t>(value.to_i64())));
388 } else if (is_uint32(value.to_i64())) {
389 // Zero extend low word.
390 movl(kScratchRegister, Immediate(static_cast<int32_t>(value.to_i64())));
391 movq(dst, kScratchRegister);
392 } else {
393 movq(kScratchRegister, value.to_i64());
394 movq(dst, kScratchRegister);
395 }
396 break;
397 }
398 default:
399 // We do not track f32 and f64 constants, hence they are unreachable.
400 UNREACHABLE();
401 }
402}
403
404void LiftoffAssembler::Fill(LiftoffRegister reg, uint32_t index,
405 ValueType type) {
406 Operand src = liftoff::GetStackSlot(index);
407 switch (type) {
408 case kWasmI32:
409 movl(reg.gp(), src);
410 break;
411 case kWasmI64:
412 movq(reg.gp(), src);
413 break;
414 case kWasmF32:
415 Movss(reg.fp(), src);
416 break;
417 case kWasmF64:
418 Movsd(reg.fp(), src);
419 break;
420 default:
421 UNREACHABLE();
422 }
423}
424
425void LiftoffAssembler::FillI64Half(Register, uint32_t index, RegPairHalf) {
426 UNREACHABLE();
427}
428
429void LiftoffAssembler::emit_i32_add(Register dst, Register lhs, Register rhs) {
430 if (lhs != dst) {
431 leal(dst, Operand(lhs, rhs, times_1, 0));
432 } else {
433 addl(dst, rhs);
434 }
435}
436
437void LiftoffAssembler::emit_i32_add(Register dst, Register lhs, int32_t imm) {
438 if (lhs != dst) {
439 leal(dst, Operand(lhs, imm));
440 } else {
441 addl(dst, Immediate(imm));
442 }
443}
444
445void LiftoffAssembler::emit_i32_sub(Register dst, Register lhs, Register rhs) {
446 if (dst != rhs) {
447 // Default path.
448 if (dst != lhs) movl(dst, lhs);
449 subl(dst, rhs);
450 } else if (lhs == rhs) {
451 // Degenerate case.
452 xorl(dst, dst);
453 } else {
454 // Emit {dst = lhs + -rhs} if dst == rhs.
455 negl(dst);
456 addl(dst, lhs);
457 }
458}
459
460namespace liftoff {
461template <void (Assembler::*op)(Register, Register),
462 void (Assembler::*mov)(Register, Register)>
463void EmitCommutativeBinOp(LiftoffAssembler* assm, Register dst, Register lhs,
464 Register rhs) {
465 if (dst == rhs) {
466 (assm->*op)(dst, lhs);
467 } else {
468 if (dst != lhs) (assm->*mov)(dst, lhs);
469 (assm->*op)(dst, rhs);
470 }
471}
472} // namespace liftoff
473
474void LiftoffAssembler::emit_i32_mul(Register dst, Register lhs, Register rhs) {
475 liftoff::EmitCommutativeBinOp<&Assembler::imull, &Assembler::movl>(this, dst,
476 lhs, rhs);
477}
478
479namespace liftoff {
480enum class DivOrRem : uint8_t { kDiv, kRem };
481template <typename type, DivOrRem div_or_rem>
482void EmitIntDivOrRem(LiftoffAssembler* assm, Register dst, Register lhs,
483 Register rhs, Label* trap_div_by_zero,
484 Label* trap_div_unrepresentable) {
485 constexpr bool needs_unrepresentable_check =
486 std::is_signed<type>::value && div_or_rem == DivOrRem::kDiv;
487 constexpr bool special_case_minus_1 =
488 std::is_signed<type>::value && div_or_rem == DivOrRem::kRem;
489 DCHECK_EQ(needs_unrepresentable_check, trap_div_unrepresentable != nullptr);
490
491#define iop(name, ...) \
492 do { \
493 if (sizeof(type) == 4) { \
494 assm->name##l(__VA_ARGS__); \
495 } else { \
496 assm->name##q(__VA_ARGS__); \
497 } \
498 } while (false)
499
500 // For division, the lhs is always taken from {edx:eax}. Thus, make sure that
501 // these registers are unused. If {rhs} is stored in one of them, move it to
502 // another temporary register.
503 // Do all this before any branch, such that the code is executed
504 // unconditionally, as the cache state will also be modified unconditionally.
505 liftoff::SpillRegisters(assm, rdx, rax);
506 if (rhs == rax || rhs == rdx) {
507 iop(mov, kScratchRegister, rhs);
508 rhs = kScratchRegister;
509 }
510
511 // Check for division by zero.
512 iop(test, rhs, rhs);
513 assm->j(zero, trap_div_by_zero);
514
515 Label done;
516 if (needs_unrepresentable_check) {
517 // Check for {kMinInt / -1}. This is unrepresentable.
518 Label do_div;
519 iop(cmp, rhs, Immediate(-1));
520 assm->j(not_equal, &do_div);
521 // {lhs} is min int if {lhs - 1} overflows.
522 iop(cmp, lhs, Immediate(1));
523 assm->j(overflow, trap_div_unrepresentable);
524 assm->bind(&do_div);
525 } else if (special_case_minus_1) {
526 // {lhs % -1} is always 0 (needs to be special cased because {kMinInt / -1}
527 // cannot be computed).
528 Label do_rem;
529 iop(cmp, rhs, Immediate(-1));
530 assm->j(not_equal, &do_rem);
531 // clang-format off
532 // (conflicts with presubmit checks because it is confused about "xor")
533 iop(xor, dst, dst);
534 // clang-format on
535 assm->jmp(&done);
536 assm->bind(&do_rem);
537 }
538
539 // Now move {lhs} into {eax}, then zero-extend or sign-extend into {edx}, then
540 // do the division.
541 if (lhs != rax) iop(mov, rax, lhs);
542 if (std::is_same<int32_t, type>::value) { // i32
543 assm->cdq();
544 assm->idivl(rhs);
545 } else if (std::is_same<uint32_t, type>::value) { // u32
546 assm->xorl(rdx, rdx);
547 assm->divl(rhs);
548 } else if (std::is_same<int64_t, type>::value) { // i64
549 assm->cqo();
550 assm->idivq(rhs);
551 } else { // u64
552 assm->xorq(rdx, rdx);
553 assm->divq(rhs);
554 }
555
556 // Move back the result (in {eax} or {edx}) into the {dst} register.
557 constexpr Register kResultReg = div_or_rem == DivOrRem::kDiv ? rax : rdx;
558 if (dst != kResultReg) {
559 iop(mov, dst, kResultReg);
560 }
561 if (special_case_minus_1) assm->bind(&done);
562}
563} // namespace liftoff
564
565void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs,
566 Label* trap_div_by_zero,
567 Label* trap_div_unrepresentable) {
568 liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kDiv>(
569 this, dst, lhs, rhs, trap_div_by_zero, trap_div_unrepresentable);
570}
571
572void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs,
573 Label* trap_div_by_zero) {
574 liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kDiv>(
575 this, dst, lhs, rhs, trap_div_by_zero, nullptr);
576}
577
578void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs,
579 Label* trap_div_by_zero) {
580 liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kRem>(
581 this, dst, lhs, rhs, trap_div_by_zero, nullptr);
582}
583
584void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs,
585 Label* trap_div_by_zero) {
586 liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kRem>(
587 this, dst, lhs, rhs, trap_div_by_zero, nullptr);
588}
589
590void LiftoffAssembler::emit_i32_and(Register dst, Register lhs, Register rhs) {
591 liftoff::EmitCommutativeBinOp<&Assembler::andl, &Assembler::movl>(this, dst,
592 lhs, rhs);
593}
594
595void LiftoffAssembler::emit_i32_or(Register dst, Register lhs, Register rhs) {
596 liftoff::EmitCommutativeBinOp<&Assembler::orl, &Assembler::movl>(this, dst,
597 lhs, rhs);
598}
599
600void LiftoffAssembler::emit_i32_xor(Register dst, Register lhs, Register rhs) {
601 liftoff::EmitCommutativeBinOp<&Assembler::xorl, &Assembler::movl>(this, dst,
602 lhs, rhs);
603}
604
605namespace liftoff {
606template <ValueType type>
607inline void EmitShiftOperation(LiftoffAssembler* assm, Register dst,
608 Register src, Register amount,
609 void (Assembler::*emit_shift)(Register),
610 LiftoffRegList pinned) {
611 // If dst is rcx, compute into the scratch register first, then move to rcx.
612 if (dst == rcx) {
613 assm->Move(kScratchRegister, src, type);
614 if (amount != rcx) assm->Move(rcx, amount, type);
615 (assm->*emit_shift)(kScratchRegister);
616 assm->Move(rcx, kScratchRegister, type);
617 return;
618 }
619
620 // Move amount into rcx. If rcx is in use, move its content into the scratch
621 // register. If src is rcx, src is now the scratch register.
622 bool use_scratch = false;
623 if (amount != rcx) {
624 use_scratch = src == rcx ||
625 assm->cache_state()->is_used(LiftoffRegister(rcx)) ||
626 pinned.has(LiftoffRegister(rcx));
627 if (use_scratch) assm->movq(kScratchRegister, rcx);
628 if (src == rcx) src = kScratchRegister;
629 assm->Move(rcx, amount, type);
630 }
631
632 // Do the actual shift.
633 if (dst != src) assm->Move(dst, src, type);
634 (assm->*emit_shift)(dst);
635
636 // Restore rcx if needed.
637 if (use_scratch) assm->movq(rcx, kScratchRegister);
638}
639} // namespace liftoff
640
641void LiftoffAssembler::emit_i32_shl(Register dst, Register src, Register amount,
642 LiftoffRegList pinned) {
643 liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
644 &Assembler::shll_cl, pinned);
645}
646
647void LiftoffAssembler::emit_i32_sar(Register dst, Register src, Register amount,
648 LiftoffRegList pinned) {
649 liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
650 &Assembler::sarl_cl, pinned);
651}
652
653void LiftoffAssembler::emit_i32_shr(Register dst, Register src, Register amount,
654 LiftoffRegList pinned) {
655 liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
656 &Assembler::shrl_cl, pinned);
657}
658
659void LiftoffAssembler::emit_i32_shr(Register dst, Register src, int amount) {
660 if (dst != src) movl(dst, src);
661 DCHECK(is_uint5(amount));
662 shrl(dst, Immediate(amount));
663}
664
665bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) {
666 Label nonzero_input;
667 Label continuation;
668 testl(src, src);
669 j(not_zero, &nonzero_input, Label::kNear);
670 movl(dst, Immediate(32));
671 jmp(&continuation, Label::kNear);
672
673 bind(&nonzero_input);
674 // Get most significant bit set (MSBS).
675 bsrl(dst, src);
676 // CLZ = 31 - MSBS = MSBS ^ 31.
677 xorl(dst, Immediate(31));
678
679 bind(&continuation);
680 return true;
681}
682
683bool LiftoffAssembler::emit_i32_ctz(Register dst, Register src) {
684 Label nonzero_input;
685 Label continuation;
686 testl(src, src);
687 j(not_zero, &nonzero_input, Label::kNear);
688 movl(dst, Immediate(32));
689 jmp(&continuation, Label::kNear);
690
691 bind(&nonzero_input);
692 // Get least significant bit set, which equals number of trailing zeros.
693 bsfl(dst, src);
694
695 bind(&continuation);
696 return true;
697}
698
699bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) {
700 if (!CpuFeatures::IsSupported(POPCNT)) return false;
701 CpuFeatureScope scope(this, POPCNT);
702 popcntl(dst, src);
703 return true;
704}
705
706void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs,
707 LiftoffRegister rhs) {
708 if (lhs.gp() != dst.gp()) {
709 leaq(dst.gp(), Operand(lhs.gp(), rhs.gp(), times_1, 0));
710 } else {
711 addq(dst.gp(), rhs.gp());
712 }
713}
714
715void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs,
716 int32_t imm) {
717 if (lhs.gp() != dst.gp()) {
718 leaq(dst.gp(), Operand(lhs.gp(), imm));
719 } else {
720 addq(dst.gp(), Immediate(imm));
721 }
722}
723
724void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs,
725 LiftoffRegister rhs) {
726 if (dst.gp() == rhs.gp()) {
727 negq(dst.gp());
728 addq(dst.gp(), lhs.gp());
729 } else {
730 if (dst.gp() != lhs.gp()) movq(dst.gp(), lhs.gp());
731 subq(dst.gp(), rhs.gp());
732 }
733}
734
735void LiftoffAssembler::emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs,
736 LiftoffRegister rhs) {
737 liftoff::EmitCommutativeBinOp<&Assembler::imulq, &Assembler::movq>(
738 this, dst.gp(), lhs.gp(), rhs.gp());
739}
740
741bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs,
742 LiftoffRegister rhs,
743 Label* trap_div_by_zero,
744 Label* trap_div_unrepresentable) {
745 liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kDiv>(
746 this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero,
747 trap_div_unrepresentable);
748 return true;
749}
750
751bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs,
752 LiftoffRegister rhs,
753 Label* trap_div_by_zero) {
754 liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kDiv>(
755 this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
756 return true;
757}
758
759bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs,
760 LiftoffRegister rhs,
761 Label* trap_div_by_zero) {
762 liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kRem>(
763 this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
764 return true;
765}
766
767bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs,
768 LiftoffRegister rhs,
769 Label* trap_div_by_zero) {
770 liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kRem>(
771 this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
772 return true;
773}
774
775void LiftoffAssembler::emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs,
776 LiftoffRegister rhs) {
777 liftoff::EmitCommutativeBinOp<&Assembler::andq, &Assembler::movq>(
778 this, dst.gp(), lhs.gp(), rhs.gp());
779}
780
781void LiftoffAssembler::emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs,
782 LiftoffRegister rhs) {
783 liftoff::EmitCommutativeBinOp<&Assembler::orq, &Assembler::movq>(
784 this, dst.gp(), lhs.gp(), rhs.gp());
785}
786
787void LiftoffAssembler::emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs,
788 LiftoffRegister rhs) {
789 liftoff::EmitCommutativeBinOp<&Assembler::xorq, &Assembler::movq>(
790 this, dst.gp(), lhs.gp(), rhs.gp());
791}
792
793void LiftoffAssembler::emit_i64_shl(LiftoffRegister dst, LiftoffRegister src,
794 Register amount, LiftoffRegList pinned) {
795 liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
796 &Assembler::shlq_cl, pinned);
797}
798
799void LiftoffAssembler::emit_i64_sar(LiftoffRegister dst, LiftoffRegister src,
800 Register amount, LiftoffRegList pinned) {
801 liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
802 &Assembler::sarq_cl, pinned);
803}
804
805void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src,
806 Register amount, LiftoffRegList pinned) {
807 liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
808 &Assembler::shrq_cl, pinned);
809}
810
811void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src,
812 int amount) {
813 if (dst.gp() != src.gp()) movl(dst.gp(), src.gp());
814 DCHECK(is_uint6(amount));
815 shrq(dst.gp(), Immediate(amount));
816}
817
818void LiftoffAssembler::emit_i32_to_intptr(Register dst, Register src) {
819 movsxlq(dst, src);
820}
821
822void LiftoffAssembler::emit_f32_add(DoubleRegister dst, DoubleRegister lhs,
823 DoubleRegister rhs) {
824 if (CpuFeatures::IsSupported(AVX)) {
825 CpuFeatureScope scope(this, AVX);
826 vaddss(dst, lhs, rhs);
827 } else if (dst == rhs) {
828 addss(dst, lhs);
829 } else {
830 if (dst != lhs) movss(dst, lhs);
831 addss(dst, rhs);
832 }
833}
834
835void LiftoffAssembler::emit_f32_sub(DoubleRegister dst, DoubleRegister lhs,
836 DoubleRegister rhs) {
837 if (CpuFeatures::IsSupported(AVX)) {
838 CpuFeatureScope scope(this, AVX);
839 vsubss(dst, lhs, rhs);
840 } else if (dst == rhs) {
841 movss(kScratchDoubleReg, rhs);
842 movss(dst, lhs);
843 subss(dst, kScratchDoubleReg);
844 } else {
845 if (dst != lhs) movss(dst, lhs);
846 subss(dst, rhs);
847 }
848}
849
850void LiftoffAssembler::emit_f32_mul(DoubleRegister dst, DoubleRegister lhs,
851 DoubleRegister rhs) {
852 if (CpuFeatures::IsSupported(AVX)) {
853 CpuFeatureScope scope(this, AVX);
854 vmulss(dst, lhs, rhs);
855 } else if (dst == rhs) {
856 mulss(dst, lhs);
857 } else {
858 if (dst != lhs) movss(dst, lhs);
859 mulss(dst, rhs);
860 }
861}
862
863void LiftoffAssembler::emit_f32_div(DoubleRegister dst, DoubleRegister lhs,
864 DoubleRegister rhs) {
865 if (CpuFeatures::IsSupported(AVX)) {
866 CpuFeatureScope scope(this, AVX);
867 vdivss(dst, lhs, rhs);
868 } else if (dst == rhs) {
869 movss(kScratchDoubleReg, rhs);
870 movss(dst, lhs);
871 divss(dst, kScratchDoubleReg);
872 } else {
873 if (dst != lhs) movss(dst, lhs);
874 divss(dst, rhs);
875 }
876}
877
878namespace liftoff {
879enum class MinOrMax : uint8_t { kMin, kMax };
880template <typename type>
881inline void EmitFloatMinOrMax(LiftoffAssembler* assm, DoubleRegister dst,
882 DoubleRegister lhs, DoubleRegister rhs,
883 MinOrMax min_or_max) {
884 Label is_nan;
885 Label lhs_below_rhs;
886 Label lhs_above_rhs;
887 Label done;
888
889#define dop(name, ...) \
890 do { \
891 if (sizeof(type) == 4) { \
892 assm->name##s(__VA_ARGS__); \
893 } else { \
894 assm->name##d(__VA_ARGS__); \
895 } \
896 } while (false)
897
898 // Check the easy cases first: nan (e.g. unordered), smaller and greater.
899 // NaN has to be checked first, because PF=1 implies CF=1.
900 dop(Ucomis, lhs, rhs);
901 assm->j(parity_even, &is_nan, Label::kNear); // PF=1
902 assm->j(below, &lhs_below_rhs, Label::kNear); // CF=1
903 assm->j(above, &lhs_above_rhs, Label::kNear); // CF=0 && ZF=0
904
905 // If we get here, then either
906 // a) {lhs == rhs},
907 // b) {lhs == -0.0} and {rhs == 0.0}, or
908 // c) {lhs == 0.0} and {rhs == -0.0}.
909 // For a), it does not matter whether we return {lhs} or {rhs}. Check the sign
910 // bit of {rhs} to differentiate b) and c).
911 dop(Movmskp, kScratchRegister, rhs);
912 assm->testl(kScratchRegister, Immediate(1));
913 assm->j(zero, &lhs_below_rhs, Label::kNear);
914 assm->jmp(&lhs_above_rhs, Label::kNear);
915
916 assm->bind(&is_nan);
917 // Create a NaN output.
918 dop(Xorp, dst, dst);
919 dop(Divs, dst, dst);
920 assm->jmp(&done, Label::kNear);
921
922 assm->bind(&lhs_below_rhs);
923 DoubleRegister lhs_below_rhs_src = min_or_max == MinOrMax::kMin ? lhs : rhs;
924 if (dst != lhs_below_rhs_src) dop(Movs, dst, lhs_below_rhs_src);
925 assm->jmp(&done, Label::kNear);
926
927 assm->bind(&lhs_above_rhs);
928 DoubleRegister lhs_above_rhs_src = min_or_max == MinOrMax::kMin ? rhs : lhs;
929 if (dst != lhs_above_rhs_src) dop(Movs, dst, lhs_above_rhs_src);
930
931 assm->bind(&done);
932}
933} // namespace liftoff
934
935void LiftoffAssembler::emit_f32_min(DoubleRegister dst, DoubleRegister lhs,
936 DoubleRegister rhs) {
937 liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs,
938 liftoff::MinOrMax::kMin);
939}
940
941void LiftoffAssembler::emit_f32_max(DoubleRegister dst, DoubleRegister lhs,
942 DoubleRegister rhs) {
943 liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs,
944 liftoff::MinOrMax::kMax);
945}
946
947void LiftoffAssembler::emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs,
948 DoubleRegister rhs) {
949 static constexpr int kF32SignBit = 1 << 31;
950 Movd(kScratchRegister, lhs);
951 andl(kScratchRegister, Immediate(~kF32SignBit));
952 Movd(liftoff::kScratchRegister2, rhs);
953 andl(liftoff::kScratchRegister2, Immediate(kF32SignBit));
954 orl(kScratchRegister, liftoff::kScratchRegister2);
955 Movd(dst, kScratchRegister);
956}
957
958void LiftoffAssembler::emit_f32_abs(DoubleRegister dst, DoubleRegister src) {
959 static constexpr uint32_t kSignBit = uint32_t{1} << 31;
960 if (dst == src) {
961 TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1);
962 Andps(dst, kScratchDoubleReg);
963 } else {
964 TurboAssembler::Move(dst, kSignBit - 1);
965 Andps(dst, src);
966 }
967}
968
969void LiftoffAssembler::emit_f32_neg(DoubleRegister dst, DoubleRegister src) {
970 static constexpr uint32_t kSignBit = uint32_t{1} << 31;
971 if (dst == src) {
972 TurboAssembler::Move(kScratchDoubleReg, kSignBit);
973 Xorps(dst, kScratchDoubleReg);
974 } else {
975 TurboAssembler::Move(dst, kSignBit);
976 Xorps(dst, src);
977 }
978}
979
980bool LiftoffAssembler::emit_f32_ceil(DoubleRegister dst, DoubleRegister src) {
981 if (CpuFeatures::IsSupported(SSE4_1)) {
982 CpuFeatureScope feature(this, SSE4_1);
983 Roundss(dst, src, kRoundUp);
984 return true;
985 }
986 return false;
987}
988
989bool LiftoffAssembler::emit_f32_floor(DoubleRegister dst, DoubleRegister src) {
990 if (CpuFeatures::IsSupported(SSE4_1)) {
991 CpuFeatureScope feature(this, SSE4_1);
992 Roundss(dst, src, kRoundDown);
993 return true;
994 }
995 return false;
996}
997
998bool LiftoffAssembler::emit_f32_trunc(DoubleRegister dst, DoubleRegister src) {
999 if (CpuFeatures::IsSupported(SSE4_1)) {
1000 CpuFeatureScope feature(this, SSE4_1);
1001 Roundss(dst, src, kRoundToZero);
1002 return true;
1003 }
1004 return false;
1005}
1006
1007bool LiftoffAssembler::emit_f32_nearest_int(DoubleRegister dst,
1008 DoubleRegister src) {
1009 if (CpuFeatures::IsSupported(SSE4_1)) {
1010 CpuFeatureScope feature(this, SSE4_1);
1011 Roundss(dst, src, kRoundToNearest);
1012 return true;
1013 }
1014 return false;
1015}
1016
1017void LiftoffAssembler::emit_f32_sqrt(DoubleRegister dst, DoubleRegister src) {
1018 Sqrtss(dst, src);
1019}
1020
1021void LiftoffAssembler::emit_f64_add(DoubleRegister dst, DoubleRegister lhs,
1022 DoubleRegister rhs) {
1023 if (CpuFeatures::IsSupported(AVX)) {
1024 CpuFeatureScope scope(this, AVX);
1025 vaddsd(dst, lhs, rhs);
1026 } else if (dst == rhs) {
1027 addsd(dst, lhs);
1028 } else {
1029 if (dst != lhs) movsd(dst, lhs);
1030 addsd(dst, rhs);
1031 }
1032}
1033
1034void LiftoffAssembler::emit_f64_sub(DoubleRegister dst, DoubleRegister lhs,
1035 DoubleRegister rhs) {
1036 if (CpuFeatures::IsSupported(AVX)) {
1037 CpuFeatureScope scope(this, AVX);
1038 vsubsd(dst, lhs, rhs);
1039 } else if (dst == rhs) {
1040 movsd(kScratchDoubleReg, rhs);
1041 movsd(dst, lhs);
1042 subsd(dst, kScratchDoubleReg);
1043 } else {
1044 if (dst != lhs) movsd(dst, lhs);
1045 subsd(dst, rhs);
1046 }
1047}
1048
1049void LiftoffAssembler::emit_f64_mul(DoubleRegister dst, DoubleRegister lhs,
1050 DoubleRegister rhs) {
1051 if (CpuFeatures::IsSupported(AVX)) {
1052 CpuFeatureScope scope(this, AVX);
1053 vmulsd(dst, lhs, rhs);
1054 } else if (dst == rhs) {
1055 mulsd(dst, lhs);
1056 } else {
1057 if (dst != lhs) movsd(dst, lhs);
1058 mulsd(dst, rhs);
1059 }
1060}
1061
1062void LiftoffAssembler::emit_f64_div(DoubleRegister dst, DoubleRegister lhs,
1063 DoubleRegister rhs) {
1064 if (CpuFeatures::IsSupported(AVX)) {
1065 CpuFeatureScope scope(this, AVX);
1066 vdivsd(dst, lhs, rhs);
1067 } else if (dst == rhs) {
1068 movsd(kScratchDoubleReg, rhs);
1069 movsd(dst, lhs);
1070 divsd(dst, kScratchDoubleReg);
1071 } else {
1072 if (dst != lhs) movsd(dst, lhs);
1073 divsd(dst, rhs);
1074 }
1075}
1076
1077void LiftoffAssembler::emit_f64_min(DoubleRegister dst, DoubleRegister lhs,
1078 DoubleRegister rhs) {
1079 liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs,
1080 liftoff::MinOrMax::kMin);
1081}
1082
1083void LiftoffAssembler::emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs,
1084 DoubleRegister rhs) {
1085 // Extract sign bit from {rhs} into {kScratchRegister2}.
1086 Movq(liftoff::kScratchRegister2, rhs);
1087 shrq(liftoff::kScratchRegister2, Immediate(63));
1088 shlq(liftoff::kScratchRegister2, Immediate(63));
1089 // Reset sign bit of {lhs} (in {kScratchRegister}).
1090 Movq(kScratchRegister, lhs);
1091 btrq(kScratchRegister, Immediate(63));
1092 // Combine both values into {kScratchRegister} and move into {dst}.
1093 orq(kScratchRegister, liftoff::kScratchRegister2);
1094 Movq(dst, kScratchRegister);
1095}
1096
1097void LiftoffAssembler::emit_f64_max(DoubleRegister dst, DoubleRegister lhs,
1098 DoubleRegister rhs) {
1099 liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs,
1100 liftoff::MinOrMax::kMax);
1101}
1102
1103void LiftoffAssembler::emit_f64_abs(DoubleRegister dst, DoubleRegister src) {
1104 static constexpr uint64_t kSignBit = uint64_t{1} << 63;
1105 if (dst == src) {
1106 TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1);
1107 Andpd(dst, kScratchDoubleReg);
1108 } else {
1109 TurboAssembler::Move(dst, kSignBit - 1);
1110 Andpd(dst, src);
1111 }
1112}
1113
1114void LiftoffAssembler::emit_f64_neg(DoubleRegister dst, DoubleRegister src) {
1115 static constexpr uint64_t kSignBit = uint64_t{1} << 63;
1116 if (dst == src) {
1117 TurboAssembler::Move(kScratchDoubleReg, kSignBit);
1118 Xorpd(dst, kScratchDoubleReg);
1119 } else {
1120 TurboAssembler::Move(dst, kSignBit);
1121 Xorpd(dst, src);
1122 }
1123}
1124
1125bool LiftoffAssembler::emit_f64_ceil(DoubleRegister dst, DoubleRegister src) {
1126 REQUIRE_CPU_FEATURE(SSE4_1, true);
1127 Roundsd(dst, src, kRoundUp);
1128 return true;
1129}
1130
1131bool LiftoffAssembler::emit_f64_floor(DoubleRegister dst, DoubleRegister src) {
1132 REQUIRE_CPU_FEATURE(SSE4_1, true);
1133 Roundsd(dst, src, kRoundDown);
1134 return true;
1135}
1136
1137bool LiftoffAssembler::emit_f64_trunc(DoubleRegister dst, DoubleRegister src) {
1138 REQUIRE_CPU_FEATURE(SSE4_1, true);
1139 Roundsd(dst, src, kRoundToZero);
1140 return true;
1141}
1142
1143bool LiftoffAssembler::emit_f64_nearest_int(DoubleRegister dst,
1144 DoubleRegister src) {
1145 REQUIRE_CPU_FEATURE(SSE4_1, true);
1146 Roundsd(dst, src, kRoundToNearest);
1147 return true;
1148}
1149
1150void LiftoffAssembler::emit_f64_sqrt(DoubleRegister dst, DoubleRegister src) {
1151 Sqrtsd(dst, src);
1152}
1153
1154namespace liftoff {
1155// Used for float to int conversions. If the value in {converted_back} equals
1156// {src} afterwards, the conversion succeeded.
1157template <typename dst_type, typename src_type>
1158inline void ConvertFloatToIntAndBack(LiftoffAssembler* assm, Register dst,
1159 DoubleRegister src,
1160 DoubleRegister converted_back) {
1161 if (std::is_same<double, src_type>::value) { // f64
1162 if (std::is_same<int32_t, dst_type>::value) { // f64 -> i32
1163 assm->Cvttsd2si(dst, src);
1164 assm->Cvtlsi2sd(converted_back, dst);
1165 } else if (std::is_same<uint32_t, dst_type>::value) { // f64 -> u32
1166 assm->Cvttsd2siq(dst, src);
1167 assm->movl(dst, dst);
1168 assm->Cvtqsi2sd(converted_back, dst);
1169 } else if (std::is_same<int64_t, dst_type>::value) { // f64 -> i64
1170 assm->Cvttsd2siq(dst, src);
1171 assm->Cvtqsi2sd(converted_back, dst);
1172 } else {
1173 UNREACHABLE();
1174 }
1175 } else { // f32
1176 if (std::is_same<int32_t, dst_type>::value) { // f32 -> i32
1177 assm->Cvttss2si(dst, src);
1178 assm->Cvtlsi2ss(converted_back, dst);
1179 } else if (std::is_same<uint32_t, dst_type>::value) { // f32 -> u32
1180 assm->Cvttss2siq(dst, src);
1181 assm->movl(dst, dst);
1182 assm->Cvtqsi2ss(converted_back, dst);
1183 } else if (std::is_same<int64_t, dst_type>::value) { // f32 -> i64
1184 assm->Cvttss2siq(dst, src);
1185 assm->Cvtqsi2ss(converted_back, dst);
1186 } else {
1187 UNREACHABLE();
1188 }
1189 }
1190}
1191
1192template <typename dst_type, typename src_type>
1193inline bool EmitTruncateFloatToInt(LiftoffAssembler* assm, Register dst,
1194 DoubleRegister src, Label* trap) {
1195 if (!CpuFeatures::IsSupported(SSE4_1)) {
1196 assm->bailout("no SSE4.1");
1197 return true;
1198 }
1199 CpuFeatureScope feature(assm, SSE4_1);
1200
1201 DoubleRegister rounded = kScratchDoubleReg;
1202 DoubleRegister converted_back = kScratchDoubleReg2;
1203
1204 if (std::is_same<double, src_type>::value) { // f64
1205 assm->Roundsd(rounded, src, kRoundToZero);
1206 } else { // f32
1207 assm->Roundss(rounded, src, kRoundToZero);
1208 }
1209 ConvertFloatToIntAndBack<dst_type, src_type>(assm, dst, rounded,
1210 converted_back);
1211 if (std::is_same<double, src_type>::value) { // f64
1212 assm->Ucomisd(converted_back, rounded);
1213 } else { // f32
1214 assm->Ucomiss(converted_back, rounded);
1215 }
1216
1217 // Jump to trap if PF is 0 (one of the operands was NaN) or they are not
1218 // equal.
1219 assm->j(parity_even, trap);
1220 assm->j(not_equal, trap);
1221 return true;
1222}
1223} // namespace liftoff
1224
1225bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
1226 LiftoffRegister dst,
1227 LiftoffRegister src, Label* trap) {
1228 switch (opcode) {
1229 case kExprI32ConvertI64:
1230 movl(dst.gp(), src.gp());
1231 return true;
1232 case kExprI32SConvertF32:
1233 return liftoff::EmitTruncateFloatToInt<int32_t, float>(this, dst.gp(),
1234 src.fp(), trap);
1235 case kExprI32UConvertF32:
1236 return liftoff::EmitTruncateFloatToInt<uint32_t, float>(this, dst.gp(),
1237 src.fp(), trap);
1238 case kExprI32SConvertF64:
1239 return liftoff::EmitTruncateFloatToInt<int32_t, double>(this, dst.gp(),
1240 src.fp(), trap);
1241 case kExprI32UConvertF64:
1242 return liftoff::EmitTruncateFloatToInt<uint32_t, double>(this, dst.gp(),
1243 src.fp(), trap);
1244 case kExprI32ReinterpretF32:
1245 Movd(dst.gp(), src.fp());
1246 return true;
1247 case kExprI64SConvertI32:
1248 movsxlq(dst.gp(), src.gp());
1249 return true;
1250 case kExprI64SConvertF32:
1251 return liftoff::EmitTruncateFloatToInt<int64_t, float>(this, dst.gp(),
1252 src.fp(), trap);
1253 case kExprI64UConvertF32: {
1254 REQUIRE_CPU_FEATURE(SSE4_1, true);
1255 Cvttss2uiq(dst.gp(), src.fp(), trap);
1256 return true;
1257 }
1258 case kExprI64SConvertF64:
1259 return liftoff::EmitTruncateFloatToInt<int64_t, double>(this, dst.gp(),
1260 src.fp(), trap);
1261 case kExprI64UConvertF64: {
1262 REQUIRE_CPU_FEATURE(SSE4_1, true);
1263 Cvttsd2uiq(dst.gp(), src.fp(), trap);
1264 return true;
1265 }
1266 case kExprI64UConvertI32:
1267 AssertZeroExtended(src.gp());
1268 if (dst.gp() != src.gp()) movl(dst.gp(), src.gp());
1269 return true;
1270 case kExprI64ReinterpretF64:
1271 Movq(dst.gp(), src.fp());
1272 return true;
1273 case kExprF32SConvertI32:
1274 Cvtlsi2ss(dst.fp(), src.gp());
1275 return true;
1276 case kExprF32UConvertI32:
1277 movl(kScratchRegister, src.gp());
1278 Cvtqsi2ss(dst.fp(), kScratchRegister);
1279 return true;
1280 case kExprF32SConvertI64:
1281 Cvtqsi2ss(dst.fp(), src.gp());
1282 return true;
1283 case kExprF32UConvertI64:
1284 Cvtqui2ss(dst.fp(), src.gp());
1285 return true;
1286 case kExprF32ConvertF64:
1287 Cvtsd2ss(dst.fp(), src.fp());
1288 return true;
1289 case kExprF32ReinterpretI32:
1290 Movd(dst.fp(), src.gp());
1291 return true;
1292 case kExprF64SConvertI32:
1293 Cvtlsi2sd(dst.fp(), src.gp());
1294 return true;
1295 case kExprF64UConvertI32:
1296 movl(kScratchRegister, src.gp());
1297 Cvtqsi2sd(dst.fp(), kScratchRegister);
1298 return true;
1299 case kExprF64SConvertI64:
1300 Cvtqsi2sd(dst.fp(), src.gp());
1301 return true;
1302 case kExprF64UConvertI64:
1303 Cvtqui2sd(dst.fp(), src.gp());
1304 return true;
1305 case kExprF64ConvertF32:
1306 Cvtss2sd(dst.fp(), src.fp());
1307 return true;
1308 case kExprF64ReinterpretI64:
1309 Movq(dst.fp(), src.gp());
1310 return true;
1311 default:
1312 UNREACHABLE();
1313 }
1314}
1315
1316void LiftoffAssembler::emit_i32_signextend_i8(Register dst, Register src) {
1317 movsxbl(dst, src);
1318}
1319
1320void LiftoffAssembler::emit_i32_signextend_i16(Register dst, Register src) {
1321 movsxwl(dst, src);
1322}
1323
1324void LiftoffAssembler::emit_i64_signextend_i8(LiftoffRegister dst,
1325 LiftoffRegister src) {
1326 movsxbq(dst.gp(), src.gp());
1327}
1328
1329void LiftoffAssembler::emit_i64_signextend_i16(LiftoffRegister dst,
1330 LiftoffRegister src) {
1331 movsxwq(dst.gp(), src.gp());
1332}
1333
1334void LiftoffAssembler::emit_i64_signextend_i32(LiftoffRegister dst,
1335 LiftoffRegister src) {
1336 movsxlq(dst.gp(), src.gp());
1337}
1338
1339void LiftoffAssembler::emit_jump(Label* label) { jmp(label); }
1340
1341void LiftoffAssembler::emit_jump(Register target) { jmp(target); }
1342
1343void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
1344 ValueType type, Register lhs,
1345 Register rhs) {
1346 if (rhs != no_reg) {
1347 switch (type) {
1348 case kWasmI32:
1349 cmpl(lhs, rhs);
1350 break;
1351 case kWasmI64:
1352 cmpq(lhs, rhs);
1353 break;
1354 default:
1355 UNREACHABLE();
1356 }
1357 } else {
1358 DCHECK_EQ(type, kWasmI32);
1359 testl(lhs, lhs);
1360 }
1361
1362 j(cond, label);
1363}
1364
1365void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) {
1366 testl(src, src);
1367 setcc(equal, dst);
1368 movzxbl(dst, dst);
1369}
1370
1371void LiftoffAssembler::emit_i32_set_cond(Condition cond, Register dst,
1372 Register lhs, Register rhs) {
1373 cmpl(lhs, rhs);
1374 setcc(cond, dst);
1375 movzxbl(dst, dst);
1376}
1377
1378void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) {
1379 testq(src.gp(), src.gp());
1380 setcc(equal, dst);
1381 movzxbl(dst, dst);
1382}
1383
1384void LiftoffAssembler::emit_i64_set_cond(Condition cond, Register dst,
1385 LiftoffRegister lhs,
1386 LiftoffRegister rhs) {
1387 cmpq(lhs.gp(), rhs.gp());
1388 setcc(cond, dst);
1389 movzxbl(dst, dst);
1390}
1391
1392namespace liftoff {
1393template <void (TurboAssembler::*cmp_op)(DoubleRegister, DoubleRegister)>
1394void EmitFloatSetCond(LiftoffAssembler* assm, Condition cond, Register dst,
1395 DoubleRegister lhs, DoubleRegister rhs) {
1396 Label cont;
1397 Label not_nan;
1398
1399 (assm->*cmp_op)(lhs, rhs);
1400 // If PF is one, one of the operands was NaN. This needs special handling.
1401 assm->j(parity_odd, &not_nan, Label::kNear);
1402 // Return 1 for f32.ne, 0 for all other cases.
1403 if (cond == not_equal) {
1404 assm->movl(dst, Immediate(1));
1405 } else {
1406 assm->xorl(dst, dst);
1407 }
1408 assm->jmp(&cont, Label::kNear);
1409 assm->bind(&not_nan);
1410
1411 assm->setcc(cond, dst);
1412 assm->movzxbl(dst, dst);
1413 assm->bind(&cont);
1414}
1415} // namespace liftoff
1416
1417void LiftoffAssembler::emit_f32_set_cond(Condition cond, Register dst,
1418 DoubleRegister lhs,
1419 DoubleRegister rhs) {
1420 liftoff::EmitFloatSetCond<&TurboAssembler::Ucomiss>(this, cond, dst, lhs,
1421 rhs);
1422}
1423
1424void LiftoffAssembler::emit_f64_set_cond(Condition cond, Register dst,
1425 DoubleRegister lhs,
1426 DoubleRegister rhs) {
1427 liftoff::EmitFloatSetCond<&TurboAssembler::Ucomisd>(this, cond, dst, lhs,
1428 rhs);
1429}
1430
1431void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) {
1432 cmpq(rsp, Operand(limit_address, 0));
1433 j(below_equal, ool_code);
1434}
1435
1436void LiftoffAssembler::CallTrapCallbackForTesting() {
1437 PrepareCallCFunction(0);
1438 CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0);
1439}
1440
1441void LiftoffAssembler::AssertUnreachable(AbortReason reason) {
1442 TurboAssembler::AssertUnreachable(reason);
1443}
1444
1445void LiftoffAssembler::PushRegisters(LiftoffRegList regs) {
1446 LiftoffRegList gp_regs = regs & kGpCacheRegList;
1447 while (!gp_regs.is_empty()) {
1448 LiftoffRegister reg = gp_regs.GetFirstRegSet();
1449 pushq(reg.gp());
1450 gp_regs.clear(reg);
1451 }
1452 LiftoffRegList fp_regs = regs & kFpCacheRegList;
1453 unsigned num_fp_regs = fp_regs.GetNumRegsSet();
1454 if (num_fp_regs) {
1455 subq(rsp, Immediate(num_fp_regs * kStackSlotSize));
1456 unsigned offset = 0;
1457 while (!fp_regs.is_empty()) {
1458 LiftoffRegister reg = fp_regs.GetFirstRegSet();
1459 Movsd(Operand(rsp, offset), reg.fp());
1460 fp_regs.clear(reg);
1461 offset += sizeof(double);
1462 }
1463 DCHECK_EQ(offset, num_fp_regs * sizeof(double));
1464 }
1465}
1466
1467void LiftoffAssembler::PopRegisters(LiftoffRegList regs) {
1468 LiftoffRegList fp_regs = regs & kFpCacheRegList;
1469 unsigned fp_offset = 0;
1470 while (!fp_regs.is_empty()) {
1471 LiftoffRegister reg = fp_regs.GetFirstRegSet();
1472 Movsd(reg.fp(), Operand(rsp, fp_offset));
1473 fp_regs.clear(reg);
1474 fp_offset += sizeof(double);
1475 }
1476 if (fp_offset) addq(rsp, Immediate(fp_offset));
1477 LiftoffRegList gp_regs = regs & kGpCacheRegList;
1478 while (!gp_regs.is_empty()) {
1479 LiftoffRegister reg = gp_regs.GetLastRegSet();
1480 popq(reg.gp());
1481 gp_regs.clear(reg);
1482 }
1483}
1484
1485void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) {
1486 DCHECK_LT(num_stack_slots,
1487 (1 << 16) / kSystemPointerSize); // 16 bit immediate
1488 ret(static_cast<int>(num_stack_slots * kSystemPointerSize));
1489}
1490
1491void LiftoffAssembler::CallC(wasm::FunctionSig* sig,
1492 const LiftoffRegister* args,
1493 const LiftoffRegister* rets,
1494 ValueType out_argument_type, int stack_bytes,
1495 ExternalReference ext_ref) {
1496 subq(rsp, Immediate(stack_bytes));
1497
1498 int arg_bytes = 0;
1499 for (ValueType param_type : sig->parameters()) {
1500 liftoff::Store(this, Operand(rsp, arg_bytes), *args++, param_type);
1501 arg_bytes += ValueTypes::MemSize(param_type);
1502 }
1503 DCHECK_LE(arg_bytes, stack_bytes);
1504
1505 // Pass a pointer to the buffer with the arguments to the C function.
1506 movq(arg_reg_1, rsp);
1507
1508 constexpr int kNumCCallArgs = 1;
1509
1510 // Now call the C function.
1511 PrepareCallCFunction(kNumCCallArgs);
1512 CallCFunction(ext_ref, kNumCCallArgs);
1513
1514 // Move return value to the right register.
1515 const LiftoffRegister* next_result_reg = rets;
1516 if (sig->return_count() > 0) {
1517 DCHECK_EQ(1, sig->return_count());
1518 constexpr Register kReturnReg = rax;
1519 if (kReturnReg != next_result_reg->gp()) {
1520 Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0));
1521 }
1522 ++next_result_reg;
1523 }
1524
1525 // Load potential output value from the buffer on the stack.
1526 if (out_argument_type != kWasmStmt) {
1527 liftoff::Load(this, *next_result_reg, Operand(rsp, 0), out_argument_type);
1528 }
1529
1530 addq(rsp, Immediate(stack_bytes));
1531}
1532
1533void LiftoffAssembler::CallNativeWasmCode(Address addr) {
1534 near_call(addr, RelocInfo::WASM_CALL);
1535}
1536
1537void LiftoffAssembler::CallIndirect(wasm::FunctionSig* sig,
1538 compiler::CallDescriptor* call_descriptor,
1539 Register target) {
1540 if (target == no_reg) {
1541 popq(kScratchRegister);
1542 target = kScratchRegister;
1543 }
1544 if (FLAG_untrusted_code_mitigations) {
1545 RetpolineCall(target);
1546 } else {
1547 call(target);
1548 }
1549}
1550
1551void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) {
1552 // A direct call to a wasm runtime stub defined in this module.
1553 // Just encode the stub index. This will be patched at relocation.
1554 near_call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL);
1555}
1556
1557void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) {
1558 subq(rsp, Immediate(size));
1559 movq(addr, rsp);
1560}
1561
1562void LiftoffAssembler::DeallocateStackSlot(uint32_t size) {
1563 addq(rsp, Immediate(size));
1564}
1565
1566void LiftoffStackSlots::Construct() {
1567 for (auto& slot : slots_) {
1568 const LiftoffAssembler::VarState& src = slot.src_;
1569 switch (src.loc()) {
1570 case LiftoffAssembler::VarState::kStack:
1571 if (src.type() == kWasmI32) {
1572 // Load i32 values to a register first to ensure they are zero
1573 // extended.
1574 asm_->movl(kScratchRegister, liftoff::GetStackSlot(slot.src_index_));
1575 asm_->pushq(kScratchRegister);
1576 } else {
1577 // For all other types, just push the whole (8-byte) stack slot.
1578 // This is also ok for f32 values (even though we copy 4 uninitialized
1579 // bytes), because f32 and f64 values are clearly distinguished in
1580 // Turbofan, so the uninitialized bytes are never accessed.
1581 asm_->pushq(liftoff::GetStackSlot(slot.src_index_));
1582 }
1583 break;
1584 case LiftoffAssembler::VarState::kRegister:
1585 liftoff::push(asm_, src.reg(), src.type());
1586 break;
1587 case LiftoffAssembler::VarState::kIntConst:
1588 asm_->pushq(Immediate(src.i32_const()));
1589 break;
1590 }
1591 }
1592}
1593
1594#undef REQUIRE_CPU_FEATURE
1595
1596} // namespace wasm
1597} // namespace internal
1598} // namespace v8
1599
1600#endif // V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
1601