1 | // Copyright 2017 the V8 project authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | #ifndef V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_ |
6 | #define V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_ |
7 | |
8 | #include "src/wasm/baseline/liftoff-assembler.h" |
9 | |
10 | #include "src/assembler.h" |
11 | #include "src/wasm/value-type.h" |
12 | |
13 | namespace v8 { |
14 | namespace internal { |
15 | namespace wasm { |
16 | |
17 | #define REQUIRE_CPU_FEATURE(name, ...) \ |
18 | if (!CpuFeatures::IsSupported(name)) { \ |
19 | bailout("no " #name); \ |
20 | return __VA_ARGS__; \ |
21 | } \ |
22 | CpuFeatureScope feature(this, name); |
23 | |
24 | namespace liftoff { |
25 | |
26 | constexpr Register kScratchRegister2 = r11; |
27 | static_assert(kScratchRegister != kScratchRegister2, "collision" ); |
28 | static_assert((kLiftoffAssemblerGpCacheRegs & |
29 | Register::ListOf<kScratchRegister, kScratchRegister2>()) == 0, |
30 | "scratch registers must not be used as cache registers" ); |
31 | |
32 | constexpr DoubleRegister kScratchDoubleReg2 = xmm14; |
33 | static_assert(kScratchDoubleReg != kScratchDoubleReg2, "collision" ); |
34 | static_assert( |
35 | (kLiftoffAssemblerFpCacheRegs & |
36 | DoubleRegister::ListOf<kScratchDoubleReg, kScratchDoubleReg2>()) == 0, |
37 | "scratch registers must not be used as cache registers" ); |
38 | |
39 | // rbp-8 holds the stack marker, rbp-16 is the instance parameter, first stack |
40 | // slot is located at rbp-24. |
41 | constexpr int32_t kConstantStackSpace = 16; |
42 | constexpr int32_t kFirstStackSlotOffset = |
43 | kConstantStackSpace + LiftoffAssembler::kStackSlotSize; |
44 | |
45 | inline Operand GetStackSlot(uint32_t index) { |
46 | int32_t offset = index * LiftoffAssembler::kStackSlotSize; |
47 | return Operand(rbp, -kFirstStackSlotOffset - offset); |
48 | } |
49 | |
50 | // TODO(clemensh): Make this a constexpr variable once Operand is constexpr. |
51 | inline Operand GetInstanceOperand() { return Operand(rbp, -16); } |
52 | |
53 | inline Operand GetMemOp(LiftoffAssembler* assm, Register addr, Register offset, |
54 | uint32_t offset_imm) { |
55 | if (is_uint31(offset_imm)) { |
56 | if (offset == no_reg) return Operand(addr, offset_imm); |
57 | return Operand(addr, offset, times_1, offset_imm); |
58 | } |
59 | // Offset immediate does not fit in 31 bits. |
60 | Register scratch = kScratchRegister; |
61 | assm->movl(scratch, Immediate(offset_imm)); |
62 | if (offset != no_reg) { |
63 | assm->addq(scratch, offset); |
64 | } |
65 | return Operand(addr, scratch, times_1, 0); |
66 | } |
67 | |
68 | inline void Load(LiftoffAssembler* assm, LiftoffRegister dst, Operand src, |
69 | ValueType type) { |
70 | switch (type) { |
71 | case kWasmI32: |
72 | assm->movl(dst.gp(), src); |
73 | break; |
74 | case kWasmI64: |
75 | assm->movq(dst.gp(), src); |
76 | break; |
77 | case kWasmF32: |
78 | assm->Movss(dst.fp(), src); |
79 | break; |
80 | case kWasmF64: |
81 | assm->Movsd(dst.fp(), src); |
82 | break; |
83 | default: |
84 | UNREACHABLE(); |
85 | } |
86 | } |
87 | |
88 | inline void Store(LiftoffAssembler* assm, Operand dst, LiftoffRegister src, |
89 | ValueType type) { |
90 | switch (type) { |
91 | case kWasmI32: |
92 | assm->movl(dst, src.gp()); |
93 | break; |
94 | case kWasmI64: |
95 | assm->movq(dst, src.gp()); |
96 | break; |
97 | case kWasmF32: |
98 | assm->Movss(dst, src.fp()); |
99 | break; |
100 | case kWasmF64: |
101 | assm->Movsd(dst, src.fp()); |
102 | break; |
103 | default: |
104 | UNREACHABLE(); |
105 | } |
106 | } |
107 | |
108 | inline void push(LiftoffAssembler* assm, LiftoffRegister reg, ValueType type) { |
109 | switch (type) { |
110 | case kWasmI32: |
111 | case kWasmI64: |
112 | assm->pushq(reg.gp()); |
113 | break; |
114 | case kWasmF32: |
115 | assm->subq(rsp, Immediate(kSystemPointerSize)); |
116 | assm->Movss(Operand(rsp, 0), reg.fp()); |
117 | break; |
118 | case kWasmF64: |
119 | assm->subq(rsp, Immediate(kSystemPointerSize)); |
120 | assm->Movsd(Operand(rsp, 0), reg.fp()); |
121 | break; |
122 | default: |
123 | UNREACHABLE(); |
124 | } |
125 | } |
126 | |
127 | template <typename... Regs> |
128 | inline void SpillRegisters(LiftoffAssembler* assm, Regs... regs) { |
129 | for (LiftoffRegister r : {LiftoffRegister(regs)...}) { |
130 | if (assm->cache_state()->is_used(r)) assm->SpillRegister(r); |
131 | } |
132 | } |
133 | |
134 | } // namespace liftoff |
135 | |
136 | int LiftoffAssembler::PrepareStackFrame() { |
137 | int offset = pc_offset(); |
138 | sub_sp_32(0); |
139 | return offset; |
140 | } |
141 | |
142 | void LiftoffAssembler::PatchPrepareStackFrame(int offset, |
143 | uint32_t stack_slots) { |
144 | uint32_t bytes = liftoff::kConstantStackSpace + kStackSlotSize * stack_slots; |
145 | DCHECK_LE(bytes, kMaxInt); |
146 | // We can't run out of space, just pass anything big enough to not cause the |
147 | // assembler to try to grow the buffer. |
148 | constexpr int kAvailableSpace = 64; |
149 | Assembler patching_assembler( |
150 | AssemblerOptions{}, |
151 | ExternalAssemblerBuffer(buffer_start_ + offset, kAvailableSpace)); |
152 | patching_assembler.sub_sp_32(bytes); |
153 | } |
154 | |
155 | void LiftoffAssembler::FinishCode() {} |
156 | |
157 | void LiftoffAssembler::AbortCompilation() {} |
158 | |
159 | void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, |
160 | RelocInfo::Mode rmode) { |
161 | switch (value.type()) { |
162 | case kWasmI32: |
163 | if (value.to_i32() == 0 && RelocInfo::IsNone(rmode)) { |
164 | xorl(reg.gp(), reg.gp()); |
165 | } else { |
166 | movl(reg.gp(), Immediate(value.to_i32(), rmode)); |
167 | } |
168 | break; |
169 | case kWasmI64: |
170 | if (RelocInfo::IsNone(rmode)) { |
171 | TurboAssembler::Set(reg.gp(), value.to_i64()); |
172 | } else { |
173 | movq(reg.gp(), Immediate64(value.to_i64(), rmode)); |
174 | } |
175 | break; |
176 | case kWasmF32: |
177 | TurboAssembler::Move(reg.fp(), value.to_f32_boxed().get_bits()); |
178 | break; |
179 | case kWasmF64: |
180 | TurboAssembler::Move(reg.fp(), value.to_f64_boxed().get_bits()); |
181 | break; |
182 | default: |
183 | UNREACHABLE(); |
184 | } |
185 | } |
186 | |
187 | void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, |
188 | int size) { |
189 | DCHECK_LE(offset, kMaxInt); |
190 | movq(dst, liftoff::GetInstanceOperand()); |
191 | DCHECK(size == 4 || size == 8); |
192 | if (size == 4) { |
193 | movl(dst, Operand(dst, offset)); |
194 | } else { |
195 | movq(dst, Operand(dst, offset)); |
196 | } |
197 | } |
198 | |
199 | void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, |
200 | uint32_t offset) { |
201 | DCHECK_LE(offset, kMaxInt); |
202 | movq(dst, liftoff::GetInstanceOperand()); |
203 | LoadTaggedPointerField(dst, Operand(dst, offset)); |
204 | } |
205 | |
206 | void LiftoffAssembler::SpillInstance(Register instance) { |
207 | movq(liftoff::GetInstanceOperand(), instance); |
208 | } |
209 | |
210 | void LiftoffAssembler::FillInstanceInto(Register dst) { |
211 | movq(dst, liftoff::GetInstanceOperand()); |
212 | } |
213 | |
214 | void LiftoffAssembler::LoadTaggedPointer(Register dst, Register src_addr, |
215 | Register offset_reg, |
216 | uint32_t offset_imm, |
217 | LiftoffRegList pinned) { |
218 | if (emit_debug_code() && offset_reg != no_reg) { |
219 | AssertZeroExtended(offset_reg); |
220 | } |
221 | Operand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm); |
222 | LoadTaggedPointerField(dst, src_op); |
223 | } |
224 | |
225 | void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr, |
226 | Register offset_reg, uint32_t offset_imm, |
227 | LoadType type, LiftoffRegList pinned, |
228 | uint32_t* protected_load_pc, bool is_load_mem) { |
229 | if (emit_debug_code() && offset_reg != no_reg) { |
230 | AssertZeroExtended(offset_reg); |
231 | } |
232 | Operand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm); |
233 | if (protected_load_pc) *protected_load_pc = pc_offset(); |
234 | switch (type.value()) { |
235 | case LoadType::kI32Load8U: |
236 | case LoadType::kI64Load8U: |
237 | movzxbl(dst.gp(), src_op); |
238 | break; |
239 | case LoadType::kI32Load8S: |
240 | movsxbl(dst.gp(), src_op); |
241 | break; |
242 | case LoadType::kI64Load8S: |
243 | movsxbq(dst.gp(), src_op); |
244 | break; |
245 | case LoadType::kI32Load16U: |
246 | case LoadType::kI64Load16U: |
247 | movzxwl(dst.gp(), src_op); |
248 | break; |
249 | case LoadType::kI32Load16S: |
250 | movsxwl(dst.gp(), src_op); |
251 | break; |
252 | case LoadType::kI64Load16S: |
253 | movsxwq(dst.gp(), src_op); |
254 | break; |
255 | case LoadType::kI32Load: |
256 | case LoadType::kI64Load32U: |
257 | movl(dst.gp(), src_op); |
258 | break; |
259 | case LoadType::kI64Load32S: |
260 | movsxlq(dst.gp(), src_op); |
261 | break; |
262 | case LoadType::kI64Load: |
263 | movq(dst.gp(), src_op); |
264 | break; |
265 | case LoadType::kF32Load: |
266 | Movss(dst.fp(), src_op); |
267 | break; |
268 | case LoadType::kF64Load: |
269 | Movsd(dst.fp(), src_op); |
270 | break; |
271 | default: |
272 | UNREACHABLE(); |
273 | } |
274 | } |
275 | |
276 | void LiftoffAssembler::Store(Register dst_addr, Register offset_reg, |
277 | uint32_t offset_imm, LiftoffRegister src, |
278 | StoreType type, LiftoffRegList /* pinned */, |
279 | uint32_t* protected_store_pc, bool is_store_mem) { |
280 | if (emit_debug_code() && offset_reg != no_reg) { |
281 | AssertZeroExtended(offset_reg); |
282 | } |
283 | Operand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm); |
284 | if (protected_store_pc) *protected_store_pc = pc_offset(); |
285 | switch (type.value()) { |
286 | case StoreType::kI32Store8: |
287 | case StoreType::kI64Store8: |
288 | movb(dst_op, src.gp()); |
289 | break; |
290 | case StoreType::kI32Store16: |
291 | case StoreType::kI64Store16: |
292 | movw(dst_op, src.gp()); |
293 | break; |
294 | case StoreType::kI32Store: |
295 | case StoreType::kI64Store32: |
296 | movl(dst_op, src.gp()); |
297 | break; |
298 | case StoreType::kI64Store: |
299 | movq(dst_op, src.gp()); |
300 | break; |
301 | case StoreType::kF32Store: |
302 | Movss(dst_op, src.fp()); |
303 | break; |
304 | case StoreType::kF64Store: |
305 | Movsd(dst_op, src.fp()); |
306 | break; |
307 | default: |
308 | UNREACHABLE(); |
309 | } |
310 | } |
311 | |
312 | void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst, |
313 | uint32_t caller_slot_idx, |
314 | ValueType type) { |
315 | Operand src(rbp, kSystemPointerSize * (caller_slot_idx + 1)); |
316 | liftoff::Load(this, dst, src, type); |
317 | } |
318 | |
319 | void LiftoffAssembler::MoveStackValue(uint32_t dst_index, uint32_t src_index, |
320 | ValueType type) { |
321 | DCHECK_NE(dst_index, src_index); |
322 | Operand src = liftoff::GetStackSlot(src_index); |
323 | Operand dst = liftoff::GetStackSlot(dst_index); |
324 | if (ValueTypes::ElementSizeLog2Of(type) == 2) { |
325 | movl(kScratchRegister, src); |
326 | movl(dst, kScratchRegister); |
327 | } else { |
328 | DCHECK_EQ(3, ValueTypes::ElementSizeLog2Of(type)); |
329 | movq(kScratchRegister, src); |
330 | movq(dst, kScratchRegister); |
331 | } |
332 | } |
333 | |
334 | void LiftoffAssembler::Move(Register dst, Register src, ValueType type) { |
335 | DCHECK_NE(dst, src); |
336 | if (type == kWasmI32) { |
337 | movl(dst, src); |
338 | } else { |
339 | DCHECK_EQ(kWasmI64, type); |
340 | movq(dst, src); |
341 | } |
342 | } |
343 | |
344 | void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src, |
345 | ValueType type) { |
346 | DCHECK_NE(dst, src); |
347 | if (type == kWasmF32) { |
348 | Movss(dst, src); |
349 | } else { |
350 | DCHECK_EQ(kWasmF64, type); |
351 | Movsd(dst, src); |
352 | } |
353 | } |
354 | |
355 | void LiftoffAssembler::Spill(uint32_t index, LiftoffRegister reg, |
356 | ValueType type) { |
357 | RecordUsedSpillSlot(index); |
358 | Operand dst = liftoff::GetStackSlot(index); |
359 | switch (type) { |
360 | case kWasmI32: |
361 | movl(dst, reg.gp()); |
362 | break; |
363 | case kWasmI64: |
364 | movq(dst, reg.gp()); |
365 | break; |
366 | case kWasmF32: |
367 | Movss(dst, reg.fp()); |
368 | break; |
369 | case kWasmF64: |
370 | Movsd(dst, reg.fp()); |
371 | break; |
372 | default: |
373 | UNREACHABLE(); |
374 | } |
375 | } |
376 | |
377 | void LiftoffAssembler::Spill(uint32_t index, WasmValue value) { |
378 | RecordUsedSpillSlot(index); |
379 | Operand dst = liftoff::GetStackSlot(index); |
380 | switch (value.type()) { |
381 | case kWasmI32: |
382 | movl(dst, Immediate(value.to_i32())); |
383 | break; |
384 | case kWasmI64: { |
385 | if (is_int32(value.to_i64())) { |
386 | // Sign extend low word. |
387 | movq(dst, Immediate(static_cast<int32_t>(value.to_i64()))); |
388 | } else if (is_uint32(value.to_i64())) { |
389 | // Zero extend low word. |
390 | movl(kScratchRegister, Immediate(static_cast<int32_t>(value.to_i64()))); |
391 | movq(dst, kScratchRegister); |
392 | } else { |
393 | movq(kScratchRegister, value.to_i64()); |
394 | movq(dst, kScratchRegister); |
395 | } |
396 | break; |
397 | } |
398 | default: |
399 | // We do not track f32 and f64 constants, hence they are unreachable. |
400 | UNREACHABLE(); |
401 | } |
402 | } |
403 | |
404 | void LiftoffAssembler::Fill(LiftoffRegister reg, uint32_t index, |
405 | ValueType type) { |
406 | Operand src = liftoff::GetStackSlot(index); |
407 | switch (type) { |
408 | case kWasmI32: |
409 | movl(reg.gp(), src); |
410 | break; |
411 | case kWasmI64: |
412 | movq(reg.gp(), src); |
413 | break; |
414 | case kWasmF32: |
415 | Movss(reg.fp(), src); |
416 | break; |
417 | case kWasmF64: |
418 | Movsd(reg.fp(), src); |
419 | break; |
420 | default: |
421 | UNREACHABLE(); |
422 | } |
423 | } |
424 | |
425 | void LiftoffAssembler::FillI64Half(Register, uint32_t index, RegPairHalf) { |
426 | UNREACHABLE(); |
427 | } |
428 | |
429 | void LiftoffAssembler::emit_i32_add(Register dst, Register lhs, Register rhs) { |
430 | if (lhs != dst) { |
431 | leal(dst, Operand(lhs, rhs, times_1, 0)); |
432 | } else { |
433 | addl(dst, rhs); |
434 | } |
435 | } |
436 | |
437 | void LiftoffAssembler::emit_i32_add(Register dst, Register lhs, int32_t imm) { |
438 | if (lhs != dst) { |
439 | leal(dst, Operand(lhs, imm)); |
440 | } else { |
441 | addl(dst, Immediate(imm)); |
442 | } |
443 | } |
444 | |
445 | void LiftoffAssembler::emit_i32_sub(Register dst, Register lhs, Register rhs) { |
446 | if (dst != rhs) { |
447 | // Default path. |
448 | if (dst != lhs) movl(dst, lhs); |
449 | subl(dst, rhs); |
450 | } else if (lhs == rhs) { |
451 | // Degenerate case. |
452 | xorl(dst, dst); |
453 | } else { |
454 | // Emit {dst = lhs + -rhs} if dst == rhs. |
455 | negl(dst); |
456 | addl(dst, lhs); |
457 | } |
458 | } |
459 | |
460 | namespace liftoff { |
461 | template <void (Assembler::*op)(Register, Register), |
462 | void (Assembler::*mov)(Register, Register)> |
463 | void EmitCommutativeBinOp(LiftoffAssembler* assm, Register dst, Register lhs, |
464 | Register rhs) { |
465 | if (dst == rhs) { |
466 | (assm->*op)(dst, lhs); |
467 | } else { |
468 | if (dst != lhs) (assm->*mov)(dst, lhs); |
469 | (assm->*op)(dst, rhs); |
470 | } |
471 | } |
472 | } // namespace liftoff |
473 | |
474 | void LiftoffAssembler::emit_i32_mul(Register dst, Register lhs, Register rhs) { |
475 | liftoff::EmitCommutativeBinOp<&Assembler::imull, &Assembler::movl>(this, dst, |
476 | lhs, rhs); |
477 | } |
478 | |
479 | namespace liftoff { |
480 | enum class DivOrRem : uint8_t { kDiv, kRem }; |
481 | template <typename type, DivOrRem div_or_rem> |
482 | void EmitIntDivOrRem(LiftoffAssembler* assm, Register dst, Register lhs, |
483 | Register rhs, Label* trap_div_by_zero, |
484 | Label* trap_div_unrepresentable) { |
485 | constexpr bool needs_unrepresentable_check = |
486 | std::is_signed<type>::value && div_or_rem == DivOrRem::kDiv; |
487 | constexpr bool special_case_minus_1 = |
488 | std::is_signed<type>::value && div_or_rem == DivOrRem::kRem; |
489 | DCHECK_EQ(needs_unrepresentable_check, trap_div_unrepresentable != nullptr); |
490 | |
491 | #define iop(name, ...) \ |
492 | do { \ |
493 | if (sizeof(type) == 4) { \ |
494 | assm->name##l(__VA_ARGS__); \ |
495 | } else { \ |
496 | assm->name##q(__VA_ARGS__); \ |
497 | } \ |
498 | } while (false) |
499 | |
500 | // For division, the lhs is always taken from {edx:eax}. Thus, make sure that |
501 | // these registers are unused. If {rhs} is stored in one of them, move it to |
502 | // another temporary register. |
503 | // Do all this before any branch, such that the code is executed |
504 | // unconditionally, as the cache state will also be modified unconditionally. |
505 | liftoff::SpillRegisters(assm, rdx, rax); |
506 | if (rhs == rax || rhs == rdx) { |
507 | iop(mov, kScratchRegister, rhs); |
508 | rhs = kScratchRegister; |
509 | } |
510 | |
511 | // Check for division by zero. |
512 | iop(test, rhs, rhs); |
513 | assm->j(zero, trap_div_by_zero); |
514 | |
515 | Label done; |
516 | if (needs_unrepresentable_check) { |
517 | // Check for {kMinInt / -1}. This is unrepresentable. |
518 | Label do_div; |
519 | iop(cmp, rhs, Immediate(-1)); |
520 | assm->j(not_equal, &do_div); |
521 | // {lhs} is min int if {lhs - 1} overflows. |
522 | iop(cmp, lhs, Immediate(1)); |
523 | assm->j(overflow, trap_div_unrepresentable); |
524 | assm->bind(&do_div); |
525 | } else if (special_case_minus_1) { |
526 | // {lhs % -1} is always 0 (needs to be special cased because {kMinInt / -1} |
527 | // cannot be computed). |
528 | Label do_rem; |
529 | iop(cmp, rhs, Immediate(-1)); |
530 | assm->j(not_equal, &do_rem); |
531 | // clang-format off |
532 | // (conflicts with presubmit checks because it is confused about "xor") |
533 | iop(xor, dst, dst); |
534 | // clang-format on |
535 | assm->jmp(&done); |
536 | assm->bind(&do_rem); |
537 | } |
538 | |
539 | // Now move {lhs} into {eax}, then zero-extend or sign-extend into {edx}, then |
540 | // do the division. |
541 | if (lhs != rax) iop(mov, rax, lhs); |
542 | if (std::is_same<int32_t, type>::value) { // i32 |
543 | assm->cdq(); |
544 | assm->idivl(rhs); |
545 | } else if (std::is_same<uint32_t, type>::value) { // u32 |
546 | assm->xorl(rdx, rdx); |
547 | assm->divl(rhs); |
548 | } else if (std::is_same<int64_t, type>::value) { // i64 |
549 | assm->cqo(); |
550 | assm->idivq(rhs); |
551 | } else { // u64 |
552 | assm->xorq(rdx, rdx); |
553 | assm->divq(rhs); |
554 | } |
555 | |
556 | // Move back the result (in {eax} or {edx}) into the {dst} register. |
557 | constexpr Register kResultReg = div_or_rem == DivOrRem::kDiv ? rax : rdx; |
558 | if (dst != kResultReg) { |
559 | iop(mov, dst, kResultReg); |
560 | } |
561 | if (special_case_minus_1) assm->bind(&done); |
562 | } |
563 | } // namespace liftoff |
564 | |
565 | void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs, |
566 | Label* trap_div_by_zero, |
567 | Label* trap_div_unrepresentable) { |
568 | liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kDiv>( |
569 | this, dst, lhs, rhs, trap_div_by_zero, trap_div_unrepresentable); |
570 | } |
571 | |
572 | void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs, |
573 | Label* trap_div_by_zero) { |
574 | liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kDiv>( |
575 | this, dst, lhs, rhs, trap_div_by_zero, nullptr); |
576 | } |
577 | |
578 | void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs, |
579 | Label* trap_div_by_zero) { |
580 | liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kRem>( |
581 | this, dst, lhs, rhs, trap_div_by_zero, nullptr); |
582 | } |
583 | |
584 | void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs, |
585 | Label* trap_div_by_zero) { |
586 | liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kRem>( |
587 | this, dst, lhs, rhs, trap_div_by_zero, nullptr); |
588 | } |
589 | |
590 | void LiftoffAssembler::emit_i32_and(Register dst, Register lhs, Register rhs) { |
591 | liftoff::EmitCommutativeBinOp<&Assembler::andl, &Assembler::movl>(this, dst, |
592 | lhs, rhs); |
593 | } |
594 | |
595 | void LiftoffAssembler::emit_i32_or(Register dst, Register lhs, Register rhs) { |
596 | liftoff::EmitCommutativeBinOp<&Assembler::orl, &Assembler::movl>(this, dst, |
597 | lhs, rhs); |
598 | } |
599 | |
600 | void LiftoffAssembler::emit_i32_xor(Register dst, Register lhs, Register rhs) { |
601 | liftoff::EmitCommutativeBinOp<&Assembler::xorl, &Assembler::movl>(this, dst, |
602 | lhs, rhs); |
603 | } |
604 | |
605 | namespace liftoff { |
606 | template <ValueType type> |
607 | inline void EmitShiftOperation(LiftoffAssembler* assm, Register dst, |
608 | Register src, Register amount, |
609 | void (Assembler::*emit_shift)(Register), |
610 | LiftoffRegList pinned) { |
611 | // If dst is rcx, compute into the scratch register first, then move to rcx. |
612 | if (dst == rcx) { |
613 | assm->Move(kScratchRegister, src, type); |
614 | if (amount != rcx) assm->Move(rcx, amount, type); |
615 | (assm->*emit_shift)(kScratchRegister); |
616 | assm->Move(rcx, kScratchRegister, type); |
617 | return; |
618 | } |
619 | |
620 | // Move amount into rcx. If rcx is in use, move its content into the scratch |
621 | // register. If src is rcx, src is now the scratch register. |
622 | bool use_scratch = false; |
623 | if (amount != rcx) { |
624 | use_scratch = src == rcx || |
625 | assm->cache_state()->is_used(LiftoffRegister(rcx)) || |
626 | pinned.has(LiftoffRegister(rcx)); |
627 | if (use_scratch) assm->movq(kScratchRegister, rcx); |
628 | if (src == rcx) src = kScratchRegister; |
629 | assm->Move(rcx, amount, type); |
630 | } |
631 | |
632 | // Do the actual shift. |
633 | if (dst != src) assm->Move(dst, src, type); |
634 | (assm->*emit_shift)(dst); |
635 | |
636 | // Restore rcx if needed. |
637 | if (use_scratch) assm->movq(rcx, kScratchRegister); |
638 | } |
639 | } // namespace liftoff |
640 | |
641 | void LiftoffAssembler::emit_i32_shl(Register dst, Register src, Register amount, |
642 | LiftoffRegList pinned) { |
643 | liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount, |
644 | &Assembler::shll_cl, pinned); |
645 | } |
646 | |
647 | void LiftoffAssembler::emit_i32_sar(Register dst, Register src, Register amount, |
648 | LiftoffRegList pinned) { |
649 | liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount, |
650 | &Assembler::sarl_cl, pinned); |
651 | } |
652 | |
653 | void LiftoffAssembler::emit_i32_shr(Register dst, Register src, Register amount, |
654 | LiftoffRegList pinned) { |
655 | liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount, |
656 | &Assembler::shrl_cl, pinned); |
657 | } |
658 | |
659 | void LiftoffAssembler::emit_i32_shr(Register dst, Register src, int amount) { |
660 | if (dst != src) movl(dst, src); |
661 | DCHECK(is_uint5(amount)); |
662 | shrl(dst, Immediate(amount)); |
663 | } |
664 | |
665 | bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) { |
666 | Label nonzero_input; |
667 | Label continuation; |
668 | testl(src, src); |
669 | j(not_zero, &nonzero_input, Label::kNear); |
670 | movl(dst, Immediate(32)); |
671 | jmp(&continuation, Label::kNear); |
672 | |
673 | bind(&nonzero_input); |
674 | // Get most significant bit set (MSBS). |
675 | bsrl(dst, src); |
676 | // CLZ = 31 - MSBS = MSBS ^ 31. |
677 | xorl(dst, Immediate(31)); |
678 | |
679 | bind(&continuation); |
680 | return true; |
681 | } |
682 | |
683 | bool LiftoffAssembler::emit_i32_ctz(Register dst, Register src) { |
684 | Label nonzero_input; |
685 | Label continuation; |
686 | testl(src, src); |
687 | j(not_zero, &nonzero_input, Label::kNear); |
688 | movl(dst, Immediate(32)); |
689 | jmp(&continuation, Label::kNear); |
690 | |
691 | bind(&nonzero_input); |
692 | // Get least significant bit set, which equals number of trailing zeros. |
693 | bsfl(dst, src); |
694 | |
695 | bind(&continuation); |
696 | return true; |
697 | } |
698 | |
699 | bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) { |
700 | if (!CpuFeatures::IsSupported(POPCNT)) return false; |
701 | CpuFeatureScope scope(this, POPCNT); |
702 | popcntl(dst, src); |
703 | return true; |
704 | } |
705 | |
706 | void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, |
707 | LiftoffRegister rhs) { |
708 | if (lhs.gp() != dst.gp()) { |
709 | leaq(dst.gp(), Operand(lhs.gp(), rhs.gp(), times_1, 0)); |
710 | } else { |
711 | addq(dst.gp(), rhs.gp()); |
712 | } |
713 | } |
714 | |
715 | void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs, |
716 | int32_t imm) { |
717 | if (lhs.gp() != dst.gp()) { |
718 | leaq(dst.gp(), Operand(lhs.gp(), imm)); |
719 | } else { |
720 | addq(dst.gp(), Immediate(imm)); |
721 | } |
722 | } |
723 | |
724 | void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs, |
725 | LiftoffRegister rhs) { |
726 | if (dst.gp() == rhs.gp()) { |
727 | negq(dst.gp()); |
728 | addq(dst.gp(), lhs.gp()); |
729 | } else { |
730 | if (dst.gp() != lhs.gp()) movq(dst.gp(), lhs.gp()); |
731 | subq(dst.gp(), rhs.gp()); |
732 | } |
733 | } |
734 | |
735 | void LiftoffAssembler::emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs, |
736 | LiftoffRegister rhs) { |
737 | liftoff::EmitCommutativeBinOp<&Assembler::imulq, &Assembler::movq>( |
738 | this, dst.gp(), lhs.gp(), rhs.gp()); |
739 | } |
740 | |
741 | bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs, |
742 | LiftoffRegister rhs, |
743 | Label* trap_div_by_zero, |
744 | Label* trap_div_unrepresentable) { |
745 | liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kDiv>( |
746 | this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, |
747 | trap_div_unrepresentable); |
748 | return true; |
749 | } |
750 | |
751 | bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs, |
752 | LiftoffRegister rhs, |
753 | Label* trap_div_by_zero) { |
754 | liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kDiv>( |
755 | this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr); |
756 | return true; |
757 | } |
758 | |
759 | bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs, |
760 | LiftoffRegister rhs, |
761 | Label* trap_div_by_zero) { |
762 | liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kRem>( |
763 | this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr); |
764 | return true; |
765 | } |
766 | |
767 | bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs, |
768 | LiftoffRegister rhs, |
769 | Label* trap_div_by_zero) { |
770 | liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kRem>( |
771 | this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr); |
772 | return true; |
773 | } |
774 | |
775 | void LiftoffAssembler::emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs, |
776 | LiftoffRegister rhs) { |
777 | liftoff::EmitCommutativeBinOp<&Assembler::andq, &Assembler::movq>( |
778 | this, dst.gp(), lhs.gp(), rhs.gp()); |
779 | } |
780 | |
781 | void LiftoffAssembler::emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs, |
782 | LiftoffRegister rhs) { |
783 | liftoff::EmitCommutativeBinOp<&Assembler::orq, &Assembler::movq>( |
784 | this, dst.gp(), lhs.gp(), rhs.gp()); |
785 | } |
786 | |
787 | void LiftoffAssembler::emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs, |
788 | LiftoffRegister rhs) { |
789 | liftoff::EmitCommutativeBinOp<&Assembler::xorq, &Assembler::movq>( |
790 | this, dst.gp(), lhs.gp(), rhs.gp()); |
791 | } |
792 | |
793 | void LiftoffAssembler::emit_i64_shl(LiftoffRegister dst, LiftoffRegister src, |
794 | Register amount, LiftoffRegList pinned) { |
795 | liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount, |
796 | &Assembler::shlq_cl, pinned); |
797 | } |
798 | |
799 | void LiftoffAssembler::emit_i64_sar(LiftoffRegister dst, LiftoffRegister src, |
800 | Register amount, LiftoffRegList pinned) { |
801 | liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount, |
802 | &Assembler::sarq_cl, pinned); |
803 | } |
804 | |
805 | void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, |
806 | Register amount, LiftoffRegList pinned) { |
807 | liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount, |
808 | &Assembler::shrq_cl, pinned); |
809 | } |
810 | |
811 | void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src, |
812 | int amount) { |
813 | if (dst.gp() != src.gp()) movl(dst.gp(), src.gp()); |
814 | DCHECK(is_uint6(amount)); |
815 | shrq(dst.gp(), Immediate(amount)); |
816 | } |
817 | |
818 | void LiftoffAssembler::emit_i32_to_intptr(Register dst, Register src) { |
819 | movsxlq(dst, src); |
820 | } |
821 | |
822 | void LiftoffAssembler::emit_f32_add(DoubleRegister dst, DoubleRegister lhs, |
823 | DoubleRegister rhs) { |
824 | if (CpuFeatures::IsSupported(AVX)) { |
825 | CpuFeatureScope scope(this, AVX); |
826 | vaddss(dst, lhs, rhs); |
827 | } else if (dst == rhs) { |
828 | addss(dst, lhs); |
829 | } else { |
830 | if (dst != lhs) movss(dst, lhs); |
831 | addss(dst, rhs); |
832 | } |
833 | } |
834 | |
835 | void LiftoffAssembler::emit_f32_sub(DoubleRegister dst, DoubleRegister lhs, |
836 | DoubleRegister rhs) { |
837 | if (CpuFeatures::IsSupported(AVX)) { |
838 | CpuFeatureScope scope(this, AVX); |
839 | vsubss(dst, lhs, rhs); |
840 | } else if (dst == rhs) { |
841 | movss(kScratchDoubleReg, rhs); |
842 | movss(dst, lhs); |
843 | subss(dst, kScratchDoubleReg); |
844 | } else { |
845 | if (dst != lhs) movss(dst, lhs); |
846 | subss(dst, rhs); |
847 | } |
848 | } |
849 | |
850 | void LiftoffAssembler::emit_f32_mul(DoubleRegister dst, DoubleRegister lhs, |
851 | DoubleRegister rhs) { |
852 | if (CpuFeatures::IsSupported(AVX)) { |
853 | CpuFeatureScope scope(this, AVX); |
854 | vmulss(dst, lhs, rhs); |
855 | } else if (dst == rhs) { |
856 | mulss(dst, lhs); |
857 | } else { |
858 | if (dst != lhs) movss(dst, lhs); |
859 | mulss(dst, rhs); |
860 | } |
861 | } |
862 | |
863 | void LiftoffAssembler::emit_f32_div(DoubleRegister dst, DoubleRegister lhs, |
864 | DoubleRegister rhs) { |
865 | if (CpuFeatures::IsSupported(AVX)) { |
866 | CpuFeatureScope scope(this, AVX); |
867 | vdivss(dst, lhs, rhs); |
868 | } else if (dst == rhs) { |
869 | movss(kScratchDoubleReg, rhs); |
870 | movss(dst, lhs); |
871 | divss(dst, kScratchDoubleReg); |
872 | } else { |
873 | if (dst != lhs) movss(dst, lhs); |
874 | divss(dst, rhs); |
875 | } |
876 | } |
877 | |
878 | namespace liftoff { |
879 | enum class MinOrMax : uint8_t { kMin, kMax }; |
880 | template <typename type> |
881 | inline void EmitFloatMinOrMax(LiftoffAssembler* assm, DoubleRegister dst, |
882 | DoubleRegister lhs, DoubleRegister rhs, |
883 | MinOrMax min_or_max) { |
884 | Label is_nan; |
885 | Label lhs_below_rhs; |
886 | Label lhs_above_rhs; |
887 | Label done; |
888 | |
889 | #define dop(name, ...) \ |
890 | do { \ |
891 | if (sizeof(type) == 4) { \ |
892 | assm->name##s(__VA_ARGS__); \ |
893 | } else { \ |
894 | assm->name##d(__VA_ARGS__); \ |
895 | } \ |
896 | } while (false) |
897 | |
898 | // Check the easy cases first: nan (e.g. unordered), smaller and greater. |
899 | // NaN has to be checked first, because PF=1 implies CF=1. |
900 | dop(Ucomis, lhs, rhs); |
901 | assm->j(parity_even, &is_nan, Label::kNear); // PF=1 |
902 | assm->j(below, &lhs_below_rhs, Label::kNear); // CF=1 |
903 | assm->j(above, &lhs_above_rhs, Label::kNear); // CF=0 && ZF=0 |
904 | |
905 | // If we get here, then either |
906 | // a) {lhs == rhs}, |
907 | // b) {lhs == -0.0} and {rhs == 0.0}, or |
908 | // c) {lhs == 0.0} and {rhs == -0.0}. |
909 | // For a), it does not matter whether we return {lhs} or {rhs}. Check the sign |
910 | // bit of {rhs} to differentiate b) and c). |
911 | dop(Movmskp, kScratchRegister, rhs); |
912 | assm->testl(kScratchRegister, Immediate(1)); |
913 | assm->j(zero, &lhs_below_rhs, Label::kNear); |
914 | assm->jmp(&lhs_above_rhs, Label::kNear); |
915 | |
916 | assm->bind(&is_nan); |
917 | // Create a NaN output. |
918 | dop(Xorp, dst, dst); |
919 | dop(Divs, dst, dst); |
920 | assm->jmp(&done, Label::kNear); |
921 | |
922 | assm->bind(&lhs_below_rhs); |
923 | DoubleRegister lhs_below_rhs_src = min_or_max == MinOrMax::kMin ? lhs : rhs; |
924 | if (dst != lhs_below_rhs_src) dop(Movs, dst, lhs_below_rhs_src); |
925 | assm->jmp(&done, Label::kNear); |
926 | |
927 | assm->bind(&lhs_above_rhs); |
928 | DoubleRegister lhs_above_rhs_src = min_or_max == MinOrMax::kMin ? rhs : lhs; |
929 | if (dst != lhs_above_rhs_src) dop(Movs, dst, lhs_above_rhs_src); |
930 | |
931 | assm->bind(&done); |
932 | } |
933 | } // namespace liftoff |
934 | |
935 | void LiftoffAssembler::emit_f32_min(DoubleRegister dst, DoubleRegister lhs, |
936 | DoubleRegister rhs) { |
937 | liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs, |
938 | liftoff::MinOrMax::kMin); |
939 | } |
940 | |
941 | void LiftoffAssembler::emit_f32_max(DoubleRegister dst, DoubleRegister lhs, |
942 | DoubleRegister rhs) { |
943 | liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs, |
944 | liftoff::MinOrMax::kMax); |
945 | } |
946 | |
947 | void LiftoffAssembler::emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs, |
948 | DoubleRegister rhs) { |
949 | static constexpr int kF32SignBit = 1 << 31; |
950 | Movd(kScratchRegister, lhs); |
951 | andl(kScratchRegister, Immediate(~kF32SignBit)); |
952 | Movd(liftoff::kScratchRegister2, rhs); |
953 | andl(liftoff::kScratchRegister2, Immediate(kF32SignBit)); |
954 | orl(kScratchRegister, liftoff::kScratchRegister2); |
955 | Movd(dst, kScratchRegister); |
956 | } |
957 | |
958 | void LiftoffAssembler::emit_f32_abs(DoubleRegister dst, DoubleRegister src) { |
959 | static constexpr uint32_t kSignBit = uint32_t{1} << 31; |
960 | if (dst == src) { |
961 | TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1); |
962 | Andps(dst, kScratchDoubleReg); |
963 | } else { |
964 | TurboAssembler::Move(dst, kSignBit - 1); |
965 | Andps(dst, src); |
966 | } |
967 | } |
968 | |
969 | void LiftoffAssembler::emit_f32_neg(DoubleRegister dst, DoubleRegister src) { |
970 | static constexpr uint32_t kSignBit = uint32_t{1} << 31; |
971 | if (dst == src) { |
972 | TurboAssembler::Move(kScratchDoubleReg, kSignBit); |
973 | Xorps(dst, kScratchDoubleReg); |
974 | } else { |
975 | TurboAssembler::Move(dst, kSignBit); |
976 | Xorps(dst, src); |
977 | } |
978 | } |
979 | |
980 | bool LiftoffAssembler::emit_f32_ceil(DoubleRegister dst, DoubleRegister src) { |
981 | if (CpuFeatures::IsSupported(SSE4_1)) { |
982 | CpuFeatureScope feature(this, SSE4_1); |
983 | Roundss(dst, src, kRoundUp); |
984 | return true; |
985 | } |
986 | return false; |
987 | } |
988 | |
989 | bool LiftoffAssembler::emit_f32_floor(DoubleRegister dst, DoubleRegister src) { |
990 | if (CpuFeatures::IsSupported(SSE4_1)) { |
991 | CpuFeatureScope feature(this, SSE4_1); |
992 | Roundss(dst, src, kRoundDown); |
993 | return true; |
994 | } |
995 | return false; |
996 | } |
997 | |
998 | bool LiftoffAssembler::emit_f32_trunc(DoubleRegister dst, DoubleRegister src) { |
999 | if (CpuFeatures::IsSupported(SSE4_1)) { |
1000 | CpuFeatureScope feature(this, SSE4_1); |
1001 | Roundss(dst, src, kRoundToZero); |
1002 | return true; |
1003 | } |
1004 | return false; |
1005 | } |
1006 | |
1007 | bool LiftoffAssembler::emit_f32_nearest_int(DoubleRegister dst, |
1008 | DoubleRegister src) { |
1009 | if (CpuFeatures::IsSupported(SSE4_1)) { |
1010 | CpuFeatureScope feature(this, SSE4_1); |
1011 | Roundss(dst, src, kRoundToNearest); |
1012 | return true; |
1013 | } |
1014 | return false; |
1015 | } |
1016 | |
1017 | void LiftoffAssembler::emit_f32_sqrt(DoubleRegister dst, DoubleRegister src) { |
1018 | Sqrtss(dst, src); |
1019 | } |
1020 | |
1021 | void LiftoffAssembler::emit_f64_add(DoubleRegister dst, DoubleRegister lhs, |
1022 | DoubleRegister rhs) { |
1023 | if (CpuFeatures::IsSupported(AVX)) { |
1024 | CpuFeatureScope scope(this, AVX); |
1025 | vaddsd(dst, lhs, rhs); |
1026 | } else if (dst == rhs) { |
1027 | addsd(dst, lhs); |
1028 | } else { |
1029 | if (dst != lhs) movsd(dst, lhs); |
1030 | addsd(dst, rhs); |
1031 | } |
1032 | } |
1033 | |
1034 | void LiftoffAssembler::emit_f64_sub(DoubleRegister dst, DoubleRegister lhs, |
1035 | DoubleRegister rhs) { |
1036 | if (CpuFeatures::IsSupported(AVX)) { |
1037 | CpuFeatureScope scope(this, AVX); |
1038 | vsubsd(dst, lhs, rhs); |
1039 | } else if (dst == rhs) { |
1040 | movsd(kScratchDoubleReg, rhs); |
1041 | movsd(dst, lhs); |
1042 | subsd(dst, kScratchDoubleReg); |
1043 | } else { |
1044 | if (dst != lhs) movsd(dst, lhs); |
1045 | subsd(dst, rhs); |
1046 | } |
1047 | } |
1048 | |
1049 | void LiftoffAssembler::emit_f64_mul(DoubleRegister dst, DoubleRegister lhs, |
1050 | DoubleRegister rhs) { |
1051 | if (CpuFeatures::IsSupported(AVX)) { |
1052 | CpuFeatureScope scope(this, AVX); |
1053 | vmulsd(dst, lhs, rhs); |
1054 | } else if (dst == rhs) { |
1055 | mulsd(dst, lhs); |
1056 | } else { |
1057 | if (dst != lhs) movsd(dst, lhs); |
1058 | mulsd(dst, rhs); |
1059 | } |
1060 | } |
1061 | |
1062 | void LiftoffAssembler::emit_f64_div(DoubleRegister dst, DoubleRegister lhs, |
1063 | DoubleRegister rhs) { |
1064 | if (CpuFeatures::IsSupported(AVX)) { |
1065 | CpuFeatureScope scope(this, AVX); |
1066 | vdivsd(dst, lhs, rhs); |
1067 | } else if (dst == rhs) { |
1068 | movsd(kScratchDoubleReg, rhs); |
1069 | movsd(dst, lhs); |
1070 | divsd(dst, kScratchDoubleReg); |
1071 | } else { |
1072 | if (dst != lhs) movsd(dst, lhs); |
1073 | divsd(dst, rhs); |
1074 | } |
1075 | } |
1076 | |
1077 | void LiftoffAssembler::emit_f64_min(DoubleRegister dst, DoubleRegister lhs, |
1078 | DoubleRegister rhs) { |
1079 | liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs, |
1080 | liftoff::MinOrMax::kMin); |
1081 | } |
1082 | |
1083 | void LiftoffAssembler::emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs, |
1084 | DoubleRegister rhs) { |
1085 | // Extract sign bit from {rhs} into {kScratchRegister2}. |
1086 | Movq(liftoff::kScratchRegister2, rhs); |
1087 | shrq(liftoff::kScratchRegister2, Immediate(63)); |
1088 | shlq(liftoff::kScratchRegister2, Immediate(63)); |
1089 | // Reset sign bit of {lhs} (in {kScratchRegister}). |
1090 | Movq(kScratchRegister, lhs); |
1091 | btrq(kScratchRegister, Immediate(63)); |
1092 | // Combine both values into {kScratchRegister} and move into {dst}. |
1093 | orq(kScratchRegister, liftoff::kScratchRegister2); |
1094 | Movq(dst, kScratchRegister); |
1095 | } |
1096 | |
1097 | void LiftoffAssembler::emit_f64_max(DoubleRegister dst, DoubleRegister lhs, |
1098 | DoubleRegister rhs) { |
1099 | liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs, |
1100 | liftoff::MinOrMax::kMax); |
1101 | } |
1102 | |
1103 | void LiftoffAssembler::emit_f64_abs(DoubleRegister dst, DoubleRegister src) { |
1104 | static constexpr uint64_t kSignBit = uint64_t{1} << 63; |
1105 | if (dst == src) { |
1106 | TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1); |
1107 | Andpd(dst, kScratchDoubleReg); |
1108 | } else { |
1109 | TurboAssembler::Move(dst, kSignBit - 1); |
1110 | Andpd(dst, src); |
1111 | } |
1112 | } |
1113 | |
1114 | void LiftoffAssembler::emit_f64_neg(DoubleRegister dst, DoubleRegister src) { |
1115 | static constexpr uint64_t kSignBit = uint64_t{1} << 63; |
1116 | if (dst == src) { |
1117 | TurboAssembler::Move(kScratchDoubleReg, kSignBit); |
1118 | Xorpd(dst, kScratchDoubleReg); |
1119 | } else { |
1120 | TurboAssembler::Move(dst, kSignBit); |
1121 | Xorpd(dst, src); |
1122 | } |
1123 | } |
1124 | |
1125 | bool LiftoffAssembler::emit_f64_ceil(DoubleRegister dst, DoubleRegister src) { |
1126 | REQUIRE_CPU_FEATURE(SSE4_1, true); |
1127 | Roundsd(dst, src, kRoundUp); |
1128 | return true; |
1129 | } |
1130 | |
1131 | bool LiftoffAssembler::emit_f64_floor(DoubleRegister dst, DoubleRegister src) { |
1132 | REQUIRE_CPU_FEATURE(SSE4_1, true); |
1133 | Roundsd(dst, src, kRoundDown); |
1134 | return true; |
1135 | } |
1136 | |
1137 | bool LiftoffAssembler::emit_f64_trunc(DoubleRegister dst, DoubleRegister src) { |
1138 | REQUIRE_CPU_FEATURE(SSE4_1, true); |
1139 | Roundsd(dst, src, kRoundToZero); |
1140 | return true; |
1141 | } |
1142 | |
1143 | bool LiftoffAssembler::emit_f64_nearest_int(DoubleRegister dst, |
1144 | DoubleRegister src) { |
1145 | REQUIRE_CPU_FEATURE(SSE4_1, true); |
1146 | Roundsd(dst, src, kRoundToNearest); |
1147 | return true; |
1148 | } |
1149 | |
1150 | void LiftoffAssembler::emit_f64_sqrt(DoubleRegister dst, DoubleRegister src) { |
1151 | Sqrtsd(dst, src); |
1152 | } |
1153 | |
1154 | namespace liftoff { |
1155 | // Used for float to int conversions. If the value in {converted_back} equals |
1156 | // {src} afterwards, the conversion succeeded. |
1157 | template <typename dst_type, typename src_type> |
1158 | inline void ConvertFloatToIntAndBack(LiftoffAssembler* assm, Register dst, |
1159 | DoubleRegister src, |
1160 | DoubleRegister converted_back) { |
1161 | if (std::is_same<double, src_type>::value) { // f64 |
1162 | if (std::is_same<int32_t, dst_type>::value) { // f64 -> i32 |
1163 | assm->Cvttsd2si(dst, src); |
1164 | assm->Cvtlsi2sd(converted_back, dst); |
1165 | } else if (std::is_same<uint32_t, dst_type>::value) { // f64 -> u32 |
1166 | assm->Cvttsd2siq(dst, src); |
1167 | assm->movl(dst, dst); |
1168 | assm->Cvtqsi2sd(converted_back, dst); |
1169 | } else if (std::is_same<int64_t, dst_type>::value) { // f64 -> i64 |
1170 | assm->Cvttsd2siq(dst, src); |
1171 | assm->Cvtqsi2sd(converted_back, dst); |
1172 | } else { |
1173 | UNREACHABLE(); |
1174 | } |
1175 | } else { // f32 |
1176 | if (std::is_same<int32_t, dst_type>::value) { // f32 -> i32 |
1177 | assm->Cvttss2si(dst, src); |
1178 | assm->Cvtlsi2ss(converted_back, dst); |
1179 | } else if (std::is_same<uint32_t, dst_type>::value) { // f32 -> u32 |
1180 | assm->Cvttss2siq(dst, src); |
1181 | assm->movl(dst, dst); |
1182 | assm->Cvtqsi2ss(converted_back, dst); |
1183 | } else if (std::is_same<int64_t, dst_type>::value) { // f32 -> i64 |
1184 | assm->Cvttss2siq(dst, src); |
1185 | assm->Cvtqsi2ss(converted_back, dst); |
1186 | } else { |
1187 | UNREACHABLE(); |
1188 | } |
1189 | } |
1190 | } |
1191 | |
1192 | template <typename dst_type, typename src_type> |
1193 | inline bool EmitTruncateFloatToInt(LiftoffAssembler* assm, Register dst, |
1194 | DoubleRegister src, Label* trap) { |
1195 | if (!CpuFeatures::IsSupported(SSE4_1)) { |
1196 | assm->bailout("no SSE4.1" ); |
1197 | return true; |
1198 | } |
1199 | CpuFeatureScope feature(assm, SSE4_1); |
1200 | |
1201 | DoubleRegister rounded = kScratchDoubleReg; |
1202 | DoubleRegister converted_back = kScratchDoubleReg2; |
1203 | |
1204 | if (std::is_same<double, src_type>::value) { // f64 |
1205 | assm->Roundsd(rounded, src, kRoundToZero); |
1206 | } else { // f32 |
1207 | assm->Roundss(rounded, src, kRoundToZero); |
1208 | } |
1209 | ConvertFloatToIntAndBack<dst_type, src_type>(assm, dst, rounded, |
1210 | converted_back); |
1211 | if (std::is_same<double, src_type>::value) { // f64 |
1212 | assm->Ucomisd(converted_back, rounded); |
1213 | } else { // f32 |
1214 | assm->Ucomiss(converted_back, rounded); |
1215 | } |
1216 | |
1217 | // Jump to trap if PF is 0 (one of the operands was NaN) or they are not |
1218 | // equal. |
1219 | assm->j(parity_even, trap); |
1220 | assm->j(not_equal, trap); |
1221 | return true; |
1222 | } |
1223 | } // namespace liftoff |
1224 | |
1225 | bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode, |
1226 | LiftoffRegister dst, |
1227 | LiftoffRegister src, Label* trap) { |
1228 | switch (opcode) { |
1229 | case kExprI32ConvertI64: |
1230 | movl(dst.gp(), src.gp()); |
1231 | return true; |
1232 | case kExprI32SConvertF32: |
1233 | return liftoff::EmitTruncateFloatToInt<int32_t, float>(this, dst.gp(), |
1234 | src.fp(), trap); |
1235 | case kExprI32UConvertF32: |
1236 | return liftoff::EmitTruncateFloatToInt<uint32_t, float>(this, dst.gp(), |
1237 | src.fp(), trap); |
1238 | case kExprI32SConvertF64: |
1239 | return liftoff::EmitTruncateFloatToInt<int32_t, double>(this, dst.gp(), |
1240 | src.fp(), trap); |
1241 | case kExprI32UConvertF64: |
1242 | return liftoff::EmitTruncateFloatToInt<uint32_t, double>(this, dst.gp(), |
1243 | src.fp(), trap); |
1244 | case kExprI32ReinterpretF32: |
1245 | Movd(dst.gp(), src.fp()); |
1246 | return true; |
1247 | case kExprI64SConvertI32: |
1248 | movsxlq(dst.gp(), src.gp()); |
1249 | return true; |
1250 | case kExprI64SConvertF32: |
1251 | return liftoff::EmitTruncateFloatToInt<int64_t, float>(this, dst.gp(), |
1252 | src.fp(), trap); |
1253 | case kExprI64UConvertF32: { |
1254 | REQUIRE_CPU_FEATURE(SSE4_1, true); |
1255 | Cvttss2uiq(dst.gp(), src.fp(), trap); |
1256 | return true; |
1257 | } |
1258 | case kExprI64SConvertF64: |
1259 | return liftoff::EmitTruncateFloatToInt<int64_t, double>(this, dst.gp(), |
1260 | src.fp(), trap); |
1261 | case kExprI64UConvertF64: { |
1262 | REQUIRE_CPU_FEATURE(SSE4_1, true); |
1263 | Cvttsd2uiq(dst.gp(), src.fp(), trap); |
1264 | return true; |
1265 | } |
1266 | case kExprI64UConvertI32: |
1267 | AssertZeroExtended(src.gp()); |
1268 | if (dst.gp() != src.gp()) movl(dst.gp(), src.gp()); |
1269 | return true; |
1270 | case kExprI64ReinterpretF64: |
1271 | Movq(dst.gp(), src.fp()); |
1272 | return true; |
1273 | case kExprF32SConvertI32: |
1274 | Cvtlsi2ss(dst.fp(), src.gp()); |
1275 | return true; |
1276 | case kExprF32UConvertI32: |
1277 | movl(kScratchRegister, src.gp()); |
1278 | Cvtqsi2ss(dst.fp(), kScratchRegister); |
1279 | return true; |
1280 | case kExprF32SConvertI64: |
1281 | Cvtqsi2ss(dst.fp(), src.gp()); |
1282 | return true; |
1283 | case kExprF32UConvertI64: |
1284 | Cvtqui2ss(dst.fp(), src.gp()); |
1285 | return true; |
1286 | case kExprF32ConvertF64: |
1287 | Cvtsd2ss(dst.fp(), src.fp()); |
1288 | return true; |
1289 | case kExprF32ReinterpretI32: |
1290 | Movd(dst.fp(), src.gp()); |
1291 | return true; |
1292 | case kExprF64SConvertI32: |
1293 | Cvtlsi2sd(dst.fp(), src.gp()); |
1294 | return true; |
1295 | case kExprF64UConvertI32: |
1296 | movl(kScratchRegister, src.gp()); |
1297 | Cvtqsi2sd(dst.fp(), kScratchRegister); |
1298 | return true; |
1299 | case kExprF64SConvertI64: |
1300 | Cvtqsi2sd(dst.fp(), src.gp()); |
1301 | return true; |
1302 | case kExprF64UConvertI64: |
1303 | Cvtqui2sd(dst.fp(), src.gp()); |
1304 | return true; |
1305 | case kExprF64ConvertF32: |
1306 | Cvtss2sd(dst.fp(), src.fp()); |
1307 | return true; |
1308 | case kExprF64ReinterpretI64: |
1309 | Movq(dst.fp(), src.gp()); |
1310 | return true; |
1311 | default: |
1312 | UNREACHABLE(); |
1313 | } |
1314 | } |
1315 | |
1316 | void LiftoffAssembler::emit_i32_signextend_i8(Register dst, Register src) { |
1317 | movsxbl(dst, src); |
1318 | } |
1319 | |
1320 | void LiftoffAssembler::emit_i32_signextend_i16(Register dst, Register src) { |
1321 | movsxwl(dst, src); |
1322 | } |
1323 | |
1324 | void LiftoffAssembler::emit_i64_signextend_i8(LiftoffRegister dst, |
1325 | LiftoffRegister src) { |
1326 | movsxbq(dst.gp(), src.gp()); |
1327 | } |
1328 | |
1329 | void LiftoffAssembler::emit_i64_signextend_i16(LiftoffRegister dst, |
1330 | LiftoffRegister src) { |
1331 | movsxwq(dst.gp(), src.gp()); |
1332 | } |
1333 | |
1334 | void LiftoffAssembler::emit_i64_signextend_i32(LiftoffRegister dst, |
1335 | LiftoffRegister src) { |
1336 | movsxlq(dst.gp(), src.gp()); |
1337 | } |
1338 | |
1339 | void LiftoffAssembler::emit_jump(Label* label) { jmp(label); } |
1340 | |
1341 | void LiftoffAssembler::emit_jump(Register target) { jmp(target); } |
1342 | |
1343 | void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label, |
1344 | ValueType type, Register lhs, |
1345 | Register rhs) { |
1346 | if (rhs != no_reg) { |
1347 | switch (type) { |
1348 | case kWasmI32: |
1349 | cmpl(lhs, rhs); |
1350 | break; |
1351 | case kWasmI64: |
1352 | cmpq(lhs, rhs); |
1353 | break; |
1354 | default: |
1355 | UNREACHABLE(); |
1356 | } |
1357 | } else { |
1358 | DCHECK_EQ(type, kWasmI32); |
1359 | testl(lhs, lhs); |
1360 | } |
1361 | |
1362 | j(cond, label); |
1363 | } |
1364 | |
1365 | void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) { |
1366 | testl(src, src); |
1367 | setcc(equal, dst); |
1368 | movzxbl(dst, dst); |
1369 | } |
1370 | |
1371 | void LiftoffAssembler::emit_i32_set_cond(Condition cond, Register dst, |
1372 | Register lhs, Register rhs) { |
1373 | cmpl(lhs, rhs); |
1374 | setcc(cond, dst); |
1375 | movzxbl(dst, dst); |
1376 | } |
1377 | |
1378 | void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) { |
1379 | testq(src.gp(), src.gp()); |
1380 | setcc(equal, dst); |
1381 | movzxbl(dst, dst); |
1382 | } |
1383 | |
1384 | void LiftoffAssembler::emit_i64_set_cond(Condition cond, Register dst, |
1385 | LiftoffRegister lhs, |
1386 | LiftoffRegister rhs) { |
1387 | cmpq(lhs.gp(), rhs.gp()); |
1388 | setcc(cond, dst); |
1389 | movzxbl(dst, dst); |
1390 | } |
1391 | |
1392 | namespace liftoff { |
1393 | template <void (TurboAssembler::*cmp_op)(DoubleRegister, DoubleRegister)> |
1394 | void EmitFloatSetCond(LiftoffAssembler* assm, Condition cond, Register dst, |
1395 | DoubleRegister lhs, DoubleRegister rhs) { |
1396 | Label cont; |
1397 | Label not_nan; |
1398 | |
1399 | (assm->*cmp_op)(lhs, rhs); |
1400 | // If PF is one, one of the operands was NaN. This needs special handling. |
1401 | assm->j(parity_odd, ¬_nan, Label::kNear); |
1402 | // Return 1 for f32.ne, 0 for all other cases. |
1403 | if (cond == not_equal) { |
1404 | assm->movl(dst, Immediate(1)); |
1405 | } else { |
1406 | assm->xorl(dst, dst); |
1407 | } |
1408 | assm->jmp(&cont, Label::kNear); |
1409 | assm->bind(¬_nan); |
1410 | |
1411 | assm->setcc(cond, dst); |
1412 | assm->movzxbl(dst, dst); |
1413 | assm->bind(&cont); |
1414 | } |
1415 | } // namespace liftoff |
1416 | |
1417 | void LiftoffAssembler::emit_f32_set_cond(Condition cond, Register dst, |
1418 | DoubleRegister lhs, |
1419 | DoubleRegister rhs) { |
1420 | liftoff::EmitFloatSetCond<&TurboAssembler::Ucomiss>(this, cond, dst, lhs, |
1421 | rhs); |
1422 | } |
1423 | |
1424 | void LiftoffAssembler::emit_f64_set_cond(Condition cond, Register dst, |
1425 | DoubleRegister lhs, |
1426 | DoubleRegister rhs) { |
1427 | liftoff::EmitFloatSetCond<&TurboAssembler::Ucomisd>(this, cond, dst, lhs, |
1428 | rhs); |
1429 | } |
1430 | |
1431 | void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) { |
1432 | cmpq(rsp, Operand(limit_address, 0)); |
1433 | j(below_equal, ool_code); |
1434 | } |
1435 | |
1436 | void LiftoffAssembler::CallTrapCallbackForTesting() { |
1437 | PrepareCallCFunction(0); |
1438 | CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0); |
1439 | } |
1440 | |
1441 | void LiftoffAssembler::AssertUnreachable(AbortReason reason) { |
1442 | TurboAssembler::AssertUnreachable(reason); |
1443 | } |
1444 | |
1445 | void LiftoffAssembler::PushRegisters(LiftoffRegList regs) { |
1446 | LiftoffRegList gp_regs = regs & kGpCacheRegList; |
1447 | while (!gp_regs.is_empty()) { |
1448 | LiftoffRegister reg = gp_regs.GetFirstRegSet(); |
1449 | pushq(reg.gp()); |
1450 | gp_regs.clear(reg); |
1451 | } |
1452 | LiftoffRegList fp_regs = regs & kFpCacheRegList; |
1453 | unsigned num_fp_regs = fp_regs.GetNumRegsSet(); |
1454 | if (num_fp_regs) { |
1455 | subq(rsp, Immediate(num_fp_regs * kStackSlotSize)); |
1456 | unsigned offset = 0; |
1457 | while (!fp_regs.is_empty()) { |
1458 | LiftoffRegister reg = fp_regs.GetFirstRegSet(); |
1459 | Movsd(Operand(rsp, offset), reg.fp()); |
1460 | fp_regs.clear(reg); |
1461 | offset += sizeof(double); |
1462 | } |
1463 | DCHECK_EQ(offset, num_fp_regs * sizeof(double)); |
1464 | } |
1465 | } |
1466 | |
1467 | void LiftoffAssembler::PopRegisters(LiftoffRegList regs) { |
1468 | LiftoffRegList fp_regs = regs & kFpCacheRegList; |
1469 | unsigned fp_offset = 0; |
1470 | while (!fp_regs.is_empty()) { |
1471 | LiftoffRegister reg = fp_regs.GetFirstRegSet(); |
1472 | Movsd(reg.fp(), Operand(rsp, fp_offset)); |
1473 | fp_regs.clear(reg); |
1474 | fp_offset += sizeof(double); |
1475 | } |
1476 | if (fp_offset) addq(rsp, Immediate(fp_offset)); |
1477 | LiftoffRegList gp_regs = regs & kGpCacheRegList; |
1478 | while (!gp_regs.is_empty()) { |
1479 | LiftoffRegister reg = gp_regs.GetLastRegSet(); |
1480 | popq(reg.gp()); |
1481 | gp_regs.clear(reg); |
1482 | } |
1483 | } |
1484 | |
1485 | void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) { |
1486 | DCHECK_LT(num_stack_slots, |
1487 | (1 << 16) / kSystemPointerSize); // 16 bit immediate |
1488 | ret(static_cast<int>(num_stack_slots * kSystemPointerSize)); |
1489 | } |
1490 | |
1491 | void LiftoffAssembler::CallC(wasm::FunctionSig* sig, |
1492 | const LiftoffRegister* args, |
1493 | const LiftoffRegister* rets, |
1494 | ValueType out_argument_type, int stack_bytes, |
1495 | ExternalReference ext_ref) { |
1496 | subq(rsp, Immediate(stack_bytes)); |
1497 | |
1498 | int arg_bytes = 0; |
1499 | for (ValueType param_type : sig->parameters()) { |
1500 | liftoff::Store(this, Operand(rsp, arg_bytes), *args++, param_type); |
1501 | arg_bytes += ValueTypes::MemSize(param_type); |
1502 | } |
1503 | DCHECK_LE(arg_bytes, stack_bytes); |
1504 | |
1505 | // Pass a pointer to the buffer with the arguments to the C function. |
1506 | movq(arg_reg_1, rsp); |
1507 | |
1508 | constexpr int kNumCCallArgs = 1; |
1509 | |
1510 | // Now call the C function. |
1511 | PrepareCallCFunction(kNumCCallArgs); |
1512 | CallCFunction(ext_ref, kNumCCallArgs); |
1513 | |
1514 | // Move return value to the right register. |
1515 | const LiftoffRegister* next_result_reg = rets; |
1516 | if (sig->return_count() > 0) { |
1517 | DCHECK_EQ(1, sig->return_count()); |
1518 | constexpr Register kReturnReg = rax; |
1519 | if (kReturnReg != next_result_reg->gp()) { |
1520 | Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0)); |
1521 | } |
1522 | ++next_result_reg; |
1523 | } |
1524 | |
1525 | // Load potential output value from the buffer on the stack. |
1526 | if (out_argument_type != kWasmStmt) { |
1527 | liftoff::Load(this, *next_result_reg, Operand(rsp, 0), out_argument_type); |
1528 | } |
1529 | |
1530 | addq(rsp, Immediate(stack_bytes)); |
1531 | } |
1532 | |
1533 | void LiftoffAssembler::CallNativeWasmCode(Address addr) { |
1534 | near_call(addr, RelocInfo::WASM_CALL); |
1535 | } |
1536 | |
1537 | void LiftoffAssembler::CallIndirect(wasm::FunctionSig* sig, |
1538 | compiler::CallDescriptor* call_descriptor, |
1539 | Register target) { |
1540 | if (target == no_reg) { |
1541 | popq(kScratchRegister); |
1542 | target = kScratchRegister; |
1543 | } |
1544 | if (FLAG_untrusted_code_mitigations) { |
1545 | RetpolineCall(target); |
1546 | } else { |
1547 | call(target); |
1548 | } |
1549 | } |
1550 | |
1551 | void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) { |
1552 | // A direct call to a wasm runtime stub defined in this module. |
1553 | // Just encode the stub index. This will be patched at relocation. |
1554 | near_call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL); |
1555 | } |
1556 | |
1557 | void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) { |
1558 | subq(rsp, Immediate(size)); |
1559 | movq(addr, rsp); |
1560 | } |
1561 | |
1562 | void LiftoffAssembler::DeallocateStackSlot(uint32_t size) { |
1563 | addq(rsp, Immediate(size)); |
1564 | } |
1565 | |
1566 | void LiftoffStackSlots::Construct() { |
1567 | for (auto& slot : slots_) { |
1568 | const LiftoffAssembler::VarState& src = slot.src_; |
1569 | switch (src.loc()) { |
1570 | case LiftoffAssembler::VarState::kStack: |
1571 | if (src.type() == kWasmI32) { |
1572 | // Load i32 values to a register first to ensure they are zero |
1573 | // extended. |
1574 | asm_->movl(kScratchRegister, liftoff::GetStackSlot(slot.src_index_)); |
1575 | asm_->pushq(kScratchRegister); |
1576 | } else { |
1577 | // For all other types, just push the whole (8-byte) stack slot. |
1578 | // This is also ok for f32 values (even though we copy 4 uninitialized |
1579 | // bytes), because f32 and f64 values are clearly distinguished in |
1580 | // Turbofan, so the uninitialized bytes are never accessed. |
1581 | asm_->pushq(liftoff::GetStackSlot(slot.src_index_)); |
1582 | } |
1583 | break; |
1584 | case LiftoffAssembler::VarState::kRegister: |
1585 | liftoff::push(asm_, src.reg(), src.type()); |
1586 | break; |
1587 | case LiftoffAssembler::VarState::kIntConst: |
1588 | asm_->pushq(Immediate(src.i32_const())); |
1589 | break; |
1590 | } |
1591 | } |
1592 | } |
1593 | |
1594 | #undef REQUIRE_CPU_FEATURE |
1595 | |
1596 | } // namespace wasm |
1597 | } // namespace internal |
1598 | } // namespace v8 |
1599 | |
1600 | #endif // V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_ |
1601 | |