1// Copyright 2017 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/wasm/baseline/liftoff-assembler.h"
6
7#include <sstream>
8
9#include "src/assembler-inl.h"
10#include "src/base/optional.h"
11#include "src/compiler/linkage.h"
12#include "src/compiler/wasm-compiler.h"
13#include "src/macro-assembler-inl.h"
14#include "src/ostreams.h"
15#include "src/wasm/function-body-decoder-impl.h"
16#include "src/wasm/wasm-linkage.h"
17#include "src/wasm/wasm-opcodes.h"
18
19namespace v8 {
20namespace internal {
21namespace wasm {
22
23using VarState = LiftoffAssembler::VarState;
24
25namespace {
26
27#define __ asm_->
28
29#define TRACE(...) \
30 do { \
31 if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
32 } while (false)
33
34class StackTransferRecipe {
35 struct RegisterMove {
36 LiftoffRegister src;
37 ValueType type;
38 constexpr RegisterMove(LiftoffRegister src, ValueType type)
39 : src(src), type(type) {}
40 };
41
42 struct RegisterLoad {
43 enum LoadKind : uint8_t {
44 kConstant, // load a constant value into a register.
45 kStack, // fill a register from a stack slot.
46 kLowHalfStack, // fill a register from the low half of a stack slot.
47 kHighHalfStack // fill a register from the high half of a stack slot.
48 };
49
50 LoadKind kind;
51 ValueType type;
52 int32_t value; // i32 constant value or stack index, depending on kind.
53
54 // Named constructors.
55 static RegisterLoad Const(WasmValue constant) {
56 if (constant.type() == kWasmI32) {
57 return {kConstant, kWasmI32, constant.to_i32()};
58 }
59 DCHECK_EQ(kWasmI64, constant.type());
60 DCHECK_EQ(constant.to_i32_unchecked(), constant.to_i64_unchecked());
61 return {kConstant, kWasmI64, constant.to_i32_unchecked()};
62 }
63 static RegisterLoad Stack(int32_t stack_index, ValueType type) {
64 return {kStack, type, stack_index};
65 }
66 static RegisterLoad HalfStack(int32_t stack_index, RegPairHalf half) {
67 return {half == kLowWord ? kLowHalfStack : kHighHalfStack, kWasmI32,
68 stack_index};
69 }
70
71 private:
72 RegisterLoad(LoadKind kind, ValueType type, int32_t value)
73 : kind(kind), type(type), value(value) {}
74 };
75
76 public:
77 explicit StackTransferRecipe(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {}
78 ~StackTransferRecipe() { Execute(); }
79
80 void Execute() {
81 // First, execute register moves. Then load constants and stack values into
82 // registers.
83 ExecuteMoves();
84 DCHECK(move_dst_regs_.is_empty());
85 ExecuteLoads();
86 DCHECK(load_dst_regs_.is_empty());
87 }
88
89 void TransferStackSlot(const LiftoffAssembler::CacheState& dst_state,
90 uint32_t dst_index,
91 const LiftoffAssembler::CacheState& src_state,
92 uint32_t src_index) {
93 const VarState& dst = dst_state.stack_state[dst_index];
94 const VarState& src = src_state.stack_state[src_index];
95 DCHECK_EQ(dst.type(), src.type());
96 switch (dst.loc()) {
97 case VarState::kStack:
98 switch (src.loc()) {
99 case VarState::kStack:
100 if (src_index == dst_index) break;
101 asm_->MoveStackValue(dst_index, src_index, src.type());
102 break;
103 case VarState::kRegister:
104 asm_->Spill(dst_index, src.reg(), src.type());
105 break;
106 case VarState::kIntConst:
107 asm_->Spill(dst_index, src.constant());
108 break;
109 }
110 break;
111 case VarState::kRegister:
112 LoadIntoRegister(dst.reg(), src, src_index);
113 break;
114 case VarState::kIntConst:
115 DCHECK_EQ(dst, src);
116 break;
117 }
118 }
119
120 void LoadIntoRegister(LiftoffRegister dst,
121 const LiftoffAssembler::VarState& src,
122 uint32_t src_index) {
123 switch (src.loc()) {
124 case VarState::kStack:
125 LoadStackSlot(dst, src_index, src.type());
126 break;
127 case VarState::kRegister:
128 DCHECK_EQ(dst.reg_class(), src.reg_class());
129 if (dst != src.reg()) MoveRegister(dst, src.reg(), src.type());
130 break;
131 case VarState::kIntConst:
132 LoadConstant(dst, src.constant());
133 break;
134 }
135 }
136
137 void LoadI64HalfIntoRegister(LiftoffRegister dst,
138 const LiftoffAssembler::VarState& src,
139 uint32_t index, RegPairHalf half) {
140 // Use CHECK such that the remaining code is statically dead if
141 // {kNeedI64RegPair} is false.
142 CHECK(kNeedI64RegPair);
143 DCHECK_EQ(kWasmI64, src.type());
144 switch (src.loc()) {
145 case VarState::kStack:
146 LoadI64HalfStackSlot(dst, index, half);
147 break;
148 case VarState::kRegister: {
149 LiftoffRegister src_half =
150 half == kLowWord ? src.reg().low() : src.reg().high();
151 if (dst != src_half) MoveRegister(dst, src_half, kWasmI32);
152 break;
153 }
154 case VarState::kIntConst:
155 int32_t value = src.i32_const();
156 // The high word is the sign extension of the low word.
157 if (half == kHighWord) value = value >> 31;
158 LoadConstant(dst, WasmValue(value));
159 break;
160 }
161 }
162
163 void MoveRegister(LiftoffRegister dst, LiftoffRegister src, ValueType type) {
164 DCHECK_NE(dst, src);
165 DCHECK_EQ(dst.reg_class(), src.reg_class());
166 DCHECK_EQ(reg_class_for(type), src.reg_class());
167 if (src.is_pair()) {
168 DCHECK_EQ(kWasmI64, type);
169 if (dst.low() != src.low()) MoveRegister(dst.low(), src.low(), kWasmI32);
170 if (dst.high() != src.high())
171 MoveRegister(dst.high(), src.high(), kWasmI32);
172 return;
173 }
174 if (move_dst_regs_.has(dst)) {
175 DCHECK_EQ(register_move(dst)->src, src);
176 // Non-fp registers can only occur with the exact same type.
177 DCHECK_IMPLIES(!dst.is_fp(), register_move(dst)->type == type);
178 // It can happen that one fp register holds both the f32 zero and the f64
179 // zero, as the initial value for local variables. Move the value as f64
180 // in that case.
181 if (type == kWasmF64) register_move(dst)->type = kWasmF64;
182 return;
183 }
184 move_dst_regs_.set(dst);
185 ++*src_reg_use_count(src);
186 *register_move(dst) = {src, type};
187 }
188
189 void LoadConstant(LiftoffRegister dst, WasmValue value) {
190 DCHECK(!load_dst_regs_.has(dst));
191 load_dst_regs_.set(dst);
192 if (dst.is_pair()) {
193 DCHECK_EQ(kWasmI64, value.type());
194 int64_t i64 = value.to_i64();
195 *register_load(dst.low()) =
196 RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64)));
197 *register_load(dst.high()) =
198 RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64 >> 32)));
199 } else {
200 *register_load(dst) = RegisterLoad::Const(value);
201 }
202 }
203
204 void LoadStackSlot(LiftoffRegister dst, uint32_t stack_index,
205 ValueType type) {
206 if (load_dst_regs_.has(dst)) {
207 // It can happen that we spilled the same register to different stack
208 // slots, and then we reload them later into the same dst register.
209 // In that case, it is enough to load one of the stack slots.
210 return;
211 }
212 load_dst_regs_.set(dst);
213 if (dst.is_pair()) {
214 DCHECK_EQ(kWasmI64, type);
215 *register_load(dst.low()) =
216 RegisterLoad::HalfStack(stack_index, kLowWord);
217 *register_load(dst.high()) =
218 RegisterLoad::HalfStack(stack_index, kHighWord);
219 } else {
220 *register_load(dst) = RegisterLoad::Stack(stack_index, type);
221 }
222 }
223
224 void LoadI64HalfStackSlot(LiftoffRegister dst, uint32_t stack_index,
225 RegPairHalf half) {
226 if (load_dst_regs_.has(dst)) {
227 // It can happen that we spilled the same register to different stack
228 // slots, and then we reload them later into the same dst register.
229 // In that case, it is enough to load one of the stack slots.
230 return;
231 }
232 load_dst_regs_.set(dst);
233 *register_load(dst) = RegisterLoad::HalfStack(stack_index, half);
234 }
235
236 private:
237 using MovesStorage =
238 std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterMove),
239 alignof(RegisterMove)>::type;
240 using LoadsStorage =
241 std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterLoad),
242 alignof(RegisterLoad)>::type;
243
244 ASSERT_TRIVIALLY_COPYABLE(RegisterMove);
245 ASSERT_TRIVIALLY_COPYABLE(RegisterLoad);
246
247 MovesStorage register_moves_; // uninitialized
248 LoadsStorage register_loads_; // uninitialized
249 int src_reg_use_count_[kAfterMaxLiftoffRegCode] = {0};
250 LiftoffRegList move_dst_regs_;
251 LiftoffRegList load_dst_regs_;
252 LiftoffAssembler* const asm_;
253
254 RegisterMove* register_move(LiftoffRegister reg) {
255 return reinterpret_cast<RegisterMove*>(&register_moves_) +
256 reg.liftoff_code();
257 }
258 RegisterLoad* register_load(LiftoffRegister reg) {
259 return reinterpret_cast<RegisterLoad*>(&register_loads_) +
260 reg.liftoff_code();
261 }
262 int* src_reg_use_count(LiftoffRegister reg) {
263 return src_reg_use_count_ + reg.liftoff_code();
264 }
265
266 void ExecuteMove(LiftoffRegister dst) {
267 RegisterMove* move = register_move(dst);
268 DCHECK_EQ(0, *src_reg_use_count(dst));
269 asm_->Move(dst, move->src, move->type);
270 ClearExecutedMove(dst);
271 }
272
273 void ClearExecutedMove(LiftoffRegister dst) {
274 DCHECK(move_dst_regs_.has(dst));
275 move_dst_regs_.clear(dst);
276 RegisterMove* move = register_move(dst);
277 DCHECK_LT(0, *src_reg_use_count(move->src));
278 if (--*src_reg_use_count(move->src)) return;
279 // src count dropped to zero. If this is a destination register, execute
280 // that move now.
281 if (!move_dst_regs_.has(move->src)) return;
282 ExecuteMove(move->src);
283 }
284
285 void ExecuteMoves() {
286 // Execute all moves whose {dst} is not being used as src in another move.
287 // If any src count drops to zero, also (transitively) execute the
288 // corresponding move to that register.
289 for (LiftoffRegister dst : move_dst_regs_) {
290 // Check if already handled via transitivity in {ClearExecutedMove}.
291 if (!move_dst_regs_.has(dst)) continue;
292 if (*src_reg_use_count(dst)) continue;
293 ExecuteMove(dst);
294 }
295
296 // All remaining moves are parts of a cycle. Just spill the first one, then
297 // process all remaining moves in that cycle. Repeat for all cycles.
298 uint32_t next_spill_slot = asm_->cache_state()->stack_height();
299 while (!move_dst_regs_.is_empty()) {
300 // TODO(clemensh): Use an unused register if available.
301 LiftoffRegister dst = move_dst_regs_.GetFirstRegSet();
302 RegisterMove* move = register_move(dst);
303 LiftoffRegister spill_reg = move->src;
304 asm_->Spill(next_spill_slot, spill_reg, move->type);
305 // Remember to reload into the destination register later.
306 LoadStackSlot(dst, next_spill_slot, move->type);
307 ++next_spill_slot;
308 ClearExecutedMove(dst);
309 }
310 }
311
312 void ExecuteLoads() {
313 for (LiftoffRegister dst : load_dst_regs_) {
314 RegisterLoad* load = register_load(dst);
315 switch (load->kind) {
316 case RegisterLoad::kConstant:
317 asm_->LoadConstant(dst, load->type == kWasmI64
318 ? WasmValue(int64_t{load->value})
319 : WasmValue(int32_t{load->value}));
320 break;
321 case RegisterLoad::kStack:
322 asm_->Fill(dst, load->value, load->type);
323 break;
324 case RegisterLoad::kLowHalfStack:
325 // Half of a register pair, {dst} must be a gp register.
326 asm_->FillI64Half(dst.gp(), load->value, kLowWord);
327 break;
328 case RegisterLoad::kHighHalfStack:
329 // Half of a register pair, {dst} must be a gp register.
330 asm_->FillI64Half(dst.gp(), load->value, kHighWord);
331 break;
332 }
333 }
334 load_dst_regs_ = {};
335 }
336
337 DISALLOW_COPY_AND_ASSIGN(StackTransferRecipe);
338};
339
340class RegisterReuseMap {
341 public:
342 void Add(LiftoffRegister src, LiftoffRegister dst) {
343 if (auto previous = Lookup(src)) {
344 DCHECK_EQ(previous, dst);
345 return;
346 }
347 map_.emplace_back(src);
348 map_.emplace_back(dst);
349 }
350
351 base::Optional<LiftoffRegister> Lookup(LiftoffRegister src) {
352 for (auto it = map_.begin(), end = map_.end(); it != end; it += 2) {
353 if (it->is_pair() == src.is_pair() && *it == src) return *(it + 1);
354 }
355 return {};
356 }
357
358 private:
359 // {map_} holds pairs of <src, dst>.
360 base::SmallVector<LiftoffRegister, 8> map_;
361};
362
363enum MergeKeepStackSlots : bool {
364 kKeepStackSlots = true,
365 kTurnStackSlotsIntoRegisters = false
366};
367enum MergeAllowConstants : bool {
368 kConstantsAllowed = true,
369 kConstantsNotAllowed = false
370};
371enum ReuseRegisters : bool {
372 kReuseRegisters = true,
373 kNoReuseRegisters = false
374};
375void InitMergeRegion(LiftoffAssembler::CacheState* state,
376 const VarState* source, VarState* target, uint32_t count,
377 MergeKeepStackSlots keep_stack_slots,
378 MergeAllowConstants allow_constants,
379 ReuseRegisters reuse_registers, LiftoffRegList used_regs) {
380 RegisterReuseMap register_reuse_map;
381 for (const VarState* source_end = source + count; source < source_end;
382 ++source, ++target) {
383 if ((source->is_stack() && keep_stack_slots) ||
384 (source->is_const() && allow_constants)) {
385 *target = *source;
386 continue;
387 }
388 base::Optional<LiftoffRegister> reg;
389 // First try: Keep the same register, if it's free.
390 if (source->is_reg() && state->is_free(source->reg())) {
391 reg = source->reg();
392 }
393 // Second try: Use the same register we used before (if we reuse registers).
394 if (!reg && reuse_registers) {
395 reg = register_reuse_map.Lookup(source->reg());
396 }
397 // Third try: Use any free register.
398 RegClass rc = reg_class_for(source->type());
399 if (!reg && state->has_unused_register(rc, used_regs)) {
400 reg = state->unused_register(rc, used_regs);
401 }
402 if (!reg) {
403 // No free register; make this a stack slot.
404 *target = VarState(source->type());
405 continue;
406 }
407 if (reuse_registers) register_reuse_map.Add(source->reg(), *reg);
408 state->inc_used(*reg);
409 *target = VarState(source->type(), *reg);
410 }
411}
412
413} // namespace
414
415// TODO(clemensh): Don't copy the full parent state (this makes us N^2).
416void LiftoffAssembler::CacheState::InitMerge(const CacheState& source,
417 uint32_t num_locals,
418 uint32_t arity,
419 uint32_t stack_depth) {
420 // |------locals------|---(in between)----|--(discarded)--|----merge----|
421 // <-- num_locals --> <-- stack_depth -->^stack_base <-- arity -->
422
423 uint32_t stack_base = stack_depth + num_locals;
424 uint32_t target_height = stack_base + arity;
425 uint32_t discarded = source.stack_height() - target_height;
426 DCHECK(stack_state.empty());
427
428 DCHECK_GE(source.stack_height(), stack_base);
429 stack_state.resize_no_init(target_height);
430
431 const VarState* source_begin = source.stack_state.data();
432 VarState* target_begin = stack_state.data();
433
434 // Try to keep locals and the merge region in their registers. Register used
435 // multiple times need to be copied to another free register. Compute the list
436 // of used registers.
437 LiftoffRegList used_regs;
438 for (auto& src : VectorOf(source_begin, num_locals)) {
439 if (src.is_reg()) used_regs.set(src.reg());
440 }
441 for (auto& src : VectorOf(source_begin + stack_base + discarded, arity)) {
442 if (src.is_reg()) used_regs.set(src.reg());
443 }
444
445 // Initialize the merge region. If this region moves, try to turn stack slots
446 // into registers since we need to load the value anyways.
447 MergeKeepStackSlots keep_merge_stack_slots =
448 discarded == 0 ? kKeepStackSlots : kTurnStackSlotsIntoRegisters;
449 InitMergeRegion(this, source_begin + stack_base + discarded,
450 target_begin + stack_base, arity, keep_merge_stack_slots,
451 kConstantsNotAllowed, kNoReuseRegisters, used_regs);
452
453 // Initialize the locals region. Here, stack slots stay stack slots (because
454 // they do not move). Try to keep register in registers, but avoid duplicates.
455 InitMergeRegion(this, source_begin, target_begin, num_locals, kKeepStackSlots,
456 kConstantsNotAllowed, kNoReuseRegisters, used_regs);
457 // Sanity check: All the {used_regs} are really in use now.
458 DCHECK_EQ(used_regs, used_registers & used_regs);
459
460 // Last, initialize the section in between. Here, constants are allowed, but
461 // registers which are already used for the merge region or locals must be
462 // moved to other registers or spilled. If a register appears twice in the
463 // source region, ensure to use the same register twice in the target region.
464 InitMergeRegion(this, source_begin + num_locals, target_begin + num_locals,
465 stack_depth, kKeepStackSlots, kConstantsAllowed,
466 kReuseRegisters, used_regs);
467}
468
469void LiftoffAssembler::CacheState::Steal(const CacheState& source) {
470 // Just use the move assignment operator.
471 *this = std::move(source);
472}
473
474void LiftoffAssembler::CacheState::Split(const CacheState& source) {
475 // Call the private copy assignment operator.
476 *this = source;
477}
478
479namespace {
480
481constexpr AssemblerOptions DefaultLiftoffOptions() {
482 return AssemblerOptions{};
483}
484
485} // namespace
486
487// TODO(clemensh): Provide a reasonably sized buffer, based on wasm function
488// size.
489LiftoffAssembler::LiftoffAssembler(std::unique_ptr<AssemblerBuffer> buffer)
490 : TurboAssembler(nullptr, DefaultLiftoffOptions(), CodeObjectRequired::kNo,
491 std::move(buffer)) {
492 set_abort_hard(true); // Avoid calls to Abort.
493}
494
495LiftoffAssembler::~LiftoffAssembler() {
496 if (num_locals_ > kInlineLocalTypes) {
497 free(more_local_types_);
498 }
499}
500
501LiftoffRegister LiftoffAssembler::PopToRegister(LiftoffRegList pinned) {
502 DCHECK(!cache_state_.stack_state.empty());
503 VarState slot = cache_state_.stack_state.back();
504 cache_state_.stack_state.pop_back();
505 switch (slot.loc()) {
506 case VarState::kStack: {
507 LiftoffRegister reg =
508 GetUnusedRegister(reg_class_for(slot.type()), pinned);
509 Fill(reg, cache_state_.stack_height(), slot.type());
510 return reg;
511 }
512 case VarState::kRegister:
513 cache_state_.dec_used(slot.reg());
514 return slot.reg();
515 case VarState::kIntConst: {
516 RegClass rc =
517 kNeedI64RegPair && slot.type() == kWasmI64 ? kGpRegPair : kGpReg;
518 LiftoffRegister reg = GetUnusedRegister(rc, pinned);
519 LoadConstant(reg, slot.constant());
520 return reg;
521 }
522 }
523 UNREACHABLE();
524}
525
526void LiftoffAssembler::MergeFullStackWith(const CacheState& target,
527 const CacheState& source) {
528 DCHECK_EQ(source.stack_height(), target.stack_height());
529 // TODO(clemensh): Reuse the same StackTransferRecipe object to save some
530 // allocations.
531 StackTransferRecipe transfers(this);
532 for (uint32_t i = 0, e = source.stack_height(); i < e; ++i) {
533 transfers.TransferStackSlot(target, i, source, i);
534 }
535}
536
537void LiftoffAssembler::MergeStackWith(const CacheState& target,
538 uint32_t arity) {
539 // Before: ----------------|----- (discarded) ----|--- arity ---|
540 // ^target_stack_height ^stack_base ^stack_height
541 // After: ----|-- arity --|
542 // ^ ^target_stack_height
543 // ^target_stack_base
544 uint32_t stack_height = cache_state_.stack_height();
545 uint32_t target_stack_height = target.stack_height();
546 DCHECK_LE(target_stack_height, stack_height);
547 DCHECK_LE(arity, target_stack_height);
548 uint32_t stack_base = stack_height - arity;
549 uint32_t target_stack_base = target_stack_height - arity;
550 StackTransferRecipe transfers(this);
551 for (uint32_t i = 0; i < target_stack_base; ++i) {
552 transfers.TransferStackSlot(target, i, cache_state_, i);
553 }
554 for (uint32_t i = 0; i < arity; ++i) {
555 transfers.TransferStackSlot(target, target_stack_base + i, cache_state_,
556 stack_base + i);
557 }
558}
559
560void LiftoffAssembler::Spill(uint32_t index) {
561 auto& slot = cache_state_.stack_state[index];
562 switch (slot.loc()) {
563 case VarState::kStack:
564 return;
565 case VarState::kRegister:
566 Spill(index, slot.reg(), slot.type());
567 cache_state_.dec_used(slot.reg());
568 break;
569 case VarState::kIntConst:
570 Spill(index, slot.constant());
571 break;
572 }
573 slot.MakeStack();
574}
575
576void LiftoffAssembler::SpillLocals() {
577 for (uint32_t i = 0; i < num_locals_; ++i) {
578 Spill(i);
579 }
580}
581
582void LiftoffAssembler::SpillAllRegisters() {
583 for (uint32_t i = 0, e = cache_state_.stack_height(); i < e; ++i) {
584 auto& slot = cache_state_.stack_state[i];
585 if (!slot.is_reg()) continue;
586 Spill(i, slot.reg(), slot.type());
587 slot.MakeStack();
588 }
589 cache_state_.reset_used_registers();
590}
591
592void LiftoffAssembler::PrepareCall(FunctionSig* sig,
593 compiler::CallDescriptor* call_descriptor,
594 Register* target,
595 Register* target_instance) {
596 uint32_t num_params = static_cast<uint32_t>(sig->parameter_count());
597 // Input 0 is the call target.
598 constexpr size_t kInputShift = 1;
599
600 // Spill all cache slots which are not being used as parameters.
601 // Don't update any register use counters, they will be reset later anyway.
602 for (uint32_t idx = 0, end = cache_state_.stack_height() - num_params;
603 idx < end; ++idx) {
604 VarState& slot = cache_state_.stack_state[idx];
605 if (!slot.is_reg()) continue;
606 Spill(idx, slot.reg(), slot.type());
607 slot.MakeStack();
608 }
609
610 LiftoffStackSlots stack_slots(this);
611 StackTransferRecipe stack_transfers(this);
612 LiftoffRegList param_regs;
613
614 // Move the target instance (if supplied) into the correct instance register.
615 compiler::LinkageLocation instance_loc =
616 call_descriptor->GetInputLocation(kInputShift);
617 DCHECK(instance_loc.IsRegister() && !instance_loc.IsAnyRegister());
618 Register instance_reg = Register::from_code(instance_loc.AsRegister());
619 param_regs.set(instance_reg);
620 if (target_instance && *target_instance != instance_reg) {
621 stack_transfers.MoveRegister(LiftoffRegister(instance_reg),
622 LiftoffRegister(*target_instance),
623 kWasmIntPtr);
624 }
625
626 // Now move all parameter values into the right slot for the call.
627 // Don't pop values yet, such that the stack height is still correct when
628 // executing the {stack_transfers}.
629 // Process parameters backwards, such that pushes of caller frame slots are
630 // in the correct order.
631 uint32_t param_base = cache_state_.stack_height() - num_params;
632 uint32_t call_desc_input_idx =
633 static_cast<uint32_t>(call_descriptor->InputCount());
634 for (uint32_t i = num_params; i > 0; --i) {
635 const uint32_t param = i - 1;
636 ValueType type = sig->GetParam(param);
637 const bool is_pair = kNeedI64RegPair && type == kWasmI64;
638 const int num_lowered_params = is_pair ? 2 : 1;
639 const uint32_t stack_idx = param_base + param;
640 const VarState& slot = cache_state_.stack_state[stack_idx];
641 // Process both halfs of a register pair separately, because they are passed
642 // as separate parameters. One or both of them could end up on the stack.
643 for (int lowered_idx = 0; lowered_idx < num_lowered_params; ++lowered_idx) {
644 const RegPairHalf half =
645 is_pair && lowered_idx == 0 ? kHighWord : kLowWord;
646 --call_desc_input_idx;
647 compiler::LinkageLocation loc =
648 call_descriptor->GetInputLocation(call_desc_input_idx);
649 if (loc.IsRegister()) {
650 DCHECK(!loc.IsAnyRegister());
651 RegClass rc = is_pair ? kGpReg : reg_class_for(type);
652 int reg_code = loc.AsRegister();
653#if V8_TARGET_ARCH_ARM
654 // Liftoff assumes a one-to-one mapping between float registers and
655 // double registers, and so does not distinguish between f32 and f64
656 // registers. The f32 register code must therefore be halved in order to
657 // pass the f64 code to Liftoff.
658 DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0);
659 LiftoffRegister reg = LiftoffRegister::from_code(
660 rc, (type == kWasmF32) ? (reg_code / 2) : reg_code);
661#else
662 LiftoffRegister reg = LiftoffRegister::from_code(rc, reg_code);
663#endif
664 param_regs.set(reg);
665 if (is_pair) {
666 stack_transfers.LoadI64HalfIntoRegister(reg, slot, stack_idx, half);
667 } else {
668 stack_transfers.LoadIntoRegister(reg, slot, stack_idx);
669 }
670 } else {
671 DCHECK(loc.IsCallerFrameSlot());
672 stack_slots.Add(slot, stack_idx, half);
673 }
674 }
675 }
676 // {call_desc_input_idx} should point after the instance parameter now.
677 DCHECK_EQ(call_desc_input_idx, kInputShift + 1);
678
679 // If the target register overlaps with a parameter register, then move the
680 // target to another free register, or spill to the stack.
681 if (target && param_regs.has(LiftoffRegister(*target))) {
682 // Try to find another free register.
683 LiftoffRegList free_regs = kGpCacheRegList.MaskOut(param_regs);
684 if (!free_regs.is_empty()) {
685 LiftoffRegister new_target = free_regs.GetFirstRegSet();
686 stack_transfers.MoveRegister(new_target, LiftoffRegister(*target),
687 kWasmIntPtr);
688 *target = new_target.gp();
689 } else {
690 stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr,
691 LiftoffRegister(*target)));
692 *target = no_reg;
693 }
694 }
695
696 // Create all the slots.
697 stack_slots.Construct();
698 // Execute the stack transfers before filling the instance register.
699 stack_transfers.Execute();
700
701 // Pop parameters from the value stack.
702 cache_state_.stack_state.pop_back(num_params);
703
704 // Reset register use counters.
705 cache_state_.reset_used_registers();
706
707 // Reload the instance from the stack.
708 if (!target_instance) {
709 FillInstanceInto(instance_reg);
710 }
711}
712
713void LiftoffAssembler::FinishCall(FunctionSig* sig,
714 compiler::CallDescriptor* call_descriptor) {
715 const size_t return_count = sig->return_count();
716 if (return_count != 0) {
717 DCHECK_EQ(1, return_count);
718 ValueType return_type = sig->GetReturn(0);
719 const bool need_pair = kNeedI64RegPair && return_type == kWasmI64;
720 DCHECK_EQ(need_pair ? 2 : 1, call_descriptor->ReturnCount());
721 RegClass rc = need_pair ? kGpReg : reg_class_for(return_type);
722#if V8_TARGET_ARCH_ARM
723 // If the return register was not d0 for f32, the code value would have to
724 // be halved as is done for the parameter registers.
725 DCHECK_EQ(call_descriptor->GetReturnLocation(0).AsRegister(), 0);
726#endif
727 LiftoffRegister return_reg = LiftoffRegister::from_code(
728 rc, call_descriptor->GetReturnLocation(0).AsRegister());
729 DCHECK(GetCacheRegList(rc).has(return_reg));
730 if (need_pair) {
731 LiftoffRegister high_reg = LiftoffRegister::from_code(
732 rc, call_descriptor->GetReturnLocation(1).AsRegister());
733 DCHECK(GetCacheRegList(rc).has(high_reg));
734 return_reg = LiftoffRegister::ForPair(return_reg.gp(), high_reg.gp());
735 }
736 DCHECK(!cache_state_.is_used(return_reg));
737 PushRegister(return_type, return_reg);
738 }
739}
740
741void LiftoffAssembler::Move(LiftoffRegister dst, LiftoffRegister src,
742 ValueType type) {
743 DCHECK_EQ(dst.reg_class(), src.reg_class());
744 DCHECK_NE(dst, src);
745 if (kNeedI64RegPair && dst.is_pair()) {
746 // Use the {StackTransferRecipe} to move pairs, as the registers in the
747 // pairs might overlap.
748 StackTransferRecipe(this).MoveRegister(dst, src, type);
749 } else if (dst.is_gp()) {
750 Move(dst.gp(), src.gp(), type);
751 } else {
752 Move(dst.fp(), src.fp(), type);
753 }
754}
755
756void LiftoffAssembler::ParallelRegisterMove(
757 Vector<ParallelRegisterMoveTuple> tuples) {
758 StackTransferRecipe stack_transfers(this);
759 for (auto tuple : tuples) {
760 if (tuple.dst == tuple.src) continue;
761 stack_transfers.MoveRegister(tuple.dst, tuple.src, tuple.type);
762 }
763}
764
765void LiftoffAssembler::MoveToReturnRegisters(FunctionSig* sig) {
766 // We do not support multi-value yet.
767 DCHECK_EQ(1, sig->return_count());
768 ValueType return_type = sig->GetReturn(0);
769 StackTransferRecipe stack_transfers(this);
770 LiftoffRegister return_reg =
771 needs_reg_pair(return_type)
772 ? LiftoffRegister::ForPair(kGpReturnRegisters[0],
773 kGpReturnRegisters[1])
774 : reg_class_for(return_type) == kGpReg
775 ? LiftoffRegister(kGpReturnRegisters[0])
776 : LiftoffRegister(kFpReturnRegisters[0]);
777 stack_transfers.LoadIntoRegister(return_reg, cache_state_.stack_state.back(),
778 cache_state_.stack_height() - 1);
779}
780
781#ifdef ENABLE_SLOW_DCHECKS
782bool LiftoffAssembler::ValidateCacheState() const {
783 uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0};
784 LiftoffRegList used_regs;
785 for (const VarState& var : cache_state_.stack_state) {
786 if (!var.is_reg()) continue;
787 LiftoffRegister reg = var.reg();
788 if (kNeedI64RegPair && reg.is_pair()) {
789 ++register_use_count[reg.low().liftoff_code()];
790 ++register_use_count[reg.high().liftoff_code()];
791 } else {
792 ++register_use_count[reg.liftoff_code()];
793 }
794 used_regs.set(reg);
795 }
796 bool valid = memcmp(register_use_count, cache_state_.register_use_count,
797 sizeof(register_use_count)) == 0 &&
798 used_regs == cache_state_.used_registers;
799 if (valid) return true;
800 std::ostringstream os;
801 os << "Error in LiftoffAssembler::ValidateCacheState().\n";
802 os << "expected: used_regs " << used_regs << ", counts "
803 << PrintCollection(register_use_count) << "\n";
804 os << "found: used_regs " << cache_state_.used_registers << ", counts "
805 << PrintCollection(cache_state_.register_use_count) << "\n";
806 os << "Use --trace-wasm-decoder and --trace-liftoff to debug.";
807 FATAL("%s", os.str().c_str());
808}
809#endif
810
811LiftoffRegister LiftoffAssembler::SpillOneRegister(LiftoffRegList candidates,
812 LiftoffRegList pinned) {
813 // Spill one cached value to free a register.
814 LiftoffRegister spill_reg = cache_state_.GetNextSpillReg(candidates, pinned);
815 SpillRegister(spill_reg);
816 return spill_reg;
817}
818
819void LiftoffAssembler::SpillRegister(LiftoffRegister reg) {
820 int remaining_uses = cache_state_.get_use_count(reg);
821 DCHECK_LT(0, remaining_uses);
822 for (uint32_t idx = cache_state_.stack_height() - 1;; --idx) {
823 DCHECK_GT(cache_state_.stack_height(), idx);
824 auto* slot = &cache_state_.stack_state[idx];
825 if (!slot->is_reg() || !slot->reg().overlaps(reg)) continue;
826 if (slot->reg().is_pair()) {
827 // Make sure to decrement *both* registers in a pair, because the
828 // {clear_used} call below only clears one of them.
829 cache_state_.dec_used(slot->reg().low());
830 cache_state_.dec_used(slot->reg().high());
831 }
832 Spill(idx, slot->reg(), slot->type());
833 slot->MakeStack();
834 if (--remaining_uses == 0) break;
835 }
836 cache_state_.clear_used(reg);
837}
838
839void LiftoffAssembler::set_num_locals(uint32_t num_locals) {
840 DCHECK_EQ(0, num_locals_); // only call this once.
841 num_locals_ = num_locals;
842 if (num_locals > kInlineLocalTypes) {
843 more_local_types_ =
844 reinterpret_cast<ValueType*>(malloc(num_locals * sizeof(ValueType)));
845 DCHECK_NOT_NULL(more_local_types_);
846 }
847}
848
849std::ostream& operator<<(std::ostream& os, VarState slot) {
850 os << ValueTypes::TypeName(slot.type()) << ":";
851 switch (slot.loc()) {
852 case VarState::kStack:
853 return os << "s";
854 case VarState::kRegister:
855 return os << slot.reg();
856 case VarState::kIntConst:
857 return os << "c" << slot.i32_const();
858 }
859 UNREACHABLE();
860}
861
862#undef __
863#undef TRACE
864
865} // namespace wasm
866} // namespace internal
867} // namespace v8
868