1 | // Copyright 2017 the V8 project authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | #include "src/wasm/baseline/liftoff-assembler.h" |
6 | |
7 | #include <sstream> |
8 | |
9 | #include "src/assembler-inl.h" |
10 | #include "src/base/optional.h" |
11 | #include "src/compiler/linkage.h" |
12 | #include "src/compiler/wasm-compiler.h" |
13 | #include "src/macro-assembler-inl.h" |
14 | #include "src/ostreams.h" |
15 | #include "src/wasm/function-body-decoder-impl.h" |
16 | #include "src/wasm/wasm-linkage.h" |
17 | #include "src/wasm/wasm-opcodes.h" |
18 | |
19 | namespace v8 { |
20 | namespace internal { |
21 | namespace wasm { |
22 | |
23 | using VarState = LiftoffAssembler::VarState; |
24 | |
25 | namespace { |
26 | |
27 | #define __ asm_-> |
28 | |
29 | #define TRACE(...) \ |
30 | do { \ |
31 | if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \ |
32 | } while (false) |
33 | |
34 | class StackTransferRecipe { |
35 | struct RegisterMove { |
36 | LiftoffRegister src; |
37 | ValueType type; |
38 | constexpr RegisterMove(LiftoffRegister src, ValueType type) |
39 | : src(src), type(type) {} |
40 | }; |
41 | |
42 | struct RegisterLoad { |
43 | enum LoadKind : uint8_t { |
44 | kConstant, // load a constant value into a register. |
45 | kStack, // fill a register from a stack slot. |
46 | kLowHalfStack, // fill a register from the low half of a stack slot. |
47 | kHighHalfStack // fill a register from the high half of a stack slot. |
48 | }; |
49 | |
50 | LoadKind kind; |
51 | ValueType type; |
52 | int32_t value; // i32 constant value or stack index, depending on kind. |
53 | |
54 | // Named constructors. |
55 | static RegisterLoad Const(WasmValue constant) { |
56 | if (constant.type() == kWasmI32) { |
57 | return {kConstant, kWasmI32, constant.to_i32()}; |
58 | } |
59 | DCHECK_EQ(kWasmI64, constant.type()); |
60 | DCHECK_EQ(constant.to_i32_unchecked(), constant.to_i64_unchecked()); |
61 | return {kConstant, kWasmI64, constant.to_i32_unchecked()}; |
62 | } |
63 | static RegisterLoad Stack(int32_t stack_index, ValueType type) { |
64 | return {kStack, type, stack_index}; |
65 | } |
66 | static RegisterLoad HalfStack(int32_t stack_index, RegPairHalf half) { |
67 | return {half == kLowWord ? kLowHalfStack : kHighHalfStack, kWasmI32, |
68 | stack_index}; |
69 | } |
70 | |
71 | private: |
72 | RegisterLoad(LoadKind kind, ValueType type, int32_t value) |
73 | : kind(kind), type(type), value(value) {} |
74 | }; |
75 | |
76 | public: |
77 | explicit StackTransferRecipe(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {} |
78 | ~StackTransferRecipe() { Execute(); } |
79 | |
80 | void Execute() { |
81 | // First, execute register moves. Then load constants and stack values into |
82 | // registers. |
83 | ExecuteMoves(); |
84 | DCHECK(move_dst_regs_.is_empty()); |
85 | ExecuteLoads(); |
86 | DCHECK(load_dst_regs_.is_empty()); |
87 | } |
88 | |
89 | void TransferStackSlot(const LiftoffAssembler::CacheState& dst_state, |
90 | uint32_t dst_index, |
91 | const LiftoffAssembler::CacheState& src_state, |
92 | uint32_t src_index) { |
93 | const VarState& dst = dst_state.stack_state[dst_index]; |
94 | const VarState& src = src_state.stack_state[src_index]; |
95 | DCHECK_EQ(dst.type(), src.type()); |
96 | switch (dst.loc()) { |
97 | case VarState::kStack: |
98 | switch (src.loc()) { |
99 | case VarState::kStack: |
100 | if (src_index == dst_index) break; |
101 | asm_->MoveStackValue(dst_index, src_index, src.type()); |
102 | break; |
103 | case VarState::kRegister: |
104 | asm_->Spill(dst_index, src.reg(), src.type()); |
105 | break; |
106 | case VarState::kIntConst: |
107 | asm_->Spill(dst_index, src.constant()); |
108 | break; |
109 | } |
110 | break; |
111 | case VarState::kRegister: |
112 | LoadIntoRegister(dst.reg(), src, src_index); |
113 | break; |
114 | case VarState::kIntConst: |
115 | DCHECK_EQ(dst, src); |
116 | break; |
117 | } |
118 | } |
119 | |
120 | void LoadIntoRegister(LiftoffRegister dst, |
121 | const LiftoffAssembler::VarState& src, |
122 | uint32_t src_index) { |
123 | switch (src.loc()) { |
124 | case VarState::kStack: |
125 | LoadStackSlot(dst, src_index, src.type()); |
126 | break; |
127 | case VarState::kRegister: |
128 | DCHECK_EQ(dst.reg_class(), src.reg_class()); |
129 | if (dst != src.reg()) MoveRegister(dst, src.reg(), src.type()); |
130 | break; |
131 | case VarState::kIntConst: |
132 | LoadConstant(dst, src.constant()); |
133 | break; |
134 | } |
135 | } |
136 | |
137 | void LoadI64HalfIntoRegister(LiftoffRegister dst, |
138 | const LiftoffAssembler::VarState& src, |
139 | uint32_t index, RegPairHalf half) { |
140 | // Use CHECK such that the remaining code is statically dead if |
141 | // {kNeedI64RegPair} is false. |
142 | CHECK(kNeedI64RegPair); |
143 | DCHECK_EQ(kWasmI64, src.type()); |
144 | switch (src.loc()) { |
145 | case VarState::kStack: |
146 | LoadI64HalfStackSlot(dst, index, half); |
147 | break; |
148 | case VarState::kRegister: { |
149 | LiftoffRegister src_half = |
150 | half == kLowWord ? src.reg().low() : src.reg().high(); |
151 | if (dst != src_half) MoveRegister(dst, src_half, kWasmI32); |
152 | break; |
153 | } |
154 | case VarState::kIntConst: |
155 | int32_t value = src.i32_const(); |
156 | // The high word is the sign extension of the low word. |
157 | if (half == kHighWord) value = value >> 31; |
158 | LoadConstant(dst, WasmValue(value)); |
159 | break; |
160 | } |
161 | } |
162 | |
163 | void MoveRegister(LiftoffRegister dst, LiftoffRegister src, ValueType type) { |
164 | DCHECK_NE(dst, src); |
165 | DCHECK_EQ(dst.reg_class(), src.reg_class()); |
166 | DCHECK_EQ(reg_class_for(type), src.reg_class()); |
167 | if (src.is_pair()) { |
168 | DCHECK_EQ(kWasmI64, type); |
169 | if (dst.low() != src.low()) MoveRegister(dst.low(), src.low(), kWasmI32); |
170 | if (dst.high() != src.high()) |
171 | MoveRegister(dst.high(), src.high(), kWasmI32); |
172 | return; |
173 | } |
174 | if (move_dst_regs_.has(dst)) { |
175 | DCHECK_EQ(register_move(dst)->src, src); |
176 | // Non-fp registers can only occur with the exact same type. |
177 | DCHECK_IMPLIES(!dst.is_fp(), register_move(dst)->type == type); |
178 | // It can happen that one fp register holds both the f32 zero and the f64 |
179 | // zero, as the initial value for local variables. Move the value as f64 |
180 | // in that case. |
181 | if (type == kWasmF64) register_move(dst)->type = kWasmF64; |
182 | return; |
183 | } |
184 | move_dst_regs_.set(dst); |
185 | ++*src_reg_use_count(src); |
186 | *register_move(dst) = {src, type}; |
187 | } |
188 | |
189 | void LoadConstant(LiftoffRegister dst, WasmValue value) { |
190 | DCHECK(!load_dst_regs_.has(dst)); |
191 | load_dst_regs_.set(dst); |
192 | if (dst.is_pair()) { |
193 | DCHECK_EQ(kWasmI64, value.type()); |
194 | int64_t i64 = value.to_i64(); |
195 | *register_load(dst.low()) = |
196 | RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64))); |
197 | *register_load(dst.high()) = |
198 | RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64 >> 32))); |
199 | } else { |
200 | *register_load(dst) = RegisterLoad::Const(value); |
201 | } |
202 | } |
203 | |
204 | void LoadStackSlot(LiftoffRegister dst, uint32_t stack_index, |
205 | ValueType type) { |
206 | if (load_dst_regs_.has(dst)) { |
207 | // It can happen that we spilled the same register to different stack |
208 | // slots, and then we reload them later into the same dst register. |
209 | // In that case, it is enough to load one of the stack slots. |
210 | return; |
211 | } |
212 | load_dst_regs_.set(dst); |
213 | if (dst.is_pair()) { |
214 | DCHECK_EQ(kWasmI64, type); |
215 | *register_load(dst.low()) = |
216 | RegisterLoad::HalfStack(stack_index, kLowWord); |
217 | *register_load(dst.high()) = |
218 | RegisterLoad::HalfStack(stack_index, kHighWord); |
219 | } else { |
220 | *register_load(dst) = RegisterLoad::Stack(stack_index, type); |
221 | } |
222 | } |
223 | |
224 | void LoadI64HalfStackSlot(LiftoffRegister dst, uint32_t stack_index, |
225 | RegPairHalf half) { |
226 | if (load_dst_regs_.has(dst)) { |
227 | // It can happen that we spilled the same register to different stack |
228 | // slots, and then we reload them later into the same dst register. |
229 | // In that case, it is enough to load one of the stack slots. |
230 | return; |
231 | } |
232 | load_dst_regs_.set(dst); |
233 | *register_load(dst) = RegisterLoad::HalfStack(stack_index, half); |
234 | } |
235 | |
236 | private: |
237 | using MovesStorage = |
238 | std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterMove), |
239 | alignof(RegisterMove)>::type; |
240 | using LoadsStorage = |
241 | std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterLoad), |
242 | alignof(RegisterLoad)>::type; |
243 | |
244 | ASSERT_TRIVIALLY_COPYABLE(RegisterMove); |
245 | ASSERT_TRIVIALLY_COPYABLE(RegisterLoad); |
246 | |
247 | MovesStorage register_moves_; // uninitialized |
248 | LoadsStorage register_loads_; // uninitialized |
249 | int src_reg_use_count_[kAfterMaxLiftoffRegCode] = {0}; |
250 | LiftoffRegList move_dst_regs_; |
251 | LiftoffRegList load_dst_regs_; |
252 | LiftoffAssembler* const asm_; |
253 | |
254 | RegisterMove* register_move(LiftoffRegister reg) { |
255 | return reinterpret_cast<RegisterMove*>(®ister_moves_) + |
256 | reg.liftoff_code(); |
257 | } |
258 | RegisterLoad* register_load(LiftoffRegister reg) { |
259 | return reinterpret_cast<RegisterLoad*>(®ister_loads_) + |
260 | reg.liftoff_code(); |
261 | } |
262 | int* src_reg_use_count(LiftoffRegister reg) { |
263 | return src_reg_use_count_ + reg.liftoff_code(); |
264 | } |
265 | |
266 | void ExecuteMove(LiftoffRegister dst) { |
267 | RegisterMove* move = register_move(dst); |
268 | DCHECK_EQ(0, *src_reg_use_count(dst)); |
269 | asm_->Move(dst, move->src, move->type); |
270 | ClearExecutedMove(dst); |
271 | } |
272 | |
273 | void ClearExecutedMove(LiftoffRegister dst) { |
274 | DCHECK(move_dst_regs_.has(dst)); |
275 | move_dst_regs_.clear(dst); |
276 | RegisterMove* move = register_move(dst); |
277 | DCHECK_LT(0, *src_reg_use_count(move->src)); |
278 | if (--*src_reg_use_count(move->src)) return; |
279 | // src count dropped to zero. If this is a destination register, execute |
280 | // that move now. |
281 | if (!move_dst_regs_.has(move->src)) return; |
282 | ExecuteMove(move->src); |
283 | } |
284 | |
285 | void ExecuteMoves() { |
286 | // Execute all moves whose {dst} is not being used as src in another move. |
287 | // If any src count drops to zero, also (transitively) execute the |
288 | // corresponding move to that register. |
289 | for (LiftoffRegister dst : move_dst_regs_) { |
290 | // Check if already handled via transitivity in {ClearExecutedMove}. |
291 | if (!move_dst_regs_.has(dst)) continue; |
292 | if (*src_reg_use_count(dst)) continue; |
293 | ExecuteMove(dst); |
294 | } |
295 | |
296 | // All remaining moves are parts of a cycle. Just spill the first one, then |
297 | // process all remaining moves in that cycle. Repeat for all cycles. |
298 | uint32_t next_spill_slot = asm_->cache_state()->stack_height(); |
299 | while (!move_dst_regs_.is_empty()) { |
300 | // TODO(clemensh): Use an unused register if available. |
301 | LiftoffRegister dst = move_dst_regs_.GetFirstRegSet(); |
302 | RegisterMove* move = register_move(dst); |
303 | LiftoffRegister spill_reg = move->src; |
304 | asm_->Spill(next_spill_slot, spill_reg, move->type); |
305 | // Remember to reload into the destination register later. |
306 | LoadStackSlot(dst, next_spill_slot, move->type); |
307 | ++next_spill_slot; |
308 | ClearExecutedMove(dst); |
309 | } |
310 | } |
311 | |
312 | void ExecuteLoads() { |
313 | for (LiftoffRegister dst : load_dst_regs_) { |
314 | RegisterLoad* load = register_load(dst); |
315 | switch (load->kind) { |
316 | case RegisterLoad::kConstant: |
317 | asm_->LoadConstant(dst, load->type == kWasmI64 |
318 | ? WasmValue(int64_t{load->value}) |
319 | : WasmValue(int32_t{load->value})); |
320 | break; |
321 | case RegisterLoad::kStack: |
322 | asm_->Fill(dst, load->value, load->type); |
323 | break; |
324 | case RegisterLoad::kLowHalfStack: |
325 | // Half of a register pair, {dst} must be a gp register. |
326 | asm_->FillI64Half(dst.gp(), load->value, kLowWord); |
327 | break; |
328 | case RegisterLoad::kHighHalfStack: |
329 | // Half of a register pair, {dst} must be a gp register. |
330 | asm_->FillI64Half(dst.gp(), load->value, kHighWord); |
331 | break; |
332 | } |
333 | } |
334 | load_dst_regs_ = {}; |
335 | } |
336 | |
337 | DISALLOW_COPY_AND_ASSIGN(StackTransferRecipe); |
338 | }; |
339 | |
340 | class RegisterReuseMap { |
341 | public: |
342 | void Add(LiftoffRegister src, LiftoffRegister dst) { |
343 | if (auto previous = Lookup(src)) { |
344 | DCHECK_EQ(previous, dst); |
345 | return; |
346 | } |
347 | map_.emplace_back(src); |
348 | map_.emplace_back(dst); |
349 | } |
350 | |
351 | base::Optional<LiftoffRegister> Lookup(LiftoffRegister src) { |
352 | for (auto it = map_.begin(), end = map_.end(); it != end; it += 2) { |
353 | if (it->is_pair() == src.is_pair() && *it == src) return *(it + 1); |
354 | } |
355 | return {}; |
356 | } |
357 | |
358 | private: |
359 | // {map_} holds pairs of <src, dst>. |
360 | base::SmallVector<LiftoffRegister, 8> map_; |
361 | }; |
362 | |
363 | enum MergeKeepStackSlots : bool { |
364 | kKeepStackSlots = true, |
365 | kTurnStackSlotsIntoRegisters = false |
366 | }; |
367 | enum MergeAllowConstants : bool { |
368 | kConstantsAllowed = true, |
369 | kConstantsNotAllowed = false |
370 | }; |
371 | enum ReuseRegisters : bool { |
372 | kReuseRegisters = true, |
373 | kNoReuseRegisters = false |
374 | }; |
375 | void InitMergeRegion(LiftoffAssembler::CacheState* state, |
376 | const VarState* source, VarState* target, uint32_t count, |
377 | MergeKeepStackSlots keep_stack_slots, |
378 | MergeAllowConstants allow_constants, |
379 | ReuseRegisters reuse_registers, LiftoffRegList used_regs) { |
380 | RegisterReuseMap register_reuse_map; |
381 | for (const VarState* source_end = source + count; source < source_end; |
382 | ++source, ++target) { |
383 | if ((source->is_stack() && keep_stack_slots) || |
384 | (source->is_const() && allow_constants)) { |
385 | *target = *source; |
386 | continue; |
387 | } |
388 | base::Optional<LiftoffRegister> reg; |
389 | // First try: Keep the same register, if it's free. |
390 | if (source->is_reg() && state->is_free(source->reg())) { |
391 | reg = source->reg(); |
392 | } |
393 | // Second try: Use the same register we used before (if we reuse registers). |
394 | if (!reg && reuse_registers) { |
395 | reg = register_reuse_map.Lookup(source->reg()); |
396 | } |
397 | // Third try: Use any free register. |
398 | RegClass rc = reg_class_for(source->type()); |
399 | if (!reg && state->has_unused_register(rc, used_regs)) { |
400 | reg = state->unused_register(rc, used_regs); |
401 | } |
402 | if (!reg) { |
403 | // No free register; make this a stack slot. |
404 | *target = VarState(source->type()); |
405 | continue; |
406 | } |
407 | if (reuse_registers) register_reuse_map.Add(source->reg(), *reg); |
408 | state->inc_used(*reg); |
409 | *target = VarState(source->type(), *reg); |
410 | } |
411 | } |
412 | |
413 | } // namespace |
414 | |
415 | // TODO(clemensh): Don't copy the full parent state (this makes us N^2). |
416 | void LiftoffAssembler::CacheState::InitMerge(const CacheState& source, |
417 | uint32_t num_locals, |
418 | uint32_t arity, |
419 | uint32_t stack_depth) { |
420 | // |------locals------|---(in between)----|--(discarded)--|----merge----| |
421 | // <-- num_locals --> <-- stack_depth -->^stack_base <-- arity --> |
422 | |
423 | uint32_t stack_base = stack_depth + num_locals; |
424 | uint32_t target_height = stack_base + arity; |
425 | uint32_t discarded = source.stack_height() - target_height; |
426 | DCHECK(stack_state.empty()); |
427 | |
428 | DCHECK_GE(source.stack_height(), stack_base); |
429 | stack_state.resize_no_init(target_height); |
430 | |
431 | const VarState* source_begin = source.stack_state.data(); |
432 | VarState* target_begin = stack_state.data(); |
433 | |
434 | // Try to keep locals and the merge region in their registers. Register used |
435 | // multiple times need to be copied to another free register. Compute the list |
436 | // of used registers. |
437 | LiftoffRegList used_regs; |
438 | for (auto& src : VectorOf(source_begin, num_locals)) { |
439 | if (src.is_reg()) used_regs.set(src.reg()); |
440 | } |
441 | for (auto& src : VectorOf(source_begin + stack_base + discarded, arity)) { |
442 | if (src.is_reg()) used_regs.set(src.reg()); |
443 | } |
444 | |
445 | // Initialize the merge region. If this region moves, try to turn stack slots |
446 | // into registers since we need to load the value anyways. |
447 | MergeKeepStackSlots keep_merge_stack_slots = |
448 | discarded == 0 ? kKeepStackSlots : kTurnStackSlotsIntoRegisters; |
449 | InitMergeRegion(this, source_begin + stack_base + discarded, |
450 | target_begin + stack_base, arity, keep_merge_stack_slots, |
451 | kConstantsNotAllowed, kNoReuseRegisters, used_regs); |
452 | |
453 | // Initialize the locals region. Here, stack slots stay stack slots (because |
454 | // they do not move). Try to keep register in registers, but avoid duplicates. |
455 | InitMergeRegion(this, source_begin, target_begin, num_locals, kKeepStackSlots, |
456 | kConstantsNotAllowed, kNoReuseRegisters, used_regs); |
457 | // Sanity check: All the {used_regs} are really in use now. |
458 | DCHECK_EQ(used_regs, used_registers & used_regs); |
459 | |
460 | // Last, initialize the section in between. Here, constants are allowed, but |
461 | // registers which are already used for the merge region or locals must be |
462 | // moved to other registers or spilled. If a register appears twice in the |
463 | // source region, ensure to use the same register twice in the target region. |
464 | InitMergeRegion(this, source_begin + num_locals, target_begin + num_locals, |
465 | stack_depth, kKeepStackSlots, kConstantsAllowed, |
466 | kReuseRegisters, used_regs); |
467 | } |
468 | |
469 | void LiftoffAssembler::CacheState::Steal(const CacheState& source) { |
470 | // Just use the move assignment operator. |
471 | *this = std::move(source); |
472 | } |
473 | |
474 | void LiftoffAssembler::CacheState::Split(const CacheState& source) { |
475 | // Call the private copy assignment operator. |
476 | *this = source; |
477 | } |
478 | |
479 | namespace { |
480 | |
481 | constexpr AssemblerOptions DefaultLiftoffOptions() { |
482 | return AssemblerOptions{}; |
483 | } |
484 | |
485 | } // namespace |
486 | |
487 | // TODO(clemensh): Provide a reasonably sized buffer, based on wasm function |
488 | // size. |
489 | LiftoffAssembler::LiftoffAssembler(std::unique_ptr<AssemblerBuffer> buffer) |
490 | : TurboAssembler(nullptr, DefaultLiftoffOptions(), CodeObjectRequired::kNo, |
491 | std::move(buffer)) { |
492 | set_abort_hard(true); // Avoid calls to Abort. |
493 | } |
494 | |
495 | LiftoffAssembler::~LiftoffAssembler() { |
496 | if (num_locals_ > kInlineLocalTypes) { |
497 | free(more_local_types_); |
498 | } |
499 | } |
500 | |
501 | LiftoffRegister LiftoffAssembler::PopToRegister(LiftoffRegList pinned) { |
502 | DCHECK(!cache_state_.stack_state.empty()); |
503 | VarState slot = cache_state_.stack_state.back(); |
504 | cache_state_.stack_state.pop_back(); |
505 | switch (slot.loc()) { |
506 | case VarState::kStack: { |
507 | LiftoffRegister reg = |
508 | GetUnusedRegister(reg_class_for(slot.type()), pinned); |
509 | Fill(reg, cache_state_.stack_height(), slot.type()); |
510 | return reg; |
511 | } |
512 | case VarState::kRegister: |
513 | cache_state_.dec_used(slot.reg()); |
514 | return slot.reg(); |
515 | case VarState::kIntConst: { |
516 | RegClass rc = |
517 | kNeedI64RegPair && slot.type() == kWasmI64 ? kGpRegPair : kGpReg; |
518 | LiftoffRegister reg = GetUnusedRegister(rc, pinned); |
519 | LoadConstant(reg, slot.constant()); |
520 | return reg; |
521 | } |
522 | } |
523 | UNREACHABLE(); |
524 | } |
525 | |
526 | void LiftoffAssembler::MergeFullStackWith(const CacheState& target, |
527 | const CacheState& source) { |
528 | DCHECK_EQ(source.stack_height(), target.stack_height()); |
529 | // TODO(clemensh): Reuse the same StackTransferRecipe object to save some |
530 | // allocations. |
531 | StackTransferRecipe transfers(this); |
532 | for (uint32_t i = 0, e = source.stack_height(); i < e; ++i) { |
533 | transfers.TransferStackSlot(target, i, source, i); |
534 | } |
535 | } |
536 | |
537 | void LiftoffAssembler::MergeStackWith(const CacheState& target, |
538 | uint32_t arity) { |
539 | // Before: ----------------|----- (discarded) ----|--- arity ---| |
540 | // ^target_stack_height ^stack_base ^stack_height |
541 | // After: ----|-- arity --| |
542 | // ^ ^target_stack_height |
543 | // ^target_stack_base |
544 | uint32_t stack_height = cache_state_.stack_height(); |
545 | uint32_t target_stack_height = target.stack_height(); |
546 | DCHECK_LE(target_stack_height, stack_height); |
547 | DCHECK_LE(arity, target_stack_height); |
548 | uint32_t stack_base = stack_height - arity; |
549 | uint32_t target_stack_base = target_stack_height - arity; |
550 | StackTransferRecipe transfers(this); |
551 | for (uint32_t i = 0; i < target_stack_base; ++i) { |
552 | transfers.TransferStackSlot(target, i, cache_state_, i); |
553 | } |
554 | for (uint32_t i = 0; i < arity; ++i) { |
555 | transfers.TransferStackSlot(target, target_stack_base + i, cache_state_, |
556 | stack_base + i); |
557 | } |
558 | } |
559 | |
560 | void LiftoffAssembler::Spill(uint32_t index) { |
561 | auto& slot = cache_state_.stack_state[index]; |
562 | switch (slot.loc()) { |
563 | case VarState::kStack: |
564 | return; |
565 | case VarState::kRegister: |
566 | Spill(index, slot.reg(), slot.type()); |
567 | cache_state_.dec_used(slot.reg()); |
568 | break; |
569 | case VarState::kIntConst: |
570 | Spill(index, slot.constant()); |
571 | break; |
572 | } |
573 | slot.MakeStack(); |
574 | } |
575 | |
576 | void LiftoffAssembler::SpillLocals() { |
577 | for (uint32_t i = 0; i < num_locals_; ++i) { |
578 | Spill(i); |
579 | } |
580 | } |
581 | |
582 | void LiftoffAssembler::SpillAllRegisters() { |
583 | for (uint32_t i = 0, e = cache_state_.stack_height(); i < e; ++i) { |
584 | auto& slot = cache_state_.stack_state[i]; |
585 | if (!slot.is_reg()) continue; |
586 | Spill(i, slot.reg(), slot.type()); |
587 | slot.MakeStack(); |
588 | } |
589 | cache_state_.reset_used_registers(); |
590 | } |
591 | |
592 | void LiftoffAssembler::PrepareCall(FunctionSig* sig, |
593 | compiler::CallDescriptor* call_descriptor, |
594 | Register* target, |
595 | Register* target_instance) { |
596 | uint32_t num_params = static_cast<uint32_t>(sig->parameter_count()); |
597 | // Input 0 is the call target. |
598 | constexpr size_t kInputShift = 1; |
599 | |
600 | // Spill all cache slots which are not being used as parameters. |
601 | // Don't update any register use counters, they will be reset later anyway. |
602 | for (uint32_t idx = 0, end = cache_state_.stack_height() - num_params; |
603 | idx < end; ++idx) { |
604 | VarState& slot = cache_state_.stack_state[idx]; |
605 | if (!slot.is_reg()) continue; |
606 | Spill(idx, slot.reg(), slot.type()); |
607 | slot.MakeStack(); |
608 | } |
609 | |
610 | LiftoffStackSlots stack_slots(this); |
611 | StackTransferRecipe stack_transfers(this); |
612 | LiftoffRegList param_regs; |
613 | |
614 | // Move the target instance (if supplied) into the correct instance register. |
615 | compiler::LinkageLocation instance_loc = |
616 | call_descriptor->GetInputLocation(kInputShift); |
617 | DCHECK(instance_loc.IsRegister() && !instance_loc.IsAnyRegister()); |
618 | Register instance_reg = Register::from_code(instance_loc.AsRegister()); |
619 | param_regs.set(instance_reg); |
620 | if (target_instance && *target_instance != instance_reg) { |
621 | stack_transfers.MoveRegister(LiftoffRegister(instance_reg), |
622 | LiftoffRegister(*target_instance), |
623 | kWasmIntPtr); |
624 | } |
625 | |
626 | // Now move all parameter values into the right slot for the call. |
627 | // Don't pop values yet, such that the stack height is still correct when |
628 | // executing the {stack_transfers}. |
629 | // Process parameters backwards, such that pushes of caller frame slots are |
630 | // in the correct order. |
631 | uint32_t param_base = cache_state_.stack_height() - num_params; |
632 | uint32_t call_desc_input_idx = |
633 | static_cast<uint32_t>(call_descriptor->InputCount()); |
634 | for (uint32_t i = num_params; i > 0; --i) { |
635 | const uint32_t param = i - 1; |
636 | ValueType type = sig->GetParam(param); |
637 | const bool is_pair = kNeedI64RegPair && type == kWasmI64; |
638 | const int num_lowered_params = is_pair ? 2 : 1; |
639 | const uint32_t stack_idx = param_base + param; |
640 | const VarState& slot = cache_state_.stack_state[stack_idx]; |
641 | // Process both halfs of a register pair separately, because they are passed |
642 | // as separate parameters. One or both of them could end up on the stack. |
643 | for (int lowered_idx = 0; lowered_idx < num_lowered_params; ++lowered_idx) { |
644 | const RegPairHalf half = |
645 | is_pair && lowered_idx == 0 ? kHighWord : kLowWord; |
646 | --call_desc_input_idx; |
647 | compiler::LinkageLocation loc = |
648 | call_descriptor->GetInputLocation(call_desc_input_idx); |
649 | if (loc.IsRegister()) { |
650 | DCHECK(!loc.IsAnyRegister()); |
651 | RegClass rc = is_pair ? kGpReg : reg_class_for(type); |
652 | int reg_code = loc.AsRegister(); |
653 | #if V8_TARGET_ARCH_ARM |
654 | // Liftoff assumes a one-to-one mapping between float registers and |
655 | // double registers, and so does not distinguish between f32 and f64 |
656 | // registers. The f32 register code must therefore be halved in order to |
657 | // pass the f64 code to Liftoff. |
658 | DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0); |
659 | LiftoffRegister reg = LiftoffRegister::from_code( |
660 | rc, (type == kWasmF32) ? (reg_code / 2) : reg_code); |
661 | #else |
662 | LiftoffRegister reg = LiftoffRegister::from_code(rc, reg_code); |
663 | #endif |
664 | param_regs.set(reg); |
665 | if (is_pair) { |
666 | stack_transfers.LoadI64HalfIntoRegister(reg, slot, stack_idx, half); |
667 | } else { |
668 | stack_transfers.LoadIntoRegister(reg, slot, stack_idx); |
669 | } |
670 | } else { |
671 | DCHECK(loc.IsCallerFrameSlot()); |
672 | stack_slots.Add(slot, stack_idx, half); |
673 | } |
674 | } |
675 | } |
676 | // {call_desc_input_idx} should point after the instance parameter now. |
677 | DCHECK_EQ(call_desc_input_idx, kInputShift + 1); |
678 | |
679 | // If the target register overlaps with a parameter register, then move the |
680 | // target to another free register, or spill to the stack. |
681 | if (target && param_regs.has(LiftoffRegister(*target))) { |
682 | // Try to find another free register. |
683 | LiftoffRegList free_regs = kGpCacheRegList.MaskOut(param_regs); |
684 | if (!free_regs.is_empty()) { |
685 | LiftoffRegister new_target = free_regs.GetFirstRegSet(); |
686 | stack_transfers.MoveRegister(new_target, LiftoffRegister(*target), |
687 | kWasmIntPtr); |
688 | *target = new_target.gp(); |
689 | } else { |
690 | stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr, |
691 | LiftoffRegister(*target))); |
692 | *target = no_reg; |
693 | } |
694 | } |
695 | |
696 | // Create all the slots. |
697 | stack_slots.Construct(); |
698 | // Execute the stack transfers before filling the instance register. |
699 | stack_transfers.Execute(); |
700 | |
701 | // Pop parameters from the value stack. |
702 | cache_state_.stack_state.pop_back(num_params); |
703 | |
704 | // Reset register use counters. |
705 | cache_state_.reset_used_registers(); |
706 | |
707 | // Reload the instance from the stack. |
708 | if (!target_instance) { |
709 | FillInstanceInto(instance_reg); |
710 | } |
711 | } |
712 | |
713 | void LiftoffAssembler::FinishCall(FunctionSig* sig, |
714 | compiler::CallDescriptor* call_descriptor) { |
715 | const size_t return_count = sig->return_count(); |
716 | if (return_count != 0) { |
717 | DCHECK_EQ(1, return_count); |
718 | ValueType return_type = sig->GetReturn(0); |
719 | const bool need_pair = kNeedI64RegPair && return_type == kWasmI64; |
720 | DCHECK_EQ(need_pair ? 2 : 1, call_descriptor->ReturnCount()); |
721 | RegClass rc = need_pair ? kGpReg : reg_class_for(return_type); |
722 | #if V8_TARGET_ARCH_ARM |
723 | // If the return register was not d0 for f32, the code value would have to |
724 | // be halved as is done for the parameter registers. |
725 | DCHECK_EQ(call_descriptor->GetReturnLocation(0).AsRegister(), 0); |
726 | #endif |
727 | LiftoffRegister return_reg = LiftoffRegister::from_code( |
728 | rc, call_descriptor->GetReturnLocation(0).AsRegister()); |
729 | DCHECK(GetCacheRegList(rc).has(return_reg)); |
730 | if (need_pair) { |
731 | LiftoffRegister high_reg = LiftoffRegister::from_code( |
732 | rc, call_descriptor->GetReturnLocation(1).AsRegister()); |
733 | DCHECK(GetCacheRegList(rc).has(high_reg)); |
734 | return_reg = LiftoffRegister::ForPair(return_reg.gp(), high_reg.gp()); |
735 | } |
736 | DCHECK(!cache_state_.is_used(return_reg)); |
737 | PushRegister(return_type, return_reg); |
738 | } |
739 | } |
740 | |
741 | void LiftoffAssembler::Move(LiftoffRegister dst, LiftoffRegister src, |
742 | ValueType type) { |
743 | DCHECK_EQ(dst.reg_class(), src.reg_class()); |
744 | DCHECK_NE(dst, src); |
745 | if (kNeedI64RegPair && dst.is_pair()) { |
746 | // Use the {StackTransferRecipe} to move pairs, as the registers in the |
747 | // pairs might overlap. |
748 | StackTransferRecipe(this).MoveRegister(dst, src, type); |
749 | } else if (dst.is_gp()) { |
750 | Move(dst.gp(), src.gp(), type); |
751 | } else { |
752 | Move(dst.fp(), src.fp(), type); |
753 | } |
754 | } |
755 | |
756 | void LiftoffAssembler::ParallelRegisterMove( |
757 | Vector<ParallelRegisterMoveTuple> tuples) { |
758 | StackTransferRecipe stack_transfers(this); |
759 | for (auto tuple : tuples) { |
760 | if (tuple.dst == tuple.src) continue; |
761 | stack_transfers.MoveRegister(tuple.dst, tuple.src, tuple.type); |
762 | } |
763 | } |
764 | |
765 | void LiftoffAssembler::MoveToReturnRegisters(FunctionSig* sig) { |
766 | // We do not support multi-value yet. |
767 | DCHECK_EQ(1, sig->return_count()); |
768 | ValueType return_type = sig->GetReturn(0); |
769 | StackTransferRecipe stack_transfers(this); |
770 | LiftoffRegister return_reg = |
771 | needs_reg_pair(return_type) |
772 | ? LiftoffRegister::ForPair(kGpReturnRegisters[0], |
773 | kGpReturnRegisters[1]) |
774 | : reg_class_for(return_type) == kGpReg |
775 | ? LiftoffRegister(kGpReturnRegisters[0]) |
776 | : LiftoffRegister(kFpReturnRegisters[0]); |
777 | stack_transfers.LoadIntoRegister(return_reg, cache_state_.stack_state.back(), |
778 | cache_state_.stack_height() - 1); |
779 | } |
780 | |
781 | #ifdef ENABLE_SLOW_DCHECKS |
782 | bool LiftoffAssembler::ValidateCacheState() const { |
783 | uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0}; |
784 | LiftoffRegList used_regs; |
785 | for (const VarState& var : cache_state_.stack_state) { |
786 | if (!var.is_reg()) continue; |
787 | LiftoffRegister reg = var.reg(); |
788 | if (kNeedI64RegPair && reg.is_pair()) { |
789 | ++register_use_count[reg.low().liftoff_code()]; |
790 | ++register_use_count[reg.high().liftoff_code()]; |
791 | } else { |
792 | ++register_use_count[reg.liftoff_code()]; |
793 | } |
794 | used_regs.set(reg); |
795 | } |
796 | bool valid = memcmp(register_use_count, cache_state_.register_use_count, |
797 | sizeof(register_use_count)) == 0 && |
798 | used_regs == cache_state_.used_registers; |
799 | if (valid) return true; |
800 | std::ostringstream os; |
801 | os << "Error in LiftoffAssembler::ValidateCacheState().\n" ; |
802 | os << "expected: used_regs " << used_regs << ", counts " |
803 | << PrintCollection(register_use_count) << "\n" ; |
804 | os << "found: used_regs " << cache_state_.used_registers << ", counts " |
805 | << PrintCollection(cache_state_.register_use_count) << "\n" ; |
806 | os << "Use --trace-wasm-decoder and --trace-liftoff to debug." ; |
807 | FATAL("%s" , os.str().c_str()); |
808 | } |
809 | #endif |
810 | |
811 | LiftoffRegister LiftoffAssembler::SpillOneRegister(LiftoffRegList candidates, |
812 | LiftoffRegList pinned) { |
813 | // Spill one cached value to free a register. |
814 | LiftoffRegister spill_reg = cache_state_.GetNextSpillReg(candidates, pinned); |
815 | SpillRegister(spill_reg); |
816 | return spill_reg; |
817 | } |
818 | |
819 | void LiftoffAssembler::SpillRegister(LiftoffRegister reg) { |
820 | int remaining_uses = cache_state_.get_use_count(reg); |
821 | DCHECK_LT(0, remaining_uses); |
822 | for (uint32_t idx = cache_state_.stack_height() - 1;; --idx) { |
823 | DCHECK_GT(cache_state_.stack_height(), idx); |
824 | auto* slot = &cache_state_.stack_state[idx]; |
825 | if (!slot->is_reg() || !slot->reg().overlaps(reg)) continue; |
826 | if (slot->reg().is_pair()) { |
827 | // Make sure to decrement *both* registers in a pair, because the |
828 | // {clear_used} call below only clears one of them. |
829 | cache_state_.dec_used(slot->reg().low()); |
830 | cache_state_.dec_used(slot->reg().high()); |
831 | } |
832 | Spill(idx, slot->reg(), slot->type()); |
833 | slot->MakeStack(); |
834 | if (--remaining_uses == 0) break; |
835 | } |
836 | cache_state_.clear_used(reg); |
837 | } |
838 | |
839 | void LiftoffAssembler::set_num_locals(uint32_t num_locals) { |
840 | DCHECK_EQ(0, num_locals_); // only call this once. |
841 | num_locals_ = num_locals; |
842 | if (num_locals > kInlineLocalTypes) { |
843 | more_local_types_ = |
844 | reinterpret_cast<ValueType*>(malloc(num_locals * sizeof(ValueType))); |
845 | DCHECK_NOT_NULL(more_local_types_); |
846 | } |
847 | } |
848 | |
849 | std::ostream& operator<<(std::ostream& os, VarState slot) { |
850 | os << ValueTypes::TypeName(slot.type()) << ":" ; |
851 | switch (slot.loc()) { |
852 | case VarState::kStack: |
853 | return os << "s" ; |
854 | case VarState::kRegister: |
855 | return os << slot.reg(); |
856 | case VarState::kIntConst: |
857 | return os << "c" << slot.i32_const(); |
858 | } |
859 | UNREACHABLE(); |
860 | } |
861 | |
862 | #undef __ |
863 | #undef TRACE |
864 | |
865 | } // namespace wasm |
866 | } // namespace internal |
867 | } // namespace v8 |
868 | |