1 | // Copyright 2017 the V8 project authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | #include "src/wasm/baseline/liftoff-compiler.h" |
6 | |
7 | #include "src/assembler-inl.h" |
8 | #include "src/base/optional.h" |
9 | // TODO(clemensh): Remove dependences on compiler stuff. |
10 | #include "src/compiler/linkage.h" |
11 | #include "src/compiler/wasm-compiler.h" |
12 | #include "src/counters.h" |
13 | #include "src/interface-descriptors.h" |
14 | #include "src/log.h" |
15 | #include "src/macro-assembler-inl.h" |
16 | #include "src/objects/smi.h" |
17 | #include "src/ostreams.h" |
18 | #include "src/tracing/trace-event.h" |
19 | #include "src/utils.h" |
20 | #include "src/wasm/baseline/liftoff-assembler.h" |
21 | #include "src/wasm/function-body-decoder-impl.h" |
22 | #include "src/wasm/function-compiler.h" |
23 | #include "src/wasm/memory-tracing.h" |
24 | #include "src/wasm/object-access.h" |
25 | #include "src/wasm/wasm-engine.h" |
26 | #include "src/wasm/wasm-linkage.h" |
27 | #include "src/wasm/wasm-objects.h" |
28 | #include "src/wasm/wasm-opcodes.h" |
29 | |
30 | namespace v8 { |
31 | namespace internal { |
32 | namespace wasm { |
33 | |
34 | constexpr auto kRegister = LiftoffAssembler::VarState::kRegister; |
35 | constexpr auto kIntConst = LiftoffAssembler::VarState::kIntConst; |
36 | constexpr auto kStack = LiftoffAssembler::VarState::kStack; |
37 | |
38 | namespace { |
39 | |
40 | #define __ asm_. |
41 | |
42 | #define TRACE(...) \ |
43 | do { \ |
44 | if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \ |
45 | } while (false) |
46 | |
47 | #define WASM_INSTANCE_OBJECT_FIELD_OFFSET(name) \ |
48 | ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset) |
49 | |
50 | template <int expected_size, int actual_size> |
51 | struct assert_field_size { |
52 | static_assert(expected_size == actual_size, |
53 | "field in WasmInstance does not have the expected size" ); |
54 | static constexpr int size = actual_size; |
55 | }; |
56 | |
57 | #define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \ |
58 | FIELD_SIZE(WasmInstanceObject::k##name##Offset) |
59 | |
60 | #define LOAD_INSTANCE_FIELD(dst, name, load_size) \ |
61 | __ LoadFromInstance(dst, WASM_INSTANCE_OBJECT_FIELD_OFFSET(name), \ |
62 | assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \ |
63 | load_size>::size); |
64 | |
65 | #define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name) \ |
66 | static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize, \ |
67 | "field in WasmInstance does not have the expected size"); \ |
68 | __ LoadTaggedPointerFromInstance(dst, \ |
69 | WASM_INSTANCE_OBJECT_FIELD_OFFSET(name)); |
70 | |
71 | #ifdef DEBUG |
72 | #define (str) \ |
73 | do { \ |
74 | __ RecordComment(str); \ |
75 | } while (false) |
76 | #else |
77 | #define DEBUG_CODE_COMMENT(str) ((void)0) |
78 | #endif |
79 | |
80 | constexpr LoadType::LoadTypeValue kPointerLoadType = |
81 | kSystemPointerSize == 8 ? LoadType::kI64Load : LoadType::kI32Load; |
82 | |
83 | #if V8_TARGET_ARCH_ARM64 |
84 | // On ARM64, the Assembler keeps track of pointers to Labels to resolve |
85 | // branches to distant targets. Moving labels would confuse the Assembler, |
86 | // thus store the label on the heap and keep a unique_ptr. |
87 | class MovableLabel { |
88 | public: |
89 | MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(MovableLabel); |
90 | MovableLabel() : label_(new Label()) {} |
91 | |
92 | Label* get() { return label_.get(); } |
93 | |
94 | private: |
95 | std::unique_ptr<Label> label_; |
96 | }; |
97 | #else |
98 | // On all other platforms, just store the Label directly. |
99 | class MovableLabel { |
100 | public: |
101 | MOVE_ONLY_WITH_DEFAULT_CONSTRUCTORS(MovableLabel); |
102 | |
103 | Label* get() { return &label_; } |
104 | |
105 | private: |
106 | Label label_; |
107 | }; |
108 | #endif |
109 | |
110 | compiler::CallDescriptor* GetLoweredCallDescriptor( |
111 | Zone* zone, compiler::CallDescriptor* call_desc) { |
112 | return kSystemPointerSize == 4 |
113 | ? compiler::GetI32WasmCallDescriptor(zone, call_desc) |
114 | : call_desc; |
115 | } |
116 | |
117 | constexpr ValueType kSupportedTypesArr[] = {kWasmI32, kWasmI64, kWasmF32, |
118 | kWasmF64}; |
119 | constexpr Vector<const ValueType> kSupportedTypes = |
120 | ArrayVector(kSupportedTypesArr); |
121 | |
122 | class LiftoffCompiler { |
123 | public: |
124 | // TODO(clemensh): Make this a template parameter. |
125 | static constexpr Decoder::ValidateFlag validate = Decoder::kValidate; |
126 | |
127 | using Value = ValueBase; |
128 | |
129 | struct ElseState { |
130 | MovableLabel label; |
131 | LiftoffAssembler::CacheState state; |
132 | }; |
133 | |
134 | struct Control : public ControlBase<Value> { |
135 | std::unique_ptr<ElseState> else_state; |
136 | LiftoffAssembler::CacheState label_state; |
137 | MovableLabel label; |
138 | |
139 | MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(Control); |
140 | |
141 | template <typename... Args> |
142 | explicit Control(Args&&... args) V8_NOEXCEPT |
143 | : ControlBase(std::forward<Args>(args)...) {} |
144 | }; |
145 | |
146 | using FullDecoder = WasmFullDecoder<validate, LiftoffCompiler>; |
147 | |
148 | struct OutOfLineCode { |
149 | MovableLabel label; |
150 | MovableLabel continuation; |
151 | WasmCode::RuntimeStubId stub; |
152 | WasmCodePosition position; |
153 | LiftoffRegList regs_to_save; |
154 | uint32_t pc; // for trap handler. |
155 | |
156 | // Named constructors: |
157 | static OutOfLineCode Trap(WasmCode::RuntimeStubId s, WasmCodePosition pos, |
158 | uint32_t pc) { |
159 | DCHECK_LT(0, pos); |
160 | return {{}, {}, s, pos, {}, pc}; |
161 | } |
162 | static OutOfLineCode StackCheck(WasmCodePosition pos, LiftoffRegList regs) { |
163 | return {{}, {}, WasmCode::kWasmStackGuard, pos, regs, 0}; |
164 | } |
165 | }; |
166 | |
167 | LiftoffCompiler(compiler::CallDescriptor* call_descriptor, |
168 | CompilationEnv* env, Zone* compilation_zone, |
169 | std::unique_ptr<AssemblerBuffer> buffer) |
170 | : asm_(std::move(buffer)), |
171 | descriptor_( |
172 | GetLoweredCallDescriptor(compilation_zone, call_descriptor)), |
173 | env_(env), |
174 | compilation_zone_(compilation_zone), |
175 | safepoint_table_builder_(compilation_zone_) {} |
176 | |
177 | bool ok() const { return ok_; } |
178 | |
179 | void GetCode(CodeDesc* desc) { |
180 | asm_.GetCode(nullptr, desc, &safepoint_table_builder_, |
181 | Assembler::kNoHandlerTable); |
182 | } |
183 | |
184 | OwnedVector<uint8_t> GetSourcePositionTable() { |
185 | return source_position_table_builder_.ToSourcePositionTableVector(); |
186 | } |
187 | |
188 | OwnedVector<trap_handler::ProtectedInstructionData> GetProtectedInstructions() |
189 | const { |
190 | return OwnedVector<trap_handler::ProtectedInstructionData>::Of( |
191 | protected_instructions_); |
192 | } |
193 | |
194 | uint32_t GetTotalFrameSlotCount() const { |
195 | return __ GetTotalFrameSlotCount(); |
196 | } |
197 | |
198 | void unsupported(FullDecoder* decoder, const char* reason) { |
199 | ok_ = false; |
200 | TRACE("unsupported: %s\n" , reason); |
201 | decoder->errorf(decoder->pc_offset(), "unsupported liftoff operation: %s" , |
202 | reason); |
203 | UnuseLabels(decoder); |
204 | } |
205 | |
206 | bool DidAssemblerBailout(FullDecoder* decoder) { |
207 | if (decoder->failed() || !__ did_bailout()) return false; |
208 | unsupported(decoder, __ bailout_reason()); |
209 | return true; |
210 | } |
211 | |
212 | bool CheckSupportedType(FullDecoder* decoder, |
213 | Vector<const ValueType> supported_types, |
214 | ValueType type, const char* context) { |
215 | char buffer[128]; |
216 | // Check supported types. |
217 | for (ValueType supported : supported_types) { |
218 | if (type == supported) return true; |
219 | } |
220 | SNPrintF(ArrayVector(buffer), "%s %s" , ValueTypes::TypeName(type), context); |
221 | unsupported(decoder, buffer); |
222 | return false; |
223 | } |
224 | |
225 | int GetSafepointTableOffset() const { |
226 | return safepoint_table_builder_.GetCodeOffset(); |
227 | } |
228 | |
229 | void UnuseLabels(FullDecoder* decoder) { |
230 | #ifdef DEBUG |
231 | auto Unuse = [](Label* label) { |
232 | label->Unuse(); |
233 | label->UnuseNear(); |
234 | }; |
235 | // Unuse all labels now, otherwise their destructor will fire a DCHECK error |
236 | // if they where referenced before. |
237 | uint32_t control_depth = decoder ? decoder->control_depth() : 0; |
238 | for (uint32_t i = 0; i < control_depth; ++i) { |
239 | Control* c = decoder->control_at(i); |
240 | Unuse(c->label.get()); |
241 | if (c->else_state) Unuse(c->else_state->label.get()); |
242 | } |
243 | for (auto& ool : out_of_line_code_) Unuse(ool.label.get()); |
244 | #endif |
245 | } |
246 | |
247 | void StartFunction(FullDecoder* decoder) { |
248 | int num_locals = decoder->num_locals(); |
249 | __ set_num_locals(num_locals); |
250 | for (int i = 0; i < num_locals; ++i) { |
251 | __ set_local_type(i, decoder->GetLocalType(i)); |
252 | } |
253 | } |
254 | |
255 | // Returns the number of inputs processed (1 or 2). |
256 | uint32_t ProcessParameter(ValueType type, uint32_t input_idx) { |
257 | const int num_lowered_params = 1 + needs_reg_pair(type); |
258 | ValueType lowered_type = needs_reg_pair(type) ? kWasmI32 : type; |
259 | RegClass rc = reg_class_for(lowered_type); |
260 | // Initialize to anything, will be set in the loop and used afterwards. |
261 | LiftoffRegister reg = kGpCacheRegList.GetFirstRegSet(); |
262 | LiftoffRegList pinned; |
263 | for (int pair_idx = 0; pair_idx < num_lowered_params; ++pair_idx) { |
264 | compiler::LinkageLocation param_loc = |
265 | descriptor_->GetInputLocation(input_idx + pair_idx); |
266 | // Initialize to anything, will be set in both arms of the if. |
267 | LiftoffRegister in_reg = kGpCacheRegList.GetFirstRegSet(); |
268 | if (param_loc.IsRegister()) { |
269 | DCHECK(!param_loc.IsAnyRegister()); |
270 | int reg_code = param_loc.AsRegister(); |
271 | #if V8_TARGET_ARCH_ARM |
272 | // Liftoff assumes a one-to-one mapping between float registers and |
273 | // double registers, and so does not distinguish between f32 and f64 |
274 | // registers. The f32 register code must therefore be halved in order to |
275 | // pass the f64 code to Liftoff. |
276 | DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0); |
277 | if (type == kWasmF32) { |
278 | reg_code /= 2; |
279 | } |
280 | #endif |
281 | RegList cache_regs = rc == kGpReg ? kLiftoffAssemblerGpCacheRegs |
282 | : kLiftoffAssemblerFpCacheRegs; |
283 | if (cache_regs & (1ULL << reg_code)) { |
284 | // This is a cache register, just use it. |
285 | in_reg = LiftoffRegister::from_code(rc, reg_code); |
286 | } else { |
287 | // Move to a cache register (spill one if necessary). |
288 | // Note that we cannot create a {LiftoffRegister} for reg_code, since |
289 | // {LiftoffRegister} can only store cache regs. |
290 | in_reg = __ GetUnusedRegister(rc, pinned); |
291 | if (rc == kGpReg) { |
292 | __ Move(in_reg.gp(), Register::from_code(reg_code), lowered_type); |
293 | } else { |
294 | __ Move(in_reg.fp(), DoubleRegister::from_code(reg_code), |
295 | lowered_type); |
296 | } |
297 | } |
298 | } else if (param_loc.IsCallerFrameSlot()) { |
299 | in_reg = __ GetUnusedRegister(rc, pinned); |
300 | __ LoadCallerFrameSlot(in_reg, -param_loc.AsCallerFrameSlot(), |
301 | lowered_type); |
302 | } |
303 | reg = pair_idx == 0 ? in_reg |
304 | : LiftoffRegister::ForPair(reg.gp(), in_reg.gp()); |
305 | pinned.set(reg); |
306 | } |
307 | __ PushRegister(type, reg); |
308 | return num_lowered_params; |
309 | } |
310 | |
311 | void StackCheck(WasmCodePosition position) { |
312 | if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) return; |
313 | out_of_line_code_.push_back( |
314 | OutOfLineCode::StackCheck(position, __ cache_state()->used_registers)); |
315 | OutOfLineCode& ool = out_of_line_code_.back(); |
316 | Register limit_address = __ GetUnusedRegister(kGpReg).gp(); |
317 | LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize); |
318 | __ StackCheck(ool.label.get(), limit_address); |
319 | __ bind(ool.continuation.get()); |
320 | } |
321 | |
322 | void StartFunctionBody(FullDecoder* decoder, Control* block) { |
323 | for (uint32_t i = 0; i < __ num_locals(); ++i) { |
324 | if (!CheckSupportedType(decoder, kSupportedTypes, __ local_type(i), |
325 | "param" )) |
326 | return; |
327 | } |
328 | |
329 | // Input 0 is the call target, the instance is at 1. |
330 | constexpr int kInstanceParameterIndex = 1; |
331 | // Store the instance parameter to a special stack slot. |
332 | compiler::LinkageLocation instance_loc = |
333 | descriptor_->GetInputLocation(kInstanceParameterIndex); |
334 | DCHECK(instance_loc.IsRegister()); |
335 | DCHECK(!instance_loc.IsAnyRegister()); |
336 | Register instance_reg = Register::from_code(instance_loc.AsRegister()); |
337 | DCHECK_EQ(kWasmInstanceRegister, instance_reg); |
338 | |
339 | // Parameter 0 is the instance parameter. |
340 | uint32_t num_params = |
341 | static_cast<uint32_t>(decoder->sig_->parameter_count()); |
342 | |
343 | __ EnterFrame(StackFrame::WASM_COMPILED); |
344 | __ set_has_frame(true); |
345 | pc_offset_stack_frame_construction_ = __ PrepareStackFrame(); |
346 | // {PrepareStackFrame} is the first platform-specific assembler method. |
347 | // If this failed, we can bail out immediately, avoiding runtime overhead |
348 | // and potential failures because of other unimplemented methods. |
349 | // A platform implementing {PrepareStackFrame} must ensure that we can |
350 | // finish compilation without errors even if we hit unimplemented |
351 | // LiftoffAssembler methods. |
352 | if (DidAssemblerBailout(decoder)) return; |
353 | |
354 | __ SpillInstance(instance_reg); |
355 | // Input 0 is the code target, 1 is the instance. First parameter at 2. |
356 | uint32_t input_idx = kInstanceParameterIndex + 1; |
357 | for (uint32_t param_idx = 0; param_idx < num_params; ++param_idx) { |
358 | input_idx += ProcessParameter(__ local_type(param_idx), input_idx); |
359 | } |
360 | DCHECK_EQ(input_idx, descriptor_->InputCount()); |
361 | // Set to a gp register, to mark this uninitialized. |
362 | LiftoffRegister zero_double_reg = kGpCacheRegList.GetFirstRegSet(); |
363 | DCHECK(zero_double_reg.is_gp()); |
364 | for (uint32_t param_idx = num_params; param_idx < __ num_locals(); |
365 | ++param_idx) { |
366 | ValueType type = decoder->GetLocalType(param_idx); |
367 | switch (type) { |
368 | case kWasmI32: |
369 | __ cache_state()->stack_state.emplace_back(kWasmI32, uint32_t{0}); |
370 | break; |
371 | case kWasmI64: |
372 | __ cache_state()->stack_state.emplace_back(kWasmI64, uint32_t{0}); |
373 | break; |
374 | case kWasmF32: |
375 | case kWasmF64: |
376 | if (zero_double_reg.is_gp()) { |
377 | // Note: This might spill one of the registers used to hold |
378 | // parameters. |
379 | zero_double_reg = __ GetUnusedRegister(kFpReg); |
380 | // Zero is represented by the bit pattern 0 for both f32 and f64. |
381 | __ LoadConstant(zero_double_reg, WasmValue(0.)); |
382 | } |
383 | __ PushRegister(type, zero_double_reg); |
384 | break; |
385 | default: |
386 | UNIMPLEMENTED(); |
387 | } |
388 | } |
389 | |
390 | // The function-prologue stack check is associated with position 0, which |
391 | // is never a position of any instruction in the function. |
392 | StackCheck(0); |
393 | |
394 | DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height()); |
395 | } |
396 | |
397 | void GenerateOutOfLineCode(OutOfLineCode& ool) { |
398 | __ bind(ool.label.get()); |
399 | const bool is_stack_check = ool.stub == WasmCode::kWasmStackGuard; |
400 | const bool is_mem_out_of_bounds = |
401 | ool.stub == WasmCode::kThrowWasmTrapMemOutOfBounds; |
402 | |
403 | if (is_mem_out_of_bounds && env_->use_trap_handler) { |
404 | uint32_t pc = static_cast<uint32_t>(__ pc_offset()); |
405 | DCHECK_EQ(pc, __ pc_offset()); |
406 | protected_instructions_.emplace_back( |
407 | trap_handler::ProtectedInstructionData{ool.pc, pc}); |
408 | } |
409 | |
410 | if (!env_->runtime_exception_support) { |
411 | // We cannot test calls to the runtime in cctest/test-run-wasm. |
412 | // Therefore we emit a call to C here instead of a call to the runtime. |
413 | // In this mode, we never generate stack checks. |
414 | DCHECK(!is_stack_check); |
415 | __ CallTrapCallbackForTesting(); |
416 | __ LeaveFrame(StackFrame::WASM_COMPILED); |
417 | __ DropStackSlotsAndRet( |
418 | static_cast<uint32_t>(descriptor_->StackParameterCount())); |
419 | return; |
420 | } |
421 | |
422 | if (!ool.regs_to_save.is_empty()) __ PushRegisters(ool.regs_to_save); |
423 | |
424 | source_position_table_builder_.AddPosition( |
425 | __ pc_offset(), SourcePosition(ool.position), false); |
426 | __ CallRuntimeStub(ool.stub); |
427 | safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple, |
428 | Safepoint::kNoLazyDeopt); |
429 | DCHECK_EQ(ool.continuation.get()->is_bound(), is_stack_check); |
430 | if (!ool.regs_to_save.is_empty()) __ PopRegisters(ool.regs_to_save); |
431 | if (is_stack_check) { |
432 | __ emit_jump(ool.continuation.get()); |
433 | } else { |
434 | __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap); |
435 | } |
436 | } |
437 | |
438 | void FinishFunction(FullDecoder* decoder) { |
439 | if (DidAssemblerBailout(decoder)) return; |
440 | for (OutOfLineCode& ool : out_of_line_code_) { |
441 | GenerateOutOfLineCode(ool); |
442 | } |
443 | __ PatchPrepareStackFrame(pc_offset_stack_frame_construction_, |
444 | __ GetTotalFrameSlotCount()); |
445 | __ FinishCode(); |
446 | safepoint_table_builder_.Emit(&asm_, __ GetTotalFrameSlotCount()); |
447 | __ MaybeEmitOutOfLineConstantPool(); |
448 | // The previous calls may have also generated a bailout. |
449 | DidAssemblerBailout(decoder); |
450 | } |
451 | |
452 | void OnFirstError(FullDecoder* decoder) { |
453 | ok_ = false; |
454 | UnuseLabels(decoder); |
455 | asm_.AbortCompilation(); |
456 | } |
457 | |
458 | void NextInstruction(FullDecoder* decoder, WasmOpcode opcode) { |
459 | TraceCacheState(decoder); |
460 | SLOW_DCHECK(__ ValidateCacheState()); |
461 | DEBUG_CODE_COMMENT(WasmOpcodes::OpcodeName(opcode)); |
462 | } |
463 | |
464 | void Block(FullDecoder* decoder, Control* block) {} |
465 | |
466 | void Loop(FullDecoder* decoder, Control* loop) { |
467 | // Before entering a loop, spill all locals to the stack, in order to free |
468 | // the cache registers, and to avoid unnecessarily reloading stack values |
469 | // into registers at branches. |
470 | // TODO(clemensh): Come up with a better strategy here, involving |
471 | // pre-analysis of the function. |
472 | __ SpillLocals(); |
473 | |
474 | // Loop labels bind at the beginning of the block. |
475 | __ bind(loop->label.get()); |
476 | |
477 | // Save the current cache state for the merge when jumping to this loop. |
478 | loop->label_state.Split(*__ cache_state()); |
479 | |
480 | // Execute a stack check in the loop header. |
481 | StackCheck(decoder->position()); |
482 | } |
483 | |
484 | void Try(FullDecoder* decoder, Control* block) { |
485 | unsupported(decoder, "try" ); |
486 | } |
487 | |
488 | void Catch(FullDecoder* decoder, Control* block, Value* exception) { |
489 | unsupported(decoder, "catch" ); |
490 | } |
491 | |
492 | void If(FullDecoder* decoder, const Value& cond, Control* if_block) { |
493 | DCHECK_EQ(if_block, decoder->control_at(0)); |
494 | DCHECK(if_block->is_if()); |
495 | |
496 | if (if_block->start_merge.arity > 0 || if_block->end_merge.arity > 1) |
497 | return unsupported(decoder, "multi-value if" ); |
498 | |
499 | // Allocate the else state. |
500 | if_block->else_state = base::make_unique<ElseState>(); |
501 | |
502 | // Test the condition, jump to else if zero. |
503 | Register value = __ PopToRegister().gp(); |
504 | __ emit_cond_jump(kEqual, if_block->else_state->label.get(), kWasmI32, |
505 | value); |
506 | |
507 | // Store the state (after popping the value) for executing the else branch. |
508 | if_block->else_state->state.Split(*__ cache_state()); |
509 | } |
510 | |
511 | void FallThruTo(FullDecoder* decoder, Control* c) { |
512 | if (c->end_merge.reached) { |
513 | __ MergeFullStackWith(c->label_state, *__ cache_state()); |
514 | } else { |
515 | c->label_state.Split(*__ cache_state()); |
516 | } |
517 | TraceCacheState(decoder); |
518 | } |
519 | |
520 | void FinishOneArmedIf(FullDecoder* decoder, Control* c) { |
521 | DCHECK(c->is_onearmed_if()); |
522 | if (c->end_merge.reached) { |
523 | // Someone already merged to the end of the if. Merge both arms into that. |
524 | if (c->reachable()) { |
525 | // Merge the if state into the end state. |
526 | __ MergeFullStackWith(c->label_state, *__ cache_state()); |
527 | __ emit_jump(c->label.get()); |
528 | } |
529 | // Merge the else state into the end state. |
530 | __ bind(c->else_state->label.get()); |
531 | __ MergeFullStackWith(c->label_state, c->else_state->state); |
532 | __ cache_state()->Steal(c->label_state); |
533 | } else if (c->reachable()) { |
534 | // No merge yet at the end of the if, but we need to create a merge for |
535 | // the both arms of this if. Thus init the merge point from the else |
536 | // state, then merge the if state into that. |
537 | DCHECK_EQ(0, c->end_merge.arity); |
538 | c->label_state.InitMerge(c->else_state->state, __ num_locals(), 0, |
539 | c->stack_depth); |
540 | __ MergeFullStackWith(c->label_state, *__ cache_state()); |
541 | __ emit_jump(c->label.get()); |
542 | // Merge the else state into the end state. |
543 | __ bind(c->else_state->label.get()); |
544 | __ MergeFullStackWith(c->label_state, c->else_state->state); |
545 | __ cache_state()->Steal(c->label_state); |
546 | } else { |
547 | // No merge needed, just continue with the else state. |
548 | __ bind(c->else_state->label.get()); |
549 | __ cache_state()->Steal(c->else_state->state); |
550 | } |
551 | } |
552 | |
553 | void PopControl(FullDecoder* decoder, Control* c) { |
554 | if (c->is_loop()) return; // A loop just falls through. |
555 | if (c->is_onearmed_if()) { |
556 | // Special handling for one-armed ifs. |
557 | FinishOneArmedIf(decoder, c); |
558 | } else if (c->end_merge.reached) { |
559 | // There is a merge already. Merge our state into that, then continue with |
560 | // that state. |
561 | if (c->reachable()) { |
562 | __ MergeFullStackWith(c->label_state, *__ cache_state()); |
563 | } |
564 | __ cache_state()->Steal(c->label_state); |
565 | } else { |
566 | // No merge, just continue with our current state. |
567 | } |
568 | |
569 | if (!c->label.get()->is_bound()) __ bind(c->label.get()); |
570 | } |
571 | |
572 | void EndControl(FullDecoder* decoder, Control* c) {} |
573 | |
574 | enum CCallReturn : bool { kHasReturn = true, kNoReturn = false }; |
575 | |
576 | void GenerateCCall(const LiftoffRegister* result_regs, FunctionSig* sig, |
577 | ValueType out_argument_type, |
578 | const LiftoffRegister* arg_regs, |
579 | ExternalReference ext_ref) { |
580 | // Before making a call, spill all cache registers. |
581 | __ SpillAllRegisters(); |
582 | |
583 | // Store arguments on our stack, then align the stack for calling to C. |
584 | int param_bytes = 0; |
585 | for (ValueType param_type : sig->parameters()) { |
586 | param_bytes += ValueTypes::MemSize(param_type); |
587 | } |
588 | int out_arg_bytes = out_argument_type == kWasmStmt |
589 | ? 0 |
590 | : ValueTypes::MemSize(out_argument_type); |
591 | int stack_bytes = std::max(param_bytes, out_arg_bytes); |
592 | __ CallC(sig, arg_regs, result_regs, out_argument_type, stack_bytes, |
593 | ext_ref); |
594 | } |
595 | |
596 | template <ValueType src_type, ValueType result_type, class EmitFn> |
597 | void EmitUnOp(EmitFn fn) { |
598 | static RegClass src_rc = reg_class_for(src_type); |
599 | static RegClass result_rc = reg_class_for(result_type); |
600 | LiftoffRegister src = __ PopToRegister(); |
601 | LiftoffRegister dst = src_rc == result_rc |
602 | ? __ GetUnusedRegister(result_rc, {src}) |
603 | : __ GetUnusedRegister(result_rc); |
604 | fn(dst, src); |
605 | __ PushRegister(result_type, dst); |
606 | } |
607 | |
608 | void EmitI32UnOpWithCFallback(bool (LiftoffAssembler::*emit_fn)(Register, |
609 | Register), |
610 | ExternalReference (*fallback_fn)()) { |
611 | auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) { |
612 | if (emit_fn && (asm_.*emit_fn)(dst.gp(), src.gp())) return; |
613 | ExternalReference ext_ref = fallback_fn(); |
614 | ValueType sig_i_i_reps[] = {kWasmI32, kWasmI32}; |
615 | FunctionSig sig_i_i(1, 1, sig_i_i_reps); |
616 | GenerateCCall(&dst, &sig_i_i, kWasmStmt, &src, ext_ref); |
617 | }; |
618 | EmitUnOp<kWasmI32, kWasmI32>(emit_with_c_fallback); |
619 | } |
620 | |
621 | template <ValueType type> |
622 | void EmitFloatUnOpWithCFallback( |
623 | bool (LiftoffAssembler::*emit_fn)(DoubleRegister, DoubleRegister), |
624 | ExternalReference (*fallback_fn)()) { |
625 | auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) { |
626 | if ((asm_.*emit_fn)(dst.fp(), src.fp())) return; |
627 | ExternalReference ext_ref = fallback_fn(); |
628 | ValueType sig_reps[] = {type}; |
629 | FunctionSig sig(0, 1, sig_reps); |
630 | GenerateCCall(&dst, &sig, type, &src, ext_ref); |
631 | }; |
632 | EmitUnOp<type, type>(emit_with_c_fallback); |
633 | } |
634 | |
635 | enum TypeConversionTrapping : bool { kCanTrap = true, kNoTrap = false }; |
636 | template <ValueType dst_type, ValueType src_type, |
637 | TypeConversionTrapping can_trap> |
638 | void EmitTypeConversion(WasmOpcode opcode, ExternalReference (*fallback_fn)(), |
639 | WasmCodePosition trap_position) { |
640 | static constexpr RegClass src_rc = reg_class_for(src_type); |
641 | static constexpr RegClass dst_rc = reg_class_for(dst_type); |
642 | LiftoffRegister src = __ PopToRegister(); |
643 | LiftoffRegister dst = src_rc == dst_rc ? __ GetUnusedRegister(dst_rc, {src}) |
644 | : __ GetUnusedRegister(dst_rc); |
645 | DCHECK_EQ(!!can_trap, trap_position > 0); |
646 | Label* trap = can_trap ? AddOutOfLineTrap( |
647 | trap_position, |
648 | WasmCode::kThrowWasmTrapFloatUnrepresentable) |
649 | : nullptr; |
650 | if (!__ emit_type_conversion(opcode, dst, src, trap)) { |
651 | DCHECK_NOT_NULL(fallback_fn); |
652 | ExternalReference ext_ref = fallback_fn(); |
653 | if (can_trap) { |
654 | // External references for potentially trapping conversions return int. |
655 | ValueType sig_reps[] = {kWasmI32, src_type}; |
656 | FunctionSig sig(1, 1, sig_reps); |
657 | LiftoffRegister ret_reg = |
658 | __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst)); |
659 | LiftoffRegister dst_regs[] = {ret_reg, dst}; |
660 | GenerateCCall(dst_regs, &sig, dst_type, &src, ext_ref); |
661 | __ emit_cond_jump(kEqual, trap, kWasmI32, ret_reg.gp()); |
662 | } else { |
663 | ValueType sig_reps[] = {src_type}; |
664 | FunctionSig sig(0, 1, sig_reps); |
665 | GenerateCCall(&dst, &sig, dst_type, &src, ext_ref); |
666 | } |
667 | } |
668 | __ PushRegister(dst_type, dst); |
669 | } |
670 | |
671 | void UnOp(FullDecoder* decoder, WasmOpcode opcode, const Value& value, |
672 | Value* result) { |
673 | #define CASE_I32_UNOP(opcode, fn) \ |
674 | case WasmOpcode::kExpr##opcode: \ |
675 | EmitUnOp<kWasmI32, kWasmI32>( \ |
676 | [=](LiftoffRegister dst, LiftoffRegister src) { \ |
677 | __ emit_##fn(dst.gp(), src.gp()); \ |
678 | }); \ |
679 | break; |
680 | #define CASE_I32_SIGN_EXTENSION(opcode, fn) \ |
681 | case WasmOpcode::kExpr##opcode: \ |
682 | EmitUnOp<kWasmI32, kWasmI32>( \ |
683 | [=](LiftoffRegister dst, LiftoffRegister src) { \ |
684 | __ emit_##fn(dst.gp(), src.gp()); \ |
685 | }); \ |
686 | break; |
687 | #define CASE_I64_SIGN_EXTENSION(opcode, fn) \ |
688 | case WasmOpcode::kExpr##opcode: \ |
689 | EmitUnOp<kWasmI64, kWasmI64>( \ |
690 | [=](LiftoffRegister dst, LiftoffRegister src) { \ |
691 | __ emit_##fn(dst, src); \ |
692 | }); \ |
693 | break; |
694 | #define CASE_FLOAT_UNOP(opcode, type, fn) \ |
695 | case WasmOpcode::kExpr##opcode: \ |
696 | EmitUnOp<kWasm##type, kWasm##type>( \ |
697 | [=](LiftoffRegister dst, LiftoffRegister src) { \ |
698 | __ emit_##fn(dst.fp(), src.fp()); \ |
699 | }); \ |
700 | break; |
701 | #define CASE_FLOAT_UNOP_WITH_CFALLBACK(opcode, type, fn) \ |
702 | case WasmOpcode::kExpr##opcode: \ |
703 | EmitFloatUnOpWithCFallback<kWasm##type>(&LiftoffAssembler::emit_##fn, \ |
704 | &ExternalReference::wasm_##fn); \ |
705 | break; |
706 | #define CASE_TYPE_CONVERSION(opcode, dst_type, src_type, ext_ref, can_trap) \ |
707 | case WasmOpcode::kExpr##opcode: \ |
708 | EmitTypeConversion<kWasm##dst_type, kWasm##src_type, can_trap>( \ |
709 | kExpr##opcode, ext_ref, can_trap ? decoder->position() : 0); \ |
710 | break; |
711 | switch (opcode) { |
712 | CASE_I32_UNOP(I32Eqz, i32_eqz) |
713 | CASE_I32_UNOP(I32Clz, i32_clz) |
714 | CASE_I32_UNOP(I32Ctz, i32_ctz) |
715 | CASE_FLOAT_UNOP(F32Abs, F32, f32_abs) |
716 | CASE_FLOAT_UNOP(F32Neg, F32, f32_neg) |
717 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Ceil, F32, f32_ceil) |
718 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Floor, F32, f32_floor) |
719 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Trunc, F32, f32_trunc) |
720 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F32NearestInt, F32, f32_nearest_int) |
721 | CASE_FLOAT_UNOP(F32Sqrt, F32, f32_sqrt) |
722 | CASE_FLOAT_UNOP(F64Abs, F64, f64_abs) |
723 | CASE_FLOAT_UNOP(F64Neg, F64, f64_neg) |
724 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Ceil, F64, f64_ceil) |
725 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Floor, F64, f64_floor) |
726 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Trunc, F64, f64_trunc) |
727 | CASE_FLOAT_UNOP_WITH_CFALLBACK(F64NearestInt, F64, f64_nearest_int) |
728 | CASE_FLOAT_UNOP(F64Sqrt, F64, f64_sqrt) |
729 | CASE_TYPE_CONVERSION(I32ConvertI64, I32, I64, nullptr, kNoTrap) |
730 | CASE_TYPE_CONVERSION(I32SConvertF32, I32, F32, nullptr, kCanTrap) |
731 | CASE_TYPE_CONVERSION(I32UConvertF32, I32, F32, nullptr, kCanTrap) |
732 | CASE_TYPE_CONVERSION(I32SConvertF64, I32, F64, nullptr, kCanTrap) |
733 | CASE_TYPE_CONVERSION(I32UConvertF64, I32, F64, nullptr, kCanTrap) |
734 | CASE_TYPE_CONVERSION(I32ReinterpretF32, I32, F32, nullptr, kNoTrap) |
735 | CASE_TYPE_CONVERSION(I64SConvertI32, I64, I32, nullptr, kNoTrap) |
736 | CASE_TYPE_CONVERSION(I64UConvertI32, I64, I32, nullptr, kNoTrap) |
737 | CASE_TYPE_CONVERSION(I64SConvertF32, I64, F32, |
738 | &ExternalReference::wasm_float32_to_int64, kCanTrap) |
739 | CASE_TYPE_CONVERSION(I64UConvertF32, I64, F32, |
740 | &ExternalReference::wasm_float32_to_uint64, kCanTrap) |
741 | CASE_TYPE_CONVERSION(I64SConvertF64, I64, F64, |
742 | &ExternalReference::wasm_float64_to_int64, kCanTrap) |
743 | CASE_TYPE_CONVERSION(I64UConvertF64, I64, F64, |
744 | &ExternalReference::wasm_float64_to_uint64, kCanTrap) |
745 | CASE_TYPE_CONVERSION(I64ReinterpretF64, I64, F64, nullptr, kNoTrap) |
746 | CASE_TYPE_CONVERSION(F32SConvertI32, F32, I32, nullptr, kNoTrap) |
747 | CASE_TYPE_CONVERSION(F32UConvertI32, F32, I32, nullptr, kNoTrap) |
748 | CASE_TYPE_CONVERSION(F32SConvertI64, F32, I64, |
749 | &ExternalReference::wasm_int64_to_float32, kNoTrap) |
750 | CASE_TYPE_CONVERSION(F32UConvertI64, F32, I64, |
751 | &ExternalReference::wasm_uint64_to_float32, kNoTrap) |
752 | CASE_TYPE_CONVERSION(F32ConvertF64, F32, F64, nullptr, kNoTrap) |
753 | CASE_TYPE_CONVERSION(F32ReinterpretI32, F32, I32, nullptr, kNoTrap) |
754 | CASE_TYPE_CONVERSION(F64SConvertI32, F64, I32, nullptr, kNoTrap) |
755 | CASE_TYPE_CONVERSION(F64UConvertI32, F64, I32, nullptr, kNoTrap) |
756 | CASE_TYPE_CONVERSION(F64SConvertI64, F64, I64, |
757 | &ExternalReference::wasm_int64_to_float64, kNoTrap) |
758 | CASE_TYPE_CONVERSION(F64UConvertI64, F64, I64, |
759 | &ExternalReference::wasm_uint64_to_float64, kNoTrap) |
760 | CASE_TYPE_CONVERSION(F64ConvertF32, F64, F32, nullptr, kNoTrap) |
761 | CASE_TYPE_CONVERSION(F64ReinterpretI64, F64, I64, nullptr, kNoTrap) |
762 | CASE_I32_SIGN_EXTENSION(I32SExtendI8, i32_signextend_i8) |
763 | CASE_I32_SIGN_EXTENSION(I32SExtendI16, i32_signextend_i16) |
764 | CASE_I64_SIGN_EXTENSION(I64SExtendI8, i64_signextend_i8) |
765 | CASE_I64_SIGN_EXTENSION(I64SExtendI16, i64_signextend_i16) |
766 | CASE_I64_SIGN_EXTENSION(I64SExtendI32, i64_signextend_i32) |
767 | case kExprI32Popcnt: |
768 | EmitI32UnOpWithCFallback(&LiftoffAssembler::emit_i32_popcnt, |
769 | &ExternalReference::wasm_word32_popcnt); |
770 | break; |
771 | case WasmOpcode::kExprI64Eqz: |
772 | EmitUnOp<kWasmI64, kWasmI32>( |
773 | [=](LiftoffRegister dst, LiftoffRegister src) { |
774 | __ emit_i64_eqz(dst.gp(), src); |
775 | }); |
776 | break; |
777 | default: |
778 | return unsupported(decoder, WasmOpcodes::OpcodeName(opcode)); |
779 | } |
780 | #undef CASE_I32_UNOP |
781 | #undef CASE_I32_SIGN_EXTENSION |
782 | #undef CASE_I64_SIGN_EXTENSION |
783 | #undef CASE_FLOAT_UNOP |
784 | #undef CASE_FLOAT_UNOP_WITH_CFALLBACK |
785 | #undef CASE_TYPE_CONVERSION |
786 | } |
787 | |
788 | template <ValueType src_type, ValueType result_type, typename EmitFn, |
789 | typename EmitFnImm> |
790 | void EmitBinOpImm(EmitFn fn, EmitFnImm fnImm) { |
791 | static constexpr RegClass src_rc = reg_class_for(src_type); |
792 | static constexpr RegClass result_rc = reg_class_for(result_type); |
793 | |
794 | LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back(); |
795 | // Check if the RHS is an immediate. |
796 | if (rhs_slot.loc() == LiftoffAssembler::VarState::kIntConst) { |
797 | __ cache_state()->stack_state.pop_back(); |
798 | int32_t imm = rhs_slot.i32_const(); |
799 | |
800 | LiftoffRegister lhs = __ PopToRegister(); |
801 | LiftoffRegister dst = src_rc == result_rc |
802 | ? __ GetUnusedRegister(result_rc, {lhs}) |
803 | : __ GetUnusedRegister(result_rc); |
804 | |
805 | fnImm(dst, lhs, imm); |
806 | __ PushRegister(result_type, dst); |
807 | } else { |
808 | // The RHS was not an immediate. |
809 | LiftoffRegister rhs = __ PopToRegister(); |
810 | LiftoffRegister lhs = __ PopToRegister(LiftoffRegList::ForRegs(rhs)); |
811 | LiftoffRegister dst = src_rc == result_rc |
812 | ? __ GetUnusedRegister(result_rc, {lhs, rhs}) |
813 | : __ GetUnusedRegister(result_rc); |
814 | fn(dst, lhs, rhs); |
815 | __ PushRegister(result_type, dst); |
816 | } |
817 | } |
818 | |
819 | template <ValueType src_type, ValueType result_type, typename EmitFn> |
820 | void EmitBinOp(EmitFn fn) { |
821 | static constexpr RegClass src_rc = reg_class_for(src_type); |
822 | static constexpr RegClass result_rc = reg_class_for(result_type); |
823 | LiftoffRegister rhs = __ PopToRegister(); |
824 | LiftoffRegister lhs = __ PopToRegister(LiftoffRegList::ForRegs(rhs)); |
825 | LiftoffRegister dst = src_rc == result_rc |
826 | ? __ GetUnusedRegister(result_rc, {lhs, rhs}) |
827 | : __ GetUnusedRegister(result_rc); |
828 | fn(dst, lhs, rhs); |
829 | __ PushRegister(result_type, dst); |
830 | } |
831 | |
832 | void EmitDivOrRem64CCall(LiftoffRegister dst, LiftoffRegister lhs, |
833 | LiftoffRegister rhs, ExternalReference ext_ref, |
834 | Label* trap_by_zero, |
835 | Label* trap_unrepresentable = nullptr) { |
836 | // Cannot emit native instructions, build C call. |
837 | LiftoffRegister ret = |
838 | __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst)); |
839 | LiftoffRegister tmp = |
840 | __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst, ret)); |
841 | LiftoffRegister arg_regs[] = {lhs, rhs}; |
842 | LiftoffRegister result_regs[] = {ret, dst}; |
843 | ValueType sig_types[] = {kWasmI32, kWasmI64, kWasmI64}; |
844 | // <i64, i64> -> i32 (with i64 output argument) |
845 | FunctionSig sig(1, 2, sig_types); |
846 | GenerateCCall(result_regs, &sig, kWasmI64, arg_regs, ext_ref); |
847 | __ LoadConstant(tmp, WasmValue(int32_t{0})); |
848 | __ emit_cond_jump(kEqual, trap_by_zero, kWasmI32, ret.gp(), tmp.gp()); |
849 | if (trap_unrepresentable) { |
850 | __ LoadConstant(tmp, WasmValue(int32_t{-1})); |
851 | __ emit_cond_jump(kEqual, trap_unrepresentable, kWasmI32, ret.gp(), |
852 | tmp.gp()); |
853 | } |
854 | } |
855 | |
856 | void BinOp(FullDecoder* decoder, WasmOpcode opcode, const Value& lhs, |
857 | const Value& rhs, Value* result) { |
858 | #define CASE_I32_BINOP(opcode, fn) \ |
859 | case WasmOpcode::kExpr##opcode: \ |
860 | return EmitBinOp<kWasmI32, kWasmI32>( \ |
861 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
862 | __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp()); \ |
863 | }); |
864 | #define CASE_I32_BINOPI(opcode, fn) \ |
865 | case WasmOpcode::kExpr##opcode: \ |
866 | return EmitBinOpImm<kWasmI32, kWasmI32>( \ |
867 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
868 | __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp()); \ |
869 | }, \ |
870 | [=](LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) { \ |
871 | __ emit_##fn(dst.gp(), lhs.gp(), imm); \ |
872 | }); |
873 | #define CASE_I64_BINOP(opcode, fn) \ |
874 | case WasmOpcode::kExpr##opcode: \ |
875 | return EmitBinOp<kWasmI64, kWasmI64>( \ |
876 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
877 | __ emit_##fn(dst, lhs, rhs); \ |
878 | }); |
879 | #define CASE_I64_BINOPI(opcode, fn) \ |
880 | case WasmOpcode::kExpr##opcode: \ |
881 | return EmitBinOpImm<kWasmI64, kWasmI64>( \ |
882 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
883 | __ emit_##fn(dst, lhs, rhs); \ |
884 | }, \ |
885 | [=](LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) { \ |
886 | __ emit_##fn(dst, lhs, imm); \ |
887 | }); |
888 | #define CASE_FLOAT_BINOP(opcode, type, fn) \ |
889 | case WasmOpcode::kExpr##opcode: \ |
890 | return EmitBinOp<kWasm##type, kWasm##type>( \ |
891 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
892 | __ emit_##fn(dst.fp(), lhs.fp(), rhs.fp()); \ |
893 | }); |
894 | #define CASE_I32_CMPOP(opcode, cond) \ |
895 | case WasmOpcode::kExpr##opcode: \ |
896 | return EmitBinOp<kWasmI32, kWasmI32>( \ |
897 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
898 | __ emit_i32_set_cond(cond, dst.gp(), lhs.gp(), rhs.gp()); \ |
899 | }); |
900 | #define CASE_I64_CMPOP(opcode, cond) \ |
901 | case WasmOpcode::kExpr##opcode: \ |
902 | return EmitBinOp<kWasmI64, kWasmI32>( \ |
903 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
904 | __ emit_i64_set_cond(cond, dst.gp(), lhs, rhs); \ |
905 | }); |
906 | #define CASE_F32_CMPOP(opcode, cond) \ |
907 | case WasmOpcode::kExpr##opcode: \ |
908 | return EmitBinOp<kWasmF32, kWasmI32>( \ |
909 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
910 | __ emit_f32_set_cond(cond, dst.gp(), lhs.fp(), rhs.fp()); \ |
911 | }); |
912 | #define CASE_F64_CMPOP(opcode, cond) \ |
913 | case WasmOpcode::kExpr##opcode: \ |
914 | return EmitBinOp<kWasmF64, kWasmI32>( \ |
915 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
916 | __ emit_f64_set_cond(cond, dst.gp(), lhs.fp(), rhs.fp()); \ |
917 | }); |
918 | #define CASE_I32_SHIFTOP(opcode, fn) \ |
919 | case WasmOpcode::kExpr##opcode: \ |
920 | return EmitBinOp<kWasmI32, kWasmI32>( \ |
921 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
922 | __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp(), {}); \ |
923 | }); |
924 | #define CASE_I64_SHIFTOP(opcode, fn) \ |
925 | case WasmOpcode::kExpr##opcode: \ |
926 | return EmitBinOp<kWasmI64, kWasmI64>([=](LiftoffRegister dst, \ |
927 | LiftoffRegister src, \ |
928 | LiftoffRegister amount) { \ |
929 | __ emit_##fn(dst, src, amount.is_pair() ? amount.low_gp() : amount.gp(), \ |
930 | {}); \ |
931 | }); |
932 | #define CASE_CCALL_BINOP(opcode, type, ext_ref_fn) \ |
933 | case WasmOpcode::kExpr##opcode: \ |
934 | return EmitBinOp<kWasmI32, kWasmI32>( \ |
935 | [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \ |
936 | LiftoffRegister args[] = {lhs, rhs}; \ |
937 | auto ext_ref = ExternalReference::ext_ref_fn(); \ |
938 | ValueType sig_i_ii_reps[] = {kWasmI32, kWasmI32, kWasmI32}; \ |
939 | FunctionSig sig_i_ii(1, 2, sig_i_ii_reps); \ |
940 | GenerateCCall(&dst, &sig_i_ii, kWasmStmt, args, ext_ref); \ |
941 | }); |
942 | switch (opcode) { |
943 | CASE_I32_BINOPI(I32Add, i32_add) |
944 | CASE_I32_BINOP(I32Sub, i32_sub) |
945 | CASE_I32_BINOP(I32Mul, i32_mul) |
946 | CASE_I32_BINOP(I32And, i32_and) |
947 | CASE_I32_BINOP(I32Ior, i32_or) |
948 | CASE_I32_BINOP(I32Xor, i32_xor) |
949 | CASE_I64_BINOP(I64And, i64_and) |
950 | CASE_I64_BINOP(I64Ior, i64_or) |
951 | CASE_I64_BINOP(I64Xor, i64_xor) |
952 | CASE_I32_CMPOP(I32Eq, kEqual) |
953 | CASE_I32_CMPOP(I32Ne, kUnequal) |
954 | CASE_I32_CMPOP(I32LtS, kSignedLessThan) |
955 | CASE_I32_CMPOP(I32LtU, kUnsignedLessThan) |
956 | CASE_I32_CMPOP(I32GtS, kSignedGreaterThan) |
957 | CASE_I32_CMPOP(I32GtU, kUnsignedGreaterThan) |
958 | CASE_I32_CMPOP(I32LeS, kSignedLessEqual) |
959 | CASE_I32_CMPOP(I32LeU, kUnsignedLessEqual) |
960 | CASE_I32_CMPOP(I32GeS, kSignedGreaterEqual) |
961 | CASE_I32_CMPOP(I32GeU, kUnsignedGreaterEqual) |
962 | CASE_I64_BINOPI(I64Add, i64_add) |
963 | CASE_I64_BINOP(I64Sub, i64_sub) |
964 | CASE_I64_BINOP(I64Mul, i64_mul) |
965 | CASE_I64_CMPOP(I64Eq, kEqual) |
966 | CASE_I64_CMPOP(I64Ne, kUnequal) |
967 | CASE_I64_CMPOP(I64LtS, kSignedLessThan) |
968 | CASE_I64_CMPOP(I64LtU, kUnsignedLessThan) |
969 | CASE_I64_CMPOP(I64GtS, kSignedGreaterThan) |
970 | CASE_I64_CMPOP(I64GtU, kUnsignedGreaterThan) |
971 | CASE_I64_CMPOP(I64LeS, kSignedLessEqual) |
972 | CASE_I64_CMPOP(I64LeU, kUnsignedLessEqual) |
973 | CASE_I64_CMPOP(I64GeS, kSignedGreaterEqual) |
974 | CASE_I64_CMPOP(I64GeU, kUnsignedGreaterEqual) |
975 | CASE_F32_CMPOP(F32Eq, kEqual) |
976 | CASE_F32_CMPOP(F32Ne, kUnequal) |
977 | CASE_F32_CMPOP(F32Lt, kUnsignedLessThan) |
978 | CASE_F32_CMPOP(F32Gt, kUnsignedGreaterThan) |
979 | CASE_F32_CMPOP(F32Le, kUnsignedLessEqual) |
980 | CASE_F32_CMPOP(F32Ge, kUnsignedGreaterEqual) |
981 | CASE_F64_CMPOP(F64Eq, kEqual) |
982 | CASE_F64_CMPOP(F64Ne, kUnequal) |
983 | CASE_F64_CMPOP(F64Lt, kUnsignedLessThan) |
984 | CASE_F64_CMPOP(F64Gt, kUnsignedGreaterThan) |
985 | CASE_F64_CMPOP(F64Le, kUnsignedLessEqual) |
986 | CASE_F64_CMPOP(F64Ge, kUnsignedGreaterEqual) |
987 | CASE_I32_SHIFTOP(I32Shl, i32_shl) |
988 | CASE_I32_SHIFTOP(I32ShrS, i32_sar) |
989 | CASE_I32_SHIFTOP(I32ShrU, i32_shr) |
990 | CASE_I64_SHIFTOP(I64Shl, i64_shl) |
991 | CASE_I64_SHIFTOP(I64ShrS, i64_sar) |
992 | CASE_I64_SHIFTOP(I64ShrU, i64_shr) |
993 | CASE_CCALL_BINOP(I32Rol, I32, wasm_word32_rol) |
994 | CASE_CCALL_BINOP(I32Ror, I32, wasm_word32_ror) |
995 | CASE_FLOAT_BINOP(F32Add, F32, f32_add) |
996 | CASE_FLOAT_BINOP(F32Sub, F32, f32_sub) |
997 | CASE_FLOAT_BINOP(F32Mul, F32, f32_mul) |
998 | CASE_FLOAT_BINOP(F32Div, F32, f32_div) |
999 | CASE_FLOAT_BINOP(F32Min, F32, f32_min) |
1000 | CASE_FLOAT_BINOP(F32Max, F32, f32_max) |
1001 | CASE_FLOAT_BINOP(F32CopySign, F32, f32_copysign) |
1002 | CASE_FLOAT_BINOP(F64Add, F64, f64_add) |
1003 | CASE_FLOAT_BINOP(F64Sub, F64, f64_sub) |
1004 | CASE_FLOAT_BINOP(F64Mul, F64, f64_mul) |
1005 | CASE_FLOAT_BINOP(F64Div, F64, f64_div) |
1006 | CASE_FLOAT_BINOP(F64Min, F64, f64_min) |
1007 | CASE_FLOAT_BINOP(F64Max, F64, f64_max) |
1008 | CASE_FLOAT_BINOP(F64CopySign, F64, f64_copysign) |
1009 | case WasmOpcode::kExprI32DivS: |
1010 | EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst, |
1011 | LiftoffRegister lhs, |
1012 | LiftoffRegister rhs) { |
1013 | WasmCodePosition position = decoder->position(); |
1014 | AddOutOfLineTrap(position, WasmCode::kThrowWasmTrapDivByZero); |
1015 | // Adding the second trap might invalidate the pointer returned for |
1016 | // the first one, thus get both pointers afterwards. |
1017 | AddOutOfLineTrap(position, |
1018 | WasmCode::kThrowWasmTrapDivUnrepresentable); |
1019 | Label* div_by_zero = out_of_line_code_.end()[-2].label.get(); |
1020 | Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get(); |
1021 | __ emit_i32_divs(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero, |
1022 | div_unrepresentable); |
1023 | }); |
1024 | break; |
1025 | case WasmOpcode::kExprI32DivU: |
1026 | EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst, |
1027 | LiftoffRegister lhs, |
1028 | LiftoffRegister rhs) { |
1029 | Label* div_by_zero = AddOutOfLineTrap( |
1030 | decoder->position(), WasmCode::kThrowWasmTrapDivByZero); |
1031 | __ emit_i32_divu(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero); |
1032 | }); |
1033 | break; |
1034 | case WasmOpcode::kExprI32RemS: |
1035 | EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst, |
1036 | LiftoffRegister lhs, |
1037 | LiftoffRegister rhs) { |
1038 | Label* rem_by_zero = AddOutOfLineTrap( |
1039 | decoder->position(), WasmCode::kThrowWasmTrapRemByZero); |
1040 | __ emit_i32_rems(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero); |
1041 | }); |
1042 | break; |
1043 | case WasmOpcode::kExprI32RemU: |
1044 | EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst, |
1045 | LiftoffRegister lhs, |
1046 | LiftoffRegister rhs) { |
1047 | Label* rem_by_zero = AddOutOfLineTrap( |
1048 | decoder->position(), WasmCode::kThrowWasmTrapRemByZero); |
1049 | __ emit_i32_remu(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero); |
1050 | }); |
1051 | break; |
1052 | case WasmOpcode::kExprI64DivS: |
1053 | EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst, |
1054 | LiftoffRegister lhs, |
1055 | LiftoffRegister rhs) { |
1056 | WasmCodePosition position = decoder->position(); |
1057 | AddOutOfLineTrap(position, WasmCode::kThrowWasmTrapDivByZero); |
1058 | // Adding the second trap might invalidate the pointer returned for |
1059 | // the first one, thus get both pointers afterwards. |
1060 | AddOutOfLineTrap(position, |
1061 | WasmCode::kThrowWasmTrapDivUnrepresentable); |
1062 | Label* div_by_zero = out_of_line_code_.end()[-2].label.get(); |
1063 | Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get(); |
1064 | if (!__ emit_i64_divs(dst, lhs, rhs, div_by_zero, |
1065 | div_unrepresentable)) { |
1066 | ExternalReference ext_ref = ExternalReference::wasm_int64_div(); |
1067 | EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero, |
1068 | div_unrepresentable); |
1069 | } |
1070 | }); |
1071 | break; |
1072 | case WasmOpcode::kExprI64DivU: |
1073 | EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst, |
1074 | LiftoffRegister lhs, |
1075 | LiftoffRegister rhs) { |
1076 | Label* div_by_zero = AddOutOfLineTrap( |
1077 | decoder->position(), WasmCode::kThrowWasmTrapDivByZero); |
1078 | if (!__ emit_i64_divu(dst, lhs, rhs, div_by_zero)) { |
1079 | ExternalReference ext_ref = ExternalReference::wasm_uint64_div(); |
1080 | EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero); |
1081 | } |
1082 | }); |
1083 | break; |
1084 | case WasmOpcode::kExprI64RemS: |
1085 | EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst, |
1086 | LiftoffRegister lhs, |
1087 | LiftoffRegister rhs) { |
1088 | Label* rem_by_zero = AddOutOfLineTrap( |
1089 | decoder->position(), WasmCode::kThrowWasmTrapRemByZero); |
1090 | if (!__ emit_i64_rems(dst, lhs, rhs, rem_by_zero)) { |
1091 | ExternalReference ext_ref = ExternalReference::wasm_int64_mod(); |
1092 | EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero); |
1093 | } |
1094 | }); |
1095 | break; |
1096 | case WasmOpcode::kExprI64RemU: |
1097 | EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst, |
1098 | LiftoffRegister lhs, |
1099 | LiftoffRegister rhs) { |
1100 | Label* rem_by_zero = AddOutOfLineTrap( |
1101 | decoder->position(), WasmCode::kThrowWasmTrapRemByZero); |
1102 | if (!__ emit_i64_remu(dst, lhs, rhs, rem_by_zero)) { |
1103 | ExternalReference ext_ref = ExternalReference::wasm_uint64_mod(); |
1104 | EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero); |
1105 | } |
1106 | }); |
1107 | break; |
1108 | default: |
1109 | return unsupported(decoder, WasmOpcodes::OpcodeName(opcode)); |
1110 | } |
1111 | #undef CASE_I32_BINOP |
1112 | #undef CASE_I32_BINOPI |
1113 | #undef CASE_I64_BINOP |
1114 | #undef CASE_I64_BINOPI |
1115 | #undef CASE_FLOAT_BINOP |
1116 | #undef CASE_I32_CMPOP |
1117 | #undef CASE_I64_CMPOP |
1118 | #undef CASE_F32_CMPOP |
1119 | #undef CASE_F64_CMPOP |
1120 | #undef CASE_I32_SHIFTOP |
1121 | #undef CASE_I64_SHIFTOP |
1122 | #undef CASE_CCALL_BINOP |
1123 | } |
1124 | |
1125 | void I32Const(FullDecoder* decoder, Value* result, int32_t value) { |
1126 | __ cache_state()->stack_state.emplace_back(kWasmI32, value); |
1127 | } |
1128 | |
1129 | void I64Const(FullDecoder* decoder, Value* result, int64_t value) { |
1130 | // The {VarState} stores constant values as int32_t, thus we only store |
1131 | // 64-bit constants in this field if it fits in an int32_t. Larger values |
1132 | // cannot be used as immediate value anyway, so we can also just put them in |
1133 | // a register immediately. |
1134 | int32_t value_i32 = static_cast<int32_t>(value); |
1135 | if (value_i32 == value) { |
1136 | __ cache_state()->stack_state.emplace_back(kWasmI64, value_i32); |
1137 | } else { |
1138 | LiftoffRegister reg = __ GetUnusedRegister(reg_class_for(kWasmI64)); |
1139 | __ LoadConstant(reg, WasmValue(value)); |
1140 | __ PushRegister(kWasmI64, reg); |
1141 | } |
1142 | } |
1143 | |
1144 | void F32Const(FullDecoder* decoder, Value* result, float value) { |
1145 | LiftoffRegister reg = __ GetUnusedRegister(kFpReg); |
1146 | __ LoadConstant(reg, WasmValue(value)); |
1147 | __ PushRegister(kWasmF32, reg); |
1148 | } |
1149 | |
1150 | void F64Const(FullDecoder* decoder, Value* result, double value) { |
1151 | LiftoffRegister reg = __ GetUnusedRegister(kFpReg); |
1152 | __ LoadConstant(reg, WasmValue(value)); |
1153 | __ PushRegister(kWasmF64, reg); |
1154 | } |
1155 | |
1156 | void RefNull(FullDecoder* decoder, Value* result) { |
1157 | unsupported(decoder, "ref_null" ); |
1158 | } |
1159 | |
1160 | void Drop(FullDecoder* decoder, const Value& value) { |
1161 | auto& slot = __ cache_state()->stack_state.back(); |
1162 | // If the dropped slot contains a register, decrement it's use count. |
1163 | if (slot.is_reg()) __ cache_state()->dec_used(slot.reg()); |
1164 | __ cache_state()->stack_state.pop_back(); |
1165 | } |
1166 | |
1167 | void ReturnImpl(FullDecoder* decoder) { |
1168 | size_t num_returns = decoder->sig_->return_count(); |
1169 | if (num_returns > 1) return unsupported(decoder, "multi-return" ); |
1170 | if (num_returns > 0) __ MoveToReturnRegisters(decoder->sig_); |
1171 | __ LeaveFrame(StackFrame::WASM_COMPILED); |
1172 | __ DropStackSlotsAndRet( |
1173 | static_cast<uint32_t>(descriptor_->StackParameterCount())); |
1174 | } |
1175 | |
1176 | void DoReturn(FullDecoder* decoder, Vector<Value> /*values*/) { |
1177 | ReturnImpl(decoder); |
1178 | } |
1179 | |
1180 | void GetLocal(FullDecoder* decoder, Value* result, |
1181 | const LocalIndexImmediate<validate>& imm) { |
1182 | auto& slot = __ cache_state()->stack_state[imm.index]; |
1183 | DCHECK_EQ(slot.type(), imm.type); |
1184 | switch (slot.loc()) { |
1185 | case kRegister: |
1186 | __ PushRegister(slot.type(), slot.reg()); |
1187 | break; |
1188 | case kIntConst: |
1189 | __ cache_state()->stack_state.emplace_back(imm.type, slot.i32_const()); |
1190 | break; |
1191 | case kStack: { |
1192 | auto rc = reg_class_for(imm.type); |
1193 | LiftoffRegister reg = __ GetUnusedRegister(rc); |
1194 | __ Fill(reg, imm.index, imm.type); |
1195 | __ PushRegister(slot.type(), reg); |
1196 | break; |
1197 | } |
1198 | } |
1199 | } |
1200 | |
1201 | void SetLocalFromStackSlot(LiftoffAssembler::VarState& dst_slot, |
1202 | uint32_t local_index) { |
1203 | auto& state = *__ cache_state(); |
1204 | ValueType type = dst_slot.type(); |
1205 | if (dst_slot.is_reg()) { |
1206 | LiftoffRegister slot_reg = dst_slot.reg(); |
1207 | if (state.get_use_count(slot_reg) == 1) { |
1208 | __ Fill(dst_slot.reg(), state.stack_height() - 1, type); |
1209 | return; |
1210 | } |
1211 | state.dec_used(slot_reg); |
1212 | dst_slot.MakeStack(); |
1213 | } |
1214 | DCHECK_EQ(type, __ local_type(local_index)); |
1215 | RegClass rc = reg_class_for(type); |
1216 | LiftoffRegister dst_reg = __ GetUnusedRegister(rc); |
1217 | __ Fill(dst_reg, __ cache_state()->stack_height() - 1, type); |
1218 | dst_slot = LiftoffAssembler::VarState(type, dst_reg); |
1219 | __ cache_state()->inc_used(dst_reg); |
1220 | } |
1221 | |
1222 | void SetLocal(uint32_t local_index, bool is_tee) { |
1223 | auto& state = *__ cache_state(); |
1224 | auto& source_slot = state.stack_state.back(); |
1225 | auto& target_slot = state.stack_state[local_index]; |
1226 | switch (source_slot.loc()) { |
1227 | case kRegister: |
1228 | if (target_slot.is_reg()) state.dec_used(target_slot.reg()); |
1229 | target_slot = source_slot; |
1230 | if (is_tee) state.inc_used(target_slot.reg()); |
1231 | break; |
1232 | case kIntConst: |
1233 | if (target_slot.is_reg()) state.dec_used(target_slot.reg()); |
1234 | target_slot = source_slot; |
1235 | break; |
1236 | case kStack: |
1237 | SetLocalFromStackSlot(target_slot, local_index); |
1238 | break; |
1239 | } |
1240 | if (!is_tee) __ cache_state()->stack_state.pop_back(); |
1241 | } |
1242 | |
1243 | void SetLocal(FullDecoder* decoder, const Value& value, |
1244 | const LocalIndexImmediate<validate>& imm) { |
1245 | SetLocal(imm.index, false); |
1246 | } |
1247 | |
1248 | void TeeLocal(FullDecoder* decoder, const Value& value, Value* result, |
1249 | const LocalIndexImmediate<validate>& imm) { |
1250 | SetLocal(imm.index, true); |
1251 | } |
1252 | |
1253 | Register GetGlobalBaseAndOffset(const WasmGlobal* global, |
1254 | LiftoffRegList& pinned, uint32_t* offset) { |
1255 | Register addr = pinned.set(__ GetUnusedRegister(kGpReg)).gp(); |
1256 | if (global->mutability && global->imported) { |
1257 | LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize); |
1258 | __ Load(LiftoffRegister(addr), addr, no_reg, |
1259 | global->index * sizeof(Address), kPointerLoadType, pinned); |
1260 | *offset = 0; |
1261 | } else { |
1262 | LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize); |
1263 | *offset = global->offset; |
1264 | } |
1265 | return addr; |
1266 | } |
1267 | |
1268 | void GetGlobal(FullDecoder* decoder, Value* result, |
1269 | const GlobalIndexImmediate<validate>& imm) { |
1270 | const auto* global = &env_->module->globals[imm.index]; |
1271 | if (!CheckSupportedType(decoder, kSupportedTypes, global->type, "global" )) |
1272 | return; |
1273 | LiftoffRegList pinned; |
1274 | uint32_t offset = 0; |
1275 | Register addr = GetGlobalBaseAndOffset(global, pinned, &offset); |
1276 | LiftoffRegister value = |
1277 | pinned.set(__ GetUnusedRegister(reg_class_for(global->type), pinned)); |
1278 | LoadType type = LoadType::ForValueType(global->type); |
1279 | __ Load(value, addr, no_reg, offset, type, pinned, nullptr, true); |
1280 | __ PushRegister(global->type, value); |
1281 | } |
1282 | |
1283 | void SetGlobal(FullDecoder* decoder, const Value& value, |
1284 | const GlobalIndexImmediate<validate>& imm) { |
1285 | auto* global = &env_->module->globals[imm.index]; |
1286 | if (!CheckSupportedType(decoder, kSupportedTypes, global->type, "global" )) |
1287 | return; |
1288 | LiftoffRegList pinned; |
1289 | uint32_t offset = 0; |
1290 | Register addr = GetGlobalBaseAndOffset(global, pinned, &offset); |
1291 | LiftoffRegister reg = pinned.set(__ PopToRegister(pinned)); |
1292 | StoreType type = StoreType::ForValueType(global->type); |
1293 | __ Store(addr, no_reg, offset, reg, type, {}, nullptr, true); |
1294 | } |
1295 | |
1296 | void GetTable(FullDecoder* decoder, const Value& index, Value* result, |
1297 | TableIndexImmediate<validate>& imm) { |
1298 | unsupported(decoder, "table_get" ); |
1299 | } |
1300 | |
1301 | void SetTable(FullDecoder* decoder, const Value& index, const Value& value, |
1302 | TableIndexImmediate<validate>& imm) { |
1303 | unsupported(decoder, "table_set" ); |
1304 | } |
1305 | |
1306 | void Unreachable(FullDecoder* decoder) { |
1307 | Label* unreachable_label = AddOutOfLineTrap( |
1308 | decoder->position(), WasmCode::kThrowWasmTrapUnreachable); |
1309 | __ emit_jump(unreachable_label); |
1310 | __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap); |
1311 | } |
1312 | |
1313 | void Select(FullDecoder* decoder, const Value& cond, const Value& fval, |
1314 | const Value& tval, Value* result) { |
1315 | LiftoffRegList pinned; |
1316 | Register condition = pinned.set(__ PopToRegister()).gp(); |
1317 | ValueType type = __ cache_state()->stack_state.end()[-1].type(); |
1318 | DCHECK_EQ(type, __ cache_state()->stack_state.end()[-2].type()); |
1319 | LiftoffRegister false_value = pinned.set(__ PopToRegister(pinned)); |
1320 | LiftoffRegister true_value = __ PopToRegister(pinned); |
1321 | LiftoffRegister dst = |
1322 | __ GetUnusedRegister(true_value.reg_class(), {true_value, false_value}); |
1323 | __ PushRegister(type, dst); |
1324 | |
1325 | // Now emit the actual code to move either {true_value} or {false_value} |
1326 | // into {dst}. |
1327 | Label cont; |
1328 | Label case_false; |
1329 | __ emit_cond_jump(kEqual, &case_false, kWasmI32, condition); |
1330 | if (dst != true_value) __ Move(dst, true_value, type); |
1331 | __ emit_jump(&cont); |
1332 | |
1333 | __ bind(&case_false); |
1334 | if (dst != false_value) __ Move(dst, false_value, type); |
1335 | __ bind(&cont); |
1336 | } |
1337 | |
1338 | void BrImpl(Control* target) { |
1339 | if (!target->br_merge()->reached) { |
1340 | target->label_state.InitMerge(*__ cache_state(), __ num_locals(), |
1341 | target->br_merge()->arity, |
1342 | target->stack_depth); |
1343 | } |
1344 | __ MergeStackWith(target->label_state, target->br_merge()->arity); |
1345 | __ jmp(target->label.get()); |
1346 | } |
1347 | |
1348 | void Br(FullDecoder* decoder, Control* target) { BrImpl(target); } |
1349 | |
1350 | void BrOrRet(FullDecoder* decoder, uint32_t depth) { |
1351 | if (depth == decoder->control_depth() - 1) { |
1352 | ReturnImpl(decoder); |
1353 | } else { |
1354 | BrImpl(decoder->control_at(depth)); |
1355 | } |
1356 | } |
1357 | |
1358 | void BrIf(FullDecoder* decoder, const Value& cond, uint32_t depth) { |
1359 | Label cont_false; |
1360 | Register value = __ PopToRegister().gp(); |
1361 | __ emit_cond_jump(kEqual, &cont_false, kWasmI32, value); |
1362 | |
1363 | BrOrRet(decoder, depth); |
1364 | __ bind(&cont_false); |
1365 | } |
1366 | |
1367 | // Generate a branch table case, potentially reusing previously generated |
1368 | // stack transfer code. |
1369 | void GenerateBrCase(FullDecoder* decoder, uint32_t br_depth, |
1370 | std::map<uint32_t, MovableLabel>& br_targets) { |
1371 | MovableLabel& label = br_targets[br_depth]; |
1372 | if (label.get()->is_bound()) { |
1373 | __ jmp(label.get()); |
1374 | } else { |
1375 | __ bind(label.get()); |
1376 | BrOrRet(decoder, br_depth); |
1377 | } |
1378 | } |
1379 | |
1380 | // Generate a branch table for input in [min, max). |
1381 | // TODO(wasm): Generate a real branch table (like TF TableSwitch). |
1382 | void GenerateBrTable(FullDecoder* decoder, LiftoffRegister tmp, |
1383 | LiftoffRegister value, uint32_t min, uint32_t max, |
1384 | BranchTableIterator<validate>& table_iterator, |
1385 | std::map<uint32_t, MovableLabel>& br_targets) { |
1386 | DCHECK_LT(min, max); |
1387 | // Check base case. |
1388 | if (max == min + 1) { |
1389 | DCHECK_EQ(min, table_iterator.cur_index()); |
1390 | GenerateBrCase(decoder, table_iterator.next(), br_targets); |
1391 | return; |
1392 | } |
1393 | |
1394 | uint32_t split = min + (max - min) / 2; |
1395 | Label upper_half; |
1396 | __ LoadConstant(tmp, WasmValue(split)); |
1397 | __ emit_cond_jump(kUnsignedGreaterEqual, &upper_half, kWasmI32, value.gp(), |
1398 | tmp.gp()); |
1399 | // Emit br table for lower half: |
1400 | GenerateBrTable(decoder, tmp, value, min, split, table_iterator, |
1401 | br_targets); |
1402 | __ bind(&upper_half); |
1403 | // Emit br table for upper half: |
1404 | GenerateBrTable(decoder, tmp, value, split, max, table_iterator, |
1405 | br_targets); |
1406 | } |
1407 | |
1408 | void BrTable(FullDecoder* decoder, const BranchTableImmediate<validate>& imm, |
1409 | const Value& key) { |
1410 | LiftoffRegList pinned; |
1411 | LiftoffRegister value = pinned.set(__ PopToRegister()); |
1412 | BranchTableIterator<validate> table_iterator(decoder, imm); |
1413 | std::map<uint32_t, MovableLabel> br_targets; |
1414 | |
1415 | if (imm.table_count > 0) { |
1416 | LiftoffRegister tmp = __ GetUnusedRegister(kGpReg, pinned); |
1417 | __ LoadConstant(tmp, WasmValue(uint32_t{imm.table_count})); |
1418 | Label case_default; |
1419 | __ emit_cond_jump(kUnsignedGreaterEqual, &case_default, kWasmI32, |
1420 | value.gp(), tmp.gp()); |
1421 | |
1422 | GenerateBrTable(decoder, tmp, value, 0, imm.table_count, table_iterator, |
1423 | br_targets); |
1424 | |
1425 | __ bind(&case_default); |
1426 | } |
1427 | |
1428 | // Generate the default case. |
1429 | GenerateBrCase(decoder, table_iterator.next(), br_targets); |
1430 | DCHECK(!table_iterator.has_next()); |
1431 | } |
1432 | |
1433 | void Else(FullDecoder* decoder, Control* c) { |
1434 | if (c->reachable()) { |
1435 | if (!c->end_merge.reached) { |
1436 | c->label_state.InitMerge(*__ cache_state(), __ num_locals(), |
1437 | c->end_merge.arity, c->stack_depth); |
1438 | } |
1439 | __ MergeFullStackWith(c->label_state, *__ cache_state()); |
1440 | __ emit_jump(c->label.get()); |
1441 | } |
1442 | __ bind(c->else_state->label.get()); |
1443 | __ cache_state()->Steal(c->else_state->state); |
1444 | } |
1445 | |
1446 | Label* AddOutOfLineTrap(WasmCodePosition position, |
1447 | WasmCode::RuntimeStubId stub, uint32_t pc = 0) { |
1448 | DCHECK(!FLAG_wasm_no_bounds_checks); |
1449 | // The pc is needed for memory OOB trap with trap handler enabled. Other |
1450 | // callers should not even compute it. |
1451 | DCHECK_EQ(pc != 0, stub == WasmCode::kThrowWasmTrapMemOutOfBounds && |
1452 | env_->use_trap_handler); |
1453 | |
1454 | out_of_line_code_.push_back(OutOfLineCode::Trap(stub, position, pc)); |
1455 | return out_of_line_code_.back().label.get(); |
1456 | } |
1457 | |
1458 | // Returns true if the memory access is statically known to be out of bounds |
1459 | // (a jump to the trap was generated then); return false otherwise. |
1460 | bool BoundsCheckMem(FullDecoder* decoder, uint32_t access_size, |
1461 | uint32_t offset, Register index, LiftoffRegList pinned) { |
1462 | const bool statically_oob = |
1463 | !IsInBounds(offset, access_size, env_->max_memory_size); |
1464 | |
1465 | if (!statically_oob && |
1466 | (FLAG_wasm_no_bounds_checks || env_->use_trap_handler)) { |
1467 | return false; |
1468 | } |
1469 | |
1470 | // TODO(wasm): This adds protected instruction information for the jump |
1471 | // instruction we are about to generate. It would be better to just not add |
1472 | // protected instruction info when the pc is 0. |
1473 | Label* trap_label = AddOutOfLineTrap( |
1474 | decoder->position(), WasmCode::kThrowWasmTrapMemOutOfBounds, |
1475 | env_->use_trap_handler ? __ pc_offset() : 0); |
1476 | |
1477 | if (statically_oob) { |
1478 | __ emit_jump(trap_label); |
1479 | Control* current_block = decoder->control_at(0); |
1480 | if (current_block->reachable()) { |
1481 | current_block->reachability = kSpecOnlyReachable; |
1482 | } |
1483 | return true; |
1484 | } |
1485 | |
1486 | DCHECK(!env_->use_trap_handler); |
1487 | DCHECK(!FLAG_wasm_no_bounds_checks); |
1488 | |
1489 | uint64_t end_offset = uint64_t{offset} + access_size - 1u; |
1490 | |
1491 | // If the end offset is larger than the smallest memory, dynamically check |
1492 | // the end offset against the actual memory size, which is not known at |
1493 | // compile time. Otherwise, only one check is required (see below). |
1494 | LiftoffRegister end_offset_reg = |
1495 | pinned.set(__ GetUnusedRegister(kGpReg, pinned)); |
1496 | Register mem_size = __ GetUnusedRegister(kGpReg, pinned).gp(); |
1497 | LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize); |
1498 | |
1499 | if (kSystemPointerSize == 8) { |
1500 | __ LoadConstant(end_offset_reg, WasmValue(end_offset)); |
1501 | } else { |
1502 | __ LoadConstant(end_offset_reg, |
1503 | WasmValue(static_cast<uint32_t>(end_offset))); |
1504 | } |
1505 | |
1506 | if (end_offset >= env_->min_memory_size) { |
1507 | __ emit_cond_jump(kUnsignedGreaterEqual, trap_label, |
1508 | LiftoffAssembler::kWasmIntPtr, end_offset_reg.gp(), |
1509 | mem_size); |
1510 | } |
1511 | |
1512 | // Just reuse the end_offset register for computing the effective size. |
1513 | LiftoffRegister effective_size_reg = end_offset_reg; |
1514 | __ emit_ptrsize_sub(effective_size_reg.gp(), mem_size, end_offset_reg.gp()); |
1515 | |
1516 | __ emit_i32_to_intptr(index, index); |
1517 | |
1518 | __ emit_cond_jump(kUnsignedGreaterEqual, trap_label, |
1519 | LiftoffAssembler::kWasmIntPtr, index, |
1520 | effective_size_reg.gp()); |
1521 | return false; |
1522 | } |
1523 | |
1524 | void TraceMemoryOperation(bool is_store, MachineRepresentation rep, |
1525 | Register index, uint32_t offset, |
1526 | WasmCodePosition position) { |
1527 | // Before making the runtime call, spill all cache registers. |
1528 | __ SpillAllRegisters(); |
1529 | |
1530 | LiftoffRegList pinned = LiftoffRegList::ForRegs(index); |
1531 | // Get one register for computing the address (offset + index). |
1532 | LiftoffRegister address = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); |
1533 | // Compute offset+index in address. |
1534 | __ LoadConstant(address, WasmValue(offset)); |
1535 | __ emit_i32_add(address.gp(), address.gp(), index); |
1536 | |
1537 | // Get a register to hold the stack slot for MemoryTracingInfo. |
1538 | LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); |
1539 | // Allocate stack slot for MemoryTracingInfo. |
1540 | __ AllocateStackSlot(info.gp(), sizeof(MemoryTracingInfo)); |
1541 | |
1542 | // Now store all information into the MemoryTracingInfo struct. |
1543 | __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, address), address, |
1544 | StoreType::kI32Store, pinned); |
1545 | __ LoadConstant(address, WasmValue(is_store ? 1 : 0)); |
1546 | __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), address, |
1547 | StoreType::kI32Store8, pinned); |
1548 | __ LoadConstant(address, WasmValue(static_cast<int>(rep))); |
1549 | __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), address, |
1550 | StoreType::kI32Store8, pinned); |
1551 | |
1552 | source_position_table_builder_.AddPosition(__ pc_offset(), |
1553 | SourcePosition(position), false); |
1554 | |
1555 | Register args[] = {info.gp()}; |
1556 | GenerateRuntimeCall(Runtime::kWasmTraceMemory, arraysize(args), args); |
1557 | __ DeallocateStackSlot(sizeof(MemoryTracingInfo)); |
1558 | } |
1559 | |
1560 | void GenerateRuntimeCall(Runtime::FunctionId runtime_function, int num_args, |
1561 | Register* args) { |
1562 | auto call_descriptor = compiler::Linkage::GetRuntimeCallDescriptor( |
1563 | compilation_zone_, runtime_function, num_args, |
1564 | compiler::Operator::kNoProperties, compiler::CallDescriptor::kNoFlags); |
1565 | // Currently, only one argument is supported. More arguments require some |
1566 | // caution for the parallel register moves (reuse StackTransferRecipe). |
1567 | DCHECK_EQ(1, num_args); |
1568 | constexpr size_t kInputShift = 1; // Input 0 is the call target. |
1569 | compiler::LinkageLocation param_loc = |
1570 | call_descriptor->GetInputLocation(kInputShift); |
1571 | if (param_loc.IsRegister()) { |
1572 | Register reg = Register::from_code(param_loc.AsRegister()); |
1573 | __ Move(LiftoffRegister(reg), LiftoffRegister(args[0]), |
1574 | LiftoffAssembler::kWasmIntPtr); |
1575 | } else { |
1576 | DCHECK(param_loc.IsCallerFrameSlot()); |
1577 | LiftoffStackSlots stack_slots(&asm_); |
1578 | stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr, |
1579 | LiftoffRegister(args[0]))); |
1580 | stack_slots.Construct(); |
1581 | } |
1582 | |
1583 | // Set context to "no context" for the runtime call. |
1584 | __ TurboAssembler::Move(kContextRegister, |
1585 | Smi::FromInt(Context::kNoContext)); |
1586 | Register centry = kJavaScriptCallCodeStartRegister; |
1587 | LOAD_TAGGED_PTR_INSTANCE_FIELD(centry, CEntryStub); |
1588 | __ CallRuntimeWithCEntry(runtime_function, centry); |
1589 | safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple, |
1590 | Safepoint::kNoLazyDeopt); |
1591 | } |
1592 | |
1593 | Register AddMemoryMasking(Register index, uint32_t* offset, |
1594 | LiftoffRegList& pinned) { |
1595 | if (!FLAG_untrusted_code_mitigations || env_->use_trap_handler) { |
1596 | return index; |
1597 | } |
1598 | DEBUG_CODE_COMMENT("Mask memory index" ); |
1599 | // Make sure that we can overwrite {index}. |
1600 | if (__ cache_state()->is_used(LiftoffRegister(index))) { |
1601 | Register old_index = index; |
1602 | pinned.clear(LiftoffRegister(old_index)); |
1603 | index = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1604 | if (index != old_index) __ Move(index, old_index, kWasmI32); |
1605 | } |
1606 | Register tmp = __ GetUnusedRegister(kGpReg, pinned).gp(); |
1607 | __ emit_ptrsize_add(index, index, *offset); |
1608 | LOAD_INSTANCE_FIELD(tmp, MemoryMask, kSystemPointerSize); |
1609 | __ emit_ptrsize_and(index, index, tmp); |
1610 | *offset = 0; |
1611 | return index; |
1612 | } |
1613 | |
1614 | void LoadMem(FullDecoder* decoder, LoadType type, |
1615 | const MemoryAccessImmediate<validate>& imm, |
1616 | const Value& index_val, Value* result) { |
1617 | ValueType value_type = type.value_type(); |
1618 | if (!CheckSupportedType(decoder, kSupportedTypes, value_type, "load" )) |
1619 | return; |
1620 | LiftoffRegList pinned; |
1621 | Register index = pinned.set(__ PopToRegister()).gp(); |
1622 | if (BoundsCheckMem(decoder, type.size(), imm.offset, index, pinned)) { |
1623 | return; |
1624 | } |
1625 | uint32_t offset = imm.offset; |
1626 | index = AddMemoryMasking(index, &offset, pinned); |
1627 | DEBUG_CODE_COMMENT("Load from memory" ); |
1628 | Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1629 | LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize); |
1630 | RegClass rc = reg_class_for(value_type); |
1631 | LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned)); |
1632 | uint32_t protected_load_pc = 0; |
1633 | __ Load(value, addr, index, offset, type, pinned, &protected_load_pc, true); |
1634 | if (env_->use_trap_handler) { |
1635 | AddOutOfLineTrap(decoder->position(), |
1636 | WasmCode::kThrowWasmTrapMemOutOfBounds, |
1637 | protected_load_pc); |
1638 | } |
1639 | __ PushRegister(value_type, value); |
1640 | |
1641 | if (FLAG_trace_wasm_memory) { |
1642 | TraceMemoryOperation(false, type.mem_type().representation(), index, |
1643 | offset, decoder->position()); |
1644 | } |
1645 | } |
1646 | |
1647 | void StoreMem(FullDecoder* decoder, StoreType type, |
1648 | const MemoryAccessImmediate<validate>& imm, |
1649 | const Value& index_val, const Value& value_val) { |
1650 | ValueType value_type = type.value_type(); |
1651 | if (!CheckSupportedType(decoder, kSupportedTypes, value_type, "store" )) |
1652 | return; |
1653 | LiftoffRegList pinned; |
1654 | LiftoffRegister value = pinned.set(__ PopToRegister()); |
1655 | Register index = pinned.set(__ PopToRegister(pinned)).gp(); |
1656 | if (BoundsCheckMem(decoder, type.size(), imm.offset, index, pinned)) { |
1657 | return; |
1658 | } |
1659 | uint32_t offset = imm.offset; |
1660 | index = AddMemoryMasking(index, &offset, pinned); |
1661 | DEBUG_CODE_COMMENT("Store to memory" ); |
1662 | Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1663 | LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize); |
1664 | uint32_t protected_store_pc = 0; |
1665 | LiftoffRegList outer_pinned; |
1666 | if (FLAG_trace_wasm_memory) outer_pinned.set(index); |
1667 | __ Store(addr, index, offset, value, type, outer_pinned, |
1668 | &protected_store_pc, true); |
1669 | if (env_->use_trap_handler) { |
1670 | AddOutOfLineTrap(decoder->position(), |
1671 | WasmCode::kThrowWasmTrapMemOutOfBounds, |
1672 | protected_store_pc); |
1673 | } |
1674 | if (FLAG_trace_wasm_memory) { |
1675 | TraceMemoryOperation(true, type.mem_rep(), index, offset, |
1676 | decoder->position()); |
1677 | } |
1678 | } |
1679 | |
1680 | void CurrentMemoryPages(FullDecoder* decoder, Value* result) { |
1681 | Register mem_size = __ GetUnusedRegister(kGpReg).gp(); |
1682 | LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize); |
1683 | __ emit_ptrsize_shr(mem_size, mem_size, kWasmPageSizeLog2); |
1684 | __ PushRegister(kWasmI32, LiftoffRegister(mem_size)); |
1685 | } |
1686 | |
1687 | void MemoryGrow(FullDecoder* decoder, const Value& value, Value* result_val) { |
1688 | // Pop the input, then spill all cache registers to make the runtime call. |
1689 | LiftoffRegList pinned; |
1690 | LiftoffRegister input = pinned.set(__ PopToRegister()); |
1691 | __ SpillAllRegisters(); |
1692 | |
1693 | constexpr Register kGpReturnReg = kGpReturnRegisters[0]; |
1694 | static_assert(kLiftoffAssemblerGpCacheRegs & Register::bit<kGpReturnReg>(), |
1695 | "first return register is a cache register (needs more " |
1696 | "complex code here otherwise)" ); |
1697 | LiftoffRegister result = pinned.set(LiftoffRegister(kGpReturnReg)); |
1698 | |
1699 | WasmMemoryGrowDescriptor descriptor; |
1700 | DCHECK_EQ(0, descriptor.GetStackParameterCount()); |
1701 | DCHECK_EQ(1, descriptor.GetRegisterParameterCount()); |
1702 | DCHECK_EQ(ValueTypes::MachineTypeFor(kWasmI32), |
1703 | descriptor.GetParameterType(0)); |
1704 | |
1705 | Register param_reg = descriptor.GetRegisterParameter(0); |
1706 | if (input.gp() != param_reg) __ Move(param_reg, input.gp(), kWasmI32); |
1707 | |
1708 | __ CallRuntimeStub(WasmCode::kWasmMemoryGrow); |
1709 | safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple, |
1710 | Safepoint::kNoLazyDeopt); |
1711 | |
1712 | if (kReturnRegister0 != result.gp()) { |
1713 | __ Move(result.gp(), kReturnRegister0, kWasmI32); |
1714 | } |
1715 | |
1716 | __ PushRegister(kWasmI32, result); |
1717 | } |
1718 | |
1719 | void CallDirect(FullDecoder* decoder, |
1720 | const CallFunctionImmediate<validate>& imm, |
1721 | const Value args[], Value returns[]) { |
1722 | if (imm.sig->return_count() > 1) |
1723 | return unsupported(decoder, "multi-return" ); |
1724 | if (imm.sig->return_count() == 1 && |
1725 | !CheckSupportedType(decoder, kSupportedTypes, imm.sig->GetReturn(0), |
1726 | "return" )) |
1727 | return; |
1728 | |
1729 | auto call_descriptor = |
1730 | compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig); |
1731 | call_descriptor = |
1732 | GetLoweredCallDescriptor(compilation_zone_, call_descriptor); |
1733 | |
1734 | if (imm.index < env_->module->num_imported_functions) { |
1735 | // A direct call to an imported function. |
1736 | LiftoffRegList pinned; |
1737 | Register tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1738 | Register target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1739 | |
1740 | Register imported_targets = tmp; |
1741 | LOAD_INSTANCE_FIELD(imported_targets, ImportedFunctionTargets, |
1742 | kSystemPointerSize); |
1743 | __ Load(LiftoffRegister(target), imported_targets, no_reg, |
1744 | imm.index * sizeof(Address), kPointerLoadType, pinned); |
1745 | |
1746 | Register imported_function_refs = tmp; |
1747 | LOAD_TAGGED_PTR_INSTANCE_FIELD(imported_function_refs, |
1748 | ImportedFunctionRefs); |
1749 | Register imported_function_ref = tmp; |
1750 | __ LoadTaggedPointer( |
1751 | imported_function_ref, imported_function_refs, no_reg, |
1752 | ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned); |
1753 | |
1754 | Register* explicit_instance = &imported_function_ref; |
1755 | __ PrepareCall(imm.sig, call_descriptor, &target, explicit_instance); |
1756 | source_position_table_builder_.AddPosition( |
1757 | __ pc_offset(), SourcePosition(decoder->position()), false); |
1758 | |
1759 | __ CallIndirect(imm.sig, call_descriptor, target); |
1760 | |
1761 | safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple, |
1762 | Safepoint::kNoLazyDeopt); |
1763 | |
1764 | __ FinishCall(imm.sig, call_descriptor); |
1765 | } else { |
1766 | // A direct call within this module just gets the current instance. |
1767 | __ PrepareCall(imm.sig, call_descriptor); |
1768 | |
1769 | source_position_table_builder_.AddPosition( |
1770 | __ pc_offset(), SourcePosition(decoder->position()), false); |
1771 | |
1772 | // Just encode the function index. This will be patched at instantiation. |
1773 | Address addr = static_cast<Address>(imm.index); |
1774 | __ CallNativeWasmCode(addr); |
1775 | |
1776 | safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple, |
1777 | Safepoint::kNoLazyDeopt); |
1778 | |
1779 | __ FinishCall(imm.sig, call_descriptor); |
1780 | } |
1781 | } |
1782 | |
1783 | void CallIndirect(FullDecoder* decoder, const Value& index_val, |
1784 | const CallIndirectImmediate<validate>& imm, |
1785 | const Value args[], Value returns[]) { |
1786 | if (imm.sig->return_count() > 1) { |
1787 | return unsupported(decoder, "multi-return" ); |
1788 | } |
1789 | if (imm.table_index != 0) { |
1790 | return unsupported(decoder, "table index != 0" ); |
1791 | } |
1792 | if (imm.sig->return_count() == 1 && |
1793 | !CheckSupportedType(decoder, kSupportedTypes, imm.sig->GetReturn(0), |
1794 | "return" )) { |
1795 | return; |
1796 | } |
1797 | |
1798 | // Pop the index. |
1799 | Register index = __ PopToRegister().gp(); |
1800 | // If that register is still being used after popping, we move it to another |
1801 | // register, because we want to modify that register. |
1802 | if (__ cache_state()->is_used(LiftoffRegister(index))) { |
1803 | Register new_index = |
1804 | __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(index)).gp(); |
1805 | __ Move(new_index, index, kWasmI32); |
1806 | index = new_index; |
1807 | } |
1808 | |
1809 | LiftoffRegList pinned = LiftoffRegList::ForRegs(index); |
1810 | // Get three temporary registers. |
1811 | Register table = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1812 | Register tmp_const = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1813 | Register scratch = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); |
1814 | |
1815 | // Bounds check against the table size. |
1816 | Label* invalid_func_label = AddOutOfLineTrap( |
1817 | decoder->position(), WasmCode::kThrowWasmTrapFuncInvalid); |
1818 | |
1819 | uint32_t canonical_sig_num = env_->module->signature_ids[imm.sig_index]; |
1820 | DCHECK_GE(canonical_sig_num, 0); |
1821 | DCHECK_GE(kMaxInt, canonical_sig_num); |
1822 | |
1823 | // Compare against table size stored in |
1824 | // {instance->indirect_function_table_size}. |
1825 | LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size); |
1826 | __ emit_cond_jump(kUnsignedGreaterEqual, invalid_func_label, kWasmI32, |
1827 | index, tmp_const); |
1828 | |
1829 | // Mask the index to prevent SSCA. |
1830 | if (FLAG_untrusted_code_mitigations) { |
1831 | DEBUG_CODE_COMMENT("Mask indirect call index" ); |
1832 | // mask = ((index - size) & ~index) >> 31 |
1833 | // Reuse allocated registers; note: size is still stored in {tmp_const}. |
1834 | Register diff = table; |
1835 | Register neg_index = tmp_const; |
1836 | Register mask = scratch; |
1837 | // 1) diff = index - size |
1838 | __ emit_i32_sub(diff, index, tmp_const); |
1839 | // 2) neg_index = ~index |
1840 | __ LoadConstant(LiftoffRegister(neg_index), WasmValue(int32_t{-1})); |
1841 | __ emit_i32_xor(neg_index, neg_index, index); |
1842 | // 3) mask = diff & neg_index |
1843 | __ emit_i32_and(mask, diff, neg_index); |
1844 | // 4) mask = mask >> 31 |
1845 | __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(int32_t{31})); |
1846 | __ emit_i32_sar(mask, mask, tmp_const, pinned); |
1847 | |
1848 | // Apply mask. |
1849 | __ emit_i32_and(index, index, mask); |
1850 | } |
1851 | |
1852 | DEBUG_CODE_COMMENT("Check indirect call signature" ); |
1853 | // Load the signature from {instance->ift_sig_ids[key]} |
1854 | LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds, kSystemPointerSize); |
1855 | // Multiply {index} by 4 to represent kInt32Size items. |
1856 | STATIC_ASSERT(kInt32Size == 4); |
1857 | // TODO(wasm): use a emit_i32_shli() instead of two adds. |
1858 | // (currently cannot use shl on ia32/x64 because it clobbers %rcx). |
1859 | __ emit_i32_add(index, index, index); |
1860 | __ emit_i32_add(index, index, index); |
1861 | __ Load(LiftoffRegister(scratch), table, index, 0, LoadType::kI32Load, |
1862 | pinned); |
1863 | |
1864 | // Compare against expected signature. |
1865 | __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(canonical_sig_num)); |
1866 | |
1867 | Label* sig_mismatch_label = AddOutOfLineTrap( |
1868 | decoder->position(), WasmCode::kThrowWasmTrapFuncSigMismatch); |
1869 | __ emit_cond_jump(kUnequal, sig_mismatch_label, |
1870 | LiftoffAssembler::kWasmIntPtr, scratch, tmp_const); |
1871 | |
1872 | // At this point {index} has already been multiplied by 4. |
1873 | DEBUG_CODE_COMMENT("Execute indirect call" ); |
1874 | if (kTaggedSize != kInt32Size) { |
1875 | DCHECK_EQ(kTaggedSize, kInt32Size * 2); |
1876 | // Multiply {index} by another 2 to represent kTaggedSize items. |
1877 | __ emit_i32_add(index, index, index); |
1878 | } |
1879 | // At this point {index} has already been multiplied by kTaggedSize. |
1880 | |
1881 | // Load the instance from {instance->ift_instances[key]} |
1882 | LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs); |
1883 | __ LoadTaggedPointer(tmp_const, table, index, |
1884 | ObjectAccess::ElementOffsetInTaggedFixedArray(0), |
1885 | pinned); |
1886 | |
1887 | if (kTaggedSize != kSystemPointerSize) { |
1888 | DCHECK_EQ(kSystemPointerSize, kTaggedSize * 2); |
1889 | // Multiply {index} by another 2 to represent kSystemPointerSize items. |
1890 | __ emit_i32_add(index, index, index); |
1891 | } |
1892 | // At this point {index} has already been multiplied by kSystemPointerSize. |
1893 | |
1894 | Register* explicit_instance = &tmp_const; |
1895 | |
1896 | // Load the target from {instance->ift_targets[key]} |
1897 | LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets, |
1898 | kSystemPointerSize); |
1899 | __ Load(LiftoffRegister(scratch), table, index, 0, kPointerLoadType, |
1900 | pinned); |
1901 | |
1902 | source_position_table_builder_.AddPosition( |
1903 | __ pc_offset(), SourcePosition(decoder->position()), false); |
1904 | |
1905 | auto call_descriptor = |
1906 | compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig); |
1907 | call_descriptor = |
1908 | GetLoweredCallDescriptor(compilation_zone_, call_descriptor); |
1909 | |
1910 | Register target = scratch; |
1911 | __ PrepareCall(imm.sig, call_descriptor, &target, explicit_instance); |
1912 | __ CallIndirect(imm.sig, call_descriptor, target); |
1913 | |
1914 | safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple, |
1915 | Safepoint::kNoLazyDeopt); |
1916 | |
1917 | __ FinishCall(imm.sig, call_descriptor); |
1918 | } |
1919 | |
1920 | void ReturnCall(FullDecoder* decoder, |
1921 | const CallFunctionImmediate<validate>& imm, |
1922 | const Value args[]) { |
1923 | unsupported(decoder, "return_call" ); |
1924 | } |
1925 | void ReturnCallIndirect(FullDecoder* decoder, const Value& index_val, |
1926 | const CallIndirectImmediate<validate>& imm, |
1927 | const Value args[]) { |
1928 | unsupported(decoder, "return_call_indirect" ); |
1929 | } |
1930 | void SimdOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args, |
1931 | Value* result) { |
1932 | unsupported(decoder, "simd" ); |
1933 | } |
1934 | void SimdLaneOp(FullDecoder* decoder, WasmOpcode opcode, |
1935 | const SimdLaneImmediate<validate>& imm, |
1936 | const Vector<Value> inputs, Value* result) { |
1937 | unsupported(decoder, "simd" ); |
1938 | } |
1939 | void SimdShiftOp(FullDecoder* decoder, WasmOpcode opcode, |
1940 | const SimdShiftImmediate<validate>& imm, const Value& input, |
1941 | Value* result) { |
1942 | unsupported(decoder, "simd" ); |
1943 | } |
1944 | void Simd8x16ShuffleOp(FullDecoder* decoder, |
1945 | const Simd8x16ShuffleImmediate<validate>& imm, |
1946 | const Value& input0, const Value& input1, |
1947 | Value* result) { |
1948 | unsupported(decoder, "simd" ); |
1949 | } |
1950 | void Throw(FullDecoder* decoder, const ExceptionIndexImmediate<validate>&, |
1951 | const Vector<Value>& args) { |
1952 | unsupported(decoder, "throw" ); |
1953 | } |
1954 | void Rethrow(FullDecoder* decoder, const Value& exception) { |
1955 | unsupported(decoder, "rethrow" ); |
1956 | } |
1957 | void BrOnException(FullDecoder* decoder, const Value& exception, |
1958 | const ExceptionIndexImmediate<validate>& imm, |
1959 | uint32_t depth, Vector<Value> values) { |
1960 | unsupported(decoder, "br_on_exn" ); |
1961 | } |
1962 | void AtomicOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args, |
1963 | const MemoryAccessImmediate<validate>& imm, Value* result) { |
1964 | unsupported(decoder, "atomicop" ); |
1965 | } |
1966 | void MemoryInit(FullDecoder* decoder, |
1967 | const MemoryInitImmediate<validate>& imm, const Value& dst, |
1968 | const Value& src, const Value& size) { |
1969 | unsupported(decoder, "memory.init" ); |
1970 | } |
1971 | void DataDrop(FullDecoder* decoder, const DataDropImmediate<validate>& imm) { |
1972 | unsupported(decoder, "data.drop" ); |
1973 | } |
1974 | void MemoryCopy(FullDecoder* decoder, |
1975 | const MemoryCopyImmediate<validate>& imm, const Value& dst, |
1976 | const Value& src, const Value& size) { |
1977 | unsupported(decoder, "memory.copy" ); |
1978 | } |
1979 | void MemoryFill(FullDecoder* decoder, |
1980 | const MemoryIndexImmediate<validate>& imm, const Value& dst, |
1981 | const Value& value, const Value& size) { |
1982 | unsupported(decoder, "memory.fill" ); |
1983 | } |
1984 | void TableInit(FullDecoder* decoder, const TableInitImmediate<validate>& imm, |
1985 | Vector<Value> args) { |
1986 | unsupported(decoder, "table.init" ); |
1987 | } |
1988 | void ElemDrop(FullDecoder* decoder, const ElemDropImmediate<validate>& imm) { |
1989 | unsupported(decoder, "elem.drop" ); |
1990 | } |
1991 | void TableCopy(FullDecoder* decoder, const TableCopyImmediate<validate>& imm, |
1992 | Vector<Value> args) { |
1993 | unsupported(decoder, "table.copy" ); |
1994 | } |
1995 | |
1996 | private: |
1997 | LiftoffAssembler asm_; |
1998 | compiler::CallDescriptor* const descriptor_; |
1999 | CompilationEnv* const env_; |
2000 | bool ok_ = true; |
2001 | std::vector<OutOfLineCode> out_of_line_code_; |
2002 | SourcePositionTableBuilder source_position_table_builder_; |
2003 | std::vector<trap_handler::ProtectedInstructionData> protected_instructions_; |
2004 | // Zone used to store information during compilation. The result will be |
2005 | // stored independently, such that this zone can die together with the |
2006 | // LiftoffCompiler after compilation. |
2007 | Zone* compilation_zone_; |
2008 | SafepointTableBuilder safepoint_table_builder_; |
2009 | // The pc offset of the instructions to reserve the stack frame. Needed to |
2010 | // patch the actually needed stack size in the end. |
2011 | uint32_t pc_offset_stack_frame_construction_ = 0; |
2012 | |
2013 | void TraceCacheState(FullDecoder* decoder) const { |
2014 | #ifdef DEBUG |
2015 | if (!FLAG_trace_liftoff || !FLAG_trace_wasm_decoder) return; |
2016 | StdoutStream os; |
2017 | for (int control_depth = decoder->control_depth() - 1; control_depth >= -1; |
2018 | --control_depth) { |
2019 | auto* cache_state = |
2020 | control_depth == -1 ? __ cache_state() |
2021 | : &decoder->control_at(control_depth) |
2022 | ->label_state; |
2023 | os << PrintCollection(cache_state->stack_state); |
2024 | if (control_depth != -1) PrintF("; " ); |
2025 | } |
2026 | os << "\n" ; |
2027 | #endif |
2028 | } |
2029 | |
2030 | DISALLOW_IMPLICIT_CONSTRUCTORS(LiftoffCompiler); |
2031 | }; |
2032 | |
2033 | } // namespace |
2034 | |
2035 | WasmCompilationResult LiftoffCompilationUnit::ExecuteCompilation( |
2036 | AccountingAllocator* allocator, CompilationEnv* env, |
2037 | const FunctionBody& func_body, Counters* counters, WasmFeatures* detected) { |
2038 | TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm" ), |
2039 | "ExecuteLiftoffCompilation" ); |
2040 | base::ElapsedTimer compile_timer; |
2041 | if (FLAG_trace_wasm_decode_time) { |
2042 | compile_timer.Start(); |
2043 | } |
2044 | |
2045 | Zone zone(allocator, "LiftoffCompilationZone" ); |
2046 | const WasmModule* module = env ? env->module : nullptr; |
2047 | auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig); |
2048 | base::Optional<TimedHistogramScope> liftoff_compile_time_scope( |
2049 | base::in_place, counters->liftoff_compile_time()); |
2050 | std::unique_ptr<wasm::WasmInstructionBuffer> instruction_buffer = |
2051 | wasm::WasmInstructionBuffer::New(); |
2052 | WasmFullDecoder<Decoder::kValidate, LiftoffCompiler> decoder( |
2053 | &zone, module, env->enabled_features, detected, func_body, |
2054 | call_descriptor, env, &zone, instruction_buffer->CreateView()); |
2055 | decoder.Decode(); |
2056 | liftoff_compile_time_scope.reset(); |
2057 | LiftoffCompiler* compiler = &decoder.interface(); |
2058 | if (decoder.failed()) { |
2059 | compiler->OnFirstError(&decoder); |
2060 | return WasmCompilationResult{}; |
2061 | } |
2062 | if (!compiler->ok()) { |
2063 | // Liftoff compilation failed. |
2064 | counters->liftoff_unsupported_functions()->Increment(); |
2065 | return WasmCompilationResult{}; |
2066 | } |
2067 | |
2068 | counters->liftoff_compiled_functions()->Increment(); |
2069 | |
2070 | if (FLAG_trace_wasm_decode_time) { |
2071 | double compile_ms = compile_timer.Elapsed().InMillisecondsF(); |
2072 | PrintF( |
2073 | "wasm-compilation liftoff phase 1 ok: %u bytes, %0.3f ms decode and " |
2074 | "compile\n" , |
2075 | static_cast<unsigned>(func_body.end - func_body.start), compile_ms); |
2076 | } |
2077 | |
2078 | WasmCompilationResult result; |
2079 | compiler->GetCode(&result.code_desc); |
2080 | result.instr_buffer = instruction_buffer->ReleaseBuffer(); |
2081 | result.source_positions = compiler->GetSourcePositionTable(); |
2082 | result.protected_instructions = compiler->GetProtectedInstructions(); |
2083 | result.frame_slot_count = compiler->GetTotalFrameSlotCount(); |
2084 | result.tagged_parameter_slots = call_descriptor->GetTaggedParameterSlots(); |
2085 | result.result_tier = ExecutionTier::kLiftoff; |
2086 | |
2087 | DCHECK(result.succeeded()); |
2088 | return result; |
2089 | } |
2090 | |
2091 | #undef __ |
2092 | #undef TRACE |
2093 | #undef WASM_INSTANCE_OBJECT_FIELD_OFFSET |
2094 | #undef WASM_INSTANCE_OBJECT_FIELD_SIZE |
2095 | #undef LOAD_INSTANCE_FIELD |
2096 | #undef LOAD_TAGGED_PTR_INSTANCE_FIELD |
2097 | #undef DEBUG_CODE_COMMENT |
2098 | |
2099 | } // namespace wasm |
2100 | } // namespace internal |
2101 | } // namespace v8 |
2102 | |