1// Copyright 2017 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/wasm/baseline/liftoff-compiler.h"
6
7#include "src/assembler-inl.h"
8#include "src/base/optional.h"
9// TODO(clemensh): Remove dependences on compiler stuff.
10#include "src/compiler/linkage.h"
11#include "src/compiler/wasm-compiler.h"
12#include "src/counters.h"
13#include "src/interface-descriptors.h"
14#include "src/log.h"
15#include "src/macro-assembler-inl.h"
16#include "src/objects/smi.h"
17#include "src/ostreams.h"
18#include "src/tracing/trace-event.h"
19#include "src/utils.h"
20#include "src/wasm/baseline/liftoff-assembler.h"
21#include "src/wasm/function-body-decoder-impl.h"
22#include "src/wasm/function-compiler.h"
23#include "src/wasm/memory-tracing.h"
24#include "src/wasm/object-access.h"
25#include "src/wasm/wasm-engine.h"
26#include "src/wasm/wasm-linkage.h"
27#include "src/wasm/wasm-objects.h"
28#include "src/wasm/wasm-opcodes.h"
29
30namespace v8 {
31namespace internal {
32namespace wasm {
33
34constexpr auto kRegister = LiftoffAssembler::VarState::kRegister;
35constexpr auto kIntConst = LiftoffAssembler::VarState::kIntConst;
36constexpr auto kStack = LiftoffAssembler::VarState::kStack;
37
38namespace {
39
40#define __ asm_.
41
42#define TRACE(...) \
43 do { \
44 if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
45 } while (false)
46
47#define WASM_INSTANCE_OBJECT_FIELD_OFFSET(name) \
48 ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
49
50template <int expected_size, int actual_size>
51struct assert_field_size {
52 static_assert(expected_size == actual_size,
53 "field in WasmInstance does not have the expected size");
54 static constexpr int size = actual_size;
55};
56
57#define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \
58 FIELD_SIZE(WasmInstanceObject::k##name##Offset)
59
60#define LOAD_INSTANCE_FIELD(dst, name, load_size) \
61 __ LoadFromInstance(dst, WASM_INSTANCE_OBJECT_FIELD_OFFSET(name), \
62 assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \
63 load_size>::size);
64
65#define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name) \
66 static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize, \
67 "field in WasmInstance does not have the expected size"); \
68 __ LoadTaggedPointerFromInstance(dst, \
69 WASM_INSTANCE_OBJECT_FIELD_OFFSET(name));
70
71#ifdef DEBUG
72#define DEBUG_CODE_COMMENT(str) \
73 do { \
74 __ RecordComment(str); \
75 } while (false)
76#else
77#define DEBUG_CODE_COMMENT(str) ((void)0)
78#endif
79
80constexpr LoadType::LoadTypeValue kPointerLoadType =
81 kSystemPointerSize == 8 ? LoadType::kI64Load : LoadType::kI32Load;
82
83#if V8_TARGET_ARCH_ARM64
84// On ARM64, the Assembler keeps track of pointers to Labels to resolve
85// branches to distant targets. Moving labels would confuse the Assembler,
86// thus store the label on the heap and keep a unique_ptr.
87class MovableLabel {
88 public:
89 MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(MovableLabel);
90 MovableLabel() : label_(new Label()) {}
91
92 Label* get() { return label_.get(); }
93
94 private:
95 std::unique_ptr<Label> label_;
96};
97#else
98// On all other platforms, just store the Label directly.
99class MovableLabel {
100 public:
101 MOVE_ONLY_WITH_DEFAULT_CONSTRUCTORS(MovableLabel);
102
103 Label* get() { return &label_; }
104
105 private:
106 Label label_;
107};
108#endif
109
110compiler::CallDescriptor* GetLoweredCallDescriptor(
111 Zone* zone, compiler::CallDescriptor* call_desc) {
112 return kSystemPointerSize == 4
113 ? compiler::GetI32WasmCallDescriptor(zone, call_desc)
114 : call_desc;
115}
116
117constexpr ValueType kSupportedTypesArr[] = {kWasmI32, kWasmI64, kWasmF32,
118 kWasmF64};
119constexpr Vector<const ValueType> kSupportedTypes =
120 ArrayVector(kSupportedTypesArr);
121
122class LiftoffCompiler {
123 public:
124 // TODO(clemensh): Make this a template parameter.
125 static constexpr Decoder::ValidateFlag validate = Decoder::kValidate;
126
127 using Value = ValueBase;
128
129 struct ElseState {
130 MovableLabel label;
131 LiftoffAssembler::CacheState state;
132 };
133
134 struct Control : public ControlBase<Value> {
135 std::unique_ptr<ElseState> else_state;
136 LiftoffAssembler::CacheState label_state;
137 MovableLabel label;
138
139 MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(Control);
140
141 template <typename... Args>
142 explicit Control(Args&&... args) V8_NOEXCEPT
143 : ControlBase(std::forward<Args>(args)...) {}
144 };
145
146 using FullDecoder = WasmFullDecoder<validate, LiftoffCompiler>;
147
148 struct OutOfLineCode {
149 MovableLabel label;
150 MovableLabel continuation;
151 WasmCode::RuntimeStubId stub;
152 WasmCodePosition position;
153 LiftoffRegList regs_to_save;
154 uint32_t pc; // for trap handler.
155
156 // Named constructors:
157 static OutOfLineCode Trap(WasmCode::RuntimeStubId s, WasmCodePosition pos,
158 uint32_t pc) {
159 DCHECK_LT(0, pos);
160 return {{}, {}, s, pos, {}, pc};
161 }
162 static OutOfLineCode StackCheck(WasmCodePosition pos, LiftoffRegList regs) {
163 return {{}, {}, WasmCode::kWasmStackGuard, pos, regs, 0};
164 }
165 };
166
167 LiftoffCompiler(compiler::CallDescriptor* call_descriptor,
168 CompilationEnv* env, Zone* compilation_zone,
169 std::unique_ptr<AssemblerBuffer> buffer)
170 : asm_(std::move(buffer)),
171 descriptor_(
172 GetLoweredCallDescriptor(compilation_zone, call_descriptor)),
173 env_(env),
174 compilation_zone_(compilation_zone),
175 safepoint_table_builder_(compilation_zone_) {}
176
177 bool ok() const { return ok_; }
178
179 void GetCode(CodeDesc* desc) {
180 asm_.GetCode(nullptr, desc, &safepoint_table_builder_,
181 Assembler::kNoHandlerTable);
182 }
183
184 OwnedVector<uint8_t> GetSourcePositionTable() {
185 return source_position_table_builder_.ToSourcePositionTableVector();
186 }
187
188 OwnedVector<trap_handler::ProtectedInstructionData> GetProtectedInstructions()
189 const {
190 return OwnedVector<trap_handler::ProtectedInstructionData>::Of(
191 protected_instructions_);
192 }
193
194 uint32_t GetTotalFrameSlotCount() const {
195 return __ GetTotalFrameSlotCount();
196 }
197
198 void unsupported(FullDecoder* decoder, const char* reason) {
199 ok_ = false;
200 TRACE("unsupported: %s\n", reason);
201 decoder->errorf(decoder->pc_offset(), "unsupported liftoff operation: %s",
202 reason);
203 UnuseLabels(decoder);
204 }
205
206 bool DidAssemblerBailout(FullDecoder* decoder) {
207 if (decoder->failed() || !__ did_bailout()) return false;
208 unsupported(decoder, __ bailout_reason());
209 return true;
210 }
211
212 bool CheckSupportedType(FullDecoder* decoder,
213 Vector<const ValueType> supported_types,
214 ValueType type, const char* context) {
215 char buffer[128];
216 // Check supported types.
217 for (ValueType supported : supported_types) {
218 if (type == supported) return true;
219 }
220 SNPrintF(ArrayVector(buffer), "%s %s", ValueTypes::TypeName(type), context);
221 unsupported(decoder, buffer);
222 return false;
223 }
224
225 int GetSafepointTableOffset() const {
226 return safepoint_table_builder_.GetCodeOffset();
227 }
228
229 void UnuseLabels(FullDecoder* decoder) {
230#ifdef DEBUG
231 auto Unuse = [](Label* label) {
232 label->Unuse();
233 label->UnuseNear();
234 };
235 // Unuse all labels now, otherwise their destructor will fire a DCHECK error
236 // if they where referenced before.
237 uint32_t control_depth = decoder ? decoder->control_depth() : 0;
238 for (uint32_t i = 0; i < control_depth; ++i) {
239 Control* c = decoder->control_at(i);
240 Unuse(c->label.get());
241 if (c->else_state) Unuse(c->else_state->label.get());
242 }
243 for (auto& ool : out_of_line_code_) Unuse(ool.label.get());
244#endif
245 }
246
247 void StartFunction(FullDecoder* decoder) {
248 int num_locals = decoder->num_locals();
249 __ set_num_locals(num_locals);
250 for (int i = 0; i < num_locals; ++i) {
251 __ set_local_type(i, decoder->GetLocalType(i));
252 }
253 }
254
255 // Returns the number of inputs processed (1 or 2).
256 uint32_t ProcessParameter(ValueType type, uint32_t input_idx) {
257 const int num_lowered_params = 1 + needs_reg_pair(type);
258 ValueType lowered_type = needs_reg_pair(type) ? kWasmI32 : type;
259 RegClass rc = reg_class_for(lowered_type);
260 // Initialize to anything, will be set in the loop and used afterwards.
261 LiftoffRegister reg = kGpCacheRegList.GetFirstRegSet();
262 LiftoffRegList pinned;
263 for (int pair_idx = 0; pair_idx < num_lowered_params; ++pair_idx) {
264 compiler::LinkageLocation param_loc =
265 descriptor_->GetInputLocation(input_idx + pair_idx);
266 // Initialize to anything, will be set in both arms of the if.
267 LiftoffRegister in_reg = kGpCacheRegList.GetFirstRegSet();
268 if (param_loc.IsRegister()) {
269 DCHECK(!param_loc.IsAnyRegister());
270 int reg_code = param_loc.AsRegister();
271#if V8_TARGET_ARCH_ARM
272 // Liftoff assumes a one-to-one mapping between float registers and
273 // double registers, and so does not distinguish between f32 and f64
274 // registers. The f32 register code must therefore be halved in order to
275 // pass the f64 code to Liftoff.
276 DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0);
277 if (type == kWasmF32) {
278 reg_code /= 2;
279 }
280#endif
281 RegList cache_regs = rc == kGpReg ? kLiftoffAssemblerGpCacheRegs
282 : kLiftoffAssemblerFpCacheRegs;
283 if (cache_regs & (1ULL << reg_code)) {
284 // This is a cache register, just use it.
285 in_reg = LiftoffRegister::from_code(rc, reg_code);
286 } else {
287 // Move to a cache register (spill one if necessary).
288 // Note that we cannot create a {LiftoffRegister} for reg_code, since
289 // {LiftoffRegister} can only store cache regs.
290 in_reg = __ GetUnusedRegister(rc, pinned);
291 if (rc == kGpReg) {
292 __ Move(in_reg.gp(), Register::from_code(reg_code), lowered_type);
293 } else {
294 __ Move(in_reg.fp(), DoubleRegister::from_code(reg_code),
295 lowered_type);
296 }
297 }
298 } else if (param_loc.IsCallerFrameSlot()) {
299 in_reg = __ GetUnusedRegister(rc, pinned);
300 __ LoadCallerFrameSlot(in_reg, -param_loc.AsCallerFrameSlot(),
301 lowered_type);
302 }
303 reg = pair_idx == 0 ? in_reg
304 : LiftoffRegister::ForPair(reg.gp(), in_reg.gp());
305 pinned.set(reg);
306 }
307 __ PushRegister(type, reg);
308 return num_lowered_params;
309 }
310
311 void StackCheck(WasmCodePosition position) {
312 if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) return;
313 out_of_line_code_.push_back(
314 OutOfLineCode::StackCheck(position, __ cache_state()->used_registers));
315 OutOfLineCode& ool = out_of_line_code_.back();
316 Register limit_address = __ GetUnusedRegister(kGpReg).gp();
317 LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize);
318 __ StackCheck(ool.label.get(), limit_address);
319 __ bind(ool.continuation.get());
320 }
321
322 void StartFunctionBody(FullDecoder* decoder, Control* block) {
323 for (uint32_t i = 0; i < __ num_locals(); ++i) {
324 if (!CheckSupportedType(decoder, kSupportedTypes, __ local_type(i),
325 "param"))
326 return;
327 }
328
329 // Input 0 is the call target, the instance is at 1.
330 constexpr int kInstanceParameterIndex = 1;
331 // Store the instance parameter to a special stack slot.
332 compiler::LinkageLocation instance_loc =
333 descriptor_->GetInputLocation(kInstanceParameterIndex);
334 DCHECK(instance_loc.IsRegister());
335 DCHECK(!instance_loc.IsAnyRegister());
336 Register instance_reg = Register::from_code(instance_loc.AsRegister());
337 DCHECK_EQ(kWasmInstanceRegister, instance_reg);
338
339 // Parameter 0 is the instance parameter.
340 uint32_t num_params =
341 static_cast<uint32_t>(decoder->sig_->parameter_count());
342
343 __ EnterFrame(StackFrame::WASM_COMPILED);
344 __ set_has_frame(true);
345 pc_offset_stack_frame_construction_ = __ PrepareStackFrame();
346 // {PrepareStackFrame} is the first platform-specific assembler method.
347 // If this failed, we can bail out immediately, avoiding runtime overhead
348 // and potential failures because of other unimplemented methods.
349 // A platform implementing {PrepareStackFrame} must ensure that we can
350 // finish compilation without errors even if we hit unimplemented
351 // LiftoffAssembler methods.
352 if (DidAssemblerBailout(decoder)) return;
353
354 __ SpillInstance(instance_reg);
355 // Input 0 is the code target, 1 is the instance. First parameter at 2.
356 uint32_t input_idx = kInstanceParameterIndex + 1;
357 for (uint32_t param_idx = 0; param_idx < num_params; ++param_idx) {
358 input_idx += ProcessParameter(__ local_type(param_idx), input_idx);
359 }
360 DCHECK_EQ(input_idx, descriptor_->InputCount());
361 // Set to a gp register, to mark this uninitialized.
362 LiftoffRegister zero_double_reg = kGpCacheRegList.GetFirstRegSet();
363 DCHECK(zero_double_reg.is_gp());
364 for (uint32_t param_idx = num_params; param_idx < __ num_locals();
365 ++param_idx) {
366 ValueType type = decoder->GetLocalType(param_idx);
367 switch (type) {
368 case kWasmI32:
369 __ cache_state()->stack_state.emplace_back(kWasmI32, uint32_t{0});
370 break;
371 case kWasmI64:
372 __ cache_state()->stack_state.emplace_back(kWasmI64, uint32_t{0});
373 break;
374 case kWasmF32:
375 case kWasmF64:
376 if (zero_double_reg.is_gp()) {
377 // Note: This might spill one of the registers used to hold
378 // parameters.
379 zero_double_reg = __ GetUnusedRegister(kFpReg);
380 // Zero is represented by the bit pattern 0 for both f32 and f64.
381 __ LoadConstant(zero_double_reg, WasmValue(0.));
382 }
383 __ PushRegister(type, zero_double_reg);
384 break;
385 default:
386 UNIMPLEMENTED();
387 }
388 }
389
390 // The function-prologue stack check is associated with position 0, which
391 // is never a position of any instruction in the function.
392 StackCheck(0);
393
394 DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height());
395 }
396
397 void GenerateOutOfLineCode(OutOfLineCode& ool) {
398 __ bind(ool.label.get());
399 const bool is_stack_check = ool.stub == WasmCode::kWasmStackGuard;
400 const bool is_mem_out_of_bounds =
401 ool.stub == WasmCode::kThrowWasmTrapMemOutOfBounds;
402
403 if (is_mem_out_of_bounds && env_->use_trap_handler) {
404 uint32_t pc = static_cast<uint32_t>(__ pc_offset());
405 DCHECK_EQ(pc, __ pc_offset());
406 protected_instructions_.emplace_back(
407 trap_handler::ProtectedInstructionData{ool.pc, pc});
408 }
409
410 if (!env_->runtime_exception_support) {
411 // We cannot test calls to the runtime in cctest/test-run-wasm.
412 // Therefore we emit a call to C here instead of a call to the runtime.
413 // In this mode, we never generate stack checks.
414 DCHECK(!is_stack_check);
415 __ CallTrapCallbackForTesting();
416 __ LeaveFrame(StackFrame::WASM_COMPILED);
417 __ DropStackSlotsAndRet(
418 static_cast<uint32_t>(descriptor_->StackParameterCount()));
419 return;
420 }
421
422 if (!ool.regs_to_save.is_empty()) __ PushRegisters(ool.regs_to_save);
423
424 source_position_table_builder_.AddPosition(
425 __ pc_offset(), SourcePosition(ool.position), false);
426 __ CallRuntimeStub(ool.stub);
427 safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
428 Safepoint::kNoLazyDeopt);
429 DCHECK_EQ(ool.continuation.get()->is_bound(), is_stack_check);
430 if (!ool.regs_to_save.is_empty()) __ PopRegisters(ool.regs_to_save);
431 if (is_stack_check) {
432 __ emit_jump(ool.continuation.get());
433 } else {
434 __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
435 }
436 }
437
438 void FinishFunction(FullDecoder* decoder) {
439 if (DidAssemblerBailout(decoder)) return;
440 for (OutOfLineCode& ool : out_of_line_code_) {
441 GenerateOutOfLineCode(ool);
442 }
443 __ PatchPrepareStackFrame(pc_offset_stack_frame_construction_,
444 __ GetTotalFrameSlotCount());
445 __ FinishCode();
446 safepoint_table_builder_.Emit(&asm_, __ GetTotalFrameSlotCount());
447 __ MaybeEmitOutOfLineConstantPool();
448 // The previous calls may have also generated a bailout.
449 DidAssemblerBailout(decoder);
450 }
451
452 void OnFirstError(FullDecoder* decoder) {
453 ok_ = false;
454 UnuseLabels(decoder);
455 asm_.AbortCompilation();
456 }
457
458 void NextInstruction(FullDecoder* decoder, WasmOpcode opcode) {
459 TraceCacheState(decoder);
460 SLOW_DCHECK(__ ValidateCacheState());
461 DEBUG_CODE_COMMENT(WasmOpcodes::OpcodeName(opcode));
462 }
463
464 void Block(FullDecoder* decoder, Control* block) {}
465
466 void Loop(FullDecoder* decoder, Control* loop) {
467 // Before entering a loop, spill all locals to the stack, in order to free
468 // the cache registers, and to avoid unnecessarily reloading stack values
469 // into registers at branches.
470 // TODO(clemensh): Come up with a better strategy here, involving
471 // pre-analysis of the function.
472 __ SpillLocals();
473
474 // Loop labels bind at the beginning of the block.
475 __ bind(loop->label.get());
476
477 // Save the current cache state for the merge when jumping to this loop.
478 loop->label_state.Split(*__ cache_state());
479
480 // Execute a stack check in the loop header.
481 StackCheck(decoder->position());
482 }
483
484 void Try(FullDecoder* decoder, Control* block) {
485 unsupported(decoder, "try");
486 }
487
488 void Catch(FullDecoder* decoder, Control* block, Value* exception) {
489 unsupported(decoder, "catch");
490 }
491
492 void If(FullDecoder* decoder, const Value& cond, Control* if_block) {
493 DCHECK_EQ(if_block, decoder->control_at(0));
494 DCHECK(if_block->is_if());
495
496 if (if_block->start_merge.arity > 0 || if_block->end_merge.arity > 1)
497 return unsupported(decoder, "multi-value if");
498
499 // Allocate the else state.
500 if_block->else_state = base::make_unique<ElseState>();
501
502 // Test the condition, jump to else if zero.
503 Register value = __ PopToRegister().gp();
504 __ emit_cond_jump(kEqual, if_block->else_state->label.get(), kWasmI32,
505 value);
506
507 // Store the state (after popping the value) for executing the else branch.
508 if_block->else_state->state.Split(*__ cache_state());
509 }
510
511 void FallThruTo(FullDecoder* decoder, Control* c) {
512 if (c->end_merge.reached) {
513 __ MergeFullStackWith(c->label_state, *__ cache_state());
514 } else {
515 c->label_state.Split(*__ cache_state());
516 }
517 TraceCacheState(decoder);
518 }
519
520 void FinishOneArmedIf(FullDecoder* decoder, Control* c) {
521 DCHECK(c->is_onearmed_if());
522 if (c->end_merge.reached) {
523 // Someone already merged to the end of the if. Merge both arms into that.
524 if (c->reachable()) {
525 // Merge the if state into the end state.
526 __ MergeFullStackWith(c->label_state, *__ cache_state());
527 __ emit_jump(c->label.get());
528 }
529 // Merge the else state into the end state.
530 __ bind(c->else_state->label.get());
531 __ MergeFullStackWith(c->label_state, c->else_state->state);
532 __ cache_state()->Steal(c->label_state);
533 } else if (c->reachable()) {
534 // No merge yet at the end of the if, but we need to create a merge for
535 // the both arms of this if. Thus init the merge point from the else
536 // state, then merge the if state into that.
537 DCHECK_EQ(0, c->end_merge.arity);
538 c->label_state.InitMerge(c->else_state->state, __ num_locals(), 0,
539 c->stack_depth);
540 __ MergeFullStackWith(c->label_state, *__ cache_state());
541 __ emit_jump(c->label.get());
542 // Merge the else state into the end state.
543 __ bind(c->else_state->label.get());
544 __ MergeFullStackWith(c->label_state, c->else_state->state);
545 __ cache_state()->Steal(c->label_state);
546 } else {
547 // No merge needed, just continue with the else state.
548 __ bind(c->else_state->label.get());
549 __ cache_state()->Steal(c->else_state->state);
550 }
551 }
552
553 void PopControl(FullDecoder* decoder, Control* c) {
554 if (c->is_loop()) return; // A loop just falls through.
555 if (c->is_onearmed_if()) {
556 // Special handling for one-armed ifs.
557 FinishOneArmedIf(decoder, c);
558 } else if (c->end_merge.reached) {
559 // There is a merge already. Merge our state into that, then continue with
560 // that state.
561 if (c->reachable()) {
562 __ MergeFullStackWith(c->label_state, *__ cache_state());
563 }
564 __ cache_state()->Steal(c->label_state);
565 } else {
566 // No merge, just continue with our current state.
567 }
568
569 if (!c->label.get()->is_bound()) __ bind(c->label.get());
570 }
571
572 void EndControl(FullDecoder* decoder, Control* c) {}
573
574 enum CCallReturn : bool { kHasReturn = true, kNoReturn = false };
575
576 void GenerateCCall(const LiftoffRegister* result_regs, FunctionSig* sig,
577 ValueType out_argument_type,
578 const LiftoffRegister* arg_regs,
579 ExternalReference ext_ref) {
580 // Before making a call, spill all cache registers.
581 __ SpillAllRegisters();
582
583 // Store arguments on our stack, then align the stack for calling to C.
584 int param_bytes = 0;
585 for (ValueType param_type : sig->parameters()) {
586 param_bytes += ValueTypes::MemSize(param_type);
587 }
588 int out_arg_bytes = out_argument_type == kWasmStmt
589 ? 0
590 : ValueTypes::MemSize(out_argument_type);
591 int stack_bytes = std::max(param_bytes, out_arg_bytes);
592 __ CallC(sig, arg_regs, result_regs, out_argument_type, stack_bytes,
593 ext_ref);
594 }
595
596 template <ValueType src_type, ValueType result_type, class EmitFn>
597 void EmitUnOp(EmitFn fn) {
598 static RegClass src_rc = reg_class_for(src_type);
599 static RegClass result_rc = reg_class_for(result_type);
600 LiftoffRegister src = __ PopToRegister();
601 LiftoffRegister dst = src_rc == result_rc
602 ? __ GetUnusedRegister(result_rc, {src})
603 : __ GetUnusedRegister(result_rc);
604 fn(dst, src);
605 __ PushRegister(result_type, dst);
606 }
607
608 void EmitI32UnOpWithCFallback(bool (LiftoffAssembler::*emit_fn)(Register,
609 Register),
610 ExternalReference (*fallback_fn)()) {
611 auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) {
612 if (emit_fn && (asm_.*emit_fn)(dst.gp(), src.gp())) return;
613 ExternalReference ext_ref = fallback_fn();
614 ValueType sig_i_i_reps[] = {kWasmI32, kWasmI32};
615 FunctionSig sig_i_i(1, 1, sig_i_i_reps);
616 GenerateCCall(&dst, &sig_i_i, kWasmStmt, &src, ext_ref);
617 };
618 EmitUnOp<kWasmI32, kWasmI32>(emit_with_c_fallback);
619 }
620
621 template <ValueType type>
622 void EmitFloatUnOpWithCFallback(
623 bool (LiftoffAssembler::*emit_fn)(DoubleRegister, DoubleRegister),
624 ExternalReference (*fallback_fn)()) {
625 auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) {
626 if ((asm_.*emit_fn)(dst.fp(), src.fp())) return;
627 ExternalReference ext_ref = fallback_fn();
628 ValueType sig_reps[] = {type};
629 FunctionSig sig(0, 1, sig_reps);
630 GenerateCCall(&dst, &sig, type, &src, ext_ref);
631 };
632 EmitUnOp<type, type>(emit_with_c_fallback);
633 }
634
635 enum TypeConversionTrapping : bool { kCanTrap = true, kNoTrap = false };
636 template <ValueType dst_type, ValueType src_type,
637 TypeConversionTrapping can_trap>
638 void EmitTypeConversion(WasmOpcode opcode, ExternalReference (*fallback_fn)(),
639 WasmCodePosition trap_position) {
640 static constexpr RegClass src_rc = reg_class_for(src_type);
641 static constexpr RegClass dst_rc = reg_class_for(dst_type);
642 LiftoffRegister src = __ PopToRegister();
643 LiftoffRegister dst = src_rc == dst_rc ? __ GetUnusedRegister(dst_rc, {src})
644 : __ GetUnusedRegister(dst_rc);
645 DCHECK_EQ(!!can_trap, trap_position > 0);
646 Label* trap = can_trap ? AddOutOfLineTrap(
647 trap_position,
648 WasmCode::kThrowWasmTrapFloatUnrepresentable)
649 : nullptr;
650 if (!__ emit_type_conversion(opcode, dst, src, trap)) {
651 DCHECK_NOT_NULL(fallback_fn);
652 ExternalReference ext_ref = fallback_fn();
653 if (can_trap) {
654 // External references for potentially trapping conversions return int.
655 ValueType sig_reps[] = {kWasmI32, src_type};
656 FunctionSig sig(1, 1, sig_reps);
657 LiftoffRegister ret_reg =
658 __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst));
659 LiftoffRegister dst_regs[] = {ret_reg, dst};
660 GenerateCCall(dst_regs, &sig, dst_type, &src, ext_ref);
661 __ emit_cond_jump(kEqual, trap, kWasmI32, ret_reg.gp());
662 } else {
663 ValueType sig_reps[] = {src_type};
664 FunctionSig sig(0, 1, sig_reps);
665 GenerateCCall(&dst, &sig, dst_type, &src, ext_ref);
666 }
667 }
668 __ PushRegister(dst_type, dst);
669 }
670
671 void UnOp(FullDecoder* decoder, WasmOpcode opcode, const Value& value,
672 Value* result) {
673#define CASE_I32_UNOP(opcode, fn) \
674 case WasmOpcode::kExpr##opcode: \
675 EmitUnOp<kWasmI32, kWasmI32>( \
676 [=](LiftoffRegister dst, LiftoffRegister src) { \
677 __ emit_##fn(dst.gp(), src.gp()); \
678 }); \
679 break;
680#define CASE_I32_SIGN_EXTENSION(opcode, fn) \
681 case WasmOpcode::kExpr##opcode: \
682 EmitUnOp<kWasmI32, kWasmI32>( \
683 [=](LiftoffRegister dst, LiftoffRegister src) { \
684 __ emit_##fn(dst.gp(), src.gp()); \
685 }); \
686 break;
687#define CASE_I64_SIGN_EXTENSION(opcode, fn) \
688 case WasmOpcode::kExpr##opcode: \
689 EmitUnOp<kWasmI64, kWasmI64>( \
690 [=](LiftoffRegister dst, LiftoffRegister src) { \
691 __ emit_##fn(dst, src); \
692 }); \
693 break;
694#define CASE_FLOAT_UNOP(opcode, type, fn) \
695 case WasmOpcode::kExpr##opcode: \
696 EmitUnOp<kWasm##type, kWasm##type>( \
697 [=](LiftoffRegister dst, LiftoffRegister src) { \
698 __ emit_##fn(dst.fp(), src.fp()); \
699 }); \
700 break;
701#define CASE_FLOAT_UNOP_WITH_CFALLBACK(opcode, type, fn) \
702 case WasmOpcode::kExpr##opcode: \
703 EmitFloatUnOpWithCFallback<kWasm##type>(&LiftoffAssembler::emit_##fn, \
704 &ExternalReference::wasm_##fn); \
705 break;
706#define CASE_TYPE_CONVERSION(opcode, dst_type, src_type, ext_ref, can_trap) \
707 case WasmOpcode::kExpr##opcode: \
708 EmitTypeConversion<kWasm##dst_type, kWasm##src_type, can_trap>( \
709 kExpr##opcode, ext_ref, can_trap ? decoder->position() : 0); \
710 break;
711 switch (opcode) {
712 CASE_I32_UNOP(I32Eqz, i32_eqz)
713 CASE_I32_UNOP(I32Clz, i32_clz)
714 CASE_I32_UNOP(I32Ctz, i32_ctz)
715 CASE_FLOAT_UNOP(F32Abs, F32, f32_abs)
716 CASE_FLOAT_UNOP(F32Neg, F32, f32_neg)
717 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Ceil, F32, f32_ceil)
718 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Floor, F32, f32_floor)
719 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Trunc, F32, f32_trunc)
720 CASE_FLOAT_UNOP_WITH_CFALLBACK(F32NearestInt, F32, f32_nearest_int)
721 CASE_FLOAT_UNOP(F32Sqrt, F32, f32_sqrt)
722 CASE_FLOAT_UNOP(F64Abs, F64, f64_abs)
723 CASE_FLOAT_UNOP(F64Neg, F64, f64_neg)
724 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Ceil, F64, f64_ceil)
725 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Floor, F64, f64_floor)
726 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Trunc, F64, f64_trunc)
727 CASE_FLOAT_UNOP_WITH_CFALLBACK(F64NearestInt, F64, f64_nearest_int)
728 CASE_FLOAT_UNOP(F64Sqrt, F64, f64_sqrt)
729 CASE_TYPE_CONVERSION(I32ConvertI64, I32, I64, nullptr, kNoTrap)
730 CASE_TYPE_CONVERSION(I32SConvertF32, I32, F32, nullptr, kCanTrap)
731 CASE_TYPE_CONVERSION(I32UConvertF32, I32, F32, nullptr, kCanTrap)
732 CASE_TYPE_CONVERSION(I32SConvertF64, I32, F64, nullptr, kCanTrap)
733 CASE_TYPE_CONVERSION(I32UConvertF64, I32, F64, nullptr, kCanTrap)
734 CASE_TYPE_CONVERSION(I32ReinterpretF32, I32, F32, nullptr, kNoTrap)
735 CASE_TYPE_CONVERSION(I64SConvertI32, I64, I32, nullptr, kNoTrap)
736 CASE_TYPE_CONVERSION(I64UConvertI32, I64, I32, nullptr, kNoTrap)
737 CASE_TYPE_CONVERSION(I64SConvertF32, I64, F32,
738 &ExternalReference::wasm_float32_to_int64, kCanTrap)
739 CASE_TYPE_CONVERSION(I64UConvertF32, I64, F32,
740 &ExternalReference::wasm_float32_to_uint64, kCanTrap)
741 CASE_TYPE_CONVERSION(I64SConvertF64, I64, F64,
742 &ExternalReference::wasm_float64_to_int64, kCanTrap)
743 CASE_TYPE_CONVERSION(I64UConvertF64, I64, F64,
744 &ExternalReference::wasm_float64_to_uint64, kCanTrap)
745 CASE_TYPE_CONVERSION(I64ReinterpretF64, I64, F64, nullptr, kNoTrap)
746 CASE_TYPE_CONVERSION(F32SConvertI32, F32, I32, nullptr, kNoTrap)
747 CASE_TYPE_CONVERSION(F32UConvertI32, F32, I32, nullptr, kNoTrap)
748 CASE_TYPE_CONVERSION(F32SConvertI64, F32, I64,
749 &ExternalReference::wasm_int64_to_float32, kNoTrap)
750 CASE_TYPE_CONVERSION(F32UConvertI64, F32, I64,
751 &ExternalReference::wasm_uint64_to_float32, kNoTrap)
752 CASE_TYPE_CONVERSION(F32ConvertF64, F32, F64, nullptr, kNoTrap)
753 CASE_TYPE_CONVERSION(F32ReinterpretI32, F32, I32, nullptr, kNoTrap)
754 CASE_TYPE_CONVERSION(F64SConvertI32, F64, I32, nullptr, kNoTrap)
755 CASE_TYPE_CONVERSION(F64UConvertI32, F64, I32, nullptr, kNoTrap)
756 CASE_TYPE_CONVERSION(F64SConvertI64, F64, I64,
757 &ExternalReference::wasm_int64_to_float64, kNoTrap)
758 CASE_TYPE_CONVERSION(F64UConvertI64, F64, I64,
759 &ExternalReference::wasm_uint64_to_float64, kNoTrap)
760 CASE_TYPE_CONVERSION(F64ConvertF32, F64, F32, nullptr, kNoTrap)
761 CASE_TYPE_CONVERSION(F64ReinterpretI64, F64, I64, nullptr, kNoTrap)
762 CASE_I32_SIGN_EXTENSION(I32SExtendI8, i32_signextend_i8)
763 CASE_I32_SIGN_EXTENSION(I32SExtendI16, i32_signextend_i16)
764 CASE_I64_SIGN_EXTENSION(I64SExtendI8, i64_signextend_i8)
765 CASE_I64_SIGN_EXTENSION(I64SExtendI16, i64_signextend_i16)
766 CASE_I64_SIGN_EXTENSION(I64SExtendI32, i64_signextend_i32)
767 case kExprI32Popcnt:
768 EmitI32UnOpWithCFallback(&LiftoffAssembler::emit_i32_popcnt,
769 &ExternalReference::wasm_word32_popcnt);
770 break;
771 case WasmOpcode::kExprI64Eqz:
772 EmitUnOp<kWasmI64, kWasmI32>(
773 [=](LiftoffRegister dst, LiftoffRegister src) {
774 __ emit_i64_eqz(dst.gp(), src);
775 });
776 break;
777 default:
778 return unsupported(decoder, WasmOpcodes::OpcodeName(opcode));
779 }
780#undef CASE_I32_UNOP
781#undef CASE_I32_SIGN_EXTENSION
782#undef CASE_I64_SIGN_EXTENSION
783#undef CASE_FLOAT_UNOP
784#undef CASE_FLOAT_UNOP_WITH_CFALLBACK
785#undef CASE_TYPE_CONVERSION
786 }
787
788 template <ValueType src_type, ValueType result_type, typename EmitFn,
789 typename EmitFnImm>
790 void EmitBinOpImm(EmitFn fn, EmitFnImm fnImm) {
791 static constexpr RegClass src_rc = reg_class_for(src_type);
792 static constexpr RegClass result_rc = reg_class_for(result_type);
793
794 LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
795 // Check if the RHS is an immediate.
796 if (rhs_slot.loc() == LiftoffAssembler::VarState::kIntConst) {
797 __ cache_state()->stack_state.pop_back();
798 int32_t imm = rhs_slot.i32_const();
799
800 LiftoffRegister lhs = __ PopToRegister();
801 LiftoffRegister dst = src_rc == result_rc
802 ? __ GetUnusedRegister(result_rc, {lhs})
803 : __ GetUnusedRegister(result_rc);
804
805 fnImm(dst, lhs, imm);
806 __ PushRegister(result_type, dst);
807 } else {
808 // The RHS was not an immediate.
809 LiftoffRegister rhs = __ PopToRegister();
810 LiftoffRegister lhs = __ PopToRegister(LiftoffRegList::ForRegs(rhs));
811 LiftoffRegister dst = src_rc == result_rc
812 ? __ GetUnusedRegister(result_rc, {lhs, rhs})
813 : __ GetUnusedRegister(result_rc);
814 fn(dst, lhs, rhs);
815 __ PushRegister(result_type, dst);
816 }
817 }
818
819 template <ValueType src_type, ValueType result_type, typename EmitFn>
820 void EmitBinOp(EmitFn fn) {
821 static constexpr RegClass src_rc = reg_class_for(src_type);
822 static constexpr RegClass result_rc = reg_class_for(result_type);
823 LiftoffRegister rhs = __ PopToRegister();
824 LiftoffRegister lhs = __ PopToRegister(LiftoffRegList::ForRegs(rhs));
825 LiftoffRegister dst = src_rc == result_rc
826 ? __ GetUnusedRegister(result_rc, {lhs, rhs})
827 : __ GetUnusedRegister(result_rc);
828 fn(dst, lhs, rhs);
829 __ PushRegister(result_type, dst);
830 }
831
832 void EmitDivOrRem64CCall(LiftoffRegister dst, LiftoffRegister lhs,
833 LiftoffRegister rhs, ExternalReference ext_ref,
834 Label* trap_by_zero,
835 Label* trap_unrepresentable = nullptr) {
836 // Cannot emit native instructions, build C call.
837 LiftoffRegister ret =
838 __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst));
839 LiftoffRegister tmp =
840 __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst, ret));
841 LiftoffRegister arg_regs[] = {lhs, rhs};
842 LiftoffRegister result_regs[] = {ret, dst};
843 ValueType sig_types[] = {kWasmI32, kWasmI64, kWasmI64};
844 // <i64, i64> -> i32 (with i64 output argument)
845 FunctionSig sig(1, 2, sig_types);
846 GenerateCCall(result_regs, &sig, kWasmI64, arg_regs, ext_ref);
847 __ LoadConstant(tmp, WasmValue(int32_t{0}));
848 __ emit_cond_jump(kEqual, trap_by_zero, kWasmI32, ret.gp(), tmp.gp());
849 if (trap_unrepresentable) {
850 __ LoadConstant(tmp, WasmValue(int32_t{-1}));
851 __ emit_cond_jump(kEqual, trap_unrepresentable, kWasmI32, ret.gp(),
852 tmp.gp());
853 }
854 }
855
856 void BinOp(FullDecoder* decoder, WasmOpcode opcode, const Value& lhs,
857 const Value& rhs, Value* result) {
858#define CASE_I32_BINOP(opcode, fn) \
859 case WasmOpcode::kExpr##opcode: \
860 return EmitBinOp<kWasmI32, kWasmI32>( \
861 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
862 __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp()); \
863 });
864#define CASE_I32_BINOPI(opcode, fn) \
865 case WasmOpcode::kExpr##opcode: \
866 return EmitBinOpImm<kWasmI32, kWasmI32>( \
867 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
868 __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp()); \
869 }, \
870 [=](LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) { \
871 __ emit_##fn(dst.gp(), lhs.gp(), imm); \
872 });
873#define CASE_I64_BINOP(opcode, fn) \
874 case WasmOpcode::kExpr##opcode: \
875 return EmitBinOp<kWasmI64, kWasmI64>( \
876 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
877 __ emit_##fn(dst, lhs, rhs); \
878 });
879#define CASE_I64_BINOPI(opcode, fn) \
880 case WasmOpcode::kExpr##opcode: \
881 return EmitBinOpImm<kWasmI64, kWasmI64>( \
882 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
883 __ emit_##fn(dst, lhs, rhs); \
884 }, \
885 [=](LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) { \
886 __ emit_##fn(dst, lhs, imm); \
887 });
888#define CASE_FLOAT_BINOP(opcode, type, fn) \
889 case WasmOpcode::kExpr##opcode: \
890 return EmitBinOp<kWasm##type, kWasm##type>( \
891 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
892 __ emit_##fn(dst.fp(), lhs.fp(), rhs.fp()); \
893 });
894#define CASE_I32_CMPOP(opcode, cond) \
895 case WasmOpcode::kExpr##opcode: \
896 return EmitBinOp<kWasmI32, kWasmI32>( \
897 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
898 __ emit_i32_set_cond(cond, dst.gp(), lhs.gp(), rhs.gp()); \
899 });
900#define CASE_I64_CMPOP(opcode, cond) \
901 case WasmOpcode::kExpr##opcode: \
902 return EmitBinOp<kWasmI64, kWasmI32>( \
903 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
904 __ emit_i64_set_cond(cond, dst.gp(), lhs, rhs); \
905 });
906#define CASE_F32_CMPOP(opcode, cond) \
907 case WasmOpcode::kExpr##opcode: \
908 return EmitBinOp<kWasmF32, kWasmI32>( \
909 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
910 __ emit_f32_set_cond(cond, dst.gp(), lhs.fp(), rhs.fp()); \
911 });
912#define CASE_F64_CMPOP(opcode, cond) \
913 case WasmOpcode::kExpr##opcode: \
914 return EmitBinOp<kWasmF64, kWasmI32>( \
915 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
916 __ emit_f64_set_cond(cond, dst.gp(), lhs.fp(), rhs.fp()); \
917 });
918#define CASE_I32_SHIFTOP(opcode, fn) \
919 case WasmOpcode::kExpr##opcode: \
920 return EmitBinOp<kWasmI32, kWasmI32>( \
921 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
922 __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp(), {}); \
923 });
924#define CASE_I64_SHIFTOP(opcode, fn) \
925 case WasmOpcode::kExpr##opcode: \
926 return EmitBinOp<kWasmI64, kWasmI64>([=](LiftoffRegister dst, \
927 LiftoffRegister src, \
928 LiftoffRegister amount) { \
929 __ emit_##fn(dst, src, amount.is_pair() ? amount.low_gp() : amount.gp(), \
930 {}); \
931 });
932#define CASE_CCALL_BINOP(opcode, type, ext_ref_fn) \
933 case WasmOpcode::kExpr##opcode: \
934 return EmitBinOp<kWasmI32, kWasmI32>( \
935 [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
936 LiftoffRegister args[] = {lhs, rhs}; \
937 auto ext_ref = ExternalReference::ext_ref_fn(); \
938 ValueType sig_i_ii_reps[] = {kWasmI32, kWasmI32, kWasmI32}; \
939 FunctionSig sig_i_ii(1, 2, sig_i_ii_reps); \
940 GenerateCCall(&dst, &sig_i_ii, kWasmStmt, args, ext_ref); \
941 });
942 switch (opcode) {
943 CASE_I32_BINOPI(I32Add, i32_add)
944 CASE_I32_BINOP(I32Sub, i32_sub)
945 CASE_I32_BINOP(I32Mul, i32_mul)
946 CASE_I32_BINOP(I32And, i32_and)
947 CASE_I32_BINOP(I32Ior, i32_or)
948 CASE_I32_BINOP(I32Xor, i32_xor)
949 CASE_I64_BINOP(I64And, i64_and)
950 CASE_I64_BINOP(I64Ior, i64_or)
951 CASE_I64_BINOP(I64Xor, i64_xor)
952 CASE_I32_CMPOP(I32Eq, kEqual)
953 CASE_I32_CMPOP(I32Ne, kUnequal)
954 CASE_I32_CMPOP(I32LtS, kSignedLessThan)
955 CASE_I32_CMPOP(I32LtU, kUnsignedLessThan)
956 CASE_I32_CMPOP(I32GtS, kSignedGreaterThan)
957 CASE_I32_CMPOP(I32GtU, kUnsignedGreaterThan)
958 CASE_I32_CMPOP(I32LeS, kSignedLessEqual)
959 CASE_I32_CMPOP(I32LeU, kUnsignedLessEqual)
960 CASE_I32_CMPOP(I32GeS, kSignedGreaterEqual)
961 CASE_I32_CMPOP(I32GeU, kUnsignedGreaterEqual)
962 CASE_I64_BINOPI(I64Add, i64_add)
963 CASE_I64_BINOP(I64Sub, i64_sub)
964 CASE_I64_BINOP(I64Mul, i64_mul)
965 CASE_I64_CMPOP(I64Eq, kEqual)
966 CASE_I64_CMPOP(I64Ne, kUnequal)
967 CASE_I64_CMPOP(I64LtS, kSignedLessThan)
968 CASE_I64_CMPOP(I64LtU, kUnsignedLessThan)
969 CASE_I64_CMPOP(I64GtS, kSignedGreaterThan)
970 CASE_I64_CMPOP(I64GtU, kUnsignedGreaterThan)
971 CASE_I64_CMPOP(I64LeS, kSignedLessEqual)
972 CASE_I64_CMPOP(I64LeU, kUnsignedLessEqual)
973 CASE_I64_CMPOP(I64GeS, kSignedGreaterEqual)
974 CASE_I64_CMPOP(I64GeU, kUnsignedGreaterEqual)
975 CASE_F32_CMPOP(F32Eq, kEqual)
976 CASE_F32_CMPOP(F32Ne, kUnequal)
977 CASE_F32_CMPOP(F32Lt, kUnsignedLessThan)
978 CASE_F32_CMPOP(F32Gt, kUnsignedGreaterThan)
979 CASE_F32_CMPOP(F32Le, kUnsignedLessEqual)
980 CASE_F32_CMPOP(F32Ge, kUnsignedGreaterEqual)
981 CASE_F64_CMPOP(F64Eq, kEqual)
982 CASE_F64_CMPOP(F64Ne, kUnequal)
983 CASE_F64_CMPOP(F64Lt, kUnsignedLessThan)
984 CASE_F64_CMPOP(F64Gt, kUnsignedGreaterThan)
985 CASE_F64_CMPOP(F64Le, kUnsignedLessEqual)
986 CASE_F64_CMPOP(F64Ge, kUnsignedGreaterEqual)
987 CASE_I32_SHIFTOP(I32Shl, i32_shl)
988 CASE_I32_SHIFTOP(I32ShrS, i32_sar)
989 CASE_I32_SHIFTOP(I32ShrU, i32_shr)
990 CASE_I64_SHIFTOP(I64Shl, i64_shl)
991 CASE_I64_SHIFTOP(I64ShrS, i64_sar)
992 CASE_I64_SHIFTOP(I64ShrU, i64_shr)
993 CASE_CCALL_BINOP(I32Rol, I32, wasm_word32_rol)
994 CASE_CCALL_BINOP(I32Ror, I32, wasm_word32_ror)
995 CASE_FLOAT_BINOP(F32Add, F32, f32_add)
996 CASE_FLOAT_BINOP(F32Sub, F32, f32_sub)
997 CASE_FLOAT_BINOP(F32Mul, F32, f32_mul)
998 CASE_FLOAT_BINOP(F32Div, F32, f32_div)
999 CASE_FLOAT_BINOP(F32Min, F32, f32_min)
1000 CASE_FLOAT_BINOP(F32Max, F32, f32_max)
1001 CASE_FLOAT_BINOP(F32CopySign, F32, f32_copysign)
1002 CASE_FLOAT_BINOP(F64Add, F64, f64_add)
1003 CASE_FLOAT_BINOP(F64Sub, F64, f64_sub)
1004 CASE_FLOAT_BINOP(F64Mul, F64, f64_mul)
1005 CASE_FLOAT_BINOP(F64Div, F64, f64_div)
1006 CASE_FLOAT_BINOP(F64Min, F64, f64_min)
1007 CASE_FLOAT_BINOP(F64Max, F64, f64_max)
1008 CASE_FLOAT_BINOP(F64CopySign, F64, f64_copysign)
1009 case WasmOpcode::kExprI32DivS:
1010 EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
1011 LiftoffRegister lhs,
1012 LiftoffRegister rhs) {
1013 WasmCodePosition position = decoder->position();
1014 AddOutOfLineTrap(position, WasmCode::kThrowWasmTrapDivByZero);
1015 // Adding the second trap might invalidate the pointer returned for
1016 // the first one, thus get both pointers afterwards.
1017 AddOutOfLineTrap(position,
1018 WasmCode::kThrowWasmTrapDivUnrepresentable);
1019 Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
1020 Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
1021 __ emit_i32_divs(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero,
1022 div_unrepresentable);
1023 });
1024 break;
1025 case WasmOpcode::kExprI32DivU:
1026 EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
1027 LiftoffRegister lhs,
1028 LiftoffRegister rhs) {
1029 Label* div_by_zero = AddOutOfLineTrap(
1030 decoder->position(), WasmCode::kThrowWasmTrapDivByZero);
1031 __ emit_i32_divu(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero);
1032 });
1033 break;
1034 case WasmOpcode::kExprI32RemS:
1035 EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
1036 LiftoffRegister lhs,
1037 LiftoffRegister rhs) {
1038 Label* rem_by_zero = AddOutOfLineTrap(
1039 decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
1040 __ emit_i32_rems(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
1041 });
1042 break;
1043 case WasmOpcode::kExprI32RemU:
1044 EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
1045 LiftoffRegister lhs,
1046 LiftoffRegister rhs) {
1047 Label* rem_by_zero = AddOutOfLineTrap(
1048 decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
1049 __ emit_i32_remu(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
1050 });
1051 break;
1052 case WasmOpcode::kExprI64DivS:
1053 EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
1054 LiftoffRegister lhs,
1055 LiftoffRegister rhs) {
1056 WasmCodePosition position = decoder->position();
1057 AddOutOfLineTrap(position, WasmCode::kThrowWasmTrapDivByZero);
1058 // Adding the second trap might invalidate the pointer returned for
1059 // the first one, thus get both pointers afterwards.
1060 AddOutOfLineTrap(position,
1061 WasmCode::kThrowWasmTrapDivUnrepresentable);
1062 Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
1063 Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
1064 if (!__ emit_i64_divs(dst, lhs, rhs, div_by_zero,
1065 div_unrepresentable)) {
1066 ExternalReference ext_ref = ExternalReference::wasm_int64_div();
1067 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero,
1068 div_unrepresentable);
1069 }
1070 });
1071 break;
1072 case WasmOpcode::kExprI64DivU:
1073 EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
1074 LiftoffRegister lhs,
1075 LiftoffRegister rhs) {
1076 Label* div_by_zero = AddOutOfLineTrap(
1077 decoder->position(), WasmCode::kThrowWasmTrapDivByZero);
1078 if (!__ emit_i64_divu(dst, lhs, rhs, div_by_zero)) {
1079 ExternalReference ext_ref = ExternalReference::wasm_uint64_div();
1080 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero);
1081 }
1082 });
1083 break;
1084 case WasmOpcode::kExprI64RemS:
1085 EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
1086 LiftoffRegister lhs,
1087 LiftoffRegister rhs) {
1088 Label* rem_by_zero = AddOutOfLineTrap(
1089 decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
1090 if (!__ emit_i64_rems(dst, lhs, rhs, rem_by_zero)) {
1091 ExternalReference ext_ref = ExternalReference::wasm_int64_mod();
1092 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
1093 }
1094 });
1095 break;
1096 case WasmOpcode::kExprI64RemU:
1097 EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
1098 LiftoffRegister lhs,
1099 LiftoffRegister rhs) {
1100 Label* rem_by_zero = AddOutOfLineTrap(
1101 decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
1102 if (!__ emit_i64_remu(dst, lhs, rhs, rem_by_zero)) {
1103 ExternalReference ext_ref = ExternalReference::wasm_uint64_mod();
1104 EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
1105 }
1106 });
1107 break;
1108 default:
1109 return unsupported(decoder, WasmOpcodes::OpcodeName(opcode));
1110 }
1111#undef CASE_I32_BINOP
1112#undef CASE_I32_BINOPI
1113#undef CASE_I64_BINOP
1114#undef CASE_I64_BINOPI
1115#undef CASE_FLOAT_BINOP
1116#undef CASE_I32_CMPOP
1117#undef CASE_I64_CMPOP
1118#undef CASE_F32_CMPOP
1119#undef CASE_F64_CMPOP
1120#undef CASE_I32_SHIFTOP
1121#undef CASE_I64_SHIFTOP
1122#undef CASE_CCALL_BINOP
1123 }
1124
1125 void I32Const(FullDecoder* decoder, Value* result, int32_t value) {
1126 __ cache_state()->stack_state.emplace_back(kWasmI32, value);
1127 }
1128
1129 void I64Const(FullDecoder* decoder, Value* result, int64_t value) {
1130 // The {VarState} stores constant values as int32_t, thus we only store
1131 // 64-bit constants in this field if it fits in an int32_t. Larger values
1132 // cannot be used as immediate value anyway, so we can also just put them in
1133 // a register immediately.
1134 int32_t value_i32 = static_cast<int32_t>(value);
1135 if (value_i32 == value) {
1136 __ cache_state()->stack_state.emplace_back(kWasmI64, value_i32);
1137 } else {
1138 LiftoffRegister reg = __ GetUnusedRegister(reg_class_for(kWasmI64));
1139 __ LoadConstant(reg, WasmValue(value));
1140 __ PushRegister(kWasmI64, reg);
1141 }
1142 }
1143
1144 void F32Const(FullDecoder* decoder, Value* result, float value) {
1145 LiftoffRegister reg = __ GetUnusedRegister(kFpReg);
1146 __ LoadConstant(reg, WasmValue(value));
1147 __ PushRegister(kWasmF32, reg);
1148 }
1149
1150 void F64Const(FullDecoder* decoder, Value* result, double value) {
1151 LiftoffRegister reg = __ GetUnusedRegister(kFpReg);
1152 __ LoadConstant(reg, WasmValue(value));
1153 __ PushRegister(kWasmF64, reg);
1154 }
1155
1156 void RefNull(FullDecoder* decoder, Value* result) {
1157 unsupported(decoder, "ref_null");
1158 }
1159
1160 void Drop(FullDecoder* decoder, const Value& value) {
1161 auto& slot = __ cache_state()->stack_state.back();
1162 // If the dropped slot contains a register, decrement it's use count.
1163 if (slot.is_reg()) __ cache_state()->dec_used(slot.reg());
1164 __ cache_state()->stack_state.pop_back();
1165 }
1166
1167 void ReturnImpl(FullDecoder* decoder) {
1168 size_t num_returns = decoder->sig_->return_count();
1169 if (num_returns > 1) return unsupported(decoder, "multi-return");
1170 if (num_returns > 0) __ MoveToReturnRegisters(decoder->sig_);
1171 __ LeaveFrame(StackFrame::WASM_COMPILED);
1172 __ DropStackSlotsAndRet(
1173 static_cast<uint32_t>(descriptor_->StackParameterCount()));
1174 }
1175
1176 void DoReturn(FullDecoder* decoder, Vector<Value> /*values*/) {
1177 ReturnImpl(decoder);
1178 }
1179
1180 void GetLocal(FullDecoder* decoder, Value* result,
1181 const LocalIndexImmediate<validate>& imm) {
1182 auto& slot = __ cache_state()->stack_state[imm.index];
1183 DCHECK_EQ(slot.type(), imm.type);
1184 switch (slot.loc()) {
1185 case kRegister:
1186 __ PushRegister(slot.type(), slot.reg());
1187 break;
1188 case kIntConst:
1189 __ cache_state()->stack_state.emplace_back(imm.type, slot.i32_const());
1190 break;
1191 case kStack: {
1192 auto rc = reg_class_for(imm.type);
1193 LiftoffRegister reg = __ GetUnusedRegister(rc);
1194 __ Fill(reg, imm.index, imm.type);
1195 __ PushRegister(slot.type(), reg);
1196 break;
1197 }
1198 }
1199 }
1200
1201 void SetLocalFromStackSlot(LiftoffAssembler::VarState& dst_slot,
1202 uint32_t local_index) {
1203 auto& state = *__ cache_state();
1204 ValueType type = dst_slot.type();
1205 if (dst_slot.is_reg()) {
1206 LiftoffRegister slot_reg = dst_slot.reg();
1207 if (state.get_use_count(slot_reg) == 1) {
1208 __ Fill(dst_slot.reg(), state.stack_height() - 1, type);
1209 return;
1210 }
1211 state.dec_used(slot_reg);
1212 dst_slot.MakeStack();
1213 }
1214 DCHECK_EQ(type, __ local_type(local_index));
1215 RegClass rc = reg_class_for(type);
1216 LiftoffRegister dst_reg = __ GetUnusedRegister(rc);
1217 __ Fill(dst_reg, __ cache_state()->stack_height() - 1, type);
1218 dst_slot = LiftoffAssembler::VarState(type, dst_reg);
1219 __ cache_state()->inc_used(dst_reg);
1220 }
1221
1222 void SetLocal(uint32_t local_index, bool is_tee) {
1223 auto& state = *__ cache_state();
1224 auto& source_slot = state.stack_state.back();
1225 auto& target_slot = state.stack_state[local_index];
1226 switch (source_slot.loc()) {
1227 case kRegister:
1228 if (target_slot.is_reg()) state.dec_used(target_slot.reg());
1229 target_slot = source_slot;
1230 if (is_tee) state.inc_used(target_slot.reg());
1231 break;
1232 case kIntConst:
1233 if (target_slot.is_reg()) state.dec_used(target_slot.reg());
1234 target_slot = source_slot;
1235 break;
1236 case kStack:
1237 SetLocalFromStackSlot(target_slot, local_index);
1238 break;
1239 }
1240 if (!is_tee) __ cache_state()->stack_state.pop_back();
1241 }
1242
1243 void SetLocal(FullDecoder* decoder, const Value& value,
1244 const LocalIndexImmediate<validate>& imm) {
1245 SetLocal(imm.index, false);
1246 }
1247
1248 void TeeLocal(FullDecoder* decoder, const Value& value, Value* result,
1249 const LocalIndexImmediate<validate>& imm) {
1250 SetLocal(imm.index, true);
1251 }
1252
1253 Register GetGlobalBaseAndOffset(const WasmGlobal* global,
1254 LiftoffRegList& pinned, uint32_t* offset) {
1255 Register addr = pinned.set(__ GetUnusedRegister(kGpReg)).gp();
1256 if (global->mutability && global->imported) {
1257 LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize);
1258 __ Load(LiftoffRegister(addr), addr, no_reg,
1259 global->index * sizeof(Address), kPointerLoadType, pinned);
1260 *offset = 0;
1261 } else {
1262 LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize);
1263 *offset = global->offset;
1264 }
1265 return addr;
1266 }
1267
1268 void GetGlobal(FullDecoder* decoder, Value* result,
1269 const GlobalIndexImmediate<validate>& imm) {
1270 const auto* global = &env_->module->globals[imm.index];
1271 if (!CheckSupportedType(decoder, kSupportedTypes, global->type, "global"))
1272 return;
1273 LiftoffRegList pinned;
1274 uint32_t offset = 0;
1275 Register addr = GetGlobalBaseAndOffset(global, pinned, &offset);
1276 LiftoffRegister value =
1277 pinned.set(__ GetUnusedRegister(reg_class_for(global->type), pinned));
1278 LoadType type = LoadType::ForValueType(global->type);
1279 __ Load(value, addr, no_reg, offset, type, pinned, nullptr, true);
1280 __ PushRegister(global->type, value);
1281 }
1282
1283 void SetGlobal(FullDecoder* decoder, const Value& value,
1284 const GlobalIndexImmediate<validate>& imm) {
1285 auto* global = &env_->module->globals[imm.index];
1286 if (!CheckSupportedType(decoder, kSupportedTypes, global->type, "global"))
1287 return;
1288 LiftoffRegList pinned;
1289 uint32_t offset = 0;
1290 Register addr = GetGlobalBaseAndOffset(global, pinned, &offset);
1291 LiftoffRegister reg = pinned.set(__ PopToRegister(pinned));
1292 StoreType type = StoreType::ForValueType(global->type);
1293 __ Store(addr, no_reg, offset, reg, type, {}, nullptr, true);
1294 }
1295
1296 void GetTable(FullDecoder* decoder, const Value& index, Value* result,
1297 TableIndexImmediate<validate>& imm) {
1298 unsupported(decoder, "table_get");
1299 }
1300
1301 void SetTable(FullDecoder* decoder, const Value& index, const Value& value,
1302 TableIndexImmediate<validate>& imm) {
1303 unsupported(decoder, "table_set");
1304 }
1305
1306 void Unreachable(FullDecoder* decoder) {
1307 Label* unreachable_label = AddOutOfLineTrap(
1308 decoder->position(), WasmCode::kThrowWasmTrapUnreachable);
1309 __ emit_jump(unreachable_label);
1310 __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
1311 }
1312
1313 void Select(FullDecoder* decoder, const Value& cond, const Value& fval,
1314 const Value& tval, Value* result) {
1315 LiftoffRegList pinned;
1316 Register condition = pinned.set(__ PopToRegister()).gp();
1317 ValueType type = __ cache_state()->stack_state.end()[-1].type();
1318 DCHECK_EQ(type, __ cache_state()->stack_state.end()[-2].type());
1319 LiftoffRegister false_value = pinned.set(__ PopToRegister(pinned));
1320 LiftoffRegister true_value = __ PopToRegister(pinned);
1321 LiftoffRegister dst =
1322 __ GetUnusedRegister(true_value.reg_class(), {true_value, false_value});
1323 __ PushRegister(type, dst);
1324
1325 // Now emit the actual code to move either {true_value} or {false_value}
1326 // into {dst}.
1327 Label cont;
1328 Label case_false;
1329 __ emit_cond_jump(kEqual, &case_false, kWasmI32, condition);
1330 if (dst != true_value) __ Move(dst, true_value, type);
1331 __ emit_jump(&cont);
1332
1333 __ bind(&case_false);
1334 if (dst != false_value) __ Move(dst, false_value, type);
1335 __ bind(&cont);
1336 }
1337
1338 void BrImpl(Control* target) {
1339 if (!target->br_merge()->reached) {
1340 target->label_state.InitMerge(*__ cache_state(), __ num_locals(),
1341 target->br_merge()->arity,
1342 target->stack_depth);
1343 }
1344 __ MergeStackWith(target->label_state, target->br_merge()->arity);
1345 __ jmp(target->label.get());
1346 }
1347
1348 void Br(FullDecoder* decoder, Control* target) { BrImpl(target); }
1349
1350 void BrOrRet(FullDecoder* decoder, uint32_t depth) {
1351 if (depth == decoder->control_depth() - 1) {
1352 ReturnImpl(decoder);
1353 } else {
1354 BrImpl(decoder->control_at(depth));
1355 }
1356 }
1357
1358 void BrIf(FullDecoder* decoder, const Value& cond, uint32_t depth) {
1359 Label cont_false;
1360 Register value = __ PopToRegister().gp();
1361 __ emit_cond_jump(kEqual, &cont_false, kWasmI32, value);
1362
1363 BrOrRet(decoder, depth);
1364 __ bind(&cont_false);
1365 }
1366
1367 // Generate a branch table case, potentially reusing previously generated
1368 // stack transfer code.
1369 void GenerateBrCase(FullDecoder* decoder, uint32_t br_depth,
1370 std::map<uint32_t, MovableLabel>& br_targets) {
1371 MovableLabel& label = br_targets[br_depth];
1372 if (label.get()->is_bound()) {
1373 __ jmp(label.get());
1374 } else {
1375 __ bind(label.get());
1376 BrOrRet(decoder, br_depth);
1377 }
1378 }
1379
1380 // Generate a branch table for input in [min, max).
1381 // TODO(wasm): Generate a real branch table (like TF TableSwitch).
1382 void GenerateBrTable(FullDecoder* decoder, LiftoffRegister tmp,
1383 LiftoffRegister value, uint32_t min, uint32_t max,
1384 BranchTableIterator<validate>& table_iterator,
1385 std::map<uint32_t, MovableLabel>& br_targets) {
1386 DCHECK_LT(min, max);
1387 // Check base case.
1388 if (max == min + 1) {
1389 DCHECK_EQ(min, table_iterator.cur_index());
1390 GenerateBrCase(decoder, table_iterator.next(), br_targets);
1391 return;
1392 }
1393
1394 uint32_t split = min + (max - min) / 2;
1395 Label upper_half;
1396 __ LoadConstant(tmp, WasmValue(split));
1397 __ emit_cond_jump(kUnsignedGreaterEqual, &upper_half, kWasmI32, value.gp(),
1398 tmp.gp());
1399 // Emit br table for lower half:
1400 GenerateBrTable(decoder, tmp, value, min, split, table_iterator,
1401 br_targets);
1402 __ bind(&upper_half);
1403 // Emit br table for upper half:
1404 GenerateBrTable(decoder, tmp, value, split, max, table_iterator,
1405 br_targets);
1406 }
1407
1408 void BrTable(FullDecoder* decoder, const BranchTableImmediate<validate>& imm,
1409 const Value& key) {
1410 LiftoffRegList pinned;
1411 LiftoffRegister value = pinned.set(__ PopToRegister());
1412 BranchTableIterator<validate> table_iterator(decoder, imm);
1413 std::map<uint32_t, MovableLabel> br_targets;
1414
1415 if (imm.table_count > 0) {
1416 LiftoffRegister tmp = __ GetUnusedRegister(kGpReg, pinned);
1417 __ LoadConstant(tmp, WasmValue(uint32_t{imm.table_count}));
1418 Label case_default;
1419 __ emit_cond_jump(kUnsignedGreaterEqual, &case_default, kWasmI32,
1420 value.gp(), tmp.gp());
1421
1422 GenerateBrTable(decoder, tmp, value, 0, imm.table_count, table_iterator,
1423 br_targets);
1424
1425 __ bind(&case_default);
1426 }
1427
1428 // Generate the default case.
1429 GenerateBrCase(decoder, table_iterator.next(), br_targets);
1430 DCHECK(!table_iterator.has_next());
1431 }
1432
1433 void Else(FullDecoder* decoder, Control* c) {
1434 if (c->reachable()) {
1435 if (!c->end_merge.reached) {
1436 c->label_state.InitMerge(*__ cache_state(), __ num_locals(),
1437 c->end_merge.arity, c->stack_depth);
1438 }
1439 __ MergeFullStackWith(c->label_state, *__ cache_state());
1440 __ emit_jump(c->label.get());
1441 }
1442 __ bind(c->else_state->label.get());
1443 __ cache_state()->Steal(c->else_state->state);
1444 }
1445
1446 Label* AddOutOfLineTrap(WasmCodePosition position,
1447 WasmCode::RuntimeStubId stub, uint32_t pc = 0) {
1448 DCHECK(!FLAG_wasm_no_bounds_checks);
1449 // The pc is needed for memory OOB trap with trap handler enabled. Other
1450 // callers should not even compute it.
1451 DCHECK_EQ(pc != 0, stub == WasmCode::kThrowWasmTrapMemOutOfBounds &&
1452 env_->use_trap_handler);
1453
1454 out_of_line_code_.push_back(OutOfLineCode::Trap(stub, position, pc));
1455 return out_of_line_code_.back().label.get();
1456 }
1457
1458 // Returns true if the memory access is statically known to be out of bounds
1459 // (a jump to the trap was generated then); return false otherwise.
1460 bool BoundsCheckMem(FullDecoder* decoder, uint32_t access_size,
1461 uint32_t offset, Register index, LiftoffRegList pinned) {
1462 const bool statically_oob =
1463 !IsInBounds(offset, access_size, env_->max_memory_size);
1464
1465 if (!statically_oob &&
1466 (FLAG_wasm_no_bounds_checks || env_->use_trap_handler)) {
1467 return false;
1468 }
1469
1470 // TODO(wasm): This adds protected instruction information for the jump
1471 // instruction we are about to generate. It would be better to just not add
1472 // protected instruction info when the pc is 0.
1473 Label* trap_label = AddOutOfLineTrap(
1474 decoder->position(), WasmCode::kThrowWasmTrapMemOutOfBounds,
1475 env_->use_trap_handler ? __ pc_offset() : 0);
1476
1477 if (statically_oob) {
1478 __ emit_jump(trap_label);
1479 Control* current_block = decoder->control_at(0);
1480 if (current_block->reachable()) {
1481 current_block->reachability = kSpecOnlyReachable;
1482 }
1483 return true;
1484 }
1485
1486 DCHECK(!env_->use_trap_handler);
1487 DCHECK(!FLAG_wasm_no_bounds_checks);
1488
1489 uint64_t end_offset = uint64_t{offset} + access_size - 1u;
1490
1491 // If the end offset is larger than the smallest memory, dynamically check
1492 // the end offset against the actual memory size, which is not known at
1493 // compile time. Otherwise, only one check is required (see below).
1494 LiftoffRegister end_offset_reg =
1495 pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1496 Register mem_size = __ GetUnusedRegister(kGpReg, pinned).gp();
1497 LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize);
1498
1499 if (kSystemPointerSize == 8) {
1500 __ LoadConstant(end_offset_reg, WasmValue(end_offset));
1501 } else {
1502 __ LoadConstant(end_offset_reg,
1503 WasmValue(static_cast<uint32_t>(end_offset)));
1504 }
1505
1506 if (end_offset >= env_->min_memory_size) {
1507 __ emit_cond_jump(kUnsignedGreaterEqual, trap_label,
1508 LiftoffAssembler::kWasmIntPtr, end_offset_reg.gp(),
1509 mem_size);
1510 }
1511
1512 // Just reuse the end_offset register for computing the effective size.
1513 LiftoffRegister effective_size_reg = end_offset_reg;
1514 __ emit_ptrsize_sub(effective_size_reg.gp(), mem_size, end_offset_reg.gp());
1515
1516 __ emit_i32_to_intptr(index, index);
1517
1518 __ emit_cond_jump(kUnsignedGreaterEqual, trap_label,
1519 LiftoffAssembler::kWasmIntPtr, index,
1520 effective_size_reg.gp());
1521 return false;
1522 }
1523
1524 void TraceMemoryOperation(bool is_store, MachineRepresentation rep,
1525 Register index, uint32_t offset,
1526 WasmCodePosition position) {
1527 // Before making the runtime call, spill all cache registers.
1528 __ SpillAllRegisters();
1529
1530 LiftoffRegList pinned = LiftoffRegList::ForRegs(index);
1531 // Get one register for computing the address (offset + index).
1532 LiftoffRegister address = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1533 // Compute offset+index in address.
1534 __ LoadConstant(address, WasmValue(offset));
1535 __ emit_i32_add(address.gp(), address.gp(), index);
1536
1537 // Get a register to hold the stack slot for MemoryTracingInfo.
1538 LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
1539 // Allocate stack slot for MemoryTracingInfo.
1540 __ AllocateStackSlot(info.gp(), sizeof(MemoryTracingInfo));
1541
1542 // Now store all information into the MemoryTracingInfo struct.
1543 __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, address), address,
1544 StoreType::kI32Store, pinned);
1545 __ LoadConstant(address, WasmValue(is_store ? 1 : 0));
1546 __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), address,
1547 StoreType::kI32Store8, pinned);
1548 __ LoadConstant(address, WasmValue(static_cast<int>(rep)));
1549 __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), address,
1550 StoreType::kI32Store8, pinned);
1551
1552 source_position_table_builder_.AddPosition(__ pc_offset(),
1553 SourcePosition(position), false);
1554
1555 Register args[] = {info.gp()};
1556 GenerateRuntimeCall(Runtime::kWasmTraceMemory, arraysize(args), args);
1557 __ DeallocateStackSlot(sizeof(MemoryTracingInfo));
1558 }
1559
1560 void GenerateRuntimeCall(Runtime::FunctionId runtime_function, int num_args,
1561 Register* args) {
1562 auto call_descriptor = compiler::Linkage::GetRuntimeCallDescriptor(
1563 compilation_zone_, runtime_function, num_args,
1564 compiler::Operator::kNoProperties, compiler::CallDescriptor::kNoFlags);
1565 // Currently, only one argument is supported. More arguments require some
1566 // caution for the parallel register moves (reuse StackTransferRecipe).
1567 DCHECK_EQ(1, num_args);
1568 constexpr size_t kInputShift = 1; // Input 0 is the call target.
1569 compiler::LinkageLocation param_loc =
1570 call_descriptor->GetInputLocation(kInputShift);
1571 if (param_loc.IsRegister()) {
1572 Register reg = Register::from_code(param_loc.AsRegister());
1573 __ Move(LiftoffRegister(reg), LiftoffRegister(args[0]),
1574 LiftoffAssembler::kWasmIntPtr);
1575 } else {
1576 DCHECK(param_loc.IsCallerFrameSlot());
1577 LiftoffStackSlots stack_slots(&asm_);
1578 stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr,
1579 LiftoffRegister(args[0])));
1580 stack_slots.Construct();
1581 }
1582
1583 // Set context to "no context" for the runtime call.
1584 __ TurboAssembler::Move(kContextRegister,
1585 Smi::FromInt(Context::kNoContext));
1586 Register centry = kJavaScriptCallCodeStartRegister;
1587 LOAD_TAGGED_PTR_INSTANCE_FIELD(centry, CEntryStub);
1588 __ CallRuntimeWithCEntry(runtime_function, centry);
1589 safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
1590 Safepoint::kNoLazyDeopt);
1591 }
1592
1593 Register AddMemoryMasking(Register index, uint32_t* offset,
1594 LiftoffRegList& pinned) {
1595 if (!FLAG_untrusted_code_mitigations || env_->use_trap_handler) {
1596 return index;
1597 }
1598 DEBUG_CODE_COMMENT("Mask memory index");
1599 // Make sure that we can overwrite {index}.
1600 if (__ cache_state()->is_used(LiftoffRegister(index))) {
1601 Register old_index = index;
1602 pinned.clear(LiftoffRegister(old_index));
1603 index = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1604 if (index != old_index) __ Move(index, old_index, kWasmI32);
1605 }
1606 Register tmp = __ GetUnusedRegister(kGpReg, pinned).gp();
1607 __ emit_ptrsize_add(index, index, *offset);
1608 LOAD_INSTANCE_FIELD(tmp, MemoryMask, kSystemPointerSize);
1609 __ emit_ptrsize_and(index, index, tmp);
1610 *offset = 0;
1611 return index;
1612 }
1613
1614 void LoadMem(FullDecoder* decoder, LoadType type,
1615 const MemoryAccessImmediate<validate>& imm,
1616 const Value& index_val, Value* result) {
1617 ValueType value_type = type.value_type();
1618 if (!CheckSupportedType(decoder, kSupportedTypes, value_type, "load"))
1619 return;
1620 LiftoffRegList pinned;
1621 Register index = pinned.set(__ PopToRegister()).gp();
1622 if (BoundsCheckMem(decoder, type.size(), imm.offset, index, pinned)) {
1623 return;
1624 }
1625 uint32_t offset = imm.offset;
1626 index = AddMemoryMasking(index, &offset, pinned);
1627 DEBUG_CODE_COMMENT("Load from memory");
1628 Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1629 LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
1630 RegClass rc = reg_class_for(value_type);
1631 LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
1632 uint32_t protected_load_pc = 0;
1633 __ Load(value, addr, index, offset, type, pinned, &protected_load_pc, true);
1634 if (env_->use_trap_handler) {
1635 AddOutOfLineTrap(decoder->position(),
1636 WasmCode::kThrowWasmTrapMemOutOfBounds,
1637 protected_load_pc);
1638 }
1639 __ PushRegister(value_type, value);
1640
1641 if (FLAG_trace_wasm_memory) {
1642 TraceMemoryOperation(false, type.mem_type().representation(), index,
1643 offset, decoder->position());
1644 }
1645 }
1646
1647 void StoreMem(FullDecoder* decoder, StoreType type,
1648 const MemoryAccessImmediate<validate>& imm,
1649 const Value& index_val, const Value& value_val) {
1650 ValueType value_type = type.value_type();
1651 if (!CheckSupportedType(decoder, kSupportedTypes, value_type, "store"))
1652 return;
1653 LiftoffRegList pinned;
1654 LiftoffRegister value = pinned.set(__ PopToRegister());
1655 Register index = pinned.set(__ PopToRegister(pinned)).gp();
1656 if (BoundsCheckMem(decoder, type.size(), imm.offset, index, pinned)) {
1657 return;
1658 }
1659 uint32_t offset = imm.offset;
1660 index = AddMemoryMasking(index, &offset, pinned);
1661 DEBUG_CODE_COMMENT("Store to memory");
1662 Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1663 LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
1664 uint32_t protected_store_pc = 0;
1665 LiftoffRegList outer_pinned;
1666 if (FLAG_trace_wasm_memory) outer_pinned.set(index);
1667 __ Store(addr, index, offset, value, type, outer_pinned,
1668 &protected_store_pc, true);
1669 if (env_->use_trap_handler) {
1670 AddOutOfLineTrap(decoder->position(),
1671 WasmCode::kThrowWasmTrapMemOutOfBounds,
1672 protected_store_pc);
1673 }
1674 if (FLAG_trace_wasm_memory) {
1675 TraceMemoryOperation(true, type.mem_rep(), index, offset,
1676 decoder->position());
1677 }
1678 }
1679
1680 void CurrentMemoryPages(FullDecoder* decoder, Value* result) {
1681 Register mem_size = __ GetUnusedRegister(kGpReg).gp();
1682 LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize);
1683 __ emit_ptrsize_shr(mem_size, mem_size, kWasmPageSizeLog2);
1684 __ PushRegister(kWasmI32, LiftoffRegister(mem_size));
1685 }
1686
1687 void MemoryGrow(FullDecoder* decoder, const Value& value, Value* result_val) {
1688 // Pop the input, then spill all cache registers to make the runtime call.
1689 LiftoffRegList pinned;
1690 LiftoffRegister input = pinned.set(__ PopToRegister());
1691 __ SpillAllRegisters();
1692
1693 constexpr Register kGpReturnReg = kGpReturnRegisters[0];
1694 static_assert(kLiftoffAssemblerGpCacheRegs & Register::bit<kGpReturnReg>(),
1695 "first return register is a cache register (needs more "
1696 "complex code here otherwise)");
1697 LiftoffRegister result = pinned.set(LiftoffRegister(kGpReturnReg));
1698
1699 WasmMemoryGrowDescriptor descriptor;
1700 DCHECK_EQ(0, descriptor.GetStackParameterCount());
1701 DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
1702 DCHECK_EQ(ValueTypes::MachineTypeFor(kWasmI32),
1703 descriptor.GetParameterType(0));
1704
1705 Register param_reg = descriptor.GetRegisterParameter(0);
1706 if (input.gp() != param_reg) __ Move(param_reg, input.gp(), kWasmI32);
1707
1708 __ CallRuntimeStub(WasmCode::kWasmMemoryGrow);
1709 safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
1710 Safepoint::kNoLazyDeopt);
1711
1712 if (kReturnRegister0 != result.gp()) {
1713 __ Move(result.gp(), kReturnRegister0, kWasmI32);
1714 }
1715
1716 __ PushRegister(kWasmI32, result);
1717 }
1718
1719 void CallDirect(FullDecoder* decoder,
1720 const CallFunctionImmediate<validate>& imm,
1721 const Value args[], Value returns[]) {
1722 if (imm.sig->return_count() > 1)
1723 return unsupported(decoder, "multi-return");
1724 if (imm.sig->return_count() == 1 &&
1725 !CheckSupportedType(decoder, kSupportedTypes, imm.sig->GetReturn(0),
1726 "return"))
1727 return;
1728
1729 auto call_descriptor =
1730 compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
1731 call_descriptor =
1732 GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
1733
1734 if (imm.index < env_->module->num_imported_functions) {
1735 // A direct call to an imported function.
1736 LiftoffRegList pinned;
1737 Register tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1738 Register target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1739
1740 Register imported_targets = tmp;
1741 LOAD_INSTANCE_FIELD(imported_targets, ImportedFunctionTargets,
1742 kSystemPointerSize);
1743 __ Load(LiftoffRegister(target), imported_targets, no_reg,
1744 imm.index * sizeof(Address), kPointerLoadType, pinned);
1745
1746 Register imported_function_refs = tmp;
1747 LOAD_TAGGED_PTR_INSTANCE_FIELD(imported_function_refs,
1748 ImportedFunctionRefs);
1749 Register imported_function_ref = tmp;
1750 __ LoadTaggedPointer(
1751 imported_function_ref, imported_function_refs, no_reg,
1752 ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned);
1753
1754 Register* explicit_instance = &imported_function_ref;
1755 __ PrepareCall(imm.sig, call_descriptor, &target, explicit_instance);
1756 source_position_table_builder_.AddPosition(
1757 __ pc_offset(), SourcePosition(decoder->position()), false);
1758
1759 __ CallIndirect(imm.sig, call_descriptor, target);
1760
1761 safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
1762 Safepoint::kNoLazyDeopt);
1763
1764 __ FinishCall(imm.sig, call_descriptor);
1765 } else {
1766 // A direct call within this module just gets the current instance.
1767 __ PrepareCall(imm.sig, call_descriptor);
1768
1769 source_position_table_builder_.AddPosition(
1770 __ pc_offset(), SourcePosition(decoder->position()), false);
1771
1772 // Just encode the function index. This will be patched at instantiation.
1773 Address addr = static_cast<Address>(imm.index);
1774 __ CallNativeWasmCode(addr);
1775
1776 safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
1777 Safepoint::kNoLazyDeopt);
1778
1779 __ FinishCall(imm.sig, call_descriptor);
1780 }
1781 }
1782
1783 void CallIndirect(FullDecoder* decoder, const Value& index_val,
1784 const CallIndirectImmediate<validate>& imm,
1785 const Value args[], Value returns[]) {
1786 if (imm.sig->return_count() > 1) {
1787 return unsupported(decoder, "multi-return");
1788 }
1789 if (imm.table_index != 0) {
1790 return unsupported(decoder, "table index != 0");
1791 }
1792 if (imm.sig->return_count() == 1 &&
1793 !CheckSupportedType(decoder, kSupportedTypes, imm.sig->GetReturn(0),
1794 "return")) {
1795 return;
1796 }
1797
1798 // Pop the index.
1799 Register index = __ PopToRegister().gp();
1800 // If that register is still being used after popping, we move it to another
1801 // register, because we want to modify that register.
1802 if (__ cache_state()->is_used(LiftoffRegister(index))) {
1803 Register new_index =
1804 __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(index)).gp();
1805 __ Move(new_index, index, kWasmI32);
1806 index = new_index;
1807 }
1808
1809 LiftoffRegList pinned = LiftoffRegList::ForRegs(index);
1810 // Get three temporary registers.
1811 Register table = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1812 Register tmp_const = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1813 Register scratch = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
1814
1815 // Bounds check against the table size.
1816 Label* invalid_func_label = AddOutOfLineTrap(
1817 decoder->position(), WasmCode::kThrowWasmTrapFuncInvalid);
1818
1819 uint32_t canonical_sig_num = env_->module->signature_ids[imm.sig_index];
1820 DCHECK_GE(canonical_sig_num, 0);
1821 DCHECK_GE(kMaxInt, canonical_sig_num);
1822
1823 // Compare against table size stored in
1824 // {instance->indirect_function_table_size}.
1825 LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size);
1826 __ emit_cond_jump(kUnsignedGreaterEqual, invalid_func_label, kWasmI32,
1827 index, tmp_const);
1828
1829 // Mask the index to prevent SSCA.
1830 if (FLAG_untrusted_code_mitigations) {
1831 DEBUG_CODE_COMMENT("Mask indirect call index");
1832 // mask = ((index - size) & ~index) >> 31
1833 // Reuse allocated registers; note: size is still stored in {tmp_const}.
1834 Register diff = table;
1835 Register neg_index = tmp_const;
1836 Register mask = scratch;
1837 // 1) diff = index - size
1838 __ emit_i32_sub(diff, index, tmp_const);
1839 // 2) neg_index = ~index
1840 __ LoadConstant(LiftoffRegister(neg_index), WasmValue(int32_t{-1}));
1841 __ emit_i32_xor(neg_index, neg_index, index);
1842 // 3) mask = diff & neg_index
1843 __ emit_i32_and(mask, diff, neg_index);
1844 // 4) mask = mask >> 31
1845 __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(int32_t{31}));
1846 __ emit_i32_sar(mask, mask, tmp_const, pinned);
1847
1848 // Apply mask.
1849 __ emit_i32_and(index, index, mask);
1850 }
1851
1852 DEBUG_CODE_COMMENT("Check indirect call signature");
1853 // Load the signature from {instance->ift_sig_ids[key]}
1854 LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds, kSystemPointerSize);
1855 // Multiply {index} by 4 to represent kInt32Size items.
1856 STATIC_ASSERT(kInt32Size == 4);
1857 // TODO(wasm): use a emit_i32_shli() instead of two adds.
1858 // (currently cannot use shl on ia32/x64 because it clobbers %rcx).
1859 __ emit_i32_add(index, index, index);
1860 __ emit_i32_add(index, index, index);
1861 __ Load(LiftoffRegister(scratch), table, index, 0, LoadType::kI32Load,
1862 pinned);
1863
1864 // Compare against expected signature.
1865 __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(canonical_sig_num));
1866
1867 Label* sig_mismatch_label = AddOutOfLineTrap(
1868 decoder->position(), WasmCode::kThrowWasmTrapFuncSigMismatch);
1869 __ emit_cond_jump(kUnequal, sig_mismatch_label,
1870 LiftoffAssembler::kWasmIntPtr, scratch, tmp_const);
1871
1872 // At this point {index} has already been multiplied by 4.
1873 DEBUG_CODE_COMMENT("Execute indirect call");
1874 if (kTaggedSize != kInt32Size) {
1875 DCHECK_EQ(kTaggedSize, kInt32Size * 2);
1876 // Multiply {index} by another 2 to represent kTaggedSize items.
1877 __ emit_i32_add(index, index, index);
1878 }
1879 // At this point {index} has already been multiplied by kTaggedSize.
1880
1881 // Load the instance from {instance->ift_instances[key]}
1882 LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs);
1883 __ LoadTaggedPointer(tmp_const, table, index,
1884 ObjectAccess::ElementOffsetInTaggedFixedArray(0),
1885 pinned);
1886
1887 if (kTaggedSize != kSystemPointerSize) {
1888 DCHECK_EQ(kSystemPointerSize, kTaggedSize * 2);
1889 // Multiply {index} by another 2 to represent kSystemPointerSize items.
1890 __ emit_i32_add(index, index, index);
1891 }
1892 // At this point {index} has already been multiplied by kSystemPointerSize.
1893
1894 Register* explicit_instance = &tmp_const;
1895
1896 // Load the target from {instance->ift_targets[key]}
1897 LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets,
1898 kSystemPointerSize);
1899 __ Load(LiftoffRegister(scratch), table, index, 0, kPointerLoadType,
1900 pinned);
1901
1902 source_position_table_builder_.AddPosition(
1903 __ pc_offset(), SourcePosition(decoder->position()), false);
1904
1905 auto call_descriptor =
1906 compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
1907 call_descriptor =
1908 GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
1909
1910 Register target = scratch;
1911 __ PrepareCall(imm.sig, call_descriptor, &target, explicit_instance);
1912 __ CallIndirect(imm.sig, call_descriptor, target);
1913
1914 safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
1915 Safepoint::kNoLazyDeopt);
1916
1917 __ FinishCall(imm.sig, call_descriptor);
1918 }
1919
1920 void ReturnCall(FullDecoder* decoder,
1921 const CallFunctionImmediate<validate>& imm,
1922 const Value args[]) {
1923 unsupported(decoder, "return_call");
1924 }
1925 void ReturnCallIndirect(FullDecoder* decoder, const Value& index_val,
1926 const CallIndirectImmediate<validate>& imm,
1927 const Value args[]) {
1928 unsupported(decoder, "return_call_indirect");
1929 }
1930 void SimdOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args,
1931 Value* result) {
1932 unsupported(decoder, "simd");
1933 }
1934 void SimdLaneOp(FullDecoder* decoder, WasmOpcode opcode,
1935 const SimdLaneImmediate<validate>& imm,
1936 const Vector<Value> inputs, Value* result) {
1937 unsupported(decoder, "simd");
1938 }
1939 void SimdShiftOp(FullDecoder* decoder, WasmOpcode opcode,
1940 const SimdShiftImmediate<validate>& imm, const Value& input,
1941 Value* result) {
1942 unsupported(decoder, "simd");
1943 }
1944 void Simd8x16ShuffleOp(FullDecoder* decoder,
1945 const Simd8x16ShuffleImmediate<validate>& imm,
1946 const Value& input0, const Value& input1,
1947 Value* result) {
1948 unsupported(decoder, "simd");
1949 }
1950 void Throw(FullDecoder* decoder, const ExceptionIndexImmediate<validate>&,
1951 const Vector<Value>& args) {
1952 unsupported(decoder, "throw");
1953 }
1954 void Rethrow(FullDecoder* decoder, const Value& exception) {
1955 unsupported(decoder, "rethrow");
1956 }
1957 void BrOnException(FullDecoder* decoder, const Value& exception,
1958 const ExceptionIndexImmediate<validate>& imm,
1959 uint32_t depth, Vector<Value> values) {
1960 unsupported(decoder, "br_on_exn");
1961 }
1962 void AtomicOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args,
1963 const MemoryAccessImmediate<validate>& imm, Value* result) {
1964 unsupported(decoder, "atomicop");
1965 }
1966 void MemoryInit(FullDecoder* decoder,
1967 const MemoryInitImmediate<validate>& imm, const Value& dst,
1968 const Value& src, const Value& size) {
1969 unsupported(decoder, "memory.init");
1970 }
1971 void DataDrop(FullDecoder* decoder, const DataDropImmediate<validate>& imm) {
1972 unsupported(decoder, "data.drop");
1973 }
1974 void MemoryCopy(FullDecoder* decoder,
1975 const MemoryCopyImmediate<validate>& imm, const Value& dst,
1976 const Value& src, const Value& size) {
1977 unsupported(decoder, "memory.copy");
1978 }
1979 void MemoryFill(FullDecoder* decoder,
1980 const MemoryIndexImmediate<validate>& imm, const Value& dst,
1981 const Value& value, const Value& size) {
1982 unsupported(decoder, "memory.fill");
1983 }
1984 void TableInit(FullDecoder* decoder, const TableInitImmediate<validate>& imm,
1985 Vector<Value> args) {
1986 unsupported(decoder, "table.init");
1987 }
1988 void ElemDrop(FullDecoder* decoder, const ElemDropImmediate<validate>& imm) {
1989 unsupported(decoder, "elem.drop");
1990 }
1991 void TableCopy(FullDecoder* decoder, const TableCopyImmediate<validate>& imm,
1992 Vector<Value> args) {
1993 unsupported(decoder, "table.copy");
1994 }
1995
1996 private:
1997 LiftoffAssembler asm_;
1998 compiler::CallDescriptor* const descriptor_;
1999 CompilationEnv* const env_;
2000 bool ok_ = true;
2001 std::vector<OutOfLineCode> out_of_line_code_;
2002 SourcePositionTableBuilder source_position_table_builder_;
2003 std::vector<trap_handler::ProtectedInstructionData> protected_instructions_;
2004 // Zone used to store information during compilation. The result will be
2005 // stored independently, such that this zone can die together with the
2006 // LiftoffCompiler after compilation.
2007 Zone* compilation_zone_;
2008 SafepointTableBuilder safepoint_table_builder_;
2009 // The pc offset of the instructions to reserve the stack frame. Needed to
2010 // patch the actually needed stack size in the end.
2011 uint32_t pc_offset_stack_frame_construction_ = 0;
2012
2013 void TraceCacheState(FullDecoder* decoder) const {
2014#ifdef DEBUG
2015 if (!FLAG_trace_liftoff || !FLAG_trace_wasm_decoder) return;
2016 StdoutStream os;
2017 for (int control_depth = decoder->control_depth() - 1; control_depth >= -1;
2018 --control_depth) {
2019 auto* cache_state =
2020 control_depth == -1 ? __ cache_state()
2021 : &decoder->control_at(control_depth)
2022 ->label_state;
2023 os << PrintCollection(cache_state->stack_state);
2024 if (control_depth != -1) PrintF("; ");
2025 }
2026 os << "\n";
2027#endif
2028 }
2029
2030 DISALLOW_IMPLICIT_CONSTRUCTORS(LiftoffCompiler);
2031};
2032
2033} // namespace
2034
2035WasmCompilationResult LiftoffCompilationUnit::ExecuteCompilation(
2036 AccountingAllocator* allocator, CompilationEnv* env,
2037 const FunctionBody& func_body, Counters* counters, WasmFeatures* detected) {
2038 TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
2039 "ExecuteLiftoffCompilation");
2040 base::ElapsedTimer compile_timer;
2041 if (FLAG_trace_wasm_decode_time) {
2042 compile_timer.Start();
2043 }
2044
2045 Zone zone(allocator, "LiftoffCompilationZone");
2046 const WasmModule* module = env ? env->module : nullptr;
2047 auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig);
2048 base::Optional<TimedHistogramScope> liftoff_compile_time_scope(
2049 base::in_place, counters->liftoff_compile_time());
2050 std::unique_ptr<wasm::WasmInstructionBuffer> instruction_buffer =
2051 wasm::WasmInstructionBuffer::New();
2052 WasmFullDecoder<Decoder::kValidate, LiftoffCompiler> decoder(
2053 &zone, module, env->enabled_features, detected, func_body,
2054 call_descriptor, env, &zone, instruction_buffer->CreateView());
2055 decoder.Decode();
2056 liftoff_compile_time_scope.reset();
2057 LiftoffCompiler* compiler = &decoder.interface();
2058 if (decoder.failed()) {
2059 compiler->OnFirstError(&decoder);
2060 return WasmCompilationResult{};
2061 }
2062 if (!compiler->ok()) {
2063 // Liftoff compilation failed.
2064 counters->liftoff_unsupported_functions()->Increment();
2065 return WasmCompilationResult{};
2066 }
2067
2068 counters->liftoff_compiled_functions()->Increment();
2069
2070 if (FLAG_trace_wasm_decode_time) {
2071 double compile_ms = compile_timer.Elapsed().InMillisecondsF();
2072 PrintF(
2073 "wasm-compilation liftoff phase 1 ok: %u bytes, %0.3f ms decode and "
2074 "compile\n",
2075 static_cast<unsigned>(func_body.end - func_body.start), compile_ms);
2076 }
2077
2078 WasmCompilationResult result;
2079 compiler->GetCode(&result.code_desc);
2080 result.instr_buffer = instruction_buffer->ReleaseBuffer();
2081 result.source_positions = compiler->GetSourcePositionTable();
2082 result.protected_instructions = compiler->GetProtectedInstructions();
2083 result.frame_slot_count = compiler->GetTotalFrameSlotCount();
2084 result.tagged_parameter_slots = call_descriptor->GetTaggedParameterSlots();
2085 result.result_tier = ExecutionTier::kLiftoff;
2086
2087 DCHECK(result.succeeded());
2088 return result;
2089}
2090
2091#undef __
2092#undef TRACE
2093#undef WASM_INSTANCE_OBJECT_FIELD_OFFSET
2094#undef WASM_INSTANCE_OBJECT_FIELD_SIZE
2095#undef LOAD_INSTANCE_FIELD
2096#undef LOAD_TAGGED_PTR_INSTANCE_FIELD
2097#undef DEBUG_CODE_COMMENT
2098
2099} // namespace wasm
2100} // namespace internal
2101} // namespace v8
2102