1/*
2 * Copyright (C) 2013-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "FTLLowerDFGToB3.h"
28
29#if ENABLE(FTL_JIT)
30
31#include "AirCode.h"
32#include "AirGenerationContext.h"
33#include "AllowMacroScratchRegisterUsage.h"
34#include "AllowMacroScratchRegisterUsageIf.h"
35#include "AtomicsObject.h"
36#include "B3CheckValue.h"
37#include "B3FenceValue.h"
38#include "B3PatchpointValue.h"
39#include "B3SlotBaseValue.h"
40#include "B3StackmapGenerationParams.h"
41#include "B3ValueInlines.h"
42#include "ButterflyInlines.h"
43#include "CallFrameShuffler.h"
44#include "CodeBlockWithJITType.h"
45#include "DFGAbstractInterpreterInlines.h"
46#include "DFGCapabilities.h"
47#include "DFGDoesGC.h"
48#include "DFGDominators.h"
49#include "DFGInPlaceAbstractState.h"
50#include "DFGLivenessAnalysisPhase.h"
51#include "DFGMayExit.h"
52#include "DFGOSRAvailabilityAnalysisPhase.h"
53#include "DFGOSRExitFuzz.h"
54#include "DirectArguments.h"
55#include "FTLAbstractHeapRepository.h"
56#include "FTLAvailableRecovery.h"
57#include "FTLExceptionTarget.h"
58#include "FTLForOSREntryJITCode.h"
59#include "FTLFormattedValue.h"
60#include "FTLLazySlowPathCall.h"
61#include "FTLLoweredNodeValue.h"
62#include "FTLOperations.h"
63#include "FTLOutput.h"
64#include "FTLPatchpointExceptionHandle.h"
65#include "FTLSnippetParams.h"
66#include "FTLThunks.h"
67#include "FTLWeightedTarget.h"
68#include "JITAddGenerator.h"
69#include "JITBitAndGenerator.h"
70#include "JITBitOrGenerator.h"
71#include "JITBitXorGenerator.h"
72#include "JITDivGenerator.h"
73#include "JITInlineCacheGenerator.h"
74#include "JITLeftShiftGenerator.h"
75#include "JITMathIC.h"
76#include "JITMulGenerator.h"
77#include "JITRightShiftGenerator.h"
78#include "JITSubGenerator.h"
79#include "JSAsyncFunction.h"
80#include "JSAsyncGenerator.h"
81#include "JSAsyncGeneratorFunction.h"
82#include "JSCInlines.h"
83#include "JSGenerator.h"
84#include "JSGeneratorFunction.h"
85#include "JSImmutableButterfly.h"
86#include "JSLexicalEnvironment.h"
87#include "JSMap.h"
88#include "OperandsInlines.h"
89#include "ProbeContext.h"
90#include "RegExpObject.h"
91#include "ScopedArguments.h"
92#include "ScopedArgumentsTable.h"
93#include "ScratchRegisterAllocator.h"
94#include "SetupVarargsFrame.h"
95#include "ShadowChicken.h"
96#include "StructureStubInfo.h"
97#include "SuperSampler.h"
98#include "ThunkGenerators.h"
99#include "VirtualRegister.h"
100#include "Watchdog.h"
101#include <atomic>
102#include <wtf/Box.h>
103#include <wtf/Gigacage.h>
104#include <wtf/RecursableLambda.h>
105#include <wtf/StdUnorderedSet.h>
106
107#undef RELEASE_ASSERT
108#define RELEASE_ASSERT(assertion) do { \
109 if (!(assertion)) { \
110 WTFReportAssertionFailure(__FILE__, __LINE__, WTF_PRETTY_FUNCTION, #assertion); \
111 CRASH(); \
112 } \
113} while (0)
114
115namespace JSC { namespace FTL {
116
117using namespace B3;
118using namespace DFG;
119
120namespace {
121
122std::atomic<int> compileCounter;
123
124#if !ASSERT_DISABLED
125NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
126 CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
127{
128 dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
129 if (nodeIndex != UINT_MAX)
130 dataLog(", node @", nodeIndex);
131 dataLog(".\n");
132 CRASH();
133}
134#endif
135
136// Using this instead of typeCheck() helps to reduce the load on B3, by creating
137// significantly less dead code.
138#define FTL_TYPE_CHECK_WITH_EXIT_KIND(exitKind, lowValue, highValue, typesPassedThrough, failCondition) do { \
139 FormattedValue _ftc_lowValue = (lowValue); \
140 Edge _ftc_highValue = (highValue); \
141 SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough); \
142 if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
143 break; \
144 typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition), exitKind); \
145 } while (false)
146
147#define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) \
148 FTL_TYPE_CHECK_WITH_EXIT_KIND(BadType, lowValue, highValue, typesPassedThrough, failCondition)
149
150class LowerDFGToB3 {
151 WTF_MAKE_NONCOPYABLE(LowerDFGToB3);
152public:
153 LowerDFGToB3(State& state)
154 : m_graph(state.graph)
155 , m_ftlState(state)
156 , m_out(state)
157 , m_proc(*state.proc)
158 , m_availabilityCalculator(m_graph)
159 , m_state(state.graph)
160 , m_interpreter(state.graph, m_state)
161 {
162 if (Options::validateAbstractInterpreterState()) {
163 performLivenessAnalysis(m_graph);
164
165 // We only use node liveness here, not combined liveness, as we only track
166 // AI state for live nodes.
167 for (DFG::BasicBlock* block : m_graph.blocksInNaturalOrder()) {
168 NodeSet live;
169
170 for (NodeFlowProjection node : block->ssa->liveAtTail) {
171 if (node.kind() == NodeFlowProjection::Primary)
172 live.addVoid(node.node());
173 }
174
175 for (unsigned i = block->size(); i--; ) {
176 Node* node = block->at(i);
177 live.remove(node);
178 m_graph.doToChildren(node, [&] (Edge child) {
179 live.addVoid(child.node());
180 });
181 m_liveInToNode.add(node, live);
182 }
183 }
184 }
185 }
186
187 void lower()
188 {
189 State* state = &m_ftlState;
190
191 CString name;
192 if (verboseCompilationEnabled()) {
193 name = toCString(
194 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
195 "_", codeBlock()->hash());
196 } else
197 name = "jsBody";
198
199 {
200 m_proc.setNumEntrypoints(m_graph.m_numberOfEntrypoints);
201 CodeBlock* codeBlock = m_graph.m_codeBlock;
202
203 Ref<B3::Air::PrologueGenerator> catchPrologueGenerator = createSharedTask<B3::Air::PrologueGeneratorFunction>(
204 [codeBlock] (CCallHelpers& jit, B3::Air::Code& code) {
205 AllowMacroScratchRegisterUsage allowScratch(jit);
206 jit.addPtr(CCallHelpers::TrustedImm32(-code.frameSize()), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
207 if (Options::zeroStackFrame())
208 jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, code.frameSize());
209
210 jit.emitSave(code.calleeSaveRegisterAtOffsetList());
211 jit.emitPutToCallFrameHeader(codeBlock, CallFrameSlot::codeBlock);
212 });
213
214 for (unsigned catchEntrypointIndex : m_graph.m_entrypointIndexToCatchBytecodeIndex.keys()) {
215 RELEASE_ASSERT(catchEntrypointIndex != 0);
216 m_proc.code().setPrologueForEntrypoint(catchEntrypointIndex, catchPrologueGenerator.copyRef());
217 }
218
219 if (m_graph.m_maxLocalsForCatchOSREntry) {
220 uint32_t numberOfLiveLocals = std::max(*m_graph.m_maxLocalsForCatchOSREntry, 1u); // Make sure we always allocate a non-null catchOSREntryBuffer.
221 m_ftlState.jitCode->common.catchOSREntryBuffer = m_graph.m_vm.scratchBufferForSize(sizeof(JSValue) * numberOfLiveLocals);
222 }
223 }
224
225 m_graph.ensureSSADominators();
226
227 if (verboseCompilationEnabled())
228 dataLog("Function ready, beginning lowering.\n");
229
230 m_out.initialize(m_heaps);
231
232 // We use prologue frequency for all of the initialization code.
233 m_out.setFrequency(1);
234
235 bool hasMultipleEntrypoints = m_graph.m_numberOfEntrypoints > 1;
236
237 LBasicBlock prologue = m_out.newBlock();
238 LBasicBlock callEntrypointArgumentSpeculations = hasMultipleEntrypoints ? m_out.newBlock() : nullptr;
239 m_handleExceptions = m_out.newBlock();
240
241 for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
242 m_highBlock = m_graph.block(blockIndex);
243 if (!m_highBlock)
244 continue;
245 m_out.setFrequency(m_highBlock->executionCount);
246 m_blocks.add(m_highBlock, m_out.newBlock());
247 }
248
249 // Back to prologue frequency for any bocks that get sneakily created in the initialization code.
250 m_out.setFrequency(1);
251
252 m_out.appendTo(prologue, hasMultipleEntrypoints ? callEntrypointArgumentSpeculations : m_handleExceptions);
253 m_out.initializeConstants(m_proc, prologue);
254 createPhiVariables();
255
256 size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal;
257 B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured);
258 m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured));
259 state->capturedValue = capturedBase->slot();
260
261 auto preOrder = m_graph.blocksInPreOrder();
262
263 VM* vm = &this->vm();
264
265 m_callFrame = m_out.framePointer();
266 m_vmValue = m_out.constIntPtr(vm);
267 m_numberTag = m_out.constInt64(JSValue::NumberTag);
268 m_notCellMask = m_out.constInt64(JSValue::NotCellMask);
269
270 // Make sure that B3 knows that we really care about the mask registers. This forces the
271 // constants to be materialized in registers.
272 m_proc.addFastConstant(m_numberTag->key());
273 m_proc.addFastConstant(m_notCellMask->key());
274
275 // We don't want the CodeBlock to have a weak pointer to itself because
276 // that would cause it to always get collected.
277 m_out.storePtr(m_out.constIntPtr(bitwise_cast<intptr_t>(codeBlock())), addressFor(CallFrameSlot::codeBlock));
278
279 // Stack Overflow Check.
280 unsigned exitFrameSize = m_graph.requiredRegisterCountForExit() * sizeof(Register);
281 MacroAssembler::AbsoluteAddress addressOfStackLimit(vm->addressOfSoftStackLimit());
282 PatchpointValue* stackOverflowHandler = m_out.patchpoint(Void);
283 CallSiteIndex callSiteIndex = callSiteIndexForCodeOrigin(m_ftlState, CodeOrigin(BytecodeIndex(0)));
284 stackOverflowHandler->appendSomeRegister(m_callFrame);
285 stackOverflowHandler->clobber(RegisterSet::macroScratchRegisters());
286 stackOverflowHandler->numGPScratchRegisters = 1;
287 stackOverflowHandler->setGenerator(
288 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
289 AllowMacroScratchRegisterUsage allowScratch(jit);
290 GPRReg fp = params[0].gpr();
291 GPRReg scratch = params.gpScratch(0);
292
293 unsigned ftlFrameSize = params.proc().frameSize();
294 unsigned maxFrameSize = std::max(exitFrameSize, ftlFrameSize);
295
296 jit.addPtr(MacroAssembler::TrustedImm32(-maxFrameSize), fp, scratch);
297 MacroAssembler::JumpList stackOverflow;
298 if (UNLIKELY(maxFrameSize > Options::reservedZoneSize()))
299 stackOverflow.append(jit.branchPtr(MacroAssembler::Above, scratch, fp));
300 stackOverflow.append(jit.branchPtr(MacroAssembler::Above, addressOfStackLimit, scratch));
301
302 params.addLatePath([=] (CCallHelpers& jit) {
303 AllowMacroScratchRegisterUsage allowScratch(jit);
304
305 stackOverflow.link(&jit);
306
307 // FIXME: We would not have to do this if the stack check was part of the Air
308 // prologue. Then, we would know that there is no way for the callee-saves to
309 // get clobbered.
310 // https://bugs.webkit.org/show_bug.cgi?id=172456
311 jit.emitRestore(params.proc().calleeSaveRegisterAtOffsetList());
312
313 jit.store32(
314 MacroAssembler::TrustedImm32(callSiteIndex.bits()),
315 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
316 jit.copyCalleeSavesToEntryFrameCalleeSavesBuffer(vm->topEntryFrame);
317
318 jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()), GPRInfo::argumentGPR0);
319 jit.prepareCallOperation(*vm);
320 CCallHelpers::Call throwCall = jit.call(OperationPtrTag);
321
322 jit.move(CCallHelpers::TrustedImmPtr(vm), GPRInfo::argumentGPR0);
323 jit.prepareCallOperation(*vm);
324 CCallHelpers::Call lookupExceptionHandlerCall = jit.call(OperationPtrTag);
325 jit.jumpToExceptionHandler(*vm);
326
327 jit.addLinkTask(
328 [=] (LinkBuffer& linkBuffer) {
329 linkBuffer.link(throwCall, FunctionPtr<OperationPtrTag>(operationThrowStackOverflowError));
330 linkBuffer.link(lookupExceptionHandlerCall, FunctionPtr<OperationPtrTag>(operationLookupExceptionHandlerFromCallerFrame));
331 });
332 });
333 });
334
335 LBasicBlock firstDFGBasicBlock = lowBlock(m_graph.block(0));
336
337 {
338 if (hasMultipleEntrypoints) {
339 Vector<LBasicBlock> successors(m_graph.m_numberOfEntrypoints);
340 successors[0] = callEntrypointArgumentSpeculations;
341 for (unsigned i = 1; i < m_graph.m_numberOfEntrypoints; ++i) {
342 // Currently, the only other entrypoint is an op_catch entrypoint.
343 // We do OSR entry at op_catch, and we prove argument formats before
344 // jumping to FTL code, so we don't need to check argument types here
345 // for these entrypoints.
346 successors[i] = firstDFGBasicBlock;
347 }
348
349 m_out.entrySwitch(successors);
350 m_out.appendTo(callEntrypointArgumentSpeculations, m_handleExceptions);
351 }
352
353 m_node = nullptr;
354 m_origin = NodeOrigin(CodeOrigin(BytecodeIndex(0)), CodeOrigin(BytecodeIndex(0)), true);
355
356 // Check Arguments.
357 availabilityMap().clear();
358 availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
359 for (unsigned i = codeBlock()->numParameters(); i--;) {
360 availabilityMap().m_locals.argument(i) =
361 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
362 }
363
364 for (unsigned i = codeBlock()->numParameters(); i--;) {
365 MethodOfGettingAValueProfile profile(&m_graph.m_profiledBlock->valueProfileForArgument(i));
366 VirtualRegister operand = virtualRegisterForArgument(i);
367 LValue jsValue = m_out.load64(addressFor(operand));
368
369 switch (m_graph.m_argumentFormats[0][i]) {
370 case FlushedInt32:
371 speculate(BadType, jsValueValue(jsValue), profile, isNotInt32(jsValue));
372 break;
373 case FlushedBoolean:
374 speculate(BadType, jsValueValue(jsValue), profile, isNotBoolean(jsValue));
375 break;
376 case FlushedCell:
377 speculate(BadType, jsValueValue(jsValue), profile, isNotCell(jsValue));
378 break;
379 case FlushedJSValue:
380 break;
381 default:
382 DFG_CRASH(m_graph, nullptr, "Bad flush format for argument");
383 break;
384 }
385 }
386 m_out.jump(firstDFGBasicBlock);
387 }
388
389
390 m_out.appendTo(m_handleExceptions, firstDFGBasicBlock);
391 Box<CCallHelpers::Label> exceptionHandler = state->exceptionHandler;
392 m_out.patchpoint(Void)->setGenerator(
393 [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
394 CCallHelpers::Jump jump = jit.jump();
395 jit.addLinkTask(
396 [=] (LinkBuffer& linkBuffer) {
397 linkBuffer.link(jump, linkBuffer.locationOf<ExceptionHandlerPtrTag>(*exceptionHandler));
398 });
399 });
400 m_out.unreachable();
401
402 for (DFG::BasicBlock* block : preOrder)
403 compileBlock(block);
404
405 // Make sure everything is decorated. This does a bunch of deferred decorating. This has
406 // to happen last because our abstract heaps are generated lazily. They have to be
407 // generated lazily because we have an infinite number of numbered, indexed, and
408 // absolute heaps. We only become aware of the ones we actually mention while lowering.
409 m_heaps.computeRangesAndDecorateInstructions();
410
411 // We create all Phi's up front, but we may then decide not to compile the basic block
412 // that would have contained one of them. So this creates orphans, which triggers B3
413 // validation failures. Calling this fixes the issue.
414 //
415 // Note that you should avoid the temptation to make this call conditional upon
416 // validation being enabled. B3 makes no guarantees of any kind of correctness when
417 // dealing with IR that would have failed validation. For example, it would be valid to
418 // write a B3 phase that so aggressively assumes the lack of orphans that it would crash
419 // if any orphans were around. We might even have such phases already.
420 m_proc.deleteOrphans();
421
422 // We put the blocks into the B3 procedure in a super weird order. Now we reorder them.
423 m_out.applyBlockOrder();
424 }
425
426private:
427
428 void createPhiVariables()
429 {
430 for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
431 DFG::BasicBlock* block = m_graph.block(blockIndex);
432 if (!block)
433 continue;
434 for (unsigned nodeIndex = block->size(); nodeIndex--;) {
435 Node* node = block->at(nodeIndex);
436 if (node->op() != DFG::Phi)
437 continue;
438 LType type;
439 switch (node->flags() & NodeResultMask) {
440 case NodeResultDouble:
441 type = Double;
442 break;
443 case NodeResultInt32:
444 type = Int32;
445 break;
446 case NodeResultInt52:
447 type = Int64;
448 break;
449 case NodeResultBoolean:
450 type = Int32;
451 break;
452 case NodeResultJS:
453 type = Int64;
454 break;
455 default:
456 DFG_CRASH(m_graph, node, "Bad Phi node result type");
457 break;
458 }
459 m_phis.add(node, m_proc.add<Value>(B3::Phi, type, Origin(node)));
460 }
461 }
462 }
463
464 void compileBlock(DFG::BasicBlock* block)
465 {
466 if (!block)
467 return;
468
469 if (verboseCompilationEnabled())
470 dataLog("Compiling block ", *block, "\n");
471
472 m_highBlock = block;
473
474 // Make sure that any blocks created while lowering code in the high block have the frequency of
475 // the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs
476 // something roughly approximate for things like register allocation.
477 m_out.setFrequency(m_highBlock->executionCount);
478
479 LBasicBlock lowBlock = m_blocks.get(m_highBlock);
480
481 m_nextHighBlock = 0;
482 for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
483 m_nextHighBlock = m_graph.block(nextBlockIndex);
484 if (m_nextHighBlock)
485 break;
486 }
487 m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
488
489 // All of this effort to find the next block gives us the ability to keep the
490 // generated IR in roughly program order. This ought not affect the performance
491 // of the generated code (since we expect B3 to reorder things) but it will
492 // make IR dumps easier to read.
493 m_out.appendTo(lowBlock, m_nextLowBlock);
494
495 if (Options::ftlCrashes())
496 m_out.trap();
497
498 if (!m_highBlock->cfaHasVisited) {
499 if (verboseCompilationEnabled())
500 dataLog("Bailing because CFA didn't reach.\n");
501 crash(m_highBlock, nullptr);
502 return;
503 }
504
505 m_aiCheckedNodes.clear();
506
507 m_availabilityCalculator.beginBlock(m_highBlock);
508
509 m_state.reset();
510 m_state.beginBasicBlock(m_highBlock);
511
512 for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
513 if (!compileNode(m_nodeIndex))
514 break;
515 }
516 }
517
518 void safelyInvalidateAfterTermination()
519 {
520 if (verboseCompilationEnabled())
521 dataLog("Bailing.\n");
522 crash();
523
524 // Invalidate dominated blocks. Under normal circumstances we would expect
525 // them to be invalidated already. But you can have the CFA become more
526 // precise over time because the structures of objects change on the main
527 // thread. Failing to do this would result in weird crashes due to a value
528 // being used but not defined. Race conditions FTW!
529 for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
530 DFG::BasicBlock* target = m_graph.block(blockIndex);
531 if (!target)
532 continue;
533 if (m_graph.m_ssaDominators->dominates(m_highBlock, target)) {
534 if (verboseCompilationEnabled())
535 dataLog("Block ", *target, " will bail also.\n");
536 target->cfaHasVisited = false;
537 }
538 }
539 }
540
541 void validateAIState(Node* node)
542 {
543 if (!m_graphDump) {
544 StringPrintStream out;
545 m_graph.dump(out);
546 m_graphDump = out.toString();
547 }
548
549 switch (node->op()) {
550 case MovHint:
551 case ZombieHint:
552 case JSConstant:
553 case LazyJSConstant:
554 case DoubleConstant:
555 case Int52Constant:
556 case GetStack:
557 case PutStack:
558 case KillStack:
559 case ExitOK:
560 return;
561 default:
562 break;
563 }
564
565 // Before we execute node.
566 NodeSet& live = m_liveInToNode.find(node)->value;
567 unsigned highParentIndex = node->index();
568 {
569 uint64_t hash = WTF::intHash(highParentIndex);
570 if (hash >= static_cast<uint64_t>((static_cast<double>(std::numeric_limits<unsigned>::max()) + 1) * Options::validateAbstractInterpreterStateProbability()))
571 return;
572 }
573
574 for (Node* node : live) {
575 if (node->isPhantomAllocation())
576 continue;
577
578 if (node->op() == CheckInBounds)
579 continue;
580
581 AbstractValue value = m_interpreter.forNode(node);
582 {
583 auto iter = m_aiCheckedNodes.find(node);
584 if (iter != m_aiCheckedNodes.end()) {
585 AbstractValue checkedValue = iter->value;
586 if (checkedValue == value) {
587 if (!(value.m_type & SpecCell))
588 continue;
589 }
590 }
591 m_aiCheckedNodes.set(node, value);
592 }
593
594 FlushFormat flushFormat;
595 LValue input;
596 if (node->hasJSResult()) {
597 input = lowJSValue(Edge(node, UntypedUse));
598 flushFormat = FlushedJSValue;
599 } else if (node->hasDoubleResult()) {
600 input = lowDouble(Edge(node, DoubleRepUse));
601 flushFormat = FlushedDouble;
602 } else if (node->hasInt52Result()) {
603 input = strictInt52ToJSValue(lowStrictInt52(Edge(node, Int52RepUse)));
604 flushFormat = FlushedInt52;
605 } else
606 continue;
607
608 unsigned highChildIndex = node->index();
609
610 String graphDump = m_graphDump;
611
612 PatchpointValue* patchpoint = m_out.patchpoint(Void);
613 patchpoint->effects = Effects::none();
614 patchpoint->effects.writesLocalState = true;
615 patchpoint->appendSomeRegister(input);
616 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
617 GPRReg reg = InvalidGPRReg;
618 FPRReg fpReg = InvalidFPRReg;
619 if (flushFormat == FlushedDouble)
620 fpReg = params[0].fpr();
621 else
622 reg = params[0].gpr();
623 jit.probe([=] (Probe::Context& context) {
624 JSValue input;
625 double doubleInput;
626
627 auto dumpAndCrash = [&] {
628 dataLogLn("Validation failed at node: @", highParentIndex);
629 dataLogLn("Failed validating live value: @", highChildIndex);
630 dataLogLn();
631 dataLogLn("Expected AI value = ", value);
632 if (flushFormat != FlushedDouble)
633 dataLogLn("Unexpected value = ", input);
634 else
635 dataLogLn("Unexpected double value = ", doubleInput);
636 dataLogLn();
637 dataLogLn(graphDump);
638 CRASH();
639 };
640
641 if (flushFormat == FlushedDouble) {
642 doubleInput = context.fpr(fpReg);
643 SpeculatedType type;
644 if (!std::isnan(doubleInput))
645 type = speculationFromValue(jsDoubleNumber(doubleInput));
646 else if (isImpureNaN(doubleInput))
647 type = SpecDoubleImpureNaN;
648 else
649 type = SpecDoublePureNaN;
650
651 if (!value.couldBeType(type))
652 dumpAndCrash();
653 } else {
654 input = JSValue::decode(context.gpr(reg));
655 if (flushFormat == FlushedInt52) {
656 RELEASE_ASSERT(input.isAnyInt());
657 input = jsDoubleNumber(input.asAnyInt());
658 }
659 if (!value.validateOSREntryValue(input, flushFormat))
660 dumpAndCrash();
661 }
662
663 });
664 });
665 }
666 }
667
668 bool compileNode(unsigned nodeIndex)
669 {
670 if (!m_state.isValid()) {
671 safelyInvalidateAfterTermination();
672 return false;
673 }
674
675 m_node = m_highBlock->at(nodeIndex);
676 m_origin = m_node->origin;
677 m_out.setOrigin(m_node);
678
679 if (verboseCompilationEnabled())
680 dataLog("Lowering ", m_node, "\n");
681
682 m_interpreter.startExecuting();
683 m_interpreter.executeKnownEdgeTypes(m_node);
684
685 if (Options::validateAbstractInterpreterState())
686 validateAIState(m_node);
687
688 if (validateDFGDoesGC) {
689 bool expectDoesGC = doesGC(m_graph, m_node);
690 m_out.store(m_out.constBool(expectDoesGC), m_out.absolute(vm().heap.addressOfExpectDoesGC()));
691 }
692
693 switch (m_node->op()) {
694 case DFG::Upsilon:
695 compileUpsilon();
696 break;
697 case DFG::Phi:
698 compilePhi();
699 break;
700 case JSConstant:
701 break;
702 case DoubleConstant:
703 compileDoubleConstant();
704 break;
705 case Int52Constant:
706 compileInt52Constant();
707 break;
708 case LazyJSConstant:
709 compileLazyJSConstant();
710 break;
711 case DoubleRep:
712 compileDoubleRep();
713 break;
714 case DoubleAsInt32:
715 compileDoubleAsInt32();
716 break;
717 case DFG::ValueRep:
718 compileValueRep();
719 break;
720 case Int52Rep:
721 compileInt52Rep();
722 break;
723 case ValueToInt32:
724 compileValueToInt32();
725 break;
726 case BooleanToNumber:
727 compileBooleanToNumber();
728 break;
729 case ExtractOSREntryLocal:
730 compileExtractOSREntryLocal();
731 break;
732 case ExtractCatchLocal:
733 compileExtractCatchLocal();
734 break;
735 case ClearCatchLocals:
736 compileClearCatchLocals();
737 break;
738 case GetStack:
739 compileGetStack();
740 break;
741 case PutStack:
742 compilePutStack();
743 break;
744 case DFG::Check:
745 case CheckVarargs:
746 compileNoOp();
747 break;
748 case ToObject:
749 case CallObjectConstructor:
750 compileToObjectOrCallObjectConstructor();
751 break;
752 case ToThis:
753 compileToThis();
754 break;
755 case Inc:
756 case Dec:
757 compileIncOrDec();
758 break;
759 case ValueNegate:
760 compileValueNegate();
761 break;
762 case ValueAdd:
763 compileValueAdd();
764 break;
765 case ValueSub:
766 compileValueSub();
767 break;
768 case ValueMul:
769 compileValueMul();
770 break;
771 case StrCat:
772 compileStrCat();
773 break;
774 case ArithAdd:
775 case ArithSub:
776 compileArithAddOrSub();
777 break;
778 case ArithClz32:
779 compileArithClz32();
780 break;
781 case ArithMul:
782 compileArithMul();
783 break;
784 case ValueDiv:
785 compileValueDiv();
786 break;
787 case ArithDiv:
788 compileArithDiv();
789 break;
790 case ValueMod:
791 compileValueMod();
792 break;
793 case ArithMod:
794 compileArithMod();
795 break;
796 case ArithMin:
797 case ArithMax:
798 compileArithMinOrMax();
799 break;
800 case ArithAbs:
801 compileArithAbs();
802 break;
803 case ValuePow:
804 compileValuePow();
805 break;
806 case ArithPow:
807 compileArithPow();
808 break;
809 case ArithRandom:
810 compileArithRandom();
811 break;
812 case ArithRound:
813 compileArithRound();
814 break;
815 case ArithFloor:
816 compileArithFloor();
817 break;
818 case ArithCeil:
819 compileArithCeil();
820 break;
821 case ArithTrunc:
822 compileArithTrunc();
823 break;
824 case ArithSqrt:
825 compileArithSqrt();
826 break;
827 case ArithFRound:
828 compileArithFRound();
829 break;
830 case ArithNegate:
831 compileArithNegate();
832 break;
833 case ArithUnary:
834 compileArithUnary();
835 break;
836 case ValueBitNot:
837 compileValueBitNot();
838 break;
839 case ArithBitNot:
840 compileArithBitNot();
841 break;
842 case ValueBitAnd:
843 compileValueBitAnd();
844 break;
845 case ArithBitAnd:
846 compileArithBitAnd();
847 break;
848 case ValueBitOr:
849 compileValueBitOr();
850 break;
851 case ArithBitOr:
852 compileArithBitOr();
853 break;
854 case ArithBitXor:
855 compileArithBitXor();
856 break;
857 case ValueBitXor:
858 compileValueBitXor();
859 break;
860 case ValueBitRShift:
861 compileValueBitRShift();
862 break;
863 case ArithBitRShift:
864 compileArithBitRShift();
865 break;
866 case ArithBitLShift:
867 compileArithBitLShift();
868 break;
869 case ValueBitLShift:
870 compileValueBitLShift();
871 break;
872 case BitURShift:
873 compileBitURShift();
874 break;
875 case UInt32ToNumber:
876 compileUInt32ToNumber();
877 break;
878 case CheckStructure:
879 compileCheckStructure();
880 break;
881 case CheckStructureOrEmpty:
882 compileCheckStructureOrEmpty();
883 break;
884 case CheckCell:
885 compileCheckCell();
886 break;
887 case CheckNotEmpty:
888 compileCheckNotEmpty();
889 break;
890 case AssertNotEmpty:
891 compileAssertNotEmpty();
892 break;
893 case CheckBadCell:
894 compileCheckBadCell();
895 break;
896 case CheckIdent:
897 compileCheckIdent();
898 break;
899 case GetExecutable:
900 compileGetExecutable();
901 break;
902 case Arrayify:
903 case ArrayifyToStructure:
904 compileArrayify();
905 break;
906 case PutStructure:
907 compilePutStructure();
908 break;
909 case TryGetById:
910 compileGetById(AccessType::TryGetById);
911 break;
912 case GetById:
913 case GetByIdFlush:
914 compileGetById(AccessType::GetById);
915 break;
916 case GetByIdWithThis:
917 compileGetByIdWithThis();
918 break;
919 case GetByIdDirect:
920 case GetByIdDirectFlush:
921 compileGetById(AccessType::GetByIdDirect);
922 break;
923 case InById:
924 compileInById();
925 break;
926 case InByVal:
927 compileInByVal();
928 break;
929 case HasOwnProperty:
930 compileHasOwnProperty();
931 break;
932 case PutById:
933 case PutByIdDirect:
934 case PutByIdFlush:
935 compilePutById();
936 break;
937 case PutByIdWithThis:
938 compilePutByIdWithThis();
939 break;
940 case PutGetterById:
941 case PutSetterById:
942 compilePutAccessorById();
943 break;
944 case PutGetterSetterById:
945 compilePutGetterSetterById();
946 break;
947 case PutGetterByVal:
948 case PutSetterByVal:
949 compilePutAccessorByVal();
950 break;
951 case DeleteById:
952 compileDeleteById();
953 break;
954 case DeleteByVal:
955 compileDeleteByVal();
956 break;
957 case GetButterfly:
958 compileGetButterfly();
959 break;
960 case ConstantStoragePointer:
961 compileConstantStoragePointer();
962 break;
963 case GetIndexedPropertyStorage:
964 compileGetIndexedPropertyStorage();
965 break;
966 case CheckArray:
967 compileCheckArray();
968 break;
969 case GetArrayLength:
970 compileGetArrayLength();
971 break;
972 case GetVectorLength:
973 compileGetVectorLength();
974 break;
975 case CheckInBounds:
976 compileCheckInBounds();
977 break;
978 case GetByVal:
979 compileGetByVal();
980 break;
981 case GetMyArgumentByVal:
982 case GetMyArgumentByValOutOfBounds:
983 compileGetMyArgumentByVal();
984 break;
985 case GetByValWithThis:
986 compileGetByValWithThis();
987 break;
988 case PutByVal:
989 case PutByValAlias:
990 case PutByValDirect:
991 compilePutByVal();
992 break;
993 case PutByValWithThis:
994 compilePutByValWithThis();
995 break;
996 case AtomicsAdd:
997 case AtomicsAnd:
998 case AtomicsCompareExchange:
999 case AtomicsExchange:
1000 case AtomicsLoad:
1001 case AtomicsOr:
1002 case AtomicsStore:
1003 case AtomicsSub:
1004 case AtomicsXor:
1005 compileAtomicsReadModifyWrite();
1006 break;
1007 case AtomicsIsLockFree:
1008 compileAtomicsIsLockFree();
1009 break;
1010 case DefineDataProperty:
1011 compileDefineDataProperty();
1012 break;
1013 case DefineAccessorProperty:
1014 compileDefineAccessorProperty();
1015 break;
1016 case ArrayPush:
1017 compileArrayPush();
1018 break;
1019 case ArrayPop:
1020 compileArrayPop();
1021 break;
1022 case ArraySlice:
1023 compileArraySlice();
1024 break;
1025 case ArrayIndexOf:
1026 compileArrayIndexOf();
1027 break;
1028 case CreateActivation:
1029 compileCreateActivation();
1030 break;
1031 case PushWithScope:
1032 compilePushWithScope();
1033 break;
1034 case NewFunction:
1035 case NewGeneratorFunction:
1036 case NewAsyncGeneratorFunction:
1037 case NewAsyncFunction:
1038 compileNewFunction();
1039 break;
1040 case CreateDirectArguments:
1041 compileCreateDirectArguments();
1042 break;
1043 case CreateScopedArguments:
1044 compileCreateScopedArguments();
1045 break;
1046 case CreateClonedArguments:
1047 compileCreateClonedArguments();
1048 break;
1049 case ObjectCreate:
1050 compileObjectCreate();
1051 break;
1052 case ObjectKeys:
1053 compileObjectKeys();
1054 break;
1055 case NewObject:
1056 compileNewObject();
1057 break;
1058 case NewPromise:
1059 compileNewPromise();
1060 break;
1061 case NewGenerator:
1062 compileNewGenerator();
1063 break;
1064 case NewAsyncGenerator:
1065 compileNewAsyncGenerator();
1066 break;
1067 case NewStringObject:
1068 compileNewStringObject();
1069 break;
1070 case NewSymbol:
1071 compileNewSymbol();
1072 break;
1073 case NewArray:
1074 compileNewArray();
1075 break;
1076 case NewArrayWithSpread:
1077 compileNewArrayWithSpread();
1078 break;
1079 case CreateThis:
1080 compileCreateThis();
1081 break;
1082 case CreatePromise:
1083 compileCreatePromise();
1084 break;
1085 case CreateGenerator:
1086 compileCreateGenerator();
1087 break;
1088 case CreateAsyncGenerator:
1089 compileCreateAsyncGenerator();
1090 break;
1091 case Spread:
1092 compileSpread();
1093 break;
1094 case NewArrayBuffer:
1095 compileNewArrayBuffer();
1096 break;
1097 case NewArrayWithSize:
1098 compileNewArrayWithSize();
1099 break;
1100 case NewTypedArray:
1101 compileNewTypedArray();
1102 break;
1103 case GetTypedArrayByteOffset:
1104 compileGetTypedArrayByteOffset();
1105 break;
1106 case GetPrototypeOf:
1107 compileGetPrototypeOf();
1108 break;
1109 case AllocatePropertyStorage:
1110 compileAllocatePropertyStorage();
1111 break;
1112 case ReallocatePropertyStorage:
1113 compileReallocatePropertyStorage();
1114 break;
1115 case NukeStructureAndSetButterfly:
1116 compileNukeStructureAndSetButterfly();
1117 break;
1118 case ToNumber:
1119 compileToNumber();
1120 break;
1121 case ToNumeric:
1122 compileToNumeric();
1123 break;
1124 case ToString:
1125 case CallStringConstructor:
1126 case StringValueOf:
1127 compileToStringOrCallStringConstructorOrStringValueOf();
1128 break;
1129 case ToPrimitive:
1130 compileToPrimitive();
1131 break;
1132 case MakeRope:
1133 compileMakeRope();
1134 break;
1135 case StringCharAt:
1136 compileStringCharAt();
1137 break;
1138 case StringCharCodeAt:
1139 compileStringCharCodeAt();
1140 break;
1141 case StringCodePointAt:
1142 compileStringCodePointAt();
1143 break;
1144 case StringFromCharCode:
1145 compileStringFromCharCode();
1146 break;
1147 case GetByOffset:
1148 case GetGetterSetterByOffset:
1149 compileGetByOffset();
1150 break;
1151 case GetGetter:
1152 compileGetGetter();
1153 break;
1154 case GetSetter:
1155 compileGetSetter();
1156 break;
1157 case MultiGetByOffset:
1158 compileMultiGetByOffset();
1159 break;
1160 case PutByOffset:
1161 compilePutByOffset();
1162 break;
1163 case MultiPutByOffset:
1164 compileMultiPutByOffset();
1165 break;
1166 case MatchStructure:
1167 compileMatchStructure();
1168 break;
1169 case GetGlobalVar:
1170 case GetGlobalLexicalVariable:
1171 compileGetGlobalVariable();
1172 break;
1173 case PutGlobalVariable:
1174 compilePutGlobalVariable();
1175 break;
1176 case NotifyWrite:
1177 compileNotifyWrite();
1178 break;
1179 case GetCallee:
1180 compileGetCallee();
1181 break;
1182 case SetCallee:
1183 compileSetCallee();
1184 break;
1185 case GetArgumentCountIncludingThis:
1186 compileGetArgumentCountIncludingThis();
1187 break;
1188 case SetArgumentCountIncludingThis:
1189 compileSetArgumentCountIncludingThis();
1190 break;
1191 case GetScope:
1192 compileGetScope();
1193 break;
1194 case SkipScope:
1195 compileSkipScope();
1196 break;
1197 case GetGlobalObject:
1198 compileGetGlobalObject();
1199 break;
1200 case GetGlobalThis:
1201 compileGetGlobalThis();
1202 break;
1203 case GetClosureVar:
1204 compileGetClosureVar();
1205 break;
1206 case PutClosureVar:
1207 compilePutClosureVar();
1208 break;
1209 case GetInternalField:
1210 compileGetInternalField();
1211 break;
1212 case PutInternalField:
1213 compilePutInternalField();
1214 break;
1215 case GetFromArguments:
1216 compileGetFromArguments();
1217 break;
1218 case PutToArguments:
1219 compilePutToArguments();
1220 break;
1221 case GetArgument:
1222 compileGetArgument();
1223 break;
1224 case CompareEq:
1225 compileCompareEq();
1226 break;
1227 case CompareStrictEq:
1228 compileCompareStrictEq();
1229 break;
1230 case CompareLess:
1231 compileCompareLess();
1232 break;
1233 case CompareLessEq:
1234 compileCompareLessEq();
1235 break;
1236 case CompareGreater:
1237 compileCompareGreater();
1238 break;
1239 case CompareGreaterEq:
1240 compileCompareGreaterEq();
1241 break;
1242 case CompareBelow:
1243 compileCompareBelow();
1244 break;
1245 case CompareBelowEq:
1246 compileCompareBelowEq();
1247 break;
1248 case CompareEqPtr:
1249 compileCompareEqPtr();
1250 break;
1251 case SameValue:
1252 compileSameValue();
1253 break;
1254 case LogicalNot:
1255 compileLogicalNot();
1256 break;
1257 case Call:
1258 case TailCallInlinedCaller:
1259 case Construct:
1260 compileCallOrConstruct();
1261 break;
1262 case DirectCall:
1263 case DirectTailCallInlinedCaller:
1264 case DirectConstruct:
1265 case DirectTailCall:
1266 compileDirectCallOrConstruct();
1267 break;
1268 case TailCall:
1269 compileTailCall();
1270 break;
1271 case CallVarargs:
1272 case CallForwardVarargs:
1273 case TailCallVarargs:
1274 case TailCallVarargsInlinedCaller:
1275 case TailCallForwardVarargs:
1276 case TailCallForwardVarargsInlinedCaller:
1277 case ConstructVarargs:
1278 case ConstructForwardVarargs:
1279 compileCallOrConstructVarargs();
1280 break;
1281 case CallEval:
1282 compileCallEval();
1283 break;
1284 case LoadVarargs:
1285 compileLoadVarargs();
1286 break;
1287 case ForwardVarargs:
1288 compileForwardVarargs();
1289 break;
1290 case DFG::Jump:
1291 compileJump();
1292 break;
1293 case DFG::Branch:
1294 compileBranch();
1295 break;
1296 case DFG::Switch:
1297 compileSwitch();
1298 break;
1299 case DFG::EntrySwitch:
1300 compileEntrySwitch();
1301 break;
1302 case DFG::Return:
1303 compileReturn();
1304 break;
1305 case ForceOSRExit:
1306 compileForceOSRExit();
1307 break;
1308 case CPUIntrinsic:
1309#if CPU(X86_64)
1310 compileCPUIntrinsic();
1311#else
1312 RELEASE_ASSERT_NOT_REACHED();
1313#endif
1314 break;
1315 case Throw:
1316 compileThrow();
1317 break;
1318 case ThrowStaticError:
1319 compileThrowStaticError();
1320 break;
1321 case InvalidationPoint:
1322 compileInvalidationPoint();
1323 break;
1324 case IsEmpty:
1325 compileIsEmpty();
1326 break;
1327 case IsUndefined:
1328 compileIsUndefined();
1329 break;
1330 case IsUndefinedOrNull:
1331 compileIsUndefinedOrNull();
1332 break;
1333 case IsBoolean:
1334 compileIsBoolean();
1335 break;
1336 case IsNumber:
1337 compileIsNumber();
1338 break;
1339 case NumberIsInteger:
1340 compileNumberIsInteger();
1341 break;
1342 case IsCellWithType:
1343 compileIsCellWithType();
1344 break;
1345 case MapHash:
1346 compileMapHash();
1347 break;
1348 case NormalizeMapKey:
1349 compileNormalizeMapKey();
1350 break;
1351 case GetMapBucket:
1352 compileGetMapBucket();
1353 break;
1354 case GetMapBucketHead:
1355 compileGetMapBucketHead();
1356 break;
1357 case GetMapBucketNext:
1358 compileGetMapBucketNext();
1359 break;
1360 case LoadKeyFromMapBucket:
1361 compileLoadKeyFromMapBucket();
1362 break;
1363 case LoadValueFromMapBucket:
1364 compileLoadValueFromMapBucket();
1365 break;
1366 case ExtractValueFromWeakMapGet:
1367 compileExtractValueFromWeakMapGet();
1368 break;
1369 case SetAdd:
1370 compileSetAdd();
1371 break;
1372 case MapSet:
1373 compileMapSet();
1374 break;
1375 case WeakMapGet:
1376 compileWeakMapGet();
1377 break;
1378 case WeakSetAdd:
1379 compileWeakSetAdd();
1380 break;
1381 case WeakMapSet:
1382 compileWeakMapSet();
1383 break;
1384 case IsObject:
1385 compileIsObject();
1386 break;
1387 case IsObjectOrNull:
1388 compileIsObjectOrNull();
1389 break;
1390 case IsFunction:
1391 compileIsFunction();
1392 break;
1393 case IsTypedArrayView:
1394 compileIsTypedArrayView();
1395 break;
1396 case ParseInt:
1397 compileParseInt();
1398 break;
1399 case TypeOf:
1400 compileTypeOf();
1401 break;
1402 case CheckTypeInfoFlags:
1403 compileCheckTypeInfoFlags();
1404 break;
1405 case OverridesHasInstance:
1406 compileOverridesHasInstance();
1407 break;
1408 case InstanceOf:
1409 compileInstanceOf();
1410 break;
1411 case InstanceOfCustom:
1412 compileInstanceOfCustom();
1413 break;
1414 case CountExecution:
1415 compileCountExecution();
1416 break;
1417 case SuperSamplerBegin:
1418 compileSuperSamplerBegin();
1419 break;
1420 case SuperSamplerEnd:
1421 compileSuperSamplerEnd();
1422 break;
1423 case StoreBarrier:
1424 case FencedStoreBarrier:
1425 compileStoreBarrier();
1426 break;
1427 case HasIndexedProperty:
1428 compileHasIndexedProperty();
1429 break;
1430 case HasGenericProperty:
1431 compileHasGenericProperty();
1432 break;
1433 case HasStructureProperty:
1434 compileHasStructureProperty();
1435 break;
1436 case GetDirectPname:
1437 compileGetDirectPname();
1438 break;
1439 case GetEnumerableLength:
1440 compileGetEnumerableLength();
1441 break;
1442 case GetPropertyEnumerator:
1443 compileGetPropertyEnumerator();
1444 break;
1445 case GetEnumeratorStructurePname:
1446 compileGetEnumeratorStructurePname();
1447 break;
1448 case GetEnumeratorGenericPname:
1449 compileGetEnumeratorGenericPname();
1450 break;
1451 case ToIndexString:
1452 compileToIndexString();
1453 break;
1454 case CheckStructureImmediate:
1455 compileCheckStructureImmediate();
1456 break;
1457 case MaterializeNewObject:
1458 compileMaterializeNewObject();
1459 break;
1460 case MaterializeCreateActivation:
1461 compileMaterializeCreateActivation();
1462 break;
1463 case CheckTraps:
1464 compileCheckTraps();
1465 break;
1466 case CreateRest:
1467 compileCreateRest();
1468 break;
1469 case GetRestLength:
1470 compileGetRestLength();
1471 break;
1472 case RegExpExec:
1473 compileRegExpExec();
1474 break;
1475 case RegExpExecNonGlobalOrSticky:
1476 compileRegExpExecNonGlobalOrSticky();
1477 break;
1478 case RegExpTest:
1479 compileRegExpTest();
1480 break;
1481 case RegExpMatchFast:
1482 compileRegExpMatchFast();
1483 break;
1484 case RegExpMatchFastGlobal:
1485 compileRegExpMatchFastGlobal();
1486 break;
1487 case NewRegexp:
1488 compileNewRegexp();
1489 break;
1490 case SetFunctionName:
1491 compileSetFunctionName();
1492 break;
1493 case StringReplace:
1494 case StringReplaceRegExp:
1495 compileStringReplace();
1496 break;
1497 case GetRegExpObjectLastIndex:
1498 compileGetRegExpObjectLastIndex();
1499 break;
1500 case SetRegExpObjectLastIndex:
1501 compileSetRegExpObjectLastIndex();
1502 break;
1503 case LogShadowChickenPrologue:
1504 compileLogShadowChickenPrologue();
1505 break;
1506 case LogShadowChickenTail:
1507 compileLogShadowChickenTail();
1508 break;
1509 case RecordRegExpCachedResult:
1510 compileRecordRegExpCachedResult();
1511 break;
1512 case ResolveScopeForHoistingFuncDeclInEval:
1513 compileResolveScopeForHoistingFuncDeclInEval();
1514 break;
1515 case ResolveScope:
1516 compileResolveScope();
1517 break;
1518 case GetDynamicVar:
1519 compileGetDynamicVar();
1520 break;
1521 case PutDynamicVar:
1522 compilePutDynamicVar();
1523 break;
1524 case Unreachable:
1525 compileUnreachable();
1526 break;
1527 case StringSlice:
1528 compileStringSlice();
1529 break;
1530 case ToLowerCase:
1531 compileToLowerCase();
1532 break;
1533 case NumberToStringWithRadix:
1534 compileNumberToStringWithRadix();
1535 break;
1536 case NumberToStringWithValidRadixConstant:
1537 compileNumberToStringWithValidRadixConstant();
1538 break;
1539 case CheckSubClass:
1540 compileCheckSubClass();
1541 break;
1542 case CallDOM:
1543 compileCallDOM();
1544 break;
1545 case CallDOMGetter:
1546 compileCallDOMGetter();
1547 break;
1548 case FilterCallLinkStatus:
1549 case FilterGetByStatus:
1550 case FilterPutByIdStatus:
1551 case FilterInByIdStatus:
1552 compileFilterICStatus();
1553 break;
1554 case DateGetInt32OrNaN:
1555 case DateGetTime:
1556 compileDateGet();
1557 break;
1558 case DataViewGetInt:
1559 case DataViewGetFloat:
1560 compileDataViewGet();
1561 break;
1562 case DataViewSet:
1563 compileDataViewSet();
1564 break;
1565
1566 case PhantomLocal:
1567 case LoopHint:
1568 case MovHint:
1569 case ZombieHint:
1570 case ExitOK:
1571 case PhantomNewObject:
1572 case PhantomNewFunction:
1573 case PhantomNewGeneratorFunction:
1574 case PhantomNewAsyncGeneratorFunction:
1575 case PhantomNewAsyncFunction:
1576 case PhantomCreateActivation:
1577 case PhantomDirectArguments:
1578 case PhantomCreateRest:
1579 case PhantomSpread:
1580 case PhantomNewArrayWithSpread:
1581 case PhantomNewArrayBuffer:
1582 case PhantomClonedArguments:
1583 case PhantomNewRegexp:
1584 case PutHint:
1585 case BottomValue:
1586 case KillStack:
1587 case InitializeEntrypointArguments:
1588 break;
1589 default:
1590 DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
1591 break;
1592 }
1593
1594 if (m_node->isTerminal())
1595 return false;
1596
1597 if (!m_state.isValid()) {
1598 safelyInvalidateAfterTermination();
1599 return false;
1600 }
1601
1602 m_availabilityCalculator.executeNode(m_node);
1603 m_interpreter.executeEffects(nodeIndex);
1604
1605 return true;
1606 }
1607
1608 void compileUpsilon()
1609 {
1610 LValue upsilonValue = nullptr;
1611 switch (m_node->child1().useKind()) {
1612 case DoubleRepUse:
1613 upsilonValue = lowDouble(m_node->child1());
1614 break;
1615 case Int32Use:
1616 case KnownInt32Use:
1617 upsilonValue = lowInt32(m_node->child1());
1618 break;
1619 case Int52RepUse:
1620 upsilonValue = lowInt52(m_node->child1());
1621 break;
1622 case BooleanUse:
1623 case KnownBooleanUse:
1624 upsilonValue = lowBoolean(m_node->child1());
1625 break;
1626 case CellUse:
1627 case KnownCellUse:
1628 upsilonValue = lowCell(m_node->child1());
1629 break;
1630 case UntypedUse:
1631 upsilonValue = lowJSValue(m_node->child1());
1632 break;
1633 default:
1634 DFG_CRASH(m_graph, m_node, "Bad use kind");
1635 break;
1636 }
1637 ValueFromBlock upsilon = m_out.anchor(upsilonValue);
1638 LValue phiNode = m_phis.get(m_node->phi());
1639 m_out.addIncomingToPhi(phiNode, upsilon);
1640 }
1641
1642 void compilePhi()
1643 {
1644 LValue phi = m_phis.get(m_node);
1645 m_out.m_block->append(phi);
1646
1647 switch (m_node->flags() & NodeResultMask) {
1648 case NodeResultDouble:
1649 setDouble(phi);
1650 break;
1651 case NodeResultInt32:
1652 setInt32(phi);
1653 break;
1654 case NodeResultInt52:
1655 setInt52(phi);
1656 break;
1657 case NodeResultBoolean:
1658 setBoolean(phi);
1659 break;
1660 case NodeResultJS:
1661 setJSValue(phi);
1662 break;
1663 default:
1664 DFG_CRASH(m_graph, m_node, "Bad result type");
1665 break;
1666 }
1667 }
1668
1669 void compileDoubleConstant()
1670 {
1671 setDouble(m_out.constDouble(m_node->asNumber()));
1672 }
1673
1674 void compileInt52Constant()
1675 {
1676 int64_t value = m_node->asAnyInt();
1677
1678 setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
1679 setStrictInt52(m_out.constInt64(value));
1680 }
1681
1682 void compileLazyJSConstant()
1683 {
1684 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
1685 LazyJSValue value = m_node->lazyJSValue();
1686 patchpoint->setGenerator(
1687 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1688 value.emit(jit, JSValueRegs(params[0].gpr()));
1689 });
1690 patchpoint->effects = Effects::none();
1691 setJSValue(patchpoint);
1692 }
1693
1694 void compileDoubleRep()
1695 {
1696 switch (m_node->child1().useKind()) {
1697 case RealNumberUse: {
1698 LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1699
1700 LValue doubleValue = unboxDouble(value);
1701
1702 LBasicBlock intCase = m_out.newBlock();
1703 LBasicBlock continuation = m_out.newBlock();
1704
1705 ValueFromBlock fastResult = m_out.anchor(doubleValue);
1706 m_out.branch(
1707 m_out.doubleEqual(doubleValue, doubleValue),
1708 usually(continuation), rarely(intCase));
1709
1710 LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
1711
1712 FTL_TYPE_CHECK(
1713 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
1714 isNotInt32(value, provenType(m_node->child1()) & ~SpecDoubleReal));
1715 ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
1716 m_out.jump(continuation);
1717
1718 m_out.appendTo(continuation, lastNext);
1719
1720 setDouble(m_out.phi(Double, fastResult, slowResult));
1721 return;
1722 }
1723
1724 case NotCellUse:
1725 case NumberUse: {
1726 bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
1727
1728 LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
1729
1730 LBasicBlock intCase = m_out.newBlock();
1731 LBasicBlock doubleTesting = m_out.newBlock();
1732 LBasicBlock doubleCase = m_out.newBlock();
1733 LBasicBlock nonDoubleCase = m_out.newBlock();
1734 LBasicBlock continuation = m_out.newBlock();
1735
1736 m_out.branch(
1737 isNotInt32(value, provenType(m_node->child1())),
1738 unsure(doubleTesting), unsure(intCase));
1739
1740 LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1741
1742 ValueFromBlock intToDouble = m_out.anchor(
1743 m_out.intToDouble(unboxInt32(value)));
1744 m_out.jump(continuation);
1745
1746 m_out.appendTo(doubleTesting, doubleCase);
1747 LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1748 m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1749
1750 m_out.appendTo(doubleCase, nonDoubleCase);
1751 ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1752 m_out.jump(continuation);
1753
1754 if (shouldConvertNonNumber) {
1755 LBasicBlock undefinedCase = m_out.newBlock();
1756 LBasicBlock testNullCase = m_out.newBlock();
1757 LBasicBlock nullCase = m_out.newBlock();
1758 LBasicBlock testBooleanTrueCase = m_out.newBlock();
1759 LBasicBlock convertBooleanTrueCase = m_out.newBlock();
1760 LBasicBlock convertBooleanFalseCase = m_out.newBlock();
1761
1762 m_out.appendTo(nonDoubleCase, undefinedCase);
1763 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(JSValue::ValueUndefined));
1764 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1765
1766 m_out.appendTo(undefinedCase, testNullCase);
1767 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1768 m_out.jump(continuation);
1769
1770 m_out.appendTo(testNullCase, nullCase);
1771 LValue valueIsNull = m_out.equal(value, m_out.constInt64(JSValue::ValueNull));
1772 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1773
1774 m_out.appendTo(nullCase, testBooleanTrueCase);
1775 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1776 m_out.jump(continuation);
1777
1778 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1779 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(JSValue::ValueTrue));
1780 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1781
1782 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1783 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1784 m_out.jump(continuation);
1785
1786 m_out.appendTo(convertBooleanFalseCase, continuation);
1787
1788 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(JSValue::ValueFalse));
1789 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCellCheck, valueIsNotBooleanFalse);
1790 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1791 m_out.jump(continuation);
1792
1793 m_out.appendTo(continuation, lastNext);
1794 setDouble(m_out.phi(Double, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1795 return;
1796 }
1797 m_out.appendTo(nonDoubleCase, continuation);
1798 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1799 m_out.unreachable();
1800
1801 m_out.appendTo(continuation, lastNext);
1802
1803 setDouble(m_out.phi(Double, intToDouble, unboxedDouble));
1804 return;
1805 }
1806
1807 case Int52RepUse: {
1808 setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1809 return;
1810 }
1811
1812 default:
1813 DFG_CRASH(m_graph, m_node, "Bad use kind");
1814 }
1815 }
1816
1817 void compileDoubleAsInt32()
1818 {
1819 LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1820 setInt32(integerValue);
1821 }
1822
1823 void compileValueRep()
1824 {
1825 switch (m_node->child1().useKind()) {
1826 case DoubleRepUse: {
1827 LValue value = lowDouble(m_node->child1());
1828
1829 if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1830 value = m_out.select(
1831 m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1832 }
1833
1834 setJSValue(boxDouble(value));
1835 return;
1836 }
1837
1838 case Int52RepUse: {
1839 setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1840 return;
1841 }
1842
1843 default:
1844 DFG_CRASH(m_graph, m_node, "Bad use kind");
1845 }
1846 }
1847
1848 void compileInt52Rep()
1849 {
1850 switch (m_node->child1().useKind()) {
1851 case Int32Use:
1852 setStrictInt52(m_out.signExt32To64(lowInt32(m_node->child1())));
1853 return;
1854
1855 case AnyIntUse:
1856 setStrictInt52(
1857 jsValueToStrictInt52(
1858 m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1859 return;
1860
1861 case DoubleRepAnyIntUse:
1862 setStrictInt52(
1863 doubleToStrictInt52(
1864 m_node->child1(), lowDouble(m_node->child1())));
1865 return;
1866
1867 default:
1868 RELEASE_ASSERT_NOT_REACHED();
1869 }
1870 }
1871
1872 void compileValueToInt32()
1873 {
1874 switch (m_node->child1().useKind()) {
1875 case Int52RepUse:
1876 setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1877 break;
1878
1879 case DoubleRepUse:
1880 setInt32(doubleToInt32(lowDouble(m_node->child1())));
1881 break;
1882
1883 case NumberUse:
1884 case NotCellUse: {
1885 LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1886 if (isValid(value)) {
1887 setInt32(value.value());
1888 break;
1889 }
1890
1891 value = m_jsValueValues.get(m_node->child1().node());
1892 if (isValid(value)) {
1893 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1894 break;
1895 }
1896
1897 // We'll basically just get here for constants. But it's good to have this
1898 // catch-all since we often add new representations into the mix.
1899 setInt32(
1900 numberOrNotCellToInt32(
1901 m_node->child1(),
1902 lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1903 break;
1904 }
1905
1906 default:
1907 DFG_CRASH(m_graph, m_node, "Bad use kind");
1908 break;
1909 }
1910 }
1911
1912 void compileBooleanToNumber()
1913 {
1914 switch (m_node->child1().useKind()) {
1915 case BooleanUse: {
1916 setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), Int32));
1917 return;
1918 }
1919
1920 case UntypedUse: {
1921 LValue value = lowJSValue(m_node->child1());
1922
1923 if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1924 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1925 return;
1926 }
1927
1928 LBasicBlock booleanCase = m_out.newBlock();
1929 LBasicBlock continuation = m_out.newBlock();
1930
1931 ValueFromBlock notBooleanResult = m_out.anchor(value);
1932 m_out.branch(
1933 isBoolean(value, provenType(m_node->child1())),
1934 unsure(booleanCase), unsure(continuation));
1935
1936 LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1937 ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1938 m_out.zeroExt(unboxBoolean(value), Int64), m_numberTag));
1939 m_out.jump(continuation);
1940
1941 m_out.appendTo(continuation, lastNext);
1942 setJSValue(m_out.phi(Int64, booleanResult, notBooleanResult));
1943 return;
1944 }
1945
1946 default:
1947 RELEASE_ASSERT_NOT_REACHED();
1948 return;
1949 }
1950 }
1951
1952 void compileExtractOSREntryLocal()
1953 {
1954 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1955 m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1956 setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1957 }
1958
1959 void compileExtractCatchLocal()
1960 {
1961 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(m_ftlState.jitCode->common.catchOSREntryBuffer->dataBuffer());
1962 setJSValue(m_out.load64(m_out.absolute(buffer + m_node->catchOSREntryIndex())));
1963 }
1964
1965 void compileClearCatchLocals()
1966 {
1967 ScratchBuffer* scratchBuffer = m_ftlState.jitCode->common.catchOSREntryBuffer;
1968 ASSERT(scratchBuffer);
1969 m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->addressOfActiveLength()));
1970 }
1971
1972 void compileGetStack()
1973 {
1974 StackAccessData* data = m_node->stackAccessData();
1975 AbstractValue& value = m_state.operand(data->local);
1976
1977 DFG_ASSERT(m_graph, m_node, isConcrete(data->format), data->format);
1978
1979 switch (data->format) {
1980 case FlushedDouble:
1981 setDouble(m_out.loadDouble(addressFor(data->machineLocal)));
1982 break;
1983 case FlushedInt52:
1984 setInt52(m_out.load64(addressFor(data->machineLocal)));
1985 break;
1986 default:
1987 if (isInt32Speculation(value.m_type))
1988 setInt32(m_out.load32(payloadFor(data->machineLocal)));
1989 else
1990 setJSValue(m_out.load64(addressFor(data->machineLocal)));
1991 break;
1992 }
1993 }
1994
1995 void compilePutStack()
1996 {
1997 StackAccessData* data = m_node->stackAccessData();
1998 switch (data->format) {
1999 case FlushedJSValue: {
2000 LValue value = lowJSValue(m_node->child1());
2001 m_out.store64(value, addressFor(data->machineLocal));
2002 break;
2003 }
2004
2005 case FlushedDouble: {
2006 LValue value = lowDouble(m_node->child1());
2007 m_out.storeDouble(value, addressFor(data->machineLocal));
2008 break;
2009 }
2010
2011 case FlushedInt32: {
2012 LValue value = lowInt32(m_node->child1());
2013 m_out.store32(value, payloadFor(data->machineLocal));
2014 break;
2015 }
2016
2017 case FlushedInt52: {
2018 LValue value = lowInt52(m_node->child1());
2019 m_out.store64(value, addressFor(data->machineLocal));
2020 break;
2021 }
2022
2023 case FlushedCell: {
2024 LValue value = lowCell(m_node->child1());
2025 m_out.store64(value, addressFor(data->machineLocal));
2026 break;
2027 }
2028
2029 case FlushedBoolean: {
2030 speculateBoolean(m_node->child1());
2031 m_out.store64(
2032 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2033 addressFor(data->machineLocal));
2034 break;
2035 }
2036
2037 default:
2038 DFG_CRASH(m_graph, m_node, "Bad flush format");
2039 break;
2040 }
2041 }
2042
2043 void compileNoOp()
2044 {
2045 DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
2046 }
2047
2048 void compileToObjectOrCallObjectConstructor()
2049 {
2050 LValue value = lowJSValue(m_node->child1());
2051
2052 LBasicBlock isCellCase = m_out.newBlock();
2053 LBasicBlock slowCase = m_out.newBlock();
2054 LBasicBlock continuation = m_out.newBlock();
2055
2056 m_out.branch(isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
2057
2058 LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
2059 ValueFromBlock fastResult = m_out.anchor(value);
2060 m_out.branch(isObject(value), usually(continuation), rarely(slowCase));
2061
2062 m_out.appendTo(slowCase, continuation);
2063
2064 ValueFromBlock slowResult;
2065 if (m_node->op() == ToObject) {
2066 auto* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2067 slowResult = m_out.anchor(vmCall(Int64, operationToObject, weakPointer(globalObject), value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2068 } else
2069 slowResult = m_out.anchor(vmCall(Int64, operationCallObjectConstructor, frozenPointer(m_node->cellOperand()), value));
2070 m_out.jump(continuation);
2071
2072 m_out.appendTo(continuation, lastNext);
2073 setJSValue(m_out.phi(Int64, fastResult, slowResult));
2074 }
2075
2076 void compileToThis()
2077 {
2078 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2079
2080 LValue value = lowJSValue(m_node->child1());
2081
2082 LBasicBlock isCellCase = m_out.newBlock();
2083 LBasicBlock slowCase = m_out.newBlock();
2084 LBasicBlock continuation = m_out.newBlock();
2085
2086 m_out.branch(
2087 isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
2088
2089 LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
2090 ValueFromBlock fastResult = m_out.anchor(value);
2091 m_out.branch(
2092 m_out.testIsZero32(
2093 m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags),
2094 m_out.constInt32(OverridesToThis)),
2095 usually(continuation), rarely(slowCase));
2096
2097 m_out.appendTo(slowCase, continuation);
2098 J_JITOperation_GJ function;
2099 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2100 function = operationToThisStrict;
2101 else
2102 function = operationToThis;
2103 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, function, weakPointer(globalObject), value));
2104 m_out.jump(continuation);
2105
2106 m_out.appendTo(continuation, lastNext);
2107 setJSValue(m_out.phi(Int64, fastResult, slowResult));
2108 }
2109
2110 void compileValueAdd()
2111 {
2112 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2113
2114 if (m_node->isBinaryUseKind(BigIntUse)) {
2115 LValue left = lowBigInt(m_node->child1());
2116 LValue right = lowBigInt(m_node->child2());
2117
2118 LValue result = vmCall(pointerType(), operationAddBigInt, weakPointer(globalObject), left, right);
2119 setJSValue(result);
2120 return;
2121 }
2122
2123 CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2124 BytecodeIndex bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2125 BinaryArithProfile* arithProfile = baselineCodeBlock->binaryArithProfileForBytecodeIndex(bytecodeIndex);
2126 auto repatchingFunction = operationValueAddOptimize;
2127 auto nonRepatchingFunction = operationValueAdd;
2128 compileBinaryMathIC<JITAddGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2129 }
2130
2131 void compileValueSub()
2132 {
2133 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2134
2135 if (m_node->isBinaryUseKind(BigIntUse)) {
2136 LValue left = lowBigInt(m_node->child1());
2137 LValue right = lowBigInt(m_node->child2());
2138
2139 LValue result = vmCall(pointerType(), operationSubBigInt, weakPointer(globalObject), left, right);
2140 setJSValue(result);
2141 return;
2142 }
2143
2144 CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2145 BytecodeIndex bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2146 BinaryArithProfile* arithProfile = baselineCodeBlock->binaryArithProfileForBytecodeIndex(bytecodeIndex);
2147 auto repatchingFunction = operationValueSubOptimize;
2148 auto nonRepatchingFunction = operationValueSub;
2149 compileBinaryMathIC<JITSubGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2150 }
2151
2152 void compileValueMul()
2153 {
2154 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2155
2156 if (m_node->isBinaryUseKind(BigIntUse)) {
2157 LValue left = lowBigInt(m_node->child1());
2158 LValue right = lowBigInt(m_node->child2());
2159
2160 LValue result = vmCall(Int64, operationMulBigInt, weakPointer(globalObject), left, right);
2161 setJSValue(result);
2162 return;
2163 }
2164
2165 CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2166 BytecodeIndex bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2167 BinaryArithProfile* arithProfile = baselineCodeBlock->binaryArithProfileForBytecodeIndex(bytecodeIndex);
2168 auto repatchingFunction = operationValueMulOptimize;
2169 auto nonRepatchingFunction = operationValueMul;
2170 compileBinaryMathIC<JITMulGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2171 }
2172
2173 template <typename Generator, typename Func1, typename Func2,
2174 typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func1>::type>::value && std::is_function<typename std::remove_pointer<Func2>::type>::value>>
2175 void compileUnaryMathIC(UnaryArithProfile* arithProfile, Func1 repatchingFunction, Func2 nonRepatchingFunction)
2176 {
2177 Node* node = m_node;
2178
2179 LValue operand = lowJSValue(node->child1());
2180
2181 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
2182 patchpoint->appendSomeRegister(operand);
2183 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
2184 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
2185 RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
2186 patchpoint->numGPScratchRegisters = 1;
2187 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2188 State* state = &m_ftlState;
2189 patchpoint->setGenerator(
2190 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2191 AllowMacroScratchRegisterUsage allowScratch(jit);
2192
2193 Box<CCallHelpers::JumpList> exceptions =
2194 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2195
2196#if ENABLE(MATH_IC_STATS)
2197 auto inlineStart = jit.label();
2198#endif
2199
2200 Box<MathICGenerationState> mathICGenerationState = Box<MathICGenerationState>::create();
2201 JITUnaryMathIC<Generator>* mathIC = jit.codeBlock()->addMathIC<Generator>(arithProfile);
2202 mathIC->m_generator = Generator(JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), params.gpScratch(0));
2203
2204 bool shouldEmitProfiling = false;
2205 bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling);
2206
2207 if (generatedInline) {
2208 ASSERT(!mathICGenerationState->slowPathJumps.empty());
2209 auto done = jit.label();
2210 params.addLatePath([=] (CCallHelpers& jit) {
2211 AllowMacroScratchRegisterUsage allowScratch(jit);
2212 mathICGenerationState->slowPathJumps.link(&jit);
2213 mathICGenerationState->slowPathStart = jit.label();
2214#if ENABLE(MATH_IC_STATS)
2215 auto slowPathStart = jit.label();
2216#endif
2217
2218 if (mathICGenerationState->shouldSlowPathRepatch) {
2219 SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2220 repatchingFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr(), CCallHelpers::TrustedImmPtr(mathIC));
2221 mathICGenerationState->slowPathCall = call.call();
2222 } else {
2223 SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic,
2224 exceptions.get(), nonRepatchingFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr());
2225 mathICGenerationState->slowPathCall = call.call();
2226 }
2227 jit.jump().linkTo(done, &jit);
2228
2229 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2230 mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer);
2231 });
2232
2233#if ENABLE(MATH_IC_STATS)
2234 auto slowPathEnd = jit.label();
2235 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2236 size_t size = linkBuffer.locationOf(slowPathEnd).executableAddress<char*>() - linkBuffer.locationOf(slowPathStart).executableAddress<char*>();
2237 mathIC->m_generatedCodeSize += size;
2238 });
2239#endif
2240 });
2241 } else {
2242 callOperation(
2243 *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2244 nonRepatchingFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr());
2245 }
2246
2247#if ENABLE(MATH_IC_STATS)
2248 auto inlineEnd = jit.label();
2249 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2250 size_t size = linkBuffer.locationOf(inlineEnd).executableAddress<char*>() - linkBuffer.locationOf(inlineStart).executableAddress<char*>();
2251 mathIC->m_generatedCodeSize += size;
2252 });
2253#endif
2254 });
2255
2256 setJSValue(patchpoint);
2257 }
2258
2259 template <typename Generator, typename Func1, typename Func2,
2260 typename = std::enable_if_t<std::is_function<typename std::remove_pointer<Func1>::type>::value && std::is_function<typename std::remove_pointer<Func2>::type>::value>>
2261 void compileBinaryMathIC(BinaryArithProfile* arithProfile, Func1 repatchingFunction, Func2 nonRepatchingFunction)
2262 {
2263 Node* node = m_node;
2264
2265 LValue left = lowJSValue(node->child1());
2266 LValue right = lowJSValue(node->child2());
2267
2268 SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType());
2269 SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType());
2270
2271 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
2272 patchpoint->appendSomeRegister(left);
2273 patchpoint->appendSomeRegister(right);
2274 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
2275 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
2276 RefPtr<PatchpointExceptionHandle> exceptionHandle =
2277 preparePatchpointForExceptions(patchpoint);
2278 patchpoint->numGPScratchRegisters = 1;
2279 patchpoint->numFPScratchRegisters = 2;
2280 patchpoint->clobber(RegisterSet::macroScratchRegisters());
2281 State* state = &m_ftlState;
2282 patchpoint->setGenerator(
2283 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
2284 AllowMacroScratchRegisterUsage allowScratch(jit);
2285
2286
2287 Box<CCallHelpers::JumpList> exceptions =
2288 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
2289
2290#if ENABLE(MATH_IC_STATS)
2291 auto inlineStart = jit.label();
2292#endif
2293
2294 Box<MathICGenerationState> mathICGenerationState = Box<MathICGenerationState>::create();
2295 JITBinaryMathIC<Generator>* mathIC = jit.codeBlock()->addMathIC<Generator>(arithProfile);
2296 mathIC->m_generator = Generator(leftOperand, rightOperand, JSValueRegs(params[0].gpr()),
2297 JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.fpScratch(0),
2298 params.fpScratch(1), params.gpScratch(0), InvalidFPRReg);
2299
2300 bool shouldEmitProfiling = false;
2301 bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling);
2302
2303 if (generatedInline) {
2304 ASSERT(!mathICGenerationState->slowPathJumps.empty());
2305 auto done = jit.label();
2306 params.addLatePath([=] (CCallHelpers& jit) {
2307 AllowMacroScratchRegisterUsage allowScratch(jit);
2308 mathICGenerationState->slowPathJumps.link(&jit);
2309 mathICGenerationState->slowPathStart = jit.label();
2310#if ENABLE(MATH_IC_STATS)
2311 auto slowPathStart = jit.label();
2312#endif
2313
2314 if (mathICGenerationState->shouldSlowPathRepatch) {
2315 SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2316 repatchingFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr(), params[2].gpr(), CCallHelpers::TrustedImmPtr(mathIC));
2317 mathICGenerationState->slowPathCall = call.call();
2318 } else {
2319 SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic,
2320 exceptions.get(), nonRepatchingFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr(), params[2].gpr());
2321 mathICGenerationState->slowPathCall = call.call();
2322 }
2323 jit.jump().linkTo(done, &jit);
2324
2325 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2326 mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer);
2327 });
2328
2329#if ENABLE(MATH_IC_STATS)
2330 auto slowPathEnd = jit.label();
2331 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2332 size_t size = linkBuffer.locationOf(slowPathEnd).executableAddress<char*>() - linkBuffer.locationOf(slowPathStart).executableAddress<char*>();
2333 mathIC->m_generatedCodeSize += size;
2334 });
2335#endif
2336 });
2337 } else {
2338 callOperation(
2339 *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(),
2340 nonRepatchingFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr(), params[2].gpr());
2341 }
2342
2343#if ENABLE(MATH_IC_STATS)
2344 auto inlineEnd = jit.label();
2345 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
2346 size_t size = linkBuffer.locationOf(inlineEnd).executableAddress<char*>() - linkBuffer.locationOf(inlineStart).executableAddress<char*>();
2347 mathIC->m_generatedCodeSize += size;
2348 });
2349#endif
2350 });
2351
2352 setJSValue(patchpoint);
2353 }
2354
2355 void compileStrCat()
2356 {
2357 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2358
2359 LValue result;
2360 if (m_node->child3()) {
2361 result = vmCall(
2362 Int64, operationStrCat3, weakPointer(globalObject),
2363 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2364 lowJSValue(m_node->child2(), ManualOperandSpeculation),
2365 lowJSValue(m_node->child3(), ManualOperandSpeculation));
2366 } else {
2367 result = vmCall(
2368 Int64, operationStrCat2, weakPointer(globalObject),
2369 lowJSValue(m_node->child1(), ManualOperandSpeculation),
2370 lowJSValue(m_node->child2(), ManualOperandSpeculation));
2371 }
2372 setJSValue(result);
2373 }
2374
2375 void compileArithAddOrSub()
2376 {
2377 bool isSub = m_node->op() == ArithSub;
2378 switch (m_node->binaryUseKind()) {
2379 case Int32Use: {
2380 LValue left = lowInt32(m_node->child1());
2381 LValue right = lowInt32(m_node->child2());
2382
2383 if (!shouldCheckOverflow(m_node->arithMode())) {
2384 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
2385 break;
2386 }
2387
2388 CheckValue* result =
2389 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
2390 blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2391 setInt32(result);
2392 break;
2393 }
2394
2395 case Int52RepUse: {
2396 if (!abstractValue(m_node->child1()).couldBeType(SpecNonInt32AsInt52)
2397 && !abstractValue(m_node->child2()).couldBeType(SpecNonInt32AsInt52)) {
2398 Int52Kind kind;
2399 LValue left = lowWhicheverInt52(m_node->child1(), kind);
2400 LValue right = lowInt52(m_node->child2(), kind);
2401 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
2402 break;
2403 }
2404
2405 LValue left = lowInt52(m_node->child1());
2406 LValue right = lowInt52(m_node->child2());
2407 CheckValue* result =
2408 isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right);
2409 blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2410 setInt52(result);
2411 break;
2412 }
2413
2414 case DoubleRepUse: {
2415 LValue C1 = lowDouble(m_node->child1());
2416 LValue C2 = lowDouble(m_node->child2());
2417
2418 setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
2419 break;
2420 }
2421
2422 case UntypedUse: {
2423 if (!isSub) {
2424 DFG_CRASH(m_graph, m_node, "Bad use kind");
2425 break;
2426 }
2427
2428 CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
2429 BytecodeIndex bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
2430 BinaryArithProfile* arithProfile = baselineCodeBlock->binaryArithProfileForBytecodeIndex(bytecodeIndex);
2431 auto repatchingFunction = operationValueSubOptimize;
2432 auto nonRepatchingFunction = operationValueSub;
2433 compileBinaryMathIC<JITSubGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
2434 break;
2435 }
2436
2437 default:
2438 DFG_CRASH(m_graph, m_node, "Bad use kind");
2439 break;
2440 }
2441 }
2442
2443 void compileArithClz32()
2444 {
2445 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2446 if (m_node->child1().useKind() == Int32Use || m_node->child1().useKind() == KnownInt32Use) {
2447 LValue operand = lowInt32(m_node->child1());
2448 setInt32(m_out.ctlz32(operand));
2449 return;
2450 }
2451 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2452 LValue argument = lowJSValue(m_node->child1());
2453 LValue result = vmCall(Int32, operationArithClz32, weakPointer(globalObject), argument);
2454 setInt32(result);
2455 }
2456
2457 void compileArithMul()
2458 {
2459 switch (m_node->binaryUseKind()) {
2460 case Int32Use: {
2461 LValue left = lowInt32(m_node->child1());
2462 LValue right = lowInt32(m_node->child2());
2463
2464 LValue result;
2465
2466 if (!shouldCheckOverflow(m_node->arithMode()))
2467 result = m_out.mul(left, right);
2468 else {
2469 CheckValue* speculation = m_out.speculateMul(left, right);
2470 blessSpeculation(speculation, Overflow, noValue(), nullptr, m_origin);
2471 result = speculation;
2472 }
2473
2474 if (shouldCheckNegativeZero(m_node->arithMode())) {
2475 LBasicBlock slowCase = m_out.newBlock();
2476 LBasicBlock continuation = m_out.newBlock();
2477
2478 m_out.branch(
2479 m_out.notZero32(result), usually(continuation), rarely(slowCase));
2480
2481 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
2482 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int32Zero));
2483 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int32Zero));
2484 m_out.jump(continuation);
2485 m_out.appendTo(continuation, lastNext);
2486 }
2487
2488 setInt32(result);
2489 break;
2490 }
2491
2492 case Int52RepUse: {
2493 Int52Kind kind;
2494 LValue left = lowWhicheverInt52(m_node->child1(), kind);
2495 LValue right = lowInt52(m_node->child2(), opposite(kind));
2496
2497 CheckValue* result = m_out.speculateMul(left, right);
2498 blessSpeculation(result, Overflow, noValue(), nullptr, m_origin);
2499
2500 if (shouldCheckNegativeZero(m_node->arithMode())) {
2501 LBasicBlock slowCase = m_out.newBlock();
2502 LBasicBlock continuation = m_out.newBlock();
2503
2504 m_out.branch(
2505 m_out.notZero64(result), usually(continuation), rarely(slowCase));
2506
2507 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
2508 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int64Zero));
2509 speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int64Zero));
2510 m_out.jump(continuation);
2511 m_out.appendTo(continuation, lastNext);
2512 }
2513
2514 setInt52(result);
2515 break;
2516 }
2517
2518 case DoubleRepUse: {
2519 setDouble(
2520 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2521 break;
2522 }
2523
2524 default:
2525 DFG_CRASH(m_graph, m_node, "Bad use kind");
2526 break;
2527 }
2528 }
2529
2530 void compileValueDiv()
2531 {
2532 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2533 if (m_node->isBinaryUseKind(BigIntUse)) {
2534 LValue left = lowBigInt(m_node->child1());
2535 LValue right = lowBigInt(m_node->child2());
2536
2537 LValue result = vmCall(pointerType(), operationDivBigInt, weakPointer(globalObject), left, right);
2538 setJSValue(result);
2539 return;
2540 }
2541
2542 emitBinarySnippet<JITDivGenerator, NeedScratchFPR>(operationValueDiv);
2543 }
2544
2545 void compileArithDiv()
2546 {
2547 switch (m_node->binaryUseKind()) {
2548 case Int32Use: {
2549 LValue numerator = lowInt32(m_node->child1());
2550 LValue denominator = lowInt32(m_node->child2());
2551
2552 if (shouldCheckNegativeZero(m_node->arithMode())) {
2553 LBasicBlock zeroNumerator = m_out.newBlock();
2554 LBasicBlock numeratorContinuation = m_out.newBlock();
2555
2556 m_out.branch(
2557 m_out.isZero32(numerator),
2558 rarely(zeroNumerator), usually(numeratorContinuation));
2559
2560 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
2561
2562 speculate(
2563 NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
2564
2565 m_out.jump(numeratorContinuation);
2566
2567 m_out.appendTo(numeratorContinuation, innerLastNext);
2568 }
2569
2570 if (shouldCheckOverflow(m_node->arithMode())) {
2571 LBasicBlock unsafeDenominator = m_out.newBlock();
2572 LBasicBlock continuation = m_out.newBlock();
2573
2574 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
2575 m_out.branch(
2576 m_out.above(adjustedDenominator, m_out.int32One),
2577 usually(continuation), rarely(unsafeDenominator));
2578
2579 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
2580 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
2581 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
2582 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
2583 m_out.jump(continuation);
2584
2585 m_out.appendTo(continuation, lastNext);
2586 LValue result = m_out.div(numerator, denominator);
2587 speculate(
2588 Overflow, noValue(), 0,
2589 m_out.notEqual(m_out.mul(result, denominator), numerator));
2590 setInt32(result);
2591 } else
2592 setInt32(m_out.chillDiv(numerator, denominator));
2593
2594 break;
2595 }
2596
2597 case DoubleRepUse: {
2598 setDouble(m_out.doubleDiv(
2599 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2600 break;
2601 }
2602
2603 default:
2604 DFG_CRASH(m_graph, m_node, "Bad use kind");
2605 break;
2606 }
2607 }
2608
2609 void compileValueMod()
2610 {
2611 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2612 if (m_node->binaryUseKind() == BigIntUse) {
2613 LValue left = lowBigInt(m_node->child1());
2614 LValue right = lowBigInt(m_node->child2());
2615
2616 LValue result = vmCall(pointerType(), operationModBigInt, weakPointer(globalObject), left, right);
2617 setJSValue(result);
2618 return;
2619 }
2620
2621 DFG_ASSERT(m_graph, m_node, m_node->binaryUseKind() == UntypedUse, m_node->binaryUseKind());
2622 LValue left = lowJSValue(m_node->child1());
2623 LValue right = lowJSValue(m_node->child2());
2624 LValue result = vmCall(Int64, operationValueMod, weakPointer(globalObject), left, right);
2625 setJSValue(result);
2626 }
2627
2628 void compileArithMod()
2629 {
2630 switch (m_node->binaryUseKind()) {
2631 case Int32Use: {
2632 LValue numerator = lowInt32(m_node->child1());
2633 LValue denominator = lowInt32(m_node->child2());
2634
2635 LValue remainder;
2636 if (shouldCheckOverflow(m_node->arithMode())) {
2637 LBasicBlock unsafeDenominator = m_out.newBlock();
2638 LBasicBlock continuation = m_out.newBlock();
2639
2640 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
2641 m_out.branch(
2642 m_out.above(adjustedDenominator, m_out.int32One),
2643 usually(continuation), rarely(unsafeDenominator));
2644
2645 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
2646 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
2647 speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator));
2648 speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31));
2649 m_out.jump(continuation);
2650
2651 m_out.appendTo(continuation, lastNext);
2652 LValue result = m_out.mod(numerator, denominator);
2653 remainder = result;
2654 } else
2655 remainder = m_out.chillMod(numerator, denominator);
2656
2657 if (shouldCheckNegativeZero(m_node->arithMode())) {
2658 LBasicBlock negativeNumerator = m_out.newBlock();
2659 LBasicBlock numeratorContinuation = m_out.newBlock();
2660
2661 m_out.branch(
2662 m_out.lessThan(numerator, m_out.int32Zero),
2663 unsure(negativeNumerator), unsure(numeratorContinuation));
2664
2665 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
2666
2667 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
2668
2669 m_out.jump(numeratorContinuation);
2670
2671 m_out.appendTo(numeratorContinuation, innerLastNext);
2672 }
2673
2674 setInt32(remainder);
2675 break;
2676 }
2677
2678 case DoubleRepUse: {
2679 setDouble(
2680 m_out.doubleMod(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
2681 break;
2682 }
2683
2684 default:
2685 DFG_CRASH(m_graph, m_node, "Bad use kind");
2686 break;
2687 }
2688 }
2689
2690 void compileArithMinOrMax()
2691 {
2692 switch (m_node->binaryUseKind()) {
2693 case Int32Use: {
2694 LValue left = lowInt32(m_node->child1());
2695 LValue right = lowInt32(m_node->child2());
2696
2697 setInt32(
2698 m_out.select(
2699 m_node->op() == ArithMin
2700 ? m_out.lessThan(left, right)
2701 : m_out.lessThan(right, left),
2702 left, right));
2703 break;
2704 }
2705
2706 case DoubleRepUse: {
2707 LValue left = lowDouble(m_node->child1());
2708 LValue right = lowDouble(m_node->child2());
2709
2710 LBasicBlock notLessThan = m_out.newBlock();
2711 LBasicBlock continuation = m_out.newBlock();
2712
2713 Vector<ValueFromBlock, 2> results;
2714
2715 results.append(m_out.anchor(left));
2716 m_out.branch(
2717 m_node->op() == ArithMin
2718 ? m_out.doubleLessThan(left, right)
2719 : m_out.doubleGreaterThan(left, right),
2720 unsure(continuation), unsure(notLessThan));
2721
2722 LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
2723 results.append(m_out.anchor(m_out.select(
2724 m_node->op() == ArithMin
2725 ? m_out.doubleGreaterThanOrEqual(left, right)
2726 : m_out.doubleLessThanOrEqual(left, right),
2727 right, m_out.constDouble(PNaN))));
2728 m_out.jump(continuation);
2729
2730 m_out.appendTo(continuation, lastNext);
2731 setDouble(m_out.phi(Double, results));
2732 break;
2733 }
2734
2735 default:
2736 DFG_CRASH(m_graph, m_node, "Bad use kind");
2737 break;
2738 }
2739 }
2740
2741 void compileArithAbs()
2742 {
2743 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2744 switch (m_node->child1().useKind()) {
2745 case Int32Use: {
2746 LValue value = lowInt32(m_node->child1());
2747
2748 LValue mask = m_out.aShr(value, m_out.constInt32(31));
2749 LValue result = m_out.bitXor(mask, m_out.add(mask, value));
2750
2751 if (shouldCheckOverflow(m_node->arithMode()))
2752 speculate(Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2753
2754 setInt32(result);
2755 break;
2756 }
2757
2758 case DoubleRepUse: {
2759 setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
2760 break;
2761 }
2762
2763 default: {
2764 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
2765 LValue argument = lowJSValue(m_node->child1());
2766 LValue result = vmCall(Double, operationArithAbs, weakPointer(globalObject), argument);
2767 setDouble(result);
2768 break;
2769 }
2770 }
2771 }
2772
2773 void compileArithUnary()
2774 {
2775 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2776 if (m_node->child1().useKind() == DoubleRepUse) {
2777 setDouble(m_out.doubleUnary(m_node->arithUnaryType(), lowDouble(m_node->child1())));
2778 return;
2779 }
2780 LValue argument = lowJSValue(m_node->child1());
2781 LValue result = vmCall(Double, DFG::arithUnaryOperation(m_node->arithUnaryType()), weakPointer(globalObject), argument);
2782 setDouble(result);
2783 }
2784
2785 void compileValuePow()
2786 {
2787 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2788 if (m_node->isBinaryUseKind(BigIntUse)) {
2789 LValue base = lowBigInt(m_node->child1());
2790 LValue exponent = lowBigInt(m_node->child2());
2791
2792 LValue result = vmCall(pointerType(), operationPowBigInt, weakPointer(globalObject), base, exponent);
2793 setJSValue(result);
2794 return;
2795 }
2796
2797 LValue base = lowJSValue(m_node->child1());
2798 LValue exponent = lowJSValue(m_node->child2());
2799 LValue result = vmCall(Int64, operationValuePow, weakPointer(globalObject), base, exponent);
2800 setJSValue(result);
2801 }
2802
2803 void compileArithPow()
2804 {
2805 if (m_node->child2().useKind() == Int32Use)
2806 setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
2807 else {
2808 LValue base = lowDouble(m_node->child1());
2809 LValue exponent = lowDouble(m_node->child2());
2810
2811 LBasicBlock integerExponentIsSmallBlock = m_out.newBlock();
2812 LBasicBlock integerExponentPowBlock = m_out.newBlock();
2813 LBasicBlock doubleExponentPowBlockEntry = m_out.newBlock();
2814 LBasicBlock nanExceptionBaseIsOne = m_out.newBlock();
2815 LBasicBlock nanExceptionExponentIsInfinity = m_out.newBlock();
2816 LBasicBlock testExponentIsOneHalf = m_out.newBlock();
2817 LBasicBlock handleBaseZeroExponentIsOneHalf = m_out.newBlock();
2818 LBasicBlock handleInfinityForExponentIsOneHalf = m_out.newBlock();
2819 LBasicBlock exponentIsOneHalfNormal = m_out.newBlock();
2820 LBasicBlock exponentIsOneHalfInfinity = m_out.newBlock();
2821 LBasicBlock testExponentIsNegativeOneHalf = m_out.newBlock();
2822 LBasicBlock testBaseZeroExponentIsNegativeOneHalf = m_out.newBlock();
2823 LBasicBlock handleBaseZeroExponentIsNegativeOneHalf = m_out.newBlock();
2824 LBasicBlock handleInfinityForExponentIsNegativeOneHalf = m_out.newBlock();
2825 LBasicBlock exponentIsNegativeOneHalfNormal = m_out.newBlock();
2826 LBasicBlock exponentIsNegativeOneHalfInfinity = m_out.newBlock();
2827 LBasicBlock powBlock = m_out.newBlock();
2828 LBasicBlock nanExceptionResultIsNaN = m_out.newBlock();
2829 LBasicBlock continuation = m_out.newBlock();
2830
2831 LValue integerExponent = m_out.doubleToInt(exponent);
2832 LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
2833 LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
2834 m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
2835
2836 LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
2837 LValue integerExponentBelowMax = m_out.belowOrEqual(integerExponent, m_out.constInt32(maxExponentForIntegerMathPow));
2838 m_out.branch(integerExponentBelowMax, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
2839
2840 m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
2841 ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
2842 m_out.jump(continuation);
2843
2844 // If y is NaN, the result is NaN.
2845 m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionBaseIsOne);
2846 LValue exponentIsNaN;
2847 if (provenType(m_node->child2()) & SpecDoubleNaN)
2848 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
2849 else
2850 exponentIsNaN = m_out.booleanFalse;
2851 m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionBaseIsOne));
2852
2853 // If abs(x) is 1 and y is +infinity, the result is NaN.
2854 // If abs(x) is 1 and y is -infinity, the result is NaN.
2855
2856 // Test if base == 1.
2857 m_out.appendTo(nanExceptionBaseIsOne, nanExceptionExponentIsInfinity);
2858 LValue absoluteBase = m_out.doubleAbs(base);
2859 LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
2860 m_out.branch(absoluteBaseIsOne, rarely(nanExceptionExponentIsInfinity), usually(testExponentIsOneHalf));
2861
2862 // Test if abs(y) == Infinity.
2863 m_out.appendTo(nanExceptionExponentIsInfinity, testExponentIsOneHalf);
2864 LValue absoluteExponent = m_out.doubleAbs(exponent);
2865 LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
2866 m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionResultIsNaN), usually(testExponentIsOneHalf));
2867
2868 // If y == 0.5 or y == -0.5, handle it through SQRT.
2869 // We have be carefuly with -0 and -Infinity.
2870
2871 // Test if y == 0.5
2872 m_out.appendTo(testExponentIsOneHalf, handleBaseZeroExponentIsOneHalf);
2873 LValue exponentIsOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(0.5));
2874 m_out.branch(exponentIsOneHalf, rarely(handleBaseZeroExponentIsOneHalf), usually(testExponentIsNegativeOneHalf));
2875
2876 // Handle x == -0.
2877 m_out.appendTo(handleBaseZeroExponentIsOneHalf, handleInfinityForExponentIsOneHalf);
2878 LValue baseIsZeroExponentIsOneHalf = m_out.doubleEqual(base, m_out.doubleZero);
2879 ValueFromBlock zeroResultExponentIsOneHalf = m_out.anchor(m_out.doubleZero);
2880 m_out.branch(baseIsZeroExponentIsOneHalf, rarely(continuation), usually(handleInfinityForExponentIsOneHalf));
2881
2882 // Test if abs(x) == Infinity.
2883 m_out.appendTo(handleInfinityForExponentIsOneHalf, exponentIsOneHalfNormal);
2884 LValue absoluteBaseIsInfinityOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits<double>::infinity()));
2885 m_out.branch(absoluteBaseIsInfinityOneHalf, rarely(exponentIsOneHalfInfinity), usually(exponentIsOneHalfNormal));
2886
2887 // The exponent is 0.5, the base is finite or NaN, we can use SQRT.
2888 m_out.appendTo(exponentIsOneHalfNormal, exponentIsOneHalfInfinity);
2889 ValueFromBlock sqrtResult = m_out.anchor(m_out.doubleSqrt(base));
2890 m_out.jump(continuation);
2891
2892 // The exponent is 0.5, the base is infinite, the result is always infinite.
2893 m_out.appendTo(exponentIsOneHalfInfinity, testExponentIsNegativeOneHalf);
2894 ValueFromBlock sqrtInfinityResult = m_out.anchor(m_out.constDouble(std::numeric_limits<double>::infinity()));
2895 m_out.jump(continuation);
2896
2897 // Test if y == -0.5
2898 m_out.appendTo(testExponentIsNegativeOneHalf, testBaseZeroExponentIsNegativeOneHalf);
2899 LValue exponentIsNegativeOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(-0.5));
2900 m_out.branch(exponentIsNegativeOneHalf, rarely(testBaseZeroExponentIsNegativeOneHalf), usually(powBlock));
2901
2902 // Handle x == -0.
2903 m_out.appendTo(testBaseZeroExponentIsNegativeOneHalf, handleBaseZeroExponentIsNegativeOneHalf);
2904 LValue baseIsZeroExponentIsNegativeOneHalf = m_out.doubleEqual(base, m_out.doubleZero);
2905 m_out.branch(baseIsZeroExponentIsNegativeOneHalf, rarely(handleBaseZeroExponentIsNegativeOneHalf), usually(handleInfinityForExponentIsNegativeOneHalf));
2906
2907 m_out.appendTo(handleBaseZeroExponentIsNegativeOneHalf, handleInfinityForExponentIsNegativeOneHalf);
2908 ValueFromBlock oneOverSqrtZeroResult = m_out.anchor(m_out.constDouble(std::numeric_limits<double>::infinity()));
2909 m_out.jump(continuation);
2910
2911 // Test if abs(x) == Infinity.
2912 m_out.appendTo(handleInfinityForExponentIsNegativeOneHalf, exponentIsNegativeOneHalfNormal);
2913 LValue absoluteBaseIsInfinityNegativeOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits<double>::infinity()));
2914 m_out.branch(absoluteBaseIsInfinityNegativeOneHalf, rarely(exponentIsNegativeOneHalfInfinity), usually(exponentIsNegativeOneHalfNormal));
2915
2916 // The exponent is -0.5, the base is finite or NaN, we can use 1/SQRT.
2917 m_out.appendTo(exponentIsNegativeOneHalfNormal, exponentIsNegativeOneHalfInfinity);
2918 LValue sqrtBase = m_out.doubleSqrt(base);
2919 ValueFromBlock oneOverSqrtResult = m_out.anchor(m_out.div(m_out.constDouble(1.), sqrtBase));
2920 m_out.jump(continuation);
2921
2922 // The exponent is -0.5, the base is infinite, the result is always zero.
2923 m_out.appendTo(exponentIsNegativeOneHalfInfinity, powBlock);
2924 ValueFromBlock oneOverSqrtInfinityResult = m_out.anchor(m_out.doubleZero);
2925 m_out.jump(continuation);
2926
2927 m_out.appendTo(powBlock, nanExceptionResultIsNaN);
2928 ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
2929 m_out.jump(continuation);
2930
2931 m_out.appendTo(nanExceptionResultIsNaN, continuation);
2932 ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
2933 m_out.jump(continuation);
2934
2935 m_out.appendTo(continuation, lastNext);
2936 setDouble(m_out.phi(Double, powDoubleIntResult, zeroResultExponentIsOneHalf, sqrtResult, sqrtInfinityResult, oneOverSqrtZeroResult, oneOverSqrtResult, oneOverSqrtInfinityResult, powResult, pureNan));
2937 }
2938 }
2939
2940 void compileArithRandom()
2941 {
2942 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2943
2944 // Inlined WeakRandom::advance().
2945 // uint64_t x = m_low;
2946 void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
2947 LValue low = m_out.load64(m_out.absolute(lowAddress));
2948 // uint64_t y = m_high;
2949 void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();
2950 LValue high = m_out.load64(m_out.absolute(highAddress));
2951 // m_low = y;
2952 m_out.store64(high, m_out.absolute(lowAddress));
2953
2954 // x ^= x << 23;
2955 LValue phase1 = m_out.bitXor(m_out.shl(low, m_out.constInt64(23)), low);
2956
2957 // x ^= x >> 17;
2958 LValue phase2 = m_out.bitXor(m_out.lShr(phase1, m_out.constInt64(17)), phase1);
2959
2960 // x ^= y ^ (y >> 26);
2961 LValue phase3 = m_out.bitXor(m_out.bitXor(high, m_out.lShr(high, m_out.constInt64(26))), phase2);
2962
2963 // m_high = x;
2964 m_out.store64(phase3, m_out.absolute(highAddress));
2965
2966 // return x + y;
2967 LValue random64 = m_out.add(phase3, high);
2968
2969 // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
2970 LValue random53 = m_out.bitAnd(random64, m_out.constInt64((1ULL << 53) - 1));
2971
2972 LValue double53Integer = m_out.intToDouble(random53);
2973
2974 // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
2975 // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
2976 static constexpr double scale = 1.0 / (1ULL << 53);
2977
2978 // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
2979 // It just reduces the exp part of the given 53bit double integer.
2980 // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
2981 // Now we get 53bit precision random double value in [0, 1).
2982 LValue result = m_out.doubleMul(double53Integer, m_out.constDouble(scale));
2983
2984 setDouble(result);
2985 }
2986
2987 void compileArithRound()
2988 {
2989 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
2990 if (m_node->child1().useKind() == DoubleRepUse) {
2991 LValue result = nullptr;
2992 if (producesInteger(m_node->arithRoundingMode()) && !shouldCheckNegativeZero(m_node->arithRoundingMode())) {
2993 LValue value = lowDouble(m_node->child1());
2994 result = m_out.doubleFloor(m_out.doubleAdd(value, m_out.constDouble(0.5)));
2995 } else {
2996 LBasicBlock shouldRoundDown = m_out.newBlock();
2997 LBasicBlock continuation = m_out.newBlock();
2998
2999 LValue value = lowDouble(m_node->child1());
3000 LValue integerValue = m_out.doubleCeil(value);
3001 ValueFromBlock integerValueResult = m_out.anchor(integerValue);
3002
3003 LValue ceilMinusHalf = m_out.doubleSub(integerValue, m_out.constDouble(0.5));
3004 m_out.branch(m_out.doubleGreaterThanOrUnordered(ceilMinusHalf, value), unsure(shouldRoundDown), unsure(continuation));
3005
3006 LBasicBlock lastNext = m_out.appendTo(shouldRoundDown, continuation);
3007 LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
3008 ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
3009 m_out.jump(continuation);
3010 m_out.appendTo(continuation, lastNext);
3011
3012 result = m_out.phi(Double, integerValueResult, integerValueRoundedDownResult);
3013 }
3014
3015 if (producesInteger(m_node->arithRoundingMode())) {
3016 LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
3017 setInt32(integerValue);
3018 } else
3019 setDouble(result);
3020 return;
3021 }
3022
3023 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
3024 LValue argument = lowJSValue(m_node->child1());
3025 setJSValue(vmCall(Int64, operationArithRound, weakPointer(globalObject), argument));
3026 }
3027
3028 void compileArithFloor()
3029 {
3030 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3031 if (m_node->child1().useKind() == DoubleRepUse) {
3032 LValue value = lowDouble(m_node->child1());
3033 LValue integerValue = m_out.doubleFloor(value);
3034 if (producesInteger(m_node->arithRoundingMode()))
3035 setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
3036 else
3037 setDouble(integerValue);
3038 return;
3039 }
3040 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
3041 LValue argument = lowJSValue(m_node->child1());
3042 setJSValue(vmCall(Int64, operationArithFloor, weakPointer(globalObject), argument));
3043 }
3044
3045 void compileArithCeil()
3046 {
3047 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3048 if (m_node->child1().useKind() == DoubleRepUse) {
3049 LValue value = lowDouble(m_node->child1());
3050 LValue integerValue = m_out.doubleCeil(value);
3051 if (producesInteger(m_node->arithRoundingMode()))
3052 setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode())));
3053 else
3054 setDouble(integerValue);
3055 return;
3056 }
3057 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
3058 LValue argument = lowJSValue(m_node->child1());
3059 setJSValue(vmCall(Int64, operationArithCeil, weakPointer(globalObject), argument));
3060 }
3061
3062 void compileArithTrunc()
3063 {
3064 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3065 if (m_node->child1().useKind() == DoubleRepUse) {
3066 LValue value = lowDouble(m_node->child1());
3067 LValue result = m_out.doubleTrunc(value);
3068 if (producesInteger(m_node->arithRoundingMode()))
3069 setInt32(convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())));
3070 else
3071 setDouble(result);
3072 return;
3073 }
3074 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse, m_node->child1().useKind());
3075 LValue argument = lowJSValue(m_node->child1());
3076 setJSValue(vmCall(Int64, operationArithTrunc, weakPointer(globalObject), argument));
3077 }
3078
3079 void compileArithSqrt()
3080 {
3081 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3082 if (m_node->child1().useKind() == DoubleRepUse) {
3083 setDouble(m_out.doubleSqrt(lowDouble(m_node->child1())));
3084 return;
3085 }
3086 LValue argument = lowJSValue(m_node->child1());
3087 LValue result = vmCall(Double, operationArithSqrt, weakPointer(globalObject), argument);
3088 setDouble(result);
3089 }
3090
3091 void compileArithFRound()
3092 {
3093 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3094 if (m_node->child1().useKind() == DoubleRepUse) {
3095 setDouble(m_out.fround(lowDouble(m_node->child1())));
3096 return;
3097 }
3098 LValue argument = lowJSValue(m_node->child1());
3099 LValue result = vmCall(Double, operationArithFRound, weakPointer(globalObject), argument);
3100 setDouble(result);
3101 }
3102
3103 void compileIncOrDec()
3104 {
3105 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse);
3106 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3107 LValue operand = lowJSValue(m_node->child1());
3108 LValue result = vmCall(Int64, m_node->op() == Inc ? operationInc : operationDec, weakPointer(globalObject), operand);
3109 setJSValue(result);
3110 }
3111
3112 void compileValueNegate()
3113 {
3114 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse);
3115 CodeBlock* baselineCodeBlock = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic);
3116 BytecodeIndex bytecodeIndex = m_node->origin.semantic.bytecodeIndex();
3117 UnaryArithProfile* arithProfile = baselineCodeBlock->unaryArithProfileForBytecodeIndex(bytecodeIndex);
3118 auto repatchingFunction = operationArithNegateOptimize;
3119 auto nonRepatchingFunction = operationArithNegate;
3120 compileUnaryMathIC<JITNegGenerator>(arithProfile, repatchingFunction, nonRepatchingFunction);
3121 }
3122
3123 void compileArithNegate()
3124 {
3125 switch (m_node->child1().useKind()) {
3126 case Int32Use: {
3127 LValue value = lowInt32(m_node->child1());
3128
3129 LValue result;
3130 if (!shouldCheckOverflow(m_node->arithMode()))
3131 result = m_out.neg(value);
3132 else if (!shouldCheckNegativeZero(m_node->arithMode())) {
3133 CheckValue* check = m_out.speculateSub(m_out.int32Zero, value);
3134 blessSpeculation(check, Overflow, noValue(), nullptr, m_origin);
3135 result = check;
3136 } else {
3137 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
3138 result = m_out.neg(value);
3139 }
3140
3141 setInt32(result);
3142 break;
3143 }
3144
3145 case Int52RepUse: {
3146 if (!abstractValue(m_node->child1()).couldBeType(SpecNonInt32AsInt52)) {
3147 Int52Kind kind;
3148 LValue value = lowWhicheverInt52(m_node->child1(), kind);
3149 LValue result = m_out.neg(value);
3150 if (shouldCheckNegativeZero(m_node->arithMode()))
3151 speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
3152 setInt52(result, kind);
3153 break;
3154 }
3155
3156 LValue value = lowInt52(m_node->child1());
3157 CheckValue* result = m_out.speculateSub(m_out.int64Zero, value);
3158 blessSpeculation(result, Int52Overflow, noValue(), nullptr, m_origin);
3159 if (shouldCheckNegativeZero(m_node->arithMode()))
3160 speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
3161 setInt52(result);
3162 break;
3163 }
3164
3165 case DoubleRepUse: {
3166 setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
3167 break;
3168 }
3169
3170 default:
3171 DFG_CRASH(m_graph, m_node, "Bad use kind");
3172 break;
3173 }
3174 }
3175
3176 void compileValueBitNot()
3177 {
3178 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3179 if (m_node->child1().useKind() == BigIntUse) {
3180 LValue operand = lowBigInt(m_node->child1());
3181 LValue result = vmCall(pointerType(), operationBitNotBigInt, weakPointer(globalObject), operand);
3182 setJSValue(result);
3183 return;
3184 }
3185
3186 LValue operand = lowJSValue(m_node->child1());
3187 LValue result = vmCall(Int64, operationValueBitNot, weakPointer(globalObject), operand);
3188 setJSValue(result);
3189 }
3190
3191 void compileArithBitNot()
3192 {
3193 setInt32(m_out.bitNot(lowInt32(m_node->child1())));
3194 }
3195
3196 void compileValueBitAnd()
3197 {
3198 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3199 if (m_node->isBinaryUseKind(BigIntUse)) {
3200 LValue left = lowBigInt(m_node->child1());
3201 LValue right = lowBigInt(m_node->child2());
3202
3203 LValue result = vmCall(pointerType(), operationBitAndBigInt, weakPointer(globalObject), left, right);
3204 setJSValue(result);
3205 return;
3206 }
3207
3208 emitBinaryBitOpSnippet<JITBitAndGenerator>(operationValueBitAnd);
3209 }
3210
3211 void compileArithBitAnd()
3212 {
3213 setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3214 }
3215
3216 void compileValueBitOr()
3217 {
3218 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3219 if (m_node->isBinaryUseKind(BigIntUse)) {
3220 LValue left = lowBigInt(m_node->child1());
3221 LValue right = lowBigInt(m_node->child2());
3222
3223 LValue result = vmCall(pointerType(), operationBitOrBigInt, weakPointer(globalObject), left, right);
3224 setJSValue(result);
3225 return;
3226 }
3227
3228 emitBinaryBitOpSnippet<JITBitOrGenerator>(operationValueBitOr);
3229 }
3230
3231 void compileArithBitOr()
3232 {
3233 setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3234 }
3235
3236 void compileValueBitXor()
3237 {
3238 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3239 if (m_node->isBinaryUseKind(BigIntUse)) {
3240 LValue left = lowBigInt(m_node->child1());
3241 LValue right = lowBigInt(m_node->child2());
3242
3243 LValue result = vmCall(pointerType(), operationBitXorBigInt, weakPointer(globalObject), left, right);
3244 setJSValue(result);
3245 return;
3246 }
3247
3248 emitBinaryBitOpSnippet<JITBitXorGenerator>(operationValueBitXor);
3249 }
3250
3251 void compileArithBitXor()
3252 {
3253 setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
3254 }
3255
3256 void compileValueBitRShift()
3257 {
3258 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3259 if (m_node->isBinaryUseKind(BigIntUse)) {
3260 LValue left = lowBigInt(m_node->child1());
3261 LValue right = lowBigInt(m_node->child2());
3262
3263 LValue result = vmCall(pointerType(), operationBitRShiftBigInt, weakPointer(globalObject), left, right);
3264 setJSValue(result);
3265 return;
3266 }
3267
3268 emitRightShiftSnippet(JITRightShiftGenerator::SignedShift);
3269 }
3270
3271 void compileArithBitRShift()
3272 {
3273 setInt32(m_out.aShr(
3274 lowInt32(m_node->child1()),
3275 m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3276 }
3277
3278 void compileArithBitLShift()
3279 {
3280 setInt32(m_out.shl(
3281 lowInt32(m_node->child1()),
3282 m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3283 }
3284
3285 void compileValueBitLShift()
3286 {
3287 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3288 if (m_node->isBinaryUseKind(BigIntUse)) {
3289 LValue left = lowBigInt(m_node->child1());
3290 LValue right = lowBigInt(m_node->child2());
3291
3292 LValue result = vmCall(pointerType(), operationBitLShiftBigInt, weakPointer(globalObject), left, right);
3293 setJSValue(result);
3294 return;
3295 }
3296
3297 ASSERT(m_node->isBinaryUseKind(UntypedUse));
3298 emitBinaryBitOpSnippet<JITLeftShiftGenerator>(operationValueBitLShift);
3299 }
3300
3301 void compileBitURShift()
3302 {
3303 if (m_node->isBinaryUseKind(UntypedUse)) {
3304 emitRightShiftSnippet(JITRightShiftGenerator::UnsignedShift);
3305 return;
3306 }
3307 setInt32(m_out.lShr(
3308 lowInt32(m_node->child1()),
3309 m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
3310 }
3311
3312 void compileUInt32ToNumber()
3313 {
3314 LValue value = lowInt32(m_node->child1());
3315
3316 if (doesOverflow(m_node->arithMode())) {
3317 setStrictInt52(m_out.zeroExtPtr(value));
3318 return;
3319 }
3320
3321 speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
3322 setInt32(value);
3323 }
3324
3325 void compileCheckStructure()
3326 {
3327 ExitKind exitKind;
3328 if (m_node->child1()->hasConstant())
3329 exitKind = BadConstantCache;
3330 else
3331 exitKind = BadCache;
3332
3333 switch (m_node->child1().useKind()) {
3334 case CellUse:
3335 case KnownCellUse: {
3336 LValue cell = lowCell(m_node->child1());
3337
3338 checkStructure(
3339 m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
3340 exitKind, m_node->structureSet(),
3341 [&] (RegisteredStructure structure) {
3342 return weakStructureID(structure);
3343 });
3344 return;
3345 }
3346
3347 case CellOrOtherUse: {
3348 LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
3349
3350 LBasicBlock cellCase = m_out.newBlock();
3351 LBasicBlock notCellCase = m_out.newBlock();
3352 LBasicBlock continuation = m_out.newBlock();
3353
3354 m_out.branch(
3355 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
3356
3357 LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
3358 checkStructure(
3359 m_out.load32(value, m_heaps.JSCell_structureID), jsValueValue(value),
3360 exitKind, m_node->structureSet(),
3361 [&] (RegisteredStructure structure) {
3362 return weakStructureID(structure);
3363 });
3364 m_out.jump(continuation);
3365
3366 m_out.appendTo(notCellCase, continuation);
3367 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecCell | SpecOther, isNotOther(value));
3368 m_out.jump(continuation);
3369
3370 m_out.appendTo(continuation, lastNext);
3371 return;
3372 }
3373
3374 default:
3375 DFG_CRASH(m_graph, m_node, "Bad use kind");
3376 return;
3377 }
3378 }
3379
3380 void compileCheckStructureOrEmpty()
3381 {
3382 ExitKind exitKind;
3383 if (m_node->child1()->hasConstant())
3384 exitKind = BadConstantCache;
3385 else
3386 exitKind = BadCache;
3387
3388 LValue cell = lowCell(m_node->child1());
3389 bool maySeeEmptyValue = m_interpreter.forNode(m_node->child1()).m_type & SpecEmpty;
3390 LBasicBlock notEmpty;
3391 LBasicBlock continuation;
3392 LBasicBlock lastNext;
3393 if (maySeeEmptyValue) {
3394 notEmpty = m_out.newBlock();
3395 continuation = m_out.newBlock();
3396 m_out.branch(m_out.isZero64(cell), unsure(continuation), unsure(notEmpty));
3397 lastNext = m_out.appendTo(notEmpty, continuation);
3398 }
3399
3400 checkStructure(
3401 m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell),
3402 exitKind, m_node->structureSet(),
3403 [&] (RegisteredStructure structure) {
3404 return weakStructureID(structure);
3405 });
3406
3407 if (maySeeEmptyValue) {
3408 m_out.jump(continuation);
3409 m_out.appendTo(continuation, lastNext);
3410 }
3411 }
3412
3413 void compileCheckCell()
3414 {
3415 LValue cell = lowCell(m_node->child1());
3416
3417 speculate(
3418 BadCell, jsValueValue(cell), m_node->child1().node(),
3419 m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
3420 }
3421
3422 void compileCheckBadCell()
3423 {
3424 terminate(BadCell);
3425 }
3426
3427 void compileCheckNotEmpty()
3428 {
3429 speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
3430 }
3431
3432 void compileAssertNotEmpty()
3433 {
3434 if (!validationEnabled())
3435 return;
3436
3437 LValue val = lowJSValue(m_node->child1());
3438 PatchpointValue* patchpoint = m_out.patchpoint(Void);
3439 patchpoint->appendSomeRegister(val);
3440 patchpoint->setGenerator(
3441 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
3442 AllowMacroScratchRegisterUsage allowScratch(jit);
3443 GPRReg input = params[0].gpr();
3444 CCallHelpers::Jump done = jit.branchIfNotEmpty(input);
3445 jit.breakpoint();
3446 done.link(&jit);
3447 });
3448 }
3449
3450 void compileCheckIdent()
3451 {
3452 UniquedStringImpl* uid = m_node->uidOperand();
3453 LValue stringImpl;
3454 if (m_node->child1().useKind() == StringIdentUse)
3455 stringImpl = lowStringIdent(m_node->child1());
3456 else {
3457 ASSERT(m_node->child1().useKind() == SymbolUse);
3458 stringImpl = m_out.loadPtr(lowSymbol(m_node->child1()), m_heaps.Symbol_symbolImpl);
3459 }
3460 speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid)));
3461 }
3462
3463 void compileGetExecutable()
3464 {
3465 LValue cell = lowCell(m_node->child1());
3466 speculateFunction(m_node->child1(), cell);
3467 setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
3468 }
3469
3470 void compileArrayify()
3471 {
3472 LValue cell = lowCell(m_node->child1());
3473 LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
3474
3475 LBasicBlock unexpectedStructure = m_out.newBlock();
3476 LBasicBlock continuation = m_out.newBlock();
3477
3478 auto isUnexpectedArray = [&] (LValue cell) {
3479 if (m_node->op() == Arrayify)
3480 return m_out.logicalNot(isArrayTypeForArrayify(cell, m_node->arrayMode()));
3481
3482 ASSERT(m_node->op() == ArrayifyToStructure);
3483 return m_out.notEqual(m_out.load32(cell, m_heaps.JSCell_structureID), weakStructureID(m_node->structure()));
3484 };
3485
3486 m_out.branch(isUnexpectedArray(cell), rarely(unexpectedStructure), usually(continuation));
3487
3488 LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
3489
3490 if (property) {
3491 switch (m_node->arrayMode().type()) {
3492 case Array::Int32:
3493 case Array::Double:
3494 case Array::Contiguous:
3495 speculate(
3496 Uncountable, noValue(), 0,
3497 m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
3498 break;
3499 default:
3500 break;
3501 }
3502 }
3503
3504 switch (m_node->arrayMode().type()) {
3505 case Array::Int32:
3506 vmCall(Void, operationEnsureInt32, m_vmValue, cell);
3507 break;
3508 case Array::Double:
3509 vmCall(Void, operationEnsureDouble, m_vmValue, cell);
3510 break;
3511 case Array::Contiguous:
3512 vmCall(Void, operationEnsureContiguous, m_vmValue, cell);
3513 break;
3514 case Array::ArrayStorage:
3515 case Array::SlowPutArrayStorage:
3516 vmCall(Void, operationEnsureArrayStorage, m_vmValue, cell);
3517 break;
3518 default:
3519 DFG_CRASH(m_graph, m_node, "Bad array type");
3520 break;
3521 }
3522
3523 speculate(BadIndexingType, jsValueValue(cell), 0, isUnexpectedArray(cell));
3524 m_out.jump(continuation);
3525
3526 m_out.appendTo(continuation, lastNext);
3527 }
3528
3529 void compilePutStructure()
3530 {
3531 m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
3532
3533 RegisteredStructure oldStructure = m_node->transition()->previous;
3534 RegisteredStructure newStructure = m_node->transition()->next;
3535 ASSERT_UNUSED(oldStructure, oldStructure->indexingMode() == newStructure->indexingMode());
3536 ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
3537 ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
3538
3539 LValue cell = lowCell(m_node->child1());
3540 m_out.store32(
3541 weakStructureID(newStructure),
3542 cell, m_heaps.JSCell_structureID);
3543 }
3544
3545 void compileGetById(AccessType type)
3546 {
3547 ASSERT(type == AccessType::GetById || type == AccessType::TryGetById || type == AccessType::GetByIdDirect);
3548 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3549 switch (m_node->child1().useKind()) {
3550 case CellUse: {
3551 setJSValue(getById(lowCell(m_node->child1()), type));
3552 return;
3553 }
3554
3555 case UntypedUse: {
3556 // This is pretty weird, since we duplicate the slow path both here and in the
3557 // code generated by the IC. We should investigate making this less bad.
3558 // https://bugs.webkit.org/show_bug.cgi?id=127830
3559 LValue value = lowJSValue(m_node->child1());
3560
3561 LBasicBlock cellCase = m_out.newBlock();
3562 LBasicBlock notCellCase = m_out.newBlock();
3563 LBasicBlock continuation = m_out.newBlock();
3564
3565 m_out.branch(
3566 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
3567
3568 LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
3569 ValueFromBlock cellResult = m_out.anchor(getById(value, type));
3570 m_out.jump(continuation);
3571
3572 J_JITOperation_GJI getByIdFunction = appropriateGenericGetByIdFunction(type);
3573
3574 m_out.appendTo(notCellCase, continuation);
3575 ValueFromBlock notCellResult = m_out.anchor(vmCall(
3576 Int64, getByIdFunction,
3577 weakPointer(globalObject), value,
3578 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
3579 m_out.jump(continuation);
3580
3581 m_out.appendTo(continuation, lastNext);
3582 setJSValue(m_out.phi(Int64, cellResult, notCellResult));
3583 return;
3584 }
3585
3586 default:
3587 DFG_CRASH(m_graph, m_node, "Bad use kind");
3588 return;
3589 }
3590 }
3591
3592 void compileGetByIdWithThis()
3593 {
3594 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3595 if (m_node->child1().useKind() == CellUse && m_node->child2().useKind() == CellUse)
3596 setJSValue(getByIdWithThis(lowCell(m_node->child1()), lowCell(m_node->child2())));
3597 else {
3598 LValue base = lowJSValue(m_node->child1());
3599 LValue thisValue = lowJSValue(m_node->child2());
3600
3601 LBasicBlock baseCellCase = m_out.newBlock();
3602 LBasicBlock notCellCase = m_out.newBlock();
3603 LBasicBlock thisValueCellCase = m_out.newBlock();
3604 LBasicBlock continuation = m_out.newBlock();
3605
3606 m_out.branch(
3607 isCell(base, provenType(m_node->child1())), unsure(baseCellCase), unsure(notCellCase));
3608
3609 LBasicBlock lastNext = m_out.appendTo(baseCellCase, thisValueCellCase);
3610
3611 m_out.branch(
3612 isCell(thisValue, provenType(m_node->child2())), unsure(thisValueCellCase), unsure(notCellCase));
3613
3614 m_out.appendTo(thisValueCellCase, notCellCase);
3615 ValueFromBlock cellResult = m_out.anchor(getByIdWithThis(base, thisValue));
3616 m_out.jump(continuation);
3617
3618 m_out.appendTo(notCellCase, continuation);
3619 ValueFromBlock notCellResult = m_out.anchor(vmCall(
3620 Int64, operationGetByIdWithThisGeneric,
3621 weakPointer(globalObject), base, thisValue,
3622 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
3623 m_out.jump(continuation);
3624
3625 m_out.appendTo(continuation, lastNext);
3626 setJSValue(m_out.phi(Int64, cellResult, notCellResult));
3627 }
3628
3629 }
3630
3631 void compileGetByValWithThis()
3632 {
3633 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3634 LValue base = lowJSValue(m_node->child1());
3635 LValue thisValue = lowJSValue(m_node->child2());
3636 LValue subscript = lowJSValue(m_node->child3());
3637
3638 LValue result = vmCall(Int64, operationGetByValWithThis, weakPointer(globalObject), base, thisValue, subscript);
3639 setJSValue(result);
3640 }
3641
3642 void compilePutByIdWithThis()
3643 {
3644 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3645 LValue base = lowJSValue(m_node->child1());
3646 LValue thisValue = lowJSValue(m_node->child2());
3647 LValue value = lowJSValue(m_node->child3());
3648
3649 vmCall(Void, m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis,
3650 weakPointer(globalObject), base, thisValue, value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()]));
3651 }
3652
3653 void compilePutByValWithThis()
3654 {
3655 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3656 LValue base = lowJSValue(m_graph.varArgChild(m_node, 0));
3657 LValue thisValue = lowJSValue(m_graph.varArgChild(m_node, 1));
3658 LValue property = lowJSValue(m_graph.varArgChild(m_node, 2));
3659 LValue value = lowJSValue(m_graph.varArgChild(m_node, 3));
3660
3661 vmCall(Void, m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis,
3662 weakPointer(globalObject), base, thisValue, property, value);
3663 }
3664
3665 void compileAtomicsReadModifyWrite()
3666 {
3667 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3668 TypedArrayType type = m_node->arrayMode().typedArrayType();
3669 unsigned numExtraArgs = numExtraAtomicsArgs(m_node->op());
3670 Edge baseEdge = m_graph.child(m_node, 0);
3671 Edge indexEdge = m_graph.child(m_node, 1);
3672 Edge argEdges[maxNumExtraAtomicsArgs];
3673 for (unsigned i = numExtraArgs; i--;)
3674 argEdges[i] = m_graph.child(m_node, 2 + i);
3675 Edge storageEdge = m_graph.child(m_node, 2 + numExtraArgs);
3676
3677 if (!storageEdge) {
3678 auto callWith0 = [&] (auto* operation) {
3679 ASSERT(numExtraArgs == 0);
3680 return vmCall(Int64, operation, weakPointer(globalObject), lowJSValue(baseEdge), lowJSValue(indexEdge));
3681 };
3682
3683 auto callWith1 = [&] (auto* operation) {
3684 ASSERT(numExtraArgs == 1);
3685 return vmCall(Int64, operation, weakPointer(globalObject), lowJSValue(baseEdge), lowJSValue(indexEdge), lowJSValue(argEdges[0]));
3686 };
3687
3688 auto callWith2 = [&] (auto* operation) {
3689 ASSERT(numExtraArgs == 2);
3690 return vmCall(Int64, operation, weakPointer(globalObject), lowJSValue(baseEdge), lowJSValue(indexEdge), lowJSValue(argEdges[0]), lowJSValue(argEdges[1]));
3691 };
3692
3693 LValue result;
3694 switch (m_node->op()) {
3695 case AtomicsAdd:
3696 result = callWith1(operationAtomicsAdd);
3697 break;
3698 case AtomicsAnd:
3699 result = callWith1(operationAtomicsAnd);
3700 break;
3701 case AtomicsCompareExchange:
3702 result = callWith2(operationAtomicsCompareExchange);
3703 break;
3704 case AtomicsExchange:
3705 result = callWith1(operationAtomicsExchange);
3706 break;
3707 case AtomicsLoad:
3708 result = callWith0(operationAtomicsLoad);
3709 break;
3710 case AtomicsOr:
3711 result = callWith1(operationAtomicsOr);
3712 break;
3713 case AtomicsStore:
3714 result = callWith1(operationAtomicsStore);
3715 break;
3716 case AtomicsSub:
3717 result = callWith1(operationAtomicsSub);
3718 break;
3719 case AtomicsXor:
3720 result = callWith1(operationAtomicsXor);
3721 break;
3722 default:
3723 RELEASE_ASSERT_NOT_REACHED();
3724 }
3725 setJSValue(result);
3726 return;
3727 }
3728
3729 LValue index = lowInt32(indexEdge);
3730 LValue args[2];
3731 for (unsigned i = numExtraArgs; i--;)
3732 args[i] = getIntTypedArrayStoreOperand(argEdges[i]);
3733 LValue storage = lowStorage(storageEdge);
3734
3735 TypedPointer pointer = pointerIntoTypedArray(storage, index, type);
3736 Width width = widthForBytes(elementSize(type));
3737
3738 LValue atomicValue;
3739 LValue result;
3740
3741 auto sanitizeResult = [&] (LValue value) -> LValue {
3742 if (isSigned(type)) {
3743 switch (elementSize(type)) {
3744 case 1:
3745 value = m_out.bitAnd(value, m_out.constInt32(0xff));
3746 break;
3747 case 2:
3748 value = m_out.bitAnd(value, m_out.constInt32(0xffff));
3749 break;
3750 case 4:
3751 break;
3752 default:
3753 RELEASE_ASSERT_NOT_REACHED();
3754 break;
3755 }
3756 }
3757 return value;
3758 };
3759
3760 switch (m_node->op()) {
3761 case AtomicsAdd:
3762 atomicValue = m_out.atomicXchgAdd(args[0], pointer, width);
3763 result = sanitizeResult(atomicValue);
3764 break;
3765 case AtomicsAnd:
3766 atomicValue = m_out.atomicXchgAnd(args[0], pointer, width);
3767 result = sanitizeResult(atomicValue);
3768 break;
3769 case AtomicsCompareExchange:
3770 atomicValue = m_out.atomicStrongCAS(args[0], args[1], pointer, width);
3771 result = sanitizeResult(atomicValue);
3772 break;
3773 case AtomicsExchange:
3774 atomicValue = m_out.atomicXchg(args[0], pointer, width);
3775 result = sanitizeResult(atomicValue);
3776 break;
3777 case AtomicsLoad:
3778 atomicValue = m_out.atomicXchgAdd(m_out.int32Zero, pointer, width);
3779 result = sanitizeResult(atomicValue);
3780 break;
3781 case AtomicsOr:
3782 atomicValue = m_out.atomicXchgOr(args[0], pointer, width);
3783 result = sanitizeResult(atomicValue);
3784 break;
3785 case AtomicsStore:
3786 atomicValue = m_out.atomicXchg(args[0], pointer, width);
3787 result = args[0];
3788 break;
3789 case AtomicsSub:
3790 atomicValue = m_out.atomicXchgSub(args[0], pointer, width);
3791 result = sanitizeResult(atomicValue);
3792 break;
3793 case AtomicsXor:
3794 atomicValue = m_out.atomicXchgXor(args[0], pointer, width);
3795 result = sanitizeResult(atomicValue);
3796 break;
3797 default:
3798 RELEASE_ASSERT_NOT_REACHED();
3799 break;
3800 }
3801 // Signify that the state against which the atomic operations are serialized is confined to just
3802 // the typed array storage, since that's as precise of an abstraction as we can have of shared
3803 // array buffer storage.
3804 m_heaps.decorateFencedAccess(&m_heaps.typedArrayProperties, atomicValue);
3805
3806 setIntTypedArrayLoadResult(result, type);
3807 }
3808
3809 void compileAtomicsIsLockFree()
3810 {
3811 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3812 if (m_node->child1().useKind() != Int32Use) {
3813 setJSValue(vmCall(Int64, operationAtomicsIsLockFree, weakPointer(globalObject), lowJSValue(m_node->child1())));
3814 return;
3815 }
3816
3817 LValue bytes = lowInt32(m_node->child1());
3818
3819 LBasicBlock trueCase = m_out.newBlock();
3820 LBasicBlock falseCase = m_out.newBlock();
3821 LBasicBlock continuation = m_out.newBlock();
3822
3823 LBasicBlock lastNext = m_out.insertNewBlocksBefore(trueCase);
3824
3825 Vector<SwitchCase> cases;
3826 cases.append(SwitchCase(m_out.constInt32(1), trueCase, Weight()));
3827 cases.append(SwitchCase(m_out.constInt32(2), trueCase, Weight()));
3828 cases.append(SwitchCase(m_out.constInt32(4), trueCase, Weight()));
3829 m_out.switchInstruction(bytes, cases, falseCase, Weight());
3830
3831 m_out.appendTo(trueCase, falseCase);
3832 ValueFromBlock trueValue = m_out.anchor(m_out.booleanTrue);
3833 m_out.jump(continuation);
3834 m_out.appendTo(falseCase, continuation);
3835 ValueFromBlock falseValue = m_out.anchor(m_out.booleanFalse);
3836 m_out.jump(continuation);
3837
3838 m_out.appendTo(continuation, lastNext);
3839 setBoolean(m_out.phi(Int32, trueValue, falseValue));
3840 }
3841
3842 void compileDefineDataProperty()
3843 {
3844 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3845 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
3846 LValue value = lowJSValue(m_graph.varArgChild(m_node, 2));
3847 LValue attributes = lowInt32(m_graph.varArgChild(m_node, 3));
3848 Edge& propertyEdge = m_graph.varArgChild(m_node, 1);
3849 switch (propertyEdge.useKind()) {
3850 case StringUse: {
3851 LValue property = lowString(propertyEdge);
3852 vmCall(Void, operationDefineDataPropertyString, weakPointer(globalObject), base, property, value, attributes);
3853 break;
3854 }
3855 case StringIdentUse: {
3856 LValue property = lowStringIdent(propertyEdge);
3857 vmCall(Void, operationDefineDataPropertyStringIdent, weakPointer(globalObject), base, property, value, attributes);
3858 break;
3859 }
3860 case SymbolUse: {
3861 LValue property = lowSymbol(propertyEdge);
3862 vmCall(Void, operationDefineDataPropertySymbol, weakPointer(globalObject), base, property, value, attributes);
3863 break;
3864 }
3865 case UntypedUse: {
3866 LValue property = lowJSValue(propertyEdge);
3867 vmCall(Void, operationDefineDataProperty, weakPointer(globalObject), base, property, value, attributes);
3868 break;
3869 }
3870 default:
3871 RELEASE_ASSERT_NOT_REACHED();
3872 }
3873 }
3874
3875 void compileDefineAccessorProperty()
3876 {
3877 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3878 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
3879 LValue getter = lowCell(m_graph.varArgChild(m_node, 2));
3880 LValue setter = lowCell(m_graph.varArgChild(m_node, 3));
3881 LValue attributes = lowInt32(m_graph.varArgChild(m_node, 4));
3882 Edge& propertyEdge = m_graph.varArgChild(m_node, 1);
3883 switch (propertyEdge.useKind()) {
3884 case StringUse: {
3885 LValue property = lowString(propertyEdge);
3886 vmCall(Void, operationDefineAccessorPropertyString, weakPointer(globalObject), base, property, getter, setter, attributes);
3887 break;
3888 }
3889 case StringIdentUse: {
3890 LValue property = lowStringIdent(propertyEdge);
3891 vmCall(Void, operationDefineAccessorPropertyStringIdent, weakPointer(globalObject), base, property, getter, setter, attributes);
3892 break;
3893 }
3894 case SymbolUse: {
3895 LValue property = lowSymbol(propertyEdge);
3896 vmCall(Void, operationDefineAccessorPropertySymbol, weakPointer(globalObject), base, property, getter, setter, attributes);
3897 break;
3898 }
3899 case UntypedUse: {
3900 LValue property = lowJSValue(propertyEdge);
3901 vmCall(Void, operationDefineAccessorProperty, weakPointer(globalObject), base, property, getter, setter, attributes);
3902 break;
3903 }
3904 default:
3905 RELEASE_ASSERT_NOT_REACHED();
3906 }
3907 }
3908
3909 void compilePutById()
3910 {
3911 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == CellUse, m_node->child1().useKind());
3912
3913 Node* node = m_node;
3914 LValue base = lowCell(node->child1());
3915 LValue value = lowJSValue(node->child2());
3916 auto uid = m_graph.identifiers()[node->identifierNumber()];
3917
3918 PatchpointValue* patchpoint = m_out.patchpoint(Void);
3919 patchpoint->appendSomeRegister(base);
3920 patchpoint->appendSomeRegister(value);
3921 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
3922 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
3923 patchpoint->clobber(RegisterSet::macroScratchRegisters());
3924
3925 // FIXME: If this is a PutByIdFlush, we might want to late-clobber volatile registers.
3926 // https://bugs.webkit.org/show_bug.cgi?id=152848
3927
3928 RefPtr<PatchpointExceptionHandle> exceptionHandle =
3929 preparePatchpointForExceptions(patchpoint);
3930
3931 State* state = &m_ftlState;
3932 ECMAMode ecmaMode = m_graph.executableFor(node->origin.semantic)->ecmaMode();
3933
3934 patchpoint->setGenerator(
3935 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
3936 AllowMacroScratchRegisterUsage allowScratch(jit);
3937
3938 CallSiteIndex callSiteIndex =
3939 state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
3940
3941 Box<CCallHelpers::JumpList> exceptions =
3942 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
3943
3944 // JS setter call ICs generated by the PutById IC will need this.
3945 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
3946
3947 auto generator = Box<JITPutByIdGenerator>::create(
3948 jit.codeBlock(), node->origin.semantic, callSiteIndex,
3949 params.unavailableRegisters(), JSValueRegs(params[0].gpr()),
3950 JSValueRegs(params[1].gpr()), GPRInfo::patchpointScratchRegister, ecmaMode,
3951 node->op() == PutByIdDirect ? Direct : NotDirect);
3952
3953 generator->generateFastPath(jit);
3954 CCallHelpers::Label done = jit.label();
3955
3956 params.addLatePath(
3957 [=] (CCallHelpers& jit) {
3958 AllowMacroScratchRegisterUsage allowScratch(jit);
3959
3960 generator->slowPathJump().link(&jit);
3961 CCallHelpers::Label slowPathBegin = jit.label();
3962 CCallHelpers::Call slowPathCall = callOperation(
3963 *state, params.unavailableRegisters(), jit, node->origin.semantic,
3964 exceptions.get(), generator->slowPathFunction(), InvalidGPRReg,
3965 jit.codeBlock()->globalObjectFor(node->origin.semantic),
3966 CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(),
3967 params[0].gpr(), CCallHelpers::TrustedImmPtr(uid)).call();
3968 jit.jump().linkTo(done, &jit);
3969
3970 generator->reportSlowPathCall(slowPathBegin, slowPathCall);
3971
3972 jit.addLinkTask(
3973 [=] (LinkBuffer& linkBuffer) {
3974 generator->finalize(linkBuffer, linkBuffer);
3975 });
3976 });
3977 });
3978 }
3979
3980 void compileGetButterfly()
3981 {
3982 LValue butterfly = m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly);
3983 setStorage(butterfly);
3984 }
3985
3986 void compileConstantStoragePointer()
3987 {
3988 setStorage(m_out.constIntPtr(m_node->storagePointer()));
3989 }
3990
3991 void compileGetIndexedPropertyStorage()
3992 {
3993 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3994 LValue cell = lowCell(m_node->child1());
3995
3996 if (m_node->arrayMode().type() == Array::String) {
3997 LBasicBlock slowPath = m_out.newBlock();
3998 LBasicBlock continuation = m_out.newBlock();
3999
4000 LValue fastResultValue = m_out.loadPtr(cell, m_heaps.JSString_value);
4001 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
4002
4003 m_out.branch(isRopeString(cell, m_node->child1()), rarely(slowPath), usually(continuation));
4004
4005 LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
4006
4007 ValueFromBlock slowResult = m_out.anchor(vmCall(pointerType(), operationResolveRope, weakPointer(globalObject), cell));
4008
4009 m_out.jump(continuation);
4010
4011 m_out.appendTo(continuation, lastNext);
4012
4013 setStorage(m_out.loadPtr(m_out.phi(pointerType(), fastResult, slowResult), m_heaps.StringImpl_data));
4014 return;
4015 }
4016
4017 DFG_ASSERT(m_graph, m_node, isTypedView(m_node->arrayMode().typedArrayType()), m_node->arrayMode().typedArrayType());
4018 LValue vector = m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector);
4019 setStorage(caged(Gigacage::Primitive, vector, cell));
4020 }
4021
4022 void compileCheckArray()
4023 {
4024 Edge edge = m_node->child1();
4025 LValue cell = lowCell(edge);
4026
4027 if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge)))
4028 return;
4029
4030 speculate(
4031 BadIndexingType, jsValueValue(cell), 0,
4032 m_out.logicalNot(isArrayTypeForCheckArray(cell, m_node->arrayMode())));
4033 }
4034
4035 void compileGetTypedArrayByteOffset()
4036 {
4037 LValue basePtr = lowCell(m_node->child1());
4038
4039 LBasicBlock simpleCase = m_out.newBlock();
4040 LBasicBlock wastefulCase = m_out.newBlock();
4041 LBasicBlock notNull = m_out.newBlock();
4042 LBasicBlock continuation = m_out.newBlock();
4043
4044 LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
4045 m_out.branch(
4046 m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
4047 unsure(simpleCase), unsure(wastefulCase));
4048
4049 LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
4050
4051 ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
4052
4053 m_out.jump(continuation);
4054
4055 m_out.appendTo(wastefulCase, notNull);
4056
4057 LValue vector = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
4058 ValueFromBlock nullVectorOut = m_out.anchor(vector);
4059 m_out.branch(vector, unsure(notNull), unsure(continuation));
4060
4061 m_out.appendTo(notNull, continuation);
4062
4063 LValue butterflyPtr = caged(Gigacage::JSValue, m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly), basePtr);
4064 LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
4065
4066 LValue vectorPtr = caged(Gigacage::Primitive, vector, basePtr);
4067
4068 // FIXME: This needs caging.
4069 // https://bugs.webkit.org/show_bug.cgi?id=175515
4070 LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
4071 dataPtr = removeArrayPtrTag(dataPtr);
4072
4073 ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
4074
4075 m_out.jump(continuation);
4076 m_out.appendTo(continuation, lastNext);
4077
4078 setInt32(m_out.castToInt32(m_out.phi(pointerType(), simpleOut, nullVectorOut, wastefulOut)));
4079 }
4080
4081 void compileGetPrototypeOf()
4082 {
4083 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4084 switch (m_node->child1().useKind()) {
4085 case ArrayUse:
4086 case FunctionUse:
4087 case FinalObjectUse: {
4088 LValue object = lowCell(m_node->child1());
4089 switch (m_node->child1().useKind()) {
4090 case ArrayUse:
4091 speculateArray(m_node->child1(), object);
4092 break;
4093 case FunctionUse:
4094 speculateFunction(m_node->child1(), object);
4095 break;
4096 case FinalObjectUse:
4097 speculateFinalObject(m_node->child1(), object);
4098 break;
4099 default:
4100 RELEASE_ASSERT_NOT_REACHED();
4101 break;
4102 }
4103
4104 LValue structure = loadStructure(object);
4105
4106 AbstractValue& value = m_state.forNode(m_node->child1());
4107 if ((value.m_type && !(value.m_type & ~SpecObject)) && value.m_structure.isFinite()) {
4108 bool hasPolyProto = false;
4109 bool hasMonoProto = false;
4110 value.m_structure.forEach([&] (RegisteredStructure structure) {
4111 if (structure->hasPolyProto())
4112 hasPolyProto = true;
4113 else
4114 hasMonoProto = true;
4115 });
4116
4117 if (hasMonoProto && !hasPolyProto) {
4118 setJSValue(m_out.load64(structure, m_heaps.Structure_prototype));
4119 return;
4120 }
4121
4122 if (hasPolyProto && !hasMonoProto) {
4123 setJSValue(m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), object, m_out.constInt64(knownPolyProtoOffset), ScaleEight, JSObject::offsetOfInlineStorage())));
4124 return;
4125 }
4126 }
4127
4128 LBasicBlock continuation = m_out.newBlock();
4129 LBasicBlock loadPolyProto = m_out.newBlock();
4130
4131 LValue prototypeBits = m_out.load64(structure, m_heaps.Structure_prototype);
4132 ValueFromBlock directPrototype = m_out.anchor(prototypeBits);
4133 m_out.branch(m_out.isZero64(prototypeBits), unsure(loadPolyProto), unsure(continuation));
4134
4135 LBasicBlock lastNext = m_out.appendTo(loadPolyProto, continuation);
4136 ValueFromBlock polyProto = m_out.anchor(
4137 m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), object, m_out.constInt64(knownPolyProtoOffset), ScaleEight, JSObject::offsetOfInlineStorage())));
4138 m_out.jump(continuation);
4139
4140 m_out.appendTo(continuation, lastNext);
4141 setJSValue(m_out.phi(Int64, directPrototype, polyProto));
4142 return;
4143 }
4144 case ObjectUse: {
4145 setJSValue(vmCall(Int64, operationGetPrototypeOfObject, weakPointer(globalObject), lowObject(m_node->child1())));
4146 return;
4147 }
4148 default: {
4149 setJSValue(vmCall(Int64, operationGetPrototypeOf, weakPointer(globalObject), lowJSValue(m_node->child1())));
4150 return;
4151 }
4152 }
4153 }
4154
4155 void compileGetArrayLength()
4156 {
4157 switch (m_node->arrayMode().type()) {
4158 case Array::Undecided:
4159 case Array::Int32:
4160 case Array::Double:
4161 case Array::Contiguous: {
4162 setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
4163 return;
4164 }
4165
4166 case Array::ArrayStorage:
4167 case Array::SlowPutArrayStorage: {
4168 LValue length = m_out.load32(lowStorage(m_node->child2()), m_heaps.ArrayStorage_publicLength);
4169 speculate(Uncountable, noValue(), nullptr, m_out.lessThan(length, m_out.int32Zero));
4170 setInt32(length);
4171 return;
4172 }
4173
4174 case Array::String: {
4175 LValue string = lowCell(m_node->child1());
4176
4177 LBasicBlock ropePath = m_out.newBlock();
4178 LBasicBlock nonRopePath = m_out.newBlock();
4179 LBasicBlock continuation = m_out.newBlock();
4180
4181 m_out.branch(isRopeString(string, m_node->child1()), rarely(ropePath), usually(nonRopePath));
4182
4183 LBasicBlock lastNext = m_out.appendTo(ropePath, nonRopePath);
4184 ValueFromBlock ropeLength = m_out.anchor(m_out.load32NonNegative(string, m_heaps.JSRopeString_length));
4185 m_out.jump(continuation);
4186
4187 m_out.appendTo(nonRopePath, continuation);
4188 ValueFromBlock nonRopeLength = m_out.anchor(m_out.load32NonNegative(m_out.loadPtr(string, m_heaps.JSString_value), m_heaps.StringImpl_length));
4189 m_out.jump(continuation);
4190
4191 m_out.appendTo(continuation, lastNext);
4192 setInt32(m_out.phi(Int32, ropeLength, nonRopeLength));
4193 return;
4194 }
4195
4196 case Array::DirectArguments: {
4197 LValue arguments = lowCell(m_node->child1());
4198 speculate(
4199 ExoticObjectMode, noValue(), nullptr,
4200 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_mappedArguments)));
4201 setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
4202 return;
4203 }
4204
4205 case Array::ScopedArguments: {
4206 LValue arguments = lowCell(m_node->child1());
4207 speculate(
4208 ExoticObjectMode, noValue(), nullptr,
4209 m_out.notZero32(m_out.load8ZeroExt32(arguments, m_heaps.ScopedArguments_overrodeThings)));
4210 setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
4211 return;
4212 }
4213
4214 default:
4215 if (m_node->arrayMode().isSomeTypedArrayView()) {
4216 setInt32(
4217 m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
4218 return;
4219 }
4220
4221 DFG_CRASH(m_graph, m_node, "Bad array type");
4222 return;
4223 }
4224 }
4225
4226 void compileGetVectorLength()
4227 {
4228 switch (m_node->arrayMode().type()) {
4229 case Array::ArrayStorage:
4230 case Array::SlowPutArrayStorage:
4231 setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.ArrayStorage_vectorLength));
4232 return;
4233 default:
4234 return;
4235 }
4236 }
4237
4238 void compileCheckInBounds()
4239 {
4240 speculate(
4241 OutOfBounds, noValue(), 0,
4242 m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
4243
4244 // Even though we claim to have JSValue result, no user of us should
4245 // depend on our value. Users of this node just need to maintain that
4246 // we dominate them.
4247 }
4248
4249 void compileGetByVal()
4250 {
4251 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4252 switch (m_node->arrayMode().type()) {
4253 case Array::Int32:
4254 case Array::Contiguous: {
4255 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4256 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
4257
4258 IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
4259 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
4260
4261 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
4262
4263 if (m_node->arrayMode().isInBounds()) {
4264 LValue result = m_out.load64(baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
4265 LValue isHole = m_out.isZero64(result);
4266 if (m_node->arrayMode().isSaneChain()) {
4267 DFG_ASSERT(
4268 m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous, m_node->arrayMode().type());
4269 result = m_out.select(
4270 isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result);
4271 } else
4272 speculate(LoadFromHole, noValue(), 0, isHole);
4273 setJSValue(result);
4274 return;
4275 }
4276
4277 LBasicBlock fastCase = m_out.newBlock();
4278 LBasicBlock slowCase = m_out.newBlock();
4279 LBasicBlock continuation = m_out.newBlock();
4280
4281 m_out.branch(
4282 m_out.aboveOrEqual(
4283 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
4284 rarely(slowCase), usually(fastCase));
4285
4286 LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
4287
4288 LValue fastResultValue = m_out.load64(baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
4289 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
4290 m_out.branch(
4291 m_out.isZero64(fastResultValue), rarely(slowCase), usually(continuation));
4292
4293 m_out.appendTo(slowCase, continuation);
4294 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operationGetByValObjectInt, weakPointer(globalObject), base, index));
4295 m_out.jump(continuation);
4296
4297 m_out.appendTo(continuation, lastNext);
4298 setJSValue(m_out.phi(Int64, fastResult, slowResult));
4299 return;
4300 }
4301
4302 case Array::Double: {
4303 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
4304 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4305 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
4306
4307 IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
4308
4309 if (m_node->arrayMode().isInBounds()) {
4310 LValue result = m_out.loadDouble(
4311 baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
4312
4313 if (!m_node->arrayMode().isSaneChain()) {
4314 speculate(
4315 LoadFromHole, noValue(), 0,
4316 m_out.doubleNotEqualOrUnordered(result, result));
4317 }
4318 setDouble(result);
4319 break;
4320 }
4321
4322 LBasicBlock inBounds = m_out.newBlock();
4323 LBasicBlock boxPath = m_out.newBlock();
4324 LBasicBlock slowCase = m_out.newBlock();
4325 LBasicBlock continuation = m_out.newBlock();
4326
4327 m_out.branch(
4328 m_out.aboveOrEqual(
4329 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
4330 rarely(slowCase), usually(inBounds));
4331
4332 LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
4333 LValue doubleValue = m_out.loadDouble(
4334 baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
4335 m_out.branch(
4336 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
4337 rarely(slowCase), usually(boxPath));
4338
4339 m_out.appendTo(boxPath, slowCase);
4340 ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
4341 m_out.jump(continuation);
4342
4343 m_out.appendTo(slowCase, continuation);
4344 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operationGetByValObjectInt, weakPointer(globalObject), base, index));
4345 m_out.jump(continuation);
4346
4347 m_out.appendTo(continuation, lastNext);
4348 setJSValue(m_out.phi(Int64, fastResult, slowResult));
4349 return;
4350 }
4351
4352 case Array::Undecided: {
4353 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4354
4355 speculate(OutOfBounds, noValue(), m_node, m_out.lessThan(index, m_out.int32Zero));
4356 setJSValue(m_out.constInt64(JSValue::ValueUndefined));
4357 return;
4358 }
4359
4360 case Array::DirectArguments: {
4361 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
4362 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4363
4364 speculate(
4365 ExoticObjectMode, noValue(), nullptr,
4366 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_mappedArguments)));
4367
4368 LValue length = m_out.load32NonNegative(base, m_heaps.DirectArguments_length);
4369 auto isOutOfBounds = m_out.aboveOrEqual(index, length);
4370 if (m_node->arrayMode().isInBounds()) {
4371 speculate(OutOfBounds, noValue(), nullptr, isOutOfBounds);
4372 TypedPointer address = m_out.baseIndex(
4373 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
4374 setJSValue(m_out.load64(address));
4375 return;
4376 }
4377
4378 LBasicBlock inBounds = m_out.newBlock();
4379 LBasicBlock slowCase = m_out.newBlock();
4380 LBasicBlock continuation = m_out.newBlock();
4381
4382 m_out.branch(isOutOfBounds, rarely(slowCase), usually(inBounds));
4383
4384 LBasicBlock lastNext = m_out.appendTo(inBounds, slowCase);
4385 TypedPointer address = m_out.baseIndex(
4386 m_heaps.DirectArguments_storage,
4387 base,
4388 m_out.zeroExt(index, pointerType()));
4389 ValueFromBlock fastResult = m_out.anchor(m_out.load64(address));
4390 m_out.jump(continuation);
4391
4392 m_out.appendTo(slowCase, continuation);
4393 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operationGetByValObjectInt, weakPointer(globalObject), base, index));
4394 m_out.jump(continuation);
4395
4396 m_out.appendTo(continuation, lastNext);
4397 setJSValue(m_out.phi(Int64, fastResult, slowResult));
4398 return;
4399 }
4400
4401 case Array::ScopedArguments: {
4402 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
4403 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4404
4405 speculate(
4406 ExoticObjectMode, noValue(), nullptr,
4407 m_out.aboveOrEqual(
4408 index,
4409 m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
4410
4411 LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
4412 LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
4413
4414 LBasicBlock namedCase = m_out.newBlock();
4415 LBasicBlock overflowCase = m_out.newBlock();
4416 LBasicBlock continuation = m_out.newBlock();
4417
4418 m_out.branch(
4419 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
4420
4421 LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
4422
4423 LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
4424 LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
4425
4426 TypedPointer address = m_out.baseIndex(
4427 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
4428 LValue scopeOffset = m_out.load32(address);
4429
4430 speculate(
4431 ExoticObjectMode, noValue(), nullptr,
4432 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
4433
4434 address = m_out.baseIndex(
4435 m_heaps.JSLexicalEnvironment_variables, scope, m_out.zeroExtPtr(scopeOffset));
4436 ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
4437 m_out.jump(continuation);
4438
4439 m_out.appendTo(overflowCase, continuation);
4440
4441 LValue storage = m_out.loadPtr(base, m_heaps.ScopedArguments_storage);
4442 address = m_out.baseIndex(
4443 m_heaps.ScopedArguments_Storage_storage, storage,
4444 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
4445 LValue overflowValue = m_out.load64(address);
4446 speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
4447 ValueFromBlock overflowResult = m_out.anchor(overflowValue);
4448 m_out.jump(continuation);
4449
4450 m_out.appendTo(continuation, lastNext);
4451 setJSValue(m_out.phi(Int64, namedResult, overflowResult));
4452 return;
4453 }
4454
4455 case Array::Generic: {
4456 if (m_graph.m_slowGetByVal.contains(m_node)) {
4457 if (m_graph.varArgChild(m_node, 0).useKind() == ObjectUse) {
4458 if (m_graph.varArgChild(m_node, 1).useKind() == StringUse) {
4459 setJSValue(vmCall(
4460 Int64, operationGetByValObjectString, weakPointer(globalObject),
4461 lowObject(m_graph.varArgChild(m_node, 0)), lowString(m_graph.varArgChild(m_node, 1))));
4462 return;
4463 }
4464
4465 if (m_graph.varArgChild(m_node, 1).useKind() == SymbolUse) {
4466 setJSValue(vmCall(
4467 Int64, operationGetByValObjectSymbol, weakPointer(globalObject),
4468 lowObject(m_graph.varArgChild(m_node, 0)), lowSymbol(m_graph.varArgChild(m_node, 1))));
4469 return;
4470 }
4471 }
4472
4473 setJSValue(vmCall(
4474 Int64, operationGetByVal, weakPointer(globalObject),
4475 lowJSValue(m_graph.varArgChild(m_node, 0)), lowJSValue(m_graph.varArgChild(m_node, 1))));
4476 return;
4477 }
4478
4479 Node* node = m_node;
4480
4481 LValue base = lowJSValue(m_graph.varArgChild(node, 0), ManualOperandSpeculation);
4482 LValue property = lowJSValue(m_graph.varArgChild(node, 1), ManualOperandSpeculation);
4483
4484 speculate(m_graph.varArgChild(node, 0));
4485 speculate(m_graph.varArgChild(node, 1));
4486 bool baseIsCell = abstractValue(m_graph.varArgChild(node, 0)).isType(SpecCell);
4487 bool propertyIsString = false;
4488 bool propertyIsInt32 = false;
4489 bool propertyIsSymbol = false;
4490 if (abstractValue(m_graph.varArgChild(node, 1)).isType(SpecString))
4491 propertyIsString = true;
4492 else if (abstractValue(m_graph.varArgChild(node, 1)).isType(SpecInt32Only))
4493 propertyIsInt32 = true;
4494 else if (abstractValue(m_graph.varArgChild(node, 1)).isType(SpecSymbol))
4495 propertyIsSymbol = true;
4496
4497 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
4498 patchpoint->appendSomeRegister(base);
4499 patchpoint->appendSomeRegister(property);
4500 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
4501 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
4502 patchpoint->clobber(RegisterSet::macroScratchRegisters());
4503
4504 RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
4505
4506 State* state = &m_ftlState;
4507 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
4508 AllowMacroScratchRegisterUsage allowScratch(jit);
4509
4510 CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
4511
4512 // This is the direct exit target for operation calls.
4513 Box<CCallHelpers::JumpList> exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit);
4514
4515 // This is the exit for call IC's created by the IC for getters. We don't have
4516 // to do anything weird other than call this, since it will associate the exit with
4517 // the callsite index.
4518 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
4519
4520 GPRReg resultGPR = params[0].gpr();
4521 GPRReg baseGPR = params[1].gpr();
4522 GPRReg propertyGPR = params[2].gpr();
4523
4524 auto generator = Box<JITGetByValGenerator>::create(
4525 jit.codeBlock(), node->origin.semantic, callSiteIndex, params.unavailableRegisters(),
4526 JSValueRegs(baseGPR), JSValueRegs(propertyGPR), JSValueRegs(resultGPR));
4527
4528 generator->stubInfo()->propertyIsString = propertyIsString;
4529 generator->stubInfo()->propertyIsInt32 = propertyIsInt32;
4530 generator->stubInfo()->propertyIsSymbol = propertyIsSymbol;
4531
4532 CCallHelpers::Jump notCell;
4533 if (!baseIsCell)
4534 notCell = jit.branchIfNotCell(baseGPR);
4535
4536 generator->generateFastPath(jit);
4537 CCallHelpers::Label done = jit.label();
4538
4539 params.addLatePath([=] (CCallHelpers& jit) {
4540 AllowMacroScratchRegisterUsage allowScratch(jit);
4541
4542 if (notCell.isSet())
4543 notCell.link(&jit);
4544 generator->slowPathJump().link(&jit);
4545 CCallHelpers::Label slowPathBegin = jit.label();
4546 CCallHelpers::Call slowPathCall = callOperation(
4547 *state, params.unavailableRegisters(), jit, node->origin.semantic,
4548 exceptions.get(), operationGetByValOptimize, resultGPR,
4549 jit.codeBlock()->globalObjectFor(node->origin.semantic),
4550 CCallHelpers::TrustedImmPtr(generator->stubInfo()), CCallHelpers::TrustedImmPtr(nullptr), baseGPR, propertyGPR).call();
4551 jit.jump().linkTo(done, &jit);
4552
4553 generator->reportSlowPathCall(slowPathBegin, slowPathCall);
4554
4555 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4556 generator->finalize(linkBuffer, linkBuffer);
4557 });
4558 });
4559 });
4560
4561 setJSValue(patchpoint);
4562 return;
4563 }
4564
4565 case Array::ArrayStorage:
4566 case Array::SlowPutArrayStorage: {
4567 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
4568 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4569 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
4570
4571 IndexedAbstractHeap& heap = m_heaps.ArrayStorage_vector;
4572
4573 if (m_node->arrayMode().isInBounds()) {
4574 LValue result = m_out.load64(baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
4575 speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
4576 setJSValue(result);
4577 break;
4578 }
4579
4580 LBasicBlock inBounds = m_out.newBlock();
4581 LBasicBlock slowCase = m_out.newBlock();
4582 LBasicBlock continuation = m_out.newBlock();
4583
4584 m_out.branch(
4585 m_out.aboveOrEqual(index, m_out.load32NonNegative(storage, m_heaps.ArrayStorage_vectorLength)),
4586 rarely(slowCase), usually(inBounds));
4587
4588 LBasicBlock lastNext = m_out.appendTo(inBounds, slowCase);
4589 LValue result = m_out.load64(baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
4590 ValueFromBlock fastResult = m_out.anchor(result);
4591 m_out.branch(
4592 m_out.isZero64(result),
4593 rarely(slowCase), usually(continuation));
4594
4595 m_out.appendTo(slowCase, continuation);
4596 ValueFromBlock slowResult = m_out.anchor(
4597 vmCall(Int64, operationGetByValObjectInt, weakPointer(globalObject), base, index));
4598 m_out.jump(continuation);
4599
4600 m_out.appendTo(continuation, lastNext);
4601 setJSValue(m_out.phi(Int64, fastResult, slowResult));
4602 return;
4603 }
4604
4605 case Array::String: {
4606 compileStringCharAt();
4607 return;
4608 }
4609
4610 case Array::Int8Array:
4611 case Array::Int16Array:
4612 case Array::Int32Array:
4613 case Array::Uint8Array:
4614 case Array::Uint8ClampedArray:
4615 case Array::Uint16Array:
4616 case Array::Uint32Array:
4617 case Array::Float32Array:
4618 case Array::Float64Array: {
4619 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
4620 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
4621
4622 TypedArrayType type = m_node->arrayMode().typedArrayType();
4623 ASSERT(isTypedView(type));
4624 {
4625 TypedPointer pointer = pointerIntoTypedArray(storage, index, type);
4626
4627 if (isInt(type)) {
4628 LValue result = loadFromIntTypedArray(pointer, type);
4629 bool canSpeculate = true;
4630 setIntTypedArrayLoadResult(result, type, canSpeculate);
4631 return;
4632 }
4633
4634 ASSERT(isFloat(type));
4635
4636 LValue result;
4637 switch (type) {
4638 case TypeFloat32:
4639 result = m_out.floatToDouble(m_out.loadFloat(pointer));
4640 break;
4641 case TypeFloat64:
4642 result = m_out.loadDouble(pointer);
4643 break;
4644 default:
4645 DFG_CRASH(m_graph, m_node, "Bad typed array type");
4646 }
4647
4648 setDouble(result);
4649 return;
4650 }
4651 }
4652
4653 case Array::AnyTypedArray:
4654 case Array::ForceExit:
4655 case Array::SelectUsingArguments:
4656 case Array::SelectUsingPredictions:
4657 case Array::Unprofiled:
4658 DFG_CRASH(m_graph, m_node, "Bad array type");
4659 return;
4660 }
4661 }
4662
4663 void compileGetMyArgumentByVal()
4664 {
4665 InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame();
4666
4667 LValue originalIndex = lowInt32(m_node->child2());
4668
4669 LValue numberOfArgsIncludingThis;
4670 if (inlineCallFrame && !inlineCallFrame->isVarargs())
4671 numberOfArgsIncludingThis = m_out.constInt32(inlineCallFrame->argumentCountIncludingThis);
4672 else {
4673 VirtualRegister argumentCountRegister = AssemblyHelpers::argumentCount(inlineCallFrame);
4674 numberOfArgsIncludingThis = m_out.load32(payloadFor(argumentCountRegister));
4675 }
4676
4677 LValue numberOfArgs = m_out.sub(numberOfArgsIncludingThis, m_out.int32One);
4678 LValue indexToCheck = originalIndex;
4679 LValue numberOfArgumentsToSkip = m_out.int32Zero;
4680 if (m_node->numberOfArgumentsToSkip()) {
4681 numberOfArgumentsToSkip = m_out.constInt32(m_node->numberOfArgumentsToSkip());
4682 CheckValue* check = m_out.speculateAdd(indexToCheck, numberOfArgumentsToSkip);
4683 blessSpeculation(check, Overflow, noValue(), nullptr, m_origin);
4684 indexToCheck = check;
4685 }
4686
4687 LValue isOutOfBounds = m_out.bitOr(m_out.aboveOrEqual(indexToCheck, numberOfArgs), m_out.below(indexToCheck, numberOfArgumentsToSkip));
4688 LBasicBlock continuation = nullptr;
4689 LBasicBlock lastNext = nullptr;
4690 ValueFromBlock slowResult;
4691 if (m_node->op() == GetMyArgumentByValOutOfBounds) {
4692 LBasicBlock normalCase = m_out.newBlock();
4693 continuation = m_out.newBlock();
4694
4695 slowResult = m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined())));
4696 m_out.branch(isOutOfBounds, unsure(continuation), unsure(normalCase));
4697
4698 lastNext = m_out.appendTo(normalCase, continuation);
4699 } else
4700 speculate(OutOfBounds, noValue(), nullptr, isOutOfBounds);
4701
4702 LValue index = m_out.add(indexToCheck, m_out.int32One);
4703
4704 TypedPointer base;
4705 if (inlineCallFrame) {
4706 if (inlineCallFrame->argumentCountIncludingThis > 1)
4707 base = addressFor(inlineCallFrame->argumentsWithFixup[0].virtualRegister());
4708 } else
4709 base = addressFor(virtualRegisterForArgument(0));
4710
4711 LValue result;
4712 if (base) {
4713 LValue pointer = m_out.baseIndex(
4714 base.value(), m_out.zeroExt(index, pointerType()), ScaleEight);
4715 result = m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer));
4716 } else
4717 result = m_out.constInt64(JSValue::encode(jsUndefined()));
4718
4719 if (m_node->op() == GetMyArgumentByValOutOfBounds) {
4720 ValueFromBlock normalResult = m_out.anchor(result);
4721 m_out.jump(continuation);
4722
4723 m_out.appendTo(continuation, lastNext);
4724 result = m_out.phi(Int64, slowResult, normalResult);
4725 }
4726
4727 setJSValue(result);
4728 }
4729
4730 void compilePutByVal()
4731 {
4732 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4733 Edge child1 = m_graph.varArgChild(m_node, 0);
4734 Edge child2 = m_graph.varArgChild(m_node, 1);
4735 Edge child3 = m_graph.varArgChild(m_node, 2);
4736 Edge child4 = m_graph.varArgChild(m_node, 3);
4737 Edge child5 = m_graph.varArgChild(m_node, 4);
4738
4739 ArrayMode arrayMode = m_node->arrayMode().modeForPut();
4740 switch (arrayMode.type()) {
4741 case Array::Generic: {
4742 if (child1.useKind() == CellUse) {
4743 V_JITOperation_GCCJ operation = nullptr;
4744 if (child2.useKind() == StringUse) {
4745 if (m_node->op() == PutByValDirect) {
4746 if (m_graph.isStrictModeFor(m_node->origin.semantic))
4747 operation = operationPutByValDirectCellStringStrict;
4748 else
4749 operation = operationPutByValDirectCellStringNonStrict;
4750 } else {
4751 if (m_graph.isStrictModeFor(m_node->origin.semantic))
4752 operation = operationPutByValCellStringStrict;
4753 else
4754 operation = operationPutByValCellStringNonStrict;
4755 }
4756 vmCall(Void, operation, weakPointer(globalObject), lowCell(child1), lowString(child2), lowJSValue(child3));
4757 return;
4758 }
4759
4760 if (child2.useKind() == SymbolUse) {
4761 if (m_node->op() == PutByValDirect) {
4762 if (m_graph.isStrictModeFor(m_node->origin.semantic))
4763 operation = operationPutByValDirectCellSymbolStrict;
4764 else
4765 operation = operationPutByValDirectCellSymbolNonStrict;
4766 } else {
4767 if (m_graph.isStrictModeFor(m_node->origin.semantic))
4768 operation = operationPutByValCellSymbolStrict;
4769 else
4770 operation = operationPutByValCellSymbolNonStrict;
4771 }
4772 vmCall(Void, operation, weakPointer(globalObject), lowCell(child1), lowSymbol(child2), lowJSValue(child3));
4773 return;
4774 }
4775 }
4776
4777 V_JITOperation_GJJJ operation;
4778 if (m_node->op() == PutByValDirect) {
4779 if (m_graph.isStrictModeFor(m_node->origin.semantic))
4780 operation = operationPutByValDirectStrict;
4781 else
4782 operation = operationPutByValDirectNonStrict;
4783 } else {
4784 if (m_graph.isStrictModeFor(m_node->origin.semantic))
4785 operation = operationPutByValStrict;
4786 else
4787 operation = operationPutByValNonStrict;
4788 }
4789
4790 vmCall(
4791 Void, operation, weakPointer(globalObject),
4792 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
4793 return;
4794 }
4795
4796 default:
4797 break;
4798 }
4799
4800 LValue base = lowCell(child1);
4801 LValue index = lowInt32(child2);
4802 LValue storage = lowStorage(child4);
4803
4804 switch (arrayMode.type()) {
4805 case Array::Int32:
4806 case Array::Double:
4807 case Array::Contiguous: {
4808 LBasicBlock continuation = m_out.newBlock();
4809 LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
4810
4811 switch (arrayMode.type()) {
4812 case Array::Int32:
4813 case Array::Contiguous: {
4814 LValue value = lowJSValue(child3, ManualOperandSpeculation);
4815
4816 if (arrayMode.type() == Array::Int32)
4817 FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32Only, isNotInt32(value));
4818
4819 TypedPointer elementPointer = m_out.baseIndex(
4820 arrayMode.type() == Array::Int32 ?
4821 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
4822 storage, m_out.zeroExtPtr(index), provenValue(child2));
4823
4824 if (m_node->op() == PutByValAlias) {
4825 m_out.store64(value, elementPointer);
4826 break;
4827 }
4828
4829 contiguousPutByValOutOfBounds(
4830 m_graph.isStrictModeFor(m_node->origin.semantic)
4831 ? (m_node->op() == PutByValDirect ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsStrict)
4832 : (m_node->op() == PutByValDirect ? operationPutByValDirectBeyondArrayBoundsNonStrict : operationPutByValBeyondArrayBoundsNonStrict),
4833 base, storage, index, value, continuation);
4834
4835 m_out.store64(value, elementPointer);
4836 break;
4837 }
4838
4839 case Array::Double: {
4840 LValue value = lowDouble(child3);
4841
4842 FTL_TYPE_CHECK(
4843 doubleValue(value), child3, SpecDoubleReal,
4844 m_out.doubleNotEqualOrUnordered(value, value));
4845
4846 TypedPointer elementPointer = m_out.baseIndex(
4847 m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
4848 provenValue(child2));
4849
4850 if (m_node->op() == PutByValAlias) {
4851 m_out.storeDouble(value, elementPointer);
4852 break;
4853 }
4854
4855 contiguousPutByValOutOfBounds(
4856 m_graph.isStrictModeFor(m_node->origin.semantic)
4857 ? (m_node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
4858 : (m_node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
4859 base, storage, index, value, continuation);
4860
4861 m_out.storeDouble(value, elementPointer);
4862 break;
4863 }
4864
4865 default:
4866 DFG_CRASH(m_graph, m_node, "Bad array type");
4867 }
4868
4869 m_out.jump(continuation);
4870 m_out.appendTo(continuation, outerLastNext);
4871 return;
4872 }
4873
4874 case Array::ArrayStorage:
4875 case Array::SlowPutArrayStorage: {
4876 LValue value = lowJSValue(child3);
4877
4878 TypedPointer elementPointer = m_out.baseIndex(
4879 m_heaps.ArrayStorage_vector, storage, m_out.zeroExtPtr(index),
4880 provenValue(child2));
4881
4882 if (m_node->op() == PutByValAlias) {
4883 m_out.store64(value, elementPointer);
4884 return;
4885 }
4886
4887 if (arrayMode.isInBounds()) {
4888 speculate(StoreToHole, noValue(), 0, m_out.isZero64(m_out.load64(elementPointer)));
4889 m_out.store64(value, elementPointer);
4890 return;
4891 }
4892
4893 LValue isOutOfBounds = m_out.aboveOrEqual(
4894 index, m_out.load32NonNegative(storage, m_heaps.ArrayStorage_vectorLength));
4895
4896 auto slowPathFunction = m_graph.isStrictModeFor(m_node->origin.semantic)
4897 ? (m_node->op() == PutByValDirect ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsStrict)
4898 : (m_node->op() == PutByValDirect ? operationPutByValDirectBeyondArrayBoundsNonStrict : operationPutByValBeyondArrayBoundsNonStrict);
4899 if (!arrayMode.isOutOfBounds()) {
4900 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
4901 isOutOfBounds = m_out.booleanFalse;
4902 }
4903
4904 LBasicBlock inBoundCase = m_out.newBlock();
4905 LBasicBlock slowCase = m_out.newBlock();
4906 LBasicBlock holeCase = m_out.newBlock();
4907 LBasicBlock doStoreCase = m_out.newBlock();
4908 LBasicBlock lengthUpdateCase = m_out.newBlock();
4909 LBasicBlock continuation = m_out.newBlock();
4910
4911 m_out.branch(isOutOfBounds, rarely(slowCase), usually(inBoundCase));
4912
4913 LBasicBlock lastNext = m_out.appendTo(slowCase, inBoundCase);
4914 vmCall(
4915 Void, slowPathFunction,
4916 weakPointer(globalObject), base, index, value);
4917 m_out.jump(continuation);
4918
4919
4920 if (arrayMode.isSlowPut()) {
4921 m_out.appendTo(inBoundCase, doStoreCase);
4922 m_out.branch(m_out.isZero64(m_out.load64(elementPointer)), rarely(slowCase), usually(doStoreCase));
4923 } else {
4924 m_out.appendTo(inBoundCase, holeCase);
4925 m_out.branch(m_out.isZero64(m_out.load64(elementPointer)), rarely(holeCase), usually(doStoreCase));
4926
4927 m_out.appendTo(holeCase, lengthUpdateCase);
4928 m_out.store32(
4929 m_out.add(m_out.load32(storage, m_heaps.ArrayStorage_numValuesInVector), m_out.int32One),
4930 storage, m_heaps.ArrayStorage_numValuesInVector);
4931 m_out.branch(
4932 m_out.below(
4933 index, m_out.load32NonNegative(storage, m_heaps.ArrayStorage_publicLength)),
4934 unsure(doStoreCase), unsure(lengthUpdateCase));
4935
4936 m_out.appendTo(lengthUpdateCase, doStoreCase);
4937 m_out.store32(
4938 m_out.add(index, m_out.int32One),
4939 storage, m_heaps.ArrayStorage_publicLength);
4940 m_out.jump(doStoreCase);
4941 }
4942
4943 m_out.appendTo(doStoreCase, continuation);
4944 m_out.store64(value, elementPointer);
4945 m_out.jump(continuation);
4946
4947 m_out.appendTo(continuation, lastNext);
4948 return;
4949 }
4950
4951 case Array::Int8Array:
4952 case Array::Int16Array:
4953 case Array::Int32Array:
4954 case Array::Uint8Array:
4955 case Array::Uint8ClampedArray:
4956 case Array::Uint16Array:
4957 case Array::Uint32Array:
4958 case Array::Float32Array:
4959 case Array::Float64Array: {
4960 TypedArrayType type = arrayMode.typedArrayType();
4961
4962 ASSERT(isTypedView(type));
4963 {
4964 TypedPointer pointer = TypedPointer(
4965 m_heaps.typedArrayProperties,
4966 m_out.add(
4967 storage,
4968 m_out.shl(
4969 m_out.zeroExt(index, pointerType()),
4970 m_out.constIntPtr(logElementSize(type)))));
4971
4972 LValue valueToStore;
4973
4974 if (isInt(type)) {
4975 LValue intValue = getIntTypedArrayStoreOperand(child3, isClamped(type));
4976
4977 valueToStore = intValue;
4978 } else /* !isInt(type) */ {
4979 LValue value = lowDouble(child3);
4980 switch (type) {
4981 case TypeFloat32:
4982 valueToStore = m_out.doubleToFloat(value);
4983 break;
4984 case TypeFloat64:
4985 valueToStore = value;
4986 break;
4987 default:
4988 DFG_CRASH(m_graph, m_node, "Bad typed array type");
4989 }
4990 }
4991
4992 if (arrayMode.isInBounds() || m_node->op() == PutByValAlias)
4993 m_out.store(valueToStore, pointer, storeType(type));
4994 else {
4995 LBasicBlock isInBounds = m_out.newBlock();
4996 LBasicBlock isOutOfBounds = m_out.newBlock();
4997 LBasicBlock continuation = m_out.newBlock();
4998
4999 m_out.branch(
5000 m_out.aboveOrEqual(index, lowInt32(child5)),
5001 unsure(isOutOfBounds), unsure(isInBounds));
5002
5003 LBasicBlock lastNext = m_out.appendTo(isInBounds, isOutOfBounds);
5004 m_out.store(valueToStore, pointer, storeType(type));
5005 m_out.jump(continuation);
5006
5007 m_out.appendTo(isOutOfBounds, continuation);
5008 speculateTypedArrayIsNotNeutered(base);
5009 m_out.jump(continuation);
5010
5011 m_out.appendTo(continuation, lastNext);
5012 }
5013
5014 return;
5015 }
5016 }
5017
5018 case Array::AnyTypedArray:
5019 case Array::String:
5020 case Array::DirectArguments:
5021 case Array::ForceExit:
5022 case Array::Generic:
5023 case Array::ScopedArguments:
5024 case Array::SelectUsingArguments:
5025 case Array::SelectUsingPredictions:
5026 case Array::Undecided:
5027 case Array::Unprofiled:
5028 DFG_CRASH(m_graph, m_node, "Bad array type");
5029 break;
5030 }
5031 }
5032
5033 void compilePutAccessorById()
5034 {
5035 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5036 LValue base = lowCell(m_node->child1());
5037 LValue accessor = lowCell(m_node->child2());
5038 auto uid = m_graph.identifiers()[m_node->identifierNumber()];
5039 vmCall(
5040 Void,
5041 m_node->op() == PutGetterById ? operationPutGetterById : operationPutSetterById,
5042 weakPointer(globalObject), base, m_out.constIntPtr(uid), m_out.constInt32(m_node->accessorAttributes()), accessor);
5043 }
5044
5045 void compilePutGetterSetterById()
5046 {
5047 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5048 LValue base = lowCell(m_node->child1());
5049 LValue getter = lowJSValue(m_node->child2());
5050 LValue setter = lowJSValue(m_node->child3());
5051 auto uid = m_graph.identifiers()[m_node->identifierNumber()];
5052 vmCall(
5053 Void, operationPutGetterSetter,
5054 weakPointer(globalObject), base, m_out.constIntPtr(uid), m_out.constInt32(m_node->accessorAttributes()), getter, setter);
5055
5056 }
5057
5058 void compilePutAccessorByVal()
5059 {
5060 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5061 LValue base = lowCell(m_node->child1());
5062 LValue subscript = lowJSValue(m_node->child2());
5063 LValue accessor = lowCell(m_node->child3());
5064 vmCall(
5065 Void,
5066 m_node->op() == PutGetterByVal ? operationPutGetterByVal : operationPutSetterByVal,
5067 weakPointer(globalObject), base, subscript, m_out.constInt32(m_node->accessorAttributes()), accessor);
5068 }
5069
5070 void compileDeleteById()
5071 {
5072 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5073 LValue base = lowJSValue(m_node->child1());
5074 auto uid = m_graph.identifiers()[m_node->identifierNumber()];
5075 setBoolean(m_out.notZero64(vmCall(Int64, operationDeleteById, weakPointer(globalObject), base, m_out.constIntPtr(uid))));
5076 }
5077
5078 void compileDeleteByVal()
5079 {
5080 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5081 LValue base = lowJSValue(m_node->child1());
5082 LValue subscript = lowJSValue(m_node->child2());
5083 setBoolean(m_out.notZero64(vmCall(Int64, operationDeleteByVal, weakPointer(globalObject), base, subscript)));
5084 }
5085
5086 void compileArrayPush()
5087 {
5088 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5089 LValue base = lowCell(m_graph.varArgChild(m_node, 1));
5090 LValue storage = lowStorage(m_graph.varArgChild(m_node, 0));
5091 unsigned elementOffset = 2;
5092 unsigned elementCount = m_node->numChildren() - elementOffset;
5093
5094 switch (m_node->arrayMode().type()) {
5095 case Array::Int32:
5096 case Array::Contiguous:
5097 case Array::Double: {
5098 IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
5099
5100 if (elementCount == 1) {
5101 LValue value;
5102 Output::StoreType storeType;
5103
5104 Edge& element = m_graph.varArgChild(m_node, elementOffset);
5105 speculate(element);
5106 if (m_node->arrayMode().type() != Array::Double) {
5107 value = lowJSValue(element, ManualOperandSpeculation);
5108 storeType = Output::Store64;
5109 } else {
5110 value = lowDouble(element);
5111 storeType = Output::StoreDouble;
5112 }
5113
5114 LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
5115
5116 LBasicBlock fastPath = m_out.newBlock();
5117 LBasicBlock slowPath = m_out.newBlock();
5118 LBasicBlock continuation = m_out.newBlock();
5119
5120 m_out.branch(
5121 m_out.aboveOrEqual(
5122 prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
5123 unsure(slowPath), unsure(fastPath));
5124
5125 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
5126 m_out.store(
5127 value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), storeType);
5128 LValue newLength = m_out.add(prevLength, m_out.int32One);
5129 m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
5130
5131 ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
5132 m_out.jump(continuation);
5133
5134 m_out.appendTo(slowPath, continuation);
5135 LValue result;
5136 if (m_node->arrayMode().type() != Array::Double)
5137 result = vmCall(Int64, operationArrayPush, weakPointer(globalObject), value, base);
5138 else
5139 result = vmCall(Int64, operationArrayPushDouble, weakPointer(globalObject), value, base);
5140 ValueFromBlock slowResult = m_out.anchor(result);
5141 m_out.jump(continuation);
5142
5143 m_out.appendTo(continuation, lastNext);
5144 setJSValue(m_out.phi(Int64, fastResult, slowResult));
5145 return;
5146 }
5147
5148 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
5149 Edge element = m_graph.varArgChild(m_node, elementIndex + elementOffset);
5150 speculate(element);
5151 }
5152
5153 LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
5154 LValue newLength = m_out.add(prevLength, m_out.constInt32(elementCount));
5155
5156 LBasicBlock fastPath = m_out.newBlock();
5157 LBasicBlock slowPath = m_out.newBlock();
5158 LBasicBlock setup = m_out.newBlock();
5159 LBasicBlock slowCallPath = m_out.newBlock();
5160 LBasicBlock continuation = m_out.newBlock();
5161
5162 LValue beyondVectorLength = m_out.above(newLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength));
5163
5164 m_out.branch(beyondVectorLength, unsure(slowPath), unsure(fastPath));
5165
5166 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
5167 m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
5168 ValueFromBlock fastBufferResult = m_out.anchor(m_out.baseIndex(storage, m_out.zeroExtPtr(prevLength), ScaleEight));
5169 m_out.jump(setup);
5170
5171 m_out.appendTo(slowPath, setup);
5172 size_t scratchSize = sizeof(EncodedJSValue) * elementCount;
5173 static_assert(sizeof(EncodedJSValue) == sizeof(double), "");
5174 ASSERT(scratchSize);
5175 ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
5176 m_out.storePtr(m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->addressOfActiveLength()));
5177 ValueFromBlock slowBufferResult = m_out.anchor(m_out.constIntPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())));
5178 m_out.jump(setup);
5179
5180 m_out.appendTo(setup, slowCallPath);
5181 LValue buffer = m_out.phi(pointerType(), fastBufferResult, slowBufferResult);
5182 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
5183 Edge& element = m_graph.varArgChild(m_node, elementIndex + elementOffset);
5184
5185 LValue value;
5186 Output::StoreType storeType;
5187 if (m_node->arrayMode().type() != Array::Double) {
5188 value = lowJSValue(element, ManualOperandSpeculation);
5189 storeType = Output::Store64;
5190 } else {
5191 value = lowDouble(element);
5192 storeType = Output::StoreDouble;
5193 }
5194
5195 m_out.store(value, m_out.baseIndex(heap, buffer, m_out.constInt32(elementIndex), jsNumber(elementIndex)), storeType);
5196 }
5197 ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
5198
5199 m_out.branch(beyondVectorLength, unsure(slowCallPath), unsure(continuation));
5200
5201 m_out.appendTo(slowCallPath, continuation);
5202 auto* operation = &operationArrayPushMultiple;
5203 if (m_node->arrayMode().type() == Array::Double)
5204 operation = &operationArrayPushDoubleMultiple;
5205 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operation, weakPointer(globalObject), base, buffer, m_out.constInt32(elementCount)));
5206 m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->addressOfActiveLength()));
5207 m_out.jump(continuation);
5208
5209 m_out.appendTo(continuation, lastNext);
5210 setJSValue(m_out.phi(Int64, fastResult, slowResult));
5211 return;
5212 }
5213
5214 case Array::ArrayStorage: {
5215 // This ensures that the result of ArrayPush is Int32 in AI.
5216 int32_t largestPositiveInt32Length = 0x7fffffff - elementCount;
5217
5218 LValue prevLength = m_out.load32(storage, m_heaps.ArrayStorage_publicLength);
5219 // Refuse to handle bizarre lengths.
5220 speculate(Uncountable, noValue(), nullptr, m_out.above(prevLength, m_out.constInt32(largestPositiveInt32Length)));
5221
5222 if (elementCount == 1) {
5223 Edge& element = m_graph.varArgChild(m_node, elementOffset);
5224
5225 LValue value = lowJSValue(element);
5226
5227 LBasicBlock fastPath = m_out.newBlock();
5228 LBasicBlock slowPath = m_out.newBlock();
5229 LBasicBlock continuation = m_out.newBlock();
5230
5231 m_out.branch(
5232 m_out.aboveOrEqual(
5233 prevLength, m_out.load32(storage, m_heaps.ArrayStorage_vectorLength)),
5234 rarely(slowPath), usually(fastPath));
5235
5236 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
5237 m_out.store64(
5238 value, m_out.baseIndex(m_heaps.ArrayStorage_vector, storage, m_out.zeroExtPtr(prevLength)));
5239 LValue newLength = m_out.add(prevLength, m_out.int32One);
5240 m_out.store32(newLength, storage, m_heaps.ArrayStorage_publicLength);
5241 m_out.store32(
5242 m_out.add(m_out.load32(storage, m_heaps.ArrayStorage_numValuesInVector), m_out.int32One),
5243 storage, m_heaps.ArrayStorage_numValuesInVector);
5244
5245 ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
5246 m_out.jump(continuation);
5247
5248 m_out.appendTo(slowPath, continuation);
5249 ValueFromBlock slowResult = m_out.anchor(
5250 vmCall(Int64, operationArrayPush, weakPointer(globalObject), value, base));
5251 m_out.jump(continuation);
5252
5253 m_out.appendTo(continuation, lastNext);
5254 setJSValue(m_out.phi(Int64, fastResult, slowResult));
5255 return;
5256 }
5257
5258 LValue newLength = m_out.add(prevLength, m_out.constInt32(elementCount));
5259
5260 LBasicBlock fastPath = m_out.newBlock();
5261 LBasicBlock slowPath = m_out.newBlock();
5262 LBasicBlock setup = m_out.newBlock();
5263 LBasicBlock slowCallPath = m_out.newBlock();
5264 LBasicBlock continuation = m_out.newBlock();
5265
5266 LValue beyondVectorLength = m_out.above(newLength, m_out.load32(storage, m_heaps.ArrayStorage_vectorLength));
5267
5268 m_out.branch(beyondVectorLength, rarely(slowPath), usually(fastPath));
5269
5270 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
5271 m_out.store32(newLength, storage, m_heaps.ArrayStorage_publicLength);
5272 m_out.store32(
5273 m_out.add(m_out.load32(storage, m_heaps.ArrayStorage_numValuesInVector), m_out.constInt32(elementCount)),
5274 storage, m_heaps.ArrayStorage_numValuesInVector);
5275 ValueFromBlock fastBufferResult = m_out.anchor(m_out.baseIndex(storage, m_out.zeroExtPtr(prevLength), ScaleEight, ArrayStorage::vectorOffset()));
5276 m_out.jump(setup);
5277
5278 m_out.appendTo(slowPath, setup);
5279 size_t scratchSize = sizeof(EncodedJSValue) * elementCount;
5280 ASSERT(scratchSize);
5281 ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
5282 m_out.storePtr(m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->addressOfActiveLength()));
5283 ValueFromBlock slowBufferResult = m_out.anchor(m_out.constIntPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())));
5284 m_out.jump(setup);
5285
5286 m_out.appendTo(setup, slowCallPath);
5287 LValue buffer = m_out.phi(pointerType(), fastBufferResult, slowBufferResult);
5288 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
5289 Edge& element = m_graph.varArgChild(m_node, elementIndex + elementOffset);
5290
5291 LValue value = lowJSValue(element);
5292 m_out.store64(value, m_out.baseIndex(m_heaps.ArrayStorage_vector.atAnyIndex(), buffer, m_out.constIntPtr(elementIndex), ScaleEight));
5293 }
5294 ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
5295
5296 m_out.branch(beyondVectorLength, rarely(slowCallPath), usually(continuation));
5297
5298 m_out.appendTo(slowCallPath, continuation);
5299 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operationArrayPushMultiple, weakPointer(globalObject), base, buffer, m_out.constInt32(elementCount)));
5300 m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->addressOfActiveLength()));
5301 m_out.jump(continuation);
5302
5303 m_out.appendTo(continuation, lastNext);
5304 setJSValue(m_out.phi(Int64, fastResult, slowResult));
5305 return;
5306 }
5307
5308 default:
5309 DFG_CRASH(m_graph, m_node, "Bad array type");
5310 return;
5311 }
5312 }
5313
5314 std::pair<LValue, LValue> populateSliceRange(LValue start, LValue end, LValue length)
5315 {
5316 // end can be nullptr.
5317 ASSERT(start);
5318 ASSERT(length);
5319
5320 auto pickIndex = [&] (LValue index) {
5321 return m_out.select(m_out.greaterThanOrEqual(index, m_out.int32Zero),
5322 m_out.select(m_out.above(index, length), length, index),
5323 m_out.select(m_out.lessThan(m_out.add(length, index), m_out.int32Zero), m_out.int32Zero, m_out.add(length, index)));
5324 };
5325
5326 LValue endBoundary = length;
5327 if (end)
5328 endBoundary = pickIndex(end);
5329 LValue startIndex = pickIndex(start);
5330 return std::make_pair(startIndex, endBoundary);
5331 }
5332
5333 void compileArraySlice()
5334 {
5335 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5336
5337 LValue sourceArray = lowCell(m_graph.varArgChild(m_node, 0));
5338 LValue sourceStorage = lowStorage(m_graph.varArgChild(m_node, m_node->numChildren() - 1));
5339 LValue inputLength = m_out.load32(sourceStorage, m_heaps.Butterfly_publicLength);
5340
5341 LValue startIndex = nullptr;
5342 LValue resultLength = nullptr;
5343 if (m_node->numChildren() == 2) {
5344 startIndex = m_out.constInt32(0);
5345 resultLength = inputLength;
5346 } else {
5347 LValue start = lowInt32(m_graph.varArgChild(m_node, 1));
5348 LValue end = nullptr;
5349 if (m_node->numChildren() != 3)
5350 end = lowInt32(m_graph.varArgChild(m_node, 2));
5351
5352 auto range = populateSliceRange(start, end, inputLength);
5353 startIndex = range.first;
5354 LValue endBoundary = range.second;
5355
5356 resultLength = m_out.select(m_out.belowOrEqual(startIndex, endBoundary),
5357 m_out.sub(endBoundary, startIndex),
5358 m_out.constInt32(0));
5359 }
5360
5361 ArrayValues arrayResult;
5362 {
5363 LValue indexingType = m_out.load8ZeroExt32(sourceArray, m_heaps.JSCell_indexingTypeAndMisc);
5364 // We can ignore the writability of the cell since we won't write to the source.
5365 indexingType = m_out.bitAnd(indexingType, m_out.constInt32(AllWritableArrayTypesAndHistory));
5366 // When we emit an ArraySlice, we dominate the use of the array by a CheckStructure
5367 // to ensure the incoming array is one to be one of the original array structures
5368 // with one of the following indexing shapes: Int32, Contiguous, Double.
5369 LValue structure = m_out.select(
5370 m_out.equal(indexingType, m_out.constInt32(ArrayWithInt32)),
5371 weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithInt32))),
5372 m_out.select(m_out.equal(indexingType, m_out.constInt32(ArrayWithContiguous)),
5373 weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithContiguous))),
5374 weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithDouble)))));
5375 arrayResult = allocateJSArray(resultLength, resultLength, structure, indexingType, false, false);
5376 }
5377
5378 // Keep the sourceArray alive at least until after anything that can GC.
5379 keepAlive(sourceArray);
5380
5381 LBasicBlock loop = m_out.newBlock();
5382 LBasicBlock continuation = m_out.newBlock();
5383
5384 resultLength = m_out.zeroExtPtr(resultLength);
5385 ValueFromBlock startLoadIndex = m_out.anchor(m_out.zeroExtPtr(startIndex));
5386 ValueFromBlock startStoreIndex = m_out.anchor(m_out.constIntPtr(0));
5387
5388 m_out.branch(
5389 m_out.below(m_out.constIntPtr(0), resultLength), unsure(loop), unsure(continuation));
5390
5391 LBasicBlock lastNext = m_out.appendTo(loop, continuation);
5392 LValue storeIndex = m_out.phi(pointerType(), startStoreIndex);
5393 LValue loadIndex = m_out.phi(pointerType(), startLoadIndex);
5394 LValue value = m_out.load64(m_out.baseIndex(m_heaps.root, sourceStorage, loadIndex, ScaleEight));
5395 m_out.store64(value, m_out.baseIndex(m_heaps.root, arrayResult.butterfly, storeIndex, ScaleEight));
5396 LValue nextStoreIndex = m_out.add(storeIndex, m_out.constIntPtr(1));
5397 m_out.addIncomingToPhi(storeIndex, m_out.anchor(nextStoreIndex));
5398 m_out.addIncomingToPhi(loadIndex, m_out.anchor(m_out.add(loadIndex, m_out.constIntPtr(1))));
5399 m_out.branch(
5400 m_out.below(nextStoreIndex, resultLength), unsure(loop), unsure(continuation));
5401
5402 m_out.appendTo(continuation, lastNext);
5403
5404 mutatorFence();
5405 setJSValue(arrayResult.array);
5406 }
5407
5408 void compileArrayIndexOf()
5409 {
5410 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5411 LValue storage = lowStorage(m_node->numChildren() == 3 ? m_graph.varArgChild(m_node, 2) : m_graph.varArgChild(m_node, 3));
5412 LValue length = m_out.load32(storage, m_heaps.Butterfly_publicLength);
5413
5414 LValue startIndex;
5415 if (m_node->numChildren() == 4) {
5416 startIndex = lowInt32(m_graph.varArgChild(m_node, 2));
5417 startIndex = m_out.select(m_out.greaterThanOrEqual(startIndex, m_out.int32Zero),
5418 m_out.select(m_out.above(startIndex, length), length, startIndex),
5419 m_out.select(m_out.lessThan(m_out.add(length, startIndex), m_out.int32Zero), m_out.int32Zero, m_out.add(length, startIndex)));
5420 } else
5421 startIndex = m_out.int32Zero;
5422
5423 Edge& searchElementEdge = m_graph.varArgChild(m_node, 1);
5424 switch (searchElementEdge.useKind()) {
5425 case Int32Use:
5426 case ObjectUse:
5427 case SymbolUse:
5428 case OtherUse:
5429 case DoubleRepUse: {
5430 LBasicBlock loopHeader = m_out.newBlock();
5431 LBasicBlock loopBody = m_out.newBlock();
5432 LBasicBlock loopNext = m_out.newBlock();
5433 LBasicBlock notFound = m_out.newBlock();
5434 LBasicBlock continuation = m_out.newBlock();
5435
5436 LValue searchElement;
5437 switch (searchElementEdge.useKind()) {
5438 case Int32Use:
5439 ASSERT(m_node->arrayMode().type() == Array::Int32);
5440 speculate(searchElementEdge);
5441 searchElement = lowJSValue(searchElementEdge, ManualOperandSpeculation);
5442 break;
5443 case ObjectUse:
5444 ASSERT(m_node->arrayMode().type() == Array::Contiguous);
5445 searchElement = lowObject(searchElementEdge);
5446 break;
5447 case SymbolUse:
5448 ASSERT(m_node->arrayMode().type() == Array::Contiguous);
5449 searchElement = lowSymbol(searchElementEdge);
5450 break;
5451 case OtherUse:
5452 ASSERT(m_node->arrayMode().type() == Array::Contiguous);
5453 speculate(searchElementEdge);
5454 searchElement = lowJSValue(searchElementEdge, ManualOperandSpeculation);
5455 break;
5456 case DoubleRepUse:
5457 ASSERT(m_node->arrayMode().type() == Array::Double);
5458 searchElement = lowDouble(searchElementEdge);
5459 break;
5460 default:
5461 RELEASE_ASSERT_NOT_REACHED();
5462 break;
5463 }
5464
5465 startIndex = m_out.zeroExtPtr(startIndex);
5466 length = m_out.zeroExtPtr(length);
5467
5468 ValueFromBlock initialStartIndex = m_out.anchor(startIndex);
5469 m_out.jump(loopHeader);
5470
5471 LBasicBlock lastNext = m_out.appendTo(loopHeader, loopBody);
5472 LValue index = m_out.phi(pointerType(), initialStartIndex);
5473 m_out.branch(m_out.notEqual(index, length), unsure(loopBody), unsure(notFound));
5474
5475 m_out.appendTo(loopBody, loopNext);
5476 ValueFromBlock foundResult = m_out.anchor(index);
5477 switch (searchElementEdge.useKind()) {
5478 case Int32Use: {
5479 // Empty value is ignored because of JSValue::NumberTag.
5480 LValue value = m_out.load64(m_out.baseIndex(m_heaps.indexedInt32Properties, storage, index));
5481 m_out.branch(m_out.equal(value, searchElement), unsure(continuation), unsure(loopNext));
5482 break;
5483 }
5484 case ObjectUse:
5485 case SymbolUse:
5486 case OtherUse: {
5487 // Empty value never matches against non-empty JS values.
5488 LValue value = m_out.load64(m_out.baseIndex(m_heaps.indexedContiguousProperties, storage, index));
5489 m_out.branch(m_out.equal(value, searchElement), unsure(continuation), unsure(loopNext));
5490 break;
5491 }
5492 case DoubleRepUse: {
5493 // Empty value is ignored because of NaN.
5494 LValue value = m_out.loadDouble(m_out.baseIndex(m_heaps.indexedDoubleProperties, storage, index));
5495 m_out.branch(m_out.doubleEqual(value, searchElement), unsure(continuation), unsure(loopNext));
5496 break;
5497 }
5498 default:
5499 RELEASE_ASSERT_NOT_REACHED();
5500 break;
5501 }
5502
5503 m_out.appendTo(loopNext, notFound);
5504 LValue nextIndex = m_out.add(index, m_out.intPtrOne);
5505 m_out.addIncomingToPhi(index, m_out.anchor(nextIndex));
5506 m_out.jump(loopHeader);
5507
5508 m_out.appendTo(notFound, continuation);
5509 ValueFromBlock notFoundResult = m_out.anchor(m_out.constIntPtr(-1));
5510 m_out.jump(continuation);
5511
5512 m_out.appendTo(continuation, lastNext);
5513 setInt32(m_out.castToInt32(m_out.phi(pointerType(), notFoundResult, foundResult)));
5514 break;
5515 }
5516
5517 case StringUse:
5518 ASSERT(m_node->arrayMode().type() == Array::Contiguous);
5519 setInt32(vmCall(Int32, operationArrayIndexOfString, weakPointer(globalObject), storage, lowString(searchElementEdge), startIndex));
5520 break;
5521
5522 case UntypedUse:
5523 switch (m_node->arrayMode().type()) {
5524 case Array::Double:
5525 setInt32(vmCall(Int32, operationArrayIndexOfValueDouble, weakPointer(globalObject), storage, lowJSValue(searchElementEdge), startIndex));
5526 break;
5527 case Array::Int32:
5528 case Array::Contiguous:
5529 setInt32(vmCall(Int32, operationArrayIndexOfValueInt32OrContiguous, weakPointer(globalObject), storage, lowJSValue(searchElementEdge), startIndex));
5530 break;
5531 default:
5532 RELEASE_ASSERT_NOT_REACHED();
5533 break;
5534 }
5535 break;
5536
5537 default:
5538 RELEASE_ASSERT_NOT_REACHED();
5539 break;
5540 }
5541 }
5542
5543
5544 void compileArrayPop()
5545 {
5546 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5547 LValue base = lowCell(m_node->child1());
5548 LValue storage = lowStorage(m_node->child2());
5549
5550 switch (m_node->arrayMode().type()) {
5551 case Array::Int32:
5552 case Array::Double:
5553 case Array::Contiguous: {
5554 IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
5555
5556 LBasicBlock fastCase = m_out.newBlock();
5557 LBasicBlock slowCase = m_out.newBlock();
5558 LBasicBlock continuation = m_out.newBlock();
5559
5560 LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
5561
5562 Vector<ValueFromBlock, 3> results;
5563 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
5564 m_out.branch(
5565 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
5566
5567 LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
5568 LValue newLength = m_out.sub(prevLength, m_out.int32One);
5569 m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
5570 TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength));
5571 if (m_node->arrayMode().type() != Array::Double) {
5572 LValue result = m_out.load64(pointer);
5573 m_out.store64(m_out.int64Zero, pointer);
5574 results.append(m_out.anchor(result));
5575 m_out.branch(
5576 m_out.notZero64(result), usually(continuation), rarely(slowCase));
5577 } else {
5578 LValue result = m_out.loadDouble(pointer);
5579 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
5580 results.append(m_out.anchor(boxDouble(result)));
5581 m_out.branch(
5582 m_out.doubleEqual(result, result),
5583 usually(continuation), rarely(slowCase));
5584 }
5585
5586 m_out.appendTo(slowCase, continuation);
5587 results.append(m_out.anchor(vmCall(
5588 Int64, operationArrayPopAndRecoverLength, weakPointer(globalObject), base)));
5589 m_out.jump(continuation);
5590
5591 m_out.appendTo(continuation, lastNext);
5592 setJSValue(m_out.phi(Int64, results));
5593 return;
5594 }
5595
5596 case Array::ArrayStorage: {
5597 LBasicBlock vectorLengthCheckCase = m_out.newBlock();
5598 LBasicBlock popCheckCase = m_out.newBlock();
5599 LBasicBlock fastCase = m_out.newBlock();
5600 LBasicBlock slowCase = m_out.newBlock();
5601 LBasicBlock continuation = m_out.newBlock();
5602
5603 LValue prevLength = m_out.load32(storage, m_heaps.ArrayStorage_publicLength);
5604
5605 Vector<ValueFromBlock, 3> results;
5606 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
5607 m_out.branch(
5608 m_out.isZero32(prevLength), rarely(continuation), usually(vectorLengthCheckCase));
5609
5610 LBasicBlock lastNext = m_out.appendTo(vectorLengthCheckCase, popCheckCase);
5611 LValue newLength = m_out.sub(prevLength, m_out.int32One);
5612 m_out.branch(
5613 m_out.aboveOrEqual(newLength, m_out.load32(storage, m_heaps.ArrayStorage_vectorLength)), rarely(slowCase), usually(popCheckCase));
5614
5615 m_out.appendTo(popCheckCase, fastCase);
5616 TypedPointer pointer = m_out.baseIndex(m_heaps.ArrayStorage_vector, storage, m_out.zeroExtPtr(newLength));
5617 LValue result = m_out.load64(pointer);
5618 m_out.branch(m_out.notZero64(result), usually(fastCase), rarely(slowCase));
5619
5620 m_out.appendTo(fastCase, slowCase);
5621 m_out.store32(newLength, storage, m_heaps.ArrayStorage_publicLength);
5622 m_out.store64(m_out.int64Zero, pointer);
5623 m_out.store32(
5624 m_out.sub(m_out.load32(storage, m_heaps.ArrayStorage_numValuesInVector), m_out.int32One),
5625 storage, m_heaps.ArrayStorage_numValuesInVector);
5626 results.append(m_out.anchor(result));
5627 m_out.jump(continuation);
5628
5629 m_out.appendTo(slowCase, continuation);
5630 results.append(m_out.anchor(vmCall(Int64, operationArrayPop, weakPointer(globalObject), base)));
5631 m_out.jump(continuation);
5632
5633 m_out.appendTo(continuation, lastNext);
5634 setJSValue(m_out.phi(Int64, results));
5635 return;
5636 }
5637
5638 default:
5639 DFG_CRASH(m_graph, m_node, "Bad array type");
5640 return;
5641 }
5642 }
5643
5644 void compilePushWithScope()
5645 {
5646 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5647 LValue parentScope = lowCell(m_node->child1());
5648 auto objectEdge = m_node->child2();
5649 if (objectEdge.useKind() == ObjectUse) {
5650 LValue object = lowNonNullObject(objectEdge);
5651 LValue result = vmCall(Int64, operationPushWithScopeObject, weakPointer(globalObject), parentScope, object);
5652 setJSValue(result);
5653 } else {
5654 ASSERT(objectEdge.useKind() == UntypedUse);
5655 LValue object = lowJSValue(m_node->child2());
5656 LValue result = vmCall(Int64, operationPushWithScope, weakPointer(globalObject), parentScope, object);
5657 setJSValue(result);
5658 }
5659 }
5660
5661 void compileCreateActivation()
5662 {
5663 LValue scope = lowCell(m_node->child1());
5664 SymbolTable* table = m_node->castOperand<SymbolTable*>();
5665 RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure());
5666 JSValue initializationValue = m_node->initializationValueForActivation();
5667 ASSERT(initializationValue.isUndefined() || initializationValue == jsTDZValue());
5668 if (table->singleton().isStillValid()) {
5669 LValue callResult = vmCall(
5670 Int64,
5671 operationCreateActivationDirect, m_vmValue, weakStructure(structure),
5672 scope, weakPointer(table), m_out.constInt64(JSValue::encode(initializationValue)));
5673 setJSValue(callResult);
5674 return;
5675 }
5676
5677 LBasicBlock slowPath = m_out.newBlock();
5678 LBasicBlock continuation = m_out.newBlock();
5679
5680 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5681
5682 LValue fastObject = allocateObject<JSLexicalEnvironment>(
5683 JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
5684
5685 // We don't need memory barriers since we just fast-created the activation, so the
5686 // activation must be young.
5687 m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
5688 m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
5689
5690 for (unsigned i = 0; i < table->scopeSize(); ++i) {
5691 m_out.store64(
5692 m_out.constInt64(JSValue::encode(initializationValue)),
5693 fastObject, m_heaps.JSLexicalEnvironment_variables[i]);
5694 }
5695
5696 mutatorFence();
5697
5698 ValueFromBlock fastResult = m_out.anchor(fastObject);
5699 m_out.jump(continuation);
5700
5701 m_out.appendTo(slowPath, continuation);
5702 VM& vm = this->vm();
5703 LValue callResult = lazySlowPath(
5704 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
5705 return createLazyCallGenerator(vm,
5706 operationCreateActivationDirect, locations[0].directGPR(), &vm,
5707 CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(),
5708 CCallHelpers::TrustedImmPtr(table),
5709 CCallHelpers::TrustedImm64(JSValue::encode(initializationValue)));
5710 },
5711 scope);
5712 ValueFromBlock slowResult = m_out.anchor(callResult);
5713 m_out.jump(continuation);
5714
5715 m_out.appendTo(continuation, lastNext);
5716 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
5717 }
5718
5719 void compileNewFunction()
5720 {
5721 ASSERT(m_node->op() == NewFunction || m_node->op() == NewGeneratorFunction || m_node->op() == NewAsyncGeneratorFunction || m_node->op() == NewAsyncFunction);
5722 bool isGeneratorFunction = m_node->op() == NewGeneratorFunction;
5723 bool isAsyncFunction = m_node->op() == NewAsyncFunction;
5724 bool isAsyncGeneratorFunction = m_node->op() == NewAsyncGeneratorFunction;
5725
5726 LValue scope = lowCell(m_node->child1());
5727
5728 FunctionExecutable* executable = m_node->castOperand<FunctionExecutable*>();
5729 if (executable->singleton().isStillValid()) {
5730 LValue callResult =
5731 isGeneratorFunction ? vmCall(Int64, operationNewGeneratorFunction, m_vmValue, scope, weakPointer(executable)) :
5732 isAsyncFunction ? vmCall(Int64, operationNewAsyncFunction, m_vmValue, scope, weakPointer(executable)) :
5733 isAsyncGeneratorFunction ? vmCall(Int64, operationNewAsyncGeneratorFunction, m_vmValue, scope, weakPointer(executable)) :
5734 vmCall(Int64, operationNewFunction, m_vmValue, scope, weakPointer(executable));
5735 setJSValue(callResult);
5736 return;
5737 }
5738
5739 RegisteredStructure structure = m_graph.registerStructure(
5740 [&] () {
5741 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5742 switch (m_node->op()) {
5743 case NewGeneratorFunction:
5744 return globalObject->generatorFunctionStructure();
5745 case NewAsyncFunction:
5746 return globalObject->asyncFunctionStructure();
5747 case NewAsyncGeneratorFunction:
5748 return globalObject->asyncGeneratorFunctionStructure();
5749 case NewFunction:
5750 return JSFunction::selectStructureForNewFuncExp(globalObject, m_node->castOperand<FunctionExecutable*>());
5751 default:
5752 RELEASE_ASSERT_NOT_REACHED();
5753 }
5754 }());
5755
5756 LBasicBlock slowPath = m_out.newBlock();
5757 LBasicBlock continuation = m_out.newBlock();
5758
5759 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5760
5761 LValue fastObject =
5762 isGeneratorFunction ? allocateObject<JSGeneratorFunction>(structure, m_out.intPtrZero, slowPath) :
5763 isAsyncFunction ? allocateObject<JSAsyncFunction>(structure, m_out.intPtrZero, slowPath) :
5764 isAsyncGeneratorFunction ? allocateObject<JSAsyncGeneratorFunction>(structure, m_out.intPtrZero, slowPath) :
5765 allocateObject<JSFunction>(structure, m_out.intPtrZero, slowPath);
5766
5767
5768 // We don't need memory barriers since we just fast-created the function, so it
5769 // must be young.
5770 m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope);
5771 m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable);
5772 m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData);
5773 m_out.storePtr(weakPointer(structure->globalObject()), fastObject, m_heaps.JSFunction_globalObject);
5774 mutatorFence();
5775
5776 ValueFromBlock fastResult = m_out.anchor(fastObject);
5777 m_out.jump(continuation);
5778
5779 m_out.appendTo(slowPath, continuation);
5780
5781 Vector<LValue> slowPathArguments;
5782 slowPathArguments.append(scope);
5783 VM& vm = this->vm();
5784 LValue callResult = lazySlowPath(
5785 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
5786 auto* operation = operationNewFunctionWithInvalidatedReallocationWatchpoint;
5787 if (isGeneratorFunction)
5788 operation = operationNewGeneratorFunctionWithInvalidatedReallocationWatchpoint;
5789 else if (isAsyncFunction)
5790 operation = operationNewAsyncFunctionWithInvalidatedReallocationWatchpoint;
5791 else if (isAsyncGeneratorFunction)
5792 operation = operationNewAsyncGeneratorFunctionWithInvalidatedReallocationWatchpoint;
5793
5794 return createLazyCallGenerator(vm, operation,
5795 locations[0].directGPR(), &vm, locations[1].directGPR(),
5796 CCallHelpers::TrustedImmPtr(executable));
5797 },
5798 slowPathArguments);
5799 ValueFromBlock slowResult = m_out.anchor(callResult);
5800 m_out.jump(continuation);
5801
5802 m_out.appendTo(continuation, lastNext);
5803 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
5804 }
5805
5806 void compileCreateDirectArguments()
5807 {
5808 // FIXME: A more effective way of dealing with the argument count and callee is to have
5809 // them be explicit arguments to this node.
5810 // https://bugs.webkit.org/show_bug.cgi?id=142207
5811
5812 RegisteredStructure structure =
5813 m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure());
5814
5815 unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1;
5816
5817 LBasicBlock slowPath = m_out.newBlock();
5818 LBasicBlock continuation = m_out.newBlock();
5819
5820 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5821
5822 ArgumentsLength length = getArgumentsLength();
5823
5824 LValue fastObject;
5825 if (length.isKnown) {
5826 fastObject = allocateObject<DirectArguments>(
5827 DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure,
5828 m_out.intPtrZero, slowPath);
5829 } else {
5830 LValue size = m_out.add(
5831 m_out.shl(length.value, m_out.constInt32(3)),
5832 m_out.constInt32(DirectArguments::storageOffset()));
5833
5834 size = m_out.select(
5835 m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
5836 size, m_out.constInt32(DirectArguments::allocationSize(minCapacity)));
5837
5838 fastObject = allocateVariableSizedObject<DirectArguments>(
5839 m_out.zeroExtPtr(size), structure, m_out.intPtrZero, slowPath);
5840 }
5841
5842 m_out.store32(length.value, fastObject, m_heaps.DirectArguments_length);
5843 m_out.store32(m_out.constInt32(minCapacity), fastObject, m_heaps.DirectArguments_minCapacity);
5844 m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_mappedArguments);
5845 m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_modifiedArgumentsDescriptor);
5846
5847 ValueFromBlock fastResult = m_out.anchor(fastObject);
5848 m_out.jump(continuation);
5849
5850 m_out.appendTo(slowPath, continuation);
5851 VM& vm = this->vm();
5852 LValue callResult = lazySlowPath(
5853 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
5854 return createLazyCallGenerator(vm,
5855 operationCreateDirectArguments, locations[0].directGPR(), &vm,
5856 CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(),
5857 CCallHelpers::TrustedImm32(minCapacity));
5858 }, length.value);
5859 ValueFromBlock slowResult = m_out.anchor(callResult);
5860 m_out.jump(continuation);
5861
5862 m_out.appendTo(continuation, lastNext);
5863 LValue result = m_out.phi(pointerType(), fastResult, slowResult);
5864
5865 m_out.storePtr(getCurrentCallee(), result, m_heaps.DirectArguments_callee);
5866
5867 if (length.isKnown) {
5868 VirtualRegister start = AssemblyHelpers::argumentsStart(m_node->origin.semantic);
5869 for (unsigned i = 0; i < std::max(length.known, minCapacity); ++i) {
5870 m_out.store64(
5871 m_out.load64(addressFor(start + i)),
5872 result, m_heaps.DirectArguments_storage[i]);
5873 }
5874 } else {
5875 LValue stackBase = getArgumentsStart();
5876
5877 LBasicBlock loop = m_out.newBlock();
5878 LBasicBlock end = m_out.newBlock();
5879
5880 ValueFromBlock originalLength;
5881 if (minCapacity) {
5882 LValue capacity = m_out.select(
5883 m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
5884 length.value,
5885 m_out.constInt32(minCapacity));
5886 LValue originalLengthValue = m_out.zeroExtPtr(capacity);
5887 originalLength = m_out.anchor(originalLengthValue);
5888 m_out.jump(loop);
5889 } else {
5890 LValue originalLengthValue = m_out.zeroExtPtr(length.value);
5891 originalLength = m_out.anchor(originalLengthValue);
5892 m_out.branch(m_out.isNull(originalLengthValue), unsure(end), unsure(loop));
5893 }
5894
5895 lastNext = m_out.appendTo(loop, end);
5896 LValue previousIndex = m_out.phi(pointerType(), originalLength);
5897 LValue index = m_out.sub(previousIndex, m_out.intPtrOne);
5898 m_out.store64(
5899 m_out.load64(m_out.baseIndex(m_heaps.variables, stackBase, index)),
5900 m_out.baseIndex(m_heaps.DirectArguments_storage, result, index));
5901 ValueFromBlock nextIndex = m_out.anchor(index);
5902 m_out.addIncomingToPhi(previousIndex, nextIndex);
5903 m_out.branch(m_out.isNull(index), unsure(end), unsure(loop));
5904
5905 m_out.appendTo(end, lastNext);
5906 }
5907
5908 mutatorFence();
5909
5910 setJSValue(result);
5911 }
5912
5913 void compileCreateScopedArguments()
5914 {
5915 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5916 LValue scope = lowCell(m_node->child1());
5917
5918 LValue result = vmCall(
5919 Int64, operationCreateScopedArguments, weakPointer(globalObject),
5920 weakPointer(
5921 m_graph.globalObjectFor(m_node->origin.semantic)->scopedArgumentsStructure()),
5922 getArgumentsStart(), getArgumentsLength().value, getCurrentCallee(), scope);
5923
5924 setJSValue(result);
5925 }
5926
5927 void compileCreateClonedArguments()
5928 {
5929 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5930 LValue result = vmCall(
5931 Int64, operationCreateClonedArguments, weakPointer(globalObject),
5932 weakPointer(
5933 m_graph.globalObjectFor(m_node->origin.semantic)->clonedArgumentsStructure()),
5934 getArgumentsStart(), getArgumentsLength().value, getCurrentCallee());
5935
5936 setJSValue(result);
5937 }
5938
5939 void compileCreateRest()
5940 {
5941 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
5942 if (m_graph.isWatchingHavingABadTimeWatchpoint(m_node)) {
5943 LBasicBlock continuation = m_out.newBlock();
5944 LValue arrayLength = lowInt32(m_node->child1());
5945 LBasicBlock loopStart = m_out.newBlock();
5946 RegisteredStructure structure = m_graph.registerStructure(globalObject->originalRestParameterStructure());
5947 ArrayValues arrayValues = allocateUninitializedContiguousJSArray(arrayLength, structure);
5948 LValue array = arrayValues.array;
5949 LValue butterfly = arrayValues.butterfly;
5950 ValueFromBlock startLength = m_out.anchor(arrayLength);
5951 LValue argumentRegion = m_out.add(getArgumentsStart(), m_out.constInt64(sizeof(Register) * m_node->numberOfArgumentsToSkip()));
5952 m_out.branch(m_out.equal(arrayLength, m_out.constInt32(0)),
5953 unsure(continuation), unsure(loopStart));
5954
5955 LBasicBlock lastNext = m_out.appendTo(loopStart, continuation);
5956 LValue phiOffset = m_out.phi(Int32, startLength);
5957 LValue currentOffset = m_out.sub(phiOffset, m_out.int32One);
5958 m_out.addIncomingToPhi(phiOffset, m_out.anchor(currentOffset));
5959 LValue loadedValue = m_out.load64(m_out.baseIndex(m_heaps.variables, argumentRegion, m_out.zeroExtPtr(currentOffset)));
5960 IndexedAbstractHeap& heap = m_heaps.indexedContiguousProperties;
5961 m_out.store64(loadedValue, m_out.baseIndex(heap, butterfly, m_out.zeroExtPtr(currentOffset)));
5962 m_out.branch(m_out.equal(currentOffset, m_out.constInt32(0)), unsure(continuation), unsure(loopStart));
5963
5964 m_out.appendTo(continuation, lastNext);
5965 mutatorFence();
5966 setJSValue(array);
5967 return;
5968 }
5969
5970 LValue arrayLength = lowInt32(m_node->child1());
5971 LValue argumentStart = getArgumentsStart();
5972 LValue numberOfArgumentsToSkip = m_out.constInt32(m_node->numberOfArgumentsToSkip());
5973 setJSValue(vmCall(
5974 Int64, operationCreateRest, weakPointer(globalObject), argumentStart, numberOfArgumentsToSkip, arrayLength));
5975 }
5976
5977 void compileGetRestLength()
5978 {
5979 LBasicBlock nonZeroLength = m_out.newBlock();
5980 LBasicBlock continuation = m_out.newBlock();
5981
5982 ValueFromBlock zeroLengthResult = m_out.anchor(m_out.constInt32(0));
5983
5984 LValue numberOfArgumentsToSkip = m_out.constInt32(m_node->numberOfArgumentsToSkip());
5985 LValue argumentsLength = getArgumentsLength().value;
5986 m_out.branch(m_out.above(argumentsLength, numberOfArgumentsToSkip),
5987 unsure(nonZeroLength), unsure(continuation));
5988
5989 LBasicBlock lastNext = m_out.appendTo(nonZeroLength, continuation);
5990 ValueFromBlock nonZeroLengthResult = m_out.anchor(m_out.sub(argumentsLength, numberOfArgumentsToSkip));
5991 m_out.jump(continuation);
5992
5993 m_out.appendTo(continuation, lastNext);
5994 setInt32(m_out.phi(Int32, zeroLengthResult, nonZeroLengthResult));
5995 }
5996
5997 void compileObjectKeys()
5998 {
5999 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6000 switch (m_node->child1().useKind()) {
6001 case ObjectUse: {
6002 if (m_graph.isWatchingHavingABadTimeWatchpoint(m_node)) {
6003 LBasicBlock notNullCase = m_out.newBlock();
6004 LBasicBlock rareDataCase = m_out.newBlock();
6005 LBasicBlock useCacheCase = m_out.newBlock();
6006 LBasicBlock slowButArrayBufferCase = m_out.newBlock();
6007 LBasicBlock slowCase = m_out.newBlock();
6008 LBasicBlock continuation = m_out.newBlock();
6009
6010 LValue object = lowObject(m_node->child1());
6011 LValue structure = loadStructure(object);
6012 LValue previousOrRareData = m_out.loadPtr(structure, m_heaps.Structure_previousOrRareData);
6013 m_out.branch(m_out.notNull(previousOrRareData), unsure(notNullCase), unsure(slowCase));
6014
6015 LBasicBlock lastNext = m_out.appendTo(notNullCase, rareDataCase);
6016 m_out.branch(
6017 m_out.notEqual(m_out.load32(previousOrRareData, m_heaps.JSCell_structureID), m_out.constInt32(m_graph.m_vm.structureStructure->structureID())),
6018 unsure(rareDataCase), unsure(slowCase));
6019
6020 m_out.appendTo(rareDataCase, useCacheCase);
6021 ASSERT(bitwise_cast<uintptr_t>(StructureRareData::cachedOwnKeysSentinel()) == 1);
6022 LValue cachedOwnKeys = m_out.loadPtr(previousOrRareData, m_heaps.StructureRareData_cachedOwnKeys);
6023 m_out.branch(m_out.belowOrEqual(cachedOwnKeys, m_out.constIntPtr(bitwise_cast<void*>(StructureRareData::cachedOwnKeysSentinel()))), unsure(slowCase), unsure(useCacheCase));
6024
6025 m_out.appendTo(useCacheCase, slowButArrayBufferCase);
6026 RegisteredStructure arrayStructure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(CopyOnWriteArrayWithContiguous));
6027 LValue fastArray = allocateObject<JSArray>(arrayStructure, m_out.addPtr(cachedOwnKeys, JSImmutableButterfly::offsetOfData()), slowButArrayBufferCase);
6028 ValueFromBlock fastResult = m_out.anchor(fastArray);
6029 m_out.jump(continuation);
6030
6031 m_out.appendTo(slowButArrayBufferCase, slowCase);
6032 LValue slowArray = vmCall(Int64, operationNewArrayBuffer, m_vmValue, weakStructure(arrayStructure), cachedOwnKeys);
6033 ValueFromBlock slowButArrayBufferResult = m_out.anchor(slowArray);
6034 m_out.jump(continuation);
6035
6036 m_out.appendTo(slowCase, continuation);
6037 VM& vm = this->vm();
6038 LValue slowResultValue = lazySlowPath(
6039 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
6040 return createLazyCallGenerator(vm,
6041 operationObjectKeysObject, locations[0].directGPR(), globalObject, locations[1].directGPR());
6042 },
6043 object);
6044 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
6045 m_out.jump(continuation);
6046
6047 m_out.appendTo(continuation, lastNext);
6048 setJSValue(m_out.phi(pointerType(), fastResult, slowButArrayBufferResult, slowResult));
6049 break;
6050 }
6051 setJSValue(vmCall(Int64, operationObjectKeysObject, weakPointer(globalObject), lowObject(m_node->child1())));
6052 break;
6053 }
6054 case UntypedUse:
6055 setJSValue(vmCall(Int64, operationObjectKeys, weakPointer(globalObject), lowJSValue(m_node->child1())));
6056 break;
6057 default:
6058 RELEASE_ASSERT_NOT_REACHED();
6059 break;
6060 }
6061 }
6062
6063 void compileObjectCreate()
6064 {
6065 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6066 switch (m_node->child1().useKind()) {
6067 case ObjectUse:
6068 setJSValue(vmCall(Int64, operationObjectCreateObject, weakPointer(globalObject), lowObject(m_node->child1())));
6069 break;
6070 case UntypedUse:
6071 setJSValue(vmCall(Int64, operationObjectCreate, weakPointer(globalObject), lowJSValue(m_node->child1())));
6072 break;
6073 default:
6074 RELEASE_ASSERT_NOT_REACHED();
6075 break;
6076 }
6077 }
6078
6079 void compileNewObject()
6080 {
6081 setJSValue(allocateObject(m_node->structure()));
6082 mutatorFence();
6083 }
6084
6085 void compileNewPromise()
6086 {
6087 LBasicBlock slowCase = m_out.newBlock();
6088 LBasicBlock continuation = m_out.newBlock();
6089
6090 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowCase);
6091
6092 LValue promise;
6093 if (m_node->isInternalPromise())
6094 promise = allocateObject<JSInternalPromise>(m_node->structure(), m_out.intPtrZero, slowCase);
6095 else
6096 promise = allocateObject<JSPromise>(m_node->structure(), m_out.intPtrZero, slowCase);
6097 m_out.store64(m_out.constInt64(JSValue::encode(jsNumber(static_cast<unsigned>(JSPromise::Status::Pending)))), promise, m_heaps.JSInternalFieldObjectImpl_internalFields[static_cast<unsigned>(JSPromise::Field::Flags)]);
6098 m_out.store64(m_out.constInt64(JSValue::encode(jsUndefined())), promise, m_heaps.JSInternalFieldObjectImpl_internalFields[static_cast<unsigned>(JSPromise::Field::ReactionsOrResult)]);
6099 mutatorFence();
6100 ValueFromBlock fastResult = m_out.anchor(promise);
6101 m_out.jump(continuation);
6102
6103 m_out.appendTo(slowCase, continuation);
6104 ValueFromBlock slowResult = m_out.anchor(vmCall(pointerType(), m_node->isInternalPromise() ? operationNewInternalPromise : operationNewPromise, m_vmValue, frozenPointer(m_graph.freezeStrong(m_node->structure().get()))));
6105 m_out.jump(continuation);
6106
6107 m_out.appendTo(continuation, lastNext);
6108 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
6109 }
6110
6111 template<typename JSClass, typename Operation>
6112 void compileNewInternalFieldObject(Operation operation)
6113 {
6114 LBasicBlock slowCase = m_out.newBlock();
6115 LBasicBlock continuation = m_out.newBlock();
6116
6117 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowCase);
6118
6119 LValue object = allocateObject<JSClass>(m_node->structure(), m_out.intPtrZero, slowCase);
6120 auto initialValues = JSClass::initialValues();
6121 ASSERT(initialValues.size() == JSClass::numberOfInternalFields);
6122 for (unsigned index = 0; index < initialValues.size(); ++index)
6123 m_out.store64(m_out.constInt64(JSValue::encode(initialValues[index])), object, m_heaps.JSInternalFieldObjectImpl_internalFields[index]);
6124 mutatorFence();
6125 ValueFromBlock fastResult = m_out.anchor(object);
6126 m_out.jump(continuation);
6127
6128 m_out.appendTo(slowCase, continuation);
6129 ValueFromBlock slowResult = m_out.anchor(vmCall(pointerType(), operation, m_vmValue, frozenPointer(m_graph.freezeStrong(m_node->structure().get()))));
6130 m_out.jump(continuation);
6131
6132 m_out.appendTo(continuation, lastNext);
6133 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
6134 }
6135
6136 void compileNewGenerator()
6137 {
6138 compileNewInternalFieldObject<JSGenerator>(operationNewGenerator);
6139 }
6140
6141 void compileNewAsyncGenerator()
6142 {
6143 compileNewInternalFieldObject<JSAsyncGenerator>(operationNewAsyncGenerator);
6144 }
6145
6146 void compileNewStringObject()
6147 {
6148 RegisteredStructure structure = m_node->structure();
6149 LValue string = lowString(m_node->child1());
6150
6151 LBasicBlock slowCase = m_out.newBlock();
6152 LBasicBlock continuation = m_out.newBlock();
6153
6154 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowCase);
6155
6156 LValue fastResultValue = allocateObject<StringObject>(structure, m_out.intPtrZero, slowCase);
6157 m_out.storePtr(m_out.constIntPtr(StringObject::info()), fastResultValue, m_heaps.JSDestructibleObject_classInfo);
6158 m_out.store64(string, fastResultValue, m_heaps.JSWrapperObject_internalValue);
6159 mutatorFence();
6160 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
6161 m_out.jump(continuation);
6162
6163 m_out.appendTo(slowCase, continuation);
6164 VM& vm = this->vm();
6165 LValue slowResultValue = lazySlowPath(
6166 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
6167 return createLazyCallGenerator(vm,
6168 operationNewStringObject, locations[0].directGPR(), &vm, locations[1].directGPR(),
6169 CCallHelpers::TrustedImmPtr(structure.get()));
6170 },
6171 string);
6172 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
6173 m_out.jump(continuation);
6174
6175 m_out.appendTo(continuation, lastNext);
6176 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
6177 }
6178
6179 void compileNewSymbol()
6180 {
6181 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6182 if (!m_node->child1()) {
6183 setJSValue(vmCall(pointerType(), operationNewSymbol, m_vmValue));
6184 return;
6185 }
6186 ASSERT(m_node->child1().useKind() == KnownStringUse);
6187 setJSValue(vmCall(pointerType(), operationNewSymbolWithDescription, weakPointer(globalObject), lowString(m_node->child1())));
6188 }
6189
6190 void compileNewArray()
6191 {
6192 // First speculate appropriately on all of the children. Do this unconditionally up here
6193 // because some of the slow paths may otherwise forget to do it. It's sort of arguable
6194 // that doing the speculations up here might be unprofitable for RA - so we can consider
6195 // sinking this to below the allocation fast path if we find that this has a lot of
6196 // register pressure.
6197 for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex)
6198 speculate(m_graph.varArgChild(m_node, operandIndex));
6199
6200 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6201 RegisteredStructure structure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(
6202 m_node->indexingType()));
6203
6204 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
6205 unsigned numElements = m_node->numChildren();
6206 unsigned vectorLengthHint = m_node->vectorLengthHint();
6207 ASSERT(vectorLengthHint >= numElements);
6208
6209 ArrayValues arrayValues =
6210 allocateUninitializedContiguousJSArray(numElements, vectorLengthHint, structure);
6211
6212 for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
6213 Edge edge = m_graph.varArgChild(m_node, operandIndex);
6214
6215 switch (m_node->indexingType()) {
6216 case ALL_BLANK_INDEXING_TYPES:
6217 case ALL_UNDECIDED_INDEXING_TYPES:
6218 DFG_CRASH(m_graph, m_node, "Bad indexing type");
6219 break;
6220
6221 case ALL_DOUBLE_INDEXING_TYPES:
6222 m_out.storeDouble(
6223 lowDouble(edge),
6224 arrayValues.butterfly, m_heaps.indexedDoubleProperties[operandIndex]);
6225 break;
6226
6227 case ALL_INT32_INDEXING_TYPES:
6228 case ALL_CONTIGUOUS_INDEXING_TYPES:
6229 m_out.store64(
6230 lowJSValue(edge, ManualOperandSpeculation),
6231 arrayValues.butterfly,
6232 m_heaps.forIndexingType(m_node->indexingType())->at(operandIndex));
6233 break;
6234
6235 default:
6236 DFG_CRASH(m_graph, m_node, "Corrupt indexing type");
6237 break;
6238 }
6239 }
6240
6241 setJSValue(arrayValues.array);
6242 mutatorFence();
6243 return;
6244 }
6245
6246 if (!m_node->numChildren()) {
6247 setJSValue(vmCall(
6248 Int64, operationNewEmptyArray, m_vmValue,
6249 weakStructure(structure)));
6250 return;
6251 }
6252
6253 size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren();
6254 ASSERT(scratchSize);
6255 ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
6256 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
6257
6258 for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
6259 Edge edge = m_graph.varArgChild(m_node, operandIndex);
6260 LValue valueToStore;
6261 switch (m_node->indexingType()) {
6262 case ALL_DOUBLE_INDEXING_TYPES:
6263 valueToStore = boxDouble(lowDouble(edge));
6264 break;
6265 default:
6266 valueToStore = lowJSValue(edge, ManualOperandSpeculation);
6267 break;
6268 }
6269 m_out.store64(valueToStore, m_out.absolute(buffer + operandIndex));
6270 }
6271
6272 m_out.storePtr(
6273 m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->addressOfActiveLength()));
6274
6275 LValue result = vmCall(
6276 Int64, operationNewArray, weakPointer(globalObject),
6277 weakStructure(structure), m_out.constIntPtr(buffer),
6278 m_out.constIntPtr(m_node->numChildren()));
6279
6280 m_out.storePtr(m_out.intPtrZero, m_out.absolute(scratchBuffer->addressOfActiveLength()));
6281
6282 setJSValue(result);
6283 }
6284
6285 void compileNewArrayWithSpread()
6286 {
6287 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6288 if (m_graph.isWatchingHavingABadTimeWatchpoint(m_node)) {
6289 CheckedInt32 startLength = 0;
6290 BitVector* bitVector = m_node->bitVector();
6291 HashMap<InlineCallFrame*, LValue, WTF::DefaultHash<InlineCallFrame*>::Hash, WTF::NullableHashTraits<InlineCallFrame*>> cachedSpreadLengths;
6292
6293 for (unsigned i = 0; i < m_node->numChildren(); ++i) {
6294 if (!bitVector->get(i))
6295 ++startLength;
6296 else {
6297 Edge& child = m_graph.varArgChild(m_node, i);
6298 if (child->op() == PhantomSpread && child->child1()->op() == PhantomNewArrayBuffer)
6299 startLength += child->child1()->castOperand<JSImmutableButterfly*>()->length();
6300 }
6301 }
6302
6303 if (startLength.hasOverflowed()) {
6304 terminate(Overflow);
6305 return;
6306 }
6307
6308 LValue length = m_out.constInt32(startLength.unsafeGet());
6309
6310 for (unsigned i = 0; i < m_node->numChildren(); ++i) {
6311 if (bitVector->get(i)) {
6312 Edge use = m_graph.varArgChild(m_node, i);
6313 CheckValue* lengthCheck = nullptr;
6314 if (use->op() == PhantomSpread) {
6315 if (use->child1()->op() == PhantomCreateRest) {
6316 InlineCallFrame* inlineCallFrame = use->child1()->origin.semantic.inlineCallFrame();
6317 unsigned numberOfArgumentsToSkip = use->child1()->numberOfArgumentsToSkip();
6318 LValue spreadLength = cachedSpreadLengths.ensure(inlineCallFrame, [&] () {
6319 return getSpreadLengthFromInlineCallFrame(inlineCallFrame, numberOfArgumentsToSkip);
6320 }).iterator->value;
6321 lengthCheck = m_out.speculateAdd(length, spreadLength);
6322 }
6323 } else {
6324 LValue fixedArray = lowCell(use);
6325 lengthCheck = m_out.speculateAdd(length, m_out.load32(fixedArray, m_heaps.JSFixedArray_size));
6326 }
6327
6328 if (lengthCheck) {
6329 blessSpeculation(lengthCheck, Overflow, noValue(), nullptr, m_origin);
6330 length = lengthCheck;
6331 }
6332 }
6333 }
6334
6335 LValue exceedsMaxAllowedLength = m_out.aboveOrEqual(length, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH));
6336 blessSpeculation(m_out.speculate(exceedsMaxAllowedLength), Overflow, noValue(), nullptr, m_origin);
6337
6338 RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->originalArrayStructureForIndexingType(ArrayWithContiguous));
6339 ArrayValues arrayValues = allocateUninitializedContiguousJSArray(length, structure);
6340 LValue result = arrayValues.array;
6341 LValue storage = arrayValues.butterfly;
6342 LValue index = m_out.constIntPtr(0);
6343
6344 for (unsigned i = 0; i < m_node->numChildren(); ++i) {
6345 Edge use = m_graph.varArgChild(m_node, i);
6346 if (bitVector->get(i)) {
6347 if (use->op() == PhantomSpread) {
6348 if (use->child1()->op() == PhantomNewArrayBuffer) {
6349 IndexedAbstractHeap& heap = m_heaps.indexedContiguousProperties;
6350 auto* array = use->child1()->castOperand<JSImmutableButterfly*>();
6351 for (unsigned i = 0; i < array->length(); ++i) {
6352 // Because resulted array from NewArrayWithSpread is always contiguous, we should not generate value
6353 // in Double form even if PhantomNewArrayBuffer's indexingType is ArrayWithDouble.
6354 int64_t value = JSValue::encode(array->get(i));
6355 m_out.store64(m_out.constInt64(value), m_out.baseIndex(heap, storage, index, JSValue(), (Checked<int32_t>(sizeof(JSValue)) * i).unsafeGet()));
6356 }
6357 index = m_out.add(index, m_out.constIntPtr(array->length()));
6358 } else {
6359 RELEASE_ASSERT(use->child1()->op() == PhantomCreateRest);
6360 InlineCallFrame* inlineCallFrame = use->child1()->origin.semantic.inlineCallFrame();
6361 unsigned numberOfArgumentsToSkip = use->child1()->numberOfArgumentsToSkip();
6362
6363 LValue length = m_out.zeroExtPtr(cachedSpreadLengths.get(inlineCallFrame));
6364 LValue sourceStart = getArgumentsStart(inlineCallFrame, numberOfArgumentsToSkip);
6365
6366 LBasicBlock loopStart = m_out.newBlock();
6367 LBasicBlock continuation = m_out.newBlock();
6368
6369 ValueFromBlock loadIndexStart = m_out.anchor(m_out.constIntPtr(0));
6370 ValueFromBlock arrayIndexStart = m_out.anchor(index);
6371 ValueFromBlock arrayIndexStartForFinish = m_out.anchor(index);
6372
6373 m_out.branch(
6374 m_out.isZero64(length),
6375 unsure(continuation), unsure(loopStart));
6376
6377 LBasicBlock lastNext = m_out.appendTo(loopStart, continuation);
6378
6379 LValue arrayIndex = m_out.phi(pointerType(), arrayIndexStart);
6380 LValue loadIndex = m_out.phi(pointerType(), loadIndexStart);
6381
6382 LValue item = m_out.load64(m_out.baseIndex(m_heaps.variables, sourceStart, loadIndex));
6383 m_out.store64(item, m_out.baseIndex(m_heaps.indexedContiguousProperties, storage, arrayIndex));
6384
6385 LValue nextArrayIndex = m_out.add(arrayIndex, m_out.constIntPtr(1));
6386 LValue nextLoadIndex = m_out.add(loadIndex, m_out.constIntPtr(1));
6387 ValueFromBlock arrayIndexLoopForFinish = m_out.anchor(nextArrayIndex);
6388
6389 m_out.addIncomingToPhi(loadIndex, m_out.anchor(nextLoadIndex));
6390 m_out.addIncomingToPhi(arrayIndex, m_out.anchor(nextArrayIndex));
6391
6392 m_out.branch(
6393 m_out.below(nextLoadIndex, length),
6394 unsure(loopStart), unsure(continuation));
6395
6396 m_out.appendTo(continuation, lastNext);
6397 index = m_out.phi(pointerType(), arrayIndexStartForFinish, arrayIndexLoopForFinish);
6398 }
6399 } else {
6400 LBasicBlock loopStart = m_out.newBlock();
6401 LBasicBlock continuation = m_out.newBlock();
6402
6403 LValue fixedArray = lowCell(use);
6404
6405 ValueFromBlock fixedIndexStart = m_out.anchor(m_out.constIntPtr(0));
6406 ValueFromBlock arrayIndexStart = m_out.anchor(index);
6407 ValueFromBlock arrayIndexStartForFinish = m_out.anchor(index);
6408
6409 LValue fixedArraySize = m_out.zeroExtPtr(m_out.load32(fixedArray, m_heaps.JSFixedArray_size));
6410
6411 m_out.branch(
6412 m_out.isZero64(fixedArraySize),
6413 unsure(continuation), unsure(loopStart));
6414
6415 LBasicBlock lastNext = m_out.appendTo(loopStart, continuation);
6416
6417 LValue arrayIndex = m_out.phi(pointerType(), arrayIndexStart);
6418 LValue fixedArrayIndex = m_out.phi(pointerType(), fixedIndexStart);
6419
6420 LValue item = m_out.load64(m_out.baseIndex(m_heaps.JSFixedArray_buffer, fixedArray, fixedArrayIndex));
6421 m_out.store64(item, m_out.baseIndex(m_heaps.indexedContiguousProperties, storage, arrayIndex));
6422
6423 LValue nextArrayIndex = m_out.add(arrayIndex, m_out.constIntPtr(1));
6424 LValue nextFixedArrayIndex = m_out.add(fixedArrayIndex, m_out.constIntPtr(1));
6425 ValueFromBlock arrayIndexLoopForFinish = m_out.anchor(nextArrayIndex);
6426
6427 m_out.addIncomingToPhi(fixedArrayIndex, m_out.anchor(nextFixedArrayIndex));
6428 m_out.addIncomingToPhi(arrayIndex, m_out.anchor(nextArrayIndex));
6429
6430 m_out.branch(
6431 m_out.below(nextFixedArrayIndex, fixedArraySize),
6432 unsure(loopStart), unsure(continuation));
6433
6434 m_out.appendTo(continuation, lastNext);
6435 index = m_out.phi(pointerType(), arrayIndexStartForFinish, arrayIndexLoopForFinish);
6436 }
6437 } else {
6438 IndexedAbstractHeap& heap = m_heaps.indexedContiguousProperties;
6439 LValue item = lowJSValue(use);
6440 m_out.store64(item, m_out.baseIndex(heap, storage, index));
6441 index = m_out.add(index, m_out.constIntPtr(1));
6442 }
6443 }
6444
6445 mutatorFence();
6446 setJSValue(result);
6447 return;
6448 }
6449
6450 ASSERT(m_node->numChildren());
6451 size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren();
6452 ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
6453 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
6454 BitVector* bitVector = m_node->bitVector();
6455 for (unsigned i = 0; i < m_node->numChildren(); ++i) {
6456 Edge use = m_graph.m_varArgChildren[m_node->firstChild() + i];
6457 LValue value;
6458 if (bitVector->get(i))
6459 value = lowCell(use);
6460 else
6461 value = lowJSValue(use);
6462 m_out.store64(value, m_out.absolute(&buffer[i]));
6463 }
6464
6465 m_out.storePtr(m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->addressOfActiveLength()));
6466 LValue result = vmCall(Int64, operationNewArrayWithSpreadSlow, weakPointer(globalObject), m_out.constIntPtr(buffer), m_out.constInt32(m_node->numChildren()));
6467 m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->addressOfActiveLength()));
6468
6469 setJSValue(result);
6470 }
6471
6472 void compileCreateThis()
6473 {
6474 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6475 LValue callee = lowCell(m_node->child1());
6476
6477 LBasicBlock isFunctionBlock = m_out.newBlock();
6478 LBasicBlock hasRareData = m_out.newBlock();
6479 LBasicBlock slowPath = m_out.newBlock();
6480 LBasicBlock continuation = m_out.newBlock();
6481
6482 m_out.branch(isFunction(callee, provenType(m_node->child1())), usually(isFunctionBlock), rarely(slowPath));
6483
6484 LBasicBlock lastNext = m_out.appendTo(isFunctionBlock, hasRareData);
6485 LValue rareData = m_out.loadPtr(callee, m_heaps.JSFunction_rareData);
6486 m_out.branch(m_out.isZero64(rareData), rarely(slowPath), usually(hasRareData));
6487
6488 m_out.appendTo(hasRareData, slowPath);
6489 LValue allocator = m_out.loadPtr(rareData, m_heaps.FunctionRareData_allocator);
6490 LValue structure = m_out.loadPtr(rareData, m_heaps.FunctionRareData_structure);
6491 LValue butterfly = m_out.constIntPtr(0);
6492 ValueFromBlock fastResult = m_out.anchor(allocateObject(allocator, structure, butterfly, slowPath));
6493 m_out.jump(continuation);
6494
6495 m_out.appendTo(slowPath, continuation);
6496 ValueFromBlock slowResult = m_out.anchor(vmCall(
6497 Int64, operationCreateThis, weakPointer(globalObject), callee, m_out.constInt32(m_node->inlineCapacity())));
6498 m_out.jump(continuation);
6499
6500 m_out.appendTo(continuation, lastNext);
6501 LValue result = m_out.phi(Int64, fastResult, slowResult);
6502
6503 mutatorFence();
6504 setJSValue(result);
6505 }
6506
6507 void compileCreatePromise()
6508 {
6509 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6510
6511 LValue callee = lowCell(m_node->child1());
6512
6513 LBasicBlock derivedCase = m_out.newBlock();
6514 LBasicBlock isFunctionBlock = m_out.newBlock();
6515 LBasicBlock hasRareData = m_out.newBlock();
6516 LBasicBlock hasStructure = m_out.newBlock();
6517 LBasicBlock checkGlobalObjectCase = m_out.newBlock();
6518 LBasicBlock fastAllocationCase = m_out.newBlock();
6519 LBasicBlock slowCase = m_out.newBlock();
6520 LBasicBlock continuation = m_out.newBlock();
6521
6522 ValueFromBlock promiseStructure = m_out.anchor(weakStructure(m_graph.registerStructure(m_node->isInternalPromise() ? globalObject->internalPromiseStructure() : globalObject->promiseStructure())));
6523 m_out.branch(m_out.equal(callee, weakPointer(m_node->isInternalPromise() ? globalObject->internalPromiseConstructor() : globalObject->promiseConstructor())), unsure(fastAllocationCase), unsure(derivedCase));
6524
6525 LBasicBlock lastNext = m_out.appendTo(derivedCase, isFunctionBlock);
6526 m_out.branch(isFunction(callee, provenType(m_node->child1())), usually(isFunctionBlock), rarely(slowCase));
6527
6528 m_out.appendTo(isFunctionBlock, hasRareData);
6529 LValue rareData = m_out.loadPtr(callee, m_heaps.JSFunction_rareData);
6530 m_out.branch(m_out.isZero64(rareData), rarely(slowCase), usually(hasRareData));
6531
6532 m_out.appendTo(hasRareData, hasStructure);
6533 LValue structure = m_out.loadPtr(rareData, m_heaps.FunctionRareData_internalFunctionAllocationProfile_structure);
6534 m_out.branch(m_out.isZero64(structure), rarely(slowCase), usually(hasStructure));
6535
6536 m_out.appendTo(hasStructure, checkGlobalObjectCase);
6537 m_out.branch(m_out.equal(m_out.loadPtr(structure, m_heaps.Structure_classInfo), m_out.constIntPtr(m_node->isInternalPromise() ? JSInternalPromise::info() : JSPromise::info())), usually(checkGlobalObjectCase), rarely(slowCase));
6538
6539 m_out.appendTo(checkGlobalObjectCase, fastAllocationCase);
6540 ValueFromBlock derivedStructure = m_out.anchor(structure);
6541 m_out.branch(m_out.equal(m_out.loadPtr(structure, m_heaps.Structure_globalObject), weakPointer(globalObject)), usually(fastAllocationCase), rarely(slowCase));
6542
6543 m_out.appendTo(fastAllocationCase, slowCase);
6544 LValue promise;
6545 if (m_node->isInternalPromise())
6546 promise = allocateObject<JSInternalPromise>(m_out.phi(pointerType(), promiseStructure, derivedStructure), m_out.intPtrZero, slowCase);
6547 else
6548 promise = allocateObject<JSPromise>(m_out.phi(pointerType(), promiseStructure, derivedStructure), m_out.intPtrZero, slowCase);
6549 m_out.store64(m_out.constInt64(JSValue::encode(jsNumber(static_cast<unsigned>(JSPromise::Status::Pending)))), promise, m_heaps.JSInternalFieldObjectImpl_internalFields[static_cast<unsigned>(JSPromise::Field::Flags)]);
6550 m_out.store64(m_out.constInt64(JSValue::encode(jsUndefined())), promise, m_heaps.JSInternalFieldObjectImpl_internalFields[static_cast<unsigned>(JSPromise::Field::ReactionsOrResult)]);
6551 mutatorFence();
6552 ValueFromBlock fastResult = m_out.anchor(promise);
6553 m_out.jump(continuation);
6554
6555 m_out.appendTo(slowCase, continuation);
6556 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, m_node->isInternalPromise() ? operationCreateInternalPromise : operationCreatePromise, weakPointer(globalObject), callee));
6557 m_out.jump(continuation);
6558
6559 m_out.appendTo(continuation, lastNext);
6560 LValue result = m_out.phi(Int64, fastResult, slowResult);
6561
6562 setJSValue(result);
6563 }
6564
6565 template<typename JSClass, typename Operation>
6566 void compileCreateInternalFieldObject(Operation operation)
6567 {
6568 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6569
6570 LValue callee = lowCell(m_node->child1());
6571
6572 LBasicBlock isFunctionBlock = m_out.newBlock();
6573 LBasicBlock hasRareData = m_out.newBlock();
6574 LBasicBlock hasStructure = m_out.newBlock();
6575 LBasicBlock checkGlobalObjectCase = m_out.newBlock();
6576 LBasicBlock fastAllocationCase = m_out.newBlock();
6577 LBasicBlock slowCase = m_out.newBlock();
6578 LBasicBlock continuation = m_out.newBlock();
6579
6580 m_out.branch(isFunction(callee, provenType(m_node->child1())), usually(isFunctionBlock), rarely(slowCase));
6581
6582 LBasicBlock lastNext = m_out.appendTo(isFunctionBlock, hasRareData);
6583 LValue rareData = m_out.loadPtr(callee, m_heaps.JSFunction_rareData);
6584 m_out.branch(m_out.isZero64(rareData), rarely(slowCase), usually(hasRareData));
6585
6586 m_out.appendTo(hasRareData, hasStructure);
6587 LValue structure = m_out.loadPtr(rareData, m_heaps.FunctionRareData_internalFunctionAllocationProfile_structure);
6588 m_out.branch(m_out.isZero64(structure), rarely(slowCase), usually(hasStructure));
6589
6590 m_out.appendTo(hasStructure, checkGlobalObjectCase);
6591 m_out.branch(m_out.equal(m_out.loadPtr(structure, m_heaps.Structure_classInfo), m_out.constIntPtr(JSClass::info())), usually(checkGlobalObjectCase), rarely(slowCase));
6592
6593 m_out.appendTo(checkGlobalObjectCase, fastAllocationCase);
6594 m_out.branch(m_out.equal(m_out.loadPtr(structure, m_heaps.Structure_globalObject), weakPointer(globalObject)), usually(fastAllocationCase), rarely(slowCase));
6595
6596 m_out.appendTo(fastAllocationCase, slowCase);
6597 LValue object = allocateObject<JSClass>(structure, m_out.intPtrZero, slowCase);
6598 auto initialValues = JSClass::initialValues();
6599 ASSERT(initialValues.size() == JSClass::numberOfInternalFields);
6600 for (unsigned index = 0; index < initialValues.size(); ++index)
6601 m_out.store64(m_out.constInt64(JSValue::encode(initialValues[index])), object, m_heaps.JSInternalFieldObjectImpl_internalFields[index]);
6602 mutatorFence();
6603 ValueFromBlock fastResult = m_out.anchor(object);
6604 m_out.jump(continuation);
6605
6606 m_out.appendTo(slowCase, continuation);
6607 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operation, weakPointer(globalObject), callee));
6608 m_out.jump(continuation);
6609
6610 m_out.appendTo(continuation, lastNext);
6611 LValue result = m_out.phi(Int64, fastResult, slowResult);
6612
6613 setJSValue(result);
6614 }
6615
6616 void compileCreateGenerator()
6617 {
6618 compileCreateInternalFieldObject<JSGenerator>(operationCreateGenerator);
6619 }
6620
6621 void compileCreateAsyncGenerator()
6622 {
6623 compileCreateInternalFieldObject<JSAsyncGenerator>(operationCreateAsyncGenerator);
6624 }
6625
6626 void compileSpread()
6627 {
6628 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6629 if (m_node->child1()->op() == PhantomNewArrayBuffer) {
6630 LBasicBlock slowAllocation = m_out.newBlock();
6631 LBasicBlock continuation = m_out.newBlock();
6632
6633 auto* immutableButterfly = m_node->child1()->castOperand<JSImmutableButterfly*>();
6634
6635 LValue fastFixedArrayValue = allocateVariableSizedCell<JSFixedArray>(
6636 m_out.constIntPtr(JSFixedArray::allocationSize(immutableButterfly->length()).unsafeGet()),
6637 m_graph.m_vm.fixedArrayStructure.get(), slowAllocation);
6638 m_out.store32(m_out.constInt32(immutableButterfly->length()), fastFixedArrayValue, m_heaps.JSFixedArray_size);
6639 ValueFromBlock fastFixedArray = m_out.anchor(fastFixedArrayValue);
6640 m_out.jump(continuation);
6641
6642 LBasicBlock lastNext = m_out.appendTo(slowAllocation, continuation);
6643 ValueFromBlock slowFixedArray = m_out.anchor(vmCall(pointerType(), operationCreateFixedArray, weakPointer(globalObject), m_out.constInt32(immutableButterfly->length())));
6644 m_out.jump(continuation);
6645
6646 m_out.appendTo(continuation, lastNext);
6647 LValue fixedArray = m_out.phi(pointerType(), fastFixedArray, slowFixedArray);
6648 for (unsigned i = 0; i < immutableButterfly->length(); i++) {
6649 // Because forwarded values are drained as JSValue, we should not generate value
6650 // in Double form even if PhantomNewArrayBuffer's indexingType is ArrayWithDouble.
6651 int64_t value = JSValue::encode(immutableButterfly->get(i));
6652 m_out.store64(
6653 m_out.constInt64(value),
6654 m_out.baseIndex(m_heaps.JSFixedArray_buffer, fixedArray, m_out.constIntPtr(i), jsNumber(i)));
6655 }
6656 mutatorFence();
6657 setJSValue(fixedArray);
6658 return;
6659 }
6660
6661 if (m_node->child1()->op() == PhantomCreateRest) {
6662 // This IR is rare to generate since it requires escaping the Spread
6663 // but not the CreateRest. In bytecode, we have only few operations that
6664 // accept Spread's result as input. This usually leads to the Spread node not
6665 // escaping. However, this can happen if for example we generate a PutStack on
6666 // the Spread but nothing escapes the CreateRest.
6667 LBasicBlock loopHeader = m_out.newBlock();
6668 LBasicBlock loopBody = m_out.newBlock();
6669 LBasicBlock slowAllocation = m_out.newBlock();
6670 LBasicBlock continuation = m_out.newBlock();
6671 LBasicBlock lastNext = m_out.insertNewBlocksBefore(loopHeader);
6672
6673 InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame();
6674 unsigned numberOfArgumentsToSkip = m_node->child1()->numberOfArgumentsToSkip();
6675 LValue sourceStart = getArgumentsStart(inlineCallFrame, numberOfArgumentsToSkip);
6676 LValue length = getSpreadLengthFromInlineCallFrame(inlineCallFrame, numberOfArgumentsToSkip);
6677 static_assert(sizeof(JSValue) == 8 && 1 << 3 == 8, "Assumed in the code below.");
6678 LValue size = m_out.add(
6679 m_out.shl(m_out.zeroExtPtr(length), m_out.constInt32(3)),
6680 m_out.constIntPtr(JSFixedArray::offsetOfData()));
6681
6682 LValue fastArrayValue = allocateVariableSizedCell<JSFixedArray>(size, m_graph.m_vm.fixedArrayStructure.get(), slowAllocation);
6683 m_out.store32(length, fastArrayValue, m_heaps.JSFixedArray_size);
6684 ValueFromBlock fastArray = m_out.anchor(fastArrayValue);
6685 m_out.jump(loopHeader);
6686
6687 m_out.appendTo(slowAllocation, loopHeader);
6688 ValueFromBlock slowArray = m_out.anchor(vmCall(pointerType(), operationCreateFixedArray, weakPointer(globalObject), length));
6689 m_out.jump(loopHeader);
6690
6691 m_out.appendTo(loopHeader, loopBody);
6692 LValue fixedArray = m_out.phi(pointerType(), fastArray, slowArray);
6693 ValueFromBlock startIndex = m_out.anchor(m_out.constIntPtr(0));
6694 m_out.branch(m_out.isZero32(length), unsure(continuation), unsure(loopBody));
6695
6696 m_out.appendTo(loopBody, continuation);
6697 LValue index = m_out.phi(pointerType(), startIndex);
6698 LValue value = m_out.load64(
6699 m_out.baseIndex(m_heaps.variables, sourceStart, index));
6700 m_out.store64(value, m_out.baseIndex(m_heaps.JSFixedArray_buffer, fixedArray, index));
6701 LValue nextIndex = m_out.add(m_out.constIntPtr(1), index);
6702 m_out.addIncomingToPhi(index, m_out.anchor(nextIndex));
6703 m_out.branch(m_out.below(nextIndex, m_out.zeroExtPtr(length)), unsure(loopBody), unsure(continuation));
6704
6705 m_out.appendTo(continuation, lastNext);
6706 mutatorFence();
6707 setJSValue(fixedArray);
6708 return;
6709 }
6710
6711 LValue argument = lowCell(m_node->child1());
6712
6713 LValue result;
6714
6715 if (m_node->child1().useKind() == ArrayUse)
6716 speculateArray(m_node->child1());
6717
6718 if (m_graph.canDoFastSpread(m_node, m_state.forNode(m_node->child1()))) {
6719 LBasicBlock preLoop = m_out.newBlock();
6720 LBasicBlock loopSelection = m_out.newBlock();
6721 LBasicBlock contiguousLoopStart = m_out.newBlock();
6722 LBasicBlock doubleLoopStart = m_out.newBlock();
6723 LBasicBlock slowPath = m_out.newBlock();
6724 LBasicBlock continuation = m_out.newBlock();
6725
6726 LValue indexingShape = m_out.load8ZeroExt32(argument, m_heaps.JSCell_indexingTypeAndMisc);
6727 indexingShape = m_out.bitAnd(indexingShape, m_out.constInt32(IndexingShapeMask));
6728 LValue isOKIndexingType = m_out.belowOrEqual(
6729 m_out.sub(indexingShape, m_out.constInt32(Int32Shape)),
6730 m_out.constInt32(ContiguousShape - Int32Shape));
6731
6732 m_out.branch(isOKIndexingType, unsure(preLoop), unsure(slowPath));
6733 LBasicBlock lastNext = m_out.appendTo(preLoop, loopSelection);
6734
6735 LValue butterfly = m_out.loadPtr(argument, m_heaps.JSObject_butterfly);
6736 LValue length = m_out.load32NonNegative(butterfly, m_heaps.Butterfly_publicLength);
6737 static_assert(sizeof(JSValue) == 8 && 1 << 3 == 8, "Assumed in the code below.");
6738 LValue size = m_out.add(
6739 m_out.shl(m_out.zeroExtPtr(length), m_out.constInt32(3)),
6740 m_out.constIntPtr(JSFixedArray::offsetOfData()));
6741
6742 LValue fastAllocation = allocateVariableSizedCell<JSFixedArray>(size, m_graph.m_vm.fixedArrayStructure.get(), slowPath);
6743 ValueFromBlock fastResult = m_out.anchor(fastAllocation);
6744 m_out.store32(length, fastAllocation, m_heaps.JSFixedArray_size);
6745
6746 ValueFromBlock startIndexForContiguous = m_out.anchor(m_out.constIntPtr(0));
6747 ValueFromBlock startIndexForDouble = m_out.anchor(m_out.constIntPtr(0));
6748
6749 m_out.branch(m_out.isZero32(length), unsure(continuation), unsure(loopSelection));
6750
6751 m_out.appendTo(loopSelection, contiguousLoopStart);
6752 m_out.branch(m_out.equal(indexingShape, m_out.constInt32(DoubleShape)),
6753 unsure(doubleLoopStart), unsure(contiguousLoopStart));
6754
6755 {
6756 m_out.appendTo(contiguousLoopStart, doubleLoopStart);
6757 LValue index = m_out.phi(pointerType(), startIndexForContiguous);
6758
6759 TypedPointer loadSite = m_out.baseIndex(m_heaps.root, butterfly, index, ScaleEight); // We read TOP here since we can be reading either int32 or contiguous properties.
6760 LValue value = m_out.load64(loadSite);
6761 value = m_out.select(m_out.isZero64(value), m_out.constInt64(JSValue::encode(jsUndefined())), value);
6762 m_out.store64(value, m_out.baseIndex(m_heaps.JSFixedArray_buffer, fastAllocation, index));
6763
6764 LValue nextIndex = m_out.add(index, m_out.constIntPtr(1));
6765 m_out.addIncomingToPhi(index, m_out.anchor(nextIndex));
6766
6767 m_out.branch(m_out.below(nextIndex, m_out.zeroExtPtr(length)),
6768 unsure(contiguousLoopStart), unsure(continuation));
6769 }
6770
6771 {
6772 m_out.appendTo(doubleLoopStart, slowPath);
6773 LValue index = m_out.phi(pointerType(), startIndexForDouble);
6774
6775 LValue value = m_out.loadDouble(m_out.baseIndex(m_heaps.indexedDoubleProperties, butterfly, index));
6776 LValue isNaN = m_out.doubleNotEqualOrUnordered(value, value);
6777 LValue holeResult = m_out.constInt64(JSValue::encode(jsUndefined()));
6778 LValue normalResult = boxDouble(value);
6779 value = m_out.select(isNaN, holeResult, normalResult);
6780 m_out.store64(value, m_out.baseIndex(m_heaps.JSFixedArray_buffer, fastAllocation, index));
6781
6782 LValue nextIndex = m_out.add(index, m_out.constIntPtr(1));
6783 m_out.addIncomingToPhi(index, m_out.anchor(nextIndex));
6784
6785 m_out.branch(m_out.below(nextIndex, m_out.zeroExtPtr(length)),
6786 unsure(doubleLoopStart), unsure(continuation));
6787 }
6788
6789 m_out.appendTo(slowPath, continuation);
6790 ValueFromBlock slowResult = m_out.anchor(vmCall(pointerType(), operationSpreadFastArray, weakPointer(globalObject), argument));
6791 m_out.jump(continuation);
6792
6793 m_out.appendTo(continuation, lastNext);
6794 result = m_out.phi(pointerType(), fastResult, slowResult);
6795 mutatorFence();
6796 } else
6797 result = vmCall(pointerType(), operationSpreadGeneric, weakPointer(globalObject), argument);
6798
6799 setJSValue(result);
6800 }
6801
6802 void compileNewArrayBuffer()
6803 {
6804 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6805 RegisteredStructure structure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(
6806 m_node->indexingMode()));
6807 auto* immutableButterfly = m_node->castOperand<JSImmutableButterfly*>();
6808
6809 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingMode())) {
6810 LBasicBlock slowPath = m_out.newBlock();
6811 LBasicBlock continuation = m_out.newBlock();
6812
6813 LValue fastArray = allocateObject<JSArray>(structure, m_out.constIntPtr(immutableButterfly->toButterfly()), slowPath);
6814 ValueFromBlock fastResult = m_out.anchor(fastArray);
6815 m_out.jump(continuation);
6816
6817 m_out.appendTo(slowPath, continuation);
6818 LValue slowArray = vmCall(Int64, operationNewArrayBuffer, m_vmValue, weakStructure(structure), frozenPointer(m_node->cellOperand()));
6819 ValueFromBlock slowResult = m_out.anchor(slowArray);
6820 m_out.jump(continuation);
6821
6822 m_out.appendTo(continuation);
6823
6824 mutatorFence();
6825 setJSValue(m_out.phi(pointerType(), slowResult, fastResult));
6826 return;
6827 }
6828
6829 setJSValue(vmCall(
6830 Int64, operationNewArrayBuffer, m_vmValue,
6831 weakStructure(structure), frozenPointer(m_node->cellOperand())));
6832 }
6833
6834 void compileNewArrayWithSize()
6835 {
6836 LValue publicLength = lowInt32(m_node->child1());
6837
6838 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6839 RegisteredStructure structure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(
6840 m_node->indexingType()));
6841
6842 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
6843 IndexingType indexingType = m_node->indexingType();
6844 setJSValue(
6845 allocateJSArray(
6846 publicLength, publicLength, weakPointer(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType)), m_out.constInt32(indexingType)).array);
6847 mutatorFence();
6848 return;
6849 }
6850
6851 LValue structureValue = m_out.select(
6852 m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)),
6853 weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))),
6854 weakStructure(structure));
6855 setJSValue(vmCall(Int64, operationNewArrayWithSize, weakPointer(globalObject), structureValue, publicLength, m_out.intPtrZero));
6856 }
6857
6858 void compileNewTypedArray()
6859 {
6860 TypedArrayType typedArrayType = m_node->typedArrayType();
6861 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6862
6863 switch (m_node->child1().useKind()) {
6864 case Int32Use: {
6865 RegisteredStructure structure = m_graph.registerStructure(globalObject->typedArrayStructureConcurrently(typedArrayType));
6866
6867 LValue size = lowInt32(m_node->child1());
6868
6869 LBasicBlock smallEnoughCase = m_out.newBlock();
6870 LBasicBlock slowCase = m_out.newBlock();
6871 LBasicBlock continuation = m_out.newBlock();
6872
6873 ValueFromBlock noStorage = m_out.anchor(m_out.intPtrZero);
6874
6875 m_out.branch(
6876 m_out.above(size, m_out.constInt32(JSArrayBufferView::fastSizeLimit)),
6877 rarely(slowCase), usually(smallEnoughCase));
6878
6879 LBasicBlock lastNext = m_out.appendTo(smallEnoughCase, slowCase);
6880
6881 LValue byteSize =
6882 m_out.shl(m_out.zeroExtPtr(size), m_out.constInt32(logElementSize(typedArrayType)));
6883 if (elementSize(typedArrayType) < 8) {
6884 byteSize = m_out.bitAnd(
6885 m_out.add(byteSize, m_out.constIntPtr(7)),
6886 m_out.constIntPtr(~static_cast<intptr_t>(7)));
6887 }
6888
6889 LValue allocator = allocatorForSize(vm().primitiveGigacageAuxiliarySpace, byteSize, slowCase);
6890 LValue storage = allocateHeapCell(allocator, slowCase);
6891
6892 splatWords(
6893 storage,
6894 m_out.int32Zero,
6895 m_out.castToInt32(m_out.lShr(byteSize, m_out.constIntPtr(3))),
6896 m_out.int64Zero,
6897 m_heaps.typedArrayProperties);
6898
6899#if CPU(ARM64E)
6900 {
6901 LValue sizePtr = m_out.zeroExtPtr(size);
6902 PatchpointValue* authenticate = m_out.patchpoint(pointerType());
6903 authenticate->appendSomeRegister(storage);
6904 authenticate->append(sizePtr, B3::ValueRep(B3::ValueRep::SomeLateRegister));
6905 authenticate->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
6906 jit.move(params[1].gpr(), params[0].gpr());
6907 jit.tagArrayPtr(params[2].gpr(), params[0].gpr());
6908 });
6909 storage = authenticate;
6910 }
6911#endif
6912
6913 ValueFromBlock haveStorage = m_out.anchor(storage);
6914
6915 LValue fastResultValue =
6916 allocateObject<JSArrayBufferView>(structure, m_out.intPtrZero, slowCase);
6917
6918 m_out.storePtr(storage, fastResultValue, m_heaps.JSArrayBufferView_vector);
6919 m_out.store32(size, fastResultValue, m_heaps.JSArrayBufferView_length);
6920 m_out.store32(m_out.constInt32(FastTypedArray), fastResultValue, m_heaps.JSArrayBufferView_mode);
6921
6922 mutatorFence();
6923 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
6924 m_out.jump(continuation);
6925
6926 m_out.appendTo(slowCase, continuation);
6927 LValue storageValue = m_out.phi(pointerType(), noStorage, haveStorage);
6928
6929 VM& vm = this->vm();
6930 LValue slowResultValue = lazySlowPath(
6931 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
6932 return createLazyCallGenerator(vm,
6933 operationNewTypedArrayWithSizeForType(typedArrayType), locations[0].directGPR(), globalObject,
6934 CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(),
6935 locations[2].directGPR());
6936 },
6937 size, storageValue);
6938 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
6939 m_out.jump(continuation);
6940
6941 m_out.appendTo(continuation, lastNext);
6942 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
6943 return;
6944 }
6945
6946 case UntypedUse: {
6947 LValue argument = lowJSValue(m_node->child1());
6948
6949 LValue result = vmCall(
6950 pointerType(), operationNewTypedArrayWithOneArgumentForType(typedArrayType),
6951 weakPointer(globalObject), weakPointer(globalObject->typedArrayStructureConcurrently(typedArrayType)), argument);
6952
6953 setJSValue(result);
6954 return;
6955 }
6956
6957 default:
6958 DFG_CRASH(m_graph, m_node, "Bad use kind");
6959 return;
6960 }
6961 }
6962
6963 void compileAllocatePropertyStorage()
6964 {
6965 LValue object = lowCell(m_node->child1());
6966 setStorage(allocatePropertyStorage(object, m_node->transition()->previous.get()));
6967 }
6968
6969 void compileReallocatePropertyStorage()
6970 {
6971 Transition* transition = m_node->transition();
6972 LValue object = lowCell(m_node->child1());
6973 LValue oldStorage = lowStorage(m_node->child2());
6974
6975 setStorage(
6976 reallocatePropertyStorage(
6977 object, oldStorage, transition->previous.get(), transition->next.get()));
6978 }
6979
6980 void compileNukeStructureAndSetButterfly()
6981 {
6982 nukeStructureAndSetButterfly(lowStorage(m_node->child2()), lowCell(m_node->child1()));
6983 }
6984
6985 void compileToNumber()
6986 {
6987 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6988 LValue value = lowJSValue(m_node->child1());
6989
6990 if (!(abstractValue(m_node->child1()).m_type & SpecBytecodeNumber))
6991 setJSValue(vmCall(Int64, operationToNumber, weakPointer(globalObject), value));
6992 else {
6993 LBasicBlock notNumber = m_out.newBlock();
6994 LBasicBlock continuation = m_out.newBlock();
6995
6996 ValueFromBlock fastResult = m_out.anchor(value);
6997 m_out.branch(isNumber(value, provenType(m_node->child1())), unsure(continuation), unsure(notNumber));
6998
6999 // notNumber case.
7000 LBasicBlock lastNext = m_out.appendTo(notNumber, continuation);
7001 // We have several attempts to remove ToNumber. But ToNumber still exists.
7002 // It means that converting non-numbers to numbers by this ToNumber is not rare.
7003 // Instead of the lazy slow path generator, we call the operation here.
7004 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operationToNumber, weakPointer(globalObject), value));
7005 m_out.jump(continuation);
7006
7007 // continuation case.
7008 m_out.appendTo(continuation, lastNext);
7009 setJSValue(m_out.phi(Int64, fastResult, slowResult));
7010 }
7011 }
7012
7013 void compileToNumeric()
7014 {
7015 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7016 LValue value = lowJSValue(m_node->child1());
7017
7018 if (abstractValue(m_node->child1()).m_type & (SpecBytecodeNumber | SpecBigInt)) {
7019 LBasicBlock notNumber = m_out.newBlock();
7020 LBasicBlock isCellPath = m_out.newBlock();
7021 LBasicBlock slowPath = m_out.newBlock();
7022 LBasicBlock continuation = m_out.newBlock();
7023
7024 ValueFromBlock fastResult = m_out.anchor(value);
7025 m_out.branch(isNumber(value, provenType(m_node->child1())), unsure(continuation), unsure(notNumber));
7026
7027 // notNumber case.
7028 LBasicBlock lastNext = m_out.appendTo(notNumber, continuation);
7029 m_out.branch(isCell(value, provenType(m_node->child1())), unsure(isCellPath), unsure(slowPath));
7030
7031 m_out.appendTo(isCellPath);
7032 m_out.branch(isBigInt(value, provenType(m_node->child1())), unsure(continuation), unsure(slowPath));
7033
7034 m_out.appendTo(slowPath);
7035 // We have several attempts to remove ToNumeric. But ToNumeric still exists.
7036 // It means that the slow path is not rare.
7037 // Instead of the lazy slow path generator, we call the operation here.
7038 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operationToNumeric, weakPointer(globalObject), value));
7039 m_out.jump(continuation);
7040
7041 // continuation case.
7042 m_out.appendTo(continuation, lastNext);
7043 setJSValue(m_out.phi(Int64, fastResult, slowResult));
7044 } else
7045 setJSValue(vmCall(Int64, operationToNumeric, weakPointer(globalObject), value));
7046 }
7047
7048 void compileToStringOrCallStringConstructorOrStringValueOf()
7049 {
7050 ASSERT(m_node->op() != StringValueOf || m_node->child1().useKind() == UntypedUse);
7051 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7052 switch (m_node->child1().useKind()) {
7053 case StringObjectUse: {
7054 LValue cell = lowCell(m_node->child1());
7055 speculateStringObjectForCell(m_node->child1(), cell);
7056 setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
7057 return;
7058 }
7059
7060 case StringOrStringObjectUse: {
7061 LValue cell = lowCell(m_node->child1());
7062 LValue type = m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType);
7063
7064 LBasicBlock notString = m_out.newBlock();
7065 LBasicBlock continuation = m_out.newBlock();
7066
7067 ValueFromBlock simpleResult = m_out.anchor(cell);
7068 m_out.branch(
7069 m_out.equal(type, m_out.constInt32(StringType)),
7070 unsure(continuation), unsure(notString));
7071
7072 LBasicBlock lastNext = m_out.appendTo(notString, continuation);
7073 speculate(
7074 BadType, jsValueValue(cell), m_node->child1().node(),
7075 m_out.notEqual(type, m_out.constInt32(StringObjectType)));
7076 ValueFromBlock unboxedResult = m_out.anchor(
7077 m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
7078 m_out.jump(continuation);
7079
7080 m_out.appendTo(continuation, lastNext);
7081 setJSValue(m_out.phi(Int64, simpleResult, unboxedResult));
7082
7083 m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject);
7084 return;
7085 }
7086
7087 case CellUse:
7088 case NotCellUse:
7089 case UntypedUse: {
7090 LValue value;
7091 if (m_node->child1().useKind() == CellUse)
7092 value = lowCell(m_node->child1());
7093 else if (m_node->child1().useKind() == NotCellUse)
7094 value = lowNotCell(m_node->child1());
7095 else
7096 value = lowJSValue(m_node->child1());
7097
7098 LBasicBlock isCell = m_out.newBlock();
7099 LBasicBlock notString = m_out.newBlock();
7100 LBasicBlock continuation = m_out.newBlock();
7101
7102 LValue isCellPredicate;
7103 if (m_node->child1().useKind() == CellUse)
7104 isCellPredicate = m_out.booleanTrue;
7105 else if (m_node->child1().useKind() == NotCellUse)
7106 isCellPredicate = m_out.booleanFalse;
7107 else
7108 isCellPredicate = this->isCell(value, provenType(m_node->child1()));
7109 m_out.branch(isCellPredicate, unsure(isCell), unsure(notString));
7110
7111 LBasicBlock lastNext = m_out.appendTo(isCell, notString);
7112 ValueFromBlock simpleResult = m_out.anchor(value);
7113 LValue isStringPredicate;
7114 if (m_node->child1()->prediction() & SpecString) {
7115 isStringPredicate = isString(value, provenType(m_node->child1()));
7116 } else
7117 isStringPredicate = m_out.booleanFalse;
7118 m_out.branch(isStringPredicate, unsure(continuation), unsure(notString));
7119
7120 m_out.appendTo(notString, continuation);
7121 LValue result;
7122 if (m_node->child1().useKind() == CellUse) {
7123 ASSERT(m_node->op() != StringValueOf);
7124 result = vmCall(Int64, m_node->op() == ToString ? operationToStringOnCell : operationCallStringConstructorOnCell, weakPointer(globalObject), value);
7125 } else {
7126 auto* operation = m_node->op() == ToString
7127 ? operationToString : m_node->op() == StringValueOf
7128 ? operationStringValueOf : operationCallStringConstructor;
7129 result = vmCall(Int64, operation, weakPointer(globalObject), value);
7130 }
7131 ValueFromBlock convertedResult = m_out.anchor(result);
7132 m_out.jump(continuation);
7133
7134 m_out.appendTo(continuation, lastNext);
7135 setJSValue(m_out.phi(Int64, simpleResult, convertedResult));
7136 return;
7137 }
7138
7139 case Int32Use:
7140 setJSValue(vmCall(Int64, operationInt32ToStringWithValidRadix, weakPointer(globalObject), lowInt32(m_node->child1()), m_out.constInt32(10)));
7141 return;
7142
7143 case Int52RepUse:
7144 setJSValue(vmCall(Int64, operationInt52ToStringWithValidRadix, weakPointer(globalObject), lowStrictInt52(m_node->child1()), m_out.constInt32(10)));
7145 return;
7146
7147 case DoubleRepUse:
7148 setJSValue(vmCall(Int64, operationDoubleToStringWithValidRadix, weakPointer(globalObject), lowDouble(m_node->child1()), m_out.constInt32(10)));
7149 return;
7150
7151 default:
7152 DFG_CRASH(m_graph, m_node, "Bad use kind");
7153 break;
7154 }
7155 }
7156
7157 void compileToPrimitive()
7158 {
7159 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7160 LValue value = lowJSValue(m_node->child1());
7161
7162 LBasicBlock isCellCase = m_out.newBlock();
7163 LBasicBlock isObjectCase = m_out.newBlock();
7164 LBasicBlock continuation = m_out.newBlock();
7165
7166 Vector<ValueFromBlock, 3> results;
7167
7168 results.append(m_out.anchor(value));
7169 m_out.branch(
7170 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
7171
7172 LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase);
7173 results.append(m_out.anchor(value));
7174 m_out.branch(
7175 isObject(value, provenType(m_node->child1())),
7176 unsure(isObjectCase), unsure(continuation));
7177
7178 m_out.appendTo(isObjectCase, continuation);
7179 results.append(m_out.anchor(vmCall(
7180 Int64, operationToPrimitive, weakPointer(globalObject), value)));
7181 m_out.jump(continuation);
7182
7183 m_out.appendTo(continuation, lastNext);
7184 setJSValue(m_out.phi(Int64, results));
7185 }
7186
7187 void compileMakeRope()
7188 {
7189 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7190
7191 struct FlagsAndLength {
7192 LValue flags;
7193 LValue length;
7194 };
7195
7196 Edge edges[3] = {
7197 m_node->child1(),
7198 m_node->child2(),
7199 m_node->child3(),
7200 };
7201 LValue kids[3];
7202 unsigned numKids;
7203 kids[0] = lowCell(edges[0]);
7204 kids[1] = lowCell(edges[1]);
7205 if (edges[2]) {
7206 kids[2] = lowCell(edges[2]);
7207 numKids = 3;
7208 } else {
7209 kids[2] = 0;
7210 numKids = 2;
7211 }
7212
7213 LBasicBlock emptyCase = m_out.newBlock();
7214 LBasicBlock slowPath = m_out.newBlock();
7215 LBasicBlock continuation = m_out.newBlock();
7216
7217 Allocator allocator = allocatorForNonVirtualConcurrently<JSRopeString>(vm(), sizeof(JSRopeString), AllocatorForMode::AllocatorIfExists);
7218
7219 LValue result = allocateCell(
7220 m_out.constIntPtr(allocator.localAllocator()), vm().stringStructure.get(), slowPath);
7221
7222 // This puts nullptr for the first fiber. It makes visitChildren safe even if this JSRopeString is discarded due to the speculation failure in the following path.
7223 m_out.storePtr(m_out.constIntPtr(JSString::isRopeInPointer), result, m_heaps.JSRopeString_fiber0);
7224
7225 auto getFlagsAndLength = [&] (Edge& edge, LValue child) {
7226 if (JSString* string = edge->dynamicCastConstant<JSString*>(vm())) {
7227 return FlagsAndLength {
7228 m_out.constInt32(string->is8Bit() ? StringImpl::flagIs8Bit() : 0),
7229 m_out.constInt32(string->length())
7230 };
7231 }
7232
7233 LBasicBlock continuation = m_out.newBlock();
7234 LBasicBlock ropeCase = m_out.newBlock();
7235 LBasicBlock notRopeCase = m_out.newBlock();
7236
7237 m_out.branch(isRopeString(child, edge), unsure(ropeCase), unsure(notRopeCase));
7238
7239 LBasicBlock lastNext = m_out.appendTo(ropeCase, notRopeCase);
7240 ValueFromBlock flagsForRope = m_out.anchor(m_out.load32NonNegative(child, m_heaps.JSRopeString_flags));
7241 ValueFromBlock lengthForRope = m_out.anchor(m_out.load32NonNegative(child, m_heaps.JSRopeString_length));
7242 m_out.jump(continuation);
7243
7244 m_out.appendTo(notRopeCase, continuation);
7245 LValue stringImpl = m_out.loadPtr(child, m_heaps.JSString_value);
7246 ValueFromBlock flagsForNonRope = m_out.anchor(m_out.load32NonNegative(stringImpl, m_heaps.StringImpl_hashAndFlags));
7247 ValueFromBlock lengthForNonRope = m_out.anchor(m_out.load32NonNegative(stringImpl, m_heaps.StringImpl_length));
7248 m_out.jump(continuation);
7249
7250 m_out.appendTo(continuation, lastNext);
7251 return FlagsAndLength {
7252 m_out.phi(Int32, flagsForRope, flagsForNonRope),
7253 m_out.phi(Int32, lengthForRope, lengthForNonRope)
7254 };
7255 };
7256
7257 FlagsAndLength flagsAndLength = getFlagsAndLength(edges[0], kids[0]);
7258 for (unsigned i = 1; i < numKids; ++i) {
7259 auto mergeFlagsAndLength = [&] (Edge& edge, LValue child, FlagsAndLength previousFlagsAndLength) {
7260 FlagsAndLength flagsAndLength = getFlagsAndLength(edge, child);
7261 LValue flags = m_out.bitAnd(previousFlagsAndLength.flags, flagsAndLength.flags);
7262 CheckValue* lengthCheck = m_out.speculateAdd(previousFlagsAndLength.length, flagsAndLength.length);
7263 blessSpeculation(lengthCheck, Uncountable, noValue(), nullptr, m_origin);
7264 return FlagsAndLength {
7265 flags,
7266 lengthCheck
7267 };
7268 };
7269 flagsAndLength = mergeFlagsAndLength(edges[i], kids[i], flagsAndLength);
7270 }
7271
7272 m_out.storePtr(
7273 m_out.bitOr(
7274 m_out.bitOr(kids[0], m_out.constIntPtr(JSString::isRopeInPointer)),
7275 m_out.bitAnd(m_out.constIntPtr(JSRopeString::is8BitInPointer), m_out.zeroExtPtr(flagsAndLength.flags))),
7276 result, m_heaps.JSRopeString_fiber0);
7277 m_out.storePtr(
7278 m_out.bitOr(m_out.zeroExtPtr(flagsAndLength.length), m_out.shl(kids[1], m_out.constInt32(32))),
7279 result, m_heaps.JSRopeString_fiber1);
7280 if (numKids == 2)
7281 m_out.storePtr(m_out.lShr(kids[1], m_out.constInt32(32)), result, m_heaps.JSRopeString_fiber2);
7282 else
7283 m_out.storePtr(m_out.bitOr(m_out.lShr(kids[1], m_out.constInt32(32)), m_out.shl(kids[2], m_out.constInt32(16))), result, m_heaps.JSRopeString_fiber2);
7284
7285 mutatorFence();
7286 ValueFromBlock fastResult = m_out.anchor(result);
7287 m_out.branch(m_out.isZero32(flagsAndLength.length), rarely(emptyCase), usually(continuation));
7288
7289 LBasicBlock lastNext = m_out.appendTo(emptyCase, slowPath);
7290 ValueFromBlock emptyResult = m_out.anchor(weakPointer(jsEmptyString(m_graph.m_vm)));
7291 m_out.jump(continuation);
7292
7293 m_out.appendTo(slowPath, continuation);
7294 LValue slowResultValue;
7295 VM& vm = this->vm();
7296 switch (numKids) {
7297 case 2:
7298 slowResultValue = lazySlowPath(
7299 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
7300 return createLazyCallGenerator(vm,
7301 operationMakeRope2, locations[0].directGPR(), globalObject, locations[1].directGPR(),
7302 locations[2].directGPR());
7303 }, kids[0], kids[1]);
7304 break;
7305 case 3:
7306 slowResultValue = lazySlowPath(
7307 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
7308 return createLazyCallGenerator(vm,
7309 operationMakeRope3, locations[0].directGPR(), globalObject, locations[1].directGPR(),
7310 locations[2].directGPR(), locations[3].directGPR());
7311 }, kids[0], kids[1], kids[2]);
7312 break;
7313 default:
7314 DFG_CRASH(m_graph, m_node, "Bad number of children");
7315 break;
7316 }
7317 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
7318 m_out.jump(continuation);
7319
7320 m_out.appendTo(continuation, lastNext);
7321 setJSValue(m_out.phi(Int64, fastResult, emptyResult, slowResult));
7322 }
7323
7324 void compileStringCharAt()
7325 {
7326 LValue base = lowString(m_graph.child(m_node, 0));
7327 LValue index = lowInt32(m_graph.child(m_node, 1));
7328 LValue storage = lowStorage(m_graph.child(m_node, 2));
7329
7330 LBasicBlock fastPath = m_out.newBlock();
7331 LBasicBlock slowPath = m_out.newBlock();
7332 LBasicBlock continuation = m_out.newBlock();
7333
7334 LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
7335 m_out.branch(
7336 m_out.aboveOrEqual(
7337 index, m_out.load32NonNegative(stringImpl, m_heaps.StringImpl_length)),
7338 rarely(slowPath), usually(fastPath));
7339
7340 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
7341
7342 LBasicBlock is8Bit = m_out.newBlock();
7343 LBasicBlock is16Bit = m_out.newBlock();
7344 LBasicBlock bitsContinuation = m_out.newBlock();
7345 LBasicBlock bigCharacter = m_out.newBlock();
7346
7347 m_out.branch(
7348 m_out.testIsZero32(
7349 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
7350 m_out.constInt32(StringImpl::flagIs8Bit())),
7351 unsure(is16Bit), unsure(is8Bit));
7352
7353 m_out.appendTo(is8Bit, is16Bit);
7354
7355 // FIXME: Need to cage strings!
7356 // https://bugs.webkit.org/show_bug.cgi?id=174924
7357 ValueFromBlock char8Bit = m_out.anchor(
7358 m_out.load8ZeroExt32(m_out.baseIndex(
7359 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
7360 provenValue(m_graph.child(m_node, 1)))));
7361 m_out.jump(bitsContinuation);
7362
7363 m_out.appendTo(is16Bit, bigCharacter);
7364
7365 LValue char16BitValue = m_out.load16ZeroExt32(
7366 m_out.baseIndex(
7367 m_heaps.characters16, storage, m_out.zeroExtPtr(index),
7368 provenValue(m_graph.child(m_node, 1))));
7369 ValueFromBlock char16Bit = m_out.anchor(char16BitValue);
7370 m_out.branch(
7371 m_out.above(char16BitValue, m_out.constInt32(maxSingleCharacterString)),
7372 rarely(bigCharacter), usually(bitsContinuation));
7373
7374 m_out.appendTo(bigCharacter, bitsContinuation);
7375
7376 Vector<ValueFromBlock, 4> results;
7377 results.append(m_out.anchor(vmCall(
7378 Int64, operationSingleCharacterString,
7379 m_vmValue, char16BitValue)));
7380 m_out.jump(continuation);
7381
7382 m_out.appendTo(bitsContinuation, slowPath);
7383
7384 LValue character = m_out.phi(Int32, char8Bit, char16Bit);
7385
7386 LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
7387
7388 results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex(
7389 m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(character)))));
7390 m_out.jump(continuation);
7391
7392 m_out.appendTo(slowPath, continuation);
7393
7394 if (m_node->arrayMode().isInBounds()) {
7395 speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue);
7396 results.append(m_out.anchor(m_out.intPtrZero));
7397 } else {
7398 // FIXME: Revisit JSGlobalObject.
7399 // https://bugs.webkit.org/show_bug.cgi?id=203204
7400 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7401 Structure* stringPrototypeStructure = globalObject->stringPrototype()->structure(vm());
7402 Structure* objectPrototypeStructure = globalObject->objectPrototype()->structure(vm());
7403 WTF::loadLoadFence();
7404
7405 if (globalObject->stringPrototypeChainIsSane()) {
7406 // FIXME: This could be captured using a Speculation mode that means
7407 // "out-of-bounds loads return a trivial value", something like
7408 // SaneChainOutOfBounds.
7409 // https://bugs.webkit.org/show_bug.cgi?id=144668
7410
7411 m_graph.registerAndWatchStructureTransition(stringPrototypeStructure);
7412 m_graph.registerAndWatchStructureTransition(objectPrototypeStructure);
7413
7414 LBasicBlock negativeIndex = m_out.newBlock();
7415
7416 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
7417 m_out.branch(
7418 m_out.lessThan(index, m_out.int32Zero),
7419 rarely(negativeIndex), usually(continuation));
7420
7421 m_out.appendTo(negativeIndex, continuation);
7422 }
7423
7424 results.append(m_out.anchor(vmCall(Int64, operationGetByValStringInt, weakPointer(globalObject), base, index)));
7425 }
7426
7427 m_out.jump(continuation);
7428
7429 m_out.appendTo(continuation, lastNext);
7430 setJSValue(m_out.phi(Int64, results));
7431 }
7432
7433 void compileStringCharCodeAt()
7434 {
7435 LBasicBlock is8Bit = m_out.newBlock();
7436 LBasicBlock is16Bit = m_out.newBlock();
7437 LBasicBlock continuation = m_out.newBlock();
7438
7439 LValue base = lowString(m_node->child1());
7440 LValue index = lowInt32(m_node->child2());
7441 LValue storage = lowStorage(m_node->child3());
7442
7443 LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
7444
7445 speculate(
7446 Uncountable, noValue(), 0,
7447 m_out.aboveOrEqual(
7448 index, m_out.load32NonNegative(stringImpl, m_heaps.StringImpl_length)));
7449
7450 m_out.branch(
7451 m_out.testIsZero32(
7452 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
7453 m_out.constInt32(StringImpl::flagIs8Bit())),
7454 unsure(is16Bit), unsure(is8Bit));
7455
7456 LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
7457
7458 // FIXME: need to cage strings!
7459 // https://bugs.webkit.org/show_bug.cgi?id=174924
7460 ValueFromBlock char8Bit = m_out.anchor(
7461 m_out.load8ZeroExt32(m_out.baseIndex(
7462 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
7463 provenValue(m_node->child2()))));
7464 m_out.jump(continuation);
7465
7466 m_out.appendTo(is16Bit, continuation);
7467
7468 ValueFromBlock char16Bit = m_out.anchor(
7469 m_out.load16ZeroExt32(m_out.baseIndex(
7470 m_heaps.characters16, storage, m_out.zeroExtPtr(index),
7471 provenValue(m_node->child2()))));
7472 m_out.jump(continuation);
7473
7474 m_out.appendTo(continuation, lastNext);
7475
7476 setInt32(m_out.phi(Int32, char8Bit, char16Bit));
7477 }
7478
7479 void compileStringCodePointAt()
7480 {
7481 LBasicBlock is8Bit = m_out.newBlock();
7482 LBasicBlock is16Bit = m_out.newBlock();
7483 LBasicBlock isLeadSurrogate = m_out.newBlock();
7484 LBasicBlock mayHaveTrailSurrogate = m_out.newBlock();
7485 LBasicBlock hasTrailSurrogate = m_out.newBlock();
7486 LBasicBlock continuation = m_out.newBlock();
7487
7488 LValue base = lowString(m_node->child1());
7489 LValue index = lowInt32(m_node->child2());
7490 LValue storage = lowStorage(m_node->child3());
7491
7492 LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
7493 LValue length = m_out.load32NonNegative(stringImpl, m_heaps.StringImpl_length);
7494
7495 speculate(Uncountable, noValue(), 0, m_out.aboveOrEqual(index, length));
7496
7497 m_out.branch(
7498 m_out.testIsZero32(
7499 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
7500 m_out.constInt32(StringImpl::flagIs8Bit())),
7501 unsure(is16Bit), unsure(is8Bit));
7502
7503 LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
7504 // FIXME: Need to cage strings!
7505 // https://bugs.webkit.org/show_bug.cgi?id=174924
7506 ValueFromBlock char8Bit = m_out.anchor(
7507 m_out.load8ZeroExt32(m_out.baseIndex(
7508 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
7509 provenValue(m_node->child2()))));
7510 m_out.jump(continuation);
7511
7512 m_out.appendTo(is16Bit, isLeadSurrogate);
7513 LValue leadCharacter = m_out.load16ZeroExt32(m_out.baseIndex(m_heaps.characters16, storage, m_out.zeroExtPtr(index), provenValue(m_node->child2())));
7514 ValueFromBlock char16Bit = m_out.anchor(leadCharacter);
7515 LValue nextIndex = m_out.add(index, m_out.int32One);
7516 m_out.branch(m_out.aboveOrEqual(nextIndex, length), unsure(continuation), unsure(isLeadSurrogate));
7517
7518 m_out.appendTo(isLeadSurrogate, mayHaveTrailSurrogate);
7519 m_out.branch(m_out.notEqual(m_out.bitAnd(leadCharacter, m_out.constInt32(0xfffffc00)), m_out.constInt32(0xd800)), unsure(continuation), unsure(mayHaveTrailSurrogate));
7520
7521 m_out.appendTo(mayHaveTrailSurrogate, hasTrailSurrogate);
7522 JSValue indexValue = provenValue(m_node->child2());
7523 JSValue nextIndexValue;
7524 if (indexValue && indexValue.isInt32() && indexValue.asInt32() != INT32_MAX)
7525 nextIndexValue = jsNumber(indexValue.asInt32() + 1);
7526 LValue trailCharacter = m_out.load16ZeroExt32(m_out.baseIndex(m_heaps.characters16, storage, m_out.zeroExtPtr(nextIndex), nextIndexValue));
7527 m_out.branch(m_out.notEqual(m_out.bitAnd(trailCharacter, m_out.constInt32(0xfffffc00)), m_out.constInt32(0xdc00)), unsure(continuation), unsure(hasTrailSurrogate));
7528
7529 m_out.appendTo(hasTrailSurrogate, continuation);
7530 ValueFromBlock charSurrogatePair = m_out.anchor(m_out.sub(m_out.add(m_out.shl(leadCharacter, m_out.constInt32(10)), trailCharacter), m_out.constInt32(U16_SURROGATE_OFFSET)));
7531 m_out.jump(continuation);
7532
7533 m_out.appendTo(continuation, lastNext);
7534 setInt32(m_out.phi(Int32, char8Bit, char16Bit, charSurrogatePair));
7535 }
7536
7537 void compileStringFromCharCode()
7538 {
7539 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7540 Edge childEdge = m_node->child1();
7541
7542 if (childEdge.useKind() == UntypedUse) {
7543 LValue result = vmCall(
7544 Int64, operationStringFromCharCodeUntyped, weakPointer(globalObject),
7545 lowJSValue(childEdge));
7546 setJSValue(result);
7547 return;
7548 }
7549
7550 DFG_ASSERT(m_graph, m_node, childEdge.useKind() == Int32Use, childEdge.useKind());
7551
7552 LValue value = lowInt32(childEdge);
7553
7554 LBasicBlock smallIntCase = m_out.newBlock();
7555 LBasicBlock slowCase = m_out.newBlock();
7556 LBasicBlock continuation = m_out.newBlock();
7557
7558 m_out.branch(
7559 m_out.above(value, m_out.constInt32(maxSingleCharacterString)),
7560 rarely(slowCase), usually(smallIntCase));
7561
7562 LBasicBlock lastNext = m_out.appendTo(smallIntCase, slowCase);
7563
7564 LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
7565 LValue fastResultValue = m_out.loadPtr(
7566 m_out.baseIndex(m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(value)));
7567 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
7568 m_out.jump(continuation);
7569
7570 m_out.appendTo(slowCase, continuation);
7571
7572 LValue slowResultValue = vmCall(
7573 pointerType(), operationStringFromCharCode, weakPointer(globalObject), value);
7574 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
7575 m_out.jump(continuation);
7576
7577 m_out.appendTo(continuation, lastNext);
7578
7579 setJSValue(m_out.phi(Int64, fastResult, slowResult));
7580 }
7581
7582 void compileGetByOffset()
7583 {
7584 StorageAccessData& data = m_node->storageAccessData();
7585
7586 setJSValue(loadProperty(
7587 lowStorage(m_node->child1()), data.identifierNumber, data.offset));
7588 }
7589
7590 void compileGetGetter()
7591 {
7592 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_getter));
7593 }
7594
7595 void compileGetSetter()
7596 {
7597 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_setter));
7598 }
7599
7600 void compileMultiGetByOffset()
7601 {
7602 LValue base = lowCell(m_node->child1());
7603
7604 MultiGetByOffsetData& data = m_node->multiGetByOffsetData();
7605
7606 Vector<LBasicBlock, 2> blocks(data.cases.size());
7607 for (unsigned i = data.cases.size(); i--;)
7608 blocks[i] = m_out.newBlock();
7609 LBasicBlock exit = m_out.newBlock();
7610 LBasicBlock continuation = m_out.newBlock();
7611
7612 Vector<SwitchCase, 2> cases;
7613 RegisteredStructureSet baseSet;
7614 for (unsigned i = data.cases.size(); i--;) {
7615 MultiGetByOffsetCase getCase = data.cases[i];
7616 for (unsigned j = getCase.set().size(); j--;) {
7617 RegisteredStructure structure = getCase.set()[j];
7618 baseSet.add(structure);
7619 cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1)));
7620 }
7621 }
7622 bool structuresChecked = m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet);
7623 emitSwitchForMultiByOffset(base, structuresChecked, cases, exit);
7624
7625 LBasicBlock lastNext = m_out.m_nextBlock;
7626
7627 Vector<ValueFromBlock, 2> results;
7628 for (unsigned i = data.cases.size(); i--;) {
7629 MultiGetByOffsetCase getCase = data.cases[i];
7630 GetByOffsetMethod method = getCase.method();
7631
7632 m_out.appendTo(blocks[i], i + 1 < data.cases.size() ? blocks[i + 1] : exit);
7633
7634 LValue result;
7635
7636 switch (method.kind()) {
7637 case GetByOffsetMethod::Invalid:
7638 RELEASE_ASSERT_NOT_REACHED();
7639 break;
7640
7641 case GetByOffsetMethod::Constant:
7642 result = m_out.constInt64(JSValue::encode(method.constant()->value()));
7643 break;
7644
7645 case GetByOffsetMethod::Load:
7646 case GetByOffsetMethod::LoadFromPrototype: {
7647 LValue propertyBase;
7648 if (method.kind() == GetByOffsetMethod::Load)
7649 propertyBase = base;
7650 else
7651 propertyBase = weakPointer(method.prototype()->value().asCell());
7652 if (!isInlineOffset(method.offset()))
7653 propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly);
7654 result = loadProperty(
7655 propertyBase, data.identifierNumber, method.offset());
7656 break;
7657 } }
7658
7659 results.append(m_out.anchor(result));
7660 m_out.jump(continuation);
7661 }
7662
7663 m_out.appendTo(exit, continuation);
7664 if (!structuresChecked)
7665 speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
7666 m_out.unreachable();
7667
7668 m_out.appendTo(continuation, lastNext);
7669 setJSValue(m_out.phi(Int64, results));
7670 }
7671
7672 void compilePutByOffset()
7673 {
7674 StorageAccessData& data = m_node->storageAccessData();
7675
7676 storeProperty(
7677 lowJSValue(m_node->child3()),
7678 lowStorage(m_node->child1()), data.identifierNumber, data.offset);
7679 }
7680
7681 void compileMultiPutByOffset()
7682 {
7683 LValue base = lowCell(m_node->child1());
7684 LValue value = lowJSValue(m_node->child2());
7685
7686 MultiPutByOffsetData& data = m_node->multiPutByOffsetData();
7687
7688 Vector<LBasicBlock, 2> blocks(data.variants.size());
7689 for (unsigned i = data.variants.size(); i--;)
7690 blocks[i] = m_out.newBlock();
7691 LBasicBlock exit = m_out.newBlock();
7692 LBasicBlock continuation = m_out.newBlock();
7693
7694 Vector<SwitchCase, 2> cases;
7695 RegisteredStructureSet baseSet;
7696 for (unsigned i = data.variants.size(); i--;) {
7697 PutByIdVariant variant = data.variants[i];
7698 for (unsigned j = variant.oldStructure().size(); j--;) {
7699 RegisteredStructure structure = m_graph.registerStructure(variant.oldStructure()[j]);
7700 baseSet.add(structure);
7701 cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1)));
7702 }
7703 }
7704 bool structuresChecked = m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet);
7705 emitSwitchForMultiByOffset(base, structuresChecked, cases, exit);
7706
7707 LBasicBlock lastNext = m_out.m_nextBlock;
7708
7709 for (unsigned i = data.variants.size(); i--;) {
7710 m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
7711
7712 PutByIdVariant variant = data.variants[i];
7713
7714 LValue storage;
7715 if (variant.kind() == PutByIdVariant::Replace) {
7716 if (isInlineOffset(variant.offset()))
7717 storage = base;
7718 else
7719 storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
7720 } else {
7721 DFG_ASSERT(m_graph, m_node, variant.kind() == PutByIdVariant::Transition, variant.kind());
7722 m_graph.m_plan.transitions().addLazily(
7723 codeBlock(), m_node->origin.semantic.codeOriginOwner(),
7724 variant.oldStructureForTransition(), variant.newStructure());
7725
7726 storage = storageForTransition(
7727 base, variant.offset(),
7728 variant.oldStructureForTransition(), variant.newStructure());
7729 }
7730
7731 storeProperty(value, storage, data.identifierNumber, variant.offset());
7732
7733 if (variant.kind() == PutByIdVariant::Transition) {
7734 ASSERT(variant.oldStructureForTransition()->indexingType() == variant.newStructure()->indexingType());
7735 ASSERT(variant.oldStructureForTransition()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags());
7736 ASSERT(variant.oldStructureForTransition()->typeInfo().type() == variant.newStructure()->typeInfo().type());
7737 m_out.store32(
7738 weakStructureID(m_graph.registerStructure(variant.newStructure())), base, m_heaps.JSCell_structureID);
7739 }
7740
7741 m_out.jump(continuation);
7742 }
7743
7744 m_out.appendTo(exit, continuation);
7745 if (!structuresChecked)
7746 speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
7747 m_out.unreachable();
7748
7749 m_out.appendTo(continuation, lastNext);
7750 }
7751
7752 void compileMatchStructure()
7753 {
7754 LValue base = lowCell(m_node->child1());
7755
7756 MatchStructureData& data = m_node->matchStructureData();
7757
7758 LBasicBlock trueBlock = m_out.newBlock();
7759 LBasicBlock falseBlock = m_out.newBlock();
7760 LBasicBlock exitBlock = m_out.newBlock();
7761 LBasicBlock continuation = m_out.newBlock();
7762
7763 LBasicBlock lastNext = m_out.insertNewBlocksBefore(trueBlock);
7764
7765 Vector<SwitchCase, 2> cases;
7766 RegisteredStructureSet baseSet;
7767 for (MatchStructureVariant& variant : data.variants) {
7768 baseSet.add(variant.structure);
7769 cases.append(SwitchCase(
7770 weakStructureID(variant.structure),
7771 variant.result ? trueBlock : falseBlock, Weight(1)));
7772 }
7773 bool structuresChecked = m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet);
7774 emitSwitchForMultiByOffset(base, structuresChecked, cases, exitBlock);
7775
7776 m_out.appendTo(trueBlock, falseBlock);
7777 ValueFromBlock trueResult = m_out.anchor(m_out.booleanTrue);
7778 m_out.jump(continuation);
7779
7780 m_out.appendTo(falseBlock, exitBlock);
7781 ValueFromBlock falseResult = m_out.anchor(m_out.booleanFalse);
7782 m_out.jump(continuation);
7783
7784 m_out.appendTo(exitBlock, continuation);
7785 if (!structuresChecked)
7786 speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
7787 m_out.unreachable();
7788
7789 m_out.appendTo(continuation, lastNext);
7790 setBoolean(m_out.phi(Int32, trueResult, falseResult));
7791 }
7792
7793 void compileGetGlobalVariable()
7794 {
7795 setJSValue(m_out.load64(m_out.absolute(m_node->variablePointer())));
7796 }
7797
7798 void compilePutGlobalVariable()
7799 {
7800 m_out.store64(
7801 lowJSValue(m_node->child2()), m_out.absolute(m_node->variablePointer()));
7802 }
7803
7804 void compileNotifyWrite()
7805 {
7806 WatchpointSet* set = m_node->watchpointSet();
7807
7808 LBasicBlock isNotInvalidated = m_out.newBlock();
7809 LBasicBlock continuation = m_out.newBlock();
7810
7811 LValue state = m_out.load8ZeroExt32(m_out.absolute(set->addressOfState()));
7812 m_out.branch(
7813 m_out.equal(state, m_out.constInt32(IsInvalidated)),
7814 usually(continuation), rarely(isNotInvalidated));
7815
7816 LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, continuation);
7817
7818 VM& vm = this->vm();
7819 lazySlowPath(
7820 [=, &vm] (const Vector<Location>&) -> RefPtr<LazySlowPath::Generator> {
7821 return createLazyCallGenerator(vm,
7822 operationNotifyWrite, InvalidGPRReg, &vm, CCallHelpers::TrustedImmPtr(set));
7823 });
7824 m_out.jump(continuation);
7825
7826 m_out.appendTo(continuation, lastNext);
7827 }
7828
7829 void compileGetCallee()
7830 {
7831 setJSValue(m_out.loadPtr(addressFor(CallFrameSlot::callee)));
7832 }
7833
7834 void compileSetCallee()
7835 {
7836 auto callee = lowCell(m_node->child1());
7837 m_out.storePtr(callee, payloadFor(CallFrameSlot::callee));
7838 }
7839
7840 void compileGetArgumentCountIncludingThis()
7841 {
7842 VirtualRegister argumentCountRegister;
7843 if (InlineCallFrame* inlineCallFrame = m_node->argumentsInlineCallFrame())
7844 argumentCountRegister = inlineCallFrame->argumentCountRegister;
7845 else
7846 argumentCountRegister = VirtualRegister(CallFrameSlot::argumentCount);
7847 setInt32(m_out.load32(payloadFor(argumentCountRegister)));
7848 }
7849
7850 void compileSetArgumentCountIncludingThis()
7851 {
7852 m_out.store32(m_out.constInt32(m_node->argumentCountIncludingThis()), payloadFor(CallFrameSlot::argumentCount));
7853 }
7854
7855 void compileGetScope()
7856 {
7857 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope));
7858 }
7859
7860 void compileSkipScope()
7861 {
7862 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
7863 }
7864
7865 void compileGetGlobalObject()
7866 {
7867 LValue structure = loadStructure(lowCell(m_node->child1()));
7868 setJSValue(m_out.loadPtr(structure, m_heaps.Structure_globalObject));
7869 }
7870
7871 void compileGetGlobalThis()
7872 {
7873 auto* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7874 setJSValue(m_out.loadPtr(m_out.absolute(globalObject->addressOfGlobalThis())));
7875 }
7876
7877 void compileGetClosureVar()
7878 {
7879 setJSValue(
7880 m_out.load64(
7881 lowCell(m_node->child1()),
7882 m_heaps.JSLexicalEnvironment_variables[m_node->scopeOffset().offset()]));
7883 }
7884
7885 void compilePutClosureVar()
7886 {
7887 m_out.store64(
7888 lowJSValue(m_node->child2()),
7889 lowCell(m_node->child1()),
7890 m_heaps.JSLexicalEnvironment_variables[m_node->scopeOffset().offset()]);
7891 }
7892
7893 void compileGetInternalField()
7894 {
7895 setJSValue(
7896 m_out.load64(
7897 lowCell(m_node->child1()),
7898 m_heaps.JSInternalFieldObjectImpl_internalFields[m_node->internalFieldIndex()]));
7899 }
7900
7901 void compilePutInternalField()
7902 {
7903 m_out.store64(
7904 lowJSValue(m_node->child2()),
7905 lowCell(m_node->child1()),
7906 m_heaps.JSInternalFieldObjectImpl_internalFields[m_node->internalFieldIndex()]);
7907 }
7908
7909 void compileGetFromArguments()
7910 {
7911 setJSValue(
7912 m_out.load64(
7913 lowCell(m_node->child1()),
7914 m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]));
7915 }
7916
7917 void compilePutToArguments()
7918 {
7919 m_out.store64(
7920 lowJSValue(m_node->child2()),
7921 lowCell(m_node->child1()),
7922 m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]);
7923 }
7924
7925 void compileGetArgument()
7926 {
7927 LValue argumentCount = m_out.load32(payloadFor(AssemblyHelpers::argumentCount(m_node->origin.semantic)));
7928
7929 LBasicBlock inBounds = m_out.newBlock();
7930 LBasicBlock outOfBounds = m_out.newBlock();
7931 LBasicBlock continuation = m_out.newBlock();
7932
7933 m_out.branch(m_out.lessThanOrEqual(argumentCount, m_out.constInt32(m_node->argumentIndex())), unsure(outOfBounds), unsure(inBounds));
7934
7935 LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
7936 VirtualRegister arg = AssemblyHelpers::argumentsStart(m_node->origin.semantic) + m_node->argumentIndex() - 1;
7937 ValueFromBlock inBoundsResult = m_out.anchor(m_out.load64(addressFor(arg)));
7938 m_out.jump(continuation);
7939
7940 m_out.appendTo(outOfBounds, continuation);
7941 ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(JSValue::ValueUndefined));
7942 m_out.jump(continuation);
7943
7944 m_out.appendTo(continuation, lastNext);
7945 setJSValue(m_out.phi(Int64, inBoundsResult, outOfBoundsResult));
7946 }
7947
7948 void compileCompareEq()
7949 {
7950 if (m_node->isBinaryUseKind(Int32Use)
7951 || m_node->isBinaryUseKind(Int52RepUse)
7952 || m_node->isBinaryUseKind(DoubleRepUse)
7953 || m_node->isBinaryUseKind(ObjectUse)
7954 || m_node->isBinaryUseKind(BooleanUse)
7955 || m_node->isBinaryUseKind(SymbolUse)
7956 || m_node->isBinaryUseKind(StringIdentUse)
7957 || m_node->isBinaryUseKind(StringUse)) {
7958 compileCompareStrictEq();
7959 return;
7960 }
7961
7962 if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) {
7963 compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1());
7964 return;
7965 }
7966
7967 if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) {
7968 compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2());
7969 return;
7970 }
7971
7972 if (m_node->child1().useKind() == KnownOtherUse) {
7973 ASSERT(!m_interpreter.needsTypeCheck(m_node->child1(), SpecOther));
7974 setBoolean(equalNullOrUndefined(m_node->child2(), AllCellsAreFalse, EqualNullOrUndefined, ManualOperandSpeculation));
7975 return;
7976 }
7977
7978 if (m_node->child2().useKind() == KnownOtherUse) {
7979 ASSERT(!m_interpreter.needsTypeCheck(m_node->child2(), SpecOther));
7980 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined, ManualOperandSpeculation));
7981 return;
7982 }
7983
7984 DFG_ASSERT(m_graph, m_node, m_node->isBinaryUseKind(UntypedUse), m_node->child1().useKind(), m_node->child2().useKind());
7985 nonSpeculativeCompare(
7986 [&] (LValue left, LValue right) {
7987 return m_out.equal(left, right);
7988 },
7989 operationCompareEq);
7990 }
7991
7992 void compileCompareStrictEq()
7993 {
7994 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
7995 if (m_node->isBinaryUseKind(Int32Use)) {
7996 setBoolean(
7997 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
7998 return;
7999 }
8000
8001 if (m_node->isBinaryUseKind(Int52RepUse)) {
8002 Int52Kind kind;
8003 LValue left = lowWhicheverInt52(m_node->child1(), kind);
8004 LValue right = lowInt52(m_node->child2(), kind);
8005 setBoolean(m_out.equal(left, right));
8006 return;
8007 }
8008
8009 if (m_node->isBinaryUseKind(DoubleRepUse)) {
8010 setBoolean(
8011 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
8012 return;
8013 }
8014
8015 if (m_node->isBinaryUseKind(StringIdentUse)) {
8016 setBoolean(
8017 m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2())));
8018 return;
8019 }
8020
8021 if (m_node->isBinaryUseKind(StringUse)) {
8022 LValue left = lowCell(m_node->child1());
8023 LValue right = lowCell(m_node->child2());
8024
8025 LBasicBlock notTriviallyEqualCase = m_out.newBlock();
8026 LBasicBlock continuation = m_out.newBlock();
8027
8028 speculateString(m_node->child1(), left);
8029
8030 ValueFromBlock fastResult = m_out.anchor(m_out.booleanTrue);
8031 m_out.branch(
8032 m_out.equal(left, right), unsure(continuation), unsure(notTriviallyEqualCase));
8033
8034 LBasicBlock lastNext = m_out.appendTo(notTriviallyEqualCase, continuation);
8035
8036 speculateString(m_node->child2(), right);
8037
8038 ValueFromBlock slowResult = m_out.anchor(stringsEqual(left, right, m_node->child1(), m_node->child2()));
8039 m_out.jump(continuation);
8040
8041 m_out.appendTo(continuation, lastNext);
8042 setBoolean(m_out.phi(Int32, fastResult, slowResult));
8043 return;
8044 }
8045
8046 if (m_node->isBinaryUseKind(ObjectUse, UntypedUse)) {
8047 setBoolean(
8048 m_out.equal(
8049 lowNonNullObject(m_node->child1()),
8050 lowJSValue(m_node->child2())));
8051 return;
8052 }
8053
8054 if (m_node->isBinaryUseKind(UntypedUse, ObjectUse)) {
8055 setBoolean(
8056 m_out.equal(
8057 lowNonNullObject(m_node->child2()),
8058 lowJSValue(m_node->child1())));
8059 return;
8060 }
8061
8062 if (m_node->isBinaryUseKind(ObjectUse)) {
8063 setBoolean(
8064 m_out.equal(
8065 lowNonNullObject(m_node->child1()),
8066 lowNonNullObject(m_node->child2())));
8067 return;
8068 }
8069
8070 if (m_node->isBinaryUseKind(BooleanUse)) {
8071 setBoolean(
8072 m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2())));
8073 return;
8074 }
8075
8076 if (m_node->isBinaryUseKind(SymbolUse)) {
8077 LValue leftSymbol = lowSymbol(m_node->child1());
8078 LValue rightSymbol = lowSymbol(m_node->child2());
8079 setBoolean(m_out.equal(leftSymbol, rightSymbol));
8080 return;
8081 }
8082
8083 if (m_node->isBinaryUseKind(BigIntUse)) {
8084 // FIXME: [ESNext][BigInt] Create specialized version of strict equals for BigIntUse
8085 // https://bugs.webkit.org/show_bug.cgi?id=182895
8086 LValue left = lowBigInt(m_node->child1());
8087 LValue right = lowBigInt(m_node->child2());
8088
8089 LBasicBlock notTriviallyEqualCase = m_out.newBlock();
8090 LBasicBlock continuation = m_out.newBlock();
8091
8092 ValueFromBlock fastResult = m_out.anchor(m_out.booleanTrue);
8093 m_out.branch(m_out.equal(left, right), rarely(continuation), usually(notTriviallyEqualCase));
8094
8095 LBasicBlock lastNext = m_out.appendTo(notTriviallyEqualCase, continuation);
8096
8097 ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
8098 pointerType(), operationCompareStrictEq, weakPointer(globalObject), left, right)));
8099 m_out.jump(continuation);
8100
8101 m_out.appendTo(continuation, lastNext);
8102 setBoolean(m_out.phi(Int32, fastResult, slowResult));
8103 return;
8104 }
8105
8106 if (m_node->isBinaryUseKind(SymbolUse, UntypedUse)
8107 || m_node->isBinaryUseKind(UntypedUse, SymbolUse)) {
8108 Edge symbolEdge = m_node->child1();
8109 Edge untypedEdge = m_node->child2();
8110 if (symbolEdge.useKind() != SymbolUse)
8111 std::swap(symbolEdge, untypedEdge);
8112
8113 LValue leftSymbol = lowSymbol(symbolEdge);
8114 LValue untypedValue = lowJSValue(untypedEdge);
8115
8116 setBoolean(m_out.equal(leftSymbol, untypedValue));
8117 return;
8118 }
8119
8120 if (m_node->isBinaryUseKind(MiscUse, UntypedUse)
8121 || m_node->isBinaryUseKind(UntypedUse, MiscUse)) {
8122 speculate(m_node->child1());
8123 speculate(m_node->child2());
8124 LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation);
8125 LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation);
8126 setBoolean(m_out.equal(left, right));
8127 return;
8128 }
8129
8130 if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse)
8131 || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) {
8132 Edge leftEdge = m_node->childFor(StringIdentUse);
8133 Edge rightEdge = m_node->childFor(NotStringVarUse);
8134
8135 LValue left = lowStringIdent(leftEdge);
8136 LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation);
8137
8138 LBasicBlock isCellCase = m_out.newBlock();
8139 LBasicBlock isStringCase = m_out.newBlock();
8140 LBasicBlock continuation = m_out.newBlock();
8141
8142 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
8143 m_out.branch(
8144 isCell(rightValue, provenType(rightEdge)),
8145 unsure(isCellCase), unsure(continuation));
8146
8147 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
8148 ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse);
8149 m_out.branch(
8150 isString(rightValue, provenType(rightEdge)),
8151 unsure(isStringCase), unsure(continuation));
8152
8153 m_out.appendTo(isStringCase, continuation);
8154 LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value);
8155 speculateStringIdent(rightEdge, rightValue, right);
8156 ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right));
8157 m_out.jump(continuation);
8158
8159 m_out.appendTo(continuation, lastNext);
8160 setBoolean(m_out.phi(Int32, notCellResult, notStringResult, isStringResult));
8161 return;
8162 }
8163
8164 if (m_node->isBinaryUseKind(StringUse, UntypedUse)) {
8165 compileStringToUntypedStrictEquality(m_node->child1(), m_node->child2());
8166 return;
8167 }
8168 if (m_node->isBinaryUseKind(UntypedUse, StringUse)) {
8169 compileStringToUntypedStrictEquality(m_node->child2(), m_node->child1());
8170 return;
8171 }
8172
8173 DFG_ASSERT(m_graph, m_node, m_node->isBinaryUseKind(UntypedUse), m_node->child1().useKind(), m_node->child2().useKind());
8174 nonSpeculativeCompare(
8175 [&] (LValue left, LValue right) {
8176 return m_out.equal(left, right);
8177 },
8178 operationCompareStrictEq);
8179 }
8180
8181 void compileStringToUntypedStrictEquality(Edge stringEdge, Edge untypedEdge)
8182 {
8183 ASSERT(stringEdge.useKind() == StringUse);
8184 ASSERT(untypedEdge.useKind() == UntypedUse);
8185
8186 LValue leftString = lowCell(stringEdge);
8187 LValue rightValue = lowJSValue(untypedEdge);
8188 SpeculatedType rightValueType = provenType(untypedEdge);
8189
8190 // Verify left is string.
8191 speculateString(stringEdge, leftString);
8192
8193 LBasicBlock testUntypedEdgeIsCell = m_out.newBlock();
8194 LBasicBlock testUntypedEdgeIsString = m_out.newBlock();
8195 LBasicBlock testStringEquality = m_out.newBlock();
8196 LBasicBlock continuation = m_out.newBlock();
8197
8198 // Given left is string. If the value are strictly equal, rightValue has to be the same string.
8199 ValueFromBlock fastTrue = m_out.anchor(m_out.booleanTrue);
8200 m_out.branch(m_out.equal(leftString, rightValue), unsure(continuation), unsure(testUntypedEdgeIsCell));
8201
8202 LBasicBlock lastNext = m_out.appendTo(testUntypedEdgeIsCell, testUntypedEdgeIsString);
8203 ValueFromBlock fastFalse = m_out.anchor(m_out.booleanFalse);
8204 m_out.branch(isNotCell(rightValue, rightValueType), unsure(continuation), unsure(testUntypedEdgeIsString));
8205
8206 // Check if the untyped edge is a string.
8207 m_out.appendTo(testUntypedEdgeIsString, testStringEquality);
8208 m_out.branch(isNotString(rightValue, rightValueType), unsure(continuation), unsure(testStringEquality));
8209
8210 // Full String compare.
8211 m_out.appendTo(testStringEquality, continuation);
8212 ValueFromBlock slowResult = m_out.anchor(stringsEqual(leftString, rightValue, stringEdge, untypedEdge));
8213 m_out.jump(continuation);
8214
8215 // Continuation.
8216 m_out.appendTo(continuation, lastNext);
8217 setBoolean(m_out.phi(Int32, fastTrue, fastFalse, slowResult));
8218 }
8219
8220 void compileCompareEqPtr()
8221 {
8222 setBoolean(
8223 m_out.equal(
8224 lowJSValue(m_node->child1()),
8225 weakPointer(m_node->cellOperand()->cell())));
8226 }
8227
8228 void compileCompareLess()
8229 {
8230 compare(
8231 [&] (LValue left, LValue right) {
8232 return m_out.lessThan(left, right);
8233 },
8234 [&] (LValue left, LValue right) {
8235 return m_out.doubleLessThan(left, right);
8236 },
8237 operationCompareStringImplLess,
8238 operationCompareStringLess,
8239 operationCompareLess);
8240 }
8241
8242 void compileCompareLessEq()
8243 {
8244 compare(
8245 [&] (LValue left, LValue right) {
8246 return m_out.lessThanOrEqual(left, right);
8247 },
8248 [&] (LValue left, LValue right) {
8249 return m_out.doubleLessThanOrEqual(left, right);
8250 },
8251 operationCompareStringImplLessEq,
8252 operationCompareStringLessEq,
8253 operationCompareLessEq);
8254 }
8255
8256 void compileCompareGreater()
8257 {
8258 compare(
8259 [&] (LValue left, LValue right) {
8260 return m_out.greaterThan(left, right);
8261 },
8262 [&] (LValue left, LValue right) {
8263 return m_out.doubleGreaterThan(left, right);
8264 },
8265 operationCompareStringImplGreater,
8266 operationCompareStringGreater,
8267 operationCompareGreater);
8268 }
8269
8270 void compileCompareGreaterEq()
8271 {
8272 compare(
8273 [&] (LValue left, LValue right) {
8274 return m_out.greaterThanOrEqual(left, right);
8275 },
8276 [&] (LValue left, LValue right) {
8277 return m_out.doubleGreaterThanOrEqual(left, right);
8278 },
8279 operationCompareStringImplGreaterEq,
8280 operationCompareStringGreaterEq,
8281 operationCompareGreaterEq);
8282 }
8283
8284 void compileCompareBelow()
8285 {
8286 setBoolean(m_out.below(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
8287 }
8288
8289 void compileCompareBelowEq()
8290 {
8291 setBoolean(m_out.belowOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
8292 }
8293
8294 void compileSameValue()
8295 {
8296 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
8297 if (m_node->isBinaryUseKind(DoubleRepUse)) {
8298 LValue arg1 = lowDouble(m_node->child1());
8299 LValue arg2 = lowDouble(m_node->child2());
8300
8301 LBasicBlock numberCase = m_out.newBlock();
8302 LBasicBlock continuation = m_out.newBlock();
8303
8304 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
8305 patchpoint->append(arg1, ValueRep::SomeRegister);
8306 patchpoint->append(arg2, ValueRep::SomeRegister);
8307 patchpoint->numGPScratchRegisters = 1;
8308 patchpoint->setGenerator(
8309 [] (CCallHelpers& jit, const StackmapGenerationParams& params) {
8310 GPRReg scratchGPR = params.gpScratch(0);
8311 jit.moveDoubleTo64(params[1].fpr(), scratchGPR);
8312 jit.moveDoubleTo64(params[2].fpr(), params[0].gpr());
8313 jit.compare64(CCallHelpers::Equal, scratchGPR, params[0].gpr(), params[0].gpr());
8314 });
8315 patchpoint->effects = Effects::none();
8316 ValueFromBlock compareResult = m_out.anchor(patchpoint);
8317 m_out.branch(patchpoint, unsure(continuation), unsure(numberCase));
8318
8319 LBasicBlock lastNext = m_out.appendTo(numberCase, continuation);
8320 LValue isArg1NaN = m_out.doubleNotEqualOrUnordered(arg1, arg1);
8321 LValue isArg2NaN = m_out.doubleNotEqualOrUnordered(arg2, arg2);
8322 ValueFromBlock nanResult = m_out.anchor(m_out.bitAnd(isArg1NaN, isArg2NaN));
8323 m_out.jump(continuation);
8324
8325 m_out.appendTo(continuation, lastNext);
8326 setBoolean(m_out.phi(Int32, compareResult, nanResult));
8327 return;
8328 }
8329
8330 ASSERT(m_node->isBinaryUseKind(UntypedUse));
8331 setBoolean(vmCall(Int32, operationSameValue, weakPointer(globalObject), lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
8332 }
8333
8334 void compileLogicalNot()
8335 {
8336 setBoolean(m_out.logicalNot(boolify(m_node->child1())));
8337 }
8338
8339 void compileCallOrConstruct()
8340 {
8341 Node* node = m_node;
8342 unsigned numArgs = node->numChildren() - 1;
8343
8344 LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0));
8345
8346 unsigned frameSize = (CallFrame::headerSizeInRegisters + numArgs) * sizeof(EncodedJSValue);
8347 unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSize);
8348
8349 // JS->JS calling convention requires that the caller allows this much space on top of stack to
8350 // get trashed by the callee, even if not all of that space is used to pass arguments. We tell
8351 // B3 this explicitly for two reasons:
8352 //
8353 // - We will only pass frameSize worth of stuff.
8354 // - The trashed stack guarantee is logically separate from the act of passing arguments, so we
8355 // shouldn't rely on Air to infer the trashed stack property based on the arguments it ends
8356 // up seeing.
8357 m_proc.requestCallArgAreaSizeInBytes(alignedFrameSize);
8358
8359 // Collect the arguments, since this can generate code and we want to generate it before we emit
8360 // the call.
8361 Vector<ConstrainedValue> arguments;
8362
8363 // Make sure that the callee goes into GPR0 because that's where the slow path thunks expect the
8364 // callee to be.
8365 arguments.append(ConstrainedValue(jsCallee, ValueRep::reg(GPRInfo::regT0)));
8366
8367 auto addArgument = [&] (LValue value, VirtualRegister reg, int offset) {
8368 intptr_t offsetFromSP =
8369 (reg.offset() - CallerFrameAndPC::sizeInRegisters) * sizeof(EncodedJSValue) + offset;
8370 arguments.append(ConstrainedValue(value, ValueRep::stackArgument(offsetFromSP)));
8371 };
8372
8373 addArgument(jsCallee, VirtualRegister(CallFrameSlot::callee), 0);
8374 addArgument(m_out.constInt32(numArgs), VirtualRegister(CallFrameSlot::argumentCount), PayloadOffset);
8375 for (unsigned i = 0; i < numArgs; ++i)
8376 addArgument(lowJSValue(m_graph.varArgChild(node, 1 + i)), virtualRegisterForArgument(i), 0);
8377
8378 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
8379 patchpoint->appendVector(arguments);
8380
8381 RefPtr<PatchpointExceptionHandle> exceptionHandle =
8382 preparePatchpointForExceptions(patchpoint);
8383
8384 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
8385 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
8386 patchpoint->clobber(RegisterSet::macroScratchRegisters());
8387 patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
8388 patchpoint->resultConstraints = { ValueRep::reg(GPRInfo::returnValueGPR) };
8389
8390 CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite();
8391 State* state = &m_ftlState;
8392 VM* vm = &this->vm();
8393 patchpoint->setGenerator(
8394 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
8395 AllowMacroScratchRegisterUsage allowScratch(jit);
8396 CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin);
8397
8398 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
8399
8400 jit.store32(
8401 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
8402 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
8403
8404 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
8405
8406 CCallHelpers::DataLabelPtr targetToCheck;
8407 CCallHelpers::Jump slowPath = jit.branchPtrWithPatch(
8408 CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck,
8409 CCallHelpers::TrustedImmPtr(nullptr));
8410
8411 CCallHelpers::Call fastCall = jit.nearCall();
8412 CCallHelpers::Jump done = jit.jump();
8413
8414 slowPath.link(&jit);
8415
8416 jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2);
8417 jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()->globalObjectFor(node->origin.semantic)), GPRInfo::regT3);
8418 CCallHelpers::Call slowCall = jit.nearCall();
8419 done.link(&jit);
8420
8421 callLinkInfo->setUpCall(
8422 node->op() == Construct ? CallLinkInfo::Construct : CallLinkInfo::Call,
8423 node->origin.semantic, GPRInfo::regT0);
8424
8425 jit.addPtr(
8426 CCallHelpers::TrustedImm32(-params.proc().frameSize()),
8427 GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
8428
8429 jit.addLinkTask(
8430 [=] (LinkBuffer& linkBuffer) {
8431 MacroAssemblerCodePtr<JITThunkPtrTag> linkCall = vm->getCTIStub(linkCallThunkGenerator).code();
8432 linkBuffer.link(slowCall, FunctionPtr<JITThunkPtrTag>(linkCall));
8433
8434 callLinkInfo->setCallLocations(
8435 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOfNearCall<JSInternalPtrTag>(slowCall)),
8436 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOf<JSInternalPtrTag>(targetToCheck)),
8437 linkBuffer.locationOfNearCall<JSInternalPtrTag>(fastCall));
8438 });
8439 });
8440
8441 setJSValue(patchpoint);
8442 }
8443
8444 void compileDirectCallOrConstruct()
8445 {
8446 Node* node = m_node;
8447 bool isTail = node->op() == DirectTailCall;
8448 bool isConstruct = node->op() == DirectConstruct;
8449
8450 ExecutableBase* executable = node->castOperand<ExecutableBase*>();
8451 FunctionExecutable* functionExecutable = jsDynamicCast<FunctionExecutable*>(vm(), executable);
8452
8453 unsigned numPassedArgs = node->numChildren() - 1;
8454 unsigned numAllocatedArgs = numPassedArgs;
8455
8456 if (functionExecutable) {
8457 numAllocatedArgs = std::max(
8458 numAllocatedArgs,
8459 std::min(
8460 static_cast<unsigned>(functionExecutable->parameterCount()) + 1,
8461 Options::maximumDirectCallStackSize()));
8462 }
8463
8464 LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0));
8465
8466 if (!isTail) {
8467 unsigned frameSize = (CallFrame::headerSizeInRegisters + numAllocatedArgs) * sizeof(EncodedJSValue);
8468 unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSize);
8469
8470 m_proc.requestCallArgAreaSizeInBytes(alignedFrameSize);
8471 }
8472
8473 Vector<ConstrainedValue> arguments;
8474
8475 arguments.append(ConstrainedValue(jsCallee, ValueRep::SomeRegister));
8476 if (!isTail) {
8477 auto addArgument = [&] (LValue value, VirtualRegister reg, int offset) {
8478 intptr_t offsetFromSP =
8479 (reg.offset() - CallerFrameAndPC::sizeInRegisters) * sizeof(EncodedJSValue) + offset;
8480 arguments.append(ConstrainedValue(value, ValueRep::stackArgument(offsetFromSP)));
8481 };
8482
8483 addArgument(jsCallee, VirtualRegister(CallFrameSlot::callee), 0);
8484 addArgument(m_out.constInt32(numPassedArgs), VirtualRegister(CallFrameSlot::argumentCount), PayloadOffset);
8485 for (unsigned i = 0; i < numPassedArgs; ++i)
8486 addArgument(lowJSValue(m_graph.varArgChild(node, 1 + i)), virtualRegisterForArgument(i), 0);
8487 for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i)
8488 addArgument(m_out.constInt64(JSValue::encode(jsUndefined())), virtualRegisterForArgument(i), 0);
8489 } else {
8490 for (unsigned i = 0; i < numPassedArgs; ++i)
8491 arguments.append(ConstrainedValue(lowJSValue(m_graph.varArgChild(node, 1 + i)), ValueRep::WarmAny));
8492 }
8493
8494 PatchpointValue* patchpoint = m_out.patchpoint(isTail ? Void : Int64);
8495 patchpoint->appendVector(arguments);
8496
8497 RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
8498
8499 if (isTail) {
8500 // The shuffler needs tags.
8501 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
8502 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
8503 }
8504
8505 patchpoint->clobber(RegisterSet::macroScratchRegisters());
8506 if (!isTail) {
8507 patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
8508 patchpoint->resultConstraints = { ValueRep::reg(GPRInfo::returnValueGPR) };
8509 }
8510
8511 CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite();
8512 State* state = &m_ftlState;
8513 patchpoint->setGenerator(
8514 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
8515 AllowMacroScratchRegisterUsage allowScratch(jit);
8516 CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin);
8517
8518 GPRReg calleeGPR = params[!isTail].gpr();
8519
8520 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
8521
8522 Box<CCallHelpers::JumpList> exceptions =
8523 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
8524
8525 if (isTail) {
8526 CallFrameShuffleData shuffleData;
8527 shuffleData.numLocals = state->jitCode->common.frameRegisterCount;
8528
8529 RegisterSet toSave = params.unavailableRegisters();
8530 shuffleData.callee = ValueRecovery::inGPR(calleeGPR, DataFormatCell);
8531 toSave.set(calleeGPR);
8532 for (unsigned i = 0; i < numPassedArgs; ++i) {
8533 ValueRecovery recovery = params[1 + i].recoveryForJSValue();
8534 shuffleData.args.append(recovery);
8535 recovery.forEachReg(
8536 [&] (Reg reg) {
8537 toSave.set(reg);
8538 });
8539 }
8540 for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i)
8541 shuffleData.args.append(ValueRecovery::constant(jsUndefined()));
8542 shuffleData.numPassedArgs = numPassedArgs;
8543 shuffleData.setupCalleeSaveRegisters(jit.codeBlock());
8544
8545 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
8546
8547 CCallHelpers::PatchableJump patchableJump = jit.patchableJump();
8548 CCallHelpers::Label mainPath = jit.label();
8549
8550 jit.store32(
8551 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
8552 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
8553
8554 callLinkInfo->setFrameShuffleData(shuffleData);
8555 CallFrameShuffler(jit, shuffleData).prepareForTailCall();
8556
8557 CCallHelpers::Call call = jit.nearTailCall();
8558
8559 jit.abortWithReason(JITDidReturnFromTailCall);
8560
8561 CCallHelpers::Label slowPath = jit.label();
8562 patchableJump.m_jump.linkTo(slowPath, &jit);
8563 callOperation(
8564 *state, toSave, jit,
8565 node->origin.semantic, exceptions.get(), operationLinkDirectCall,
8566 InvalidGPRReg, CCallHelpers::TrustedImmPtr(callLinkInfo), calleeGPR).call();
8567 jit.jump().linkTo(mainPath, &jit);
8568
8569 callLinkInfo->setUpCall(
8570 CallLinkInfo::DirectTailCall, node->origin.semantic, InvalidGPRReg);
8571 callLinkInfo->setExecutableDuringCompilation(executable);
8572 if (numAllocatedArgs > numPassedArgs)
8573 callLinkInfo->setMaxArgumentCountIncludingThis(numAllocatedArgs);
8574
8575 jit.addLinkTask(
8576 [=] (LinkBuffer& linkBuffer) {
8577 CodeLocationLabel<JSInternalPtrTag> patchableJumpLocation = linkBuffer.locationOf<JSInternalPtrTag>(patchableJump);
8578 CodeLocationNearCall<JSInternalPtrTag> callLocation = linkBuffer.locationOfNearCall<JSInternalPtrTag>(call);
8579 CodeLocationLabel<JSInternalPtrTag> slowPathLocation = linkBuffer.locationOf<JSInternalPtrTag>(slowPath);
8580
8581 callLinkInfo->setCallLocations(
8582 patchableJumpLocation,
8583 slowPathLocation,
8584 callLocation);
8585 });
8586 return;
8587 }
8588
8589 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
8590
8591 CCallHelpers::Label mainPath = jit.label();
8592
8593 jit.store32(
8594 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
8595 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
8596
8597 CCallHelpers::Call call = jit.nearCall();
8598 jit.addPtr(
8599 CCallHelpers::TrustedImm32(-params.proc().frameSize()),
8600 GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
8601
8602 callLinkInfo->setUpCall(
8603 isConstruct ? CallLinkInfo::DirectConstruct : CallLinkInfo::DirectCall,
8604 node->origin.semantic, InvalidGPRReg);
8605 callLinkInfo->setExecutableDuringCompilation(executable);
8606 if (numAllocatedArgs > numPassedArgs)
8607 callLinkInfo->setMaxArgumentCountIncludingThis(numAllocatedArgs);
8608
8609 params.addLatePath(
8610 [=] (CCallHelpers& jit) {
8611 AllowMacroScratchRegisterUsage allowScratch(jit);
8612
8613 CCallHelpers::Label slowPath = jit.label();
8614 if (isX86())
8615 jit.pop(CCallHelpers::selectScratchGPR(calleeGPR));
8616
8617 callOperation(
8618 *state, params.unavailableRegisters(), jit,
8619 node->origin.semantic, exceptions.get(), operationLinkDirectCall,
8620 InvalidGPRReg, CCallHelpers::TrustedImmPtr(callLinkInfo),
8621 calleeGPR).call();
8622 jit.jump().linkTo(mainPath, &jit);
8623
8624 jit.addLinkTask(
8625 [=] (LinkBuffer& linkBuffer) {
8626 CodeLocationNearCall<JSInternalPtrTag> callLocation = linkBuffer.locationOfNearCall<JSInternalPtrTag>(call);
8627 CodeLocationLabel<JSInternalPtrTag> slowPathLocation = linkBuffer.locationOf<JSInternalPtrTag>(slowPath);
8628
8629 linkBuffer.link(call, slowPathLocation);
8630
8631 callLinkInfo->setCallLocations(
8632 CodeLocationLabel<JSInternalPtrTag>(),
8633 slowPathLocation,
8634 callLocation);
8635 });
8636 });
8637 });
8638
8639 if (isTail)
8640 patchpoint->effects.terminal = true;
8641 else
8642 setJSValue(patchpoint);
8643 }
8644
8645 void compileTailCall()
8646 {
8647 Node* node = m_node;
8648 unsigned numArgs = node->numChildren() - 1;
8649
8650 // It seems counterintuitive that this is needed given that tail calls don't create a new frame
8651 // on the stack. However, the tail call slow path builds the frame at SP instead of FP before
8652 // calling into the slow path C code. This slow path may decide to throw an exception because
8653 // the callee we're trying to call is not callable. Throwing an exception will cause us to walk
8654 // the stack, which may read, for the sake of the correctness of this code, arbitrary slots on the
8655 // stack to recover state. This call arg area ensures the call frame shuffler does not overwrite
8656 // any of the slots the stack walking code requires when on the slow path.
8657 m_proc.requestCallArgAreaSizeInBytes(
8658 WTF::roundUpToMultipleOf(stackAlignmentBytes(), (CallFrame::headerSizeInRegisters + numArgs) * sizeof(EncodedJSValue)));
8659
8660 LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0));
8661
8662 // We want B3 to give us all of the arguments using whatever mechanism it thinks is
8663 // convenient. The generator then shuffles those arguments into our own call frame,
8664 // destroying our frame in the process.
8665
8666 // Note that we don't have to do anything special for exceptions. A tail call is only a
8667 // tail call if it is not inside a try block.
8668
8669 Vector<ConstrainedValue> arguments;
8670
8671 arguments.append(ConstrainedValue(jsCallee, ValueRep::reg(GPRInfo::regT0)));
8672
8673 for (unsigned i = 0; i < numArgs; ++i) {
8674 // Note: we could let the shuffler do boxing for us, but it's not super clear that this
8675 // would be better. Also, if we wanted to do that, then we'd have to teach the shuffler
8676 // that 32-bit values could land at 4-byte alignment but not 8-byte alignment.
8677
8678 ConstrainedValue constrainedValue(
8679 lowJSValue(m_graph.varArgChild(node, 1 + i)),
8680 ValueRep::WarmAny);
8681 arguments.append(constrainedValue);
8682 }
8683
8684 PatchpointValue* patchpoint = m_out.patchpoint(Void);
8685 patchpoint->appendVector(arguments);
8686
8687 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
8688 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
8689
8690 // Prevent any of the arguments from using the scratch register.
8691 patchpoint->clobberEarly(RegisterSet::macroScratchRegisters());
8692
8693 patchpoint->effects.terminal = true;
8694
8695 // We don't have to tell the patchpoint that we will clobber registers, since we won't return
8696 // anyway.
8697
8698 CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite();
8699 State* state = &m_ftlState;
8700 VM* vm = &this->vm();
8701 patchpoint->setGenerator(
8702 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
8703 AllowMacroScratchRegisterUsage allowScratch(jit);
8704 CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin);
8705
8706 // Yes, this is really necessary. You could throw an exception in a host call on the
8707 // slow path. That'll route us to operationLookupExceptionHandler(), which unwinds starting
8708 // with the call site index of our frame. Bad things happen if it's not set.
8709 jit.store32(
8710 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
8711 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
8712
8713 CallFrameShuffleData shuffleData;
8714 shuffleData.numLocals = state->jitCode->common.frameRegisterCount;
8715 shuffleData.callee = ValueRecovery::inGPR(GPRInfo::regT0, DataFormatJS);
8716
8717 for (unsigned i = 0; i < numArgs; ++i)
8718 shuffleData.args.append(params[1 + i].recoveryForJSValue());
8719
8720 shuffleData.numPassedArgs = numArgs;
8721
8722 shuffleData.setupCalleeSaveRegisters(jit.codeBlock());
8723
8724 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
8725
8726 CCallHelpers::DataLabelPtr targetToCheck;
8727 CCallHelpers::Jump slowPath = jit.branchPtrWithPatch(
8728 CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck,
8729 CCallHelpers::TrustedImmPtr(nullptr));
8730
8731 callLinkInfo->setFrameShuffleData(shuffleData);
8732 CallFrameShuffler(jit, shuffleData).prepareForTailCall();
8733
8734 CCallHelpers::Call fastCall = jit.nearTailCall();
8735
8736 slowPath.link(&jit);
8737
8738 CallFrameShuffler slowPathShuffler(jit, shuffleData);
8739 slowPathShuffler.setCalleeJSValueRegs(JSValueRegs(GPRInfo::regT0));
8740 slowPathShuffler.prepareForSlowPath();
8741
8742 jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2);
8743 jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()->globalObjectFor(node->origin.semantic)), GPRInfo::regT3);
8744 CCallHelpers::Call slowCall = jit.nearCall();
8745
8746 jit.abortWithReason(JITDidReturnFromTailCall);
8747
8748 callLinkInfo->setUpCall(CallLinkInfo::TailCall, codeOrigin, GPRInfo::regT0);
8749
8750 jit.addLinkTask(
8751 [=] (LinkBuffer& linkBuffer) {
8752 MacroAssemblerCodePtr<JITThunkPtrTag> linkCall = vm->getCTIStub(linkCallThunkGenerator).code();
8753 linkBuffer.link(slowCall, FunctionPtr<JITThunkPtrTag>(linkCall));
8754
8755 callLinkInfo->setCallLocations(
8756 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOfNearCall<JSInternalPtrTag>(slowCall)),
8757 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOf<JSInternalPtrTag>(targetToCheck)),
8758 linkBuffer.locationOfNearCall<JSInternalPtrTag>(fastCall));
8759 });
8760 });
8761 }
8762
8763 void compileCallOrConstructVarargsSpread()
8764 {
8765 Node* node = m_node;
8766 Node* arguments = node->child3().node();
8767
8768 LValue jsCallee = lowJSValue(m_node->child1());
8769 LValue thisArg = lowJSValue(m_node->child2());
8770
8771 RELEASE_ASSERT(arguments->op() == PhantomNewArrayWithSpread || arguments->op() == PhantomSpread || arguments->op() == PhantomNewArrayBuffer);
8772
8773 unsigned staticArgumentCount = 0;
8774 Vector<LValue, 2> spreadLengths;
8775 Vector<LValue, 8> patchpointArguments;
8776 HashMap<InlineCallFrame*, LValue, WTF::DefaultHash<InlineCallFrame*>::Hash, WTF::NullableHashTraits<InlineCallFrame*>> cachedSpreadLengths;
8777 auto pushAndCountArgumentsFromRightToLeft = recursableLambda([&](auto self, Node* target) -> void {
8778 if (target->op() == PhantomSpread) {
8779 self(target->child1().node());
8780 return;
8781 }
8782
8783 if (target->op() == PhantomNewArrayWithSpread) {
8784 BitVector* bitVector = target->bitVector();
8785 for (unsigned i = target->numChildren(); i--; ) {
8786 if (bitVector->get(i))
8787 self(m_graph.varArgChild(target, i).node());
8788 else {
8789 ++staticArgumentCount;
8790 LValue argument = this->lowJSValue(m_graph.varArgChild(target, i));
8791 patchpointArguments.append(argument);
8792 }
8793 }
8794 return;
8795 }
8796
8797 if (target->op() == PhantomNewArrayBuffer) {
8798 staticArgumentCount += target->castOperand<JSImmutableButterfly*>()->length();
8799 return;
8800 }
8801
8802 RELEASE_ASSERT(target->op() == PhantomCreateRest);
8803 InlineCallFrame* inlineCallFrame = target->origin.semantic.inlineCallFrame();
8804 unsigned numberOfArgumentsToSkip = target->numberOfArgumentsToSkip();
8805 LValue length = cachedSpreadLengths.ensure(inlineCallFrame, [&] () {
8806 return m_out.zeroExtPtr(this->getSpreadLengthFromInlineCallFrame(inlineCallFrame, numberOfArgumentsToSkip));
8807 }).iterator->value;
8808 patchpointArguments.append(length);
8809 spreadLengths.append(length);
8810 });
8811
8812 pushAndCountArgumentsFromRightToLeft(arguments);
8813 LValue argumentCountIncludingThis = m_out.constIntPtr(staticArgumentCount + 1);
8814 for (LValue length : spreadLengths)
8815 argumentCountIncludingThis = m_out.add(length, argumentCountIncludingThis);
8816
8817 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
8818
8819 patchpoint->append(jsCallee, ValueRep::reg(GPRInfo::regT0));
8820 patchpoint->append(thisArg, ValueRep::WarmAny);
8821 patchpoint->append(argumentCountIncludingThis, ValueRep::WarmAny);
8822 patchpoint->appendVectorWithRep(patchpointArguments, ValueRep::WarmAny);
8823 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
8824 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
8825
8826 RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
8827
8828 patchpoint->clobber(RegisterSet::macroScratchRegisters());
8829 patchpoint->clobber(RegisterSet::volatileRegistersForJSCall()); // No inputs will be in a volatile register.
8830 patchpoint->resultConstraints = { ValueRep::reg(GPRInfo::returnValueGPR) };
8831
8832 patchpoint->numGPScratchRegisters = 0;
8833
8834 // This is the minimum amount of call arg area stack space that all JS->JS calls always have.
8835 unsigned minimumJSCallAreaSize =
8836 sizeof(CallerFrameAndPC) +
8837 WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(EncodedJSValue));
8838
8839 m_proc.requestCallArgAreaSizeInBytes(minimumJSCallAreaSize);
8840
8841 CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite();
8842 State* state = &m_ftlState;
8843 VM* vm = &this->vm();
8844 patchpoint->setGenerator(
8845 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
8846 AllowMacroScratchRegisterUsage allowScratch(jit);
8847 CallSiteIndex callSiteIndex =
8848 state->jitCode->common.addUniqueCallSiteIndex(codeOrigin);
8849
8850 Box<CCallHelpers::JumpList> exceptions =
8851 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
8852
8853 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
8854
8855 jit.store32(
8856 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
8857 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
8858
8859 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
8860
8861 RegisterSet usedRegisters = RegisterSet::allRegisters();
8862 usedRegisters.exclude(RegisterSet::volatileRegistersForJSCall());
8863 GPRReg calleeGPR = params[1].gpr();
8864 usedRegisters.set(calleeGPR);
8865
8866 ScratchRegisterAllocator allocator(usedRegisters);
8867 GPRReg scratchGPR1 = allocator.allocateScratchGPR();
8868 GPRReg scratchGPR2 = allocator.allocateScratchGPR();
8869 GPRReg scratchGPR3 = allocator.allocateScratchGPR();
8870 GPRReg scratchGPR4 = allocator.allocateScratchGPR();
8871 RELEASE_ASSERT(!allocator.numberOfReusedRegisters());
8872
8873 auto getValueFromRep = [&] (B3::ValueRep rep, GPRReg result) {
8874 ASSERT(!usedRegisters.get(result));
8875
8876 if (rep.isConstant()) {
8877 jit.move(CCallHelpers::Imm64(rep.value()), result);
8878 return;
8879 }
8880
8881 // Note: in this function, we only request 64 bit values.
8882 if (rep.isStack()) {
8883 jit.load64(
8884 CCallHelpers::Address(GPRInfo::callFrameRegister, rep.offsetFromFP()),
8885 result);
8886 return;
8887 }
8888
8889 RELEASE_ASSERT(rep.isGPR());
8890 ASSERT(usedRegisters.get(rep.gpr()));
8891 jit.move(rep.gpr(), result);
8892 };
8893
8894 auto callWithExceptionCheck = [&] (void* callee) {
8895 jit.move(CCallHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(callee)), GPRInfo::nonPreservedNonArgumentGPR0);
8896 jit.call(GPRInfo::nonPreservedNonArgumentGPR0, OperationPtrTag);
8897 exceptions->append(jit.emitExceptionCheck(*vm, AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth));
8898 };
8899
8900 CCallHelpers::JumpList slowCase;
8901 unsigned originalStackHeight = params.proc().frameSize();
8902
8903 {
8904 unsigned numUsedSlots = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), originalStackHeight / sizeof(EncodedJSValue));
8905 B3::ValueRep argumentCountIncludingThisRep = params[3];
8906 getValueFromRep(argumentCountIncludingThisRep, scratchGPR2);
8907 slowCase.append(jit.branch32(CCallHelpers::Above, scratchGPR2, CCallHelpers::TrustedImm32(JSC::maxArguments + 1)));
8908
8909 jit.move(scratchGPR2, scratchGPR1);
8910 jit.addPtr(CCallHelpers::TrustedImmPtr(static_cast<size_t>(numUsedSlots + CallFrame::headerSizeInRegisters)), scratchGPR1);
8911 // scratchGPR1 now has the required frame size in Register units
8912 // Round scratchGPR1 to next multiple of stackAlignmentRegisters()
8913 jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), scratchGPR1);
8914 jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), scratchGPR1);
8915 jit.negPtr(scratchGPR1);
8916 jit.getEffectiveAddress(CCallHelpers::BaseIndex(GPRInfo::callFrameRegister, scratchGPR1, CCallHelpers::TimesEight), scratchGPR1);
8917
8918 // Before touching stack values, we should update the stack pointer to protect them from signal stack.
8919 jit.addPtr(CCallHelpers::TrustedImm32(sizeof(CallerFrameAndPC)), scratchGPR1, CCallHelpers::stackPointerRegister);
8920
8921 jit.store32(scratchGPR2, CCallHelpers::Address(scratchGPR1, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset));
8922
8923 int storeOffset = CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register));
8924
8925 unsigned paramsOffset = 4;
8926 unsigned index = 0;
8927 auto emitArgumentsFromRightToLeft = recursableLambda([&](auto self, Node* target) -> void {
8928 if (target->op() == PhantomSpread) {
8929 self(target->child1().node());
8930 return;
8931 }
8932
8933 if (target->op() == PhantomNewArrayWithSpread) {
8934 BitVector* bitVector = target->bitVector();
8935 for (unsigned i = target->numChildren(); i--; ) {
8936 if (bitVector->get(i))
8937 self(state->graph.varArgChild(target, i).node());
8938 else {
8939 jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast<size_t>(1)), scratchGPR2);
8940 getValueFromRep(params[paramsOffset + (index++)], scratchGPR3);
8941 jit.store64(scratchGPR3,
8942 CCallHelpers::BaseIndex(scratchGPR1, scratchGPR2, CCallHelpers::TimesEight, storeOffset));
8943 }
8944 }
8945 return;
8946 }
8947
8948 if (target->op() == PhantomNewArrayBuffer) {
8949 auto* array = target->castOperand<JSImmutableButterfly*>();
8950 Checked<int32_t> offsetCount { 1 };
8951 for (unsigned i = array->length(); i--; ++offsetCount) {
8952 // Because varargs values are drained as JSValue, we should not generate value
8953 // in Double form even if PhantomNewArrayBuffer's indexingType is ArrayWithDouble.
8954 int64_t value = JSValue::encode(array->get(i));
8955 jit.move(CCallHelpers::TrustedImm64(value), scratchGPR3);
8956 Checked<int32_t> currentStoreOffset { storeOffset };
8957 currentStoreOffset -= (offsetCount * static_cast<int32_t>(sizeof(Register)));
8958 jit.store64(scratchGPR3,
8959 CCallHelpers::BaseIndex(scratchGPR1, scratchGPR2, CCallHelpers::TimesEight, currentStoreOffset.unsafeGet()));
8960 }
8961 jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast<size_t>(array->length())), scratchGPR2);
8962 return;
8963 }
8964
8965 RELEASE_ASSERT(target->op() == PhantomCreateRest);
8966 InlineCallFrame* inlineCallFrame = target->origin.semantic.inlineCallFrame();
8967
8968 unsigned numberOfArgumentsToSkip = target->numberOfArgumentsToSkip();
8969
8970 B3::ValueRep numArgumentsToCopy = params[paramsOffset + (index++)];
8971 getValueFromRep(numArgumentsToCopy, scratchGPR3);
8972 int loadOffset = (AssemblyHelpers::argumentsStart(inlineCallFrame).offset() + numberOfArgumentsToSkip) * static_cast<int>(sizeof(Register));
8973
8974 auto done = jit.branchTestPtr(MacroAssembler::Zero, scratchGPR3);
8975 auto loopStart = jit.label();
8976 jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast<size_t>(1)), scratchGPR3);
8977 jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast<size_t>(1)), scratchGPR2);
8978 jit.load64(CCallHelpers::BaseIndex(GPRInfo::callFrameRegister, scratchGPR3, CCallHelpers::TimesEight, loadOffset), scratchGPR4);
8979 jit.store64(scratchGPR4,
8980 CCallHelpers::BaseIndex(scratchGPR1, scratchGPR2, CCallHelpers::TimesEight, storeOffset));
8981 jit.branchTestPtr(CCallHelpers::NonZero, scratchGPR3).linkTo(loopStart, &jit);
8982 done.link(&jit);
8983 });
8984 emitArgumentsFromRightToLeft(arguments);
8985 }
8986
8987 {
8988 CCallHelpers::Jump dontThrow = jit.jump();
8989 slowCase.link(&jit);
8990 jit.setupArguments<decltype(operationThrowStackOverflowForVarargs)>(jit.codeBlock()->globalObjectFor(node->origin.semantic));
8991 jit.prepareCallOperation(jit.vm());
8992 callWithExceptionCheck(bitwise_cast<void*>(operationThrowStackOverflowForVarargs));
8993 jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow);
8994
8995 dontThrow.link(&jit);
8996 }
8997
8998 ASSERT(calleeGPR == GPRInfo::regT0);
8999 jit.store64(calleeGPR, CCallHelpers::calleeFrameSlot(CallFrameSlot::callee));
9000 getValueFromRep(params[2], scratchGPR3);
9001 jit.store64(scratchGPR3, CCallHelpers::calleeArgumentSlot(0));
9002
9003 CallLinkInfo::CallType callType;
9004 if (node->op() == ConstructVarargs || node->op() == ConstructForwardVarargs)
9005 callType = CallLinkInfo::ConstructVarargs;
9006 else if (node->op() == TailCallVarargs || node->op() == TailCallForwardVarargs)
9007 callType = CallLinkInfo::TailCallVarargs;
9008 else
9009 callType = CallLinkInfo::CallVarargs;
9010
9011 bool isTailCall = CallLinkInfo::callModeFor(callType) == CallMode::Tail;
9012
9013 CCallHelpers::DataLabelPtr targetToCheck;
9014 CCallHelpers::Jump slowPath = jit.branchPtrWithPatch(
9015 CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck,
9016 CCallHelpers::TrustedImmPtr(nullptr));
9017
9018 CCallHelpers::Call fastCall;
9019 CCallHelpers::Jump done;
9020
9021 if (isTailCall) {
9022 jit.emitRestoreCalleeSaves();
9023 jit.prepareForTailCallSlow();
9024 fastCall = jit.nearTailCall();
9025 } else {
9026 fastCall = jit.nearCall();
9027 done = jit.jump();
9028 }
9029
9030 slowPath.link(&jit);
9031
9032 if (isTailCall)
9033 jit.emitRestoreCalleeSaves();
9034 ASSERT(!usedRegisters.get(GPRInfo::regT2));
9035 jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2);
9036 jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()->globalObjectFor(node->origin.semantic)), GPRInfo::regT3);
9037 CCallHelpers::Call slowCall = jit.nearCall();
9038
9039 if (isTailCall)
9040 jit.abortWithReason(JITDidReturnFromTailCall);
9041 else
9042 done.link(&jit);
9043
9044 callLinkInfo->setUpCall(callType, node->origin.semantic, GPRInfo::regT0);
9045
9046 jit.addPtr(
9047 CCallHelpers::TrustedImm32(-originalStackHeight),
9048 GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
9049
9050 jit.addLinkTask(
9051 [=] (LinkBuffer& linkBuffer) {
9052 MacroAssemblerCodePtr<JITThunkPtrTag> linkCall = vm->getCTIStub(linkCallThunkGenerator).code();
9053 linkBuffer.link(slowCall, FunctionPtr<JITThunkPtrTag>(linkCall));
9054
9055 callLinkInfo->setCallLocations(
9056 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOfNearCall<JSInternalPtrTag>(slowCall)),
9057 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOf<JSInternalPtrTag>(targetToCheck)),
9058 linkBuffer.locationOfNearCall<JSInternalPtrTag>(fastCall));
9059 });
9060 });
9061
9062 switch (node->op()) {
9063 case TailCallForwardVarargs:
9064 m_out.unreachable();
9065 break;
9066
9067 default:
9068 setJSValue(patchpoint);
9069 break;
9070 }
9071 }
9072
9073 void compileCallOrConstructVarargs()
9074 {
9075 Node* node = m_node;
9076 LValue jsCallee = lowJSValue(m_node->child1());
9077 LValue thisArg = lowJSValue(m_node->child2());
9078
9079 LValue jsArguments = nullptr;
9080 bool forwarding = false;
9081
9082 switch (node->op()) {
9083 case CallVarargs:
9084 case TailCallVarargs:
9085 case TailCallVarargsInlinedCaller:
9086 case ConstructVarargs:
9087 jsArguments = lowJSValue(node->child3());
9088 break;
9089 case CallForwardVarargs:
9090 case TailCallForwardVarargs:
9091 case TailCallForwardVarargsInlinedCaller:
9092 case ConstructForwardVarargs:
9093 forwarding = true;
9094 break;
9095 default:
9096 DFG_CRASH(m_graph, node, "bad node type");
9097 break;
9098 }
9099
9100 if (forwarding && m_node->child3()) {
9101 Node* arguments = m_node->child3().node();
9102 if (arguments->op() == PhantomNewArrayWithSpread || arguments->op() == PhantomNewArrayBuffer || arguments->op() == PhantomSpread) {
9103 compileCallOrConstructVarargsSpread();
9104 return;
9105 }
9106 }
9107
9108
9109 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
9110
9111 // Append the forms of the arguments that we will use before any clobbering happens.
9112 patchpoint->append(jsCallee, ValueRep::reg(GPRInfo::regT0));
9113 if (jsArguments)
9114 patchpoint->appendSomeRegister(jsArguments);
9115 patchpoint->appendSomeRegister(thisArg);
9116
9117 if (!forwarding) {
9118 // Now append them again for after clobbering. Note that the compiler may ask us to use a
9119 // different register for the late for the post-clobbering version of the value. This gives
9120 // the compiler a chance to spill these values without having to burn any callee-saves.
9121 patchpoint->append(jsCallee, ValueRep::LateColdAny);
9122 patchpoint->append(jsArguments, ValueRep::LateColdAny);
9123 patchpoint->append(thisArg, ValueRep::LateColdAny);
9124 }
9125
9126 RefPtr<PatchpointExceptionHandle> exceptionHandle =
9127 preparePatchpointForExceptions(patchpoint);
9128
9129 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
9130 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
9131
9132 patchpoint->clobber(RegisterSet::macroScratchRegisters());
9133 patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
9134 patchpoint->resultConstraints = { ValueRep::reg(GPRInfo::returnValueGPR) };
9135
9136 // This is the minimum amount of call arg area stack space that all JS->JS calls always have.
9137 unsigned minimumJSCallAreaSize =
9138 sizeof(CallerFrameAndPC) +
9139 WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(EncodedJSValue));
9140
9141 m_proc.requestCallArgAreaSizeInBytes(minimumJSCallAreaSize);
9142
9143 CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite();
9144 State* state = &m_ftlState;
9145 VM* vm = &this->vm();
9146 patchpoint->setGenerator(
9147 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
9148 AllowMacroScratchRegisterUsage allowScratch(jit);
9149 CallSiteIndex callSiteIndex =
9150 state->jitCode->common.addUniqueCallSiteIndex(codeOrigin);
9151
9152 Box<CCallHelpers::JumpList> exceptions =
9153 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
9154
9155 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
9156
9157 jit.store32(
9158 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
9159 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
9160
9161 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
9162 CallVarargsData* data = node->callVarargsData();
9163
9164 unsigned argIndex = 1;
9165 GPRReg calleeGPR = params[argIndex++].gpr();
9166 ASSERT(calleeGPR == GPRInfo::regT0);
9167 GPRReg argumentsGPR = jsArguments ? params[argIndex++].gpr() : InvalidGPRReg;
9168 GPRReg thisGPR = params[argIndex++].gpr();
9169
9170 B3::ValueRep calleeLateRep;
9171 B3::ValueRep argumentsLateRep;
9172 B3::ValueRep thisLateRep;
9173 if (!forwarding) {
9174 // If we're not forwarding then we'll need callee, arguments, and this after we
9175 // have potentially clobbered calleeGPR, argumentsGPR, and thisGPR. Our technique
9176 // for this is to supply all of those operands as late uses in addition to
9177 // specifying them as early uses. It's possible that the late use uses a spill
9178 // while the early use uses a register, and it's possible for the late and early
9179 // uses to use different registers. We do know that the late uses interfere with
9180 // all volatile registers and so won't use those, but the early uses may use
9181 // volatile registers and in the case of calleeGPR, it's pinned to regT0 so it
9182 // definitely will.
9183 //
9184 // Note that we have to be super careful with these. It's possible that these
9185 // use a shuffling of the registers used for calleeGPR, argumentsGPR, and
9186 // thisGPR. If that happens and we do for example:
9187 //
9188 // calleeLateRep.emitRestore(jit, calleeGPR);
9189 // argumentsLateRep.emitRestore(jit, calleeGPR);
9190 //
9191 // Then we might end up with garbage if calleeLateRep.gpr() == argumentsGPR and
9192 // argumentsLateRep.gpr() == calleeGPR.
9193 //
9194 // We do a variety of things to prevent this from happening. For example, we use
9195 // argumentsLateRep before needing the other two and after we've already stopped
9196 // using the *GPRs. Also, we pin calleeGPR to regT0, and rely on the fact that
9197 // the *LateReps cannot use volatile registers (so they cannot be regT0, so
9198 // calleeGPR != argumentsLateRep.gpr() and calleeGPR != thisLateRep.gpr()).
9199 //
9200 // An alternative would have been to just use early uses and early-clobber all
9201 // volatile registers. But that would force callee, arguments, and this into
9202 // callee-save registers even if we have to spill them. We don't want spilling to
9203 // use up three callee-saves.
9204 //
9205 // TL;DR: The way we use LateReps here is dangerous and barely works but achieves
9206 // some desirable performance properties, so don't mistake the cleverness for
9207 // elegance.
9208 calleeLateRep = params[argIndex++];
9209 argumentsLateRep = params[argIndex++];
9210 thisLateRep = params[argIndex++];
9211 }
9212
9213 // Get some scratch registers.
9214 RegisterSet usedRegisters;
9215 usedRegisters.merge(RegisterSet::stackRegisters());
9216 usedRegisters.merge(RegisterSet::reservedHardwareRegisters());
9217 usedRegisters.merge(RegisterSet::calleeSaveRegisters());
9218 usedRegisters.set(calleeGPR);
9219 if (argumentsGPR != InvalidGPRReg)
9220 usedRegisters.set(argumentsGPR);
9221 usedRegisters.set(thisGPR);
9222 if (calleeLateRep.isReg())
9223 usedRegisters.set(calleeLateRep.reg());
9224 if (argumentsLateRep.isReg())
9225 usedRegisters.set(argumentsLateRep.reg());
9226 if (thisLateRep.isReg())
9227 usedRegisters.set(thisLateRep.reg());
9228 ScratchRegisterAllocator allocator(usedRegisters);
9229 GPRReg scratchGPR1 = allocator.allocateScratchGPR();
9230 GPRReg scratchGPR2 = allocator.allocateScratchGPR();
9231 GPRReg scratchGPR3 = forwarding ? allocator.allocateScratchGPR() : InvalidGPRReg;
9232 RELEASE_ASSERT(!allocator.numberOfReusedRegisters());
9233
9234 auto callWithExceptionCheck = [&] (void* callee) {
9235 jit.move(CCallHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(callee)), GPRInfo::nonPreservedNonArgumentGPR0);
9236 jit.call(GPRInfo::nonPreservedNonArgumentGPR0, OperationPtrTag);
9237 exceptions->append(jit.emitExceptionCheck(*vm, AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth));
9238 };
9239
9240 unsigned originalStackHeight = params.proc().frameSize();
9241
9242 if (forwarding) {
9243 jit.move(CCallHelpers::TrustedImm32(originalStackHeight / sizeof(EncodedJSValue)), scratchGPR2);
9244
9245 CCallHelpers::JumpList slowCase;
9246 InlineCallFrame* inlineCallFrame;
9247 if (node->child3())
9248 inlineCallFrame = node->child3()->origin.semantic.inlineCallFrame();
9249 else
9250 inlineCallFrame = node->origin.semantic.inlineCallFrame();
9251
9252 // emitSetupVarargsFrameFastCase modifies the stack pointer if it succeeds.
9253 emitSetupVarargsFrameFastCase(*vm, jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, inlineCallFrame, data->firstVarArgOffset, slowCase);
9254
9255 CCallHelpers::Jump done = jit.jump();
9256 slowCase.link(&jit);
9257 jit.setupArguments<decltype(operationThrowStackOverflowForVarargs)>(jit.codeBlock()->globalObjectFor(node->origin.semantic));
9258 jit.prepareCallOperation(jit.vm());
9259 callWithExceptionCheck(bitwise_cast<void*>(operationThrowStackOverflowForVarargs));
9260 jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow);
9261
9262 done.link(&jit);
9263 } else {
9264 jit.move(CCallHelpers::TrustedImm32(originalStackHeight / sizeof(EncodedJSValue)), scratchGPR1);
9265 jit.setupArguments<decltype(operationSizeFrameForVarargs)>(jit.codeBlock()->globalObjectFor(node->origin.semantic), argumentsGPR, scratchGPR1, CCallHelpers::TrustedImm32(data->firstVarArgOffset));
9266 jit.prepareCallOperation(jit.vm());
9267 callWithExceptionCheck(bitwise_cast<void*>(operationSizeFrameForVarargs));
9268
9269 jit.move(GPRInfo::returnValueGPR, scratchGPR1);
9270 jit.move(CCallHelpers::TrustedImm32(originalStackHeight / sizeof(EncodedJSValue)), scratchGPR2);
9271 argumentsLateRep.emitRestore(jit, argumentsGPR);
9272 emitSetVarargsFrame(jit, scratchGPR1, false, scratchGPR2, scratchGPR2);
9273 jit.addPtr(CCallHelpers::TrustedImm32(-minimumJSCallAreaSize), scratchGPR2, CCallHelpers::stackPointerRegister);
9274 jit.setupArguments<decltype(operationSetupVarargsFrame)>(jit.codeBlock()->globalObjectFor(node->origin.semantic), scratchGPR2, argumentsGPR, CCallHelpers::TrustedImm32(data->firstVarArgOffset), scratchGPR1);
9275 jit.prepareCallOperation(jit.vm());
9276 callWithExceptionCheck(bitwise_cast<void*>(operationSetupVarargsFrame));
9277
9278 jit.addPtr(CCallHelpers::TrustedImm32(sizeof(CallerFrameAndPC)), GPRInfo::returnValueGPR, CCallHelpers::stackPointerRegister);
9279
9280 calleeLateRep.emitRestore(jit, GPRInfo::regT0);
9281
9282 // This may not emit code if thisGPR got a callee-save. Also, we're guaranteed
9283 // that thisGPR != GPRInfo::regT0 because regT0 interferes with it.
9284 thisLateRep.emitRestore(jit, thisGPR);
9285 }
9286
9287 jit.store64(GPRInfo::regT0, CCallHelpers::calleeFrameSlot(CallFrameSlot::callee));
9288 jit.store64(thisGPR, CCallHelpers::calleeArgumentSlot(0));
9289
9290 CallLinkInfo::CallType callType;
9291 if (node->op() == ConstructVarargs || node->op() == ConstructForwardVarargs)
9292 callType = CallLinkInfo::ConstructVarargs;
9293 else if (node->op() == TailCallVarargs || node->op() == TailCallForwardVarargs)
9294 callType = CallLinkInfo::TailCallVarargs;
9295 else
9296 callType = CallLinkInfo::CallVarargs;
9297
9298 bool isTailCall = CallLinkInfo::callModeFor(callType) == CallMode::Tail;
9299
9300 CCallHelpers::DataLabelPtr targetToCheck;
9301 CCallHelpers::Jump slowPath = jit.branchPtrWithPatch(
9302 CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck,
9303 CCallHelpers::TrustedImmPtr(nullptr));
9304
9305 CCallHelpers::Call fastCall;
9306 CCallHelpers::Jump done;
9307
9308 if (isTailCall) {
9309 jit.emitRestoreCalleeSaves();
9310 jit.prepareForTailCallSlow();
9311 fastCall = jit.nearTailCall();
9312 } else {
9313 fastCall = jit.nearCall();
9314 done = jit.jump();
9315 }
9316
9317 slowPath.link(&jit);
9318
9319 if (isTailCall)
9320 jit.emitRestoreCalleeSaves();
9321 jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2);
9322 jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()->globalObjectFor(node->origin.semantic)), GPRInfo::regT3);
9323 CCallHelpers::Call slowCall = jit.nearCall();
9324
9325 if (isTailCall)
9326 jit.abortWithReason(JITDidReturnFromTailCall);
9327 else
9328 done.link(&jit);
9329
9330 callLinkInfo->setUpCall(callType, node->origin.semantic, GPRInfo::regT0);
9331
9332 jit.addPtr(
9333 CCallHelpers::TrustedImm32(-originalStackHeight),
9334 GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
9335
9336 jit.addLinkTask(
9337 [=] (LinkBuffer& linkBuffer) {
9338 MacroAssemblerCodePtr<JITThunkPtrTag> linkCall = vm->getCTIStub(linkCallThunkGenerator).code();
9339 linkBuffer.link(slowCall, FunctionPtr<JITThunkPtrTag>(linkCall));
9340
9341 callLinkInfo->setCallLocations(
9342 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOfNearCall<JSInternalPtrTag>(slowCall)),
9343 CodeLocationLabel<JSInternalPtrTag>(linkBuffer.locationOf<JSInternalPtrTag>(targetToCheck)),
9344 linkBuffer.locationOfNearCall<JSInternalPtrTag>(fastCall));
9345 });
9346 });
9347
9348 switch (node->op()) {
9349 case TailCallVarargs:
9350 case TailCallForwardVarargs:
9351 m_out.unreachable();
9352 break;
9353
9354 default:
9355 setJSValue(patchpoint);
9356 break;
9357 }
9358 }
9359
9360 void compileCallEval()
9361 {
9362 Node* node = m_node;
9363 unsigned numArgs = node->numChildren() - 1;
9364
9365 LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0));
9366
9367 unsigned frameSize = (CallFrame::headerSizeInRegisters + numArgs) * sizeof(EncodedJSValue);
9368 unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSize);
9369
9370 m_proc.requestCallArgAreaSizeInBytes(alignedFrameSize);
9371
9372 Vector<ConstrainedValue> arguments;
9373 arguments.append(ConstrainedValue(jsCallee, ValueRep::reg(GPRInfo::regT0)));
9374
9375 auto addArgument = [&] (LValue value, VirtualRegister reg, int offset) {
9376 intptr_t offsetFromSP =
9377 (reg.offset() - CallerFrameAndPC::sizeInRegisters) * sizeof(EncodedJSValue) + offset;
9378 arguments.append(ConstrainedValue(value, ValueRep::stackArgument(offsetFromSP)));
9379 };
9380
9381 addArgument(jsCallee, VirtualRegister(CallFrameSlot::callee), 0);
9382 addArgument(m_out.constInt32(numArgs), VirtualRegister(CallFrameSlot::argumentCount), PayloadOffset);
9383 for (unsigned i = 0; i < numArgs; ++i)
9384 addArgument(lowJSValue(m_graph.varArgChild(node, 1 + i)), virtualRegisterForArgument(i), 0);
9385
9386 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
9387 patchpoint->appendVector(arguments);
9388
9389 RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
9390
9391 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
9392 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
9393 patchpoint->clobber(RegisterSet::macroScratchRegisters());
9394 patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall());
9395 patchpoint->resultConstraints = { ValueRep::reg(GPRInfo::returnValueGPR) };
9396
9397 CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite();
9398 State* state = &m_ftlState;
9399 VM& vm = this->vm();
9400 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
9401 patchpoint->setGenerator(
9402 [=, &vm] (CCallHelpers& jit, const StackmapGenerationParams& params) {
9403 AllowMacroScratchRegisterUsage allowScratch(jit);
9404 CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin);
9405
9406 Box<CCallHelpers::JumpList> exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit);
9407
9408 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
9409
9410 jit.store32(
9411 CCallHelpers::TrustedImm32(callSiteIndex.bits()),
9412 CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount)));
9413
9414 CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo();
9415 callLinkInfo->setUpCall(CallLinkInfo::Call, node->origin.semantic, GPRInfo::regT0);
9416
9417 jit.addPtr(CCallHelpers::TrustedImm32(-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC))), CCallHelpers::stackPointerRegister, GPRInfo::regT1);
9418 jit.storePtr(GPRInfo::callFrameRegister, CCallHelpers::Address(GPRInfo::regT1, CallFrame::callerFrameOffset()));
9419
9420 // Now we need to make room for:
9421 // - The caller frame and PC for a call to operationCallEval.
9422 // - Potentially two arguments on the stack.
9423 unsigned requiredBytes = sizeof(CallerFrameAndPC) + sizeof(CallFrame*) * 2;
9424 requiredBytes = WTF::roundUpToMultipleOf(stackAlignmentBytes(), requiredBytes);
9425 jit.subPtr(CCallHelpers::TrustedImm32(requiredBytes), CCallHelpers::stackPointerRegister);
9426 jit.setupArguments<decltype(operationCallEval)>(globalObject, GPRInfo::regT1);
9427 jit.prepareCallOperation(vm);
9428 jit.move(CCallHelpers::TrustedImmPtr(tagCFunctionPtr<OperationPtrTag>(operationCallEval)), GPRInfo::nonPreservedNonArgumentGPR0);
9429 jit.call(GPRInfo::nonPreservedNonArgumentGPR0, OperationPtrTag);
9430 exceptions->append(jit.emitExceptionCheck(state->vm(), AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth));
9431
9432 CCallHelpers::Jump done = jit.branchTest64(CCallHelpers::NonZero, GPRInfo::returnValueGPR);
9433
9434 jit.addPtr(CCallHelpers::TrustedImm32(requiredBytes), CCallHelpers::stackPointerRegister);
9435 jit.load64(CCallHelpers::calleeFrameSlot(CallFrameSlot::callee), GPRInfo::regT0);
9436 jit.emitDumbVirtualCall(vm, globalObject, callLinkInfo);
9437
9438 done.link(&jit);
9439 jit.addPtr(
9440 CCallHelpers::TrustedImm32(-params.proc().frameSize()),
9441 GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
9442 });
9443
9444 setJSValue(patchpoint);
9445 }
9446
9447 void compileLoadVarargs()
9448 {
9449 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
9450 LoadVarargsData* data = m_node->loadVarargsData();
9451 LValue jsArguments = lowJSValue(m_node->child1());
9452
9453 LValue length = vmCall(
9454 Int32, operationSizeOfVarargs, weakPointer(globalObject), jsArguments,
9455 m_out.constInt32(data->offset));
9456
9457 // FIXME: There is a chance that we will call an effectful length property twice. This is safe
9458 // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance
9459 // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right
9460 // past the sizing.
9461 // https://bugs.webkit.org/show_bug.cgi?id=141448
9462
9463 LValue lengthIncludingThis = m_out.add(length, m_out.int32One);
9464
9465 speculate(
9466 VarargsOverflow, noValue(), nullptr,
9467 m_out.above(length, lengthIncludingThis));
9468
9469 speculate(
9470 VarargsOverflow, noValue(), nullptr,
9471 m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
9472
9473 m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
9474
9475 // FIXME: This computation is rather silly. If operationLaodVarargs just took a pointer instead
9476 // of a VirtualRegister, we wouldn't have to do this.
9477 // https://bugs.webkit.org/show_bug.cgi?id=141660
9478 LValue machineStart = m_out.lShr(
9479 m_out.sub(addressFor(data->machineStart.offset()).value(), m_callFrame),
9480 m_out.constIntPtr(3));
9481
9482 vmCall(
9483 Void, operationLoadVarargs, weakPointer(globalObject),
9484 m_out.castToInt32(machineStart), jsArguments, m_out.constInt32(data->offset),
9485 length, m_out.constInt32(data->mandatoryMinimum));
9486 }
9487
9488 void compileForwardVarargs()
9489 {
9490 if (m_node->child1()) {
9491 Node* arguments = m_node->child1().node();
9492 if (arguments->op() == PhantomNewArrayWithSpread || arguments->op() == PhantomNewArrayBuffer || arguments->op() == PhantomSpread) {
9493 compileForwardVarargsWithSpread();
9494 return;
9495 }
9496 }
9497
9498 LoadVarargsData* data = m_node->loadVarargsData();
9499 InlineCallFrame* inlineCallFrame;
9500 if (m_node->child1())
9501 inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame();
9502 else
9503 inlineCallFrame = m_node->origin.semantic.inlineCallFrame();
9504
9505 LValue length = nullptr;
9506 LValue lengthIncludingThis = nullptr;
9507 ArgumentsLength argumentsLength = getArgumentsLength(inlineCallFrame);
9508 if (argumentsLength.isKnown) {
9509 unsigned knownLength = argumentsLength.known;
9510 if (knownLength >= data->offset)
9511 knownLength = knownLength - data->offset;
9512 else
9513 knownLength = 0;
9514 length = m_out.constInt32(knownLength);
9515 lengthIncludingThis = m_out.constInt32(knownLength + 1);
9516 } else {
9517 // We need to perform the same logical operation as the code above, but through dynamic operations.
9518 if (!data->offset)
9519 length = argumentsLength.value;
9520 else {
9521 LBasicBlock isLarger = m_out.newBlock();
9522 LBasicBlock continuation = m_out.newBlock();
9523
9524 ValueFromBlock smallerOrEqualLengthResult = m_out.anchor(m_out.constInt32(0));
9525 m_out.branch(
9526 m_out.above(argumentsLength.value, m_out.constInt32(data->offset)), unsure(isLarger), unsure(continuation));
9527 LBasicBlock lastNext = m_out.appendTo(isLarger, continuation);
9528 ValueFromBlock largerLengthResult = m_out.anchor(m_out.sub(argumentsLength.value, m_out.constInt32(data->offset)));
9529 m_out.jump(continuation);
9530
9531 m_out.appendTo(continuation, lastNext);
9532 length = m_out.phi(Int32, smallerOrEqualLengthResult, largerLengthResult);
9533 }
9534 lengthIncludingThis = m_out.add(length, m_out.constInt32(1));
9535 }
9536
9537 speculate(
9538 VarargsOverflow, noValue(), nullptr,
9539 m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
9540
9541 m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
9542
9543 unsigned numberOfArgumentsToSkip = data->offset;
9544 LValue sourceStart = getArgumentsStart(inlineCallFrame, numberOfArgumentsToSkip);
9545 LValue targetStart = addressFor(data->machineStart).value();
9546
9547 LBasicBlock undefinedLoop = m_out.newBlock();
9548 LBasicBlock mainLoopEntry = m_out.newBlock();
9549 LBasicBlock mainLoop = m_out.newBlock();
9550 LBasicBlock continuation = m_out.newBlock();
9551
9552 LValue lengthAsPtr = m_out.zeroExtPtr(length);
9553 LValue loopBoundValue = m_out.constIntPtr(data->mandatoryMinimum);
9554 ValueFromBlock loopBound = m_out.anchor(loopBoundValue);
9555 m_out.branch(
9556 m_out.above(loopBoundValue, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry));
9557
9558 LBasicBlock lastNext = m_out.appendTo(undefinedLoop, mainLoopEntry);
9559 LValue previousIndex = m_out.phi(pointerType(), loopBound);
9560 LValue currentIndex = m_out.sub(previousIndex, m_out.intPtrOne);
9561 m_out.store64(
9562 m_out.constInt64(JSValue::encode(jsUndefined())),
9563 m_out.baseIndex(m_heaps.variables, targetStart, currentIndex));
9564 ValueFromBlock nextIndex = m_out.anchor(currentIndex);
9565 m_out.addIncomingToPhi(previousIndex, nextIndex);
9566 m_out.branch(
9567 m_out.above(currentIndex, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry));
9568
9569 m_out.appendTo(mainLoopEntry, mainLoop);
9570 loopBound = m_out.anchor(lengthAsPtr);
9571 m_out.branch(m_out.notNull(lengthAsPtr), unsure(mainLoop), unsure(continuation));
9572
9573 m_out.appendTo(mainLoop, continuation);
9574 previousIndex = m_out.phi(pointerType(), loopBound);
9575 currentIndex = m_out.sub(previousIndex, m_out.intPtrOne);
9576 LValue value = m_out.load64(
9577 m_out.baseIndex(m_heaps.variables, sourceStart, currentIndex));
9578 m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, currentIndex));
9579 nextIndex = m_out.anchor(currentIndex);
9580 m_out.addIncomingToPhi(previousIndex, nextIndex);
9581 m_out.branch(m_out.isNull(currentIndex), unsure(continuation), unsure(mainLoop));
9582
9583 m_out.appendTo(continuation, lastNext);
9584 }
9585
9586 LValue getSpreadLengthFromInlineCallFrame(InlineCallFrame* inlineCallFrame, unsigned numberOfArgumentsToSkip)
9587 {
9588 ArgumentsLength argumentsLength = getArgumentsLength(inlineCallFrame);
9589 if (argumentsLength.isKnown) {
9590 unsigned knownLength = argumentsLength.known;
9591 if (knownLength >= numberOfArgumentsToSkip)
9592 knownLength = knownLength - numberOfArgumentsToSkip;
9593 else
9594 knownLength = 0;
9595 return m_out.constInt32(knownLength);
9596 }
9597
9598
9599 // We need to perform the same logical operation as the code above, but through dynamic operations.
9600 if (!numberOfArgumentsToSkip)
9601 return argumentsLength.value;
9602
9603 LBasicBlock isLarger = m_out.newBlock();
9604 LBasicBlock continuation = m_out.newBlock();
9605
9606 ValueFromBlock smallerOrEqualLengthResult = m_out.anchor(m_out.constInt32(0));
9607 m_out.branch(
9608 m_out.above(argumentsLength.value, m_out.constInt32(numberOfArgumentsToSkip)), unsure(isLarger), unsure(continuation));
9609 LBasicBlock lastNext = m_out.appendTo(isLarger, continuation);
9610 ValueFromBlock largerLengthResult = m_out.anchor(m_out.sub(argumentsLength.value, m_out.constInt32(numberOfArgumentsToSkip)));
9611 m_out.jump(continuation);
9612
9613 m_out.appendTo(continuation, lastNext);
9614 return m_out.phi(Int32, smallerOrEqualLengthResult, largerLengthResult);
9615 }
9616
9617 void compileForwardVarargsWithSpread()
9618 {
9619 HashMap<InlineCallFrame*, LValue, WTF::DefaultHash<InlineCallFrame*>::Hash, WTF::NullableHashTraits<InlineCallFrame*>> cachedSpreadLengths;
9620
9621 Node* arguments = m_node->child1().node();
9622 RELEASE_ASSERT(arguments->op() == PhantomNewArrayWithSpread || arguments->op() == PhantomNewArrayBuffer || arguments->op() == PhantomSpread);
9623
9624 unsigned numberOfStaticArguments = 0;
9625 Vector<LValue, 2> spreadLengths;
9626
9627 auto collectArgumentCount = recursableLambda([&](auto self, Node* target) -> void {
9628 if (target->op() == PhantomSpread) {
9629 self(target->child1().node());
9630 return;
9631 }
9632
9633 if (target->op() == PhantomNewArrayWithSpread) {
9634 BitVector* bitVector = target->bitVector();
9635 for (unsigned i = 0; i < target->numChildren(); i++) {
9636 if (bitVector->get(i))
9637 self(m_graph.varArgChild(target, i).node());
9638 else
9639 ++numberOfStaticArguments;
9640 }
9641 return;
9642 }
9643
9644 if (target->op() == PhantomNewArrayBuffer) {
9645 numberOfStaticArguments += target->castOperand<JSImmutableButterfly*>()->length();
9646 return;
9647 }
9648
9649 ASSERT(target->op() == PhantomCreateRest);
9650 InlineCallFrame* inlineCallFrame = target->origin.semantic.inlineCallFrame();
9651 unsigned numberOfArgumentsToSkip = target->numberOfArgumentsToSkip();
9652 spreadLengths.append(cachedSpreadLengths.ensure(inlineCallFrame, [&] () {
9653 return this->getSpreadLengthFromInlineCallFrame(inlineCallFrame, numberOfArgumentsToSkip);
9654 }).iterator->value);
9655 });
9656
9657 collectArgumentCount(arguments);
9658 LValue lengthIncludingThis = m_out.constInt32(1 + numberOfStaticArguments);
9659 for (LValue length : spreadLengths)
9660 lengthIncludingThis = m_out.add(lengthIncludingThis, length);
9661
9662 LoadVarargsData* data = m_node->loadVarargsData();
9663 speculate(
9664 VarargsOverflow, noValue(), nullptr,
9665 m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
9666
9667 m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
9668
9669 LValue targetStart = addressFor(data->machineStart).value();
9670
9671 auto forwardSpread = recursableLambda([this, &cachedSpreadLengths, &targetStart](auto self, Node* target, LValue storeIndex) -> LValue {
9672 if (target->op() == PhantomSpread)
9673 return self(target->child1().node(), storeIndex);
9674
9675 if (target->op() == PhantomNewArrayWithSpread) {
9676 BitVector* bitVector = target->bitVector();
9677 for (unsigned i = 0; i < target->numChildren(); i++) {
9678 if (bitVector->get(i))
9679 storeIndex = self(m_graph.varArgChild(target, i).node(), storeIndex);
9680 else {
9681 LValue value = this->lowJSValue(m_graph.varArgChild(target, i));
9682 m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, storeIndex));
9683 storeIndex = m_out.add(m_out.constIntPtr(1), storeIndex);
9684 }
9685 }
9686 return storeIndex;
9687 }
9688
9689 if (target->op() == PhantomNewArrayBuffer) {
9690 auto* array = target->castOperand<JSImmutableButterfly*>();
9691 for (unsigned i = 0; i < array->length(); i++) {
9692 // Because forwarded values are drained as JSValue, we should not generate value
9693 // in Double form even if PhantomNewArrayBuffer's indexingType is ArrayWithDouble.
9694 int64_t value = JSValue::encode(array->get(i));
9695 m_out.store64(m_out.constInt64(value), m_out.baseIndex(m_heaps.variables, targetStart, storeIndex, JSValue(), (Checked<int32_t>(sizeof(Register)) * i).unsafeGet()));
9696 }
9697 return m_out.add(m_out.constIntPtr(array->length()), storeIndex);
9698 }
9699
9700 RELEASE_ASSERT(target->op() == PhantomCreateRest);
9701 InlineCallFrame* inlineCallFrame = target->origin.semantic.inlineCallFrame();
9702
9703 LValue sourceStart = this->getArgumentsStart(inlineCallFrame, target->numberOfArgumentsToSkip());
9704 LValue spreadLength = m_out.zeroExtPtr(cachedSpreadLengths.get(inlineCallFrame));
9705
9706 LBasicBlock loop = m_out.newBlock();
9707 LBasicBlock continuation = m_out.newBlock();
9708 ValueFromBlock startLoadIndex = m_out.anchor(m_out.constIntPtr(0));
9709 ValueFromBlock startStoreIndex = m_out.anchor(storeIndex);
9710 ValueFromBlock startStoreIndexForEnd = m_out.anchor(storeIndex);
9711
9712 m_out.branch(m_out.isZero64(spreadLength), unsure(continuation), unsure(loop));
9713
9714 LBasicBlock lastNext = m_out.appendTo(loop, continuation);
9715 LValue loopStoreIndex = m_out.phi(Int64, startStoreIndex);
9716 LValue loadIndex = m_out.phi(Int64, startLoadIndex);
9717 LValue value = m_out.load64(
9718 m_out.baseIndex(m_heaps.variables, sourceStart, loadIndex));
9719 m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, loopStoreIndex));
9720 LValue nextLoadIndex = m_out.add(m_out.constIntPtr(1), loadIndex);
9721 m_out.addIncomingToPhi(loadIndex, m_out.anchor(nextLoadIndex));
9722 LValue nextStoreIndex = m_out.add(m_out.constIntPtr(1), loopStoreIndex);
9723 m_out.addIncomingToPhi(loopStoreIndex, m_out.anchor(nextStoreIndex));
9724 ValueFromBlock loopStoreIndexForEnd = m_out.anchor(nextStoreIndex);
9725 m_out.branch(m_out.below(nextLoadIndex, spreadLength), unsure(loop), unsure(continuation));
9726
9727 m_out.appendTo(continuation, lastNext);
9728 return m_out.phi(Int64, startStoreIndexForEnd, loopStoreIndexForEnd);
9729 });
9730
9731 LValue storeIndex = forwardSpread(arguments, m_out.constIntPtr(0));
9732
9733 LBasicBlock undefinedLoop = m_out.newBlock();
9734 LBasicBlock continuation = m_out.newBlock();
9735
9736 ValueFromBlock startStoreIndex = m_out.anchor(storeIndex);
9737 LValue loopBoundValue = m_out.constIntPtr(data->mandatoryMinimum);
9738 m_out.branch(m_out.below(storeIndex, loopBoundValue),
9739 unsure(undefinedLoop), unsure(continuation));
9740
9741 LBasicBlock lastNext = m_out.appendTo(undefinedLoop, continuation);
9742 LValue loopStoreIndex = m_out.phi(Int64, startStoreIndex);
9743 m_out.store64(
9744 m_out.constInt64(JSValue::encode(jsUndefined())),
9745 m_out.baseIndex(m_heaps.variables, targetStart, loopStoreIndex));
9746 LValue nextIndex = m_out.add(loopStoreIndex, m_out.constIntPtr(1));
9747 m_out.addIncomingToPhi(loopStoreIndex, m_out.anchor(nextIndex));
9748 m_out.branch(
9749 m_out.below(nextIndex, loopBoundValue), unsure(undefinedLoop), unsure(continuation));
9750
9751 m_out.appendTo(continuation, lastNext);
9752 }
9753
9754 void compileJump()
9755 {
9756 m_out.jump(lowBlock(m_node->targetBlock()));
9757 }
9758
9759 void compileBranch()
9760 {
9761 m_out.branch(
9762 boolify(m_node->child1()),
9763 WeightedTarget(
9764 lowBlock(m_node->branchData()->taken.block),
9765 m_node->branchData()->taken.count),
9766 WeightedTarget(
9767 lowBlock(m_node->branchData()->notTaken.block),
9768 m_node->branchData()->notTaken.count));
9769 }
9770
9771 void compileSwitch()
9772 {
9773 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
9774 SwitchData* data = m_node->switchData();
9775 switch (data->kind) {
9776 case SwitchImm: {
9777 Vector<ValueFromBlock, 2> intValues;
9778 LBasicBlock switchOnInts = m_out.newBlock();
9779
9780 LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
9781
9782 switch (m_node->child1().useKind()) {
9783 case Int32Use: {
9784 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
9785 m_out.jump(switchOnInts);
9786 break;
9787 }
9788
9789 case UntypedUse: {
9790 LBasicBlock isInt = m_out.newBlock();
9791 LBasicBlock isNotInt = m_out.newBlock();
9792 LBasicBlock isDouble = m_out.newBlock();
9793
9794 LValue boxedValue = lowJSValue(m_node->child1());
9795 m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt));
9796
9797 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
9798
9799 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
9800 m_out.jump(switchOnInts);
9801
9802 m_out.appendTo(isNotInt, isDouble);
9803 m_out.branch(
9804 isCellOrMisc(boxedValue, provenType(m_node->child1())),
9805 usually(lowBlock(data->fallThrough.block)), rarely(isDouble));
9806
9807 m_out.appendTo(isDouble, innerLastNext);
9808 LValue doubleValue = unboxDouble(boxedValue);
9809 LValue intInDouble = m_out.doubleToInt(doubleValue);
9810 intValues.append(m_out.anchor(intInDouble));
9811 m_out.branch(
9812 m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
9813 unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block)));
9814 break;
9815 }
9816
9817 default:
9818 DFG_CRASH(m_graph, m_node, "Bad use kind");
9819 break;
9820 }
9821
9822 m_out.appendTo(switchOnInts, lastNext);
9823 buildSwitch(data, Int32, m_out.phi(Int32, intValues));
9824 return;
9825 }
9826
9827 case SwitchChar: {
9828 LValue stringValue;
9829
9830 // FIXME: We should use something other than unsure() for the branch weight
9831 // of the fallThrough block. The main challenge is just that we have multiple
9832 // branches to fallThrough but a single count, so we would need to divvy it up
9833 // among the different lowered branches.
9834 // https://bugs.webkit.org/show_bug.cgi?id=129082
9835
9836 switch (m_node->child1().useKind()) {
9837 case StringUse: {
9838 stringValue = lowString(m_node->child1());
9839 break;
9840 }
9841
9842 case UntypedUse: {
9843 LValue unboxedValue = lowJSValue(m_node->child1());
9844
9845 LBasicBlock isCellCase = m_out.newBlock();
9846 LBasicBlock isStringCase = m_out.newBlock();
9847
9848 m_out.branch(
9849 isNotCell(unboxedValue, provenType(m_node->child1())),
9850 unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase));
9851
9852 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
9853 LValue cellValue = unboxedValue;
9854 m_out.branch(
9855 isNotString(cellValue, provenType(m_node->child1())),
9856 unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase));
9857
9858 m_out.appendTo(isStringCase, lastNext);
9859 stringValue = cellValue;
9860 break;
9861 }
9862
9863 default:
9864 DFG_CRASH(m_graph, m_node, "Bad use kind");
9865 break;
9866 }
9867
9868 LBasicBlock lengthIs1 = m_out.newBlock();
9869 LBasicBlock needResolution = m_out.newBlock();
9870 LBasicBlock resolved = m_out.newBlock();
9871 LBasicBlock is8Bit = m_out.newBlock();
9872 LBasicBlock is16Bit = m_out.newBlock();
9873 LBasicBlock continuation = m_out.newBlock();
9874
9875 ValueFromBlock fastValue = m_out.anchor(m_out.loadPtr(stringValue, m_heaps.JSString_value));
9876 m_out.branch(
9877 isRopeString(stringValue, m_node->child1()),
9878 rarely(needResolution), usually(resolved));
9879
9880 LBasicBlock lastNext = m_out.appendTo(needResolution, resolved);
9881 ValueFromBlock slowValue = m_out.anchor(
9882 vmCall(pointerType(), operationResolveRope, weakPointer(globalObject), stringValue));
9883 m_out.jump(resolved);
9884
9885 m_out.appendTo(resolved, lengthIs1);
9886 LValue value = m_out.phi(pointerType(), fastValue, slowValue);
9887 m_out.branch(
9888 m_out.notEqual(
9889 m_out.load32NonNegative(value, m_heaps.StringImpl_length),
9890 m_out.int32One),
9891 unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1));
9892
9893 m_out.appendTo(lengthIs1, is8Bit);
9894 LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
9895 m_out.branch(
9896 m_out.testNonZero32(
9897 m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
9898 m_out.constInt32(StringImpl::flagIs8Bit())),
9899 unsure(is8Bit), unsure(is16Bit));
9900
9901 Vector<ValueFromBlock, 2> characters;
9902 m_out.appendTo(is8Bit, is16Bit);
9903 characters.append(m_out.anchor(m_out.load8ZeroExt32(characterData, m_heaps.characters8[0])));
9904 m_out.jump(continuation);
9905
9906 m_out.appendTo(is16Bit, continuation);
9907 characters.append(m_out.anchor(m_out.load16ZeroExt32(characterData, m_heaps.characters16[0])));
9908 m_out.jump(continuation);
9909
9910 m_out.appendTo(continuation, lastNext);
9911 buildSwitch(data, Int32, m_out.phi(Int32, characters));
9912 return;
9913 }
9914
9915 case SwitchString: {
9916 switch (m_node->child1().useKind()) {
9917 case StringIdentUse: {
9918 LValue stringImpl = lowStringIdent(m_node->child1());
9919
9920 Vector<SwitchCase> cases;
9921 for (unsigned i = 0; i < data->cases.size(); ++i) {
9922 LValue value = m_out.constIntPtr(data->cases[i].value.stringImpl());
9923 LBasicBlock block = lowBlock(data->cases[i].target.block);
9924 Weight weight = Weight(data->cases[i].target.count);
9925 cases.append(SwitchCase(value, block, weight));
9926 }
9927
9928 m_out.switchInstruction(
9929 stringImpl, cases, lowBlock(data->fallThrough.block),
9930 Weight(data->fallThrough.count));
9931 return;
9932 }
9933
9934 case StringUse: {
9935 switchString(data, lowString(m_node->child1()), m_node->child1());
9936 return;
9937 }
9938
9939 case UntypedUse: {
9940 LValue value = lowJSValue(m_node->child1());
9941
9942 LBasicBlock isCellBlock = m_out.newBlock();
9943 LBasicBlock isStringBlock = m_out.newBlock();
9944
9945 m_out.branch(
9946 isCell(value, provenType(m_node->child1())),
9947 unsure(isCellBlock), unsure(lowBlock(data->fallThrough.block)));
9948
9949 LBasicBlock lastNext = m_out.appendTo(isCellBlock, isStringBlock);
9950
9951 m_out.branch(
9952 isString(value, provenType(m_node->child1())),
9953 unsure(isStringBlock), unsure(lowBlock(data->fallThrough.block)));
9954
9955 m_out.appendTo(isStringBlock, lastNext);
9956
9957 switchString(data, value, m_node->child1());
9958 return;
9959 }
9960
9961 default:
9962 DFG_CRASH(m_graph, m_node, "Bad use kind");
9963 return;
9964 }
9965 return;
9966 }
9967
9968 case SwitchCell: {
9969 LValue cell;
9970 switch (m_node->child1().useKind()) {
9971 case CellUse: {
9972 cell = lowCell(m_node->child1());
9973 break;
9974 }
9975
9976 case UntypedUse: {
9977 LValue value = lowJSValue(m_node->child1());
9978 LBasicBlock cellCase = m_out.newBlock();
9979 m_out.branch(
9980 isCell(value, provenType(m_node->child1())),
9981 unsure(cellCase), unsure(lowBlock(data->fallThrough.block)));
9982 m_out.appendTo(cellCase);
9983 cell = value;
9984 break;
9985 }
9986
9987 default:
9988 DFG_CRASH(m_graph, m_node, "Bad use kind");
9989 return;
9990 }
9991
9992 buildSwitch(m_node->switchData(), pointerType(), cell);
9993 return;
9994 } }
9995
9996 DFG_CRASH(m_graph, m_node, "Bad switch kind");
9997 }
9998
9999 void compileEntrySwitch()
10000 {
10001 Vector<LBasicBlock> successors;
10002 for (DFG::BasicBlock* successor : m_node->entrySwitchData()->cases)
10003 successors.append(lowBlock(successor));
10004 m_out.entrySwitch(successors);
10005 }
10006
10007 void compileReturn()
10008 {
10009 m_out.ret(lowJSValue(m_node->child1()));
10010 }
10011
10012 void compileForceOSRExit()
10013 {
10014 terminate(InadequateCoverage);
10015 }
10016
10017 void compileCPUIntrinsic()
10018 {
10019#if CPU(X86_64)
10020 Intrinsic intrinsic = m_node->intrinsic();
10021 switch (intrinsic) {
10022 case CPUMfenceIntrinsic:
10023 case CPUCpuidIntrinsic:
10024 case CPUPauseIntrinsic: {
10025 PatchpointValue* patchpoint = m_out.patchpoint(Void);
10026 patchpoint->effects = Effects::forCall();
10027 if (intrinsic == CPUCpuidIntrinsic)
10028 patchpoint->clobber(RegisterSet { X86Registers::eax, X86Registers::ebx, X86Registers::ecx, X86Registers::edx });
10029
10030 patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
10031 switch (intrinsic) {
10032 case CPUMfenceIntrinsic:
10033 jit.mfence();
10034 break;
10035 case CPUCpuidIntrinsic:
10036 jit.cpuid();
10037 break;
10038 case CPUPauseIntrinsic:
10039 jit.pause();
10040 break;
10041 default:
10042 RELEASE_ASSERT_NOT_REACHED();
10043 }
10044 });
10045 setJSValue(m_out.constInt64(JSValue::encode(jsUndefined())));
10046 break;
10047 }
10048 case CPURdtscIntrinsic: {
10049 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
10050 patchpoint->effects = Effects::forCall();
10051 patchpoint->clobber(RegisterSet { X86Registers::eax, X86Registers::edx });
10052 // The low 32-bits of rdtsc go into rax.
10053 patchpoint->resultConstraints = { ValueRep::reg(X86Registers::eax) };
10054 patchpoint->setGenerator( [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
10055 jit.rdtsc();
10056 });
10057 setJSValue(boxInt32(patchpoint));
10058 break;
10059 }
10060 default:
10061 RELEASE_ASSERT_NOT_REACHED();
10062
10063 }
10064#endif
10065 }
10066
10067 void compileThrow()
10068 {
10069 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10070 LValue error = lowJSValue(m_node->child1());
10071 vmCall(Void, operationThrowDFG, weakPointer(globalObject), error);
10072 // vmCall() does an exception check so we should never reach this.
10073 m_out.unreachable();
10074 }
10075
10076 void compileThrowStaticError()
10077 {
10078 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10079 LValue errorMessage = lowString(m_node->child1());
10080 LValue errorType = m_out.constInt32(m_node->errorType());
10081 vmCall(Void, operationThrowStaticError, weakPointer(globalObject), errorMessage, errorType);
10082 // vmCall() does an exception check so we should never reach this.
10083 m_out.unreachable();
10084 }
10085
10086 void compileInvalidationPoint()
10087 {
10088 if (verboseCompilationEnabled())
10089 dataLog(" Invalidation point with availability: ", availabilityMap(), "\n");
10090
10091 DFG_ASSERT(m_graph, m_node, m_origin.exitOK);
10092
10093 PatchpointValue* patchpoint = m_out.patchpoint(Void);
10094 OSRExitDescriptor* descriptor = appendOSRExitDescriptor(noValue(), nullptr);
10095 NodeOrigin origin = m_origin;
10096 patchpoint->appendColdAnys(buildExitArguments(descriptor, origin.forExit, noValue()));
10097
10098 State* state = &m_ftlState;
10099
10100 patchpoint->setGenerator(
10101 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
10102 // The MacroAssembler knows more about this than B3 does. The watchpointLabel() method
10103 // will ensure that this is followed by a nop shadow but only when this is actually
10104 // necessary.
10105 CCallHelpers::Label label = jit.watchpointLabel();
10106
10107 RefPtr<OSRExitHandle> handle = descriptor->emitOSRExitLater(
10108 *state, UncountableInvalidation, origin, params);
10109
10110 RefPtr<JITCode> jitCode = state->jitCode.get();
10111
10112 jit.addLinkTask(
10113 [=] (LinkBuffer& linkBuffer) {
10114 JumpReplacement jumpReplacement(
10115 linkBuffer.locationOf<JSInternalPtrTag>(label),
10116 linkBuffer.locationOf<OSRExitPtrTag>(handle->label));
10117 jitCode->common.jumpReplacements.append(jumpReplacement);
10118 });
10119 });
10120
10121 // Set some obvious things.
10122 patchpoint->effects.terminal = false;
10123 patchpoint->effects.writesLocalState = false;
10124 patchpoint->effects.readsLocalState = false;
10125
10126 // This is how we tell B3 about the possibility of jump replacement.
10127 patchpoint->effects.exitsSideways = true;
10128
10129 // It's not possible for some prior branch to determine the safety of this operation. It's always
10130 // fine to execute this on some path that wouldn't have originally executed it before
10131 // optimization.
10132 patchpoint->effects.controlDependent = false;
10133
10134 // If this falls through then it won't write anything.
10135 patchpoint->effects.writes = HeapRange();
10136
10137 // When this abruptly terminates, it could read any heap location.
10138 patchpoint->effects.reads = HeapRange::top();
10139 }
10140
10141 void compileIsEmpty()
10142 {
10143 setBoolean(m_out.isZero64(lowJSValue(m_node->child1())));
10144 }
10145
10146 void compileIsUndefined()
10147 {
10148 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
10149 }
10150
10151 void compileIsUndefinedOrNull()
10152 {
10153 setBoolean(isOther(lowJSValue(m_node->child1()), provenType(m_node->child1())));
10154 }
10155
10156 void compileIsBoolean()
10157 {
10158 setBoolean(isBoolean(lowJSValue(m_node->child1()), provenType(m_node->child1())));
10159 }
10160
10161 void compileIsNumber()
10162 {
10163 setBoolean(isNumber(lowJSValue(m_node->child1()), provenType(m_node->child1())));
10164 }
10165
10166 void compileNumberIsInteger()
10167 {
10168 LBasicBlock notInt32 = m_out.newBlock();
10169 LBasicBlock doubleCase = m_out.newBlock();
10170 LBasicBlock doubleNotNanOrInf = m_out.newBlock();
10171 LBasicBlock continuation = m_out.newBlock();
10172
10173 LValue input = lowJSValue(m_node->child1());
10174
10175 ValueFromBlock trueResult = m_out.anchor(m_out.booleanTrue);
10176 m_out.branch(
10177 isInt32(input, provenType(m_node->child1())), unsure(continuation), unsure(notInt32));
10178
10179 LBasicBlock lastNext = m_out.appendTo(notInt32, doubleCase);
10180 ValueFromBlock falseResult = m_out.anchor(m_out.booleanFalse);
10181 m_out.branch(
10182 isNotNumber(input, provenType(m_node->child1())), unsure(continuation), unsure(doubleCase));
10183
10184 m_out.appendTo(doubleCase, doubleNotNanOrInf);
10185 LValue doubleAsInt;
10186 LValue asDouble = unboxDouble(input, &doubleAsInt);
10187 LValue expBits = m_out.bitAnd(m_out.lShr(doubleAsInt, m_out.constInt32(52)), m_out.constInt64(0x7ff));
10188 m_out.branch(
10189 m_out.equal(expBits, m_out.constInt64(0x7ff)),
10190 unsure(continuation), unsure(doubleNotNanOrInf));
10191
10192 m_out.appendTo(doubleNotNanOrInf, continuation);
10193 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
10194 patchpoint->appendSomeRegister(asDouble);
10195 patchpoint->numFPScratchRegisters = 1;
10196 patchpoint->effects = Effects::none();
10197 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
10198 GPRReg result = params[0].gpr();
10199 FPRReg input = params[1].fpr();
10200 FPRReg temp = params.fpScratch(0);
10201 jit.roundTowardZeroDouble(input, temp);
10202 jit.compareDouble(MacroAssembler::DoubleEqual, input, temp, result);
10203 });
10204 ValueFromBlock patchpointResult = m_out.anchor(patchpoint);
10205 m_out.jump(continuation);
10206
10207 m_out.appendTo(continuation, lastNext);
10208 setBoolean(m_out.phi(Int32, trueResult, falseResult, patchpointResult));
10209 }
10210
10211 void compileIsCellWithType()
10212 {
10213 if (m_node->child1().useKind() == UntypedUse) {
10214 LValue value = lowJSValue(m_node->child1());
10215
10216 LBasicBlock isCellCase = m_out.newBlock();
10217 LBasicBlock continuation = m_out.newBlock();
10218
10219 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
10220 m_out.branch(
10221 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
10222
10223 LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
10224 ValueFromBlock cellResult = m_out.anchor(isCellWithType(value, m_node->queriedType(), m_node->speculatedTypeForQuery(), provenType(m_node->child1())));
10225 m_out.jump(continuation);
10226
10227 m_out.appendTo(continuation, lastNext);
10228 setBoolean(m_out.phi(Int32, notCellResult, cellResult));
10229 } else {
10230 ASSERT(m_node->child1().useKind() == CellUse);
10231 setBoolean(isCellWithType(lowCell(m_node->child1()), m_node->queriedType(), m_node->speculatedTypeForQuery(), provenType(m_node->child1())));
10232 }
10233 }
10234
10235 void compileIsObject()
10236 {
10237 LValue value = lowJSValue(m_node->child1());
10238
10239 LBasicBlock isCellCase = m_out.newBlock();
10240 LBasicBlock continuation = m_out.newBlock();
10241
10242 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
10243 m_out.branch(
10244 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
10245
10246 LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
10247 ValueFromBlock cellResult = m_out.anchor(isObject(value, provenType(m_node->child1())));
10248 m_out.jump(continuation);
10249
10250 m_out.appendTo(continuation, lastNext);
10251 setBoolean(m_out.phi(Int32, notCellResult, cellResult));
10252 }
10253
10254 LValue wangsInt64Hash(LValue input)
10255 {
10256 // key += ~(key << 32);
10257 LValue key = input;
10258 LValue temp = key;
10259 temp = m_out.shl(temp, m_out.constInt32(32));
10260 temp = m_out.bitNot(temp);
10261 key = m_out.add(key, temp);
10262 // key ^= (key >> 22);
10263 temp = key;
10264 temp = m_out.lShr(temp, m_out.constInt32(22));
10265 key = m_out.bitXor(key, temp);
10266 // key += ~(key << 13);
10267 temp = key;
10268 temp = m_out.shl(temp, m_out.constInt32(13));
10269 temp = m_out.bitNot(temp);
10270 key = m_out.add(key, temp);
10271 // key ^= (key >> 8);
10272 temp = key;
10273 temp = m_out.lShr(temp, m_out.constInt32(8));
10274 key = m_out.bitXor(key, temp);
10275 // key += (key << 3);
10276 temp = key;
10277 temp = m_out.shl(temp, m_out.constInt32(3));
10278 key = m_out.add(key, temp);
10279 // key ^= (key >> 15);
10280 temp = key;
10281 temp = m_out.lShr(temp, m_out.constInt32(15));
10282 key = m_out.bitXor(key, temp);
10283 // key += ~(key << 27);
10284 temp = key;
10285 temp = m_out.shl(temp, m_out.constInt32(27));
10286 temp = m_out.bitNot(temp);
10287 key = m_out.add(key, temp);
10288 // key ^= (key >> 31);
10289 temp = key;
10290 temp = m_out.lShr(temp, m_out.constInt32(31));
10291 key = m_out.bitXor(key, temp);
10292 key = m_out.castToInt32(key);
10293
10294 return key;
10295 }
10296
10297 LValue mapHashString(LValue string, Edge& edge)
10298 {
10299 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10300 LBasicBlock nonEmptyStringCase = m_out.newBlock();
10301 LBasicBlock slowCase = m_out.newBlock();
10302 LBasicBlock continuation = m_out.newBlock();
10303
10304 m_out.branch(isRopeString(string, edge), rarely(slowCase), usually(nonEmptyStringCase));
10305
10306 LBasicBlock lastNext = m_out.appendTo(nonEmptyStringCase, slowCase);
10307 LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
10308 LValue hash = m_out.lShr(m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount));
10309 ValueFromBlock nonEmptyStringHashResult = m_out.anchor(hash);
10310 m_out.branch(m_out.equal(hash, m_out.constInt32(0)),
10311 unsure(slowCase), unsure(continuation));
10312
10313 m_out.appendTo(slowCase, continuation);
10314 ValueFromBlock slowResult = m_out.anchor(
10315 vmCall(Int32, operationMapHash, weakPointer(globalObject), string));
10316 m_out.jump(continuation);
10317
10318 m_out.appendTo(continuation, lastNext);
10319 return m_out.phi(Int32, slowResult, nonEmptyStringHashResult);
10320 }
10321
10322 void compileMapHash()
10323 {
10324 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10325 switch (m_node->child1().useKind()) {
10326 case BooleanUse:
10327 case Int32Use:
10328 case SymbolUse:
10329 case ObjectUse: {
10330 LValue key = lowJSValue(m_node->child1(), ManualOperandSpeculation);
10331 speculate(m_node->child1());
10332 setInt32(wangsInt64Hash(key));
10333 return;
10334 }
10335
10336 case CellUse: {
10337 LBasicBlock isString = m_out.newBlock();
10338 LBasicBlock notString = m_out.newBlock();
10339 LBasicBlock continuation = m_out.newBlock();
10340
10341 LValue value = lowCell(m_node->child1());
10342 LValue isStringValue = m_out.equal(m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoType), m_out.constInt32(StringType));
10343 m_out.branch(
10344 isStringValue, unsure(isString), unsure(notString));
10345
10346 LBasicBlock lastNext = m_out.appendTo(isString, notString);
10347 ValueFromBlock stringResult = m_out.anchor(mapHashString(value, m_node->child1()));
10348 m_out.jump(continuation);
10349
10350 m_out.appendTo(notString, continuation);
10351 ValueFromBlock notStringResult = m_out.anchor(wangsInt64Hash(value));
10352 m_out.jump(continuation);
10353
10354 m_out.appendTo(continuation, lastNext);
10355 setInt32(m_out.phi(Int32, stringResult, notStringResult));
10356 return;
10357 }
10358
10359 case StringUse: {
10360 LValue string = lowString(m_node->child1());
10361 setInt32(mapHashString(string, m_node->child1()));
10362 return;
10363 }
10364
10365 default:
10366 RELEASE_ASSERT(m_node->child1().useKind() == UntypedUse);
10367 break;
10368 }
10369
10370 LValue value = lowJSValue(m_node->child1());
10371
10372 LBasicBlock isCellCase = m_out.newBlock();
10373 LBasicBlock slowCase = m_out.newBlock();
10374 LBasicBlock straightHash = m_out.newBlock();
10375 LBasicBlock isStringCase = m_out.newBlock();
10376 LBasicBlock nonEmptyStringCase = m_out.newBlock();
10377 LBasicBlock continuation = m_out.newBlock();
10378
10379 m_out.branch(
10380 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(straightHash));
10381
10382 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
10383 LValue isString = m_out.equal(m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoType), m_out.constInt32(StringType));
10384 m_out.branch(
10385 isString, unsure(isStringCase), unsure(straightHash));
10386
10387 m_out.appendTo(isStringCase, nonEmptyStringCase);
10388 m_out.branch(isRopeString(value, m_node->child1()), rarely(slowCase), usually(nonEmptyStringCase));
10389
10390 m_out.appendTo(nonEmptyStringCase, straightHash);
10391 LValue stringImpl = m_out.loadPtr(value, m_heaps.JSString_value);
10392 LValue hash = m_out.lShr(m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount));
10393 ValueFromBlock nonEmptyStringHashResult = m_out.anchor(hash);
10394 m_out.branch(m_out.equal(hash, m_out.constInt32(0)),
10395 unsure(slowCase), unsure(continuation));
10396
10397 m_out.appendTo(straightHash, slowCase);
10398 ValueFromBlock fastResult = m_out.anchor(wangsInt64Hash(value));
10399 m_out.jump(continuation);
10400
10401 m_out.appendTo(slowCase, continuation);
10402 ValueFromBlock slowResult = m_out.anchor(
10403 vmCall(Int32, operationMapHash, weakPointer(globalObject), value));
10404 m_out.jump(continuation);
10405
10406 m_out.appendTo(continuation, lastNext);
10407 setInt32(m_out.phi(Int32, fastResult, slowResult, nonEmptyStringHashResult));
10408 }
10409
10410 void compileNormalizeMapKey()
10411 {
10412 ASSERT(m_node->child1().useKind() == UntypedUse);
10413
10414 LBasicBlock isNumberCase = m_out.newBlock();
10415 LBasicBlock notInt32NumberCase = m_out.newBlock();
10416 LBasicBlock notNaNCase = m_out.newBlock();
10417 LBasicBlock convertibleCase = m_out.newBlock();
10418 LBasicBlock continuation = m_out.newBlock();
10419
10420 LBasicBlock lastNext = m_out.insertNewBlocksBefore(isNumberCase);
10421
10422 LValue key = lowJSValue(m_node->child1());
10423 ValueFromBlock fastResult = m_out.anchor(key);
10424 m_out.branch(isNotNumber(key), unsure(continuation), unsure(isNumberCase));
10425
10426 m_out.appendTo(isNumberCase, notInt32NumberCase);
10427 m_out.branch(isInt32(key), unsure(continuation), unsure(notInt32NumberCase));
10428
10429 m_out.appendTo(notInt32NumberCase, notNaNCase);
10430 LValue doubleValue = unboxDouble(key);
10431 ValueFromBlock normalizedNaNResult = m_out.anchor(m_out.constInt64(JSValue::encode(jsNaN())));
10432 m_out.branch(m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue), unsure(continuation), unsure(notNaNCase));
10433
10434 m_out.appendTo(notNaNCase, convertibleCase);
10435 LValue integerValue = m_out.doubleToInt(doubleValue);
10436 LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue);
10437 ValueFromBlock doubleResult = m_out.anchor(key);
10438 m_out.branch(m_out.doubleNotEqualOrUnordered(doubleValue, integerValueConvertedToDouble), unsure(continuation), unsure(convertibleCase));
10439
10440 m_out.appendTo(convertibleCase, continuation);
10441 ValueFromBlock boxedIntResult = m_out.anchor(boxInt32(integerValue));
10442 m_out.jump(continuation);
10443
10444 m_out.appendTo(continuation, lastNext);
10445 setJSValue(m_out.phi(Int64, fastResult, normalizedNaNResult, doubleResult, boxedIntResult));
10446 }
10447
10448 void compileGetMapBucket()
10449 {
10450 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10451 LBasicBlock loopStart = m_out.newBlock();
10452 LBasicBlock loopAround = m_out.newBlock();
10453 LBasicBlock slowPath = m_out.newBlock();
10454 LBasicBlock notPresentInTable = m_out.newBlock();
10455 LBasicBlock notEmptyValue = m_out.newBlock();
10456 LBasicBlock notDeletedValue = m_out.newBlock();
10457 LBasicBlock continuation = m_out.newBlock();
10458
10459 LBasicBlock lastNext = m_out.insertNewBlocksBefore(loopStart);
10460
10461 LValue map;
10462 if (m_node->child1().useKind() == MapObjectUse)
10463 map = lowMapObject(m_node->child1());
10464 else if (m_node->child1().useKind() == SetObjectUse)
10465 map = lowSetObject(m_node->child1());
10466 else
10467 RELEASE_ASSERT_NOT_REACHED();
10468
10469 LValue key = lowJSValue(m_node->child2(), ManualOperandSpeculation);
10470 if (m_node->child2().useKind() != UntypedUse)
10471 speculate(m_node->child2());
10472
10473 LValue hash = lowInt32(m_node->child3());
10474
10475 LValue buffer = m_out.loadPtr(map, m_heaps.HashMapImpl_buffer);
10476 LValue mask = m_out.sub(m_out.load32(map, m_heaps.HashMapImpl_capacity), m_out.int32One);
10477
10478 ValueFromBlock indexStart = m_out.anchor(hash);
10479 m_out.jump(loopStart);
10480
10481 m_out.appendTo(loopStart, notEmptyValue);
10482 LValue unmaskedIndex = m_out.phi(Int32, indexStart);
10483 LValue index = m_out.bitAnd(mask, unmaskedIndex);
10484 // FIXME: I think these buffers are caged?
10485 // https://bugs.webkit.org/show_bug.cgi?id=174925
10486 LValue hashMapBucket = m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), buffer, m_out.zeroExt(index, Int64), ScaleEight));
10487 ValueFromBlock bucketResult = m_out.anchor(hashMapBucket);
10488 m_out.branch(m_out.equal(hashMapBucket, m_out.constIntPtr(bitwise_cast<intptr_t>(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::emptyValue()))),
10489 unsure(notPresentInTable), unsure(notEmptyValue));
10490
10491 m_out.appendTo(notEmptyValue, notDeletedValue);
10492 m_out.branch(m_out.equal(hashMapBucket, m_out.constIntPtr(bitwise_cast<intptr_t>(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::deletedValue()))),
10493 unsure(loopAround), unsure(notDeletedValue));
10494
10495 m_out.appendTo(notDeletedValue, loopAround);
10496 LValue bucketKey = m_out.load64(hashMapBucket, m_heaps.HashMapBucket_key);
10497
10498 // Perform Object.is()
10499 switch (m_node->child2().useKind()) {
10500 case BooleanUse:
10501 case Int32Use:
10502 case SymbolUse:
10503 case ObjectUse: {
10504 m_out.branch(m_out.equal(key, bucketKey),
10505 unsure(continuation), unsure(loopAround));
10506 break;
10507 }
10508 case StringUse: {
10509 LBasicBlock notBitEqual = m_out.newBlock();
10510 LBasicBlock bucketKeyIsCell = m_out.newBlock();
10511
10512 m_out.branch(m_out.equal(key, bucketKey),
10513 unsure(continuation), unsure(notBitEqual));
10514
10515 m_out.appendTo(notBitEqual, bucketKeyIsCell);
10516 m_out.branch(isCell(bucketKey),
10517 unsure(bucketKeyIsCell), unsure(loopAround));
10518
10519 m_out.appendTo(bucketKeyIsCell, loopAround);
10520 m_out.branch(isString(bucketKey),
10521 unsure(slowPath), unsure(loopAround));
10522 break;
10523 }
10524 case CellUse: {
10525 LBasicBlock notBitEqual = m_out.newBlock();
10526 LBasicBlock bucketKeyIsCell = m_out.newBlock();
10527 LBasicBlock bucketKeyIsString = m_out.newBlock();
10528
10529 m_out.branch(m_out.equal(key, bucketKey),
10530 unsure(continuation), unsure(notBitEqual));
10531
10532 m_out.appendTo(notBitEqual, bucketKeyIsCell);
10533 m_out.branch(isCell(bucketKey),
10534 unsure(bucketKeyIsCell), unsure(loopAround));
10535
10536 m_out.appendTo(bucketKeyIsCell, bucketKeyIsString);
10537 m_out.branch(isString(bucketKey),
10538 unsure(bucketKeyIsString), unsure(loopAround));
10539
10540 m_out.appendTo(bucketKeyIsString, loopAround);
10541 m_out.branch(isString(key),
10542 unsure(slowPath), unsure(loopAround));
10543 break;
10544 }
10545 case UntypedUse: {
10546 LBasicBlock notBitEqual = m_out.newBlock();
10547 LBasicBlock bucketKeyIsCell = m_out.newBlock();
10548 LBasicBlock bothAreCells = m_out.newBlock();
10549 LBasicBlock bucketKeyIsString = m_out.newBlock();
10550
10551 m_out.branch(m_out.equal(key, bucketKey),
10552 unsure(continuation), unsure(notBitEqual));
10553
10554 m_out.appendTo(notBitEqual, bucketKeyIsCell);
10555 m_out.branch(isCell(bucketKey),
10556 unsure(bucketKeyIsCell), unsure(loopAround));
10557
10558 m_out.appendTo(bucketKeyIsCell, bothAreCells);
10559 m_out.branch(isCell(key),
10560 unsure(bothAreCells), unsure(loopAround));
10561
10562 m_out.appendTo(bothAreCells, bucketKeyIsString);
10563 m_out.branch(isString(bucketKey),
10564 unsure(bucketKeyIsString), unsure(loopAround));
10565
10566 m_out.appendTo(bucketKeyIsString, loopAround);
10567 m_out.branch(isString(key),
10568 unsure(slowPath), unsure(loopAround));
10569 break;
10570 }
10571 default:
10572 RELEASE_ASSERT_NOT_REACHED();
10573 }
10574
10575 m_out.appendTo(loopAround, slowPath);
10576 m_out.addIncomingToPhi(unmaskedIndex, m_out.anchor(m_out.add(index, m_out.int32One)));
10577 m_out.jump(loopStart);
10578
10579 m_out.appendTo(slowPath, notPresentInTable);
10580 ValueFromBlock slowPathResult = m_out.anchor(vmCall(pointerType(),
10581 m_node->child1().useKind() == MapObjectUse ? operationJSMapFindBucket : operationJSSetFindBucket, weakPointer(globalObject), map, key, hash));
10582 m_out.jump(continuation);
10583
10584 m_out.appendTo(notPresentInTable, continuation);
10585 ValueFromBlock notPresentResult;
10586 if (m_node->child1().useKind() == MapObjectUse)
10587 notPresentResult = m_out.anchor(weakPointer(vm().sentinelMapBucket()));
10588 else if (m_node->child1().useKind() == SetObjectUse)
10589 notPresentResult = m_out.anchor(weakPointer(vm().sentinelSetBucket()));
10590 else
10591 RELEASE_ASSERT_NOT_REACHED();
10592 m_out.jump(continuation);
10593
10594 m_out.appendTo(continuation, lastNext);
10595 setJSValue(m_out.phi(pointerType(), bucketResult, slowPathResult, notPresentResult));
10596 }
10597
10598 void compileGetMapBucketHead()
10599 {
10600 LValue map;
10601 if (m_node->child1().useKind() == MapObjectUse)
10602 map = lowMapObject(m_node->child1());
10603 else if (m_node->child1().useKind() == SetObjectUse)
10604 map = lowSetObject(m_node->child1());
10605 else
10606 RELEASE_ASSERT_NOT_REACHED();
10607
10608 ASSERT(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfHead() == HashMapImpl<HashMapBucket<HashMapBucketDataKeyValue>>::offsetOfHead());
10609 setJSValue(m_out.loadPtr(map, m_heaps.HashMapImpl_head));
10610 }
10611
10612 void compileGetMapBucketNext()
10613 {
10614 LBasicBlock loopStart = m_out.newBlock();
10615 LBasicBlock continuation = m_out.newBlock();
10616 LBasicBlock noBucket = m_out.newBlock();
10617 LBasicBlock hasBucket = m_out.newBlock();
10618 LBasicBlock nextBucket = m_out.newBlock();
10619
10620 LBasicBlock lastNext = m_out.insertNewBlocksBefore(loopStart);
10621
10622 ASSERT(HashMapBucket<HashMapBucketDataKey>::offsetOfNext() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext());
10623 ASSERT(HashMapBucket<HashMapBucketDataKey>::offsetOfKey() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey());
10624 LValue mapBucketPrev = lowCell(m_node->child1());
10625 ValueFromBlock mapBucketStart = m_out.anchor(m_out.loadPtr(mapBucketPrev, m_heaps.HashMapBucket_next));
10626 m_out.jump(loopStart);
10627
10628 m_out.appendTo(loopStart, noBucket);
10629 LValue mapBucket = m_out.phi(pointerType(), mapBucketStart);
10630 m_out.branch(m_out.isNull(mapBucket), unsure(noBucket), unsure(hasBucket));
10631
10632 m_out.appendTo(noBucket, hasBucket);
10633 ValueFromBlock noBucketResult;
10634 if (m_node->bucketOwnerType() == BucketOwnerType::Map)
10635 noBucketResult = m_out.anchor(weakPointer(vm().sentinelMapBucket()));
10636 else {
10637 ASSERT(m_node->bucketOwnerType() == BucketOwnerType::Set);
10638 noBucketResult = m_out.anchor(weakPointer(vm().sentinelSetBucket()));
10639 }
10640 m_out.jump(continuation);
10641
10642 m_out.appendTo(hasBucket, nextBucket);
10643 ValueFromBlock bucketResult = m_out.anchor(mapBucket);
10644 m_out.branch(m_out.isZero64(m_out.load64(mapBucket, m_heaps.HashMapBucket_key)), unsure(nextBucket), unsure(continuation));
10645
10646 m_out.appendTo(nextBucket, continuation);
10647 m_out.addIncomingToPhi(mapBucket, m_out.anchor(m_out.loadPtr(mapBucket, m_heaps.HashMapBucket_next)));
10648 m_out.jump(loopStart);
10649
10650 m_out.appendTo(continuation, lastNext);
10651 setJSValue(m_out.phi(pointerType(), noBucketResult, bucketResult));
10652 }
10653
10654 void compileLoadValueFromMapBucket()
10655 {
10656 LValue mapBucket = lowCell(m_node->child1());
10657 setJSValue(m_out.load64(mapBucket, m_heaps.HashMapBucket_value));
10658 }
10659
10660 void compileExtractValueFromWeakMapGet()
10661 {
10662 LValue value = lowJSValue(m_node->child1());
10663 setJSValue(m_out.select(m_out.isZero64(value),
10664 m_out.constInt64(JSValue::encode(jsUndefined())),
10665 value));
10666 }
10667
10668 void compileLoadKeyFromMapBucket()
10669 {
10670 LValue mapBucket = lowCell(m_node->child1());
10671 setJSValue(m_out.load64(mapBucket, m_heaps.HashMapBucket_key));
10672 }
10673
10674 void compileSetAdd()
10675 {
10676 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10677 LValue set = lowSetObject(m_node->child1());
10678 LValue key = lowJSValue(m_node->child2());
10679 LValue hash = lowInt32(m_node->child3());
10680
10681 setJSValue(vmCall(pointerType(), operationSetAdd, weakPointer(globalObject), set, key, hash));
10682 }
10683
10684 void compileMapSet()
10685 {
10686 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10687 LValue map = lowMapObject(m_graph.varArgChild(m_node, 0));
10688 LValue key = lowJSValue(m_graph.varArgChild(m_node, 1));
10689 LValue value = lowJSValue(m_graph.varArgChild(m_node, 2));
10690 LValue hash = lowInt32(m_graph.varArgChild(m_node, 3));
10691
10692 setJSValue(vmCall(pointerType(), operationMapSet, weakPointer(globalObject), map, key, value, hash));
10693 }
10694
10695 void compileWeakMapGet()
10696 {
10697 LBasicBlock loopStart = m_out.newBlock();
10698 LBasicBlock loopAround = m_out.newBlock();
10699 LBasicBlock notEqualValue = m_out.newBlock();
10700 LBasicBlock continuation = m_out.newBlock();
10701
10702 LBasicBlock lastNext = m_out.insertNewBlocksBefore(loopStart);
10703
10704 LValue weakMap;
10705 if (m_node->child1().useKind() == WeakMapObjectUse)
10706 weakMap = lowWeakMapObject(m_node->child1());
10707 else if (m_node->child1().useKind() == WeakSetObjectUse)
10708 weakMap = lowWeakSetObject(m_node->child1());
10709 else
10710 RELEASE_ASSERT_NOT_REACHED();
10711 LValue key = lowObject(m_node->child2());
10712 LValue hash = lowInt32(m_node->child3());
10713
10714 LValue buffer = m_out.loadPtr(weakMap, m_heaps.WeakMapImpl_buffer);
10715 LValue mask = m_out.sub(m_out.load32(weakMap, m_heaps.WeakMapImpl_capacity), m_out.int32One);
10716
10717 ValueFromBlock indexStart = m_out.anchor(hash);
10718 m_out.jump(loopStart);
10719
10720 m_out.appendTo(loopStart, notEqualValue);
10721 LValue unmaskedIndex = m_out.phi(Int32, indexStart);
10722 LValue index = m_out.bitAnd(mask, unmaskedIndex);
10723
10724 LValue bucket;
10725
10726 if (m_node->child1().useKind() == WeakMapObjectUse) {
10727 static_assert(hasOneBitSet(sizeof(WeakMapBucket<WeakMapBucketDataKeyValue>)), "Should be a power of 2");
10728 bucket = m_out.add(buffer, m_out.shl(m_out.zeroExt(index, Int64), m_out.constInt32(getLSBSet(sizeof(WeakMapBucket<WeakMapBucketDataKeyValue>)))));
10729 } else {
10730 static_assert(hasOneBitSet(sizeof(WeakMapBucket<WeakMapBucketDataKey>)), "Should be a power of 2");
10731 bucket = m_out.add(buffer, m_out.shl(m_out.zeroExt(index, Int64), m_out.constInt32(getLSBSet(sizeof(WeakMapBucket<WeakMapBucketDataKey>)))));
10732 }
10733
10734 LValue bucketKey = m_out.load64(bucket, m_heaps.WeakMapBucket_key);
10735 m_out.branch(m_out.equal(key, bucketKey), unsure(continuation), unsure(notEqualValue));
10736
10737 m_out.appendTo(notEqualValue, loopAround);
10738 m_out.branch(m_out.isNull(bucketKey), unsure(continuation), unsure(loopAround));
10739
10740 m_out.appendTo(loopAround, continuation);
10741 m_out.addIncomingToPhi(unmaskedIndex, m_out.anchor(m_out.add(index, m_out.int32One)));
10742 m_out.jump(loopStart);
10743
10744 m_out.appendTo(continuation, lastNext);
10745 LValue result;
10746 if (m_node->child1().useKind() == WeakMapObjectUse)
10747 result = m_out.load64(bucket, m_heaps.WeakMapBucket_value);
10748 else
10749 result = bucketKey;
10750 setJSValue(result);
10751 }
10752
10753 void compileWeakSetAdd()
10754 {
10755 LValue set = lowWeakSetObject(m_node->child1());
10756 LValue key = lowObject(m_node->child2());
10757 LValue hash = lowInt32(m_node->child3());
10758
10759 vmCall(Void, operationWeakSetAdd, m_vmValue, set, key, hash);
10760 }
10761
10762 void compileWeakMapSet()
10763 {
10764 LValue map = lowWeakMapObject(m_graph.varArgChild(m_node, 0));
10765 LValue key = lowObject(m_graph.varArgChild(m_node, 1));
10766 LValue value = lowJSValue(m_graph.varArgChild(m_node, 2));
10767 LValue hash = lowInt32(m_graph.varArgChild(m_node, 3));
10768
10769 vmCall(Void, operationWeakMapSet, m_vmValue, map, key, value, hash);
10770 }
10771
10772 void compileIsObjectOrNull()
10773 {
10774 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10775
10776 Edge child = m_node->child1();
10777 LValue value = lowJSValue(child);
10778
10779 LBasicBlock cellCase = m_out.newBlock();
10780 LBasicBlock notFunctionCase = m_out.newBlock();
10781 LBasicBlock objectCase = m_out.newBlock();
10782 LBasicBlock slowPath = m_out.newBlock();
10783 LBasicBlock notCellCase = m_out.newBlock();
10784 LBasicBlock continuation = m_out.newBlock();
10785
10786 m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase));
10787
10788 LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase);
10789 ValueFromBlock isFunctionResult = m_out.anchor(m_out.booleanFalse);
10790 m_out.branch(
10791 isFunction(value, provenType(child)),
10792 unsure(continuation), unsure(notFunctionCase));
10793
10794 m_out.appendTo(notFunctionCase, objectCase);
10795 ValueFromBlock notObjectResult = m_out.anchor(m_out.booleanFalse);
10796 m_out.branch(
10797 isObject(value, provenType(child)),
10798 unsure(objectCase), unsure(continuation));
10799
10800 m_out.appendTo(objectCase, slowPath);
10801 ValueFromBlock objectResult = m_out.anchor(m_out.booleanTrue);
10802 m_out.branch(
10803 isExoticForTypeof(value, provenType(child)),
10804 rarely(slowPath), usually(continuation));
10805
10806 m_out.appendTo(slowPath, notCellCase);
10807 VM& vm = this->vm();
10808 LValue slowResultValue = lazySlowPath(
10809 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
10810 return createLazyCallGenerator(vm,
10811 operationObjectIsObject, locations[0].directGPR(),
10812 CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR());
10813 }, value);
10814 ValueFromBlock slowResult = m_out.anchor(m_out.notZero64(slowResultValue));
10815 m_out.jump(continuation);
10816
10817 m_out.appendTo(notCellCase, continuation);
10818 LValue notCellResultValue = m_out.equal(value, m_out.constInt64(JSValue::encode(jsNull())));
10819 ValueFromBlock notCellResult = m_out.anchor(notCellResultValue);
10820 m_out.jump(continuation);
10821
10822 m_out.appendTo(continuation, lastNext);
10823 LValue result = m_out.phi(
10824 Int32,
10825 isFunctionResult, notObjectResult, objectResult, slowResult, notCellResult);
10826 setBoolean(result);
10827 }
10828
10829 void compileIsFunction()
10830 {
10831 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10832
10833 Edge child = m_node->child1();
10834 LValue value = lowJSValue(child);
10835
10836 LBasicBlock cellCase = m_out.newBlock();
10837 LBasicBlock notFunctionCase = m_out.newBlock();
10838 LBasicBlock slowPath = m_out.newBlock();
10839 LBasicBlock continuation = m_out.newBlock();
10840
10841 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
10842 m_out.branch(
10843 isCell(value, provenType(child)), unsure(cellCase), unsure(continuation));
10844
10845 LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase);
10846 ValueFromBlock functionResult = m_out.anchor(m_out.booleanTrue);
10847 m_out.branch(
10848 isFunction(value, provenType(child)),
10849 unsure(continuation), unsure(notFunctionCase));
10850
10851 m_out.appendTo(notFunctionCase, slowPath);
10852 ValueFromBlock objectResult = m_out.anchor(m_out.booleanFalse);
10853 m_out.branch(
10854 isExoticForTypeof(value, provenType(child)),
10855 rarely(slowPath), usually(continuation));
10856
10857 m_out.appendTo(slowPath, continuation);
10858 VM& vm = this->vm();
10859 LValue slowResultValue = lazySlowPath(
10860 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
10861 return createLazyCallGenerator(vm,
10862 operationObjectIsFunction, locations[0].directGPR(),
10863 CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR());
10864 }, value);
10865 ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue));
10866 m_out.jump(continuation);
10867
10868 m_out.appendTo(continuation, lastNext);
10869 LValue result = m_out.phi(
10870 Int32, notCellResult, functionResult, objectResult, slowResult);
10871 setBoolean(result);
10872 }
10873
10874 void compileIsTypedArrayView()
10875 {
10876 LValue value = lowJSValue(m_node->child1());
10877
10878 LBasicBlock isCellCase = m_out.newBlock();
10879 LBasicBlock continuation = m_out.newBlock();
10880
10881 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
10882 m_out.branch(isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
10883
10884 LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
10885 ValueFromBlock cellResult = m_out.anchor(isTypedArrayView(value, provenType(m_node->child1())));
10886 m_out.jump(continuation);
10887
10888 m_out.appendTo(continuation, lastNext);
10889 setBoolean(m_out.phi(Int32, notCellResult, cellResult));
10890 }
10891
10892 void compileTypeOf()
10893 {
10894 Edge child = m_node->child1();
10895 LValue value = lowJSValue(child);
10896
10897 LBasicBlock continuation = m_out.newBlock();
10898 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
10899
10900 Vector<ValueFromBlock> results;
10901
10902 buildTypeOf(
10903 child, value,
10904 [&] (TypeofType type) {
10905 results.append(m_out.anchor(weakPointer(vm().smallStrings.typeString(type))));
10906 m_out.jump(continuation);
10907 });
10908
10909 m_out.appendTo(continuation, lastNext);
10910 setJSValue(m_out.phi(Int64, results));
10911 }
10912
10913 void compileInByVal()
10914 {
10915 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10916 setJSValue(vmCall(Int64, operationInByVal, weakPointer(globalObject), lowCell(m_node->child1()), lowJSValue(m_node->child2())));
10917 }
10918
10919 void compileInById()
10920 {
10921 Node* node = m_node;
10922 UniquedStringImpl* uid = m_graph.identifiers()[node->identifierNumber()];
10923 LValue base = lowCell(m_node->child1());
10924
10925 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
10926 patchpoint->appendSomeRegister(base);
10927 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
10928 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
10929
10930 patchpoint->clobber(RegisterSet::macroScratchRegisters());
10931
10932 RefPtr<PatchpointExceptionHandle> exceptionHandle =
10933 preparePatchpointForExceptions(patchpoint);
10934
10935 State* state = &m_ftlState;
10936 patchpoint->setGenerator(
10937 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
10938 AllowMacroScratchRegisterUsage allowScratch(jit);
10939
10940 CallSiteIndex callSiteIndex =
10941 state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
10942
10943 // This is the direct exit target for operation calls.
10944 Box<CCallHelpers::JumpList> exceptions =
10945 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
10946
10947 auto generator = Box<JITInByIdGenerator>::create(
10948 jit.codeBlock(), node->origin.semantic, callSiteIndex,
10949 params.unavailableRegisters(), uid, JSValueRegs(params[1].gpr()),
10950 JSValueRegs(params[0].gpr()));
10951
10952 generator->generateFastPath(jit);
10953 CCallHelpers::Label done = jit.label();
10954
10955 params.addLatePath(
10956 [=] (CCallHelpers& jit) {
10957 AllowMacroScratchRegisterUsage allowScratch(jit);
10958
10959 generator->slowPathJump().link(&jit);
10960 CCallHelpers::Label slowPathBegin = jit.label();
10961 CCallHelpers::Call slowPathCall = callOperation(
10962 *state, params.unavailableRegisters(), jit, node->origin.semantic,
10963 exceptions.get(), operationInByIdOptimize, params[0].gpr(),
10964 jit.codeBlock()->globalObjectFor(node->origin.semantic),
10965 CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(),
10966 CCallHelpers::TrustedImmPtr(uid)).call();
10967 jit.jump().linkTo(done, &jit);
10968
10969 generator->reportSlowPathCall(slowPathBegin, slowPathCall);
10970
10971 jit.addLinkTask(
10972 [=] (LinkBuffer& linkBuffer) {
10973 generator->finalize(linkBuffer, linkBuffer);
10974 });
10975 });
10976 });
10977
10978 setJSValue(patchpoint);
10979 }
10980
10981 void compileHasOwnProperty()
10982 {
10983 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
10984 LBasicBlock slowCase = m_out.newBlock();
10985 LBasicBlock continuation = m_out.newBlock();
10986 LBasicBlock lastNext = nullptr;
10987
10988 LValue object = lowObject(m_node->child1());
10989 LValue uniquedStringImpl;
10990 LValue keyAsValue = nullptr;
10991 switch (m_node->child2().useKind()) {
10992 case StringUse: {
10993 LBasicBlock isNonEmptyString = m_out.newBlock();
10994 LBasicBlock isAtomString = m_out.newBlock();
10995
10996 keyAsValue = lowString(m_node->child2());
10997 m_out.branch(isNotRopeString(keyAsValue, m_node->child2()), usually(isNonEmptyString), rarely(slowCase));
10998
10999 lastNext = m_out.appendTo(isNonEmptyString, isAtomString);
11000 uniquedStringImpl = m_out.loadPtr(keyAsValue, m_heaps.JSString_value);
11001 LValue isNotAtomic = m_out.testIsZero32(m_out.load32(uniquedStringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtom()));
11002 m_out.branch(isNotAtomic, rarely(slowCase), usually(isAtomString));
11003
11004 m_out.appendTo(isAtomString, slowCase);
11005 break;
11006 }
11007 case SymbolUse: {
11008 keyAsValue = lowSymbol(m_node->child2());
11009 uniquedStringImpl = m_out.loadPtr(keyAsValue, m_heaps.Symbol_symbolImpl);
11010 lastNext = m_out.insertNewBlocksBefore(slowCase);
11011 break;
11012 }
11013 case UntypedUse: {
11014 LBasicBlock isCellCase = m_out.newBlock();
11015 LBasicBlock isStringCase = m_out.newBlock();
11016 LBasicBlock notStringCase = m_out.newBlock();
11017 LBasicBlock isNonEmptyString = m_out.newBlock();
11018 LBasicBlock isSymbolCase = m_out.newBlock();
11019 LBasicBlock hasUniquedStringImpl = m_out.newBlock();
11020
11021 keyAsValue = lowJSValue(m_node->child2());
11022 m_out.branch(isCell(keyAsValue), usually(isCellCase), rarely(slowCase));
11023
11024 lastNext = m_out.appendTo(isCellCase, isStringCase);
11025 m_out.branch(isString(keyAsValue), unsure(isStringCase), unsure(notStringCase));
11026
11027 m_out.appendTo(isStringCase, isNonEmptyString);
11028 m_out.branch(isNotRopeString(keyAsValue, m_node->child2()), usually(isNonEmptyString), rarely(slowCase));
11029
11030 m_out.appendTo(isNonEmptyString, notStringCase);
11031 LValue implFromString = m_out.loadPtr(keyAsValue, m_heaps.JSString_value);
11032 ValueFromBlock stringResult = m_out.anchor(implFromString);
11033 LValue isNotAtomic = m_out.testIsZero32(m_out.load32(implFromString, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtom()));
11034 m_out.branch(isNotAtomic, rarely(slowCase), usually(hasUniquedStringImpl));
11035
11036 m_out.appendTo(notStringCase, isSymbolCase);
11037 m_out.branch(isSymbol(keyAsValue), unsure(isSymbolCase), unsure(slowCase));
11038
11039 m_out.appendTo(isSymbolCase, hasUniquedStringImpl);
11040 ValueFromBlock symbolResult = m_out.anchor(m_out.loadPtr(keyAsValue, m_heaps.Symbol_symbolImpl));
11041 m_out.jump(hasUniquedStringImpl);
11042
11043 m_out.appendTo(hasUniquedStringImpl, slowCase);
11044 uniquedStringImpl = m_out.phi(pointerType(), stringResult, symbolResult);
11045 break;
11046 }
11047 default:
11048 RELEASE_ASSERT_NOT_REACHED();
11049 }
11050
11051 ASSERT(keyAsValue);
11052
11053 // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero
11054 // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only
11055 // ever load the result from the cache if the cache entry matches what we are querying for.
11056 // So we either get super lucky and use zero for the hash and somehow collide with the entity
11057 // we're looking for, or we realize we're comparing against another entity, and go to the
11058 // slow path anyways.
11059 LValue hash = m_out.lShr(m_out.load32(uniquedStringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount));
11060
11061 LValue structureID = m_out.load32(object, m_heaps.JSCell_structureID);
11062 LValue index = m_out.add(hash, structureID);
11063 index = m_out.zeroExtPtr(m_out.bitAnd(index, m_out.constInt32(HasOwnPropertyCache::mask)));
11064 ASSERT(vm().hasOwnPropertyCache());
11065 LValue cache = m_out.constIntPtr(vm().hasOwnPropertyCache());
11066
11067 IndexedAbstractHeap& heap = m_heaps.HasOwnPropertyCache;
11068 LValue sameStructureID = m_out.equal(structureID, m_out.load32(m_out.baseIndex(heap, cache, index, JSValue(), HasOwnPropertyCache::Entry::offsetOfStructureID())));
11069 LValue sameImpl = m_out.equal(uniquedStringImpl, m_out.loadPtr(m_out.baseIndex(heap, cache, index, JSValue(), HasOwnPropertyCache::Entry::offsetOfImpl())));
11070 ValueFromBlock fastResult = m_out.anchor(m_out.load8ZeroExt32(m_out.baseIndex(heap, cache, index, JSValue(), HasOwnPropertyCache::Entry::offsetOfResult())));
11071 LValue cacheHit = m_out.bitAnd(sameStructureID, sameImpl);
11072
11073 m_out.branch(m_out.notZero32(cacheHit), usually(continuation), rarely(slowCase));
11074
11075 m_out.appendTo(slowCase, continuation);
11076 ValueFromBlock slowResult;
11077 slowResult = m_out.anchor(vmCall(Int32, operationHasOwnProperty, weakPointer(globalObject), object, keyAsValue));
11078 m_out.jump(continuation);
11079
11080 m_out.appendTo(continuation, lastNext);
11081 setBoolean(m_out.phi(Int32, fastResult, slowResult));
11082 }
11083
11084 void compileParseInt()
11085 {
11086 RELEASE_ASSERT(m_node->child1().useKind() == UntypedUse || m_node->child1().useKind() == StringUse);
11087 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11088 LValue result;
11089 if (m_node->child2()) {
11090 LValue radix = lowInt32(m_node->child2());
11091 if (m_node->child1().useKind() == UntypedUse)
11092 result = vmCall(Int64, operationParseIntGeneric, weakPointer(globalObject), lowJSValue(m_node->child1()), radix);
11093 else
11094 result = vmCall(Int64, operationParseIntString, weakPointer(globalObject), lowString(m_node->child1()), radix);
11095 } else {
11096 if (m_node->child1().useKind() == UntypedUse)
11097 result = vmCall(Int64, operationParseIntNoRadixGeneric, weakPointer(globalObject), lowJSValue(m_node->child1()));
11098 else
11099 result = vmCall(Int64, operationParseIntStringNoRadix, weakPointer(globalObject), lowString(m_node->child1()));
11100 }
11101 setJSValue(result);
11102 }
11103
11104 void compileOverridesHasInstance()
11105 {
11106 FrozenValue* defaultHasInstanceFunction = m_node->cellOperand();
11107 ASSERT(defaultHasInstanceFunction->cell()->inherits<JSFunction>(vm()));
11108
11109 LValue constructor = lowCell(m_node->child1());
11110 LValue hasInstance = lowJSValue(m_node->child2());
11111
11112 LBasicBlock defaultHasInstance = m_out.newBlock();
11113 LBasicBlock continuation = m_out.newBlock();
11114
11115 // Unlike in the DFG, we don't worry about cleaning this code up for the case where we have proven the hasInstanceValue is a constant as B3 should fix it for us.
11116
11117 ValueFromBlock notDefaultHasInstanceResult = m_out.anchor(m_out.booleanTrue);
11118 m_out.branch(m_out.notEqual(hasInstance, frozenPointer(defaultHasInstanceFunction)), unsure(continuation), unsure(defaultHasInstance));
11119
11120 LBasicBlock lastNext = m_out.appendTo(defaultHasInstance, continuation);
11121 ValueFromBlock implementsDefaultHasInstanceResult = m_out.anchor(m_out.testIsZero32(
11122 m_out.load8ZeroExt32(constructor, m_heaps.JSCell_typeInfoFlags),
11123 m_out.constInt32(ImplementsDefaultHasInstance)));
11124 m_out.jump(continuation);
11125
11126 m_out.appendTo(continuation, lastNext);
11127 setBoolean(m_out.phi(Int32, implementsDefaultHasInstanceResult, notDefaultHasInstanceResult));
11128 }
11129
11130 void compileCheckTypeInfoFlags()
11131 {
11132 speculate(
11133 BadTypeInfoFlags, noValue(), 0,
11134 m_out.testIsZero32(
11135 m_out.load8ZeroExt32(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags),
11136 m_out.constInt32(m_node->typeInfoOperand())));
11137 }
11138
11139 void compileInstanceOf()
11140 {
11141 Node* node = m_node;
11142 State* state = &m_ftlState;
11143
11144 LValue value;
11145 LValue prototype;
11146 bool valueIsCell;
11147 bool prototypeIsCell;
11148 if (m_node->child1().useKind() == CellUse
11149 && m_node->child2().useKind() == CellUse) {
11150 value = lowCell(m_node->child1());
11151 prototype = lowCell(m_node->child2());
11152
11153 valueIsCell = true;
11154 prototypeIsCell = true;
11155 } else {
11156 DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse);
11157 DFG_ASSERT(m_graph, m_node, m_node->child2().useKind() == UntypedUse);
11158
11159 value = lowJSValue(m_node->child1());
11160 prototype = lowJSValue(m_node->child2());
11161
11162 valueIsCell = abstractValue(m_node->child1()).isType(SpecCell);
11163 prototypeIsCell = abstractValue(m_node->child2()).isType(SpecCell);
11164 }
11165
11166 bool prototypeIsObject = abstractValue(m_node->child2()).isType(SpecObject | ~SpecCell);
11167
11168 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
11169 patchpoint->appendSomeRegister(value);
11170 patchpoint->appendSomeRegister(prototype);
11171 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
11172 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
11173 patchpoint->numGPScratchRegisters = 2;
11174 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
11175 patchpoint->clobber(RegisterSet::macroScratchRegisters());
11176
11177 RefPtr<PatchpointExceptionHandle> exceptionHandle =
11178 preparePatchpointForExceptions(patchpoint);
11179
11180 patchpoint->setGenerator(
11181 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
11182 AllowMacroScratchRegisterUsage allowScratch(jit);
11183
11184 GPRReg resultGPR = params[0].gpr();
11185 GPRReg valueGPR = params[1].gpr();
11186 GPRReg prototypeGPR = params[2].gpr();
11187 GPRReg scratchGPR = params.gpScratch(0);
11188 GPRReg scratch2GPR = params.gpScratch(1);
11189
11190 CCallHelpers::Jump doneJump;
11191 if (!valueIsCell) {
11192 CCallHelpers::Jump isCell = jit.branchIfCell(valueGPR);
11193 jit.boxBooleanPayload(false, resultGPR);
11194 doneJump = jit.jump();
11195 isCell.link(&jit);
11196 }
11197
11198 CCallHelpers::JumpList slowCases;
11199 if (!prototypeIsCell)
11200 slowCases.append(jit.branchIfNotCell(prototypeGPR));
11201
11202 CallSiteIndex callSiteIndex =
11203 state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
11204
11205 // This is the direct exit target for operation calls.
11206 Box<CCallHelpers::JumpList> exceptions =
11207 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
11208
11209 auto generator = Box<JITInstanceOfGenerator>::create(
11210 jit.codeBlock(), node->origin.semantic, callSiteIndex,
11211 params.unavailableRegisters(), resultGPR, valueGPR, prototypeGPR, scratchGPR,
11212 scratch2GPR, prototypeIsObject);
11213 generator->generateFastPath(jit);
11214 CCallHelpers::Label done = jit.label();
11215
11216 params.addLatePath(
11217 [=] (CCallHelpers& jit) {
11218 AllowMacroScratchRegisterUsage allowScratch(jit);
11219
11220 J_JITOperation_GSsiJJ optimizationFunction = operationInstanceOfOptimize;
11221
11222 slowCases.link(&jit);
11223 CCallHelpers::Label slowPathBegin = jit.label();
11224 CCallHelpers::Call slowPathCall = callOperation(
11225 *state, params.unavailableRegisters(), jit, node->origin.semantic,
11226 exceptions.get(), optimizationFunction, resultGPR,
11227 jit.codeBlock()->globalObjectFor(node->origin.semantic),
11228 CCallHelpers::TrustedImmPtr(generator->stubInfo()), valueGPR,
11229 prototypeGPR).call();
11230 jit.jump().linkTo(done, &jit);
11231
11232 generator->reportSlowPathCall(slowPathBegin, slowPathCall);
11233
11234 jit.addLinkTask(
11235 [=] (LinkBuffer& linkBuffer) {
11236 generator->finalize(linkBuffer, linkBuffer);
11237 });
11238 });
11239
11240 if (doneJump.isSet())
11241 doneJump.link(&jit);
11242 });
11243
11244 // This returns a boxed boolean.
11245 setJSValue(patchpoint);
11246 }
11247
11248 void compileInstanceOfCustom()
11249 {
11250 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11251 LValue value = lowJSValue(m_node->child1());
11252 LValue constructor = lowCell(m_node->child2());
11253 LValue hasInstance = lowJSValue(m_node->child3());
11254
11255 setBoolean(m_out.logicalNot(m_out.equal(m_out.constInt32(0), vmCall(Int32, operationInstanceOfCustom, weakPointer(globalObject), value, constructor, hasInstance))));
11256 }
11257
11258 void compileCountExecution()
11259 {
11260 TypedPointer counter = m_out.absolute(m_node->executionCounter()->address());
11261 m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter);
11262 }
11263
11264 void compileSuperSamplerBegin()
11265 {
11266 TypedPointer counter = m_out.absolute(bitwise_cast<void*>(&g_superSamplerCount));
11267 m_out.store32(m_out.add(m_out.load32(counter), m_out.constInt32(1)), counter);
11268 }
11269
11270 void compileSuperSamplerEnd()
11271 {
11272 TypedPointer counter = m_out.absolute(bitwise_cast<void*>(&g_superSamplerCount));
11273 m_out.store32(m_out.sub(m_out.load32(counter), m_out.constInt32(1)), counter);
11274 }
11275
11276 void compileStoreBarrier()
11277 {
11278 emitStoreBarrier(lowCell(m_node->child1()), m_node->op() == FencedStoreBarrier);
11279 }
11280
11281 void compileHasIndexedProperty()
11282 {
11283 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11284 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
11285 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
11286
11287 switch (m_node->arrayMode().type()) {
11288 case Array::Int32:
11289 case Array::Contiguous: {
11290 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
11291 LValue internalMethodType = m_out.constInt32(static_cast<int32_t>(m_node->internalMethodType()));
11292
11293 IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
11294 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
11295
11296 LBasicBlock slowCase = m_out.newBlock();
11297 LBasicBlock continuation = m_out.newBlock();
11298 LBasicBlock lastNext = nullptr;
11299
11300 if (!m_node->arrayMode().isInBounds()) {
11301 LBasicBlock checkHole = m_out.newBlock();
11302 m_out.branch(
11303 m_out.aboveOrEqual(
11304 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
11305 rarely(slowCase), usually(checkHole));
11306 lastNext = m_out.appendTo(checkHole, slowCase);
11307 } else
11308 lastNext = m_out.insertNewBlocksBefore(slowCase);
11309
11310 LValue checkHoleResultValue =
11311 m_out.notZero64(m_out.load64(baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1))));
11312 ValueFromBlock checkHoleResult = m_out.anchor(checkHoleResultValue);
11313 m_out.branch(checkHoleResultValue, usually(continuation), rarely(slowCase));
11314
11315 m_out.appendTo(slowCase, continuation);
11316 ValueFromBlock slowResult = m_out.anchor(
11317 m_out.notZero64(vmCall(Int64, operationHasIndexedPropertyByInt, weakPointer(globalObject), base, index, internalMethodType)));
11318 m_out.jump(continuation);
11319
11320 m_out.appendTo(continuation, lastNext);
11321 setBoolean(m_out.phi(Int32, checkHoleResult, slowResult));
11322 return;
11323 }
11324 case Array::Double: {
11325 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
11326 LValue internalMethodType = m_out.constInt32(static_cast<int32_t>(m_node->internalMethodType()));
11327
11328 IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
11329
11330 LBasicBlock slowCase = m_out.newBlock();
11331 LBasicBlock continuation = m_out.newBlock();
11332 LBasicBlock lastNext = nullptr;
11333
11334 if (!m_node->arrayMode().isInBounds()) {
11335 LBasicBlock checkHole = m_out.newBlock();
11336 m_out.branch(
11337 m_out.aboveOrEqual(
11338 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
11339 rarely(slowCase), usually(checkHole));
11340 lastNext = m_out.appendTo(checkHole, slowCase);
11341 } else
11342 lastNext = m_out.insertNewBlocksBefore(slowCase);
11343
11344 LValue doubleValue = m_out.loadDouble(baseIndex(heap, storage, index, m_graph.varArgChild(m_node, 1)));
11345 LValue checkHoleResultValue = m_out.doubleEqual(doubleValue, doubleValue);
11346 ValueFromBlock checkHoleResult = m_out.anchor(checkHoleResultValue);
11347 m_out.branch(checkHoleResultValue, usually(continuation), rarely(slowCase));
11348
11349 m_out.appendTo(slowCase, continuation);
11350 ValueFromBlock slowResult = m_out.anchor(
11351 m_out.notZero64(vmCall(Int64, operationHasIndexedPropertyByInt, weakPointer(globalObject), base, index, internalMethodType)));
11352 m_out.jump(continuation);
11353
11354 m_out.appendTo(continuation, lastNext);
11355 setBoolean(m_out.phi(Int32, checkHoleResult, slowResult));
11356 return;
11357 }
11358
11359 case Array::ArrayStorage: {
11360 LValue storage = lowStorage(m_graph.varArgChild(m_node, 2));
11361 LValue internalMethodType = m_out.constInt32(static_cast<int32_t>(m_node->internalMethodType()));
11362
11363 LBasicBlock slowCase = m_out.newBlock();
11364 LBasicBlock continuation = m_out.newBlock();
11365 LBasicBlock lastNext = nullptr;
11366
11367 if (!m_node->arrayMode().isInBounds()) {
11368 LBasicBlock checkHole = m_out.newBlock();
11369 m_out.branch(
11370 m_out.aboveOrEqual(
11371 index, m_out.load32NonNegative(storage, m_heaps.ArrayStorage_vectorLength)),
11372 rarely(slowCase), usually(checkHole));
11373 lastNext = m_out.appendTo(checkHole, slowCase);
11374 } else
11375 lastNext = m_out.insertNewBlocksBefore(slowCase);
11376
11377 LValue checkHoleResultValue =
11378 m_out.notZero64(m_out.load64(baseIndex(m_heaps.ArrayStorage_vector, storage, index, m_graph.varArgChild(m_node, 1))));
11379 ValueFromBlock checkHoleResult = m_out.anchor(checkHoleResultValue);
11380 m_out.branch(checkHoleResultValue, usually(continuation), rarely(slowCase));
11381
11382 m_out.appendTo(slowCase, continuation);
11383 ValueFromBlock slowResult = m_out.anchor(
11384 m_out.notZero64(vmCall(Int64, operationHasIndexedPropertyByInt, weakPointer(globalObject), base, index, internalMethodType)));
11385 m_out.jump(continuation);
11386
11387 m_out.appendTo(continuation, lastNext);
11388 setBoolean(m_out.phi(Int32, checkHoleResult, slowResult));
11389 break;
11390 }
11391
11392 default: {
11393 LValue internalMethodType = m_out.constInt32(static_cast<int32_t>(m_node->internalMethodType()));
11394 setBoolean(m_out.notZero64(vmCall(Int64, operationHasIndexedPropertyByInt, weakPointer(globalObject), base, index, internalMethodType)));
11395 break;
11396 }
11397 }
11398 }
11399
11400 void compileHasGenericProperty()
11401 {
11402 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11403 LValue base = lowJSValue(m_node->child1());
11404 LValue property = lowCell(m_node->child2());
11405 setJSValue(vmCall(Int64, operationHasGenericProperty, weakPointer(globalObject), base, property));
11406 }
11407
11408 void compileHasStructureProperty()
11409 {
11410 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11411 LValue base = lowJSValue(m_node->child1());
11412 LValue property = lowString(m_node->child2());
11413 LValue enumerator = lowCell(m_node->child3());
11414
11415 LBasicBlock correctStructure = m_out.newBlock();
11416 LBasicBlock wrongStructure = m_out.newBlock();
11417 LBasicBlock continuation = m_out.newBlock();
11418
11419 m_out.branch(m_out.notEqual(
11420 m_out.load32(base, m_heaps.JSCell_structureID),
11421 m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)),
11422 rarely(wrongStructure), usually(correctStructure));
11423
11424 LBasicBlock lastNext = m_out.appendTo(correctStructure, wrongStructure);
11425 ValueFromBlock correctStructureResult = m_out.anchor(m_out.booleanTrue);
11426 m_out.jump(continuation);
11427
11428 m_out.appendTo(wrongStructure, continuation);
11429 ValueFromBlock wrongStructureResult = m_out.anchor(
11430 m_out.equal(
11431 m_out.constInt64(JSValue::encode(jsBoolean(true))),
11432 vmCall(Int64, operationHasGenericProperty, weakPointer(globalObject), base, property)));
11433 m_out.jump(continuation);
11434
11435 m_out.appendTo(continuation, lastNext);
11436 setBoolean(m_out.phi(Int32, correctStructureResult, wrongStructureResult));
11437 }
11438
11439 void compileGetDirectPname()
11440 {
11441 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11442 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
11443 LValue property = lowCell(m_graph.varArgChild(m_node, 1));
11444 LValue index = lowInt32(m_graph.varArgChild(m_node, 2));
11445 LValue enumerator = lowCell(m_graph.varArgChild(m_node, 3));
11446
11447 LBasicBlock checkOffset = m_out.newBlock();
11448 LBasicBlock inlineLoad = m_out.newBlock();
11449 LBasicBlock outOfLineLoad = m_out.newBlock();
11450 LBasicBlock slowCase = m_out.newBlock();
11451 LBasicBlock continuation = m_out.newBlock();
11452
11453 m_out.branch(m_out.notEqual(
11454 m_out.load32(base, m_heaps.JSCell_structureID),
11455 m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)),
11456 rarely(slowCase), usually(checkOffset));
11457
11458 LBasicBlock lastNext = m_out.appendTo(checkOffset, inlineLoad);
11459 m_out.branch(m_out.aboveOrEqual(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity)),
11460 unsure(outOfLineLoad), unsure(inlineLoad));
11461
11462 m_out.appendTo(inlineLoad, outOfLineLoad);
11463 ValueFromBlock inlineResult = m_out.anchor(
11464 m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(),
11465 base, m_out.zeroExt(index, Int64), ScaleEight, JSObject::offsetOfInlineStorage())));
11466 m_out.jump(continuation);
11467
11468 m_out.appendTo(outOfLineLoad, slowCase);
11469 LValue storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
11470 LValue realIndex = m_out.signExt32To64(
11471 m_out.neg(m_out.sub(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity))));
11472 int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue);
11473 ValueFromBlock outOfLineResult = m_out.anchor(
11474 m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), storage, realIndex, ScaleEight, offsetOfFirstProperty)));
11475 m_out.jump(continuation);
11476
11477 m_out.appendTo(slowCase, continuation);
11478 ValueFromBlock slowCaseResult = m_out.anchor(
11479 vmCall(Int64, operationGetByVal, weakPointer(globalObject), base, property));
11480 m_out.jump(continuation);
11481
11482 m_out.appendTo(continuation, lastNext);
11483 setJSValue(m_out.phi(Int64, inlineResult, outOfLineResult, slowCaseResult));
11484 }
11485
11486 void compileGetEnumerableLength()
11487 {
11488 LValue enumerator = lowCell(m_node->child1());
11489 setInt32(m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_indexLength));
11490 }
11491
11492 void compileGetPropertyEnumerator()
11493 {
11494 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11495 if (m_node->child1().useKind() == CellUse)
11496 setJSValue(vmCall(Int64, operationGetPropertyEnumeratorCell, weakPointer(globalObject), lowCell(m_node->child1())));
11497 else
11498 setJSValue(vmCall(Int64, operationGetPropertyEnumerator, weakPointer(globalObject), lowJSValue(m_node->child1())));
11499 }
11500
11501 void compileGetEnumeratorStructurePname()
11502 {
11503 LValue enumerator = lowCell(m_node->child1());
11504 LValue index = lowInt32(m_node->child2());
11505
11506 LBasicBlock inBounds = m_out.newBlock();
11507 LBasicBlock outOfBounds = m_out.newBlock();
11508 LBasicBlock continuation = m_out.newBlock();
11509
11510 m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endStructurePropertyIndex)),
11511 usually(inBounds), rarely(outOfBounds));
11512
11513 LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
11514 LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector);
11515 ValueFromBlock inBoundsResult = m_out.anchor(
11516 m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index))));
11517 m_out.jump(continuation);
11518
11519 m_out.appendTo(outOfBounds, continuation);
11520 ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(JSValue::ValueNull));
11521 m_out.jump(continuation);
11522
11523 m_out.appendTo(continuation, lastNext);
11524 setJSValue(m_out.phi(Int64, inBoundsResult, outOfBoundsResult));
11525 }
11526
11527 void compileGetEnumeratorGenericPname()
11528 {
11529 LValue enumerator = lowCell(m_node->child1());
11530 LValue index = lowInt32(m_node->child2());
11531
11532 LBasicBlock inBounds = m_out.newBlock();
11533 LBasicBlock outOfBounds = m_out.newBlock();
11534 LBasicBlock continuation = m_out.newBlock();
11535
11536 m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endGenericPropertyIndex)),
11537 usually(inBounds), rarely(outOfBounds));
11538
11539 LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
11540 LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector);
11541 ValueFromBlock inBoundsResult = m_out.anchor(
11542 m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index))));
11543 m_out.jump(continuation);
11544
11545 m_out.appendTo(outOfBounds, continuation);
11546 ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(JSValue::ValueNull));
11547 m_out.jump(continuation);
11548
11549 m_out.appendTo(continuation, lastNext);
11550 setJSValue(m_out.phi(Int64, inBoundsResult, outOfBoundsResult));
11551 }
11552
11553 void compileToIndexString()
11554 {
11555 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11556 LValue index = lowInt32(m_node->child1());
11557 setJSValue(vmCall(Int64, operationToIndexString, weakPointer(globalObject), index));
11558 }
11559
11560 void compileCheckStructureImmediate()
11561 {
11562 LValue structure = lowCell(m_node->child1());
11563 checkStructure(
11564 structure, noValue(), BadCache, m_node->structureSet(),
11565 [this] (RegisteredStructure structure) {
11566 return weakStructure(structure);
11567 });
11568 }
11569
11570 void compileMaterializeNewObject()
11571 {
11572 ObjectMaterializationData& data = m_node->objectMaterializationData();
11573
11574 // Lower the values first, to avoid creating values inside a control flow diamond.
11575
11576 Vector<LValue, 8> values;
11577 for (unsigned i = 0; i < data.m_properties.size(); ++i) {
11578 Edge edge = m_graph.varArgChild(m_node, 1 + i);
11579 switch (data.m_properties[i].kind()) {
11580 case PublicLengthPLoc:
11581 case VectorLengthPLoc:
11582 values.append(lowInt32(edge));
11583 break;
11584 default:
11585 values.append(lowJSValue(edge));
11586 break;
11587 }
11588 }
11589
11590 RegisteredStructureSet set = m_node->structureSet();
11591
11592 Vector<LBasicBlock, 1> blocks(set.size());
11593 for (unsigned i = set.size(); i--;)
11594 blocks[i] = m_out.newBlock();
11595 LBasicBlock dummyDefault = m_out.newBlock();
11596 LBasicBlock outerContinuation = m_out.newBlock();
11597
11598 Vector<SwitchCase, 1> cases(set.size());
11599 for (unsigned i = set.size(); i--;)
11600 cases[i] = SwitchCase(weakStructure(set.at(i)), blocks[i], Weight(1));
11601 m_out.switchInstruction(
11602 lowCell(m_graph.varArgChild(m_node, 0)), cases, dummyDefault, Weight(0));
11603
11604 LBasicBlock outerLastNext = m_out.m_nextBlock;
11605
11606 Vector<ValueFromBlock, 1> results;
11607
11608 for (unsigned i = set.size(); i--;) {
11609 m_out.appendTo(blocks[i], i + 1 < set.size() ? blocks[i + 1] : dummyDefault);
11610
11611 RegisteredStructure structure = set.at(i);
11612
11613 LValue object;
11614 LValue butterfly;
11615
11616 if (structure->outOfLineCapacity() || hasIndexedProperties(structure->indexingType())) {
11617 size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
11618 Allocator cellAllocator = allocatorForNonVirtualConcurrently<JSFinalObject>(vm(), allocationSize, AllocatorForMode::AllocatorIfExists);
11619
11620 bool hasIndexingHeader = hasIndexedProperties(structure->indexingType());
11621 unsigned indexingHeaderSize = 0;
11622 LValue indexingPayloadSizeInBytes = m_out.intPtrZero;
11623 LValue vectorLength = m_out.int32Zero;
11624 LValue publicLength = m_out.int32Zero;
11625 if (hasIndexingHeader) {
11626 indexingHeaderSize = sizeof(IndexingHeader);
11627 for (unsigned i = data.m_properties.size(); i--;) {
11628 PromotedLocationDescriptor descriptor = data.m_properties[i];
11629 switch (descriptor.kind()) {
11630 case PublicLengthPLoc:
11631 publicLength = values[i];
11632 break;
11633 case VectorLengthPLoc:
11634 vectorLength = values[i];
11635 break;
11636 default:
11637 break;
11638 }
11639 }
11640 indexingPayloadSizeInBytes =
11641 m_out.mul(m_out.zeroExtPtr(vectorLength), m_out.intPtrEight);
11642 }
11643
11644 LValue butterflySize = m_out.add(
11645 m_out.constIntPtr(
11646 structure->outOfLineCapacity() * sizeof(JSValue) + indexingHeaderSize),
11647 indexingPayloadSizeInBytes);
11648
11649 LBasicBlock slowPath = m_out.newBlock();
11650 LBasicBlock continuation = m_out.newBlock();
11651
11652 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
11653
11654 ValueFromBlock noButterfly = m_out.anchor(m_out.intPtrZero);
11655
11656 LValue startOfStorage = allocateHeapCell(
11657 allocatorForSize(vm().jsValueGigacageAuxiliarySpace, butterflySize, slowPath),
11658 slowPath);
11659
11660 LValue fastButterflyValue = m_out.add(
11661 startOfStorage,
11662 m_out.constIntPtr(
11663 structure->outOfLineCapacity() * sizeof(JSValue) + sizeof(IndexingHeader)));
11664
11665 ValueFromBlock haveButterfly = m_out.anchor(fastButterflyValue);
11666
11667 splatWords(
11668 fastButterflyValue,
11669 m_out.constInt32(-structure->outOfLineCapacity() - 1),
11670 m_out.constInt32(-1),
11671 m_out.int64Zero, m_heaps.properties.atAnyNumber());
11672
11673 m_out.store32(vectorLength, fastButterflyValue, m_heaps.Butterfly_vectorLength);
11674
11675 LValue fastObjectValue = allocateObject(
11676 m_out.constIntPtr(cellAllocator.localAllocator()), structure, fastButterflyValue,
11677 slowPath);
11678
11679 ValueFromBlock fastObject = m_out.anchor(fastObjectValue);
11680 ValueFromBlock fastButterfly = m_out.anchor(fastButterflyValue);
11681 m_out.jump(continuation);
11682
11683 m_out.appendTo(slowPath, continuation);
11684
11685 LValue butterflyValue = m_out.phi(pointerType(), noButterfly, haveButterfly);
11686
11687 VM& vm = this->vm();
11688 LValue slowObjectValue;
11689 if (hasIndexingHeader) {
11690 slowObjectValue = lazySlowPath(
11691 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
11692 return createLazyCallGenerator(vm,
11693 operationNewObjectWithButterflyWithIndexingHeaderAndVectorLength,
11694 locations[0].directGPR(), &vm, CCallHelpers::TrustedImmPtr(structure.get()),
11695 locations[1].directGPR(), locations[2].directGPR());
11696 },
11697 vectorLength, butterflyValue);
11698 } else {
11699 slowObjectValue = lazySlowPath(
11700 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
11701 return createLazyCallGenerator(vm,
11702 operationNewObjectWithButterfly, locations[0].directGPR(), &vm,
11703 CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR());
11704 },
11705 butterflyValue);
11706 }
11707 ValueFromBlock slowObject = m_out.anchor(slowObjectValue);
11708 ValueFromBlock slowButterfly = m_out.anchor(
11709 m_out.loadPtr(slowObjectValue, m_heaps.JSObject_butterfly));
11710
11711 m_out.jump(continuation);
11712
11713 m_out.appendTo(continuation, lastNext);
11714
11715 object = m_out.phi(pointerType(), fastObject, slowObject);
11716 butterfly = m_out.phi(pointerType(), fastButterfly, slowButterfly);
11717
11718 m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength);
11719
11720 initializeArrayElements(m_out.constInt32(structure->indexingType()), m_out.int32Zero, vectorLength, butterfly);
11721
11722 HashMap<int32_t, LValue, DefaultHash<int32_t>::Hash, WTF::UnsignedWithZeroKeyHashTraits<int32_t>> indexMap;
11723 Vector<int32_t> indices;
11724 for (unsigned i = data.m_properties.size(); i--;) {
11725 PromotedLocationDescriptor descriptor = data.m_properties[i];
11726 if (descriptor.kind() != IndexedPropertyPLoc)
11727 continue;
11728 int32_t index = static_cast<int32_t>(descriptor.info());
11729
11730 auto result = indexMap.add(index, values[i]);
11731 DFG_ASSERT(m_graph, m_node, result); // Duplicates are illegal.
11732
11733 indices.append(index);
11734 }
11735
11736 if (!indices.isEmpty()) {
11737 std::sort(indices.begin(), indices.end());
11738
11739 Vector<LBasicBlock> blocksWithStores(indices.size());
11740 Vector<LBasicBlock> blocksWithChecks(indices.size());
11741
11742 for (unsigned i = indices.size(); i--;) {
11743 blocksWithStores[i] = m_out.newBlock();
11744 blocksWithChecks[i] = m_out.newBlock(); // blocksWithChecks[0] is the continuation.
11745 }
11746
11747 LBasicBlock indexLastNext = m_out.m_nextBlock;
11748
11749 for (unsigned i = indices.size(); i--;) {
11750 int32_t index = indices[i];
11751 LValue value = indexMap.get(index);
11752
11753 m_out.branch(
11754 m_out.below(m_out.constInt32(index), publicLength),
11755 unsure(blocksWithStores[i]), unsure(blocksWithChecks[i]));
11756
11757 m_out.appendTo(blocksWithStores[i], blocksWithChecks[i]);
11758
11759 // This has to type-check and convert its inputs, but it cannot do so in a
11760 // way that updates AI. That's a bit annoying, but if you think about how
11761 // sinking works, it's actually not a bad thing. We are virtually guaranteed
11762 // that these type checks will not fail, since the type checks that guarded
11763 // the original stores to the array are still somewhere above this point.
11764 Output::StoreType storeType;
11765 IndexedAbstractHeap* heap;
11766 switch (structure->indexingType()) {
11767 case ALL_INT32_INDEXING_TYPES:
11768 // FIXME: This could use the proven type if we had the Edge for the
11769 // value. https://bugs.webkit.org/show_bug.cgi?id=155311
11770 speculate(BadType, noValue(), nullptr, isNotInt32(value));
11771 storeType = Output::Store64;
11772 heap = &m_heaps.indexedInt32Properties;
11773 break;
11774
11775 case ALL_DOUBLE_INDEXING_TYPES: {
11776 // FIXME: If the source is ValueRep, we should avoid emitting any
11777 // checks. We could also avoid emitting checks if we had the Edge of
11778 // this value. https://bugs.webkit.org/show_bug.cgi?id=155311
11779
11780 LBasicBlock intCase = m_out.newBlock();
11781 LBasicBlock doubleCase = m_out.newBlock();
11782 LBasicBlock continuation = m_out.newBlock();
11783
11784 m_out.branch(isInt32(value), unsure(intCase), unsure(doubleCase));
11785
11786 LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
11787
11788 ValueFromBlock intResult =
11789 m_out.anchor(m_out.intToDouble(unboxInt32(value)));
11790 m_out.jump(continuation);
11791
11792 m_out.appendTo(doubleCase, continuation);
11793
11794 speculate(BadType, noValue(), nullptr, isNumber(value));
11795 ValueFromBlock doubleResult = m_out.anchor(unboxDouble(value));
11796 m_out.jump(continuation);
11797
11798 m_out.appendTo(continuation, lastNext);
11799 value = m_out.phi(Double, intResult, doubleResult);
11800 storeType = Output::StoreDouble;
11801 heap = &m_heaps.indexedDoubleProperties;
11802 break;
11803 }
11804
11805 case ALL_CONTIGUOUS_INDEXING_TYPES:
11806 storeType = Output::Store64;
11807 heap = &m_heaps.indexedContiguousProperties;
11808 break;
11809
11810 default:
11811 DFG_CRASH(m_graph, m_node, "Invalid indexing type");
11812 break;
11813 }
11814
11815 m_out.store(value, m_out.address(butterfly, heap->at(index)), storeType);
11816
11817 m_out.jump(blocksWithChecks[i]);
11818 m_out.appendTo(
11819 blocksWithChecks[i], i ? blocksWithStores[i - 1] : indexLastNext);
11820 }
11821 }
11822 } else {
11823 // In the easy case where we can do a one-shot allocation, we simply allocate the
11824 // object to directly have the desired structure.
11825 object = allocateObject(structure);
11826 butterfly = nullptr; // Don't have one, don't need one.
11827 }
11828
11829 BitVector setInlineOffsets;
11830 for (PropertyMapEntry entry : structure->getPropertiesConcurrently()) {
11831 for (unsigned i = data.m_properties.size(); i--;) {
11832 PromotedLocationDescriptor descriptor = data.m_properties[i];
11833 if (descriptor.kind() != NamedPropertyPLoc)
11834 continue;
11835 if (m_graph.identifiers()[descriptor.info()] != entry.key)
11836 continue;
11837
11838 LValue base;
11839 if (isInlineOffset(entry.offset)) {
11840 setInlineOffsets.set(entry.offset);
11841 base = object;
11842 } else
11843 base = butterfly;
11844 storeProperty(values[i], base, descriptor.info(), entry.offset);
11845 break;
11846 }
11847 }
11848 for (unsigned i = structure->inlineCapacity(); i--;) {
11849 if (!setInlineOffsets.get(i))
11850 m_out.store64(m_out.int64Zero, m_out.address(m_heaps.properties.atAnyNumber(), object, offsetRelativeToBase(i)));
11851 }
11852
11853 results.append(m_out.anchor(object));
11854 m_out.jump(outerContinuation);
11855 }
11856
11857 m_out.appendTo(dummyDefault, outerContinuation);
11858 m_out.unreachable();
11859
11860 m_out.appendTo(outerContinuation, outerLastNext);
11861 setJSValue(m_out.phi(pointerType(), results));
11862 mutatorFence();
11863 }
11864
11865 void compileMaterializeCreateActivation()
11866 {
11867 ObjectMaterializationData& data = m_node->objectMaterializationData();
11868
11869 Vector<LValue, 8> values;
11870 for (unsigned i = 0; i < data.m_properties.size(); ++i)
11871 values.append(lowJSValue(m_graph.varArgChild(m_node, 2 + i)));
11872
11873 LValue scope = lowCell(m_graph.varArgChild(m_node, 1));
11874 SymbolTable* table = m_node->castOperand<SymbolTable*>();
11875 RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure());
11876
11877 LBasicBlock slowPath = m_out.newBlock();
11878 LBasicBlock continuation = m_out.newBlock();
11879
11880 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
11881
11882 LValue fastObject = allocateObject<JSLexicalEnvironment>(
11883 JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
11884
11885 m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
11886 m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
11887
11888
11889 ValueFromBlock fastResult = m_out.anchor(fastObject);
11890 m_out.jump(continuation);
11891
11892 m_out.appendTo(slowPath, continuation);
11893 // We ensure allocation sinking explictly sets bottom values for all field members.
11894 // Therefore, it doesn't matter what JSValue we pass in as the initialization value
11895 // because all fields will be overwritten.
11896 // FIXME: It may be worth creating an operation that calls a constructor on JSLexicalEnvironment that
11897 // doesn't initialize every slot because we are guaranteed to do that here.
11898 VM& vm = this->vm();
11899 LValue callResult = lazySlowPath(
11900 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
11901 return createLazyCallGenerator(vm,
11902 operationCreateActivationDirect, locations[0].directGPR(), &vm,
11903 CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(),
11904 CCallHelpers::TrustedImmPtr(table),
11905 CCallHelpers::TrustedImm64(JSValue::encode(jsUndefined())));
11906 }, scope);
11907 ValueFromBlock slowResult = m_out.anchor(callResult);
11908 m_out.jump(continuation);
11909
11910 m_out.appendTo(continuation, lastNext);
11911 LValue activation = m_out.phi(pointerType(), fastResult, slowResult);
11912 RELEASE_ASSERT(data.m_properties.size() == table->scopeSize());
11913 for (unsigned i = 0; i < data.m_properties.size(); ++i) {
11914 PromotedLocationDescriptor descriptor = data.m_properties[i];
11915 ASSERT(descriptor.kind() == ClosureVarPLoc);
11916 m_out.store64(
11917 values[i], activation,
11918 m_heaps.JSLexicalEnvironment_variables[descriptor.info()]);
11919 }
11920
11921 if (validationEnabled()) {
11922 // Validate to make sure every slot in the scope has one value.
11923 ConcurrentJSLocker locker(table->m_lock);
11924 for (auto iter = table->begin(locker), end = table->end(locker); iter != end; ++iter) {
11925 bool found = false;
11926 for (unsigned i = 0; i < data.m_properties.size(); ++i) {
11927 PromotedLocationDescriptor descriptor = data.m_properties[i];
11928 ASSERT(descriptor.kind() == ClosureVarPLoc);
11929 if (iter->value.scopeOffset().offset() == descriptor.info()) {
11930 found = true;
11931 break;
11932 }
11933 }
11934 ASSERT_UNUSED(found, found);
11935 }
11936 }
11937
11938 mutatorFence();
11939 setJSValue(activation);
11940 }
11941
11942 void compileCheckTraps()
11943 {
11944 ASSERT(Options::usePollingTraps());
11945 LBasicBlock needTrapHandling = m_out.newBlock();
11946 LBasicBlock continuation = m_out.newBlock();
11947
11948 LValue state = m_out.load8ZeroExt32(m_out.absolute(vm().needTrapHandlingAddress()));
11949 m_out.branch(m_out.isZero32(state),
11950 usually(continuation), rarely(needTrapHandling));
11951
11952 LBasicBlock lastNext = m_out.appendTo(needTrapHandling, continuation);
11953
11954 VM& vm = this->vm();
11955 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
11956 lazySlowPath(
11957 [=, &vm] (const Vector<Location>&) -> RefPtr<LazySlowPath::Generator> {
11958 return createLazyCallGenerator(vm, operationHandleTraps, InvalidGPRReg, globalObject);
11959 });
11960 m_out.jump(continuation);
11961
11962 m_out.appendTo(continuation, lastNext);
11963 }
11964
11965 void compileRegExpExec()
11966 {
11967 LValue globalObject = lowCell(m_node->child1());
11968
11969 if (m_node->child2().useKind() == RegExpObjectUse) {
11970 LValue base = lowRegExpObject(m_node->child2());
11971
11972 if (m_node->child3().useKind() == StringUse) {
11973 LValue argument = lowString(m_node->child3());
11974 LValue result = vmCall(Int64, operationRegExpExecString, globalObject, base, argument);
11975 setJSValue(result);
11976 return;
11977 }
11978
11979 LValue argument = lowJSValue(m_node->child3());
11980 LValue result = vmCall(Int64, operationRegExpExec, globalObject, base, argument);
11981 setJSValue(result);
11982 return;
11983 }
11984
11985 LValue base = lowJSValue(m_node->child2());
11986 LValue argument = lowJSValue(m_node->child3());
11987 LValue result = vmCall(Int64, operationRegExpExecGeneric, globalObject, base, argument);
11988 setJSValue(result);
11989 }
11990
11991 void compileRegExpExecNonGlobalOrSticky()
11992 {
11993 LValue globalObject = lowCell(m_node->child1());
11994 LValue argument = lowString(m_node->child2());
11995 LValue result = vmCall(Int64, operationRegExpExecNonGlobalOrSticky, globalObject, frozenPointer(m_node->cellOperand()), argument);
11996 setJSValue(result);
11997 }
11998
11999 void compileRegExpMatchFastGlobal()
12000 {
12001 LValue globalObject = lowCell(m_node->child1());
12002 LValue argument = lowString(m_node->child2());
12003 LValue result = vmCall(Int64, operationRegExpMatchFastGlobalString, globalObject, frozenPointer(m_node->cellOperand()), argument);
12004 setJSValue(result);
12005 }
12006
12007 void compileRegExpTest()
12008 {
12009 LValue globalObject = lowCell(m_node->child1());
12010
12011 if (m_node->child2().useKind() == RegExpObjectUse) {
12012 LValue base = lowRegExpObject(m_node->child2());
12013
12014 if (m_node->child3().useKind() == StringUse) {
12015 LValue argument = lowString(m_node->child3());
12016 LValue result = vmCall(Int32, operationRegExpTestString, globalObject, base, argument);
12017 setBoolean(result);
12018 return;
12019 }
12020
12021 LValue argument = lowJSValue(m_node->child3());
12022 LValue result = vmCall(Int32, operationRegExpTest, globalObject, base, argument);
12023 setBoolean(result);
12024 return;
12025 }
12026
12027 LValue base = lowJSValue(m_node->child2());
12028 LValue argument = lowJSValue(m_node->child3());
12029 LValue result = vmCall(Int32, operationRegExpTestGeneric, globalObject, base, argument);
12030 setBoolean(result);
12031 }
12032
12033 void compileRegExpMatchFast()
12034 {
12035 LValue globalObject = lowCell(m_node->child1());
12036 LValue base = lowRegExpObject(m_node->child2());
12037 LValue argument = lowString(m_node->child3());
12038 LValue result = vmCall(Int64, operationRegExpMatchFastString, globalObject, base, argument);
12039 setJSValue(result);
12040 }
12041
12042 void compileNewRegexp()
12043 {
12044 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12045 FrozenValue* regexp = m_node->cellOperand();
12046 LValue lastIndex = lowJSValue(m_node->child1());
12047 ASSERT(regexp->cell()->inherits<RegExp>(vm()));
12048
12049 LBasicBlock slowCase = m_out.newBlock();
12050 LBasicBlock continuation = m_out.newBlock();
12051
12052 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowCase);
12053
12054 auto structure = m_graph.registerStructure(globalObject->regExpStructure());
12055 LValue fastResultValue = allocateObject<RegExpObject>(structure, m_out.intPtrZero, slowCase);
12056 m_out.storePtr(frozenPointer(regexp), fastResultValue, m_heaps.RegExpObject_regExpAndLastIndexIsNotWritableFlag);
12057 m_out.store64(lastIndex, fastResultValue, m_heaps.RegExpObject_lastIndex);
12058 mutatorFence();
12059 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
12060 m_out.jump(continuation);
12061
12062 m_out.appendTo(slowCase, continuation);
12063 VM& vm = this->vm();
12064 RegExp* regexpCell = regexp->cast<RegExp*>();
12065 LValue slowResultValue = lazySlowPath(
12066 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
12067 return createLazyCallGenerator(vm,
12068 operationNewRegexpWithLastIndex, locations[0].directGPR(), globalObject,
12069 CCallHelpers::TrustedImmPtr(regexpCell), locations[1].directGPR());
12070 }, lastIndex);
12071 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
12072 m_out.jump(continuation);
12073
12074 m_out.appendTo(continuation, lastNext);
12075 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
12076 }
12077
12078 void compileSetFunctionName()
12079 {
12080 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12081 vmCall(Void, operationSetFunctionName, weakPointer(globalObject),
12082 lowCell(m_node->child1()), lowJSValue(m_node->child2()));
12083 }
12084
12085 void compileStringReplace()
12086 {
12087 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12088 if (m_node->child1().useKind() == StringUse
12089 && m_node->child2().useKind() == RegExpObjectUse
12090 && m_node->child3().useKind() == StringUse) {
12091
12092 if (JSString* replace = m_node->child3()->dynamicCastConstant<JSString*>(vm())) {
12093 if (!replace->length()) {
12094 LValue string = lowString(m_node->child1());
12095 LValue regExp = lowRegExpObject(m_node->child2());
12096
12097 LValue result = vmCall(pointerType(), operationStringProtoFuncReplaceRegExpEmptyStr, weakPointer(globalObject), string, regExp);
12098
12099 setJSValue(result);
12100 return;
12101 }
12102 }
12103
12104 LValue string = lowString(m_node->child1());
12105 LValue regExp = lowRegExpObject(m_node->child2());
12106 LValue replace = lowString(m_node->child3());
12107
12108 LValue result = vmCall(pointerType(), operationStringProtoFuncReplaceRegExpString, weakPointer(globalObject), string, regExp, replace);
12109
12110 setJSValue(result);
12111 return;
12112 }
12113
12114 LValue search;
12115 if (m_node->child2().useKind() == StringUse)
12116 search = lowString(m_node->child2());
12117 else
12118 search = lowJSValue(m_node->child2());
12119
12120 LValue result = vmCall(
12121 pointerType(), operationStringProtoFuncReplaceGeneric,
12122 weakPointer(globalObject),
12123 lowJSValue(m_node->child1()), search,
12124 lowJSValue(m_node->child3()));
12125
12126 setJSValue(result);
12127 }
12128
12129 void compileGetRegExpObjectLastIndex()
12130 {
12131 setJSValue(m_out.load64(lowRegExpObject(m_node->child1()), m_heaps.RegExpObject_lastIndex));
12132 }
12133
12134 void compileSetRegExpObjectLastIndex()
12135 {
12136 if (!m_node->ignoreLastIndexIsWritable()) {
12137 LValue regExp = lowRegExpObject(m_node->child1());
12138 LValue value = lowJSValue(m_node->child2());
12139
12140 speculate(
12141 ExoticObjectMode, noValue(), nullptr,
12142 m_out.testNonZeroPtr(
12143 m_out.loadPtr(regExp, m_heaps.RegExpObject_regExpAndLastIndexIsNotWritableFlag),
12144 m_out.constIntPtr(RegExpObject::lastIndexIsNotWritableFlag)));
12145
12146 m_out.store64(value, regExp, m_heaps.RegExpObject_lastIndex);
12147 return;
12148 }
12149
12150 m_out.store64(lowJSValue(m_node->child2()), lowCell(m_node->child1()), m_heaps.RegExpObject_lastIndex);
12151 }
12152
12153 void compileLogShadowChickenPrologue()
12154 {
12155 LValue packet = ensureShadowChickenPacket();
12156 LValue scope = lowCell(m_node->child1());
12157
12158 m_out.storePtr(m_callFrame, packet, m_heaps.ShadowChicken_Packet_frame);
12159 m_out.storePtr(m_out.loadPtr(addressFor(0)), packet, m_heaps.ShadowChicken_Packet_callerFrame);
12160 m_out.storePtr(m_out.loadPtr(payloadFor(CallFrameSlot::callee)), packet, m_heaps.ShadowChicken_Packet_callee);
12161 m_out.storePtr(scope, packet, m_heaps.ShadowChicken_Packet_scope);
12162 }
12163
12164 void compileLogShadowChickenTail()
12165 {
12166 LValue packet = ensureShadowChickenPacket();
12167 LValue thisValue = lowJSValue(m_node->child1());
12168 LValue scope = lowCell(m_node->child2());
12169 CallSiteIndex callSiteIndex = m_ftlState.jitCode->common.addCodeOrigin(m_node->origin.semantic);
12170
12171 m_out.storePtr(m_callFrame, packet, m_heaps.ShadowChicken_Packet_frame);
12172 m_out.storePtr(m_out.constIntPtr(bitwise_cast<intptr_t>(ShadowChicken::Packet::tailMarker())), packet, m_heaps.ShadowChicken_Packet_callee);
12173 m_out.store64(thisValue, packet, m_heaps.ShadowChicken_Packet_thisValue);
12174 m_out.storePtr(scope, packet, m_heaps.ShadowChicken_Packet_scope);
12175 // We don't want the CodeBlock to have a weak pointer to itself because
12176 // that would cause it to always get collected.
12177 m_out.storePtr(m_out.constIntPtr(bitwise_cast<intptr_t>(codeBlock())), packet, m_heaps.ShadowChicken_Packet_codeBlock);
12178 m_out.store32(m_out.constInt32(callSiteIndex.bits()), packet, m_heaps.ShadowChicken_Packet_callSiteIndex);
12179 }
12180
12181 void compileRecordRegExpCachedResult()
12182 {
12183 Edge globalObjectEdge = m_graph.varArgChild(m_node, 0);
12184 Edge regExpEdge = m_graph.varArgChild(m_node, 1);
12185 Edge stringEdge = m_graph.varArgChild(m_node, 2);
12186 Edge startEdge = m_graph.varArgChild(m_node, 3);
12187 Edge endEdge = m_graph.varArgChild(m_node, 4);
12188
12189 LValue globalObject = lowCell(globalObjectEdge);
12190 LValue regExp = lowCell(regExpEdge);
12191 LValue string = lowCell(stringEdge);
12192 LValue start = lowInt32(startEdge);
12193 LValue end = lowInt32(endEdge);
12194
12195 m_out.storePtr(regExp, globalObject, m_heaps.JSGlobalObject_regExpGlobalData_cachedResult_lastRegExp);
12196 m_out.storePtr(string, globalObject, m_heaps.JSGlobalObject_regExpGlobalData_cachedResult_lastInput);
12197 m_out.store32(start, globalObject, m_heaps.JSGlobalObject_regExpGlobalData_cachedResult_result_start);
12198 m_out.store32(end, globalObject, m_heaps.JSGlobalObject_regExpGlobalData_cachedResult_result_end);
12199 m_out.store32As8(
12200 m_out.constInt32(0),
12201 m_out.address(globalObject, m_heaps.JSGlobalObject_regExpGlobalData_cachedResult_reified));
12202 }
12203
12204 struct ArgumentsLength {
12205 ArgumentsLength()
12206 : isKnown(false)
12207 , known(UINT_MAX)
12208 , value(nullptr)
12209 {
12210 }
12211
12212 bool isKnown;
12213 unsigned known;
12214 LValue value;
12215 };
12216 ArgumentsLength getArgumentsLength(InlineCallFrame* inlineCallFrame)
12217 {
12218 ArgumentsLength length;
12219
12220 if (inlineCallFrame && !inlineCallFrame->isVarargs()) {
12221 length.known = inlineCallFrame->argumentCountIncludingThis - 1;
12222 length.isKnown = true;
12223 length.value = m_out.constInt32(length.known);
12224 } else {
12225 length.known = UINT_MAX;
12226 length.isKnown = false;
12227
12228 VirtualRegister argumentCountRegister;
12229 if (!inlineCallFrame)
12230 argumentCountRegister = VirtualRegister(CallFrameSlot::argumentCount);
12231 else
12232 argumentCountRegister = inlineCallFrame->argumentCountRegister;
12233 length.value = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
12234 }
12235
12236 return length;
12237 }
12238
12239 ArgumentsLength getArgumentsLength()
12240 {
12241 return getArgumentsLength(m_node->origin.semantic.inlineCallFrame());
12242 }
12243
12244 LValue getCurrentCallee()
12245 {
12246 if (InlineCallFrame* frame = m_node->origin.semantic.inlineCallFrame()) {
12247 if (frame->isClosureCall)
12248 return m_out.loadPtr(addressFor(frame->calleeRecovery.virtualRegister()));
12249 return weakPointer(frame->calleeRecovery.constant().asCell());
12250 }
12251 return m_out.loadPtr(addressFor(CallFrameSlot::callee));
12252 }
12253
12254 LValue getArgumentsStart(InlineCallFrame* inlineCallFrame, unsigned offset = 0)
12255 {
12256 VirtualRegister start = AssemblyHelpers::argumentsStart(inlineCallFrame) + offset;
12257 return addressFor(start).value();
12258 }
12259
12260 LValue getArgumentsStart()
12261 {
12262 return getArgumentsStart(m_node->origin.semantic.inlineCallFrame());
12263 }
12264
12265 template<typename Functor>
12266 void checkStructure(
12267 LValue structureDiscriminant, const FormattedValue& formattedValue, ExitKind exitKind,
12268 const RegisteredStructureSet& set, const Functor& weakStructureDiscriminant)
12269 {
12270 if (set.isEmpty()) {
12271 terminate(exitKind);
12272 return;
12273 }
12274
12275 if (set.size() == 1) {
12276 speculate(
12277 exitKind, formattedValue, 0,
12278 m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set[0])));
12279 return;
12280 }
12281
12282 LBasicBlock continuation = m_out.newBlock();
12283
12284 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
12285 for (unsigned i = 0; i < set.size() - 1; ++i) {
12286 LBasicBlock nextStructure = m_out.newBlock();
12287 m_out.branch(
12288 m_out.equal(structureDiscriminant, weakStructureDiscriminant(set[i])),
12289 unsure(continuation), unsure(nextStructure));
12290 m_out.appendTo(nextStructure);
12291 }
12292
12293 speculate(
12294 exitKind, formattedValue, 0,
12295 m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set.last())));
12296
12297 m_out.jump(continuation);
12298 m_out.appendTo(continuation, lastNext);
12299 }
12300
12301 LValue numberOrNotCellToInt32(Edge edge, LValue value)
12302 {
12303 LBasicBlock intCase = m_out.newBlock();
12304 LBasicBlock notIntCase = m_out.newBlock();
12305 LBasicBlock doubleCase = 0;
12306 LBasicBlock notNumberCase = 0;
12307 if (edge.useKind() == NotCellUse) {
12308 doubleCase = m_out.newBlock();
12309 notNumberCase = m_out.newBlock();
12310 }
12311 LBasicBlock continuation = m_out.newBlock();
12312
12313 Vector<ValueFromBlock> results;
12314
12315 m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase));
12316
12317 LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase);
12318 results.append(m_out.anchor(unboxInt32(value)));
12319 m_out.jump(continuation);
12320
12321 if (edge.useKind() == NumberUse) {
12322 m_out.appendTo(notIntCase, continuation);
12323 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value));
12324 results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
12325 m_out.jump(continuation);
12326 } else {
12327 m_out.appendTo(notIntCase, doubleCase);
12328 m_out.branch(
12329 isCellOrMisc(value, provenType(edge)), unsure(notNumberCase), unsure(doubleCase));
12330
12331 m_out.appendTo(doubleCase, notNumberCase);
12332 results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
12333 m_out.jump(continuation);
12334
12335 m_out.appendTo(notNumberCase, continuation);
12336
12337 FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCellCheck, isCell(value));
12338
12339 LValue specialResult = m_out.select(
12340 m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))),
12341 m_out.int32One, m_out.int32Zero);
12342 results.append(m_out.anchor(specialResult));
12343 m_out.jump(continuation);
12344 }
12345
12346 m_out.appendTo(continuation, lastNext);
12347 return m_out.phi(Int32, results);
12348 }
12349
12350 LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset)
12351 {
12352 return m_out.load64(addressOfProperty(storage, identifierNumber, offset));
12353 }
12354
12355 void storeProperty(
12356 LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset)
12357 {
12358 m_out.store64(value, addressOfProperty(storage, identifierNumber, offset));
12359 }
12360
12361 TypedPointer addressOfProperty(
12362 LValue storage, unsigned identifierNumber, PropertyOffset offset)
12363 {
12364 return m_out.address(
12365 m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset));
12366 }
12367
12368 LValue storageForTransition(
12369 LValue object, PropertyOffset offset,
12370 Structure* previousStructure, Structure* nextStructure)
12371 {
12372 if (isInlineOffset(offset))
12373 return object;
12374
12375 if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity())
12376 return m_out.loadPtr(object, m_heaps.JSObject_butterfly);
12377
12378 LValue result;
12379 if (!previousStructure->outOfLineCapacity())
12380 result = allocatePropertyStorage(object, previousStructure);
12381 else {
12382 result = reallocatePropertyStorage(
12383 object, m_out.loadPtr(object, m_heaps.JSObject_butterfly),
12384 previousStructure, nextStructure);
12385 }
12386
12387 nukeStructureAndSetButterfly(result, object);
12388 return result;
12389 }
12390
12391 void initializeArrayElements(LValue indexingType, LValue begin, LValue end, LValue butterfly)
12392 {
12393
12394 if (begin == end)
12395 return;
12396
12397 if (indexingType->hasInt32()) {
12398 IndexingType rawIndexingType = static_cast<IndexingType>(indexingType->asInt32());
12399 if (hasUndecided(rawIndexingType))
12400 return;
12401 IndexedAbstractHeap* heap = m_heaps.forIndexingType(rawIndexingType);
12402 DFG_ASSERT(m_graph, m_node, heap);
12403
12404 LValue hole;
12405 if (hasDouble(rawIndexingType))
12406 hole = m_out.constInt64(bitwise_cast<int64_t>(PNaN));
12407 else
12408 hole = m_out.constInt64(JSValue::encode(JSValue()));
12409
12410 splatWords(butterfly, begin, end, hole, heap->atAnyIndex());
12411 } else {
12412 LValue hole = m_out.select(
12413 m_out.equal(m_out.bitAnd(indexingType, m_out.constInt32(IndexingShapeMask)), m_out.constInt32(DoubleShape)),
12414 m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
12415 m_out.constInt64(JSValue::encode(JSValue())));
12416 splatWords(butterfly, begin, end, hole, m_heaps.root);
12417 }
12418 }
12419
12420 void splatWords(LValue base, LValue begin, LValue end, LValue value, const AbstractHeap& heap)
12421 {
12422 const uint64_t unrollingLimit = 10;
12423 if (begin->hasInt() && end->hasInt()) {
12424 uint64_t beginConst = static_cast<uint64_t>(begin->asInt());
12425 uint64_t endConst = static_cast<uint64_t>(end->asInt());
12426
12427 if (endConst - beginConst <= unrollingLimit) {
12428 for (uint64_t i = beginConst; i < endConst; ++i) {
12429 LValue pointer = m_out.add(base, m_out.constIntPtr(i * sizeof(uint64_t)));
12430 m_out.store64(value, TypedPointer(heap, pointer));
12431 }
12432 return;
12433 }
12434 }
12435
12436 LBasicBlock initLoop = m_out.newBlock();
12437 LBasicBlock initDone = m_out.newBlock();
12438
12439 LBasicBlock lastNext = m_out.insertNewBlocksBefore(initLoop);
12440
12441 ValueFromBlock originalIndex = m_out.anchor(end);
12442 ValueFromBlock originalPointer = m_out.anchor(
12443 m_out.add(base, m_out.shl(m_out.signExt32ToPtr(begin), m_out.constInt32(3))));
12444 m_out.branch(m_out.notEqual(end, begin), unsure(initLoop), unsure(initDone));
12445
12446 m_out.appendTo(initLoop, initDone);
12447 LValue index = m_out.phi(Int32, originalIndex);
12448 LValue pointer = m_out.phi(pointerType(), originalPointer);
12449
12450 m_out.store64(value, TypedPointer(heap, pointer));
12451
12452 LValue nextIndex = m_out.sub(index, m_out.int32One);
12453 m_out.addIncomingToPhi(index, m_out.anchor(nextIndex));
12454 m_out.addIncomingToPhi(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight)));
12455 m_out.branch(
12456 m_out.notEqual(nextIndex, begin), unsure(initLoop), unsure(initDone));
12457
12458 m_out.appendTo(initDone, lastNext);
12459 }
12460
12461 LValue allocatePropertyStorage(LValue object, Structure* previousStructure)
12462 {
12463 if (previousStructure->couldHaveIndexingHeader()) {
12464 return vmCall(
12465 pointerType(),
12466 operationAllocateComplexPropertyStorageWithInitialCapacity,
12467 m_vmValue, object);
12468 }
12469
12470 LValue result = allocatePropertyStorageWithSizeImpl(initialOutOfLineCapacity);
12471
12472 splatWords(
12473 result,
12474 m_out.constInt32(-initialOutOfLineCapacity - 1), m_out.constInt32(-1),
12475 m_out.int64Zero, m_heaps.properties.atAnyNumber());
12476
12477 return result;
12478 }
12479
12480 LValue reallocatePropertyStorage(
12481 LValue object, LValue oldStorage, Structure* previous, Structure* next)
12482 {
12483 size_t oldSize = previous->outOfLineCapacity();
12484 size_t newSize = oldSize * outOfLineGrowthFactor;
12485
12486 ASSERT_UNUSED(next, newSize == next->outOfLineCapacity());
12487
12488 if (previous->couldHaveIndexingHeader()) {
12489 LValue newAllocSize = m_out.constIntPtr(newSize);
12490 return vmCall(pointerType(), operationAllocateComplexPropertyStorage, m_vmValue, object, newAllocSize);
12491 }
12492
12493 LValue result = allocatePropertyStorageWithSizeImpl(newSize);
12494
12495 ptrdiff_t headerSize = -sizeof(IndexingHeader) - sizeof(void*);
12496 ptrdiff_t endStorage = headerSize - static_cast<ptrdiff_t>(oldSize * sizeof(JSValue));
12497
12498 for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) {
12499 LValue loaded =
12500 m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset));
12501 m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset));
12502 }
12503
12504 splatWords(
12505 result,
12506 m_out.constInt32(-newSize - 1), m_out.constInt32(-oldSize - 1),
12507 m_out.int64Zero, m_heaps.properties.atAnyNumber());
12508
12509 return result;
12510 }
12511
12512 LValue allocatePropertyStorageWithSizeImpl(size_t sizeInValues)
12513 {
12514 LBasicBlock slowPath = m_out.newBlock();
12515 LBasicBlock continuation = m_out.newBlock();
12516
12517 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
12518
12519 size_t sizeInBytes = sizeInValues * sizeof(JSValue);
12520 Allocator allocator = vm().jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(sizeInBytes, AllocatorForMode::AllocatorIfExists);
12521 LValue startOfStorage = allocateHeapCell(
12522 m_out.constIntPtr(allocator.localAllocator()), slowPath);
12523 ValueFromBlock fastButterfly = m_out.anchor(
12524 m_out.add(m_out.constIntPtr(sizeInBytes + sizeof(IndexingHeader)), startOfStorage));
12525 m_out.jump(continuation);
12526
12527 m_out.appendTo(slowPath, continuation);
12528
12529 LValue slowButterflyValue;
12530 VM& vm = this->vm();
12531 if (sizeInValues == initialOutOfLineCapacity) {
12532 slowButterflyValue = lazySlowPath(
12533 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
12534 return createLazyCallGenerator(vm,
12535 operationAllocateSimplePropertyStorageWithInitialCapacity,
12536 locations[0].directGPR(), &vm);
12537 });
12538 } else {
12539 slowButterflyValue = lazySlowPath(
12540 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
12541 return createLazyCallGenerator(vm,
12542 operationAllocateSimplePropertyStorage, locations[0].directGPR(), &vm,
12543 CCallHelpers::TrustedImmPtr(sizeInValues));
12544 });
12545 }
12546 ValueFromBlock slowButterfly = m_out.anchor(slowButterflyValue);
12547
12548 m_out.jump(continuation);
12549
12550 m_out.appendTo(continuation, lastNext);
12551
12552 return m_out.phi(pointerType(), fastButterfly, slowButterfly);
12553 }
12554
12555 LValue getById(LValue base, AccessType type)
12556 {
12557 Node* node = m_node;
12558 UniquedStringImpl* uid = m_graph.identifiers()[node->identifierNumber()];
12559
12560 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
12561 patchpoint->appendSomeRegister(base);
12562 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
12563 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
12564
12565 // FIXME: If this is a GetByIdFlush/GetByIdDirectFlush, we might get some performance boost if we claim that it
12566 // clobbers volatile registers late. It's not necessary for correctness, though, since the
12567 // IC code is super smart about saving registers.
12568 // https://bugs.webkit.org/show_bug.cgi?id=152848
12569
12570 patchpoint->clobber(RegisterSet::macroScratchRegisters());
12571
12572 RefPtr<PatchpointExceptionHandle> exceptionHandle =
12573 preparePatchpointForExceptions(patchpoint);
12574
12575 State* state = &m_ftlState;
12576 patchpoint->setGenerator(
12577 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
12578 AllowMacroScratchRegisterUsage allowScratch(jit);
12579
12580 CallSiteIndex callSiteIndex =
12581 state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
12582
12583 // This is the direct exit target for operation calls.
12584 Box<CCallHelpers::JumpList> exceptions =
12585 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
12586
12587 // This is the exit for call IC's created by the getById for getters. We don't have
12588 // to do anything weird other than call this, since it will associate the exit with
12589 // the callsite index.
12590 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
12591
12592 auto generator = Box<JITGetByIdGenerator>::create(
12593 jit.codeBlock(), node->origin.semantic, callSiteIndex,
12594 params.unavailableRegisters(), uid, JSValueRegs(params[1].gpr()),
12595 JSValueRegs(params[0].gpr()), type);
12596
12597 generator->generateFastPath(jit);
12598 CCallHelpers::Label done = jit.label();
12599
12600 params.addLatePath(
12601 [=] (CCallHelpers& jit) {
12602 AllowMacroScratchRegisterUsage allowScratch(jit);
12603
12604 J_JITOperation_GSsiJI optimizationFunction = appropriateOptimizingGetByIdFunction(type);
12605
12606 generator->slowPathJump().link(&jit);
12607 CCallHelpers::Label slowPathBegin = jit.label();
12608 CCallHelpers::Call slowPathCall = callOperation(
12609 *state, params.unavailableRegisters(), jit, node->origin.semantic,
12610 exceptions.get(), optimizationFunction, params[0].gpr(),
12611 jit.codeBlock()->globalObjectFor(node->origin.semantic),
12612 CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(),
12613 CCallHelpers::TrustedImmPtr(uid)).call();
12614 jit.jump().linkTo(done, &jit);
12615
12616 generator->reportSlowPathCall(slowPathBegin, slowPathCall);
12617
12618 jit.addLinkTask(
12619 [=] (LinkBuffer& linkBuffer) {
12620 generator->finalize(linkBuffer, linkBuffer);
12621 });
12622 });
12623 });
12624
12625 return patchpoint;
12626 }
12627
12628 LValue getByIdWithThis(LValue base, LValue thisValue)
12629 {
12630 Node* node = m_node;
12631 UniquedStringImpl* uid = m_graph.identifiers()[node->identifierNumber()];
12632
12633 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
12634 patchpoint->appendSomeRegister(base);
12635 patchpoint->appendSomeRegister(thisValue);
12636 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
12637 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
12638
12639 patchpoint->clobber(RegisterSet::macroScratchRegisters());
12640
12641 RefPtr<PatchpointExceptionHandle> exceptionHandle =
12642 preparePatchpointForExceptions(patchpoint);
12643
12644 State* state = &m_ftlState;
12645 patchpoint->setGenerator(
12646 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
12647 AllowMacroScratchRegisterUsage allowScratch(jit);
12648
12649 CallSiteIndex callSiteIndex =
12650 state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic);
12651
12652 // This is the direct exit target for operation calls.
12653 Box<CCallHelpers::JumpList> exceptions =
12654 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
12655
12656 // This is the exit for call IC's created by the getById for getters. We don't have
12657 // to do anything weird other than call this, since it will associate the exit with
12658 // the callsite index.
12659 exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex);
12660
12661 auto generator = Box<JITGetByIdWithThisGenerator>::create(
12662 jit.codeBlock(), node->origin.semantic, callSiteIndex,
12663 params.unavailableRegisters(), uid, JSValueRegs(params[0].gpr()),
12664 JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()));
12665
12666 generator->generateFastPath(jit);
12667 CCallHelpers::Label done = jit.label();
12668
12669 params.addLatePath(
12670 [=] (CCallHelpers& jit) {
12671 AllowMacroScratchRegisterUsage allowScratch(jit);
12672
12673 J_JITOperation_GSsiJJI optimizationFunction = operationGetByIdWithThisOptimize;
12674
12675 generator->slowPathJump().link(&jit);
12676 CCallHelpers::Label slowPathBegin = jit.label();
12677 CCallHelpers::Call slowPathCall = callOperation(
12678 *state, params.unavailableRegisters(), jit, node->origin.semantic,
12679 exceptions.get(), optimizationFunction, params[0].gpr(),
12680 jit.codeBlock()->globalObjectFor(node->origin.semantic),
12681 CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(),
12682 params[2].gpr(), CCallHelpers::TrustedImmPtr(uid)).call();
12683 jit.jump().linkTo(done, &jit);
12684
12685 generator->reportSlowPathCall(slowPathBegin, slowPathCall);
12686
12687 jit.addLinkTask(
12688 [=] (LinkBuffer& linkBuffer) {
12689 generator->finalize(linkBuffer, linkBuffer);
12690 });
12691 });
12692 });
12693
12694 return patchpoint;
12695 }
12696
12697 LValue isFastTypedArray(LValue object)
12698 {
12699 return m_out.equal(
12700 m_out.load32(object, m_heaps.JSArrayBufferView_mode),
12701 m_out.constInt32(FastTypedArray));
12702 }
12703
12704 TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge, ptrdiff_t offset = 0)
12705 {
12706 return m_out.baseIndex(
12707 heap, storage, m_out.zeroExtPtr(index), provenValue(edge), offset);
12708 }
12709
12710 template<typename IntFunctor, typename DoubleFunctor>
12711 void compare(
12712 const IntFunctor& intFunctor, const DoubleFunctor& doubleFunctor,
12713 C_JITOperation_TT stringIdentFunction,
12714 C_JITOperation_B_GJssJss stringFunction,
12715 S_JITOperation_GJJ fallbackFunction)
12716 {
12717 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12718 if (m_node->isBinaryUseKind(Int32Use)) {
12719 LValue left = lowInt32(m_node->child1());
12720 LValue right = lowInt32(m_node->child2());
12721 setBoolean(intFunctor(left, right));
12722 return;
12723 }
12724
12725 if (m_node->isBinaryUseKind(Int52RepUse)) {
12726 Int52Kind kind;
12727 LValue left = lowWhicheverInt52(m_node->child1(), kind);
12728 LValue right = lowInt52(m_node->child2(), kind);
12729 setBoolean(intFunctor(left, right));
12730 return;
12731 }
12732
12733 if (m_node->isBinaryUseKind(DoubleRepUse)) {
12734 LValue left = lowDouble(m_node->child1());
12735 LValue right = lowDouble(m_node->child2());
12736 setBoolean(doubleFunctor(left, right));
12737 return;
12738 }
12739
12740 if (m_node->isBinaryUseKind(StringIdentUse)) {
12741 LValue left = lowStringIdent(m_node->child1());
12742 LValue right = lowStringIdent(m_node->child2());
12743 setBoolean(m_out.callWithoutSideEffects(Int32, stringIdentFunction, left, right));
12744 return;
12745 }
12746
12747 if (m_node->isBinaryUseKind(StringUse)) {
12748 LValue left = lowCell(m_node->child1());
12749 LValue right = lowCell(m_node->child2());
12750 speculateString(m_node->child1(), left);
12751 speculateString(m_node->child2(), right);
12752
12753 LValue result = vmCall(
12754 Int32, stringFunction,
12755 weakPointer(globalObject), left, right);
12756 setBoolean(result);
12757 return;
12758 }
12759
12760 DFG_ASSERT(m_graph, m_node, m_node->isBinaryUseKind(UntypedUse), m_node->child1().useKind(), m_node->child2().useKind());
12761 nonSpeculativeCompare(intFunctor, fallbackFunction);
12762 }
12763
12764 void compileStringSlice()
12765 {
12766 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12767 LBasicBlock lengthCheckCase = m_out.newBlock();
12768 LBasicBlock emptyCase = m_out.newBlock();
12769 LBasicBlock notEmptyCase = m_out.newBlock();
12770 LBasicBlock oneCharCase = m_out.newBlock();
12771 LBasicBlock is8Bit = m_out.newBlock();
12772 LBasicBlock is16Bit = m_out.newBlock();
12773 LBasicBlock bitsContinuation = m_out.newBlock();
12774 LBasicBlock bigCharacter = m_out.newBlock();
12775 LBasicBlock slowCase = m_out.newBlock();
12776 LBasicBlock ropeSlowCase = m_out.newBlock();
12777 LBasicBlock continuation = m_out.newBlock();
12778
12779 LValue string = lowString(m_node->child1());
12780 LValue start = lowInt32(m_node->child2());
12781 LValue end = nullptr;
12782 if (m_node->child3())
12783 end = lowInt32(m_node->child3());
12784 else
12785 end = m_out.constInt32(std::numeric_limits<int32_t>::max());
12786 m_out.branch(isRopeString(string, m_node->child1()), rarely(ropeSlowCase), usually(lengthCheckCase));
12787
12788 LBasicBlock lastNext = m_out.appendTo(lengthCheckCase, emptyCase);
12789 LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
12790 LValue length = m_out.load32NonNegative(stringImpl, m_heaps.StringImpl_length);
12791 auto range = populateSliceRange(start, end, length);
12792 LValue from = range.first;
12793 LValue to = range.second;
12794 LValue span = m_out.sub(to, from);
12795 m_out.branch(m_out.lessThanOrEqual(span, m_out.int32Zero), unsure(emptyCase), unsure(notEmptyCase));
12796
12797 Vector<ValueFromBlock, 5> results;
12798
12799 m_out.appendTo(emptyCase, notEmptyCase);
12800 results.append(m_out.anchor(weakPointer(jsEmptyString(vm()))));
12801 m_out.jump(continuation);
12802
12803 m_out.appendTo(notEmptyCase, oneCharCase);
12804 m_out.branch(m_out.equal(span, m_out.int32One), unsure(oneCharCase), unsure(slowCase));
12805
12806 m_out.appendTo(oneCharCase, is8Bit);
12807 LValue storage = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data);
12808 m_out.branch(
12809 m_out.testIsZero32(
12810 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
12811 m_out.constInt32(StringImpl::flagIs8Bit())),
12812 unsure(is16Bit), unsure(is8Bit));
12813
12814 m_out.appendTo(is8Bit, is16Bit);
12815 ValueFromBlock char8Bit = m_out.anchor(m_out.load8ZeroExt32(m_out.baseIndex(m_heaps.characters8, storage, m_out.zeroExtPtr(from))));
12816 m_out.jump(bitsContinuation);
12817
12818 m_out.appendTo(is16Bit, bigCharacter);
12819 LValue char16BitValue = m_out.load16ZeroExt32(m_out.baseIndex(m_heaps.characters16, storage, m_out.zeroExtPtr(from)));
12820 ValueFromBlock char16Bit = m_out.anchor(char16BitValue);
12821 m_out.branch(
12822 m_out.above(char16BitValue, m_out.constInt32(maxSingleCharacterString)),
12823 rarely(bigCharacter), usually(bitsContinuation));
12824
12825 m_out.appendTo(bigCharacter, bitsContinuation);
12826 results.append(m_out.anchor(vmCall(
12827 Int64, operationSingleCharacterString,
12828 m_vmValue, char16BitValue)));
12829 m_out.jump(continuation);
12830
12831 m_out.appendTo(bitsContinuation, slowCase);
12832 LValue character = m_out.phi(Int32, char8Bit, char16Bit);
12833 LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
12834 results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex(
12835 m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(character)))));
12836 m_out.jump(continuation);
12837
12838 m_out.appendTo(slowCase, ropeSlowCase);
12839 results.append(m_out.anchor(vmCall(pointerType(), operationStringSubstr, weakPointer(globalObject), string, from, span)));
12840 m_out.jump(continuation);
12841
12842 m_out.appendTo(ropeSlowCase, continuation);
12843 results.append(m_out.anchor(vmCall(pointerType(), operationStringSlice, weakPointer(globalObject), string, start, end)));
12844 m_out.jump(continuation);
12845
12846 m_out.appendTo(continuation, lastNext);
12847 setJSValue(m_out.phi(pointerType(), results));
12848 }
12849
12850 void compileToLowerCase()
12851 {
12852 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12853 LBasicBlock notRope = m_out.newBlock();
12854 LBasicBlock is8Bit = m_out.newBlock();
12855 LBasicBlock loopTop = m_out.newBlock();
12856 LBasicBlock loopBody = m_out.newBlock();
12857 LBasicBlock slowPath = m_out.newBlock();
12858 LBasicBlock continuation = m_out.newBlock();
12859
12860 LValue string = lowString(m_node->child1());
12861 ValueFromBlock startIndex = m_out.anchor(m_out.constInt32(0));
12862 ValueFromBlock startIndexForCall = m_out.anchor(m_out.constInt32(0));
12863 m_out.branch(isRopeString(string, m_node->child1()),
12864 unsure(slowPath), unsure(notRope));
12865
12866 LBasicBlock lastNext = m_out.appendTo(notRope, is8Bit);
12867 LValue impl = m_out.loadPtr(string, m_heaps.JSString_value);
12868 m_out.branch(
12869 m_out.testIsZero32(
12870 m_out.load32(impl, m_heaps.StringImpl_hashAndFlags),
12871 m_out.constInt32(StringImpl::flagIs8Bit())),
12872 unsure(slowPath), unsure(is8Bit));
12873
12874 m_out.appendTo(is8Bit, loopTop);
12875 LValue length = m_out.load32(impl, m_heaps.StringImpl_length);
12876 LValue buffer = m_out.loadPtr(impl, m_heaps.StringImpl_data);
12877 ValueFromBlock fastResult = m_out.anchor(string);
12878 m_out.jump(loopTop);
12879
12880 m_out.appendTo(loopTop, loopBody);
12881 LValue index = m_out.phi(Int32, startIndex);
12882 ValueFromBlock indexFromBlock = m_out.anchor(index);
12883 m_out.branch(m_out.below(index, length),
12884 unsure(loopBody), unsure(continuation));
12885
12886 m_out.appendTo(loopBody, slowPath);
12887
12888 // FIXME: Strings needs to be caged.
12889 // https://bugs.webkit.org/show_bug.cgi?id=174924
12890 LValue byte = m_out.load8ZeroExt32(m_out.baseIndex(m_heaps.characters8, buffer, m_out.zeroExtPtr(index)));
12891 LValue isInvalidAsciiRange = m_out.bitAnd(byte, m_out.constInt32(~0x7F));
12892 LValue isUpperCase = m_out.belowOrEqual(m_out.sub(byte, m_out.constInt32('A')), m_out.constInt32('Z' - 'A'));
12893 LValue isBadCharacter = m_out.bitOr(isInvalidAsciiRange, isUpperCase);
12894 m_out.addIncomingToPhi(index, m_out.anchor(m_out.add(index, m_out.int32One)));
12895 m_out.branch(isBadCharacter, unsure(slowPath), unsure(loopTop));
12896
12897 m_out.appendTo(slowPath, continuation);
12898 LValue slowPathIndex = m_out.phi(Int32, startIndexForCall, indexFromBlock);
12899 ValueFromBlock slowResult = m_out.anchor(vmCall(pointerType(), operationToLowerCase, weakPointer(globalObject), string, slowPathIndex));
12900 m_out.jump(continuation);
12901
12902 m_out.appendTo(continuation, lastNext);
12903 setJSValue(m_out.phi(pointerType(), fastResult, slowResult));
12904 }
12905
12906 void compileNumberToStringWithRadix()
12907 {
12908 bool validRadixIsGuaranteed = false;
12909 if (m_node->child2()->isInt32Constant()) {
12910 int32_t radix = m_node->child2()->asInt32();
12911 if (radix >= 2 && radix <= 36)
12912 validRadixIsGuaranteed = true;
12913 }
12914
12915 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12916 switch (m_node->child1().useKind()) {
12917 case Int32Use:
12918 setJSValue(vmCall(pointerType(), validRadixIsGuaranteed ? operationInt32ToStringWithValidRadix : operationInt32ToString, weakPointer(globalObject), lowInt32(m_node->child1()), lowInt32(m_node->child2())));
12919 break;
12920 case Int52RepUse:
12921 setJSValue(vmCall(pointerType(), validRadixIsGuaranteed ? operationInt52ToStringWithValidRadix : operationInt52ToString, weakPointer(globalObject), lowStrictInt52(m_node->child1()), lowInt32(m_node->child2())));
12922 break;
12923 case DoubleRepUse:
12924 setJSValue(vmCall(pointerType(), validRadixIsGuaranteed ? operationDoubleToStringWithValidRadix : operationDoubleToString, weakPointer(globalObject), lowDouble(m_node->child1()), lowInt32(m_node->child2())));
12925 break;
12926 default:
12927 RELEASE_ASSERT_NOT_REACHED();
12928 }
12929 }
12930
12931 void compileNumberToStringWithValidRadixConstant()
12932 {
12933 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12934 switch (m_node->child1().useKind()) {
12935 case Int32Use:
12936 setJSValue(vmCall(pointerType(), operationInt32ToStringWithValidRadix, weakPointer(globalObject), lowInt32(m_node->child1()), m_out.constInt32(m_node->validRadixConstant())));
12937 break;
12938 case Int52RepUse:
12939 setJSValue(vmCall(pointerType(), operationInt52ToStringWithValidRadix, weakPointer(globalObject), lowStrictInt52(m_node->child1()), m_out.constInt32(m_node->validRadixConstant())));
12940 break;
12941 case DoubleRepUse:
12942 setJSValue(vmCall(pointerType(), operationDoubleToStringWithValidRadix, weakPointer(globalObject), lowDouble(m_node->child1()), m_out.constInt32(m_node->validRadixConstant())));
12943 break;
12944 default:
12945 RELEASE_ASSERT_NOT_REACHED();
12946 }
12947 }
12948
12949 void compileResolveScopeForHoistingFuncDeclInEval()
12950 {
12951 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12952 UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
12953 setJSValue(vmCall(pointerType(), operationResolveScopeForHoistingFuncDeclInEval, weakPointer(globalObject), lowCell(m_node->child1()), m_out.constIntPtr(uid)));
12954 }
12955
12956 void compileResolveScope()
12957 {
12958 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12959 UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
12960 setJSValue(vmCall(pointerType(), operationResolveScope,
12961 weakPointer(globalObject), lowCell(m_node->child1()), m_out.constIntPtr(uid)));
12962 }
12963
12964 void compileGetDynamicVar()
12965 {
12966 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12967 UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
12968 setJSValue(vmCall(Int64, operationGetDynamicVar,
12969 weakPointer(globalObject), lowCell(m_node->child1()), m_out.constIntPtr(uid), m_out.constInt32(m_node->getPutInfo())));
12970 }
12971
12972 void compilePutDynamicVar()
12973 {
12974 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
12975 UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()];
12976 setJSValue(vmCall(Void, m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutDynamicVarStrict : operationPutDynamicVarNonStrict,
12977 weakPointer(globalObject), lowCell(m_node->child1()), lowJSValue(m_node->child2()), m_out.constIntPtr(uid), m_out.constInt32(m_node->getPutInfo())));
12978 }
12979
12980 void compileUnreachable()
12981 {
12982 // It's so tempting to assert that AI has proved that this is unreachable. But that's
12983 // simply not a requirement of the Unreachable opcode at all. If you emit an opcode that
12984 // *you* know will not return, then it's fine to end the basic block with Unreachable
12985 // after that opcode. You don't have to also prove to AI that your opcode does not return.
12986 // Hence, there is nothing to do here but emit code that will crash, so that we catch
12987 // cases where you said Unreachable but you lied.
12988 //
12989 // It's also also worth noting that some clients emit this opcode because they're not 100% sure
12990 // if the code is unreachable, but they would really prefer if we crashed rather than kept going
12991 // if it did turn out to be reachable. Hence, this needs to deterministically crash.
12992
12993 crash();
12994 }
12995
12996 void compileCheckSubClass()
12997 {
12998 LValue cell = lowCell(m_node->child1());
12999
13000 const ClassInfo* classInfo = m_node->classInfo();
13001 if (!classInfo->checkSubClassSnippet) {
13002 LBasicBlock loop = m_out.newBlock();
13003 LBasicBlock parentClass = m_out.newBlock();
13004 LBasicBlock continuation = m_out.newBlock();
13005
13006 LValue structure = loadStructure(cell);
13007 LValue classInfo = m_out.loadPtr(structure, m_heaps.Structure_classInfo);
13008 ValueFromBlock otherAtStart = m_out.anchor(classInfo);
13009 m_out.jump(loop);
13010
13011 LBasicBlock lastNext = m_out.appendTo(loop, parentClass);
13012 LValue other = m_out.phi(pointerType(), otherAtStart);
13013 m_out.branch(m_out.equal(other, m_out.constIntPtr(classInfo)), unsure(continuation), unsure(parentClass));
13014
13015 m_out.appendTo(parentClass, continuation);
13016 LValue parent = m_out.loadPtr(other, m_heaps.ClassInfo_parentClass);
13017 speculate(BadType, jsValueValue(cell), m_node->child1().node(), m_out.isNull(parent));
13018 m_out.addIncomingToPhi(other, m_out.anchor(parent));
13019 m_out.jump(loop);
13020
13021 m_out.appendTo(continuation, lastNext);
13022 return;
13023 }
13024
13025 RefPtr<Snippet> domJIT = classInfo->checkSubClassSnippet();
13026 PatchpointValue* patchpoint = m_out.patchpoint(Void);
13027 patchpoint->appendSomeRegister(cell);
13028 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
13029 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
13030
13031 NodeOrigin origin = m_origin;
13032 unsigned osrExitArgumentOffset = patchpoint->numChildren();
13033 OSRExitDescriptor* exitDescriptor = appendOSRExitDescriptor(jsValueValue(cell), m_node->child1().node());
13034 patchpoint->appendColdAnys(buildExitArguments(exitDescriptor, origin.forExit, jsValueValue(cell)));
13035
13036 patchpoint->numGPScratchRegisters = domJIT->numGPScratchRegisters;
13037 patchpoint->numFPScratchRegisters = domJIT->numFPScratchRegisters;
13038 patchpoint->clobber(RegisterSet::macroScratchRegisters());
13039
13040 State* state = &m_ftlState;
13041 Node* node = m_node;
13042 JSValue child1Constant = m_state.forNode(m_node->child1()).value();
13043
13044 patchpoint->setGenerator(
13045 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13046 AllowMacroScratchRegisterUsage allowScratch(jit);
13047
13048 Vector<GPRReg> gpScratch;
13049 Vector<FPRReg> fpScratch;
13050 Vector<SnippetParams::Value> regs;
13051
13052 regs.append(SnippetParams::Value(params[0].gpr(), child1Constant));
13053
13054 for (unsigned i = 0; i < domJIT->numGPScratchRegisters; ++i)
13055 gpScratch.append(params.gpScratch(i));
13056
13057 for (unsigned i = 0; i < domJIT->numFPScratchRegisters; ++i)
13058 fpScratch.append(params.fpScratch(i));
13059
13060 RefPtr<OSRExitHandle> handle = exitDescriptor->emitOSRExitLater(*state, BadType, origin, params, osrExitArgumentOffset);
13061
13062 SnippetParams domJITParams(*state, params, node, nullptr, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch));
13063 CCallHelpers::JumpList failureCases = domJIT->generator()->run(jit, domJITParams);
13064
13065 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
13066 linkBuffer.link(failureCases, linkBuffer.locationOf<NoPtrTag>(handle->label));
13067 });
13068 });
13069 patchpoint->effects = Effects::forCheck();
13070 }
13071
13072 void compileCallDOM()
13073 {
13074 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
13075 const DOMJIT::Signature* signature = m_node->signature();
13076
13077 // FIXME: We should have a way to call functions with the vector of registers.
13078 // https://bugs.webkit.org/show_bug.cgi?id=163099
13079 Vector<LValue, JSC_DOMJIT_SIGNATURE_MAX_ARGUMENTS_INCLUDING_THIS> operands;
13080
13081 unsigned index = 0;
13082 DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, [&](Node*, Edge edge) {
13083 if (!index)
13084 operands.append(lowCell(edge));
13085 else {
13086 switch (signature->arguments[index - 1]) {
13087 case SpecString:
13088 operands.append(lowString(edge));
13089 break;
13090 case SpecInt32Only:
13091 operands.append(lowInt32(edge));
13092 break;
13093 case SpecBoolean:
13094 operands.append(lowBoolean(edge));
13095 break;
13096 default:
13097 RELEASE_ASSERT_NOT_REACHED();
13098 break;
13099 }
13100 }
13101 ++index;
13102 });
13103
13104 unsigned argumentCountIncludingThis = signature->argumentCount + 1;
13105 LValue result;
13106 // FIXME: Revisit JSGlobalObject.
13107 // https://bugs.webkit.org/show_bug.cgi?id=203204
13108 auto function = CFunctionPtr(signature->functionWithoutTypeCheck);
13109 switch (argumentCountIncludingThis) {
13110 case 1:
13111 result = vmCall(Int64, reinterpret_cast<J_JITOperation_GP>(function.get()), weakPointer(globalObject), operands[0]);
13112 break;
13113 case 2:
13114 result = vmCall(Int64, reinterpret_cast<J_JITOperation_GPP>(function.get()), weakPointer(globalObject), operands[0], operands[1]);
13115 break;
13116 case 3:
13117 result = vmCall(Int64, reinterpret_cast<J_JITOperation_GPPP>(function.get()), weakPointer(globalObject), operands[0], operands[1], operands[2]);
13118 break;
13119 default:
13120 RELEASE_ASSERT_NOT_REACHED();
13121 break;
13122 }
13123
13124 setJSValue(result);
13125 }
13126
13127 void compileCallDOMGetter()
13128 {
13129 DOMJIT::CallDOMGetterSnippet* domJIT = m_node->callDOMGetterData()->snippet;
13130 if (!domJIT) {
13131 // The following function is not an operation: we directly call a custom accessor getter.
13132 // Since the getter does not have code setting topCallFrame, As is the same to IC, we should set topCallFrame in caller side.
13133 // FIXME: Revisit JSGlobalObject.
13134 // https://bugs.webkit.org/show_bug.cgi?id=203204
13135 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
13136 m_out.storePtr(m_callFrame, m_out.absolute(&vm().topCallFrame));
13137 setJSValue(
13138 vmCall(Int64, bitwise_cast<CustomGetterSetter::CustomGetter>(m_node->callDOMGetterData()->customAccessorGetter.retaggedExecutableAddress<CFunctionPtrTag>()),
13139 weakPointer(globalObject), lowCell(m_node->child1()), m_out.constIntPtr(m_graph.identifiers()[m_node->callDOMGetterData()->identifierNumber])));
13140 return;
13141 }
13142
13143 Edge& baseEdge = m_node->child1();
13144 LValue base = lowCell(baseEdge);
13145 JSValue baseConstant = m_state.forNode(baseEdge).value();
13146
13147 LValue globalObject;
13148 JSValue globalObjectConstant;
13149 if (domJIT->requireGlobalObject) {
13150 Edge& globalObjectEdge = m_node->child2();
13151 globalObject = lowCell(globalObjectEdge);
13152 globalObjectConstant = m_state.forNode(globalObjectEdge).value();
13153 }
13154
13155 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
13156 patchpoint->appendSomeRegister(base);
13157 if (domJIT->requireGlobalObject)
13158 patchpoint->appendSomeRegister(globalObject);
13159 patchpoint->append(m_notCellMask, ValueRep::reg(GPRInfo::notCellMaskRegister));
13160 patchpoint->append(m_numberTag, ValueRep::reg(GPRInfo::numberTagRegister));
13161 RefPtr<PatchpointExceptionHandle> exceptionHandle = preparePatchpointForExceptions(patchpoint);
13162 patchpoint->clobber(RegisterSet::macroScratchRegisters());
13163 patchpoint->numGPScratchRegisters = domJIT->numGPScratchRegisters;
13164 patchpoint->numFPScratchRegisters = domJIT->numFPScratchRegisters;
13165 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
13166
13167 State* state = &m_ftlState;
13168 Node* node = m_node;
13169 patchpoint->setGenerator(
13170 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13171 AllowMacroScratchRegisterUsage allowScratch(jit);
13172
13173 Vector<GPRReg> gpScratch;
13174 Vector<FPRReg> fpScratch;
13175 Vector<SnippetParams::Value> regs;
13176
13177 regs.append(JSValueRegs(params[0].gpr()));
13178 regs.append(SnippetParams::Value(params[1].gpr(), baseConstant));
13179 if (domJIT->requireGlobalObject)
13180 regs.append(SnippetParams::Value(params[2].gpr(), globalObjectConstant));
13181
13182 for (unsigned i = 0; i < domJIT->numGPScratchRegisters; ++i)
13183 gpScratch.append(params.gpScratch(i));
13184
13185 for (unsigned i = 0; i < domJIT->numFPScratchRegisters; ++i)
13186 fpScratch.append(params.fpScratch(i));
13187
13188 Box<CCallHelpers::JumpList> exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit);
13189
13190 SnippetParams domJITParams(*state, params, node, exceptions, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch));
13191 domJIT->generator()->run(jit, domJITParams);
13192 });
13193 patchpoint->effects = Effects::forCall();
13194 setJSValue(patchpoint);
13195 }
13196
13197 void compileFilterICStatus()
13198 {
13199 m_interpreter.filterICStatus(m_node);
13200 }
13201
13202 LValue byteSwap32(LValue value)
13203 {
13204 // FIXME: teach B3 byteswap
13205 // https://bugs.webkit.org/show_bug.cgi?id=188759
13206
13207 RELEASE_ASSERT(value->type() == Int32);
13208 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
13209 patchpoint->appendSomeRegister(value);
13210 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13211 jit.move(params[1].gpr(), params[0].gpr());
13212 jit.byteSwap32(params[0].gpr());
13213 });
13214 patchpoint->effects = Effects::none();
13215 return patchpoint;
13216 }
13217
13218 LValue byteSwap64(LValue value)
13219 {
13220 // FIXME: teach B3 byteswap
13221 // https://bugs.webkit.org/show_bug.cgi?id=188759
13222
13223 RELEASE_ASSERT(value->type() == Int64);
13224 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
13225 patchpoint->appendSomeRegister(value);
13226 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13227 jit.move(params[1].gpr(), params[0].gpr());
13228 jit.byteSwap64(params[0].gpr());
13229 });
13230 patchpoint->effects = Effects::none();
13231 return patchpoint;
13232 }
13233
13234 template <typename F1, typename F2>
13235 LValue emitCodeBasedOnEndiannessBranch(LValue isLittleEndian, const F1& emitLittleEndianCode, const F2& emitBigEndianCode)
13236 {
13237 LType type;
13238
13239 LBasicBlock bigEndianCase = m_out.newBlock();
13240 LBasicBlock littleEndianCase = m_out.newBlock();
13241 LBasicBlock continuation = m_out.newBlock();
13242
13243 m_out.branch(m_out.testIsZero32(isLittleEndian, m_out.constInt32(1)),
13244 unsure(bigEndianCase), unsure(littleEndianCase));
13245
13246 LBasicBlock lastNext = m_out.appendTo(bigEndianCase, littleEndianCase);
13247 LValue bigEndianValue = emitBigEndianCode();
13248 type = bigEndianValue ? bigEndianValue->type() : Void;
13249 ValueFromBlock bigEndianResult = bigEndianValue ? m_out.anchor(bigEndianValue) : ValueFromBlock();
13250 m_out.jump(continuation);
13251
13252 m_out.appendTo(littleEndianCase, continuation);
13253 LValue littleEndianValue = emitLittleEndianCode();
13254 ValueFromBlock littleEndianResult = littleEndianValue ? m_out.anchor(littleEndianValue) : ValueFromBlock();
13255 RELEASE_ASSERT((!littleEndianValue && !bigEndianValue) || type == littleEndianValue->type());
13256 m_out.jump(continuation);
13257
13258 m_out.appendTo(continuation, lastNext);
13259 RELEASE_ASSERT(!!bigEndianResult == !!littleEndianResult);
13260 if (bigEndianResult)
13261 return m_out.phi(type, bigEndianResult, littleEndianResult);
13262 return nullptr;
13263 }
13264
13265 void compileDataViewGet()
13266 {
13267 LValue dataView = lowDataViewObject(m_node->child1());
13268 LValue index = lowInt32(m_node->child2());
13269 LValue isLittleEndian = nullptr;
13270 if (m_node->child3())
13271 isLittleEndian = lowBoolean(m_node->child3());
13272
13273 DataViewData data = m_node->dataViewData();
13274
13275 LValue length = m_out.zeroExtPtr(m_out.load32NonNegative(dataView, m_heaps.JSArrayBufferView_length));
13276 LValue indexToCheck = m_out.zeroExtPtr(index);
13277 if (data.byteSize > 1)
13278 indexToCheck = m_out.add(indexToCheck, m_out.constInt64(data.byteSize - 1));
13279 speculate(OutOfBounds, noValue(), nullptr, m_out.aboveOrEqual(indexToCheck, length));
13280
13281 LValue vector = caged(Gigacage::Primitive, m_out.loadPtr(dataView, m_heaps.JSArrayBufferView_vector), dataView);
13282
13283 TypedPointer pointer(m_heaps.typedArrayProperties, m_out.add(vector, m_out.zeroExtPtr(index)));
13284
13285 if (m_node->op() == DataViewGetInt) {
13286 switch (data.byteSize) {
13287 case 1:
13288 if (data.isSigned)
13289 setInt32(m_out.load8SignExt32(pointer));
13290 else
13291 setInt32(m_out.load8ZeroExt32(pointer));
13292 break;
13293 case 2: {
13294 auto emitLittleEndianLoad = [&] {
13295 if (data.isSigned)
13296 return m_out.load16SignExt32(pointer);
13297 return m_out.load16ZeroExt32(pointer);
13298 };
13299
13300 auto emitBigEndianLoad = [&] {
13301 LValue val = m_out.load16ZeroExt32(pointer);
13302
13303 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
13304 patchpoint->appendSomeRegister(val);
13305 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13306 jit.move(params[1].gpr(), params[0].gpr());
13307 jit.byteSwap16(params[0].gpr());
13308 if (data.isSigned)
13309 jit.signExtend16To32(params[0].gpr(), params[0].gpr());
13310 });
13311 patchpoint->effects = Effects::none();
13312
13313 return patchpoint;
13314 };
13315
13316 if (data.isLittleEndian == FalseTriState)
13317 setInt32(emitBigEndianLoad());
13318 else if (data.isLittleEndian == TrueTriState)
13319 setInt32(emitLittleEndianLoad());
13320 else
13321 setInt32(emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianLoad, emitBigEndianLoad));
13322
13323 break;
13324 }
13325 case 4: {
13326 LValue loadedValue = m_out.load32(pointer);
13327
13328 if (data.isLittleEndian == FalseTriState)
13329 loadedValue = byteSwap32(loadedValue);
13330 else if (data.isLittleEndian == MixedTriState) {
13331 auto emitLittleEndianCode = [&] {
13332 return loadedValue;
13333 };
13334 auto emitBigEndianCode = [&] {
13335 return byteSwap32(loadedValue);
13336 };
13337
13338 loadedValue = emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode);
13339 }
13340
13341 if (data.isSigned)
13342 setInt32(loadedValue);
13343 else
13344 setStrictInt52(m_out.zeroExt(loadedValue, Int64));
13345
13346 break;
13347 }
13348 default:
13349 RELEASE_ASSERT_NOT_REACHED();
13350 }
13351 } else {
13352 switch (data.byteSize) {
13353 case 4: {
13354 auto emitLittleEndianCode = [&] {
13355 return m_out.floatToDouble(m_out.loadFloat(pointer));
13356 };
13357
13358 auto emitBigEndianCode = [&] {
13359 LValue loadedValue = m_out.load32(pointer);
13360 PatchpointValue* patchpoint = m_out.patchpoint(Double);
13361 patchpoint->appendSomeRegister(loadedValue);
13362 patchpoint->numGPScratchRegisters = 1;
13363 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13364 jit.move(params[1].gpr(), params.gpScratch(0));
13365 jit.byteSwap32(params.gpScratch(0));
13366 jit.move32ToFloat(params.gpScratch(0), params[0].fpr());
13367 jit.convertFloatToDouble(params[0].fpr(), params[0].fpr());
13368 });
13369 patchpoint->effects = Effects::none();
13370 return patchpoint;
13371 };
13372
13373 if (data.isLittleEndian == TrueTriState)
13374 setDouble(emitLittleEndianCode());
13375 else if (data.isLittleEndian == FalseTriState)
13376 setDouble(emitBigEndianCode());
13377 else
13378 setDouble(emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode));
13379
13380 break;
13381 }
13382 case 8: {
13383 auto emitLittleEndianCode = [&] {
13384 return m_out.loadDouble(pointer);
13385 };
13386
13387 auto emitBigEndianCode = [&] {
13388 LValue loadedValue = m_out.load64(pointer);
13389 loadedValue = byteSwap64(loadedValue);
13390 return m_out.bitCast(loadedValue, Double);
13391 };
13392
13393 if (data.isLittleEndian == TrueTriState)
13394 setDouble(emitLittleEndianCode());
13395 else if (data.isLittleEndian == FalseTriState)
13396 setDouble(emitBigEndianCode());
13397 else
13398 setDouble(emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode));
13399
13400 break;
13401 }
13402 default:
13403 RELEASE_ASSERT_NOT_REACHED();
13404 }
13405 }
13406 }
13407
13408 void compileDataViewSet()
13409 {
13410 LValue dataView = lowDataViewObject(m_graph.varArgChild(m_node, 0));
13411 LValue index = lowInt32(m_graph.varArgChild(m_node, 1));
13412 LValue isLittleEndian = nullptr;
13413 if (m_graph.varArgChild(m_node, 3))
13414 isLittleEndian = lowBoolean(m_graph.varArgChild(m_node, 3));
13415
13416 DataViewData data = m_node->dataViewData();
13417
13418 LValue length = m_out.zeroExtPtr(m_out.load32NonNegative(dataView, m_heaps.JSArrayBufferView_length));
13419 LValue indexToCheck = m_out.zeroExtPtr(index);
13420 if (data.byteSize > 1)
13421 indexToCheck = m_out.add(indexToCheck, m_out.constInt64(data.byteSize - 1));
13422 speculate(OutOfBounds, noValue(), nullptr, m_out.aboveOrEqual(indexToCheck, length));
13423
13424 Edge& valueEdge = m_graph.varArgChild(m_node, 2);
13425 LValue valueToStore;
13426 switch (valueEdge.useKind()) {
13427 case Int32Use:
13428 valueToStore = lowInt32(valueEdge);
13429 break;
13430 case DoubleRepUse:
13431 valueToStore = lowDouble(valueEdge);
13432 break;
13433 case Int52RepUse:
13434 valueToStore = lowStrictInt52(valueEdge);
13435 break;
13436 default:
13437 RELEASE_ASSERT_NOT_REACHED();
13438 }
13439
13440 LValue vector = caged(Gigacage::Primitive, m_out.loadPtr(dataView, m_heaps.JSArrayBufferView_vector), dataView);
13441 TypedPointer pointer(m_heaps.typedArrayProperties, m_out.add(vector, m_out.zeroExtPtr(index)));
13442
13443 if (data.isFloatingPoint) {
13444 if (data.byteSize == 4) {
13445 valueToStore = m_out.doubleToFloat(valueToStore);
13446
13447 auto emitLittleEndianCode = [&] () -> LValue {
13448 m_out.storeFloat(valueToStore, pointer);
13449 return nullptr;
13450 };
13451
13452 auto emitBigEndianCode = [&] () -> LValue {
13453 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
13454 patchpoint->appendSomeRegister(valueToStore);
13455 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13456 jit.moveFloatTo32(params[1].fpr(), params[0].gpr());
13457 jit.byteSwap32(params[0].gpr());
13458 });
13459 patchpoint->effects = Effects::none();
13460 m_out.store32(patchpoint, pointer);
13461 return nullptr;
13462 };
13463
13464 if (data.isLittleEndian == FalseTriState)
13465 emitBigEndianCode();
13466 else if (data.isLittleEndian == TrueTriState)
13467 emitLittleEndianCode();
13468 else
13469 emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode);
13470
13471 } else {
13472 RELEASE_ASSERT(data.byteSize == 8);
13473 auto emitLittleEndianCode = [&] () -> LValue {
13474 m_out.storeDouble(valueToStore, pointer);
13475 return nullptr;
13476 };
13477 auto emitBigEndianCode = [&] () -> LValue {
13478 m_out.store64(byteSwap64(m_out.bitCast(valueToStore, Int64)), pointer);
13479 return nullptr;
13480 };
13481
13482 if (data.isLittleEndian == FalseTriState)
13483 emitBigEndianCode();
13484 else if (data.isLittleEndian == TrueTriState)
13485 emitLittleEndianCode();
13486 else
13487 emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode);
13488 }
13489 } else {
13490 switch (data.byteSize) {
13491 case 1:
13492 RELEASE_ASSERT(valueEdge.useKind() == Int32Use);
13493 m_out.store32As8(valueToStore, pointer);
13494 break;
13495 case 2: {
13496 RELEASE_ASSERT(valueEdge.useKind() == Int32Use);
13497
13498 auto emitLittleEndianCode = [&] () -> LValue {
13499 m_out.store32As16(valueToStore, pointer);
13500 return nullptr;
13501 };
13502 auto emitBigEndianCode = [&] () -> LValue {
13503 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
13504 patchpoint->appendSomeRegister(valueToStore);
13505 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13506 jit.move(params[1].gpr(), params[0].gpr());
13507 jit.byteSwap16(params[0].gpr());
13508 });
13509 patchpoint->effects = Effects::none();
13510
13511 m_out.store32As16(patchpoint, pointer);
13512 return nullptr;
13513 };
13514
13515 if (data.isLittleEndian == FalseTriState)
13516 emitBigEndianCode();
13517 else if (data.isLittleEndian == TrueTriState)
13518 emitLittleEndianCode();
13519 else
13520 emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode);
13521 break;
13522 }
13523 case 4: {
13524 RELEASE_ASSERT(valueEdge.useKind() == Int32Use || valueEdge.useKind() == Int52RepUse);
13525
13526 if (valueEdge.useKind() == Int52RepUse)
13527 valueToStore = m_out.castToInt32(valueToStore);
13528
13529 auto emitLittleEndianCode = [&] () -> LValue {
13530 m_out.store32(valueToStore, pointer);
13531 return nullptr;
13532 };
13533 auto emitBigEndianCode = [&] () -> LValue {
13534 m_out.store32(byteSwap32(valueToStore), pointer);
13535 return nullptr;
13536 };
13537
13538 if (data.isLittleEndian == FalseTriState)
13539 emitBigEndianCode();
13540 else if (data.isLittleEndian == TrueTriState)
13541 emitLittleEndianCode();
13542 else
13543 emitCodeBasedOnEndiannessBranch(isLittleEndian, emitLittleEndianCode, emitBigEndianCode);
13544
13545 break;
13546 }
13547 default:
13548 RELEASE_ASSERT_NOT_REACHED();
13549 }
13550 }
13551 }
13552
13553 void compileDateGet()
13554 {
13555 LValue base = lowDateObject(m_node->child1());
13556
13557 auto emitGetCodeWithCallback = [&] (const AbstractHeap& cachedDoubleOffset, const AbstractHeap& cachedDataOffset, auto* operation, auto callback) {
13558 LBasicBlock dataExistsCase = m_out.newBlock();
13559 LBasicBlock fastCase = m_out.newBlock();
13560 LBasicBlock slowCase = m_out.newBlock();
13561 LBasicBlock continuation = m_out.newBlock();
13562
13563 LValue data = m_out.loadPtr(base, m_heaps.DateInstance_data);
13564 m_out.branch(m_out.notZero64(data), unsure(dataExistsCase), unsure(slowCase));
13565
13566 LBasicBlock lastNext = m_out.appendTo(dataExistsCase, fastCase);
13567 LValue milliseconds = m_out.loadDouble(base, m_heaps.DateInstance_internalNumber);
13568 LValue cachedMilliseconds = m_out.loadDouble(data, cachedDoubleOffset);
13569 m_out.branch(m_out.doubleNotEqualOrUnordered(milliseconds, cachedMilliseconds), unsure(slowCase), unsure(fastCase));
13570
13571 m_out.appendTo(fastCase, slowCase);
13572 ValueFromBlock fastResult = m_out.anchor(boxInt32(callback(m_out.load32(data, cachedDataOffset))));
13573 m_out.jump(continuation);
13574
13575 m_out.appendTo(slowCase, continuation);
13576 ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, operation, m_vmValue, base));
13577 m_out.jump(continuation);
13578
13579 m_out.appendTo(continuation, lastNext);
13580 setJSValue(m_out.phi(Int64, fastResult, slowResult));
13581 };
13582
13583 auto emitGetCode = [&] (const AbstractHeap& cachedDoubleOffset, const AbstractHeap& cachedDataOffset, auto* operation) {
13584 emitGetCodeWithCallback(cachedDoubleOffset, cachedDataOffset, operation, [] (LValue value) { return value; });
13585 };
13586
13587 switch (m_node->intrinsic()) {
13588 case DatePrototypeGetTimeIntrinsic:
13589 setDouble(m_out.loadDouble(base, m_heaps.DateInstance_internalNumber));
13590 break;
13591
13592 case DatePrototypeGetMillisecondsIntrinsic:
13593 case DatePrototypeGetUTCMillisecondsIntrinsic: {
13594 LValue milliseconds = m_out.loadDouble(base, m_heaps.DateInstance_internalNumber);
13595 LValue msPerSecondConstant = m_out.constDouble(msPerSecond);
13596 LValue seconds = m_out.doubleFloor(m_out.doubleDiv(milliseconds, msPerSecondConstant));
13597 LValue result = m_out.doubleToInt(m_out.doubleSub(milliseconds, m_out.doubleMul(seconds, msPerSecondConstant)));
13598 setJSValue(m_out.select(m_out.doubleNotEqualOrUnordered(milliseconds, milliseconds), m_out.constInt64(JSValue::encode(jsNaN())), boxInt32(result)));
13599 break;
13600 }
13601
13602 case DatePrototypeGetFullYearIntrinsic:
13603 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_year, operationDateGetFullYear);
13604 break;
13605 case DatePrototypeGetUTCFullYearIntrinsic:
13606 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_year, operationDateGetUTCFullYear);
13607 break;
13608 case DatePrototypeGetMonthIntrinsic:
13609 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_month, operationDateGetMonth);
13610 break;
13611 case DatePrototypeGetUTCMonthIntrinsic:
13612 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_month, operationDateGetUTCMonth);
13613 break;
13614 case DatePrototypeGetDateIntrinsic:
13615 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_monthDay, operationDateGetDate);
13616 break;
13617 case DatePrototypeGetUTCDateIntrinsic:
13618 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_monthDay, operationDateGetUTCDate);
13619 break;
13620 case DatePrototypeGetDayIntrinsic:
13621 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_weekDay, operationDateGetDay);
13622 break;
13623 case DatePrototypeGetUTCDayIntrinsic:
13624 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_weekDay, operationDateGetUTCDay);
13625 break;
13626 case DatePrototypeGetHoursIntrinsic:
13627 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_hour, operationDateGetHours);
13628 break;
13629 case DatePrototypeGetUTCHoursIntrinsic:
13630 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_hour, operationDateGetUTCHours);
13631 break;
13632 case DatePrototypeGetMinutesIntrinsic:
13633 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_minute, operationDateGetMinutes);
13634 break;
13635 case DatePrototypeGetUTCMinutesIntrinsic:
13636 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_minute, operationDateGetUTCMinutes);
13637 break;
13638 case DatePrototypeGetSecondsIntrinsic:
13639 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_second, operationDateGetSeconds);
13640 break;
13641 case DatePrototypeGetUTCSecondsIntrinsic:
13642 emitGetCode(m_heaps.DateInstanceData_gregorianDateTimeUTCCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTimeUTC_second, operationDateGetUTCSeconds);
13643 break;
13644
13645 case DatePrototypeGetTimezoneOffsetIntrinsic:
13646 emitGetCodeWithCallback(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_utcOffsetInMinute, operationDateGetTimezoneOffset, [&] (LValue offset) {
13647 return m_out.neg(offset);
13648 });
13649 break;
13650
13651 case DatePrototypeGetYearIntrinsic:
13652 emitGetCodeWithCallback(m_heaps.DateInstanceData_gregorianDateTimeCachedForMS, m_heaps.DateInstanceData_cachedGregorianDateTime_year, operationDateGetYear, [&] (LValue year) {
13653 return m_out.sub(year, m_out.constInt32(1900));
13654 });
13655 break;
13656
13657 default:
13658 RELEASE_ASSERT_NOT_REACHED();
13659 }
13660 }
13661
13662 void emitSwitchForMultiByOffset(LValue base, bool structuresChecked, Vector<SwitchCase, 2>& cases, LBasicBlock exit)
13663 {
13664 if (cases.isEmpty()) {
13665 m_out.jump(exit);
13666 return;
13667 }
13668
13669 if (structuresChecked) {
13670 std::sort(
13671 cases.begin(), cases.end(),
13672 [&] (const SwitchCase& a, const SwitchCase& b) -> bool {
13673 return a.value()->asInt() < b.value()->asInt();
13674 });
13675 SwitchCase last = cases.takeLast();
13676 m_out.switchInstruction(
13677 m_out.load32(base, m_heaps.JSCell_structureID), cases, last.target(), Weight(0));
13678 return;
13679 }
13680
13681 m_out.switchInstruction(
13682 m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
13683 }
13684
13685 void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild)
13686 {
13687 LValue rightCell = lowCell(rightChild);
13688 LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation);
13689
13690 speculateTruthyObject(rightChild, rightCell, SpecObject);
13691
13692 LBasicBlock leftCellCase = m_out.newBlock();
13693 LBasicBlock leftNotCellCase = m_out.newBlock();
13694 LBasicBlock continuation = m_out.newBlock();
13695
13696 m_out.branch(
13697 isCell(leftValue, provenType(leftChild)),
13698 unsure(leftCellCase), unsure(leftNotCellCase));
13699
13700 LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase);
13701 speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCellCheck));
13702 ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue));
13703 m_out.jump(continuation);
13704
13705 m_out.appendTo(leftNotCellCase, continuation);
13706 FTL_TYPE_CHECK(
13707 jsValueValue(leftValue), leftChild, SpecOther | SpecCellCheck, isNotOther(leftValue));
13708 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
13709 m_out.jump(continuation);
13710
13711 m_out.appendTo(continuation, lastNext);
13712 setBoolean(m_out.phi(Int32, cellResult, notCellResult));
13713 }
13714
13715 void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter)
13716 {
13717 if (masqueradesAsUndefinedWatchpointIsStillValid()) {
13718 FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
13719 return;
13720 }
13721
13722 FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
13723 speculate(
13724 BadType, jsValueValue(cell), edge.node(),
13725 m_out.testNonZero32(
13726 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoFlags),
13727 m_out.constInt32(MasqueradesAsUndefined)));
13728 }
13729
13730 template<typename IntFunctor>
13731 void nonSpeculativeCompare(const IntFunctor& intFunctor, S_JITOperation_GJJ helperFunction)
13732 {
13733 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
13734 LValue left = lowJSValue(m_node->child1());
13735 LValue right = lowJSValue(m_node->child2());
13736
13737 LBasicBlock leftIsInt = m_out.newBlock();
13738 LBasicBlock fastPath = m_out.newBlock();
13739 LBasicBlock slowPath = m_out.newBlock();
13740 LBasicBlock continuation = m_out.newBlock();
13741
13742 m_out.branch(isNotInt32(left, provenType(m_node->child1())), rarely(slowPath), usually(leftIsInt));
13743
13744 LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath);
13745 m_out.branch(isNotInt32(right, provenType(m_node->child2())), rarely(slowPath), usually(fastPath));
13746
13747 m_out.appendTo(fastPath, slowPath);
13748 ValueFromBlock fastResult = m_out.anchor(intFunctor(unboxInt32(left), unboxInt32(right)));
13749 m_out.jump(continuation);
13750
13751 m_out.appendTo(slowPath, continuation);
13752 ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
13753 pointerType(), helperFunction, weakPointer(globalObject), left, right)));
13754 m_out.jump(continuation);
13755
13756 m_out.appendTo(continuation, lastNext);
13757 setBoolean(m_out.phi(Int32, fastResult, slowResult));
13758 }
13759
13760 LValue stringsEqual(LValue leftJSString, LValue rightJSString, Edge leftJSStringEdge = Edge(), Edge rightJSStringEdge = Edge())
13761 {
13762 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
13763 LBasicBlock notTriviallyUnequalCase = m_out.newBlock();
13764 LBasicBlock notEmptyCase = m_out.newBlock();
13765 LBasicBlock leftReadyCase = m_out.newBlock();
13766 LBasicBlock rightReadyCase = m_out.newBlock();
13767 LBasicBlock left8BitCase = m_out.newBlock();
13768 LBasicBlock right8BitCase = m_out.newBlock();
13769 LBasicBlock loop = m_out.newBlock();
13770 LBasicBlock bytesEqual = m_out.newBlock();
13771 LBasicBlock trueCase = m_out.newBlock();
13772 LBasicBlock falseCase = m_out.newBlock();
13773 LBasicBlock slowCase = m_out.newBlock();
13774 LBasicBlock continuation = m_out.newBlock();
13775
13776 m_out.branch(isRopeString(leftJSString, leftJSStringEdge), rarely(slowCase), usually(leftReadyCase));
13777
13778 LBasicBlock lastNext = m_out.appendTo(leftReadyCase, rightReadyCase);
13779 m_out.branch(isRopeString(rightJSString, rightJSStringEdge), rarely(slowCase), usually(rightReadyCase));
13780
13781 m_out.appendTo(rightReadyCase, notTriviallyUnequalCase);
13782 LValue left = m_out.loadPtr(leftJSString, m_heaps.JSString_value);
13783 LValue right = m_out.loadPtr(rightJSString, m_heaps.JSString_value);
13784 LValue length = m_out.load32(left, m_heaps.StringImpl_length);
13785 m_out.branch(
13786 m_out.notEqual(length, m_out.load32(right, m_heaps.StringImpl_length)),
13787 unsure(falseCase), unsure(notTriviallyUnequalCase));
13788
13789 m_out.appendTo(notTriviallyUnequalCase, notEmptyCase);
13790 m_out.branch(m_out.isZero32(length), unsure(trueCase), unsure(notEmptyCase));
13791
13792 m_out.appendTo(notEmptyCase, left8BitCase);
13793 m_out.branch(
13794 m_out.testIsZero32(
13795 m_out.load32(left, m_heaps.StringImpl_hashAndFlags),
13796 m_out.constInt32(StringImpl::flagIs8Bit())),
13797 unsure(slowCase), unsure(left8BitCase));
13798
13799 m_out.appendTo(left8BitCase, right8BitCase);
13800 m_out.branch(
13801 m_out.testIsZero32(
13802 m_out.load32(right, m_heaps.StringImpl_hashAndFlags),
13803 m_out.constInt32(StringImpl::flagIs8Bit())),
13804 unsure(slowCase), unsure(right8BitCase));
13805
13806 m_out.appendTo(right8BitCase, loop);
13807
13808 LValue leftData = m_out.loadPtr(left, m_heaps.StringImpl_data);
13809 LValue rightData = m_out.loadPtr(right, m_heaps.StringImpl_data);
13810
13811 ValueFromBlock indexAtStart = m_out.anchor(length);
13812
13813 m_out.jump(loop);
13814
13815 m_out.appendTo(loop, bytesEqual);
13816
13817 LValue indexAtLoopTop = m_out.phi(Int32, indexAtStart);
13818 LValue indexInLoop = m_out.sub(indexAtLoopTop, m_out.int32One);
13819
13820 LValue leftByte = m_out.load8ZeroExt32(
13821 m_out.baseIndex(m_heaps.characters8, leftData, m_out.zeroExtPtr(indexInLoop)));
13822 LValue rightByte = m_out.load8ZeroExt32(
13823 m_out.baseIndex(m_heaps.characters8, rightData, m_out.zeroExtPtr(indexInLoop)));
13824
13825 m_out.branch(m_out.notEqual(leftByte, rightByte), unsure(falseCase), unsure(bytesEqual));
13826
13827 m_out.appendTo(bytesEqual, trueCase);
13828
13829 ValueFromBlock indexForNextIteration = m_out.anchor(indexInLoop);
13830 m_out.addIncomingToPhi(indexAtLoopTop, indexForNextIteration);
13831 m_out.branch(m_out.notZero32(indexInLoop), unsure(loop), unsure(trueCase));
13832
13833 m_out.appendTo(trueCase, falseCase);
13834
13835 ValueFromBlock trueResult = m_out.anchor(m_out.booleanTrue);
13836 m_out.jump(continuation);
13837
13838 m_out.appendTo(falseCase, slowCase);
13839
13840 ValueFromBlock falseResult = m_out.anchor(m_out.booleanFalse);
13841 m_out.jump(continuation);
13842
13843 m_out.appendTo(slowCase, continuation);
13844
13845 LValue slowResultValue = vmCall(
13846 Int64, operationCompareStringEq, weakPointer(globalObject),
13847 leftJSString, rightJSString);
13848 ValueFromBlock slowResult = m_out.anchor(unboxBoolean(slowResultValue));
13849 m_out.jump(continuation);
13850
13851 m_out.appendTo(continuation, lastNext);
13852 return m_out.phi(Int32, trueResult, falseResult, slowResult);
13853 }
13854
13855 enum ScratchFPRUsage {
13856 DontNeedScratchFPR,
13857 NeedScratchFPR
13858 };
13859 template<typename BinaryArithOpGenerator, ScratchFPRUsage scratchFPRUsage = DontNeedScratchFPR>
13860 void emitBinarySnippet(J_JITOperation_GJJ slowPathFunction)
13861 {
13862 Node* node = m_node;
13863
13864 LValue left = lowJSValue(node->child1());
13865 LValue right = lowJSValue(node->child2());
13866
13867 SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType());
13868 SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType());
13869
13870 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
13871 patchpoint->appendSomeRegister(left);
13872 patchpoint->appendSomeRegister(right);
13873 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
13874 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
13875 RefPtr<PatchpointExceptionHandle> exceptionHandle =
13876 preparePatchpointForExceptions(patchpoint);
13877 patchpoint->numGPScratchRegisters = 1;
13878 patchpoint->numFPScratchRegisters = 2;
13879 if (scratchFPRUsage == NeedScratchFPR)
13880 patchpoint->numFPScratchRegisters++;
13881 patchpoint->clobber(RegisterSet::macroScratchRegisters());
13882 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
13883 State* state = &m_ftlState;
13884 patchpoint->setGenerator(
13885 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13886 AllowMacroScratchRegisterUsage allowScratch(jit);
13887
13888 Box<CCallHelpers::JumpList> exceptions =
13889 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
13890
13891 auto generator = Box<BinaryArithOpGenerator>::create(
13892 leftOperand, rightOperand, JSValueRegs(params[0].gpr()),
13893 JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()),
13894 params.fpScratch(0), params.fpScratch(1), params.gpScratch(0),
13895 scratchFPRUsage == NeedScratchFPR ? params.fpScratch(2) : InvalidFPRReg);
13896
13897 generator->generateFastPath(jit);
13898
13899 if (generator->didEmitFastPath()) {
13900 generator->endJumpList().link(&jit);
13901 CCallHelpers::Label done = jit.label();
13902
13903 params.addLatePath(
13904 [=] (CCallHelpers& jit) {
13905 AllowMacroScratchRegisterUsage allowScratch(jit);
13906
13907 generator->slowPathJumpList().link(&jit);
13908 callOperation(
13909 *state, params.unavailableRegisters(), jit, node->origin.semantic,
13910 exceptions.get(), slowPathFunction, params[0].gpr(),
13911 jit.codeBlock()->globalObjectFor(node->origin.semantic),
13912 params[1].gpr(), params[2].gpr());
13913 jit.jump().linkTo(done, &jit);
13914 });
13915 } else {
13916 callOperation(
13917 *state, params.unavailableRegisters(), jit, node->origin.semantic,
13918 exceptions.get(), slowPathFunction, params[0].gpr(), jit.codeBlock()->globalObjectFor(node->origin.semantic), params[1].gpr(),
13919 params[2].gpr());
13920 }
13921 });
13922
13923 setJSValue(patchpoint);
13924 }
13925
13926 template<typename BinaryBitOpGenerator>
13927 void emitBinaryBitOpSnippet(J_JITOperation_GJJ slowPathFunction)
13928 {
13929 Node* node = m_node;
13930
13931 LValue left = lowJSValue(node->child1());
13932 LValue right = lowJSValue(node->child2());
13933
13934 SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType());
13935 SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType());
13936
13937 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
13938 patchpoint->appendSomeRegister(left);
13939 patchpoint->appendSomeRegister(right);
13940 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
13941 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
13942 RefPtr<PatchpointExceptionHandle> exceptionHandle =
13943 preparePatchpointForExceptions(patchpoint);
13944 patchpoint->numGPScratchRegisters = 1;
13945 patchpoint->clobber(RegisterSet::macroScratchRegisters());
13946 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
13947 State* state = &m_ftlState;
13948 patchpoint->setGenerator(
13949 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
13950 AllowMacroScratchRegisterUsage allowScratch(jit);
13951
13952 Box<CCallHelpers::JumpList> exceptions =
13953 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
13954
13955 auto generator = Box<BinaryBitOpGenerator>::create(
13956 leftOperand, rightOperand, JSValueRegs(params[0].gpr()),
13957 JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.gpScratch(0));
13958
13959 generator->generateFastPath(jit);
13960 generator->endJumpList().link(&jit);
13961 CCallHelpers::Label done = jit.label();
13962
13963 params.addLatePath(
13964 [=] (CCallHelpers& jit) {
13965 AllowMacroScratchRegisterUsage allowScratch(jit);
13966
13967 generator->slowPathJumpList().link(&jit);
13968 callOperation(
13969 *state, params.unavailableRegisters(), jit, node->origin.semantic,
13970 exceptions.get(), slowPathFunction, params[0].gpr(),
13971 jit.codeBlock()->globalObjectFor(node->origin.semantic),
13972 params[1].gpr(), params[2].gpr());
13973 jit.jump().linkTo(done, &jit);
13974 });
13975 });
13976
13977 setJSValue(patchpoint);
13978 }
13979
13980 void emitRightShiftSnippet(JITRightShiftGenerator::ShiftType shiftType)
13981 {
13982 Node* node = m_node;
13983
13984 // FIXME: Make this do exceptions.
13985 // https://bugs.webkit.org/show_bug.cgi?id=151686
13986
13987 LValue left = lowJSValue(node->child1());
13988 LValue right = lowJSValue(node->child2());
13989
13990 SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType());
13991 SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType());
13992
13993 PatchpointValue* patchpoint = m_out.patchpoint(Int64);
13994 patchpoint->appendSomeRegister(left);
13995 patchpoint->appendSomeRegister(right);
13996 patchpoint->append(m_notCellMask, ValueRep::lateReg(GPRInfo::notCellMaskRegister));
13997 patchpoint->append(m_numberTag, ValueRep::lateReg(GPRInfo::numberTagRegister));
13998 RefPtr<PatchpointExceptionHandle> exceptionHandle =
13999 preparePatchpointForExceptions(patchpoint);
14000 patchpoint->numGPScratchRegisters = 1;
14001 patchpoint->numFPScratchRegisters = 1;
14002 patchpoint->clobber(RegisterSet::macroScratchRegisters());
14003 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
14004 State* state = &m_ftlState;
14005 patchpoint->setGenerator(
14006 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
14007 AllowMacroScratchRegisterUsage allowScratch(jit);
14008
14009 Box<CCallHelpers::JumpList> exceptions =
14010 exceptionHandle->scheduleExitCreation(params)->jumps(jit);
14011
14012 auto generator = Box<JITRightShiftGenerator>::create(
14013 leftOperand, rightOperand, JSValueRegs(params[0].gpr()),
14014 JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()),
14015 params.fpScratch(0), params.gpScratch(0), InvalidFPRReg, shiftType);
14016
14017 generator->generateFastPath(jit);
14018 generator->endJumpList().link(&jit);
14019 CCallHelpers::Label done = jit.label();
14020
14021 params.addLatePath(
14022 [=] (CCallHelpers& jit) {
14023 AllowMacroScratchRegisterUsage allowScratch(jit);
14024
14025 generator->slowPathJumpList().link(&jit);
14026
14027 J_JITOperation_GJJ slowPathFunction =
14028 shiftType == JITRightShiftGenerator::SignedShift
14029 ? operationValueBitRShift : operationValueBitURShift;
14030
14031 callOperation(
14032 *state, params.unavailableRegisters(), jit, node->origin.semantic,
14033 exceptions.get(), slowPathFunction, params[0].gpr(),
14034 jit.codeBlock()->globalObjectFor(node->origin.semantic),
14035 params[1].gpr(), params[2].gpr());
14036 jit.jump().linkTo(done, &jit);
14037 });
14038 });
14039
14040 setJSValue(patchpoint);
14041 }
14042
14043 LValue allocateHeapCell(LValue allocator, LBasicBlock slowPath)
14044 {
14045 JITAllocator actualAllocator;
14046 if (allocator->hasIntPtr())
14047 actualAllocator = JITAllocator::constant(Allocator(bitwise_cast<LocalAllocator*>(allocator->asIntPtr())));
14048 else
14049 actualAllocator = JITAllocator::variable();
14050
14051 if (actualAllocator.isConstant()) {
14052 if (!actualAllocator.allocator()) {
14053 LBasicBlock haveAllocator = m_out.newBlock();
14054 LBasicBlock lastNext = m_out.insertNewBlocksBefore(haveAllocator);
14055 m_out.jump(slowPath);
14056 m_out.appendTo(haveAllocator, lastNext);
14057 return m_out.intPtrZero;
14058 }
14059 } else {
14060 // This means that either we know that the allocator is null or we don't know what the
14061 // allocator is. In either case, we need the null check.
14062 LBasicBlock haveAllocator = m_out.newBlock();
14063 LBasicBlock lastNext = m_out.insertNewBlocksBefore(haveAllocator);
14064 m_out.branch(
14065 m_out.notEqual(allocator, m_out.intPtrZero),
14066 usually(haveAllocator), rarely(slowPath));
14067 m_out.appendTo(haveAllocator, lastNext);
14068 }
14069
14070 LBasicBlock continuation = m_out.newBlock();
14071
14072 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
14073
14074 PatchpointValue* patchpoint = m_out.patchpoint(pointerType());
14075 if (isARM64()) {
14076 // emitAllocateWithNonNullAllocator uses the scratch registers on ARM.
14077 patchpoint->clobber(RegisterSet::macroScratchRegisters());
14078 }
14079 patchpoint->effects.terminal = true;
14080 if (actualAllocator.isConstant())
14081 patchpoint->numGPScratchRegisters++;
14082 else
14083 patchpoint->appendSomeRegisterWithClobber(allocator);
14084 patchpoint->numGPScratchRegisters++;
14085 patchpoint->resultConstraints = { ValueRep::SomeEarlyRegister };
14086
14087 m_out.appendSuccessor(usually(continuation));
14088 m_out.appendSuccessor(rarely(slowPath));
14089
14090 patchpoint->setGenerator(
14091 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
14092 AllowMacroScratchRegisterUsageIf allowScratchIf(jit, isARM64());
14093 CCallHelpers::JumpList jumpToSlowPath;
14094
14095 GPRReg allocatorGPR;
14096 if (actualAllocator.isConstant())
14097 allocatorGPR = params.gpScratch(1);
14098 else
14099 allocatorGPR = params[1].gpr();
14100
14101 // We use a patchpoint to emit the allocation path because whenever we mess with
14102 // allocation paths, we already reason about them at the machine code level. We know
14103 // exactly what instruction sequence we want. We're confident that no compiler
14104 // optimization could make this code better. So, it's best to have the code in
14105 // AssemblyHelpers::emitAllocate(). That way, the same optimized path is shared by
14106 // all of the compiler tiers.
14107 jit.emitAllocateWithNonNullAllocator(
14108 params[0].gpr(), actualAllocator, allocatorGPR, params.gpScratch(0),
14109 jumpToSlowPath);
14110
14111 CCallHelpers::Jump jumpToSuccess;
14112 if (!params.fallsThroughToSuccessor(0))
14113 jumpToSuccess = jit.jump();
14114
14115 Vector<Box<CCallHelpers::Label>> labels = params.successorLabels();
14116
14117 params.addLatePath(
14118 [=] (CCallHelpers& jit) {
14119 jumpToSlowPath.linkTo(*labels[1], &jit);
14120 if (jumpToSuccess.isSet())
14121 jumpToSuccess.linkTo(*labels[0], &jit);
14122 });
14123 });
14124
14125 m_out.appendTo(continuation, lastNext);
14126 return patchpoint;
14127 }
14128
14129 void storeStructure(LValue object, Structure* structure)
14130 {
14131 m_out.store32(m_out.constInt32(structure->id()), object, m_heaps.JSCell_structureID);
14132 m_out.store32(
14133 m_out.constInt32(structure->objectInitializationBlob()),
14134 object, m_heaps.JSCell_usefulBytes);
14135 }
14136
14137 void storeStructure(LValue object, LValue structure)
14138 {
14139 if (structure->hasIntPtr()) {
14140 storeStructure(object, bitwise_cast<Structure*>(structure->asIntPtr()));
14141 return;
14142 }
14143
14144 LValue id = m_out.load32(structure, m_heaps.Structure_structureID);
14145 m_out.store32(id, object, m_heaps.JSCell_structureID);
14146
14147 LValue blob = m_out.load32(structure, m_heaps.Structure_indexingModeIncludingHistory);
14148 m_out.store32(blob, object, m_heaps.JSCell_usefulBytes);
14149 }
14150
14151 template <typename StructureType>
14152 LValue allocateCell(LValue allocator, StructureType structure, LBasicBlock slowPath)
14153 {
14154 LValue result = allocateHeapCell(allocator, slowPath);
14155 storeStructure(result, structure);
14156 return result;
14157 }
14158
14159 LValue allocateObject(LValue allocator, RegisteredStructure structure, LValue butterfly, LBasicBlock slowPath)
14160 {
14161 return allocateObject(allocator, weakStructure(structure), butterfly, slowPath);
14162 }
14163
14164 LValue allocateObject(LValue allocator, LValue structure, LValue butterfly, LBasicBlock slowPath)
14165 {
14166 LValue result = allocateCell(allocator, structure, slowPath);
14167 if (structure->hasIntPtr()) {
14168 splatWords(
14169 result,
14170 m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8),
14171 m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8 + bitwise_cast<Structure*>(structure->asIntPtr())->inlineCapacity()),
14172 m_out.int64Zero,
14173 m_heaps.properties.atAnyNumber());
14174 } else {
14175 LValue end = m_out.add(
14176 m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8),
14177 m_out.load8ZeroExt32(structure, m_heaps.Structure_inlineCapacity));
14178 splatWords(
14179 result,
14180 m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8),
14181 end,
14182 m_out.int64Zero,
14183 m_heaps.properties.atAnyNumber());
14184 }
14185
14186 m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly);
14187 return result;
14188 }
14189
14190 template<typename ClassType, typename StructureType>
14191 LValue allocateObject(
14192 size_t size, StructureType structure, LValue butterfly, LBasicBlock slowPath)
14193 {
14194 Allocator allocator = allocatorForNonVirtualConcurrently<ClassType>(vm(), size, AllocatorForMode::AllocatorIfExists);
14195 return allocateObject(
14196 m_out.constIntPtr(allocator.localAllocator()), structure, butterfly, slowPath);
14197 }
14198
14199 template<typename ClassType, typename StructureType>
14200 LValue allocateObject(StructureType structure, LValue butterfly, LBasicBlock slowPath)
14201 {
14202 return allocateObject<ClassType>(
14203 ClassType::allocationSize(0), structure, butterfly, slowPath);
14204 }
14205
14206 LValue allocatorForSize(LValue subspace, LValue size, LBasicBlock slowPath)
14207 {
14208 static_assert(!(MarkedSpace::sizeStep & (MarkedSpace::sizeStep - 1)), "MarkedSpace::sizeStep must be a power of two.");
14209
14210 // Try to do some constant-folding here.
14211 if (subspace->hasIntPtr() && size->hasIntPtr()) {
14212 CompleteSubspace* actualSubspace = bitwise_cast<CompleteSubspace*>(subspace->asIntPtr());
14213 size_t actualSize = size->asIntPtr();
14214
14215 Allocator actualAllocator = actualSubspace->allocatorForNonVirtual(actualSize, AllocatorForMode::AllocatorIfExists);
14216 if (!actualAllocator) {
14217 LBasicBlock continuation = m_out.newBlock();
14218 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
14219 m_out.jump(slowPath);
14220 m_out.appendTo(continuation, lastNext);
14221 return m_out.intPtrZero;
14222 }
14223
14224 return m_out.constIntPtr(actualAllocator.localAllocator());
14225 }
14226
14227 unsigned stepShift = getLSBSet(MarkedSpace::sizeStep);
14228
14229 LBasicBlock continuation = m_out.newBlock();
14230
14231 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
14232
14233 LValue sizeClassIndex = m_out.lShr(
14234 m_out.add(size, m_out.constIntPtr(MarkedSpace::sizeStep - 1)),
14235 m_out.constInt32(stepShift));
14236
14237 m_out.branch(
14238 m_out.above(sizeClassIndex, m_out.constIntPtr(MarkedSpace::largeCutoff >> stepShift)),
14239 rarely(slowPath), usually(continuation));
14240
14241 m_out.appendTo(continuation, lastNext);
14242
14243 return m_out.loadPtr(
14244 m_out.baseIndex(
14245 m_heaps.CompleteSubspace_allocatorForSizeStep,
14246 subspace, sizeClassIndex));
14247 }
14248
14249 LValue allocatorForSize(CompleteSubspace& subspace, LValue size, LBasicBlock slowPath)
14250 {
14251 return allocatorForSize(m_out.constIntPtr(&subspace), size, slowPath);
14252 }
14253
14254 template<typename ClassType>
14255 LValue allocateVariableSizedObject(
14256 LValue size, RegisteredStructure structure, LValue butterfly, LBasicBlock slowPath)
14257 {
14258 CompleteSubspace* subspace = subspaceForConcurrently<ClassType>(vm());
14259 RELEASE_ASSERT_WITH_MESSAGE(subspace, "CompleteSubspace is always allocated");
14260 LValue allocator = allocatorForSize(*subspace, size, slowPath);
14261 return allocateObject(allocator, structure, butterfly, slowPath);
14262 }
14263
14264 template<typename ClassType>
14265 LValue allocateVariableSizedCell(
14266 LValue size, Structure* structure, LBasicBlock slowPath)
14267 {
14268 CompleteSubspace* subspace = subspaceForConcurrently<ClassType>(vm());
14269 RELEASE_ASSERT_WITH_MESSAGE(subspace, "CompleteSubspace is always allocated");
14270 LValue allocator = allocatorForSize(*subspace, size, slowPath);
14271 return allocateCell(allocator, structure, slowPath);
14272 }
14273
14274 LValue allocateObject(RegisteredStructure structure)
14275 {
14276 size_t allocationSize = JSFinalObject::allocationSize(structure.get()->inlineCapacity());
14277 Allocator allocator = allocatorForNonVirtualConcurrently<JSFinalObject>(vm(), allocationSize, AllocatorForMode::AllocatorIfExists);
14278
14279 // FIXME: If the allocator is null, we could simply emit a normal C call to the allocator
14280 // instead of putting it on the slow path.
14281 // https://bugs.webkit.org/show_bug.cgi?id=161062
14282
14283 LBasicBlock slowPath = m_out.newBlock();
14284 LBasicBlock continuation = m_out.newBlock();
14285
14286 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
14287
14288 ValueFromBlock fastResult = m_out.anchor(allocateObject(
14289 m_out.constIntPtr(allocator.localAllocator()), structure, m_out.intPtrZero, slowPath));
14290
14291 m_out.jump(continuation);
14292
14293 m_out.appendTo(slowPath, continuation);
14294
14295 VM& vm = this->vm();
14296 LValue slowResultValue = lazySlowPath(
14297 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
14298 return createLazyCallGenerator(vm,
14299 operationNewObject, locations[0].directGPR(), &vm,
14300 CCallHelpers::TrustedImmPtr(structure.get()));
14301 });
14302 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
14303 m_out.jump(continuation);
14304
14305 m_out.appendTo(continuation, lastNext);
14306 return m_out.phi(pointerType(), fastResult, slowResult);
14307 }
14308
14309 struct ArrayValues {
14310 ArrayValues()
14311 : array(0)
14312 , butterfly(0)
14313 {
14314 }
14315
14316 ArrayValues(LValue array, LValue butterfly)
14317 : array(array)
14318 , butterfly(butterfly)
14319 {
14320 }
14321
14322 LValue array;
14323 LValue butterfly;
14324 };
14325
14326 ArrayValues allocateJSArray(LValue publicLength, LValue vectorLength, LValue structure, LValue indexingType, bool shouldInitializeElements = true, bool shouldLargeArraySizeCreateArrayStorage = true)
14327 {
14328 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
14329 if (indexingType->hasInt32()) {
14330 IndexingType type = static_cast<IndexingType>(indexingType->asInt32());
14331 ASSERT_UNUSED(type,
14332 hasUndecided(type)
14333 || hasInt32(type)
14334 || hasDouble(type)
14335 || hasContiguous(type));
14336 }
14337
14338 LBasicBlock fastCase = m_out.newBlock();
14339 LBasicBlock largeCase = m_out.newBlock();
14340 LBasicBlock failCase = m_out.newBlock();
14341 LBasicBlock continuation = m_out.newBlock();
14342 LBasicBlock slowCase = m_out.newBlock();
14343
14344 LBasicBlock lastNext = m_out.insertNewBlocksBefore(fastCase);
14345
14346 Optional<unsigned> staticVectorLength;
14347 Optional<unsigned> staticVectorLengthFromPublicLength;
14348 if (structure->hasIntPtr()) {
14349 if (publicLength->hasInt32()) {
14350 unsigned publicLengthConst = static_cast<unsigned>(publicLength->asInt32());
14351 if (publicLengthConst <= MAX_STORAGE_VECTOR_LENGTH) {
14352 publicLengthConst = Butterfly::optimalContiguousVectorLength(
14353 bitwise_cast<Structure*>(structure->asIntPtr())->outOfLineCapacity(), publicLengthConst);
14354 staticVectorLengthFromPublicLength = publicLengthConst;
14355 }
14356
14357 }
14358 if (vectorLength->hasInt32()) {
14359 unsigned vectorLengthConst = static_cast<unsigned>(vectorLength->asInt32());
14360 if (vectorLengthConst <= MAX_STORAGE_VECTOR_LENGTH) {
14361 vectorLengthConst = Butterfly::optimalContiguousVectorLength(
14362 bitwise_cast<Structure*>(structure->asIntPtr())->outOfLineCapacity(), vectorLengthConst);
14363 vectorLength = m_out.constInt32(vectorLengthConst);
14364 staticVectorLength = vectorLengthConst;
14365 }
14366 }
14367 } else {
14368 // We don't compute the optimal vector length for new Array(blah) where blah is not
14369 // statically known, since the compute effort of doing it here is probably not worth it.
14370 }
14371
14372 ValueFromBlock noButterfly = m_out.anchor(m_out.intPtrZero);
14373
14374 LValue predicate;
14375 if (shouldLargeArraySizeCreateArrayStorage)
14376 predicate = m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH));
14377 else
14378 predicate = m_out.booleanFalse;
14379
14380 m_out.branch(predicate, rarely(largeCase), usually(fastCase));
14381
14382 m_out.appendTo(fastCase, largeCase);
14383
14384 LValue payloadSize =
14385 m_out.shl(m_out.zeroExt(vectorLength, pointerType()), m_out.constIntPtr(3));
14386
14387 LValue butterflySize = m_out.add(
14388 payloadSize, m_out.constIntPtr(sizeof(IndexingHeader)));
14389
14390 LValue allocator = allocatorForSize(vm().jsValueGigacageAuxiliarySpace, butterflySize, failCase);
14391 LValue startOfStorage = allocateHeapCell(allocator, failCase);
14392
14393 LValue butterfly = m_out.add(startOfStorage, m_out.constIntPtr(sizeof(IndexingHeader)));
14394
14395 m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength);
14396 m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength);
14397
14398 initializeArrayElements(
14399 indexingType,
14400 shouldInitializeElements ? m_out.int32Zero : publicLength, vectorLength,
14401 butterfly);
14402
14403 ValueFromBlock haveButterfly = m_out.anchor(butterfly);
14404
14405 LValue object = allocateObject<JSArray>(structure, butterfly, failCase);
14406
14407 ValueFromBlock fastResult = m_out.anchor(object);
14408 ValueFromBlock fastButterfly = m_out.anchor(butterfly);
14409 m_out.jump(continuation);
14410
14411 m_out.appendTo(largeCase, failCase);
14412 ValueFromBlock largeStructure = m_out.anchor(
14413 weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))));
14414 m_out.jump(slowCase);
14415
14416 m_out.appendTo(failCase, slowCase);
14417 ValueFromBlock failStructure = m_out.anchor(structure);
14418 m_out.jump(slowCase);
14419
14420 m_out.appendTo(slowCase, continuation);
14421 LValue structureValue = m_out.phi(pointerType(), largeStructure, failStructure);
14422 LValue butterflyValue = m_out.phi(pointerType(), noButterfly, haveButterfly);
14423
14424 VM& vm = this->vm();
14425 LValue slowResultValue = nullptr;
14426 if (vectorLength == publicLength
14427 || (staticVectorLengthFromPublicLength && staticVectorLength && staticVectorLength.value() == staticVectorLengthFromPublicLength.value())) {
14428 slowResultValue = lazySlowPath(
14429 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
14430 return createLazyCallGenerator(vm,
14431 operationNewArrayWithSize, locations[0].directGPR(), globalObject,
14432 locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR());
14433 },
14434 structureValue, publicLength, butterflyValue);
14435 } else {
14436 slowResultValue = lazySlowPath(
14437 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
14438 return createLazyCallGenerator(vm,
14439 operationNewArrayWithSizeAndHint, locations[0].directGPR(), globalObject,
14440 locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR(), locations[4].directGPR());
14441 },
14442 structureValue, publicLength, vectorLength, butterflyValue);
14443 }
14444
14445 ValueFromBlock slowResult = m_out.anchor(slowResultValue);
14446 ValueFromBlock slowButterfly = m_out.anchor(
14447 m_out.loadPtr(slowResultValue, m_heaps.JSObject_butterfly));
14448 m_out.jump(continuation);
14449
14450 m_out.appendTo(continuation, lastNext);
14451 return ArrayValues(
14452 m_out.phi(pointerType(), fastResult, slowResult),
14453 m_out.phi(pointerType(), fastButterfly, slowButterfly));
14454 }
14455
14456 ArrayValues allocateUninitializedContiguousJSArrayInternal(LValue publicLength, LValue vectorLength, RegisteredStructure structure)
14457 {
14458 bool shouldInitializeElements = false;
14459 bool shouldLargeArraySizeCreateArrayStorage = false;
14460 return allocateJSArray(
14461 publicLength, vectorLength, weakStructure(structure), m_out.constInt32(structure->indexingType()), shouldInitializeElements,
14462 shouldLargeArraySizeCreateArrayStorage);
14463 }
14464
14465 ArrayValues allocateUninitializedContiguousJSArray(LValue publicLength, RegisteredStructure structure)
14466 {
14467 return allocateUninitializedContiguousJSArrayInternal(publicLength, publicLength, structure);
14468 }
14469
14470 ArrayValues allocateUninitializedContiguousJSArray(unsigned publicLength, unsigned vectorLength, RegisteredStructure structure)
14471 {
14472 ASSERT(vectorLength >= publicLength);
14473 return allocateUninitializedContiguousJSArrayInternal(m_out.constInt32(publicLength), m_out.constInt32(vectorLength), structure);
14474 }
14475
14476 LValue ensureShadowChickenPacket()
14477 {
14478 ShadowChicken* shadowChicken = vm().shadowChicken();
14479 RELEASE_ASSERT(shadowChicken);
14480 LBasicBlock slowCase = m_out.newBlock();
14481 LBasicBlock continuation = m_out.newBlock();
14482
14483 TypedPointer addressOfLogCursor = m_out.absolute(shadowChicken->addressOfLogCursor());
14484 LValue logCursor = m_out.loadPtr(addressOfLogCursor);
14485
14486 ValueFromBlock fastResult = m_out.anchor(logCursor);
14487
14488 m_out.branch(
14489 m_out.below(logCursor, m_out.constIntPtr(shadowChicken->logEnd())),
14490 usually(continuation), rarely(slowCase));
14491
14492 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
14493
14494 vmCall(Void, operationProcessShadowChickenLog, m_vmValue);
14495
14496 ValueFromBlock slowResult = m_out.anchor(m_out.loadPtr(addressOfLogCursor));
14497 m_out.jump(continuation);
14498
14499 m_out.appendTo(continuation, lastNext);
14500 LValue result = m_out.phi(pointerType(), fastResult, slowResult);
14501
14502 m_out.storePtr(
14503 m_out.add(result, m_out.constIntPtr(sizeof(ShadowChicken::Packet))),
14504 addressOfLogCursor);
14505
14506 return result;
14507 }
14508
14509 LValue boolify(Edge edge)
14510 {
14511 switch (edge.useKind()) {
14512 case BooleanUse:
14513 case KnownBooleanUse:
14514 return lowBoolean(edge);
14515 case Int32Use:
14516 return m_out.notZero32(lowInt32(edge));
14517 case DoubleRepUse:
14518 return m_out.doubleNotEqualAndOrdered(lowDouble(edge), m_out.doubleZero);
14519 case ObjectOrOtherUse:
14520 return m_out.logicalNot(
14521 equalNullOrUndefined(
14522 edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
14523 ManualOperandSpeculation));
14524 case StringUse:
14525 return m_out.notEqual(lowString(edge), weakPointer(jsEmptyString(m_graph.m_vm)));
14526 case StringOrOtherUse: {
14527 LValue value = lowJSValue(edge, ManualOperandSpeculation);
14528
14529 LBasicBlock cellCase = m_out.newBlock();
14530 LBasicBlock notCellCase = m_out.newBlock();
14531 LBasicBlock continuation = m_out.newBlock();
14532
14533 m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase));
14534
14535 LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
14536 FTL_TYPE_CHECK(jsValueValue(value), edge, (~SpecCellCheck) | SpecString, isNotString(value));
14537 ValueFromBlock stringResult = m_out.anchor(m_out.notEqual(value, weakPointer(jsEmptyString(m_graph.m_vm))));
14538 m_out.jump(continuation);
14539
14540 m_out.appendTo(notCellCase, continuation);
14541 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecCellCheck | SpecOther, isNotOther(value));
14542 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
14543 m_out.jump(continuation);
14544
14545 m_out.appendTo(continuation, lastNext);
14546 return m_out.phi(Int32, stringResult, notCellResult);
14547 }
14548 case UntypedUse: {
14549 LValue value = lowJSValue(edge);
14550
14551 // Implements the following control flow structure:
14552 // if (value is cell) {
14553 // if (value is string or value is BigInt)
14554 // result = !!value->length
14555 // else {
14556 // do evil things for masquerades-as-undefined
14557 // result = true
14558 // }
14559 // } else if (value is int32) {
14560 // result = !!unboxInt32(value)
14561 // } else if (value is number) {
14562 // result = !!unboxDouble(value)
14563 // } else {
14564 // result = value == jsTrue
14565 // }
14566
14567 LBasicBlock cellCase = m_out.newBlock();
14568 LBasicBlock notStringCase = m_out.newBlock();
14569 LBasicBlock stringCase = m_out.newBlock();
14570 LBasicBlock bigIntCase = m_out.newBlock();
14571 LBasicBlock notStringOrBigIntCase = m_out.newBlock();
14572 LBasicBlock notCellCase = m_out.newBlock();
14573 LBasicBlock int32Case = m_out.newBlock();
14574 LBasicBlock notInt32Case = m_out.newBlock();
14575 LBasicBlock doubleCase = m_out.newBlock();
14576 LBasicBlock notDoubleCase = m_out.newBlock();
14577 LBasicBlock continuation = m_out.newBlock();
14578
14579 Vector<ValueFromBlock> results;
14580
14581 m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase));
14582
14583 LBasicBlock lastNext = m_out.appendTo(cellCase, notStringCase);
14584 m_out.branch(
14585 isString(value, provenType(edge) & SpecCell),
14586 unsure(stringCase), unsure(notStringCase));
14587
14588 m_out.appendTo(notStringCase, stringCase);
14589 m_out.branch(
14590 isBigInt(value, provenType(edge) & (SpecCell - SpecString)),
14591 unsure(bigIntCase), unsure(notStringOrBigIntCase));
14592
14593 m_out.appendTo(stringCase, bigIntCase);
14594 results.append(m_out.anchor(m_out.notEqual(value, weakPointer(jsEmptyString(m_graph.m_vm)))));
14595 m_out.jump(continuation);
14596
14597 m_out.appendTo(bigIntCase, notStringOrBigIntCase);
14598 LValue nonZeroBigInt = m_out.notZero32(
14599 m_out.load32NonNegative(value, m_heaps.JSBigInt_length));
14600 results.append(m_out.anchor(nonZeroBigInt));
14601 m_out.jump(continuation);
14602
14603 m_out.appendTo(notStringOrBigIntCase, notCellCase);
14604 LValue isTruthyObject;
14605 if (masqueradesAsUndefinedWatchpointIsStillValid())
14606 isTruthyObject = m_out.booleanTrue;
14607 else {
14608 LBasicBlock masqueradesCase = m_out.newBlock();
14609
14610 results.append(m_out.anchor(m_out.booleanTrue));
14611
14612 m_out.branch(
14613 m_out.testIsZero32(
14614 m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags),
14615 m_out.constInt32(MasqueradesAsUndefined)),
14616 usually(continuation), rarely(masqueradesCase));
14617
14618 m_out.appendTo(masqueradesCase);
14619
14620 isTruthyObject = m_out.notEqual(
14621 weakPointer(m_graph.globalObjectFor(m_node->origin.semantic)),
14622 m_out.loadPtr(loadStructure(value), m_heaps.Structure_globalObject));
14623 }
14624 results.append(m_out.anchor(isTruthyObject));
14625 m_out.jump(continuation);
14626
14627 m_out.appendTo(notCellCase, int32Case);
14628 m_out.branch(
14629 isInt32(value, provenType(edge) & ~SpecCell),
14630 unsure(int32Case), unsure(notInt32Case));
14631
14632 m_out.appendTo(int32Case, notInt32Case);
14633 results.append(m_out.anchor(m_out.notZero32(unboxInt32(value))));
14634 m_out.jump(continuation);
14635
14636 m_out.appendTo(notInt32Case, doubleCase);
14637 m_out.branch(
14638 isNumber(value, provenType(edge) & ~SpecCell),
14639 unsure(doubleCase), unsure(notDoubleCase));
14640
14641 m_out.appendTo(doubleCase, notDoubleCase);
14642 LValue doubleIsTruthy = m_out.doubleNotEqualAndOrdered(
14643 unboxDouble(value), m_out.constDouble(0));
14644 results.append(m_out.anchor(doubleIsTruthy));
14645 m_out.jump(continuation);
14646
14647 m_out.appendTo(notDoubleCase, continuation);
14648 LValue miscIsTruthy = m_out.equal(
14649 value, m_out.constInt64(JSValue::encode(jsBoolean(true))));
14650 results.append(m_out.anchor(miscIsTruthy));
14651 m_out.jump(continuation);
14652
14653 m_out.appendTo(continuation, lastNext);
14654 return m_out.phi(Int32, results);
14655 }
14656 default:
14657 DFG_CRASH(m_graph, m_node, "Bad use kind");
14658 return 0;
14659 }
14660 }
14661
14662 enum StringOrObjectMode {
14663 AllCellsAreFalse,
14664 CellCaseSpeculatesObject
14665 };
14666 enum EqualNullOrUndefinedMode {
14667 EqualNull,
14668 EqualUndefined,
14669 EqualNullOrUndefined,
14670 SpeculateNullOrUndefined
14671 };
14672 LValue equalNullOrUndefined(
14673 Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
14674 OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
14675 {
14676 bool validWatchpoint = masqueradesAsUndefinedWatchpointIsStillValid();
14677
14678 LValue value = lowJSValue(edge, operandMode);
14679
14680 LBasicBlock cellCase = m_out.newBlock();
14681 LBasicBlock primitiveCase = m_out.newBlock();
14682 LBasicBlock continuation = m_out.newBlock();
14683
14684 m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase));
14685
14686 LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
14687
14688 Vector<ValueFromBlock, 3> results;
14689
14690 switch (cellMode) {
14691 case AllCellsAreFalse:
14692 break;
14693 case CellCaseSpeculatesObject:
14694 FTL_TYPE_CHECK(
14695 jsValueValue(value), edge, (~SpecCellCheck) | SpecObject, isNotObject(value));
14696 break;
14697 }
14698
14699 if (validWatchpoint) {
14700 results.append(m_out.anchor(m_out.booleanFalse));
14701 m_out.jump(continuation);
14702 } else {
14703 LBasicBlock masqueradesCase =
14704 m_out.newBlock();
14705
14706 results.append(m_out.anchor(m_out.booleanFalse));
14707
14708 m_out.branch(
14709 m_out.testNonZero32(
14710 m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags),
14711 m_out.constInt32(MasqueradesAsUndefined)),
14712 rarely(masqueradesCase), usually(continuation));
14713
14714 m_out.appendTo(masqueradesCase, primitiveCase);
14715
14716 LValue structure = loadStructure(value);
14717
14718 results.append(m_out.anchor(
14719 m_out.equal(
14720 weakPointer(m_graph.globalObjectFor(m_node->origin.semantic)),
14721 m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
14722 m_out.jump(continuation);
14723 }
14724
14725 m_out.appendTo(primitiveCase, continuation);
14726
14727 LValue primitiveResult;
14728 switch (primitiveMode) {
14729 case EqualNull:
14730 primitiveResult = m_out.equal(value, m_out.constInt64(JSValue::ValueNull));
14731 break;
14732 case EqualUndefined:
14733 primitiveResult = m_out.equal(value, m_out.constInt64(JSValue::ValueUndefined));
14734 break;
14735 case EqualNullOrUndefined:
14736 primitiveResult = isOther(value, provenType(edge));
14737 break;
14738 case SpeculateNullOrUndefined:
14739 FTL_TYPE_CHECK(
14740 jsValueValue(value), edge, SpecCellCheck | SpecOther, isNotOther(value));
14741 primitiveResult = m_out.booleanTrue;
14742 break;
14743 }
14744 results.append(m_out.anchor(primitiveResult));
14745 m_out.jump(continuation);
14746
14747 m_out.appendTo(continuation, lastNext);
14748
14749 return m_out.phi(Int32, results);
14750 }
14751
14752 template<typename FunctionType>
14753 void contiguousPutByValOutOfBounds(
14754 FunctionType slowPathFunction, LValue base, LValue storage, LValue index, LValue value,
14755 LBasicBlock continuation)
14756 {
14757 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
14758 if (!m_node->arrayMode().isInBounds()) {
14759 LBasicBlock notInBoundsCase =
14760 m_out.newBlock();
14761 LBasicBlock performStore =
14762 m_out.newBlock();
14763
14764 LValue isNotInBounds = m_out.aboveOrEqual(
14765 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength));
14766 m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore));
14767
14768 LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
14769
14770 LValue isOutOfBounds = m_out.aboveOrEqual(
14771 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength));
14772
14773 if (!m_node->arrayMode().isOutOfBounds())
14774 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
14775 else {
14776 LBasicBlock outOfBoundsCase =
14777 m_out.newBlock();
14778 LBasicBlock holeCase =
14779 m_out.newBlock();
14780
14781 m_out.branch(isOutOfBounds, rarely(outOfBoundsCase), usually(holeCase));
14782
14783 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
14784
14785 vmCall(
14786 Void, slowPathFunction,
14787 weakPointer(globalObject), base, index, value);
14788
14789 m_out.jump(continuation);
14790
14791 m_out.appendTo(holeCase, innerLastNext);
14792 }
14793
14794 m_out.store32(
14795 m_out.add(index, m_out.int32One),
14796 storage, m_heaps.Butterfly_publicLength);
14797
14798 m_out.jump(performStore);
14799 m_out.appendTo(performStore, lastNext);
14800 }
14801 }
14802
14803 LValue untagArrayPtr(LValue ptr, LValue size)
14804 {
14805#if CPU(ARM64E)
14806 PatchpointValue* authenticate = m_out.patchpoint(pointerType());
14807 authenticate->appendSomeRegister(ptr);
14808 authenticate->append(size, B3::ValueRep(B3::ValueRep::SomeLateRegister));
14809 authenticate->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
14810 jit.move(params[1].gpr(), params[0].gpr());
14811 jit.untagArrayPtr(params[2].gpr(), params[0].gpr());
14812 });
14813 return authenticate;
14814#else
14815 UNUSED_PARAM(size);
14816 return ptr;
14817#endif
14818 }
14819
14820 LValue removeArrayPtrTag(LValue ptr)
14821 {
14822#if CPU(ARM64E)
14823 PatchpointValue* authenticate = m_out.patchpoint(pointerType());
14824 authenticate->appendSomeRegister(ptr);
14825 authenticate->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
14826 jit.move(params[1].gpr(), params[0].gpr());
14827 jit.removeArrayPtrTag(params[0].gpr());
14828 });
14829 return authenticate;
14830#endif
14831 return ptr;
14832 }
14833
14834 LValue caged(Gigacage::Kind kind, LValue ptr, LValue base)
14835 {
14836 auto doUntagArrayPtr = [&](LValue taggedPtr) {
14837#if CPU(ARM64E)
14838 if (kind == Gigacage::Primitive) {
14839 LValue size = m_out.load32(base, m_heaps.JSArrayBufferView_length);
14840 return untagArrayPtr(taggedPtr, size);
14841 }
14842 return ptr;
14843#else
14844 UNUSED_PARAM(taggedPtr);
14845 return ptr;
14846#endif
14847 };
14848
14849#if GIGACAGE_ENABLED
14850 if (!Gigacage::isEnabled(kind))
14851 return doUntagArrayPtr(ptr);
14852
14853 if (kind == Gigacage::Primitive && Gigacage::canPrimitiveGigacageBeDisabled()) {
14854 if (vm().primitiveGigacageEnabled().isStillValid())
14855 m_graph.watchpoints().addLazily(vm().primitiveGigacageEnabled());
14856 else
14857 return doUntagArrayPtr(ptr);
14858 }
14859
14860 LValue basePtr = m_out.constIntPtr(Gigacage::basePtr(kind));
14861 LValue mask = m_out.constIntPtr(Gigacage::mask(kind));
14862
14863 LValue masked = m_out.bitAnd(ptr, mask);
14864 LValue result = m_out.add(masked, basePtr);
14865
14866#if CPU(ARM64E)
14867 if (kind == Gigacage::Primitive) {
14868 PatchpointValue* merge = m_out.patchpoint(pointerType());
14869 merge->append(result, B3::ValueRep(B3::ValueRep::SomeLateRegister));
14870 merge->appendSomeRegister(ptr);
14871 merge->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
14872 jit.move(params[2].gpr(), params[0].gpr());
14873 jit.bitFieldInsert64(params[1].gpr(), 0, 64 - MacroAssembler::numberOfPACBits, params[0].gpr());
14874 });
14875
14876 result = doUntagArrayPtr(merge);
14877 }
14878#endif // CPU(ARM64E)
14879
14880 // Make sure that B3 doesn't try to do smart reassociation of these pointer bits.
14881 // FIXME: In an ideal world, B3 would not do harmful reassociations, and if it did, it would be able
14882 // to undo them during constant hoisting and regalloc. As it stands, if you remove this then Octane
14883 // gets 1.6% slower and Kraken gets 5% slower. It's all because the basePtr, which is a constant,
14884 // gets reassociated out of the add above and into the address arithmetic. This disables hoisting of
14885 // the basePtr constant. Hoisting that constant is worth a lot more perf than the reassociation. One
14886 // way to make this all work happily is to combine offset legalization with constant hoisting, and
14887 // then teach it reassociation. So, Add(Add(a, b), const) where a is loop-invariant while b isn't
14888 // will turn into Add(Add(a, const), b) by the constant hoister. We would have to teach B3 to do this
14889 // and possibly other smart things if we want to be able to remove this opaque.
14890 // https://bugs.webkit.org/show_bug.cgi?id=175493
14891 return m_out.opaque(result);
14892#endif
14893
14894 UNUSED_PARAM(kind);
14895 UNUSED_PARAM(base);
14896 return doUntagArrayPtr(ptr);
14897 }
14898
14899 void buildSwitch(SwitchData* data, LType type, LValue switchValue)
14900 {
14901 ASSERT(type == pointerType() || type == Int32);
14902
14903 Vector<SwitchCase> cases;
14904 for (unsigned i = 0; i < data->cases.size(); ++i) {
14905 SwitchCase newCase;
14906
14907 if (type == pointerType()) {
14908 newCase = SwitchCase(m_out.constIntPtr(data->cases[i].value.switchLookupValue(data->kind)),
14909 lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count));
14910 } else if (type == Int32) {
14911 newCase = SwitchCase(m_out.constInt32(data->cases[i].value.switchLookupValue(data->kind)),
14912 lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count));
14913 } else
14914 CRASH();
14915
14916 cases.append(newCase);
14917 }
14918
14919 m_out.switchInstruction(
14920 switchValue, cases,
14921 lowBlock(data->fallThrough.block), Weight(data->fallThrough.count));
14922 }
14923
14924 void switchString(SwitchData* data, LValue string, Edge& edge)
14925 {
14926 bool canDoBinarySwitch = true;
14927 unsigned totalLength = 0;
14928
14929 for (DFG::SwitchCase myCase : data->cases) {
14930 StringImpl* string = myCase.value.stringImpl();
14931 if (!string->is8Bit()) {
14932 canDoBinarySwitch = false;
14933 break;
14934 }
14935 if (string->length() > Options::maximumBinaryStringSwitchCaseLength()) {
14936 canDoBinarySwitch = false;
14937 break;
14938 }
14939 totalLength += string->length();
14940 }
14941
14942 if (!canDoBinarySwitch || totalLength > Options::maximumBinaryStringSwitchTotalLength()) {
14943 switchStringSlow(data, string);
14944 return;
14945 }
14946
14947 LBasicBlock hasImplBlock = m_out.newBlock();
14948 LBasicBlock is8BitBlock = m_out.newBlock();
14949 LBasicBlock slowBlock = m_out.newBlock();
14950
14951 m_out.branch(isRopeString(string, edge), unsure(slowBlock), unsure(hasImplBlock));
14952
14953 LBasicBlock lastNext = m_out.appendTo(hasImplBlock, is8BitBlock);
14954
14955 LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
14956 LValue length = m_out.load32(stringImpl, m_heaps.StringImpl_length);
14957
14958 m_out.branch(
14959 m_out.testIsZero32(
14960 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
14961 m_out.constInt32(StringImpl::flagIs8Bit())),
14962 unsure(slowBlock), unsure(is8BitBlock));
14963
14964 m_out.appendTo(is8BitBlock, slowBlock);
14965
14966 LValue buffer = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data);
14967
14968 // FIXME: We should propagate branch weight data to the cases of this switch.
14969 // https://bugs.webkit.org/show_bug.cgi?id=144368
14970
14971 Vector<StringSwitchCase> cases;
14972 for (DFG::SwitchCase myCase : data->cases)
14973 cases.append(StringSwitchCase(myCase.value.stringImpl(), lowBlock(myCase.target.block)));
14974 std::sort(cases.begin(), cases.end());
14975 switchStringRecurse(data, buffer, length, cases, 0, 0, cases.size(), 0, false);
14976
14977 m_out.appendTo(slowBlock, lastNext);
14978 switchStringSlow(data, string);
14979 }
14980
14981 // The code for string switching is based closely on the same code in the DFG backend. While it
14982 // would be nice to reduce the amount of similar-looking code, it seems like this is one of
14983 // those algorithms where factoring out the common bits would result in more code than just
14984 // duplicating.
14985
14986 struct StringSwitchCase {
14987 StringSwitchCase() { }
14988
14989 StringSwitchCase(StringImpl* string, LBasicBlock target)
14990 : string(string)
14991 , target(target)
14992 {
14993 }
14994
14995 bool operator<(const StringSwitchCase& other) const
14996 {
14997 return stringLessThan(*string, *other.string);
14998 }
14999
15000 StringImpl* string;
15001 LBasicBlock target;
15002 };
15003
15004 struct CharacterCase {
15005 CharacterCase()
15006 : character(0)
15007 , begin(0)
15008 , end(0)
15009 {
15010 }
15011
15012 CharacterCase(LChar character, unsigned begin, unsigned end)
15013 : character(character)
15014 , begin(begin)
15015 , end(end)
15016 {
15017 }
15018
15019 bool operator<(const CharacterCase& other) const
15020 {
15021 return character < other.character;
15022 }
15023
15024 LChar character;
15025 unsigned begin;
15026 unsigned end;
15027 };
15028
15029 void switchStringRecurse(
15030 SwitchData* data, LValue buffer, LValue length, const Vector<StringSwitchCase>& cases,
15031 unsigned numChecked, unsigned begin, unsigned end, unsigned alreadyCheckedLength,
15032 unsigned checkedExactLength)
15033 {
15034 LBasicBlock fallThrough = lowBlock(data->fallThrough.block);
15035
15036 if (begin == end) {
15037 m_out.jump(fallThrough);
15038 return;
15039 }
15040
15041 unsigned minLength = cases[begin].string->length();
15042 unsigned commonChars = minLength;
15043 bool allLengthsEqual = true;
15044 for (unsigned i = begin + 1; i < end; ++i) {
15045 unsigned myCommonChars = numChecked;
15046 unsigned limit = std::min(cases[begin].string->length(), cases[i].string->length());
15047 for (unsigned j = numChecked; j < limit; ++j) {
15048 if (cases[begin].string->at(j) != cases[i].string->at(j))
15049 break;
15050 myCommonChars++;
15051 }
15052 commonChars = std::min(commonChars, myCommonChars);
15053 if (minLength != cases[i].string->length())
15054 allLengthsEqual = false;
15055 minLength = std::min(minLength, cases[i].string->length());
15056 }
15057
15058 if (checkedExactLength) {
15059 DFG_ASSERT(m_graph, m_node, alreadyCheckedLength == minLength, alreadyCheckedLength, minLength);
15060 DFG_ASSERT(m_graph, m_node, allLengthsEqual);
15061 }
15062
15063 DFG_ASSERT(m_graph, m_node, minLength >= commonChars, minLength, commonChars);
15064
15065 if (!allLengthsEqual && alreadyCheckedLength < minLength)
15066 m_out.check(m_out.below(length, m_out.constInt32(minLength)), unsure(fallThrough));
15067 if (allLengthsEqual && (alreadyCheckedLength < minLength || !checkedExactLength))
15068 m_out.check(m_out.notEqual(length, m_out.constInt32(minLength)), unsure(fallThrough));
15069
15070 for (unsigned i = numChecked; i < commonChars; ++i) {
15071 m_out.check(
15072 m_out.notEqual(
15073 m_out.load8ZeroExt32(buffer, m_heaps.characters8[i]),
15074 m_out.constInt32(static_cast<uint16_t>(cases[begin].string->at(i)))),
15075 unsure(fallThrough));
15076 }
15077
15078 if (minLength == commonChars) {
15079 // This is the case where one of the cases is a prefix of all of the other cases.
15080 // We've already checked that the input string is a prefix of all of the cases,
15081 // so we just check length to jump to that case.
15082
15083 DFG_ASSERT(m_graph, m_node, cases[begin].string->length() == commonChars, cases[begin].string->length(), commonChars);
15084 for (unsigned i = begin + 1; i < end; ++i)
15085 DFG_ASSERT(m_graph, m_node, cases[i].string->length() > commonChars, cases[i].string->length(), commonChars);
15086
15087 if (allLengthsEqual) {
15088 DFG_ASSERT(m_graph, m_node, end == begin + 1, end, begin);
15089 m_out.jump(cases[begin].target);
15090 return;
15091 }
15092
15093 m_out.check(
15094 m_out.equal(length, m_out.constInt32(commonChars)),
15095 unsure(cases[begin].target));
15096
15097 // We've checked if the length is >= minLength, and then we checked if the length is
15098 // == commonChars. We get to this point if it is >= minLength but not == commonChars.
15099 // Hence we know that it now must be > minLength, i.e. that it's >= minLength + 1.
15100 switchStringRecurse(
15101 data, buffer, length, cases, commonChars, begin + 1, end, minLength + 1, false);
15102 return;
15103 }
15104
15105 // At this point we know that the string is longer than commonChars, and we've only verified
15106 // commonChars. Use a binary switch on the next unchecked character, i.e.
15107 // string[commonChars].
15108
15109 DFG_ASSERT(m_graph, m_node, end >= begin + 2, end, begin);
15110
15111 LValue uncheckedChar = m_out.load8ZeroExt32(buffer, m_heaps.characters8[commonChars]);
15112
15113 Vector<CharacterCase> characterCases;
15114 CharacterCase currentCase(cases[begin].string->at(commonChars), begin, begin + 1);
15115 for (unsigned i = begin + 1; i < end; ++i) {
15116 LChar currentChar = cases[i].string->at(commonChars);
15117 if (currentChar != currentCase.character) {
15118 currentCase.end = i;
15119 characterCases.append(currentCase);
15120 currentCase = CharacterCase(currentChar, i, i + 1);
15121 } else
15122 currentCase.end = i + 1;
15123 }
15124 characterCases.append(currentCase);
15125
15126 Vector<LBasicBlock> characterBlocks;
15127 for (unsigned i = characterCases.size(); i--;)
15128 characterBlocks.append(m_out.newBlock());
15129
15130 Vector<SwitchCase> switchCases;
15131 for (unsigned i = 0; i < characterCases.size(); ++i) {
15132 if (i)
15133 DFG_ASSERT(m_graph, m_node, characterCases[i - 1].character < characterCases[i].character);
15134 switchCases.append(SwitchCase(
15135 m_out.constInt32(characterCases[i].character), characterBlocks[i], Weight()));
15136 }
15137 m_out.switchInstruction(uncheckedChar, switchCases, fallThrough, Weight());
15138
15139 LBasicBlock lastNext = m_out.m_nextBlock;
15140 characterBlocks.append(lastNext); // Makes it convenient to set nextBlock.
15141 for (unsigned i = 0; i < characterCases.size(); ++i) {
15142 m_out.appendTo(characterBlocks[i], characterBlocks[i + 1]);
15143 switchStringRecurse(
15144 data, buffer, length, cases, commonChars + 1,
15145 characterCases[i].begin, characterCases[i].end, minLength, allLengthsEqual);
15146 }
15147
15148 DFG_ASSERT(m_graph, m_node, m_out.m_nextBlock == lastNext);
15149 }
15150
15151 void switchStringSlow(SwitchData* data, LValue string)
15152 {
15153 // FIXME: We ought to be able to use computed gotos here. We would save the labels of the
15154 // blocks we want to jump to, and then request their addresses after compilation completes.
15155 // https://bugs.webkit.org/show_bug.cgi?id=144369
15156
15157 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
15158
15159 LValue branchOffset = vmCall(
15160 Int32, operationSwitchStringAndGetBranchOffset,
15161 weakPointer(globalObject), m_out.constIntPtr(data->switchTableIndex), string);
15162
15163 StringJumpTable& table = codeBlock()->stringSwitchJumpTable(data->switchTableIndex);
15164
15165 Vector<SwitchCase> cases;
15166 // These may be negative, or zero, or probably other stuff, too. We don't want to mess with HashSet's corner cases and we don't really care about throughput here.
15167 StdUnorderedSet<int32_t> alreadyHandled;
15168 for (unsigned i = 0; i < data->cases.size(); ++i) {
15169 // FIXME: The fact that we're using the bytecode's switch table means that the
15170 // following DFG IR transformation would be invalid.
15171 //
15172 // Original code:
15173 // switch (v) {
15174 // case "foo":
15175 // case "bar":
15176 // things();
15177 // break;
15178 // default:
15179 // break;
15180 // }
15181 //
15182 // New code:
15183 // switch (v) {
15184 // case "foo":
15185 // instrumentFoo();
15186 // goto _things;
15187 // case "bar":
15188 // instrumentBar();
15189 // _things:
15190 // things();
15191 // break;
15192 // default:
15193 // break;
15194 // }
15195 //
15196 // Luckily, we don't currently do any such transformation. But it's kind of silly that
15197 // this is an issue.
15198 // https://bugs.webkit.org/show_bug.cgi?id=144635
15199
15200 DFG::SwitchCase myCase = data->cases[i];
15201 StringJumpTable::StringOffsetTable::iterator iter =
15202 table.offsetTable.find(myCase.value.stringImpl());
15203 DFG_ASSERT(m_graph, m_node, iter != table.offsetTable.end());
15204
15205 if (!alreadyHandled.insert(iter->value.branchOffset).second)
15206 continue;
15207
15208 cases.append(SwitchCase(
15209 m_out.constInt32(iter->value.branchOffset),
15210 lowBlock(myCase.target.block), Weight(myCase.target.count)));
15211 }
15212
15213 m_out.switchInstruction(
15214 branchOffset, cases, lowBlock(data->fallThrough.block),
15215 Weight(data->fallThrough.count));
15216 }
15217
15218 // Calls the functor at the point of code generation where we know what the result type is.
15219 // You can emit whatever code you like at that point. Expects you to terminate the basic block.
15220 // When buildTypeOf() returns, it will have terminated all basic blocks that it created. So, if
15221 // you aren't using this as the terminator of a high-level block, you should create your own
15222 // contination and set it as the nextBlock (m_out.insertNewBlocksBefore(continuation)) before
15223 // calling this. For example:
15224 //
15225 // LBasicBlock continuation = m_out.newBlock();
15226 // LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
15227 // buildTypeOf(
15228 // child, value,
15229 // [&] (TypeofType type) {
15230 // do things;
15231 // m_out.jump(continuation);
15232 // });
15233 // m_out.appendTo(continuation, lastNext);
15234 template<typename Functor>
15235 void buildTypeOf(Edge child, LValue value, const Functor& functor)
15236 {
15237 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
15238
15239 // Implements the following branching structure:
15240 //
15241 // if (is cell) {
15242 // if (is object) {
15243 // if (is function) {
15244 // return function;
15245 // } else if (doesn't have call trap and doesn't masquerade as undefined) {
15246 // return object
15247 // } else {
15248 // return slowPath();
15249 // }
15250 // } else if (is string) {
15251 // return string
15252 // } else if (is bigint) {
15253 // return bigint
15254 // } else {
15255 // return symbol
15256 // }
15257 // } else if (is number) {
15258 // return number
15259 // } else if (is null) {
15260 // return object
15261 // } else if (is boolean) {
15262 // return boolean
15263 // } else {
15264 // return undefined
15265 // }
15266 //
15267 // FIXME: typeof Symbol should be more frequently seen than BigInt.
15268 // We should change the order of type detection based on this frequency.
15269 // https://bugs.webkit.org/show_bug.cgi?id=192650
15270
15271 LBasicBlock cellCase = m_out.newBlock();
15272 LBasicBlock objectCase = m_out.newBlock();
15273 LBasicBlock functionCase = m_out.newBlock();
15274 LBasicBlock notFunctionCase = m_out.newBlock();
15275 LBasicBlock reallyObjectCase = m_out.newBlock();
15276 LBasicBlock slowPath = m_out.newBlock();
15277 LBasicBlock unreachable = m_out.newBlock();
15278 LBasicBlock notObjectCase = m_out.newBlock();
15279 LBasicBlock stringCase = m_out.newBlock();
15280 LBasicBlock notStringCase = m_out.newBlock();
15281 LBasicBlock bigIntCase = m_out.newBlock();
15282 LBasicBlock symbolCase = m_out.newBlock();
15283 LBasicBlock notCellCase = m_out.newBlock();
15284 LBasicBlock numberCase = m_out.newBlock();
15285 LBasicBlock notNumberCase = m_out.newBlock();
15286 LBasicBlock notNullCase = m_out.newBlock();
15287 LBasicBlock booleanCase = m_out.newBlock();
15288 LBasicBlock undefinedCase = m_out.newBlock();
15289
15290 m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase));
15291
15292 LBasicBlock lastNext = m_out.appendTo(cellCase, objectCase);
15293 m_out.branch(isObject(value, provenType(child)), unsure(objectCase), unsure(notObjectCase));
15294
15295 m_out.appendTo(objectCase, functionCase);
15296 m_out.branch(
15297 isFunction(value, provenType(child) & SpecObject),
15298 unsure(functionCase), unsure(notFunctionCase));
15299
15300 m_out.appendTo(functionCase, notFunctionCase);
15301 functor(TypeofType::Function);
15302
15303 m_out.appendTo(notFunctionCase, reallyObjectCase);
15304 m_out.branch(
15305 isExoticForTypeof(value, provenType(child) & (SpecObject - SpecFunction)),
15306 rarely(slowPath), usually(reallyObjectCase));
15307
15308 m_out.appendTo(reallyObjectCase, slowPath);
15309 functor(TypeofType::Object);
15310
15311 m_out.appendTo(slowPath, unreachable);
15312 VM& vm = this->vm();
15313 LValue result = lazySlowPath(
15314 [=, &vm] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
15315 return createLazyCallGenerator(vm,
15316 operationTypeOfObjectAsTypeofType, locations[0].directGPR(),
15317 CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR());
15318 }, value);
15319 Vector<SwitchCase, 3> cases;
15320 cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Undefined)), undefinedCase));
15321 cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Object)), reallyObjectCase));
15322 cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Function)), functionCase));
15323 m_out.switchInstruction(m_out.castToInt32(result), cases, unreachable, Weight());
15324
15325 m_out.appendTo(unreachable, notObjectCase);
15326 m_out.unreachable();
15327
15328 m_out.appendTo(notObjectCase, stringCase);
15329 m_out.branch(
15330 isString(value, provenType(child) & (SpecCell - SpecObject)),
15331 unsure(stringCase), unsure(notStringCase));
15332
15333 m_out.appendTo(stringCase, notStringCase);
15334 functor(TypeofType::String);
15335
15336 m_out.appendTo(notStringCase, bigIntCase);
15337 m_out.branch(
15338 isBigInt(value, provenType(child) & (SpecCell - SpecObject - SpecString)),
15339 unsure(bigIntCase), unsure(symbolCase));
15340
15341 m_out.appendTo(bigIntCase, symbolCase);
15342 functor(TypeofType::BigInt);
15343
15344 m_out.appendTo(symbolCase, notCellCase);
15345 functor(TypeofType::Symbol);
15346
15347 m_out.appendTo(notCellCase, numberCase);
15348 m_out.branch(
15349 isNumber(value, provenType(child) & ~SpecCell),
15350 unsure(numberCase), unsure(notNumberCase));
15351
15352 m_out.appendTo(numberCase, notNumberCase);
15353 functor(TypeofType::Number);
15354
15355 m_out.appendTo(notNumberCase, notNullCase);
15356 LValue isNull;
15357 if (provenType(child) & SpecOther)
15358 isNull = m_out.equal(value, m_out.constInt64(JSValue::ValueNull));
15359 else
15360 isNull = m_out.booleanFalse;
15361 m_out.branch(isNull, unsure(reallyObjectCase), unsure(notNullCase));
15362
15363 m_out.appendTo(notNullCase, booleanCase);
15364 m_out.branch(
15365 isBoolean(value, provenType(child) & ~(SpecCell | SpecFullNumber)),
15366 unsure(booleanCase), unsure(undefinedCase));
15367
15368 m_out.appendTo(booleanCase, undefinedCase);
15369 functor(TypeofType::Boolean);
15370
15371 m_out.appendTo(undefinedCase, lastNext);
15372 functor(TypeofType::Undefined);
15373 }
15374
15375 TypedPointer pointerIntoTypedArray(LValue storage, LValue index, TypedArrayType type)
15376 {
15377 LValue offset = m_out.shl(m_out.zeroExtPtr(index), m_out.constIntPtr(logElementSize(type)));
15378
15379 return TypedPointer(
15380 m_heaps.typedArrayProperties,
15381 m_out.add(
15382 storage,
15383 offset
15384 ));
15385 }
15386
15387 LValue loadFromIntTypedArray(TypedPointer pointer, TypedArrayType type)
15388 {
15389 switch (elementSize(type)) {
15390 case 1:
15391 return isSigned(type) ? m_out.load8SignExt32(pointer) : m_out.load8ZeroExt32(pointer);
15392 case 2:
15393 return isSigned(type) ? m_out.load16SignExt32(pointer) : m_out.load16ZeroExt32(pointer);
15394 case 4:
15395 return m_out.load32(pointer);
15396 default:
15397 DFG_CRASH(m_graph, m_node, "Bad element size");
15398 }
15399 }
15400
15401 Output::StoreType storeType(TypedArrayType type)
15402 {
15403 if (isInt(type)) {
15404 switch (elementSize(type)) {
15405 case 1:
15406 return Output::Store32As8;
15407 case 2:
15408 return Output::Store32As16;
15409 case 4:
15410 return Output::Store32;
15411 default:
15412 DFG_CRASH(m_graph, m_node, "Bad element size");
15413 return Output::Store32;
15414 }
15415 }
15416 switch (type) {
15417 case TypeFloat32:
15418 return Output::StoreFloat;
15419 case TypeFloat64:
15420 return Output::StoreDouble;
15421 default:
15422 DFG_CRASH(m_graph, m_node, "Bad typed array type");
15423 }
15424 }
15425
15426 void setIntTypedArrayLoadResult(LValue result, TypedArrayType type, bool canSpeculate = false)
15427 {
15428 if (elementSize(type) < 4 || isSigned(type)) {
15429 setInt32(result);
15430 return;
15431 }
15432
15433 if (m_node->shouldSpeculateInt32() && canSpeculate) {
15434 speculate(
15435 Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
15436 setInt32(result);
15437 return;
15438 }
15439
15440 if (m_node->shouldSpeculateInt52()) {
15441 setStrictInt52(m_out.zeroExt(result, Int64));
15442 return;
15443 }
15444
15445 setDouble(m_out.unsignedToDouble(result));
15446 }
15447
15448 LValue getIntTypedArrayStoreOperand(Edge edge, bool isClamped = false)
15449 {
15450 LValue intValue;
15451 switch (edge.useKind()) {
15452 case Int52RepUse:
15453 case Int32Use: {
15454 if (edge.useKind() == Int32Use)
15455 intValue = lowInt32(edge);
15456 else
15457 intValue = m_out.castToInt32(lowStrictInt52(edge));
15458
15459 if (isClamped) {
15460 LBasicBlock atLeastZero = m_out.newBlock();
15461 LBasicBlock continuation = m_out.newBlock();
15462
15463 Vector<ValueFromBlock, 2> intValues;
15464 intValues.append(m_out.anchor(m_out.int32Zero));
15465 m_out.branch(
15466 m_out.lessThan(intValue, m_out.int32Zero),
15467 unsure(continuation), unsure(atLeastZero));
15468
15469 LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
15470
15471 intValues.append(m_out.anchor(m_out.select(
15472 m_out.greaterThan(intValue, m_out.constInt32(255)),
15473 m_out.constInt32(255),
15474 intValue)));
15475 m_out.jump(continuation);
15476
15477 m_out.appendTo(continuation, lastNext);
15478 intValue = m_out.phi(Int32, intValues);
15479 }
15480 break;
15481 }
15482
15483 case DoubleRepUse: {
15484 LValue doubleValue = lowDouble(edge);
15485
15486 if (isClamped) {
15487 LBasicBlock atLeastZero = m_out.newBlock();
15488 LBasicBlock withinRange = m_out.newBlock();
15489 LBasicBlock continuation = m_out.newBlock();
15490
15491 Vector<ValueFromBlock, 3> intValues;
15492 intValues.append(m_out.anchor(m_out.int32Zero));
15493 m_out.branch(
15494 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
15495 unsure(continuation), unsure(atLeastZero));
15496
15497 LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
15498 intValues.append(m_out.anchor(m_out.constInt32(255)));
15499 m_out.branch(
15500 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
15501 unsure(continuation), unsure(withinRange));
15502
15503 m_out.appendTo(withinRange, continuation);
15504 intValues.append(m_out.anchor(m_out.doubleToInt(doubleValue)));
15505 m_out.jump(continuation);
15506
15507 m_out.appendTo(continuation, lastNext);
15508 intValue = m_out.phi(Int32, intValues);
15509 } else
15510 intValue = doubleToInt32(doubleValue);
15511 break;
15512 }
15513
15514 default:
15515 DFG_CRASH(m_graph, m_node, "Bad use kind");
15516 }
15517
15518 return intValue;
15519 }
15520
15521 LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
15522 {
15523 LBasicBlock greatEnough = m_out.newBlock();
15524 LBasicBlock withinRange = m_out.newBlock();
15525 LBasicBlock slowPath = m_out.newBlock();
15526 LBasicBlock continuation = m_out.newBlock();
15527
15528 Vector<ValueFromBlock, 2> results;
15529
15530 m_out.branch(
15531 m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)),
15532 unsure(greatEnough), unsure(slowPath));
15533
15534 LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
15535 m_out.branch(
15536 m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)),
15537 unsure(withinRange), unsure(slowPath));
15538
15539 m_out.appendTo(withinRange, slowPath);
15540 LValue fastResult;
15541 if (isSigned)
15542 fastResult = m_out.doubleToInt(doubleValue);
15543 else
15544 fastResult = m_out.doubleToUInt(doubleValue);
15545 results.append(m_out.anchor(fastResult));
15546 m_out.jump(continuation);
15547
15548 m_out.appendTo(slowPath, continuation);
15549 results.append(m_out.anchor(m_out.callWithoutSideEffects(Int32, operationToInt32, doubleValue)));
15550 m_out.jump(continuation);
15551
15552 m_out.appendTo(continuation, lastNext);
15553 return m_out.phi(Int32, results);
15554 }
15555
15556 LValue doubleToInt32(LValue doubleValue)
15557 {
15558#if CPU(ARM64)
15559 if (MacroAssemblerARM64::supportsDoubleToInt32ConversionUsingJavaScriptSemantics()) {
15560 PatchpointValue* patchpoint = m_out.patchpoint(Int32);
15561 patchpoint->append(ConstrainedValue(doubleValue, B3::ValueRep::SomeRegister));
15562 patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
15563 jit.convertDoubleToInt32UsingJavaScriptSemantics(params[1].fpr(), params[0].gpr());
15564 });
15565 patchpoint->effects = Effects::none();
15566 return patchpoint;
15567 }
15568#endif
15569
15570 if (hasSensibleDoubleToInt())
15571 return sensibleDoubleToInt32(doubleValue);
15572
15573 double limit = pow(2, 31) - 1;
15574 return doubleToInt32(doubleValue, -limit, limit);
15575 }
15576
15577 LValue sensibleDoubleToInt32(LValue doubleValue)
15578 {
15579 LBasicBlock slowPath = m_out.newBlock();
15580 LBasicBlock continuation = m_out.newBlock();
15581
15582 LValue fastResultValue = m_out.doubleToInt(doubleValue);
15583 ValueFromBlock fastResult = m_out.anchor(fastResultValue);
15584 m_out.branch(
15585 m_out.equal(fastResultValue, m_out.constInt32(0x80000000)),
15586 rarely(slowPath), usually(continuation));
15587
15588 LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
15589 ValueFromBlock slowResult = m_out.anchor(m_out.callWithoutSideEffects(Int32, operationToInt32SensibleSlow, doubleValue));
15590 m_out.jump(continuation);
15591
15592 m_out.appendTo(continuation, lastNext);
15593 return m_out.phi(Int32, fastResult, slowResult);
15594 }
15595
15596 // This is a mechanism for creating a code generator that fills in a gap in the code using our
15597 // own MacroAssembler. This is useful for slow paths that involve a lot of code and we don't want
15598 // to pay the price of B3 optimizing it. A lazy slow path will only be generated if it actually
15599 // executes. On the other hand, a lazy slow path always incurs the cost of two additional jumps.
15600 // Also, the lazy slow path's register allocation state is slaved to whatever B3 did, so you
15601 // have to use a ScratchRegisterAllocator to try to use some unused registers and you may have
15602 // to spill to top of stack if there aren't enough registers available.
15603 //
15604 // Lazy slow paths involve three different stages of execution. Each stage has unique
15605 // capabilities and knowledge. The stages are:
15606 //
15607 // 1) DFG->B3 lowering, i.e. code that runs in this phase. Lowering is the last time you will
15608 // have access to LValues. If there is an LValue that needs to be fed as input to a lazy slow
15609 // path, then you must pass it as an argument here (as one of the varargs arguments after the
15610 // functor). But, lowering doesn't know which registers will be used for those LValues. Hence
15611 // you pass a lambda to lazySlowPath() and that lambda will run during stage (2):
15612 //
15613 // 2) FTLCompile.cpp's fixFunctionBasedOnStackMaps. This code is the only stage at which we know
15614 // the mapping from arguments passed to this method in (1) and the registers that B3
15615 // selected for those arguments. You don't actually want to generate any code here, since then
15616 // the slow path wouldn't actually be lazily generated. Instead, you want to save the
15617 // registers being used for the arguments and defer code generation to stage (3) by creating
15618 // and returning a LazySlowPath::Generator:
15619 //
15620 // 3) LazySlowPath's generate() method. This code runs in response to the lazy slow path
15621 // executing for the first time. It will call the generator you created in stage (2).
15622 //
15623 // Note that each time you invoke stage (1), stage (2) may be invoked zero, one, or many times.
15624 // Stage (2) will usually be invoked once for stage (1). But, B3 may kill the code, in which
15625 // case stage (2) won't run. B3 may duplicate the code (for example via tail duplication),
15626 // leading to many calls to your stage (2) lambda. Stage (3) may be called zero or once for each
15627 // stage (2). It will be called zero times if the slow path never runs. This is what you hope for
15628 // whenever you use the lazySlowPath() mechanism.
15629 //
15630 // A typical use of lazySlowPath() will look like the example below, which just creates a slow
15631 // path that adds some value to the input and returns it.
15632 //
15633 // // Stage (1) is here. This is your last chance to figure out which LValues to use as inputs.
15634 // // Notice how we pass "input" as an argument to lazySlowPath().
15635 // LValue input = ...;
15636 // int addend = ...;
15637 // LValue output = lazySlowPath(
15638 // [=] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
15639 // // Stage (2) is here. This is your last chance to figure out which registers are used
15640 // // for which values. Location zero is always the return value. You can ignore it if
15641 // // you don't want to return anything. Location 1 is the register for the first
15642 // // argument to the lazySlowPath(), i.e. "input". Note that the Location object could
15643 // // also hold an FPR, if you are passing a double.
15644 // GPRReg outputGPR = locations[0].directGPR();
15645 // GPRReg inputGPR = locations[1].directGPR();
15646 // return LazySlowPath::createGenerator(
15647 // [=] (CCallHelpers& jit, LazySlowPath::GenerationParams& params) {
15648 // // Stage (3) is here. This is when you generate code. You have access to the
15649 // // registers you collected in stage (2) because this lambda closes over those
15650 // // variables (outputGPR and inputGPR). You also have access to whatever extra
15651 // // data you collected in stage (1), such as the addend in this case.
15652 // jit.add32(TrustedImm32(addend), inputGPR, outputGPR);
15653 // // You have to end by jumping to done. There is nothing to fall through to.
15654 // // You can also jump to the exception handler (see LazySlowPath.h for more
15655 // // info). Note that currently you cannot OSR exit.
15656 // params.doneJumps.append(jit.jump());
15657 // });
15658 // },
15659 // input);
15660 //
15661 // You can basically pass as many inputs as you like, either using this varargs form, or by
15662 // passing a Vector of LValues.
15663 //
15664 // Note that if your slow path is only doing a call, you can use the createLazyCallGenerator()
15665 // helper. For example:
15666 //
15667 // LValue input = ...;
15668 // LValue output = lazySlowPath(
15669 // [=] (const Vector<Location>& locations) -> RefPtr<LazySlowPath::Generator> {
15670 // return createLazyCallGenerator(
15671 // operationDoThings, locations[0].directGPR(), locations[1].directGPR());
15672 // }, input);
15673 //
15674 // Finally, note that all of the lambdas - both the stage (2) lambda and the stage (3) lambda -
15675 // run after the function that created them returns. Hence, you should not use by-reference
15676 // capture (i.e. [&]) in any of these lambdas.
15677 template<typename Functor, typename... ArgumentTypes>
15678 PatchpointValue* lazySlowPath(const Functor& functor, ArgumentTypes... arguments)
15679 {
15680 return lazySlowPath(functor, Vector<LValue>{ arguments... });
15681 }
15682
15683 template<typename Functor>
15684 PatchpointValue* lazySlowPath(const Functor& functor, const Vector<LValue>& userArguments)
15685 {
15686 CodeOrigin origin = m_node->origin.semantic;
15687
15688 PatchpointValue* result = m_out.patchpoint(B3::Int64);
15689 for (LValue arg : userArguments)
15690 result->append(ConstrainedValue(arg, B3::ValueRep::SomeRegister));
15691
15692 RefPtr<PatchpointExceptionHandle> exceptionHandle =
15693 preparePatchpointForExceptions(result);
15694
15695 result->clobber(RegisterSet::macroScratchRegisters());
15696 State* state = &m_ftlState;
15697
15698 result->setGenerator(
15699 [=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
15700 Vector<Location> locations;
15701 for (const B3::ValueRep& rep : params)
15702 locations.append(Location::forValueRep(rep));
15703
15704 RefPtr<LazySlowPath::Generator> generator = functor(locations);
15705
15706 CCallHelpers::PatchableJump patchableJump = jit.patchableJump();
15707 CCallHelpers::Label done = jit.label();
15708
15709 RegisterSet usedRegisters = params.unavailableRegisters();
15710
15711 RefPtr<ExceptionTarget> exceptionTarget =
15712 exceptionHandle->scheduleExitCreation(params);
15713
15714 // FIXME: As part of handling exceptions, we need to create a concrete OSRExit here.
15715 // Doing so should automagically register late paths that emit exit thunks.
15716
15717 params.addLatePath(
15718 [=] (CCallHelpers& jit) {
15719 AllowMacroScratchRegisterUsage allowScratch(jit);
15720 patchableJump.m_jump.link(&jit);
15721 unsigned index = state->jitCode->lazySlowPaths.size();
15722 state->jitCode->lazySlowPaths.append(nullptr);
15723 jit.pushToSaveImmediateWithoutTouchingRegisters(
15724 CCallHelpers::TrustedImm32(index));
15725 CCallHelpers::Jump generatorJump = jit.jump();
15726
15727 // Note that so long as we're here, we don't really know if our late path
15728 // runs before or after any other late paths that we might depend on, like
15729 // the exception thunk.
15730
15731 RefPtr<JITCode> jitCode = state->jitCode;
15732 VM* vm = &state->graph.m_vm;
15733
15734 jit.addLinkTask(
15735 [=] (LinkBuffer& linkBuffer) {
15736 linkBuffer.link(generatorJump,
15737 CodeLocationLabel<JITThunkPtrTag>(vm->getCTIStub(lazySlowPathGenerationThunkGenerator).code()));
15738
15739 std::unique_ptr<LazySlowPath> lazySlowPath = makeUnique<LazySlowPath>();
15740
15741 auto linkedPatchableJump = CodeLocationJump<JSInternalPtrTag>(linkBuffer.locationOf<JSInternalPtrTag>(patchableJump));
15742
15743 CodeLocationLabel<JSInternalPtrTag> linkedDone = linkBuffer.locationOf<JSInternalPtrTag>(done);
15744
15745 CallSiteIndex callSiteIndex =
15746 jitCode->common.addUniqueCallSiteIndex(origin);
15747
15748 lazySlowPath->initialize(
15749 linkedPatchableJump, linkedDone,
15750 exceptionTarget->label(linkBuffer), usedRegisters,
15751 callSiteIndex, generator);
15752
15753 jitCode->lazySlowPaths[index] = WTFMove(lazySlowPath);
15754 });
15755 });
15756 });
15757 return result;
15758 }
15759
15760 void speculate(
15761 ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
15762 {
15763 appendOSRExit(kind, lowValue, highValue, failCondition, m_origin);
15764 }
15765
15766 void speculate(
15767 ExitKind kind, FormattedValue lowValue, const MethodOfGettingAValueProfile& profile, LValue failCondition)
15768 {
15769 appendOSRExit(kind, lowValue, profile, failCondition, m_origin);
15770 }
15771
15772 void terminate(ExitKind kind)
15773 {
15774 speculate(kind, noValue(), nullptr, m_out.booleanTrue);
15775 didAlreadyTerminate();
15776 }
15777
15778 void didAlreadyTerminate()
15779 {
15780 m_state.setIsValid(false);
15781 }
15782
15783 void simulatedTypeCheck(Edge highValue, SpeculatedType typesPassedThrough)
15784 {
15785 m_interpreter.filter(highValue, typesPassedThrough);
15786 }
15787
15788 void typeCheck(
15789 FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
15790 LValue failCondition, ExitKind exitKind = BadType)
15791 {
15792 appendTypeCheck(lowValue, highValue, typesPassedThrough, failCondition, exitKind);
15793 }
15794
15795 void appendTypeCheck(
15796 FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
15797 LValue failCondition, ExitKind exitKind)
15798 {
15799 if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
15800 return;
15801 ASSERT(mayHaveTypeCheck(highValue.useKind()));
15802 appendOSRExit(exitKind, lowValue, highValue.node(), failCondition, m_origin);
15803 m_interpreter.filter(highValue, typesPassedThrough);
15804 }
15805
15806 LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
15807 {
15808 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
15809
15810 if (edge->hasConstant()) {
15811 JSValue value = edge->asJSValue();
15812 simulatedTypeCheck(edge, SpecInt32Only);
15813 if (!value.isInt32()) {
15814 if (mayHaveTypeCheck(edge.useKind()))
15815 terminate(Uncountable);
15816 return m_out.int32Zero;
15817 }
15818 LValue result = m_out.constInt32(value.asInt32());
15819 result->setOrigin(B3::Origin(edge.node()));
15820 return result;
15821 }
15822
15823 LoweredNodeValue value = m_int32Values.get(edge.node());
15824 if (isValid(value)) {
15825 simulatedTypeCheck(edge, SpecInt32Only);
15826 return value.value();
15827 }
15828
15829 value = m_strictInt52Values.get(edge.node());
15830 if (isValid(value))
15831 return strictInt52ToInt32(edge, value.value());
15832
15833 value = m_int52Values.get(edge.node());
15834 if (isValid(value))
15835 return strictInt52ToInt32(edge, int52ToStrictInt52(value.value()));
15836
15837 value = m_jsValueValues.get(edge.node());
15838 if (isValid(value)) {
15839 LValue boxedResult = value.value();
15840 FTL_TYPE_CHECK(
15841 jsValueValue(boxedResult), edge, SpecInt32Only, isNotInt32(boxedResult));
15842 LValue result = unboxInt32(boxedResult);
15843 setInt32(edge.node(), result);
15844 return result;
15845 }
15846
15847 DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecInt32Only), provenType(edge));
15848 if (mayHaveTypeCheck(edge.useKind()))
15849 terminate(Uncountable);
15850 return m_out.int32Zero;
15851 }
15852
15853 enum Int52Kind { StrictInt52, Int52 };
15854 LValue lowInt52(Edge edge, Int52Kind kind)
15855 {
15856 DFG_ASSERT(m_graph, m_node, edge.useKind() == Int52RepUse, edge.useKind());
15857
15858 LoweredNodeValue value;
15859
15860 switch (kind) {
15861 case Int52:
15862 value = m_int52Values.get(edge.node());
15863 if (isValid(value))
15864 return value.value();
15865
15866 value = m_strictInt52Values.get(edge.node());
15867 if (isValid(value))
15868 return strictInt52ToInt52(value.value());
15869 break;
15870
15871 case StrictInt52:
15872 value = m_strictInt52Values.get(edge.node());
15873 if (isValid(value))
15874 return value.value();
15875
15876 value = m_int52Values.get(edge.node());
15877 if (isValid(value))
15878 return int52ToStrictInt52(value.value());
15879 break;
15880 }
15881
15882 DFG_ASSERT(m_graph, m_node, !provenType(edge), provenType(edge));
15883 if (mayHaveTypeCheck(edge.useKind()))
15884 terminate(Uncountable);
15885 return m_out.int64Zero;
15886 }
15887
15888 LValue lowInt52(Edge edge)
15889 {
15890 return lowInt52(edge, Int52);
15891 }
15892
15893 LValue lowStrictInt52(Edge edge)
15894 {
15895 return lowInt52(edge, StrictInt52);
15896 }
15897
15898 bool betterUseStrictInt52(Node* node)
15899 {
15900 return !isValid(m_int52Values.get(node));
15901 }
15902 bool betterUseStrictInt52(Edge edge)
15903 {
15904 return betterUseStrictInt52(edge.node());
15905 }
15906 template<typename T>
15907 Int52Kind bestInt52Kind(T node)
15908 {
15909 return betterUseStrictInt52(node) ? StrictInt52 : Int52;
15910 }
15911 Int52Kind opposite(Int52Kind kind)
15912 {
15913 switch (kind) {
15914 case Int52:
15915 return StrictInt52;
15916 case StrictInt52:
15917 return Int52;
15918 }
15919 DFG_CRASH(m_graph, m_node, "Bad use kind");
15920 return Int52;
15921 }
15922
15923 LValue lowWhicheverInt52(Edge edge, Int52Kind& kind)
15924 {
15925 kind = bestInt52Kind(edge);
15926 return lowInt52(edge, kind);
15927 }
15928
15929 LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
15930 {
15931 DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind()), edge.useKind());
15932
15933 if (edge->op() == JSConstant) {
15934 FrozenValue* value = edge->constant();
15935 simulatedTypeCheck(edge, SpecCellCheck);
15936 if (!value->value().isCell()) {
15937 if (mayHaveTypeCheck(edge.useKind()))
15938 terminate(Uncountable);
15939 return m_out.intPtrZero;
15940 }
15941 LValue result = frozenPointer(value);
15942 result->setOrigin(B3::Origin(edge.node()));
15943 return result;
15944 }
15945
15946 LoweredNodeValue value = m_jsValueValues.get(edge.node());
15947 if (isValid(value)) {
15948 LValue uncheckedValue = value.value();
15949 FTL_TYPE_CHECK(
15950 jsValueValue(uncheckedValue), edge, SpecCellCheck, isNotCell(uncheckedValue));
15951 return uncheckedValue;
15952 }
15953
15954 DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecCellCheck), provenType(edge));
15955 if (mayHaveTypeCheck(edge.useKind()))
15956 terminate(Uncountable);
15957 return m_out.intPtrZero;
15958 }
15959
15960 LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
15961 {
15962 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
15963
15964 LValue result = lowCell(edge, mode);
15965 speculateObject(edge, result);
15966 return result;
15967 }
15968
15969 LValue lowRegExpObject(Edge edge)
15970 {
15971 LValue result = lowCell(edge);
15972 speculateRegExpObject(edge, result);
15973 return result;
15974 }
15975
15976 LValue lowMapObject(Edge edge)
15977 {
15978 LValue result = lowCell(edge);
15979 speculateMapObject(edge, result);
15980 return result;
15981 }
15982
15983 LValue lowSetObject(Edge edge)
15984 {
15985 LValue result = lowCell(edge);
15986 speculateSetObject(edge, result);
15987 return result;
15988 }
15989
15990 LValue lowWeakMapObject(Edge edge)
15991 {
15992 LValue result = lowCell(edge);
15993 speculateWeakMapObject(edge, result);
15994 return result;
15995 }
15996
15997 LValue lowWeakSetObject(Edge edge)
15998 {
15999 LValue result = lowCell(edge);
16000 speculateWeakSetObject(edge, result);
16001 return result;
16002 }
16003
16004 LValue lowDataViewObject(Edge edge)
16005 {
16006 LValue result = lowCell(edge);
16007 speculateDataViewObject(edge, result);
16008 return result;
16009 }
16010
16011 LValue lowDateObject(Edge edge)
16012 {
16013 LValue result = lowCell(edge);
16014 speculateDateObject(edge, result);
16015 return result;
16016 }
16017
16018 LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16019 {
16020 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse);
16021
16022 LValue result = lowCell(edge, mode);
16023 speculateString(edge, result);
16024 return result;
16025 }
16026
16027 LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16028 {
16029 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse);
16030
16031 LValue string = lowString(edge, mode);
16032 LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
16033 speculateStringIdent(edge, string, stringImpl);
16034 return stringImpl;
16035 }
16036
16037 LValue lowSymbol(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16038 {
16039 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == SymbolUse);
16040
16041 LValue result = lowCell(edge, mode);
16042 speculateSymbol(edge, result);
16043 return result;
16044 }
16045
16046 LValue lowBigInt(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16047 {
16048 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BigIntUse);
16049
16050 LValue result = lowCell(edge, mode);
16051 speculateBigInt(edge, result);
16052 return result;
16053 }
16054
16055 LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16056 {
16057 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
16058
16059 LValue result = lowCell(edge, mode);
16060 speculateNonNullObject(edge, result);
16061 return result;
16062 }
16063
16064 LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16065 {
16066 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse || edge.useKind() == KnownBooleanUse);
16067
16068 if (edge->hasConstant()) {
16069 JSValue value = edge->asJSValue();
16070 simulatedTypeCheck(edge, SpecBoolean);
16071 if (!value.isBoolean()) {
16072 if (mayHaveTypeCheck(edge.useKind()))
16073 terminate(Uncountable);
16074 return m_out.booleanFalse;
16075 }
16076 LValue result = m_out.constBool(value.asBoolean());
16077 result->setOrigin(B3::Origin(edge.node()));
16078 return result;
16079 }
16080
16081 LoweredNodeValue value = m_booleanValues.get(edge.node());
16082 if (isValid(value)) {
16083 simulatedTypeCheck(edge, SpecBoolean);
16084 return value.value();
16085 }
16086
16087 value = m_jsValueValues.get(edge.node());
16088 if (isValid(value)) {
16089 LValue unboxedResult = value.value();
16090 FTL_TYPE_CHECK(
16091 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
16092 LValue result = unboxBoolean(unboxedResult);
16093 setBoolean(edge.node(), result);
16094 return result;
16095 }
16096
16097 DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecBoolean), provenType(edge));
16098 if (mayHaveTypeCheck(edge.useKind()))
16099 terminate(Uncountable);
16100 return m_out.booleanFalse;
16101 }
16102
16103 LValue lowDouble(Edge edge)
16104 {
16105 DFG_ASSERT(m_graph, m_node, isDouble(edge.useKind()), edge.useKind());
16106
16107 LoweredNodeValue value = m_doubleValues.get(edge.node());
16108 if (isValid(value))
16109 return value.value();
16110 DFG_ASSERT(m_graph, m_node, !provenType(edge), provenType(edge));
16111 if (mayHaveTypeCheck(edge.useKind()))
16112 terminate(Uncountable);
16113 return m_out.doubleZero;
16114 }
16115
16116 LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
16117 {
16118 DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse, m_node->op(), edge.useKind());
16119 DFG_ASSERT(m_graph, m_node, !isDouble(edge.useKind()), m_node->op(), edge.useKind());
16120 DFG_ASSERT(m_graph, m_node, edge.useKind() != Int52RepUse, m_node->op(), edge.useKind());
16121
16122 if (edge->hasConstant()) {
16123 LValue result = m_out.constInt64(JSValue::encode(edge->asJSValue()));
16124 result->setOrigin(B3::Origin(edge.node()));
16125 return result;
16126 }
16127
16128 LoweredNodeValue value = m_jsValueValues.get(edge.node());
16129 if (isValid(value))
16130 return value.value();
16131
16132 value = m_int32Values.get(edge.node());
16133 if (isValid(value)) {
16134 LValue result = boxInt32(value.value());
16135 setJSValue(edge.node(), result);
16136 return result;
16137 }
16138
16139 value = m_booleanValues.get(edge.node());
16140 if (isValid(value)) {
16141 LValue result = boxBoolean(value.value());
16142 setJSValue(edge.node(), result);
16143 return result;
16144 }
16145
16146 DFG_CRASH(m_graph, m_node, makeString("Value not defined: ", String::number(edge.node()->index())).ascii().data());
16147 return 0;
16148 }
16149
16150 LValue lowNotCell(Edge edge)
16151 {
16152 LValue result = lowJSValue(edge, ManualOperandSpeculation);
16153 FTL_TYPE_CHECK(jsValueValue(result), edge, ~SpecCellCheck, isCell(result));
16154 return result;
16155 }
16156
16157 LValue lowStorage(Edge edge)
16158 {
16159 LoweredNodeValue value = m_storageValues.get(edge.node());
16160 if (isValid(value))
16161 return value.value();
16162
16163 LValue result = lowCell(edge);
16164 setStorage(edge.node(), result);
16165 return result;
16166 }
16167
16168 LValue strictInt52ToInt32(Edge edge, LValue value)
16169 {
16170 LValue result = m_out.castToInt32(value);
16171 FTL_TYPE_CHECK(
16172 noValue(), edge, SpecInt32Only,
16173 m_out.notEqual(m_out.signExt32To64(result), value));
16174 setInt32(edge.node(), result);
16175 return result;
16176 }
16177
16178 LValue strictInt52ToDouble(LValue value)
16179 {
16180 return m_out.intToDouble(value);
16181 }
16182
16183 LValue strictInt52ToJSValue(LValue value)
16184 {
16185 LBasicBlock isInt32 = m_out.newBlock();
16186 LBasicBlock isDouble = m_out.newBlock();
16187 LBasicBlock continuation = m_out.newBlock();
16188
16189 Vector<ValueFromBlock, 2> results;
16190
16191 LValue int32Value = m_out.castToInt32(value);
16192 m_out.branch(
16193 m_out.equal(m_out.signExt32To64(int32Value), value),
16194 unsure(isInt32), unsure(isDouble));
16195
16196 LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble);
16197
16198 results.append(m_out.anchor(boxInt32(int32Value)));
16199 m_out.jump(continuation);
16200
16201 m_out.appendTo(isDouble, continuation);
16202
16203 results.append(m_out.anchor(boxDouble(m_out.intToDouble(value))));
16204 m_out.jump(continuation);
16205
16206 m_out.appendTo(continuation, lastNext);
16207 return m_out.phi(Int64, results);
16208 }
16209
16210 LValue strictInt52ToInt52(LValue value)
16211 {
16212 return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount));
16213 }
16214
16215 LValue int52ToStrictInt52(LValue value)
16216 {
16217 return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount));
16218 }
16219
16220 LValue isInt32(LValue jsValue, SpeculatedType type = SpecFullTop)
16221 {
16222 if (LValue proven = isProvenValue(type, SpecInt32Only))
16223 return proven;
16224 return m_out.aboveOrEqual(jsValue, m_numberTag);
16225 }
16226 LValue isNotInt32(LValue jsValue, SpeculatedType type = SpecFullTop)
16227 {
16228 if (LValue proven = isProvenValue(type, ~SpecInt32Only))
16229 return proven;
16230 return m_out.below(jsValue, m_numberTag);
16231 }
16232 LValue unboxInt32(LValue jsValue)
16233 {
16234 return m_out.castToInt32(jsValue);
16235 }
16236 LValue boxInt32(LValue value)
16237 {
16238 return m_out.add(m_out.zeroExt(value, Int64), m_numberTag);
16239 }
16240
16241 LValue isCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop)
16242 {
16243 if (LValue proven = isProvenValue(type, SpecCellCheck | SpecMisc))
16244 return proven;
16245 return m_out.testIsZero64(jsValue, m_numberTag);
16246 }
16247 LValue isNotCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop)
16248 {
16249 if (LValue proven = isProvenValue(type, ~(SpecCellCheck | SpecMisc)))
16250 return proven;
16251 return m_out.testNonZero64(jsValue, m_numberTag);
16252 }
16253
16254 LValue unboxDouble(LValue jsValue, LValue* unboxedAsInt = nullptr)
16255 {
16256 LValue asInt = m_out.add(jsValue, m_numberTag);
16257 if (unboxedAsInt)
16258 *unboxedAsInt = asInt;
16259 return m_out.bitCast(asInt, Double);
16260 }
16261 LValue boxDouble(LValue doubleValue)
16262 {
16263 return m_out.sub(m_out.bitCast(doubleValue, Int64), m_numberTag);
16264 }
16265
16266 LValue jsValueToStrictInt52(Edge edge, LValue boxedValue)
16267 {
16268 LBasicBlock intCase = m_out.newBlock();
16269 LBasicBlock doubleCase = m_out.newBlock();
16270 LBasicBlock continuation = m_out.newBlock();
16271
16272 LValue isNotInt32;
16273 if (!m_interpreter.needsTypeCheck(edge, SpecInt32Only))
16274 isNotInt32 = m_out.booleanFalse;
16275 else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32Only))
16276 isNotInt32 = m_out.booleanTrue;
16277 else
16278 isNotInt32 = this->isNotInt32(boxedValue);
16279 m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
16280
16281 LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
16282
16283 ValueFromBlock intToInt52 = m_out.anchor(
16284 m_out.signExt32To64(unboxInt32(boxedValue)));
16285 m_out.jump(continuation);
16286
16287 m_out.appendTo(doubleCase, continuation);
16288
16289 LValue possibleResult = m_out.callWithoutSideEffects(Int64, operationConvertBoxedDoubleToInt52, boxedValue);
16290 FTL_TYPE_CHECK(
16291 jsValueValue(boxedValue), edge, SpecInt32Only | SpecAnyIntAsDouble,
16292 m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
16293
16294 ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult);
16295 m_out.jump(continuation);
16296
16297 m_out.appendTo(continuation, lastNext);
16298
16299 return m_out.phi(Int64, intToInt52, doubleToInt52);
16300 }
16301
16302 LValue doubleToStrictInt52(Edge edge, LValue value)
16303 {
16304 LValue integerValue = m_out.doubleToInt64(value);
16305 LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue);
16306 LValue valueNotConvertibleToInteger = m_out.doubleNotEqualOrUnordered(value, integerValueConvertedToDouble);
16307 speculate(Int52Overflow, doubleValue(value), edge.node(), valueNotConvertibleToInteger);
16308
16309 LBasicBlock valueIsZero = m_out.newBlock();
16310 LBasicBlock valueIsNotZero = m_out.newBlock();
16311 LBasicBlock continuation = m_out.newBlock();
16312 m_out.branch(m_out.isZero64(integerValue), unsure(valueIsZero), unsure(valueIsNotZero));
16313
16314 LBasicBlock lastNext = m_out.appendTo(valueIsZero, valueIsNotZero);
16315 LValue doubleBitcastToInt64 = m_out.bitCast(value, Int64);
16316 LValue signBitSet = m_out.lessThan(doubleBitcastToInt64, m_out.constInt64(0));
16317 speculate(Int52Overflow, doubleValue(value), edge.node(), signBitSet);
16318 m_out.jump(continuation);
16319
16320 m_out.appendTo(valueIsNotZero, continuation);
16321 speculate(Int52Overflow, doubleValue(value), edge.node(), m_out.greaterThanOrEqual(integerValue, m_out.constInt64(static_cast<int64_t>(1) << (JSValue::numberOfInt52Bits - 1))));
16322 speculate(Int52Overflow, doubleValue(value), edge.node(), m_out.lessThan(integerValue, m_out.constInt64(-(static_cast<int64_t>(1) << (JSValue::numberOfInt52Bits - 1)))));
16323 m_out.jump(continuation);
16324
16325 m_out.appendTo(continuation, lastNext);
16326 m_interpreter.filter(edge, SpecAnyIntAsDouble);
16327 return integerValue;
16328 }
16329
16330 LValue convertDoubleToInt32(LValue value, bool shouldCheckNegativeZero)
16331 {
16332 LValue integerValue = m_out.doubleToInt(value);
16333 LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue);
16334 LValue valueNotConvertibleToInteger = m_out.doubleNotEqualOrUnordered(value, integerValueConvertedToDouble);
16335 speculate(Overflow, FormattedValue(DataFormatDouble, value), m_node, valueNotConvertibleToInteger);
16336
16337 if (shouldCheckNegativeZero) {
16338 LBasicBlock valueIsZero = m_out.newBlock();
16339 LBasicBlock continuation = m_out.newBlock();
16340 m_out.branch(m_out.isZero32(integerValue), unsure(valueIsZero), unsure(continuation));
16341
16342 LBasicBlock lastNext = m_out.appendTo(valueIsZero, continuation);
16343
16344 LValue doubleBitcastToInt64 = m_out.bitCast(value, Int64);
16345 LValue signBitSet = m_out.lessThan(doubleBitcastToInt64, m_out.constInt64(0));
16346
16347 speculate(NegativeZero, FormattedValue(DataFormatDouble, value), m_node, signBitSet);
16348 m_out.jump(continuation);
16349 m_out.appendTo(continuation, lastNext);
16350 }
16351 return integerValue;
16352 }
16353
16354 LValue isNumber(LValue jsValue, SpeculatedType type = SpecFullTop)
16355 {
16356 if (LValue proven = isProvenValue(type, SpecFullNumber))
16357 return proven;
16358 return isNotCellOrMisc(jsValue);
16359 }
16360 LValue isNotNumber(LValue jsValue, SpeculatedType type = SpecFullTop)
16361 {
16362 if (LValue proven = isProvenValue(type, ~SpecFullNumber))
16363 return proven;
16364 return isCellOrMisc(jsValue);
16365 }
16366
16367 LValue isNotCell(LValue jsValue, SpeculatedType type = SpecFullTop)
16368 {
16369 if (LValue proven = isProvenValue(type, ~SpecCellCheck))
16370 return proven;
16371 return m_out.testNonZero64(jsValue, m_notCellMask);
16372 }
16373
16374 LValue isCell(LValue jsValue, SpeculatedType type = SpecFullTop)
16375 {
16376 if (LValue proven = isProvenValue(type, SpecCellCheck))
16377 return proven;
16378 return m_out.testIsZero64(jsValue, m_notCellMask);
16379 }
16380
16381 LValue isNotMisc(LValue value, SpeculatedType type = SpecFullTop)
16382 {
16383 if (LValue proven = isProvenValue(type, ~SpecMisc))
16384 return proven;
16385 return m_out.above(value, m_out.constInt64(JSValue::MiscTag));
16386 }
16387
16388 LValue isMisc(LValue value, SpeculatedType type = SpecFullTop)
16389 {
16390 if (LValue proven = isProvenValue(type, SpecMisc))
16391 return proven;
16392 return m_out.logicalNot(isNotMisc(value));
16393 }
16394
16395 LValue isNotBoolean(LValue jsValue, SpeculatedType type = SpecFullTop)
16396 {
16397 if (LValue proven = isProvenValue(type, ~SpecBoolean))
16398 return proven;
16399 return m_out.testNonZero64(
16400 m_out.bitXor(jsValue, m_out.constInt64(JSValue::ValueFalse)),
16401 m_out.constInt64(~1));
16402 }
16403 LValue isBoolean(LValue jsValue, SpeculatedType type = SpecFullTop)
16404 {
16405 if (LValue proven = isProvenValue(type, SpecBoolean))
16406 return proven;
16407 return m_out.logicalNot(isNotBoolean(jsValue));
16408 }
16409 LValue unboxBoolean(LValue jsValue)
16410 {
16411 // We want to use a cast that guarantees that B3 knows that even the integer
16412 // value is just 0 or 1. But for now we do it the dumb way.
16413 return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
16414 }
16415 LValue boxBoolean(LValue value)
16416 {
16417 return m_out.select(
16418 value, m_out.constInt64(JSValue::ValueTrue), m_out.constInt64(JSValue::ValueFalse));
16419 }
16420
16421 LValue isNotOther(LValue value, SpeculatedType type = SpecFullTop)
16422 {
16423 if (LValue proven = isProvenValue(type, ~SpecOther))
16424 return proven;
16425 return m_out.notEqual(
16426 m_out.bitAnd(value, m_out.constInt64(~JSValue::UndefinedTag)),
16427 m_out.constInt64(JSValue::ValueNull));
16428 }
16429 LValue isOther(LValue value, SpeculatedType type = SpecFullTop)
16430 {
16431 if (LValue proven = isProvenValue(type, SpecOther))
16432 return proven;
16433 return m_out.equal(
16434 m_out.bitAnd(value, m_out.constInt64(~JSValue::UndefinedTag)),
16435 m_out.constInt64(JSValue::ValueNull));
16436 }
16437
16438 LValue isProvenValue(SpeculatedType provenType, SpeculatedType wantedType)
16439 {
16440 if (!(provenType & ~wantedType))
16441 return m_out.booleanTrue;
16442 if (!(provenType & wantedType))
16443 return m_out.booleanFalse;
16444 return nullptr;
16445 }
16446
16447 void speculate(Edge edge)
16448 {
16449 switch (edge.useKind()) {
16450 case UntypedUse:
16451 break;
16452 case KnownInt32Use:
16453 case KnownStringUse:
16454 case KnownPrimitiveUse:
16455 case KnownOtherUse:
16456 case DoubleRepUse:
16457 case Int52RepUse:
16458 case KnownCellUse:
16459 case KnownBooleanUse:
16460 ASSERT(!m_interpreter.needsTypeCheck(edge));
16461 break;
16462 case Int32Use:
16463 speculateInt32(edge);
16464 break;
16465 case CellUse:
16466 speculateCell(edge);
16467 break;
16468 case CellOrOtherUse:
16469 speculateCellOrOther(edge);
16470 break;
16471 case AnyIntUse:
16472 speculateAnyInt(edge);
16473 break;
16474 case ObjectUse:
16475 speculateObject(edge);
16476 break;
16477 case ArrayUse:
16478 speculateArray(edge);
16479 break;
16480 case FunctionUse:
16481 speculateFunction(edge);
16482 break;
16483 case ObjectOrOtherUse:
16484 speculateObjectOrOther(edge);
16485 break;
16486 case FinalObjectUse:
16487 speculateFinalObject(edge);
16488 break;
16489 case RegExpObjectUse:
16490 speculateRegExpObject(edge);
16491 break;
16492 case PromiseObjectUse:
16493 speculatePromiseObject(edge);
16494 break;
16495 case ProxyObjectUse:
16496 speculateProxyObject(edge);
16497 break;
16498 case DerivedArrayUse:
16499 speculateDerivedArray(edge);
16500 break;
16501 case DateObjectUse:
16502 speculateDateObject(edge);
16503 break;
16504 case MapObjectUse:
16505 speculateMapObject(edge);
16506 break;
16507 case SetObjectUse:
16508 speculateSetObject(edge);
16509 break;
16510 case WeakMapObjectUse:
16511 speculateWeakMapObject(edge);
16512 break;
16513 case WeakSetObjectUse:
16514 speculateWeakSetObject(edge);
16515 break;
16516 case DataViewObjectUse:
16517 speculateDataViewObject(edge);
16518 break;
16519 case StringUse:
16520 speculateString(edge);
16521 break;
16522 case StringOrOtherUse:
16523 speculateStringOrOther(edge);
16524 break;
16525 case StringIdentUse:
16526 speculateStringIdent(edge);
16527 break;
16528 case SymbolUse:
16529 speculateSymbol(edge);
16530 break;
16531 case StringObjectUse:
16532 speculateStringObject(edge);
16533 break;
16534 case StringOrStringObjectUse:
16535 speculateStringOrStringObject(edge);
16536 break;
16537 case NumberUse:
16538 speculateNumber(edge);
16539 break;
16540 case RealNumberUse:
16541 speculateRealNumber(edge);
16542 break;
16543 case DoubleRepRealUse:
16544 speculateDoubleRepReal(edge);
16545 break;
16546 case DoubleRepAnyIntUse:
16547 speculateDoubleRepAnyInt(edge);
16548 break;
16549 case BooleanUse:
16550 speculateBoolean(edge);
16551 break;
16552 case BigIntUse:
16553 speculateBigInt(edge);
16554 break;
16555 case NotStringVarUse:
16556 speculateNotStringVar(edge);
16557 break;
16558 case NotSymbolUse:
16559 speculateNotSymbol(edge);
16560 break;
16561 case NotCellUse:
16562 speculateNotCell(edge);
16563 break;
16564 case OtherUse:
16565 speculateOther(edge);
16566 break;
16567 case MiscUse:
16568 speculateMisc(edge);
16569 break;
16570 default:
16571 DFG_CRASH(m_graph, m_node, "Unsupported speculation use kind");
16572 }
16573 }
16574
16575 void speculate(Node*, Edge edge)
16576 {
16577 speculate(edge);
16578 }
16579
16580 void speculateInt32(Edge edge)
16581 {
16582 lowInt32(edge);
16583 }
16584
16585 void speculateCell(Edge edge)
16586 {
16587 lowCell(edge);
16588 }
16589
16590 void speculateNotCell(Edge edge)
16591 {
16592 if (!m_interpreter.needsTypeCheck(edge))
16593 return;
16594 lowNotCell(edge);
16595 }
16596
16597 void speculateCellOrOther(Edge edge)
16598 {
16599 if (shouldNotHaveTypeCheck(edge.useKind()))
16600 return;
16601
16602 LValue value = lowJSValue(edge, ManualOperandSpeculation);
16603
16604 LBasicBlock isNotCell = m_out.newBlock();
16605 LBasicBlock continuation = m_out.newBlock();
16606
16607 m_out.branch(isCell(value, provenType(edge)), unsure(continuation), unsure(isNotCell));
16608
16609 LBasicBlock lastNext = m_out.appendTo(isNotCell, continuation);
16610 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecCellCheck | SpecOther, isNotOther(value));
16611 m_out.jump(continuation);
16612
16613 m_out.appendTo(continuation, lastNext);
16614 }
16615
16616 void speculateAnyInt(Edge edge)
16617 {
16618 if (!m_interpreter.needsTypeCheck(edge))
16619 return;
16620
16621 jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation));
16622 }
16623
16624 LValue isCellWithType(LValue cell, JSType queriedType, Optional<SpeculatedType> speculatedTypeForQuery, SpeculatedType type = SpecFullTop)
16625 {
16626 if (speculatedTypeForQuery) {
16627 if (LValue proven = isProvenValue(type & SpecCell, speculatedTypeForQuery.value()))
16628 return proven;
16629 }
16630 return m_out.equal(
16631 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16632 m_out.constInt32(queriedType));
16633 }
16634
16635 LValue isTypedArrayView(LValue cell, SpeculatedType type = SpecFullTop)
16636 {
16637 if (LValue proven = isProvenValue(type & SpecCell, SpecTypedArrayView))
16638 return proven;
16639 LValue jsType = m_out.sub(
16640 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16641 m_out.constInt32(FirstTypedArrayType));
16642 return m_out.below(
16643 jsType,
16644 m_out.constInt32(NumberOfTypedArrayTypesExcludingDataView));
16645 }
16646
16647 LValue isObject(LValue cell, SpeculatedType type = SpecFullTop)
16648 {
16649 if (LValue proven = isProvenValue(type & SpecCell, SpecObject))
16650 return proven;
16651 return m_out.aboveOrEqual(
16652 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16653 m_out.constInt32(ObjectType));
16654 }
16655
16656 LValue isNotObject(LValue cell, SpeculatedType type = SpecFullTop)
16657 {
16658 if (LValue proven = isProvenValue(type & SpecCell, ~SpecObject))
16659 return proven;
16660 return m_out.below(
16661 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16662 m_out.constInt32(ObjectType));
16663 }
16664
16665 LValue isNotString(LValue cell, SpeculatedType type = SpecFullTop)
16666 {
16667 if (LValue proven = isProvenValue(type & SpecCell, ~SpecString))
16668 return proven;
16669 return m_out.notEqual(
16670 m_out.load32(cell, m_heaps.JSCell_structureID),
16671 m_out.constInt32(vm().stringStructure->id()));
16672 }
16673
16674 LValue isString(LValue cell, SpeculatedType type = SpecFullTop)
16675 {
16676 if (LValue proven = isProvenValue(type & SpecCell, SpecString))
16677 return proven;
16678 return m_out.equal(
16679 m_out.load32(cell, m_heaps.JSCell_structureID),
16680 m_out.constInt32(vm().stringStructure->id()));
16681 }
16682
16683 LValue isRopeString(LValue string, Edge edge = Edge())
16684 {
16685 if (edge) {
16686 if (!((provenType(edge) & SpecString) & ~SpecStringIdent))
16687 return m_out.booleanFalse;
16688 if (JSValue value = provenValue(edge)) {
16689 if (value.isCell() && value.asCell()->type() == StringType && !asString(value)->isRope())
16690 return m_out.booleanFalse;
16691 }
16692 String value = edge->tryGetString(m_graph);
16693 if (!value.isNull()) {
16694 // If this value is LazyValue, it will be converted to JSString, and the result must be non-rope string.
16695 return m_out.booleanFalse;
16696 }
16697 }
16698
16699 return m_out.testNonZeroPtr(m_out.loadPtr(string, m_heaps.JSString_value), m_out.constIntPtr(JSString::isRopeInPointer));
16700 }
16701
16702 LValue isNotRopeString(LValue string, Edge edge = Edge())
16703 {
16704 if (edge) {
16705 if (!((provenType(edge) & SpecString) & ~SpecStringIdent))
16706 return m_out.booleanTrue;
16707 if (JSValue value = provenValue(edge)) {
16708 if (value.isCell() && value.asCell()->type() == StringType && !asString(value)->isRope())
16709 return m_out.booleanTrue;
16710 }
16711 String value = edge->tryGetString(m_graph);
16712 if (!value.isNull()) {
16713 // If this value is LazyValue, it will be converted to JSString, and the result must be non-rope string.
16714 return m_out.booleanTrue;
16715 }
16716 }
16717
16718 return m_out.testIsZeroPtr(m_out.loadPtr(string, m_heaps.JSString_value), m_out.constIntPtr(JSString::isRopeInPointer));
16719 }
16720
16721 LValue isNotSymbol(LValue cell, SpeculatedType type = SpecFullTop)
16722 {
16723 if (LValue proven = isProvenValue(type & SpecCell, ~SpecSymbol))
16724 return proven;
16725 return m_out.notEqual(
16726 m_out.load32(cell, m_heaps.JSCell_structureID),
16727 m_out.constInt32(vm().symbolStructure->id()));
16728 }
16729
16730 LValue isSymbol(LValue cell, SpeculatedType type = SpecFullTop)
16731 {
16732 if (LValue proven = isProvenValue(type & SpecCell, SpecSymbol))
16733 return proven;
16734 return m_out.equal(
16735 m_out.load32(cell, m_heaps.JSCell_structureID),
16736 m_out.constInt32(vm().symbolStructure->id()));
16737 }
16738
16739 LValue isNotBigInt(LValue cell, SpeculatedType type = SpecFullTop)
16740 {
16741 if (LValue proven = isProvenValue(type & SpecCell, ~SpecBigInt))
16742 return proven;
16743 return m_out.notEqual(
16744 m_out.load32(cell, m_heaps.JSCell_structureID),
16745 m_out.constInt32(vm().bigIntStructure->id()));
16746 }
16747
16748 LValue isBigInt(LValue cell, SpeculatedType type = SpecFullTop)
16749 {
16750 if (LValue proven = isProvenValue(type & SpecCell, SpecBigInt))
16751 return proven;
16752 return m_out.equal(
16753 m_out.load32(cell, m_heaps.JSCell_structureID),
16754 m_out.constInt32(vm().bigIntStructure->id()));
16755 }
16756
16757 LValue isArrayTypeForArrayify(LValue cell, ArrayMode arrayMode)
16758 {
16759 switch (arrayMode.type()) {
16760 case Array::Int32:
16761 case Array::Double:
16762 case Array::Contiguous:
16763 case Array::Undecided:
16764 case Array::ArrayStorage: {
16765 IndexingType indexingModeMask = IsArray | IndexingShapeMask;
16766 if (arrayMode.action() == Array::Write)
16767 indexingModeMask |= CopyOnWrite;
16768
16769 IndexingType shape = arrayMode.shapeMask();
16770 LValue indexingType = m_out.load8ZeroExt32(cell, m_heaps.JSCell_indexingTypeAndMisc);
16771
16772 switch (arrayMode.arrayClass()) {
16773 case Array::OriginalArray:
16774 case Array::OriginalCopyOnWriteArray:
16775 DFG_CRASH(m_graph, m_node, "Unexpected original array");
16776 return nullptr;
16777
16778 case Array::Array:
16779 return m_out.equal(
16780 m_out.bitAnd(indexingType, m_out.constInt32(indexingModeMask)),
16781 m_out.constInt32(IsArray | shape));
16782
16783 case Array::NonArray:
16784 case Array::OriginalNonArray:
16785 return m_out.equal(
16786 m_out.bitAnd(indexingType, m_out.constInt32(indexingModeMask)),
16787 m_out.constInt32(shape));
16788
16789 case Array::PossiblyArray:
16790 return m_out.equal(
16791 m_out.bitAnd(indexingType, m_out.constInt32(indexingModeMask & ~IsArray)),
16792 m_out.constInt32(shape));
16793 }
16794 break;
16795 }
16796
16797 case Array::SlowPutArrayStorage: {
16798 ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
16799 LValue indexingType = m_out.load8ZeroExt32(cell, m_heaps.JSCell_indexingTypeAndMisc);
16800
16801 LBasicBlock trueCase = m_out.newBlock();
16802 LBasicBlock checkCase = m_out.newBlock();
16803 LBasicBlock continuation = m_out.newBlock();
16804
16805 ValueFromBlock falseValue = m_out.anchor(m_out.booleanFalse);
16806 LValue isAnArrayStorageShape = m_out.belowOrEqual(
16807 m_out.sub(
16808 m_out.bitAnd(indexingType, m_out.constInt32(IndexingShapeMask)),
16809 m_out.constInt32(ArrayStorageShape)),
16810 m_out.constInt32(SlowPutArrayStorageShape - ArrayStorageShape));
16811 m_out.branch(isAnArrayStorageShape, unsure(checkCase), unsure(continuation));
16812
16813 LBasicBlock lastNext = m_out.appendTo(checkCase, trueCase);
16814 switch (arrayMode.arrayClass()) {
16815 case Array::OriginalArray:
16816 case Array::OriginalCopyOnWriteArray:
16817 DFG_CRASH(m_graph, m_node, "Unexpected original array");
16818 return nullptr;
16819
16820 case Array::Array:
16821 m_out.branch(
16822 m_out.testNonZero32(indexingType, m_out.constInt32(IsArray)),
16823 unsure(trueCase), unsure(continuation));
16824 break;
16825
16826 case Array::NonArray:
16827 case Array::OriginalNonArray:
16828 m_out.branch(
16829 m_out.testIsZero32(indexingType, m_out.constInt32(IsArray)),
16830 unsure(trueCase), unsure(continuation));
16831 break;
16832
16833 case Array::PossiblyArray:
16834 m_out.jump(trueCase);
16835 break;
16836 }
16837
16838 m_out.appendTo(trueCase, continuation);
16839 ValueFromBlock trueValue = m_out.anchor(m_out.booleanTrue);
16840 m_out.jump(continuation);
16841
16842 m_out.appendTo(continuation, lastNext);
16843 return m_out.phi(Int32, falseValue, trueValue);
16844 }
16845
16846 default:
16847 break;
16848 }
16849 DFG_CRASH(m_graph, m_node, "Corrupt array class");
16850 }
16851
16852 LValue isArrayTypeForCheckArray(LValue cell, ArrayMode arrayMode)
16853 {
16854 switch (arrayMode.type()) {
16855 case Array::Int32:
16856 case Array::Double:
16857 case Array::Contiguous:
16858 case Array::Undecided:
16859 case Array::ArrayStorage:
16860 case Array::SlowPutArrayStorage:
16861 return isArrayTypeForArrayify(cell, arrayMode);
16862
16863 case Array::DirectArguments:
16864 return m_out.equal(
16865 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16866 m_out.constInt32(DirectArgumentsType));
16867
16868 case Array::ScopedArguments:
16869 return m_out.equal(
16870 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16871 m_out.constInt32(ScopedArgumentsType));
16872
16873 default:
16874 return m_out.equal(
16875 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16876 m_out.constInt32(typeForTypedArrayType(arrayMode.typedArrayType())));
16877 }
16878 }
16879
16880 LValue isFunction(LValue cell, SpeculatedType type = SpecFullTop)
16881 {
16882 if (LValue proven = isProvenValue(type & SpecCell, SpecFunction))
16883 return proven;
16884 return isType(cell, JSFunctionType);
16885 }
16886 LValue isNotFunction(LValue cell, SpeculatedType type = SpecFullTop)
16887 {
16888 if (LValue proven = isProvenValue(type & SpecCell, ~SpecFunction))
16889 return proven;
16890 return isNotType(cell, JSFunctionType);
16891 }
16892
16893 LValue isExoticForTypeof(LValue cell, SpeculatedType type = SpecFullTop)
16894 {
16895 if (!(type & SpecObjectOther))
16896 return m_out.booleanFalse;
16897 return m_out.testNonZero32(
16898 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoFlags),
16899 m_out.constInt32(MasqueradesAsUndefined | OverridesGetCallData));
16900 }
16901
16902 LValue isType(LValue cell, JSType type)
16903 {
16904 return m_out.equal(
16905 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType),
16906 m_out.constInt32(type));
16907 }
16908
16909 LValue isNotType(LValue cell, JSType type)
16910 {
16911 return m_out.logicalNot(isType(cell, type));
16912 }
16913
16914 void speculateObject(Edge edge, LValue cell)
16915 {
16916 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
16917 }
16918
16919 void speculateObject(Edge edge)
16920 {
16921 speculateObject(edge, lowCell(edge));
16922 }
16923
16924 void speculateArray(Edge edge, LValue cell)
16925 {
16926 FTL_TYPE_CHECK(
16927 jsValueValue(cell), edge, SpecArray, isNotType(cell, ArrayType));
16928 }
16929
16930 void speculateArray(Edge edge)
16931 {
16932 speculateArray(edge, lowCell(edge));
16933 }
16934
16935 void speculateFunction(Edge edge, LValue cell)
16936 {
16937 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecFunction, isNotFunction(cell));
16938 }
16939
16940 void speculateFunction(Edge edge)
16941 {
16942 speculateFunction(edge, lowCell(edge));
16943 }
16944
16945 void speculateObjectOrOther(Edge edge)
16946 {
16947 if (!m_interpreter.needsTypeCheck(edge))
16948 return;
16949
16950 LValue value = lowJSValue(edge, ManualOperandSpeculation);
16951
16952 LBasicBlock cellCase = m_out.newBlock();
16953 LBasicBlock primitiveCase = m_out.newBlock();
16954 LBasicBlock continuation = m_out.newBlock();
16955
16956 m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase));
16957
16958 LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
16959
16960 FTL_TYPE_CHECK(
16961 jsValueValue(value), edge, (~SpecCellCheck) | SpecObject, isNotObject(value));
16962
16963 m_out.jump(continuation);
16964
16965 m_out.appendTo(primitiveCase, continuation);
16966
16967 FTL_TYPE_CHECK(
16968 jsValueValue(value), edge, SpecCellCheck | SpecOther, isNotOther(value));
16969
16970 m_out.jump(continuation);
16971
16972 m_out.appendTo(continuation, lastNext);
16973 }
16974
16975 void speculateFinalObject(Edge edge, LValue cell)
16976 {
16977 FTL_TYPE_CHECK(
16978 jsValueValue(cell), edge, SpecFinalObject, isNotType(cell, FinalObjectType));
16979 }
16980
16981 void speculateFinalObject(Edge edge)
16982 {
16983 speculateFinalObject(edge, lowCell(edge));
16984 }
16985
16986 void speculateRegExpObject(Edge edge, LValue cell)
16987 {
16988 FTL_TYPE_CHECK(
16989 jsValueValue(cell), edge, SpecRegExpObject, isNotType(cell, RegExpObjectType));
16990 }
16991
16992 void speculateRegExpObject(Edge edge)
16993 {
16994 speculateRegExpObject(edge, lowCell(edge));
16995 }
16996
16997 void speculateProxyObject(Edge edge, LValue cell)
16998 {
16999 FTL_TYPE_CHECK(
17000 jsValueValue(cell), edge, SpecProxyObject, isNotType(cell, ProxyObjectType));
17001 }
17002
17003 void speculateProxyObject(Edge edge)
17004 {
17005 speculateProxyObject(edge, lowCell(edge));
17006 }
17007
17008 void speculateDerivedArray(Edge edge, LValue cell)
17009 {
17010 FTL_TYPE_CHECK(
17011 jsValueValue(cell), edge, SpecDerivedArray, isNotType(cell, DerivedArrayType));
17012 }
17013
17014 void speculateDerivedArray(Edge edge)
17015 {
17016 speculateDerivedArray(edge, lowCell(edge));
17017 }
17018
17019 void speculatePromiseObject(Edge edge, LValue cell)
17020 {
17021 FTL_TYPE_CHECK(
17022 jsValueValue(cell), edge, SpecPromiseObject, isNotType(cell, JSPromiseType));
17023 }
17024
17025 void speculatePromiseObject(Edge edge)
17026 {
17027 speculatePromiseObject(edge, lowCell(edge));
17028 }
17029
17030 void speculateDateObject(Edge edge, LValue cell)
17031 {
17032 FTL_TYPE_CHECK(
17033 jsValueValue(cell), edge, SpecDateObject, isNotType(cell, JSDateType));
17034 }
17035
17036 void speculateDateObject(Edge edge)
17037 {
17038 speculateDateObject(edge, lowCell(edge));
17039 }
17040
17041 void speculateMapObject(Edge edge, LValue cell)
17042 {
17043 FTL_TYPE_CHECK(
17044 jsValueValue(cell), edge, SpecMapObject, isNotType(cell, JSMapType));
17045 }
17046
17047 void speculateMapObject(Edge edge)
17048 {
17049 speculateMapObject(edge, lowCell(edge));
17050 }
17051
17052 void speculateSetObject(Edge edge, LValue cell)
17053 {
17054 FTL_TYPE_CHECK(
17055 jsValueValue(cell), edge, SpecSetObject, isNotType(cell, JSSetType));
17056 }
17057
17058 void speculateSetObject(Edge edge)
17059 {
17060 speculateSetObject(edge, lowCell(edge));
17061 }
17062
17063 void speculateWeakMapObject(Edge edge, LValue cell)
17064 {
17065 FTL_TYPE_CHECK(
17066 jsValueValue(cell), edge, SpecWeakMapObject, isNotType(cell, JSWeakMapType));
17067 }
17068
17069 void speculateWeakMapObject(Edge edge)
17070 {
17071 speculateWeakMapObject(edge, lowCell(edge));
17072 }
17073
17074 void speculateWeakSetObject(Edge edge, LValue cell)
17075 {
17076 FTL_TYPE_CHECK(
17077 jsValueValue(cell), edge, SpecWeakSetObject, isNotType(cell, JSWeakSetType));
17078 }
17079
17080 void speculateWeakSetObject(Edge edge)
17081 {
17082 speculateWeakSetObject(edge, lowCell(edge));
17083 }
17084
17085 void speculateDataViewObject(Edge edge, LValue cell)
17086 {
17087 FTL_TYPE_CHECK(
17088 jsValueValue(cell), edge, SpecDataViewObject, isNotType(cell, DataViewType));
17089 }
17090
17091 void speculateDataViewObject(Edge edge)
17092 {
17093 speculateDataViewObject(edge, lowCell(edge));
17094 }
17095
17096 void speculateString(Edge edge, LValue cell)
17097 {
17098 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString, isNotString(cell));
17099 }
17100
17101 void speculateString(Edge edge)
17102 {
17103 speculateString(edge, lowCell(edge));
17104 }
17105
17106 void speculateStringOrOther(Edge edge, LValue value)
17107 {
17108 if (!m_interpreter.needsTypeCheck(edge))
17109 return;
17110
17111 LBasicBlock cellCase = m_out.newBlock();
17112 LBasicBlock notCellCase = m_out.newBlock();
17113 LBasicBlock continuation = m_out.newBlock();
17114
17115 m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase));
17116
17117 LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
17118
17119 FTL_TYPE_CHECK(jsValueValue(value), edge, (~SpecCellCheck) | SpecString, isNotString(value));
17120
17121 m_out.jump(continuation);
17122 m_out.appendTo(notCellCase, continuation);
17123
17124 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecCellCheck | SpecOther, isNotOther(value));
17125
17126 m_out.jump(continuation);
17127 m_out.appendTo(continuation, lastNext);
17128 }
17129
17130 void speculateStringOrOther(Edge edge)
17131 {
17132 speculateStringOrOther(edge, lowJSValue(edge, ManualOperandSpeculation));
17133 }
17134
17135 void speculateStringIdent(Edge edge, LValue string, LValue stringImpl)
17136 {
17137 if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString))
17138 return;
17139
17140 speculate(BadType, jsValueValue(string), edge.node(), isRopeString(string));
17141 speculate(
17142 BadType, jsValueValue(string), edge.node(),
17143 m_out.testIsZero32(
17144 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
17145 m_out.constInt32(StringImpl::flagIsAtom())));
17146 m_interpreter.filter(edge, SpecStringIdent | ~SpecString);
17147 }
17148
17149 void speculateStringIdent(Edge edge)
17150 {
17151 lowStringIdent(edge);
17152 }
17153
17154 void speculateStringObject(Edge edge)
17155 {
17156 if (!m_interpreter.needsTypeCheck(edge, SpecStringObject))
17157 return;
17158
17159 speculateStringObjectForCell(edge, lowCell(edge));
17160 }
17161
17162 void speculateStringOrStringObject(Edge edge)
17163 {
17164 if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject))
17165 return;
17166
17167 LValue cellBase = lowCell(edge);
17168 if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject))
17169 return;
17170
17171 LBasicBlock notString = m_out.newBlock();
17172 LBasicBlock continuation = m_out.newBlock();
17173
17174 LValue type = m_out.load8ZeroExt32(cellBase, m_heaps.JSCell_typeInfoType);
17175 m_out.branch(
17176 m_out.equal(type, m_out.constInt32(StringType)),
17177 unsure(continuation), unsure(notString));
17178
17179 LBasicBlock lastNext = m_out.appendTo(notString, continuation);
17180 speculate(
17181 BadType, jsValueValue(cellBase), edge.node(),
17182 m_out.notEqual(type, m_out.constInt32(StringObjectType)));
17183 m_out.jump(continuation);
17184
17185 m_out.appendTo(continuation, lastNext);
17186 m_interpreter.filter(edge, SpecString | SpecStringObject);
17187 }
17188
17189 void speculateStringObjectForCell(Edge edge, LValue cell)
17190 {
17191 if (!m_interpreter.needsTypeCheck(edge, SpecStringObject))
17192 return;
17193
17194 LValue type = m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType);
17195 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecStringObject, m_out.notEqual(type, m_out.constInt32(StringObjectType)));
17196 }
17197
17198 void speculateSymbol(Edge edge, LValue cell)
17199 {
17200 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecSymbol, isNotSymbol(cell));
17201 }
17202
17203 void speculateSymbol(Edge edge)
17204 {
17205 speculateSymbol(edge, lowCell(edge));
17206 }
17207
17208 void speculateBigInt(Edge edge, LValue cell)
17209 {
17210 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecBigInt, isNotBigInt(cell));
17211 }
17212
17213 void speculateBigInt(Edge edge)
17214 {
17215 speculateBigInt(edge, lowCell(edge));
17216 }
17217
17218 void speculateNonNullObject(Edge edge, LValue cell)
17219 {
17220 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
17221 if (masqueradesAsUndefinedWatchpointIsStillValid())
17222 return;
17223
17224 speculate(
17225 BadType, jsValueValue(cell), edge.node(),
17226 m_out.testNonZero32(
17227 m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoFlags),
17228 m_out.constInt32(MasqueradesAsUndefined)));
17229 }
17230
17231 void speculateNumber(Edge edge)
17232 {
17233 LValue value = lowJSValue(edge, ManualOperandSpeculation);
17234 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value));
17235 }
17236
17237 void speculateRealNumber(Edge edge)
17238 {
17239 // Do an early return here because lowDouble() can create a lot of control flow.
17240 if (!m_interpreter.needsTypeCheck(edge))
17241 return;
17242
17243 LValue value = lowJSValue(edge, ManualOperandSpeculation);
17244 LValue doubleValue = unboxDouble(value);
17245
17246 LBasicBlock intCase = m_out.newBlock();
17247 LBasicBlock continuation = m_out.newBlock();
17248
17249 m_out.branch(
17250 m_out.doubleEqual(doubleValue, doubleValue),
17251 usually(continuation), rarely(intCase));
17252
17253 LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
17254
17255 typeCheck(
17256 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
17257 isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
17258 m_out.jump(continuation);
17259
17260 m_out.appendTo(continuation, lastNext);
17261 }
17262
17263 void speculateDoubleRepReal(Edge edge)
17264 {
17265 // Do an early return here because lowDouble() can create a lot of control flow.
17266 if (!m_interpreter.needsTypeCheck(edge))
17267 return;
17268
17269 LValue value = lowDouble(edge);
17270 FTL_TYPE_CHECK(
17271 doubleValue(value), edge, SpecDoubleReal,
17272 m_out.doubleNotEqualOrUnordered(value, value));
17273 }
17274
17275 void speculateDoubleRepAnyInt(Edge edge)
17276 {
17277 if (!m_interpreter.needsTypeCheck(edge))
17278 return;
17279
17280 doubleToStrictInt52(edge, lowDouble(edge));
17281 }
17282
17283 void speculateBoolean(Edge edge)
17284 {
17285 lowBoolean(edge);
17286 }
17287
17288 void speculateNotStringVar(Edge edge)
17289 {
17290 if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar))
17291 return;
17292
17293 LValue value = lowJSValue(edge, ManualOperandSpeculation);
17294
17295 LBasicBlock isCellCase = m_out.newBlock();
17296 LBasicBlock isStringCase = m_out.newBlock();
17297 LBasicBlock continuation = m_out.newBlock();
17298
17299 m_out.branch(isCell(value, provenType(edge)), unsure(isCellCase), unsure(continuation));
17300
17301 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
17302 m_out.branch(isString(value, provenType(edge)), unsure(isStringCase), unsure(continuation));
17303
17304 m_out.appendTo(isStringCase, continuation);
17305 speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value));
17306 m_out.jump(continuation);
17307
17308 m_out.appendTo(continuation, lastNext);
17309 }
17310
17311 void speculateNotSymbol(Edge edge)
17312 {
17313 if (!m_interpreter.needsTypeCheck(edge, ~SpecSymbol))
17314 return;
17315
17316 ASSERT(mayHaveTypeCheck(edge.useKind()));
17317 LValue value = lowJSValue(edge, ManualOperandSpeculation);
17318
17319 LBasicBlock isCellCase = m_out.newBlock();
17320 LBasicBlock continuation = m_out.newBlock();
17321
17322 m_out.branch(isCell(value, provenType(edge)), unsure(isCellCase), unsure(continuation));
17323
17324 LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
17325 speculate(BadType, jsValueValue(value), edge.node(), isSymbol(value));
17326 m_out.jump(continuation);
17327
17328 m_out.appendTo(continuation, lastNext);
17329
17330 m_interpreter.filter(edge, ~SpecSymbol);
17331 }
17332
17333 void speculateOther(Edge edge)
17334 {
17335 if (!m_interpreter.needsTypeCheck(edge))
17336 return;
17337
17338 LValue value = lowJSValue(edge, ManualOperandSpeculation);
17339 typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value));
17340 }
17341
17342 void speculateMisc(Edge edge)
17343 {
17344 if (!m_interpreter.needsTypeCheck(edge))
17345 return;
17346
17347 LValue value = lowJSValue(edge, ManualOperandSpeculation);
17348 typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value));
17349 }
17350
17351 void speculateTypedArrayIsNotNeutered(LValue base)
17352 {
17353 LBasicBlock isWasteful = m_out.newBlock();
17354 LBasicBlock continuation = m_out.newBlock();
17355
17356 LValue mode = m_out.load32(base, m_heaps.JSArrayBufferView_mode);
17357 m_out.branch(m_out.equal(mode, m_out.constInt32(WastefulTypedArray)),
17358 unsure(isWasteful), unsure(continuation));
17359
17360 LBasicBlock lastNext = m_out.appendTo(isWasteful, continuation);
17361 LValue vector = m_out.loadPtr(base, m_heaps.JSArrayBufferView_vector);
17362 // FIXME: We could probably make this a mask.
17363 // https://bugs.webkit.org/show_bug.cgi?id=197701
17364 vector = removeArrayPtrTag(vector);
17365 speculate(Uncountable, jsValueValue(vector), m_node, m_out.isZero64(vector));
17366 m_out.jump(continuation);
17367
17368 m_out.appendTo(continuation, lastNext);
17369 }
17370
17371 bool masqueradesAsUndefinedWatchpointIsStillValid()
17372 {
17373 return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic);
17374 }
17375
17376 LValue loadCellState(LValue base)
17377 {
17378 return m_out.load8ZeroExt32(base, m_heaps.JSCell_cellState);
17379 }
17380
17381 void emitStoreBarrier(LValue base, bool isFenced)
17382 {
17383 LBasicBlock recheckPath = nullptr;
17384 if (isFenced)
17385 recheckPath = m_out.newBlock();
17386 LBasicBlock slowPath = m_out.newBlock();
17387 LBasicBlock continuation = m_out.newBlock();
17388
17389 LBasicBlock lastNext = m_out.insertNewBlocksBefore(isFenced ? recheckPath : slowPath);
17390
17391 LValue threshold;
17392 if (isFenced)
17393 threshold = m_out.load32(m_out.absolute(vm().heap.addressOfBarrierThreshold()));
17394 else
17395 threshold = m_out.constInt32(blackThreshold);
17396
17397 m_out.branch(
17398 m_out.above(loadCellState(base), threshold),
17399 usually(continuation), rarely(isFenced ? recheckPath : slowPath));
17400
17401 if (isFenced) {
17402 m_out.appendTo(recheckPath, slowPath);
17403
17404 m_out.fence(&m_heaps.root, &m_heaps.JSCell_cellState);
17405
17406 m_out.branch(
17407 m_out.above(loadCellState(base), m_out.constInt32(blackThreshold)),
17408 usually(continuation), rarely(slowPath));
17409 }
17410
17411 m_out.appendTo(slowPath, continuation);
17412
17413 LValue call = vmCall(Void, operationWriteBarrierSlowPath, m_vmValue, base);
17414 m_heaps.decorateCCallRead(&m_heaps.root, call);
17415 m_heaps.decorateCCallWrite(&m_heaps.JSCell_cellState, call);
17416
17417 m_out.jump(continuation);
17418
17419 m_out.appendTo(continuation, lastNext);
17420 }
17421
17422 void mutatorFence()
17423 {
17424 if (isX86()) {
17425 m_out.fence(&m_heaps.root, nullptr);
17426 return;
17427 }
17428
17429 LBasicBlock slowPath = m_out.newBlock();
17430 LBasicBlock continuation = m_out.newBlock();
17431
17432 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
17433
17434 m_out.branch(
17435 m_out.load8ZeroExt32(m_out.absolute(vm().heap.addressOfMutatorShouldBeFenced())),
17436 rarely(slowPath), usually(continuation));
17437
17438 m_out.appendTo(slowPath, continuation);
17439
17440 m_out.fence(&m_heaps.root, nullptr);
17441 m_out.jump(continuation);
17442
17443 m_out.appendTo(continuation, lastNext);
17444 }
17445
17446 void nukeStructureAndSetButterfly(LValue butterfly, LValue object)
17447 {
17448 if (isX86()) {
17449 m_out.store32(
17450 m_out.bitOr(
17451 m_out.load32(object, m_heaps.JSCell_structureID),
17452 m_out.constInt32(nukedStructureIDBit())),
17453 object, m_heaps.JSCell_structureID);
17454 m_out.fence(&m_heaps.root, nullptr);
17455 m_out.storePtr(butterfly, object, m_heaps.JSObject_butterfly);
17456 m_out.fence(&m_heaps.root, nullptr);
17457 return;
17458 }
17459
17460 LBasicBlock fastPath = m_out.newBlock();
17461 LBasicBlock slowPath = m_out.newBlock();
17462 LBasicBlock continuation = m_out.newBlock();
17463
17464 LBasicBlock lastNext = m_out.insertNewBlocksBefore(fastPath);
17465
17466 m_out.branch(
17467 m_out.load8ZeroExt32(m_out.absolute(vm().heap.addressOfMutatorShouldBeFenced())),
17468 rarely(slowPath), usually(fastPath));
17469
17470 m_out.appendTo(fastPath, slowPath);
17471
17472 m_out.storePtr(butterfly, object, m_heaps.JSObject_butterfly);
17473 m_out.jump(continuation);
17474
17475 m_out.appendTo(slowPath, continuation);
17476
17477 m_out.store32(
17478 m_out.bitOr(
17479 m_out.load32(object, m_heaps.JSCell_structureID),
17480 m_out.constInt32(nukedStructureIDBit())),
17481 object, m_heaps.JSCell_structureID);
17482 m_out.fence(&m_heaps.root, nullptr);
17483 m_out.storePtr(butterfly, object, m_heaps.JSObject_butterfly);
17484 m_out.fence(&m_heaps.root, nullptr);
17485 m_out.jump(continuation);
17486
17487 m_out.appendTo(continuation, lastNext);
17488 }
17489
17490 template<typename OperationType, typename... Args>
17491 LValue vmCall(LType type, OperationType function, Args&&... args)
17492 {
17493 static_assert(!std::is_same<OperationType, LValue>::value);
17494 static_assert(FunctionTraits<OperationType>::cCallArity() == sizeof...(Args), "Sanity check");
17495 callPreflight();
17496 LValue result = m_out.call(type, m_out.operation(function), std::forward<Args>(args)...);
17497 if (mayExit(m_graph, m_node))
17498 callCheck();
17499 else {
17500 // We can't exit due to an exception, so we also can't throw an exception.
17501#ifndef NDEBUG
17502 LBasicBlock crash = m_out.newBlock();
17503 LBasicBlock continuation = m_out.newBlock();
17504
17505 LValue exception = m_out.load64(m_out.absolute(vm().addressOfException()));
17506 LValue hadException = m_out.notZero64(exception);
17507
17508 m_out.branch(
17509 hadException, rarely(crash), usually(continuation));
17510
17511 LBasicBlock lastNext = m_out.appendTo(crash, continuation);
17512 m_out.unreachable();
17513
17514 m_out.appendTo(continuation, lastNext);
17515#endif
17516 }
17517 return result;
17518 }
17519
17520 void callPreflight(CodeOrigin codeOrigin)
17521 {
17522 CallSiteIndex callSiteIndex = m_ftlState.jitCode->common.addCodeOrigin(codeOrigin);
17523 m_out.store32(
17524 m_out.constInt32(callSiteIndex.bits()),
17525 tagFor(CallFrameSlot::argumentCount));
17526#if !USE(BUILTIN_FRAME_ADDRESS) || !ASSERT_DISABLED
17527 m_out.storePtr(m_callFrame, m_out.absolute(&vm().topCallFrame));
17528#endif
17529 }
17530
17531 void callPreflight()
17532 {
17533 callPreflight(codeOriginDescriptionOfCallSite());
17534 }
17535
17536 CodeOrigin codeOriginDescriptionOfCallSite() const
17537 {
17538 CodeOrigin codeOrigin = m_node->origin.semantic;
17539 if (m_node->op() == TailCallInlinedCaller
17540 || m_node->op() == TailCallVarargsInlinedCaller
17541 || m_node->op() == TailCallForwardVarargsInlinedCaller
17542 || m_node->op() == DirectTailCallInlinedCaller) {
17543 // This case arises when you have a situation like this:
17544 // foo makes a call to bar, bar is inlined in foo. bar makes a call
17545 // to baz and baz is inlined in bar. And then baz makes a tail-call to jaz,
17546 // and jaz is inlined in baz. We want the callframe for jaz to appear to
17547 // have caller be bar.
17548 codeOrigin = *codeOrigin.inlineCallFrame()->getCallerSkippingTailCalls();
17549 }
17550
17551 return codeOrigin;
17552 }
17553
17554 void callCheck()
17555 {
17556 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
17557 if (Options::useExceptionFuzz()) {
17558#if !USE(BUILTIN_FRAME_ADDRESS) || !ASSERT_DISABLED
17559 m_out.storePtr(m_callFrame, m_out.absolute(&vm().topCallFrame));
17560#endif
17561 m_out.call(Void, m_out.operation(operationExceptionFuzz), weakPointer(globalObject));
17562 }
17563
17564 LValue exception = m_out.load64(m_out.absolute(vm().addressOfException()));
17565 LValue hadException = m_out.notZero64(exception);
17566
17567 CodeOrigin opCatchOrigin;
17568 HandlerInfo* exceptionHandler;
17569 if (m_graph.willCatchExceptionInMachineFrame(m_origin.forExit, opCatchOrigin, exceptionHandler)) {
17570 bool exitOK = true;
17571 bool isExceptionHandler = true;
17572 appendOSRExit(
17573 ExceptionCheck, noValue(), nullptr, hadException,
17574 m_origin.withForExitAndExitOK(opCatchOrigin, exitOK), isExceptionHandler);
17575 return;
17576 }
17577
17578 LBasicBlock continuation = m_out.newBlock();
17579
17580 m_out.branch(
17581 hadException, rarely(m_handleExceptions), usually(continuation));
17582
17583 m_out.appendTo(continuation);
17584 }
17585
17586 RefPtr<PatchpointExceptionHandle> preparePatchpointForExceptions(PatchpointValue* value)
17587 {
17588 CodeOrigin opCatchOrigin;
17589 HandlerInfo* exceptionHandler;
17590 bool willCatchException = m_graph.willCatchExceptionInMachineFrame(m_origin.forExit, opCatchOrigin, exceptionHandler);
17591 if (!willCatchException)
17592 return PatchpointExceptionHandle::defaultHandle(m_ftlState);
17593
17594 dataLogLnIf(verboseCompilationEnabled(), " Patchpoint exception OSR exit #", m_ftlState.jitCode->osrExitDescriptors.size(), " with availability: ", availabilityMap());
17595
17596 bool exitOK = true;
17597 NodeOrigin origin = m_origin.withForExitAndExitOK(opCatchOrigin, exitOK);
17598
17599 OSRExitDescriptor* exitDescriptor = appendOSRExitDescriptor(noValue(), nullptr);
17600
17601 // Compute the offset into the StackmapGenerationParams where we will find the exit arguments
17602 // we are about to append. We need to account for both the children we've already added, and
17603 // for the possibility of a result value if the patchpoint is not void.
17604 unsigned offset = value->numChildren();
17605 if (value->type() != Void)
17606 offset++;
17607
17608 // Use LateColdAny to ensure that the stackmap arguments interfere with the patchpoint's
17609 // result and with any late-clobbered registers.
17610 value->appendVectorWithRep(
17611 buildExitArguments(exitDescriptor, opCatchOrigin, noValue()),
17612 ValueRep::LateColdAny);
17613
17614 return PatchpointExceptionHandle::create(
17615 m_ftlState, exitDescriptor, origin, offset, *exceptionHandler);
17616 }
17617
17618 LBasicBlock lowBlock(DFG::BasicBlock* block)
17619 {
17620 return m_blocks.get(block);
17621 }
17622
17623 OSRExitDescriptor* appendOSRExitDescriptor(FormattedValue lowValue, Node* highValue)
17624 {
17625 return appendOSRExitDescriptor(lowValue, m_graph.methodOfGettingAValueProfileFor(m_node, highValue));
17626 }
17627
17628 OSRExitDescriptor* appendOSRExitDescriptor(FormattedValue lowValue, const MethodOfGettingAValueProfile& profile)
17629 {
17630 return &m_ftlState.jitCode->osrExitDescriptors.alloc(
17631 lowValue.format(), profile,
17632 availabilityMap().m_locals.numberOfArguments(),
17633 availabilityMap().m_locals.numberOfLocals());
17634 }
17635
17636 void appendOSRExit(
17637 ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition,
17638 NodeOrigin origin, bool isExceptionHandler = false)
17639 {
17640 return appendOSRExit(kind, lowValue, m_graph.methodOfGettingAValueProfileFor(m_node, highValue),
17641 failCondition, origin, isExceptionHandler);
17642 }
17643
17644 void appendOSRExit(
17645 ExitKind kind, FormattedValue lowValue, const MethodOfGettingAValueProfile& profile, LValue failCondition,
17646 NodeOrigin origin, bool isExceptionHandler = false)
17647 {
17648 dataLogLnIf(verboseCompilationEnabled(), " OSR exit #", m_ftlState.jitCode->osrExitDescriptors.size(), " with availability: ", availabilityMap());
17649
17650 DFG_ASSERT(m_graph, m_node, origin.exitOK);
17651
17652 if (!isExceptionHandler
17653 && Options::useOSRExitFuzz()
17654 && canUseOSRExitFuzzing(m_graph.baselineCodeBlockFor(m_node->origin.semantic))
17655 && doOSRExitFuzzing()) {
17656 LValue numberOfFuzzChecks = m_out.add(
17657 m_out.load32(m_out.absolute(&g_numberOfOSRExitFuzzChecks)),
17658 m_out.int32One);
17659
17660 m_out.store32(numberOfFuzzChecks, m_out.absolute(&g_numberOfOSRExitFuzzChecks));
17661
17662 if (unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter()) {
17663 failCondition = m_out.bitOr(
17664 failCondition,
17665 m_out.aboveOrEqual(numberOfFuzzChecks, m_out.constInt32(atOrAfter)));
17666 }
17667 if (unsigned at = Options::fireOSRExitFuzzAt()) {
17668 failCondition = m_out.bitOr(
17669 failCondition,
17670 m_out.equal(numberOfFuzzChecks, m_out.constInt32(at)));
17671 }
17672 }
17673
17674 if (failCondition == m_out.booleanFalse)
17675 return;
17676
17677 blessSpeculation(
17678 m_out.speculate(failCondition), kind, lowValue, profile, origin);
17679 }
17680
17681 void blessSpeculation(CheckValue* value, ExitKind kind, FormattedValue lowValue, Node* highValue, NodeOrigin origin)
17682 {
17683 blessSpeculation(value, kind, lowValue, m_graph.methodOfGettingAValueProfileFor(m_node, highValue), origin);
17684 }
17685
17686 void blessSpeculation(CheckValue* value, ExitKind kind, FormattedValue lowValue, const MethodOfGettingAValueProfile& profile, NodeOrigin origin)
17687 {
17688 OSRExitDescriptor* exitDescriptor = appendOSRExitDescriptor(lowValue, profile);
17689
17690 value->appendColdAnys(buildExitArguments(exitDescriptor, origin.forExit, lowValue));
17691
17692 State* state = &m_ftlState;
17693 value->setGenerator(
17694 [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
17695 exitDescriptor->emitOSRExit(
17696 *state, kind, origin, jit, params, 0);
17697 });
17698 }
17699
17700 StackmapArgumentList buildExitArguments(
17701 OSRExitDescriptor* exitDescriptor, CodeOrigin exitOrigin, FormattedValue lowValue,
17702 unsigned offsetOfExitArgumentsInStackmapLocations = 0)
17703 {
17704 StackmapArgumentList result;
17705 buildExitArguments(
17706 exitDescriptor, exitOrigin, result, lowValue, offsetOfExitArgumentsInStackmapLocations);
17707 return result;
17708 }
17709
17710 void buildExitArguments(
17711 OSRExitDescriptor* exitDescriptor, CodeOrigin exitOrigin, StackmapArgumentList& arguments, FormattedValue lowValue,
17712 unsigned offsetOfExitArgumentsInStackmapLocations = 0)
17713 {
17714 if (!!lowValue)
17715 arguments.append(lowValue.value());
17716
17717 AvailabilityMap availabilityMap = this->availabilityMap();
17718 availabilityMap.pruneByLiveness(m_graph, exitOrigin);
17719
17720 HashMap<Node*, ExitTimeObjectMaterialization*> map;
17721 availabilityMap.forEachAvailability(
17722 [&] (Availability availability) {
17723 if (!availability.shouldUseNode())
17724 return;
17725
17726 Node* node = availability.node();
17727 if (!node->isPhantomAllocation())
17728 return;
17729
17730 auto result = map.add(node, nullptr);
17731 if (result.isNewEntry) {
17732 result.iterator->value =
17733 exitDescriptor->m_materializations.add(node->op(), node->origin.semantic);
17734 }
17735 });
17736
17737 for (unsigned i = 0; i < exitDescriptor->m_values.size(); ++i) {
17738 int operand = exitDescriptor->m_values.operandForIndex(i);
17739
17740 Availability availability = availabilityMap.m_locals[i];
17741
17742 if (Options::validateFTLOSRExitLiveness()
17743 && m_graph.m_plan.mode() != FTLForOSREntryMode) {
17744
17745 if (availability.isDead() && m_graph.isLiveInBytecode(VirtualRegister(operand), exitOrigin))
17746 DFG_CRASH(m_graph, m_node, toCString("Live bytecode local not available: operand = ", VirtualRegister(operand), ", availability = ", availability, ", origin = ", exitOrigin).data());
17747 }
17748 ExitValue exitValue = exitValueForAvailability(arguments, map, availability);
17749 if (exitValue.hasIndexInStackmapLocations())
17750 exitValue.adjustStackmapLocationsIndexByOffset(offsetOfExitArgumentsInStackmapLocations);
17751 exitDescriptor->m_values[i] = exitValue;
17752 }
17753
17754 for (auto heapPair : availabilityMap.m_heap) {
17755 Node* node = heapPair.key.base();
17756 ExitTimeObjectMaterialization* materialization = map.get(node);
17757 if (!materialization)
17758 DFG_CRASH(m_graph, m_node, toCString("Could not find materialization for ", node, " in ", availabilityMap).data());
17759 ExitValue exitValue = exitValueForAvailability(arguments, map, heapPair.value);
17760 if (exitValue.hasIndexInStackmapLocations())
17761 exitValue.adjustStackmapLocationsIndexByOffset(offsetOfExitArgumentsInStackmapLocations);
17762 materialization->add(
17763 heapPair.key.descriptor(),
17764 exitValue);
17765 }
17766
17767 if (verboseCompilationEnabled()) {
17768 dataLog(" Exit values: ", exitDescriptor->m_values, "\n");
17769 if (!exitDescriptor->m_materializations.isEmpty()) {
17770 dataLog(" Materializations: \n");
17771 for (ExitTimeObjectMaterialization* materialization : exitDescriptor->m_materializations)
17772 dataLog(" ", pointerDump(materialization), "\n");
17773 }
17774 }
17775 }
17776
17777 ExitValue exitValueForAvailability(
17778 StackmapArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map,
17779 Availability availability)
17780 {
17781 FlushedAt flush = availability.flushedAt();
17782 switch (flush.format()) {
17783 case DeadFlush:
17784 case ConflictingFlush:
17785 if (availability.hasNode())
17786 return exitValueForNode(arguments, map, availability.node());
17787
17788 // This means that the value is dead. It could be dead in bytecode or it could have
17789 // been killed by our DCE, which can sometimes kill things even if they were live in
17790 // bytecode.
17791 return ExitValue::dead();
17792
17793 case FlushedJSValue:
17794 case FlushedCell:
17795 case FlushedBoolean:
17796 return ExitValue::inJSStack(flush.virtualRegister());
17797
17798 case FlushedInt32:
17799 return ExitValue::inJSStackAsInt32(flush.virtualRegister());
17800
17801 case FlushedInt52:
17802 return ExitValue::inJSStackAsInt52(flush.virtualRegister());
17803
17804 case FlushedDouble:
17805 return ExitValue::inJSStackAsDouble(flush.virtualRegister());
17806 }
17807
17808 DFG_CRASH(m_graph, m_node, "Invalid flush format");
17809 return ExitValue::dead();
17810 }
17811
17812 ExitValue exitValueForNode(
17813 StackmapArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map,
17814 Node* node)
17815 {
17816 // NOTE: In FTL->B3, we cannot generate code here, because m_output is positioned after the
17817 // stackmap value. Like all values, the stackmap value cannot use a child that is defined after
17818 // it.
17819
17820 ASSERT(node->shouldGenerate());
17821 ASSERT(node->hasResult());
17822
17823 if (node) {
17824 switch (node->op()) {
17825 case BottomValue:
17826 // This might arise in object materializations. I actually doubt that it would,
17827 // but it seems worthwhile to be conservative.
17828 return ExitValue::dead();
17829
17830 case JSConstant:
17831 case Int52Constant:
17832 case DoubleConstant:
17833 return ExitValue::constant(node->asJSValue());
17834
17835 default:
17836 if (node->isPhantomAllocation())
17837 return ExitValue::materializeNewObject(map.get(node));
17838 break;
17839 }
17840 }
17841
17842 LoweredNodeValue value = m_int32Values.get(node);
17843 if (isValid(value))
17844 return exitArgument(arguments, DataFormatInt32, value.value());
17845
17846 value = m_int52Values.get(node);
17847 if (isValid(value))
17848 return exitArgument(arguments, DataFormatInt52, value.value());
17849
17850 value = m_strictInt52Values.get(node);
17851 if (isValid(value))
17852 return exitArgument(arguments, DataFormatStrictInt52, value.value());
17853
17854 value = m_booleanValues.get(node);
17855 if (isValid(value))
17856 return exitArgument(arguments, DataFormatBoolean, value.value());
17857
17858 value = m_jsValueValues.get(node);
17859 if (isValid(value))
17860 return exitArgument(arguments, DataFormatJS, value.value());
17861
17862 value = m_doubleValues.get(node);
17863 if (isValid(value))
17864 return exitArgument(arguments, DataFormatDouble, value.value());
17865
17866 DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data());
17867 return ExitValue::dead();
17868 }
17869
17870 ExitValue exitArgument(StackmapArgumentList& arguments, DataFormat format, LValue value)
17871 {
17872 ExitValue result = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
17873 arguments.append(value);
17874 return result;
17875 }
17876
17877 ExitValue exitValueForTailCall(StackmapArgumentList& arguments, Node* node)
17878 {
17879 ASSERT(node->shouldGenerate());
17880 ASSERT(node->hasResult());
17881
17882 switch (node->op()) {
17883 case JSConstant:
17884 case Int52Constant:
17885 case DoubleConstant:
17886 return ExitValue::constant(node->asJSValue());
17887
17888 default:
17889 break;
17890 }
17891
17892 LoweredNodeValue value = m_jsValueValues.get(node);
17893 if (isValid(value))
17894 return exitArgument(arguments, DataFormatJS, value.value());
17895
17896 value = m_int32Values.get(node);
17897 if (isValid(value))
17898 return exitArgument(arguments, DataFormatJS, boxInt32(value.value()));
17899
17900 value = m_booleanValues.get(node);
17901 if (isValid(value))
17902 return exitArgument(arguments, DataFormatJS, boxBoolean(value.value()));
17903
17904 // Doubles and Int52 have been converted by ValueRep()
17905 DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data());
17906 }
17907
17908 void setInt32(Node* node, LValue value)
17909 {
17910 m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
17911 }
17912 void setInt52(Node* node, LValue value)
17913 {
17914 m_int52Values.set(node, LoweredNodeValue(value, m_highBlock));
17915 }
17916 void setStrictInt52(Node* node, LValue value)
17917 {
17918 m_strictInt52Values.set(node, LoweredNodeValue(value, m_highBlock));
17919 }
17920 void setInt52(Node* node, LValue value, Int52Kind kind)
17921 {
17922 switch (kind) {
17923 case Int52:
17924 setInt52(node, value);
17925 return;
17926
17927 case StrictInt52:
17928 setStrictInt52(node, value);
17929 return;
17930 }
17931
17932 DFG_CRASH(m_graph, m_node, "Corrupt int52 kind");
17933 }
17934 void setJSValue(Node* node, LValue value)
17935 {
17936 m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
17937 }
17938 void setBoolean(Node* node, LValue value)
17939 {
17940 m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
17941 }
17942 void setStorage(Node* node, LValue value)
17943 {
17944 m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
17945 }
17946 void setDouble(Node* node, LValue value)
17947 {
17948 m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
17949 }
17950
17951 void setInt32(LValue value)
17952 {
17953 setInt32(m_node, value);
17954 }
17955 void setInt52(LValue value)
17956 {
17957 setInt52(m_node, value);
17958 }
17959 void setStrictInt52(LValue value)
17960 {
17961 setStrictInt52(m_node, value);
17962 }
17963 void setInt52(LValue value, Int52Kind kind)
17964 {
17965 setInt52(m_node, value, kind);
17966 }
17967 void setJSValue(LValue value)
17968 {
17969 setJSValue(m_node, value);
17970 }
17971 void setBoolean(LValue value)
17972 {
17973 setBoolean(m_node, value);
17974 }
17975 void setStorage(LValue value)
17976 {
17977 setStorage(m_node, value);
17978 }
17979 void setDouble(LValue value)
17980 {
17981 setDouble(m_node, value);
17982 }
17983
17984 bool isValid(const LoweredNodeValue& value)
17985 {
17986 if (!value)
17987 return false;
17988 if (!m_graph.m_ssaDominators->dominates(value.block(), m_highBlock))
17989 return false;
17990 return true;
17991 }
17992
17993 void keepAlive(LValue value)
17994 {
17995 PatchpointValue* patchpoint = m_out.patchpoint(Void);
17996 patchpoint->effects = Effects::none();
17997 patchpoint->effects.writesLocalState = true;
17998 patchpoint->effects.reads = HeapRange::top();
17999 patchpoint->append(value, ValueRep::ColdAny);
18000 patchpoint->setGenerator([=] (CCallHelpers&, const StackmapGenerationParams&) { });
18001 }
18002
18003 void addWeakReference(JSCell* target)
18004 {
18005 m_graph.m_plan.weakReferences().addLazily(target);
18006 }
18007
18008 LValue loadStructure(LValue value)
18009 {
18010 LValue structureID = m_out.load32(value, m_heaps.JSCell_structureID);
18011 LValue tableBase = m_out.loadPtr(m_out.absolute(vm().heap.structureIDTable().base()));
18012 LValue tableIndex = m_out.aShr(structureID, m_out.constInt32(StructureIDTable::s_numberOfEntropyBits));
18013 LValue entropyBits = m_out.shl(m_out.zeroExtPtr(structureID), m_out.constInt32(StructureIDTable::s_entropyBitsShiftForStructurePointer));
18014 TypedPointer address = m_out.baseIndex(m_heaps.structureTable, tableBase, m_out.zeroExtPtr(tableIndex));
18015 LValue encodedStructureBits = m_out.loadPtr(address);
18016 return m_out.bitXor(encodedStructureBits, entropyBits);
18017 }
18018
18019 LValue weakPointer(JSCell* pointer)
18020 {
18021 addWeakReference(pointer);
18022 return m_out.alreadyRegisteredWeakPointer(m_graph, pointer);
18023 }
18024
18025 LValue frozenPointer(FrozenValue* value)
18026 {
18027 return m_out.alreadyRegisteredFrozenPointer(value);
18028 }
18029
18030 LValue weakStructureID(RegisteredStructure structure)
18031 {
18032 return m_out.constInt32(structure->id());
18033 }
18034
18035 LValue weakStructure(RegisteredStructure structure)
18036 {
18037 ASSERT(!!structure.get());
18038 return m_out.alreadyRegisteredWeakPointer(m_graph, structure.get());
18039 }
18040
18041 TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
18042 {
18043 return m_out.address(base, m_heaps.variables[operand], offset);
18044 }
18045 TypedPointer payloadFor(LValue base, int operand)
18046 {
18047 return addressFor(base, operand, PayloadOffset);
18048 }
18049 TypedPointer tagFor(LValue base, int operand)
18050 {
18051 return addressFor(base, operand, TagOffset);
18052 }
18053 TypedPointer addressFor(int operand, ptrdiff_t offset = 0)
18054 {
18055 return addressFor(VirtualRegister(operand), offset);
18056 }
18057 TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0)
18058 {
18059 if (operand.isLocal())
18060 return addressFor(m_captured, operand.offset(), offset);
18061 return addressFor(m_callFrame, operand.offset(), offset);
18062 }
18063 TypedPointer payloadFor(int operand)
18064 {
18065 return payloadFor(VirtualRegister(operand));
18066 }
18067 TypedPointer payloadFor(VirtualRegister operand)
18068 {
18069 return addressFor(operand, PayloadOffset);
18070 }
18071 TypedPointer tagFor(int operand)
18072 {
18073 return tagFor(VirtualRegister(operand));
18074 }
18075 TypedPointer tagFor(VirtualRegister operand)
18076 {
18077 return addressFor(operand, TagOffset);
18078 }
18079
18080 AbstractValue abstractValue(Node* node)
18081 {
18082 return m_state.forNode(node);
18083 }
18084 AbstractValue abstractValue(Edge edge)
18085 {
18086 return abstractValue(edge.node());
18087 }
18088
18089 SpeculatedType provenType(Node* node)
18090 {
18091 return abstractValue(node).m_type;
18092 }
18093 SpeculatedType provenType(Edge edge)
18094 {
18095 return provenType(edge.node());
18096 }
18097
18098 JSValue provenValue(Node* node)
18099 {
18100 return abstractValue(node).m_value;
18101 }
18102 JSValue provenValue(Edge edge)
18103 {
18104 return provenValue(edge.node());
18105 }
18106
18107 StructureAbstractValue abstractStructure(Node* node)
18108 {
18109 return abstractValue(node).m_structure;
18110 }
18111 StructureAbstractValue abstractStructure(Edge edge)
18112 {
18113 return abstractStructure(edge.node());
18114 }
18115
18116 void crash()
18117 {
18118 crash(m_highBlock, m_node);
18119 }
18120 void crash(DFG::BasicBlock* block, Node* node)
18121 {
18122 BlockIndex blockIndex = block->index;
18123 unsigned nodeIndex = node ? node->index() : UINT_MAX;
18124#if ASSERT_DISABLED
18125 m_out.patchpoint(Void)->setGenerator(
18126 [=] (CCallHelpers& jit, const StackmapGenerationParams&) {
18127 AllowMacroScratchRegisterUsage allowScratch(jit);
18128
18129 jit.move(CCallHelpers::TrustedImm32(blockIndex), GPRInfo::regT0);
18130 jit.move(CCallHelpers::TrustedImm32(nodeIndex), GPRInfo::regT1);
18131 if (node)
18132 jit.move(CCallHelpers::TrustedImm32(node->op()), GPRInfo::regT2);
18133 jit.abortWithReason(FTLCrash);
18134 });
18135#else
18136 m_out.call(
18137 Void,
18138 m_out.constIntPtr(ftlUnreachable),
18139 // We don't want the CodeBlock to have a weak pointer to itself because
18140 // that would cause it to always get collected.
18141 m_out.constIntPtr(bitwise_cast<intptr_t>(codeBlock())), m_out.constInt32(blockIndex),
18142 m_out.constInt32(nodeIndex));
18143#endif
18144 m_out.unreachable();
18145 }
18146
18147 AvailabilityMap& availabilityMap() { return m_availabilityCalculator.m_availability; }
18148
18149 VM& vm() { return m_graph.m_vm; }
18150 CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
18151
18152 Graph& m_graph;
18153 State& m_ftlState;
18154 AbstractHeapRepository m_heaps;
18155 Output m_out;
18156 Procedure& m_proc;
18157
18158 LBasicBlock m_handleExceptions;
18159 HashMap<DFG::BasicBlock*, LBasicBlock> m_blocks;
18160
18161 LValue m_callFrame;
18162 LValue m_vmValue;
18163 LValue m_captured;
18164 LValue m_numberTag;
18165 LValue m_notCellMask;
18166
18167 HashMap<Node*, LoweredNodeValue> m_int32Values;
18168 HashMap<Node*, LoweredNodeValue> m_strictInt52Values;
18169 HashMap<Node*, LoweredNodeValue> m_int52Values;
18170 HashMap<Node*, LoweredNodeValue> m_jsValueValues;
18171 HashMap<Node*, LoweredNodeValue> m_booleanValues;
18172 HashMap<Node*, LoweredNodeValue> m_storageValues;
18173 HashMap<Node*, LoweredNodeValue> m_doubleValues;
18174
18175 HashMap<Node*, LValue> m_phis;
18176
18177 LocalOSRAvailabilityCalculator m_availabilityCalculator;
18178
18179 InPlaceAbstractState m_state;
18180 AbstractInterpreter<InPlaceAbstractState> m_interpreter;
18181 DFG::BasicBlock* m_highBlock;
18182 DFG::BasicBlock* m_nextHighBlock;
18183 LBasicBlock m_nextLowBlock;
18184
18185 NodeOrigin m_origin;
18186 unsigned m_nodeIndex;
18187 Node* m_node;
18188
18189 // These are used for validating AI state.
18190 HashMap<Node*, NodeSet> m_liveInToNode;
18191 HashMap<Node*, AbstractValue> m_aiCheckedNodes;
18192 String m_graphDump;
18193};
18194
18195} // anonymous namespace
18196
18197void lowerDFGToB3(State& state)
18198{
18199 LowerDFGToB3 lowering(state);
18200 lowering.lower();
18201}
18202
18203} } // namespace JSC::FTL
18204
18205#endif // ENABLE(FTL_JIT)
18206
18207