1/*
2 * Copyright (C) 2011-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "DFGSpeculativeJIT.h"
28
29#if ENABLE(DFG_JIT)
30
31#include "BinarySwitch.h"
32#include "DFGAbstractInterpreterInlines.h"
33#include "DFGArrayifySlowPathGenerator.h"
34#include "DFGCallArrayAllocatorSlowPathGenerator.h"
35#include "DFGCallCreateDirectArgumentsSlowPathGenerator.h"
36#include "DFGCapabilities.h"
37#include "DFGMayExit.h"
38#include "DFGOSRExitFuzz.h"
39#include "DFGSaneStringGetByValSlowPathGenerator.h"
40#include "DFGSlowPathGenerator.h"
41#include "DFGSnippetParams.h"
42#include "DirectArguments.h"
43#include "JITAddGenerator.h"
44#include "JITBitAndGenerator.h"
45#include "JITBitOrGenerator.h"
46#include "JITBitXorGenerator.h"
47#include "JITDivGenerator.h"
48#include "JITLeftShiftGenerator.h"
49#include "JITMulGenerator.h"
50#include "JITRightShiftGenerator.h"
51#include "JITSubGenerator.h"
52#include "JSAsyncFunction.h"
53#include "JSAsyncGeneratorFunction.h"
54#include "JSCInlines.h"
55#include "JSFixedArray.h"
56#include "JSGeneratorFunction.h"
57#include "JSImmutableButterfly.h"
58#include "JSLexicalEnvironment.h"
59#include "JSPropertyNameEnumerator.h"
60#include "LinkBuffer.h"
61#include "RegExpObject.h"
62#include "ScopedArguments.h"
63#include "ScratchRegisterAllocator.h"
64#include "SuperSampler.h"
65#include "TypeProfilerLog.h"
66#include "WeakMapImpl.h"
67#include <wtf/BitVector.h>
68#include <wtf/Box.h>
69#include <wtf/MathExtras.h>
70
71namespace JSC { namespace DFG {
72
73SpeculativeJIT::SpeculativeJIT(JITCompiler& jit)
74 : m_jit(jit)
75 , m_graph(m_jit.graph())
76 , m_currentNode(0)
77 , m_lastGeneratedNode(LastNodeType)
78 , m_indexInBlock(0)
79 , m_generationInfo(m_jit.graph().frameRegisterCount())
80 , m_compileOkay(true)
81 , m_state(m_jit.graph())
82 , m_interpreter(m_jit.graph(), m_state)
83 , m_stream(&jit.jitCode()->variableEventStream)
84 , m_minifiedGraph(&jit.jitCode()->minifiedDFG)
85{
86}
87
88SpeculativeJIT::~SpeculativeJIT()
89{
90}
91
92void SpeculativeJIT::emitAllocateRawObject(GPRReg resultGPR, RegisteredStructure structure, GPRReg storageGPR, unsigned numElements, unsigned vectorLength)
93{
94 ASSERT(!isCopyOnWrite(structure->indexingMode()));
95 IndexingType indexingType = structure->indexingType();
96 bool hasIndexingHeader = hasIndexedProperties(indexingType);
97
98 unsigned inlineCapacity = structure->inlineCapacity();
99 unsigned outOfLineCapacity = structure->outOfLineCapacity();
100
101 GPRTemporary scratch(this);
102 GPRTemporary scratch2(this);
103 GPRReg scratchGPR = scratch.gpr();
104 GPRReg scratch2GPR = scratch2.gpr();
105
106 ASSERT(vectorLength >= numElements);
107 vectorLength = Butterfly::optimalContiguousVectorLength(structure.get(), vectorLength);
108
109 JITCompiler::JumpList slowCases;
110
111 size_t size = 0;
112 if (hasIndexingHeader)
113 size += vectorLength * sizeof(JSValue) + sizeof(IndexingHeader);
114 size += outOfLineCapacity * sizeof(JSValue);
115
116 m_jit.move(TrustedImmPtr(nullptr), storageGPR);
117
118 if (size) {
119 if (Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists)) {
120 m_jit.emitAllocate(storageGPR, JITAllocator::constant(allocator), scratchGPR, scratch2GPR, slowCases);
121
122 m_jit.addPtr(
123 TrustedImm32(outOfLineCapacity * sizeof(JSValue) + sizeof(IndexingHeader)),
124 storageGPR);
125
126 if (hasIndexingHeader)
127 m_jit.store32(TrustedImm32(vectorLength), MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
128 } else
129 slowCases.append(m_jit.jump());
130 }
131
132 size_t allocationSize = JSFinalObject::allocationSize(inlineCapacity);
133 Allocator allocator = allocatorForNonVirtualConcurrently<JSFinalObject>(*m_jit.vm(), allocationSize, AllocatorForMode::AllocatorIfExists);
134 if (allocator) {
135 emitAllocateJSObject(resultGPR, JITAllocator::constant(allocator), scratchGPR, TrustedImmPtr(structure), storageGPR, scratch2GPR, slowCases);
136 m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
137 } else
138 slowCases.append(m_jit.jump());
139
140 // I want a slow path that also loads out the storage pointer, and that's
141 // what this custom CallArrayAllocatorSlowPathGenerator gives me. It's a lot
142 // of work for a very small piece of functionality. :-/
143 addSlowPathGenerator(std::make_unique<CallArrayAllocatorSlowPathGenerator>(
144 slowCases, this, operationNewRawObject, resultGPR, storageGPR,
145 structure, vectorLength));
146
147 if (numElements < vectorLength) {
148#if USE(JSVALUE64)
149 if (hasDouble(structure->indexingType()))
150 m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR);
151 else
152 m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR);
153 for (unsigned i = numElements; i < vectorLength; ++i)
154 m_jit.store64(scratchGPR, MacroAssembler::Address(storageGPR, sizeof(double) * i));
155#else
156 EncodedValueDescriptor value;
157 if (hasDouble(structure->indexingType()))
158 value.asInt64 = JSValue::encode(JSValue(JSValue::EncodeAsDouble, PNaN));
159 else
160 value.asInt64 = JSValue::encode(JSValue());
161 for (unsigned i = numElements; i < vectorLength; ++i) {
162 m_jit.store32(TrustedImm32(value.asBits.tag), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
163 m_jit.store32(TrustedImm32(value.asBits.payload), MacroAssembler::Address(storageGPR, sizeof(double) * i + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
164 }
165#endif
166 }
167
168 if (hasIndexingHeader)
169 m_jit.store32(TrustedImm32(numElements), MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
170
171 m_jit.emitInitializeOutOfLineStorage(storageGPR, structure->outOfLineCapacity());
172
173 m_jit.mutatorFence(*m_jit.vm());
174}
175
176void SpeculativeJIT::emitGetLength(InlineCallFrame* inlineCallFrame, GPRReg lengthGPR, bool includeThis)
177{
178 if (inlineCallFrame && !inlineCallFrame->isVarargs())
179 m_jit.move(TrustedImm32(inlineCallFrame->argumentCountIncludingThis - !includeThis), lengthGPR);
180 else {
181 VirtualRegister argumentCountRegister = m_jit.argumentCount(inlineCallFrame);
182 m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
183 if (!includeThis)
184 m_jit.sub32(TrustedImm32(1), lengthGPR);
185 }
186}
187
188void SpeculativeJIT::emitGetLength(CodeOrigin origin, GPRReg lengthGPR, bool includeThis)
189{
190 emitGetLength(origin.inlineCallFrame(), lengthGPR, includeThis);
191}
192
193void SpeculativeJIT::emitGetCallee(CodeOrigin origin, GPRReg calleeGPR)
194{
195 auto* inlineCallFrame = origin.inlineCallFrame();
196 if (inlineCallFrame) {
197 if (inlineCallFrame->isClosureCall) {
198 m_jit.loadPtr(
199 JITCompiler::addressFor(inlineCallFrame->calleeRecovery.virtualRegister()),
200 calleeGPR);
201 } else {
202 m_jit.move(
203 TrustedImmPtr::weakPointer(m_jit.graph(), inlineCallFrame->calleeRecovery.constant().asCell()),
204 calleeGPR);
205 }
206 } else
207 m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), calleeGPR);
208}
209
210void SpeculativeJIT::emitGetArgumentStart(CodeOrigin origin, GPRReg startGPR)
211{
212 m_jit.addPtr(
213 TrustedImm32(
214 JITCompiler::argumentsStart(origin).offset() * static_cast<int>(sizeof(Register))),
215 GPRInfo::callFrameRegister, startGPR);
216}
217
218MacroAssembler::Jump SpeculativeJIT::emitOSRExitFuzzCheck()
219{
220 if (!Options::useOSRExitFuzz()
221 || !canUseOSRExitFuzzing(m_jit.graph().baselineCodeBlockFor(m_origin.semantic))
222 || !doOSRExitFuzzing())
223 return MacroAssembler::Jump();
224
225 MacroAssembler::Jump result;
226
227 m_jit.pushToSave(GPRInfo::regT0);
228 m_jit.load32(&g_numberOfOSRExitFuzzChecks, GPRInfo::regT0);
229 m_jit.add32(TrustedImm32(1), GPRInfo::regT0);
230 m_jit.store32(GPRInfo::regT0, &g_numberOfOSRExitFuzzChecks);
231 unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter();
232 unsigned at = Options::fireOSRExitFuzzAt();
233 if (at || atOrAfter) {
234 unsigned threshold;
235 MacroAssembler::RelationalCondition condition;
236 if (atOrAfter) {
237 threshold = atOrAfter;
238 condition = MacroAssembler::Below;
239 } else {
240 threshold = at;
241 condition = MacroAssembler::NotEqual;
242 }
243 MacroAssembler::Jump ok = m_jit.branch32(
244 condition, GPRInfo::regT0, MacroAssembler::TrustedImm32(threshold));
245 m_jit.popToRestore(GPRInfo::regT0);
246 result = m_jit.jump();
247 ok.link(&m_jit);
248 }
249 m_jit.popToRestore(GPRInfo::regT0);
250
251 return result;
252}
253
254void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail)
255{
256 if (!m_compileOkay)
257 return;
258 JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
259 if (fuzzJump.isSet()) {
260 JITCompiler::JumpList jumpsToFail;
261 jumpsToFail.append(fuzzJump);
262 jumpsToFail.append(jumpToFail);
263 m_jit.appendExitInfo(jumpsToFail);
264 } else
265 m_jit.appendExitInfo(jumpToFail);
266 m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
267}
268
269void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, const MacroAssembler::JumpList& jumpsToFail)
270{
271 if (!m_compileOkay)
272 return;
273 JITCompiler::Jump fuzzJump = emitOSRExitFuzzCheck();
274 if (fuzzJump.isSet()) {
275 JITCompiler::JumpList myJumpsToFail;
276 myJumpsToFail.append(jumpsToFail);
277 myJumpsToFail.append(fuzzJump);
278 m_jit.appendExitInfo(myJumpsToFail);
279 } else
280 m_jit.appendExitInfo(jumpsToFail);
281 m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
282}
283
284OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node)
285{
286 if (!m_compileOkay)
287 return OSRExitJumpPlaceholder();
288 unsigned index = m_jit.jitCode()->osrExit.size();
289 m_jit.appendExitInfo();
290 m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size()));
291 return OSRExitJumpPlaceholder(index);
292}
293
294OSRExitJumpPlaceholder SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse)
295{
296 return speculationCheck(kind, jsValueSource, nodeUse.node());
297}
298
299void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail)
300{
301 speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail);
302}
303
304void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, const MacroAssembler::JumpList& jumpsToFail)
305{
306 speculationCheck(kind, jsValueSource, nodeUse.node(), jumpsToFail);
307}
308
309void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Node* node, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
310{
311 if (!m_compileOkay)
312 return;
313 unsigned recoveryIndex = m_jit.jitCode()->appendSpeculationRecovery(recovery);
314 m_jit.appendExitInfo(jumpToFail);
315 m_jit.jitCode()->appendOSRExit(OSRExit(kind, jsValueSource, m_jit.graph().methodOfGettingAValueProfileFor(m_currentNode, node), this, m_stream->size(), recoveryIndex));
316}
317
318void SpeculativeJIT::speculationCheck(ExitKind kind, JSValueSource jsValueSource, Edge nodeUse, MacroAssembler::Jump jumpToFail, const SpeculationRecovery& recovery)
319{
320 speculationCheck(kind, jsValueSource, nodeUse.node(), jumpToFail, recovery);
321}
322
323void SpeculativeJIT::emitInvalidationPoint(Node* node)
324{
325 if (!m_compileOkay)
326 return;
327 OSRExitCompilationInfo& info = m_jit.appendExitInfo(JITCompiler::JumpList());
328 m_jit.jitCode()->appendOSRExit(OSRExit(
329 UncountableInvalidation, JSValueSource(), MethodOfGettingAValueProfile(),
330 this, m_stream->size()));
331 info.m_replacementSource = m_jit.watchpointLabel();
332 ASSERT(info.m_replacementSource.isSet());
333 noResult(node);
334}
335
336void SpeculativeJIT::unreachable(Node* node)
337{
338 m_compileOkay = false;
339 m_jit.abortWithReason(DFGUnreachableNode, node->op());
340}
341
342void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Node* node)
343{
344 if (!m_compileOkay)
345 return;
346 speculationCheck(kind, jsValueRegs, node, m_jit.jump());
347 m_compileOkay = false;
348 if (verboseCompilationEnabled())
349 dataLog("Bailing compilation.\n");
350}
351
352void SpeculativeJIT::terminateSpeculativeExecution(ExitKind kind, JSValueRegs jsValueRegs, Edge nodeUse)
353{
354 terminateSpeculativeExecution(kind, jsValueRegs, nodeUse.node());
355}
356
357void SpeculativeJIT::typeCheck(JSValueSource source, Edge edge, SpeculatedType typesPassedThrough, MacroAssembler::Jump jumpToFail, ExitKind exitKind)
358{
359 ASSERT(needsTypeCheck(edge, typesPassedThrough));
360 m_interpreter.filter(edge, typesPassedThrough);
361 speculationCheck(exitKind, source, edge.node(), jumpToFail);
362}
363
364RegisterSet SpeculativeJIT::usedRegisters()
365{
366 RegisterSet result;
367
368 for (unsigned i = GPRInfo::numberOfRegisters; i--;) {
369 GPRReg gpr = GPRInfo::toRegister(i);
370 if (m_gprs.isInUse(gpr))
371 result.set(gpr);
372 }
373 for (unsigned i = FPRInfo::numberOfRegisters; i--;) {
374 FPRReg fpr = FPRInfo::toRegister(i);
375 if (m_fprs.isInUse(fpr))
376 result.set(fpr);
377 }
378
379 // FIXME: This is overly conservative. We could subtract out those callee-saves that we
380 // actually saved.
381 // https://bugs.webkit.org/show_bug.cgi?id=185686
382 result.merge(RegisterSet::stubUnavailableRegisters());
383
384 return result;
385}
386
387void SpeculativeJIT::addSlowPathGenerator(std::unique_ptr<SlowPathGenerator> slowPathGenerator)
388{
389 m_slowPathGenerators.append(WTFMove(slowPathGenerator));
390}
391
392void SpeculativeJIT::addSlowPathGeneratorLambda(Function<void()>&& lambda)
393{
394 m_slowPathLambdas.append(SlowPathLambda{ WTFMove(lambda), m_currentNode, static_cast<unsigned>(m_stream->size()) });
395}
396
397void SpeculativeJIT::runSlowPathGenerators(PCToCodeOriginMapBuilder& pcToCodeOriginMapBuilder)
398{
399 for (auto& slowPathGenerator : m_slowPathGenerators) {
400 pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), slowPathGenerator->origin().semantic);
401 slowPathGenerator->generate(this);
402 }
403 for (auto& slowPathLambda : m_slowPathLambdas) {
404 Node* currentNode = slowPathLambda.currentNode;
405 m_currentNode = currentNode;
406 m_outOfLineStreamIndex = slowPathLambda.streamIndex;
407 pcToCodeOriginMapBuilder.appendItem(m_jit.labelIgnoringWatchpoints(), currentNode->origin.semantic);
408 slowPathLambda.generator();
409 m_outOfLineStreamIndex = WTF::nullopt;
410 }
411}
412
413void SpeculativeJIT::clearGenerationInfo()
414{
415 for (unsigned i = 0; i < m_generationInfo.size(); ++i)
416 m_generationInfo[i] = GenerationInfo();
417 m_gprs = RegisterBank<GPRInfo>();
418 m_fprs = RegisterBank<FPRInfo>();
419}
420
421SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForGPR(VirtualRegister spillMe, GPRReg source)
422{
423 GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
424 Node* node = info.node();
425 DataFormat registerFormat = info.registerFormat();
426 ASSERT(registerFormat != DataFormatNone);
427 ASSERT(registerFormat != DataFormatDouble);
428
429 SilentSpillAction spillAction;
430 SilentFillAction fillAction;
431
432 if (!info.needsSpill())
433 spillAction = DoNothingForSpill;
434 else {
435#if USE(JSVALUE64)
436 ASSERT(info.gpr() == source);
437 if (registerFormat == DataFormatInt32)
438 spillAction = Store32Payload;
439 else if (registerFormat == DataFormatCell || registerFormat == DataFormatStorage)
440 spillAction = StorePtr;
441 else if (registerFormat == DataFormatInt52 || registerFormat == DataFormatStrictInt52)
442 spillAction = Store64;
443 else {
444 ASSERT(registerFormat & DataFormatJS);
445 spillAction = Store64;
446 }
447#elif USE(JSVALUE32_64)
448 if (registerFormat & DataFormatJS) {
449 ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
450 spillAction = source == info.tagGPR() ? Store32Tag : Store32Payload;
451 } else {
452 ASSERT(info.gpr() == source);
453 spillAction = Store32Payload;
454 }
455#endif
456 }
457
458 if (registerFormat == DataFormatInt32) {
459 ASSERT(info.gpr() == source);
460 ASSERT(isJSInt32(info.registerFormat()));
461 if (node->hasConstant()) {
462 ASSERT(node->isInt32Constant());
463 fillAction = SetInt32Constant;
464 } else
465 fillAction = Load32Payload;
466 } else if (registerFormat == DataFormatBoolean) {
467#if USE(JSVALUE64)
468 RELEASE_ASSERT_NOT_REACHED();
469#if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
470 fillAction = DoNothingForFill;
471#endif
472#elif USE(JSVALUE32_64)
473 ASSERT(info.gpr() == source);
474 if (node->hasConstant()) {
475 ASSERT(node->isBooleanConstant());
476 fillAction = SetBooleanConstant;
477 } else
478 fillAction = Load32Payload;
479#endif
480 } else if (registerFormat == DataFormatCell) {
481 ASSERT(info.gpr() == source);
482 if (node->hasConstant()) {
483 DFG_ASSERT(m_jit.graph(), m_currentNode, node->isCellConstant());
484 node->asCell(); // To get the assertion.
485 fillAction = SetCellConstant;
486 } else {
487#if USE(JSVALUE64)
488 fillAction = LoadPtr;
489#else
490 fillAction = Load32Payload;
491#endif
492 }
493 } else if (registerFormat == DataFormatStorage) {
494 ASSERT(info.gpr() == source);
495 fillAction = LoadPtr;
496 } else if (registerFormat == DataFormatInt52) {
497 if (node->hasConstant())
498 fillAction = SetInt52Constant;
499 else if (info.spillFormat() == DataFormatInt52)
500 fillAction = Load64;
501 else if (info.spillFormat() == DataFormatStrictInt52)
502 fillAction = Load64ShiftInt52Left;
503 else if (info.spillFormat() == DataFormatNone)
504 fillAction = Load64;
505 else {
506 RELEASE_ASSERT_NOT_REACHED();
507#if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
508 fillAction = Load64; // Make GCC happy.
509#endif
510 }
511 } else if (registerFormat == DataFormatStrictInt52) {
512 if (node->hasConstant())
513 fillAction = SetStrictInt52Constant;
514 else if (info.spillFormat() == DataFormatInt52)
515 fillAction = Load64ShiftInt52Right;
516 else if (info.spillFormat() == DataFormatStrictInt52)
517 fillAction = Load64;
518 else if (info.spillFormat() == DataFormatNone)
519 fillAction = Load64;
520 else {
521 RELEASE_ASSERT_NOT_REACHED();
522#if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE)
523 fillAction = Load64; // Make GCC happy.
524#endif
525 }
526 } else {
527 ASSERT(registerFormat & DataFormatJS);
528#if USE(JSVALUE64)
529 ASSERT(info.gpr() == source);
530 if (node->hasConstant()) {
531 if (node->isCellConstant())
532 fillAction = SetTrustedJSConstant;
533 else
534 fillAction = SetJSConstant;
535 } else if (info.spillFormat() == DataFormatInt32) {
536 ASSERT(registerFormat == DataFormatJSInt32);
537 fillAction = Load32PayloadBoxInt;
538 } else
539 fillAction = Load64;
540#else
541 ASSERT(info.tagGPR() == source || info.payloadGPR() == source);
542 if (node->hasConstant())
543 fillAction = info.tagGPR() == source ? SetJSConstantTag : SetJSConstantPayload;
544 else if (info.payloadGPR() == source)
545 fillAction = Load32Payload;
546 else { // Fill the Tag
547 switch (info.spillFormat()) {
548 case DataFormatInt32:
549 ASSERT(registerFormat == DataFormatJSInt32);
550 fillAction = SetInt32Tag;
551 break;
552 case DataFormatCell:
553 ASSERT(registerFormat == DataFormatJSCell);
554 fillAction = SetCellTag;
555 break;
556 case DataFormatBoolean:
557 ASSERT(registerFormat == DataFormatJSBoolean);
558 fillAction = SetBooleanTag;
559 break;
560 default:
561 fillAction = Load32Tag;
562 break;
563 }
564 }
565#endif
566 }
567
568 return SilentRegisterSavePlan(spillAction, fillAction, node, source);
569}
570
571SilentRegisterSavePlan SpeculativeJIT::silentSavePlanForFPR(VirtualRegister spillMe, FPRReg source)
572{
573 GenerationInfo& info = generationInfoFromVirtualRegister(spillMe);
574 Node* node = info.node();
575 ASSERT(info.registerFormat() == DataFormatDouble);
576
577 SilentSpillAction spillAction;
578 SilentFillAction fillAction;
579
580 if (!info.needsSpill())
581 spillAction = DoNothingForSpill;
582 else {
583 ASSERT(!node->hasConstant());
584 ASSERT(info.spillFormat() == DataFormatNone);
585 ASSERT(info.fpr() == source);
586 spillAction = StoreDouble;
587 }
588
589#if USE(JSVALUE64)
590 if (node->hasConstant()) {
591 node->asNumber(); // To get the assertion.
592 fillAction = SetDoubleConstant;
593 } else {
594 ASSERT(info.spillFormat() == DataFormatNone || info.spillFormat() == DataFormatDouble);
595 fillAction = LoadDouble;
596 }
597#elif USE(JSVALUE32_64)
598 ASSERT(info.registerFormat() == DataFormatDouble);
599 if (node->hasConstant()) {
600 node->asNumber(); // To get the assertion.
601 fillAction = SetDoubleConstant;
602 } else
603 fillAction = LoadDouble;
604#endif
605
606 return SilentRegisterSavePlan(spillAction, fillAction, node, source);
607}
608
609void SpeculativeJIT::silentSpill(const SilentRegisterSavePlan& plan)
610{
611 switch (plan.spillAction()) {
612 case DoNothingForSpill:
613 break;
614 case Store32Tag:
615 m_jit.store32(plan.gpr(), JITCompiler::tagFor(plan.node()->virtualRegister()));
616 break;
617 case Store32Payload:
618 m_jit.store32(plan.gpr(), JITCompiler::payloadFor(plan.node()->virtualRegister()));
619 break;
620 case StorePtr:
621 m_jit.storePtr(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
622 break;
623#if USE(JSVALUE64)
624 case Store64:
625 m_jit.store64(plan.gpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
626 break;
627#endif
628 case StoreDouble:
629 m_jit.storeDouble(plan.fpr(), JITCompiler::addressFor(plan.node()->virtualRegister()));
630 break;
631 default:
632 RELEASE_ASSERT_NOT_REACHED();
633 }
634}
635
636void SpeculativeJIT::silentFill(const SilentRegisterSavePlan& plan)
637{
638 switch (plan.fillAction()) {
639 case DoNothingForFill:
640 break;
641 case SetInt32Constant:
642 m_jit.move(Imm32(plan.node()->asInt32()), plan.gpr());
643 break;
644#if USE(JSVALUE64)
645 case SetInt52Constant:
646 m_jit.move(Imm64(plan.node()->asAnyInt() << JSValue::int52ShiftAmount), plan.gpr());
647 break;
648 case SetStrictInt52Constant:
649 m_jit.move(Imm64(plan.node()->asAnyInt()), plan.gpr());
650 break;
651#endif // USE(JSVALUE64)
652 case SetBooleanConstant:
653 m_jit.move(TrustedImm32(plan.node()->asBoolean()), plan.gpr());
654 break;
655 case SetCellConstant:
656 ASSERT(plan.node()->constant()->value().isCell());
657 m_jit.move(TrustedImmPtr(plan.node()->constant()), plan.gpr());
658 break;
659#if USE(JSVALUE64)
660 case SetTrustedJSConstant:
661 m_jit.move(valueOfJSConstantAsImm64(plan.node()).asTrustedImm64(), plan.gpr());
662 break;
663 case SetJSConstant:
664 m_jit.move(valueOfJSConstantAsImm64(plan.node()), plan.gpr());
665 break;
666 case SetDoubleConstant:
667 m_jit.moveDouble(Imm64(reinterpretDoubleToInt64(plan.node()->asNumber())), plan.fpr());
668 break;
669 case Load32PayloadBoxInt:
670 m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
671 m_jit.or64(GPRInfo::tagTypeNumberRegister, plan.gpr());
672 break;
673 case Load32PayloadConvertToInt52:
674 m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
675 m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
676 m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
677 break;
678 case Load32PayloadSignExtend:
679 m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
680 m_jit.signExtend32ToPtr(plan.gpr(), plan.gpr());
681 break;
682#else
683 case SetJSConstantTag:
684 m_jit.move(Imm32(plan.node()->asJSValue().tag()), plan.gpr());
685 break;
686 case SetJSConstantPayload:
687 m_jit.move(Imm32(plan.node()->asJSValue().payload()), plan.gpr());
688 break;
689 case SetInt32Tag:
690 m_jit.move(TrustedImm32(JSValue::Int32Tag), plan.gpr());
691 break;
692 case SetCellTag:
693 m_jit.move(TrustedImm32(JSValue::CellTag), plan.gpr());
694 break;
695 case SetBooleanTag:
696 m_jit.move(TrustedImm32(JSValue::BooleanTag), plan.gpr());
697 break;
698 case SetDoubleConstant:
699 m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(plan.node())), plan.fpr());
700 break;
701#endif
702 case Load32Tag:
703 m_jit.load32(JITCompiler::tagFor(plan.node()->virtualRegister()), plan.gpr());
704 break;
705 case Load32Payload:
706 m_jit.load32(JITCompiler::payloadFor(plan.node()->virtualRegister()), plan.gpr());
707 break;
708 case LoadPtr:
709 m_jit.loadPtr(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
710 break;
711#if USE(JSVALUE64)
712 case Load64:
713 m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
714 break;
715 case Load64ShiftInt52Right:
716 m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
717 m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
718 break;
719 case Load64ShiftInt52Left:
720 m_jit.load64(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.gpr());
721 m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), plan.gpr());
722 break;
723#endif
724 case LoadDouble:
725 m_jit.loadDouble(JITCompiler::addressFor(plan.node()->virtualRegister()), plan.fpr());
726 break;
727 default:
728 RELEASE_ASSERT_NOT_REACHED();
729 }
730}
731
732JITCompiler::JumpList SpeculativeJIT::jumpSlowForUnwantedArrayMode(GPRReg tempGPR, ArrayMode arrayMode)
733{
734 JITCompiler::JumpList result;
735
736 IndexingType indexingModeMask = IsArray | IndexingShapeMask;
737 if (arrayMode.action() == Array::Write)
738 indexingModeMask |= CopyOnWrite;
739
740 switch (arrayMode.type()) {
741 case Array::Int32:
742 case Array::Double:
743 case Array::Contiguous:
744 case Array::Undecided:
745 case Array::ArrayStorage: {
746 IndexingType shape = arrayMode.shapeMask();
747 switch (arrayMode.arrayClass()) {
748 case Array::OriginalArray:
749 case Array::OriginalCopyOnWriteArray:
750 RELEASE_ASSERT_NOT_REACHED();
751 return result;
752
753 case Array::Array:
754 m_jit.and32(TrustedImm32(indexingModeMask), tempGPR);
755 result.append(m_jit.branch32(
756 MacroAssembler::NotEqual, tempGPR, TrustedImm32(IsArray | shape)));
757 return result;
758
759 case Array::NonArray:
760 case Array::OriginalNonArray:
761 m_jit.and32(TrustedImm32(indexingModeMask), tempGPR);
762 result.append(m_jit.branch32(
763 MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
764 return result;
765
766 case Array::PossiblyArray:
767 m_jit.and32(TrustedImm32(indexingModeMask & ~IsArray), tempGPR);
768 result.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(shape)));
769 return result;
770 }
771
772 RELEASE_ASSERT_NOT_REACHED();
773 return result;
774 }
775
776 case Array::SlowPutArrayStorage: {
777 ASSERT(!arrayMode.isJSArrayWithOriginalStructure());
778
779 switch (arrayMode.arrayClass()) {
780 case Array::OriginalArray:
781 case Array::OriginalCopyOnWriteArray:
782 RELEASE_ASSERT_NOT_REACHED();
783 return result;
784
785 case Array::Array:
786 result.append(
787 m_jit.branchTest32(
788 MacroAssembler::Zero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
789 break;
790
791 case Array::NonArray:
792 case Array::OriginalNonArray:
793 result.append(
794 m_jit.branchTest32(
795 MacroAssembler::NonZero, tempGPR, MacroAssembler::TrustedImm32(IsArray)));
796 break;
797
798 case Array::PossiblyArray:
799 break;
800 }
801
802 m_jit.and32(TrustedImm32(IndexingShapeMask), tempGPR);
803 m_jit.sub32(TrustedImm32(ArrayStorageShape), tempGPR);
804 result.append(
805 m_jit.branch32(
806 MacroAssembler::Above, tempGPR,
807 TrustedImm32(SlowPutArrayStorageShape - ArrayStorageShape)));
808 return result;
809 }
810 default:
811 CRASH();
812 break;
813 }
814
815 return result;
816}
817
818void SpeculativeJIT::checkArray(Node* node)
819{
820 ASSERT(node->arrayMode().isSpecific());
821 ASSERT(!node->arrayMode().doesConversion());
822
823 SpeculateCellOperand base(this, node->child1());
824 GPRReg baseReg = base.gpr();
825
826 if (node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1()))) {
827 noResult(m_currentNode);
828 return;
829 }
830
831 switch (node->arrayMode().type()) {
832 case Array::AnyTypedArray:
833 case Array::String:
834 RELEASE_ASSERT_NOT_REACHED(); // Should have been a Phantom(String:)
835 return;
836 case Array::Int32:
837 case Array::Double:
838 case Array::Contiguous:
839 case Array::Undecided:
840 case Array::ArrayStorage:
841 case Array::SlowPutArrayStorage: {
842 GPRTemporary temp(this);
843 GPRReg tempGPR = temp.gpr();
844 m_jit.load8(MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
845 speculationCheck(
846 BadIndexingType, JSValueSource::unboxedCell(baseReg), 0,
847 jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
848
849 noResult(m_currentNode);
850 return;
851 }
852 case Array::DirectArguments:
853 speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, DirectArgumentsType);
854 noResult(m_currentNode);
855 return;
856 case Array::ScopedArguments:
857 speculateCellTypeWithoutTypeFiltering(node->child1(), baseReg, ScopedArgumentsType);
858 noResult(m_currentNode);
859 return;
860 default:
861 speculateCellTypeWithoutTypeFiltering(
862 node->child1(), baseReg,
863 typeForTypedArrayType(node->arrayMode().typedArrayType()));
864 noResult(m_currentNode);
865 return;
866 }
867}
868
869void SpeculativeJIT::arrayify(Node* node, GPRReg baseReg, GPRReg propertyReg)
870{
871 ASSERT(node->arrayMode().doesConversion());
872
873 GPRTemporary temp(this);
874 GPRTemporary structure;
875 GPRReg tempGPR = temp.gpr();
876 GPRReg structureGPR = InvalidGPRReg;
877
878 if (node->op() != ArrayifyToStructure) {
879 GPRTemporary realStructure(this);
880 structure.adopt(realStructure);
881 structureGPR = structure.gpr();
882 }
883
884 // We can skip all that comes next if we already have array storage.
885 MacroAssembler::JumpList slowPath;
886
887 if (node->op() == ArrayifyToStructure) {
888 ASSERT(!isCopyOnWrite(node->structure()->indexingMode()));
889 ASSERT((node->structure()->indexingType() & IndexingShapeMask) == node->arrayMode().shapeMask());
890 slowPath.append(m_jit.branchWeakStructure(
891 JITCompiler::NotEqual,
892 JITCompiler::Address(baseReg, JSCell::structureIDOffset()),
893 node->structure()));
894 } else {
895 m_jit.load8(
896 MacroAssembler::Address(baseReg, JSCell::indexingTypeAndMiscOffset()), tempGPR);
897
898 slowPath.append(jumpSlowForUnwantedArrayMode(tempGPR, node->arrayMode()));
899 }
900
901 addSlowPathGenerator(std::make_unique<ArrayifySlowPathGenerator>(
902 slowPath, this, node, baseReg, propertyReg, tempGPR, structureGPR));
903
904 noResult(m_currentNode);
905}
906
907void SpeculativeJIT::arrayify(Node* node)
908{
909 ASSERT(node->arrayMode().isSpecific());
910
911 SpeculateCellOperand base(this, node->child1());
912
913 if (!node->child2()) {
914 arrayify(node, base.gpr(), InvalidGPRReg);
915 return;
916 }
917
918 SpeculateInt32Operand property(this, node->child2());
919
920 arrayify(node, base.gpr(), property.gpr());
921}
922
923GPRReg SpeculativeJIT::fillStorage(Edge edge)
924{
925 VirtualRegister virtualRegister = edge->virtualRegister();
926 GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
927
928 switch (info.registerFormat()) {
929 case DataFormatNone: {
930 if (info.spillFormat() == DataFormatStorage) {
931 GPRReg gpr = allocate();
932 m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
933 m_jit.loadPtr(JITCompiler::addressFor(virtualRegister), gpr);
934 info.fillStorage(*m_stream, gpr);
935 return gpr;
936 }
937
938 // Must be a cell; fill it as a cell and then return the pointer.
939 return fillSpeculateCell(edge);
940 }
941
942 case DataFormatStorage: {
943 GPRReg gpr = info.gpr();
944 m_gprs.lock(gpr);
945 return gpr;
946 }
947
948 default:
949 return fillSpeculateCell(edge);
950 }
951}
952
953void SpeculativeJIT::useChildren(Node* node)
954{
955 if (node->flags() & NodeHasVarArgs) {
956 for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) {
957 if (!!m_jit.graph().m_varArgChildren[childIdx])
958 use(m_jit.graph().m_varArgChildren[childIdx]);
959 }
960 } else {
961 Edge child1 = node->child1();
962 if (!child1) {
963 ASSERT(!node->child2() && !node->child3());
964 return;
965 }
966 use(child1);
967
968 Edge child2 = node->child2();
969 if (!child2) {
970 ASSERT(!node->child3());
971 return;
972 }
973 use(child2);
974
975 Edge child3 = node->child3();
976 if (!child3)
977 return;
978 use(child3);
979 }
980}
981
982void SpeculativeJIT::compileGetById(Node* node, AccessType accessType)
983{
984 ASSERT(accessType == AccessType::Get || accessType == AccessType::GetDirect || accessType == AccessType::TryGet);
985
986 switch (node->child1().useKind()) {
987 case CellUse: {
988 SpeculateCellOperand base(this, node->child1());
989 JSValueRegsTemporary result(this, Reuse, base);
990
991 JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
992 JSValueRegs resultRegs = result.regs();
993
994 base.use();
995
996 cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), NeedToSpill, accessType);
997
998 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
999 break;
1000 }
1001
1002 case UntypedUse: {
1003 JSValueOperand base(this, node->child1());
1004 JSValueRegsTemporary result(this, Reuse, base);
1005
1006 JSValueRegs baseRegs = base.jsValueRegs();
1007 JSValueRegs resultRegs = result.regs();
1008
1009 base.use();
1010
1011 JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1012
1013 cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, NeedToSpill, accessType);
1014
1015 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1016 break;
1017 }
1018
1019 default:
1020 DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1021 break;
1022 }
1023}
1024
1025void SpeculativeJIT::compileGetByIdFlush(Node* node, AccessType accessType)
1026{
1027 switch (node->child1().useKind()) {
1028 case CellUse: {
1029 SpeculateCellOperand base(this, node->child1());
1030 JSValueRegs baseRegs = JSValueRegs::payloadOnly(base.gpr());
1031
1032 JSValueRegsFlushedCallResult result(this);
1033 JSValueRegs resultRegs = result.regs();
1034
1035 base.use();
1036
1037 flushRegisters();
1038
1039 cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), JITCompiler::Jump(), DontSpill, accessType);
1040
1041 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1042 break;
1043 }
1044
1045 case UntypedUse: {
1046 JSValueOperand base(this, node->child1());
1047 JSValueRegs baseRegs = base.jsValueRegs();
1048
1049 JSValueRegsFlushedCallResult result(this);
1050 JSValueRegs resultRegs = result.regs();
1051
1052 base.use();
1053
1054 flushRegisters();
1055
1056 JITCompiler::Jump notCell = m_jit.branchIfNotCell(baseRegs);
1057
1058 cachedGetById(node->origin.semantic, baseRegs, resultRegs, node->identifierNumber(), notCell, DontSpill, accessType);
1059
1060 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
1061 break;
1062 }
1063
1064 default:
1065 DFG_CRASH(m_jit.graph(), node, "Bad use kind");
1066 break;
1067 }
1068}
1069
1070void SpeculativeJIT::compileInById(Node* node)
1071{
1072 SpeculateCellOperand base(this, node->child1());
1073 JSValueRegsTemporary result(this, Reuse, base, PayloadWord);
1074
1075 GPRReg baseGPR = base.gpr();
1076 JSValueRegs resultRegs = result.regs();
1077
1078 base.use();
1079
1080 CodeOrigin codeOrigin = node->origin.semantic;
1081 CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size());
1082 RegisterSet usedRegisters = this->usedRegisters();
1083 JITInByIdGenerator gen(
1084 m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, identifierUID(node->identifierNumber()),
1085 JSValueRegs::payloadOnly(baseGPR), resultRegs);
1086 gen.generateFastPath(m_jit);
1087
1088 auto slowPath = slowPathCall(
1089 gen.slowPathJump(), this, operationInByIdOptimize,
1090 NeedToSpill, ExceptionCheckRequirement::CheckNeeded,
1091 resultRegs, gen.stubInfo(), CCallHelpers::CellValue(baseGPR), identifierUID(node->identifierNumber()));
1092
1093 m_jit.addInById(gen, slowPath.get());
1094 addSlowPathGenerator(WTFMove(slowPath));
1095
1096 blessedBooleanResult(resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly);
1097}
1098
1099void SpeculativeJIT::compileInByVal(Node* node)
1100{
1101 SpeculateCellOperand base(this, node->child1());
1102 JSValueOperand key(this, node->child2());
1103
1104 GPRReg baseGPR = base.gpr();
1105 JSValueRegs regs = key.jsValueRegs();
1106
1107 base.use();
1108 key.use();
1109
1110 flushRegisters();
1111 JSValueRegsFlushedCallResult result(this);
1112 JSValueRegs resultRegs = result.regs();
1113 callOperation(operationInByVal, resultRegs, baseGPR, regs);
1114 m_jit.exceptionCheck();
1115 blessedBooleanResult(resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly);
1116}
1117
1118void SpeculativeJIT::compileDeleteById(Node* node)
1119{
1120 JSValueOperand value(this, node->child1());
1121 GPRFlushedCallResult result(this);
1122
1123 JSValueRegs valueRegs = value.jsValueRegs();
1124 GPRReg resultGPR = result.gpr();
1125
1126 value.use();
1127
1128 flushRegisters();
1129 callOperation(operationDeleteById, resultGPR, valueRegs, identifierUID(node->identifierNumber()));
1130 m_jit.exceptionCheck();
1131
1132 unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1133}
1134
1135void SpeculativeJIT::compileDeleteByVal(Node* node)
1136{
1137 JSValueOperand base(this, node->child1());
1138 JSValueOperand key(this, node->child2());
1139 GPRFlushedCallResult result(this);
1140
1141 JSValueRegs baseRegs = base.jsValueRegs();
1142 JSValueRegs keyRegs = key.jsValueRegs();
1143 GPRReg resultGPR = result.gpr();
1144
1145 base.use();
1146 key.use();
1147
1148 flushRegisters();
1149 callOperation(operationDeleteByVal, resultGPR, baseRegs, keyRegs);
1150 m_jit.exceptionCheck();
1151
1152 unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
1153}
1154
1155void SpeculativeJIT::compilePushWithScope(Node* node)
1156{
1157 SpeculateCellOperand currentScope(this, node->child1());
1158 GPRReg currentScopeGPR = currentScope.gpr();
1159
1160 GPRFlushedCallResult result(this);
1161 GPRReg resultGPR = result.gpr();
1162
1163 auto objectEdge = node->child2();
1164 if (objectEdge.useKind() == ObjectUse) {
1165 SpeculateCellOperand object(this, objectEdge);
1166 GPRReg objectGPR = object.gpr();
1167 speculateObject(objectEdge, objectGPR);
1168
1169 flushRegisters();
1170 callOperation(operationPushWithScopeObject, resultGPR, currentScopeGPR, objectGPR);
1171 // No exception check here as we did not have to call toObject().
1172 } else {
1173 ASSERT(objectEdge.useKind() == UntypedUse);
1174 JSValueOperand object(this, objectEdge);
1175 JSValueRegs objectRegs = object.jsValueRegs();
1176
1177 flushRegisters();
1178 callOperation(operationPushWithScope, resultGPR, currentScopeGPR, objectRegs);
1179 m_jit.exceptionCheck();
1180 }
1181
1182 cellResult(resultGPR, node);
1183}
1184
1185bool SpeculativeJIT::nonSpeculativeStrictEq(Node* node, bool invert)
1186{
1187 unsigned branchIndexInBlock = detectPeepHoleBranch();
1188 if (branchIndexInBlock != UINT_MAX) {
1189 Node* branchNode = m_block->at(branchIndexInBlock);
1190
1191 ASSERT(node->adjustedRefCount() == 1);
1192
1193 nonSpeculativePeepholeStrictEq(node, branchNode, invert);
1194
1195 m_indexInBlock = branchIndexInBlock;
1196 m_currentNode = branchNode;
1197
1198 return true;
1199 }
1200
1201 nonSpeculativeNonPeepholeStrictEq(node, invert);
1202
1203 return false;
1204}
1205
1206static const char* dataFormatString(DataFormat format)
1207{
1208 // These values correspond to the DataFormat enum.
1209 const char* strings[] = {
1210 "[ ]",
1211 "[ i]",
1212 "[ d]",
1213 "[ c]",
1214 "Err!",
1215 "Err!",
1216 "Err!",
1217 "Err!",
1218 "[J ]",
1219 "[Ji]",
1220 "[Jd]",
1221 "[Jc]",
1222 "Err!",
1223 "Err!",
1224 "Err!",
1225 "Err!",
1226 };
1227 return strings[format];
1228}
1229
1230void SpeculativeJIT::dump(const char* label)
1231{
1232 if (label)
1233 dataLogF("<%s>\n", label);
1234
1235 dataLogF(" gprs:\n");
1236 m_gprs.dump();
1237 dataLogF(" fprs:\n");
1238 m_fprs.dump();
1239 dataLogF(" VirtualRegisters:\n");
1240 for (unsigned i = 0; i < m_generationInfo.size(); ++i) {
1241 GenerationInfo& info = m_generationInfo[i];
1242 if (info.alive())
1243 dataLogF(" % 3d:%s%s", i, dataFormatString(info.registerFormat()), dataFormatString(info.spillFormat()));
1244 else
1245 dataLogF(" % 3d:[__][__]", i);
1246 if (info.registerFormat() == DataFormatDouble)
1247 dataLogF(":fpr%d\n", info.fpr());
1248 else if (info.registerFormat() != DataFormatNone
1249#if USE(JSVALUE32_64)
1250 && !(info.registerFormat() & DataFormatJS)
1251#endif
1252 ) {
1253 ASSERT(info.gpr() != InvalidGPRReg);
1254 dataLogF(":%s\n", GPRInfo::debugName(info.gpr()));
1255 } else
1256 dataLogF("\n");
1257 }
1258 if (label)
1259 dataLogF("</%s>\n", label);
1260}
1261
1262GPRTemporary::GPRTemporary()
1263 : m_jit(0)
1264 , m_gpr(InvalidGPRReg)
1265{
1266}
1267
1268GPRTemporary::GPRTemporary(SpeculativeJIT* jit)
1269 : m_jit(jit)
1270 , m_gpr(InvalidGPRReg)
1271{
1272 m_gpr = m_jit->allocate();
1273}
1274
1275GPRTemporary::GPRTemporary(SpeculativeJIT* jit, GPRReg specific)
1276 : m_jit(jit)
1277 , m_gpr(InvalidGPRReg)
1278{
1279 m_gpr = m_jit->allocate(specific);
1280}
1281
1282#if USE(JSVALUE32_64)
1283GPRTemporary::GPRTemporary(
1284 SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord which)
1285 : m_jit(jit)
1286 , m_gpr(InvalidGPRReg)
1287{
1288 if (!op1.isDouble() && m_jit->canReuse(op1.node()))
1289 m_gpr = m_jit->reuse(op1.gpr(which));
1290 else
1291 m_gpr = m_jit->allocate();
1292}
1293#else // USE(JSVALUE32_64)
1294GPRTemporary::GPRTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& op1, WhichValueWord)
1295 : GPRTemporary(jit, Reuse, op1)
1296{
1297}
1298#endif
1299
1300JSValueRegsTemporary::JSValueRegsTemporary() { }
1301
1302JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit)
1303#if USE(JSVALUE64)
1304 : m_gpr(jit)
1305#else
1306 : m_payloadGPR(jit)
1307 , m_tagGPR(jit)
1308#endif
1309{
1310}
1311
1312#if USE(JSVALUE64)
1313template<typename T>
1314JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord)
1315 : m_gpr(jit, Reuse, operand)
1316{
1317}
1318#else
1319template<typename T>
1320JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, T& operand, WhichValueWord resultWord)
1321{
1322 if (resultWord == PayloadWord) {
1323 m_payloadGPR = GPRTemporary(jit, Reuse, operand);
1324 m_tagGPR = GPRTemporary(jit);
1325 } else {
1326 m_payloadGPR = GPRTemporary(jit);
1327 m_tagGPR = GPRTemporary(jit, Reuse, operand);
1328 }
1329}
1330#endif
1331
1332#if USE(JSVALUE64)
1333JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1334{
1335 m_gpr = GPRTemporary(jit, Reuse, operand);
1336}
1337#else
1338JSValueRegsTemporary::JSValueRegsTemporary(SpeculativeJIT* jit, ReuseTag, JSValueOperand& operand)
1339{
1340 if (jit->canReuse(operand.node())) {
1341 m_payloadGPR = GPRTemporary(jit, Reuse, operand, PayloadWord);
1342 m_tagGPR = GPRTemporary(jit, Reuse, operand, TagWord);
1343 } else {
1344 m_payloadGPR = GPRTemporary(jit);
1345 m_tagGPR = GPRTemporary(jit);
1346 }
1347}
1348#endif
1349
1350JSValueRegsTemporary::~JSValueRegsTemporary() { }
1351
1352JSValueRegs JSValueRegsTemporary::regs()
1353{
1354#if USE(JSVALUE64)
1355 return JSValueRegs(m_gpr.gpr());
1356#else
1357 return JSValueRegs(m_tagGPR.gpr(), m_payloadGPR.gpr());
1358#endif
1359}
1360
1361void GPRTemporary::adopt(GPRTemporary& other)
1362{
1363 ASSERT(!m_jit);
1364 ASSERT(m_gpr == InvalidGPRReg);
1365 ASSERT(other.m_jit);
1366 ASSERT(other.m_gpr != InvalidGPRReg);
1367 m_jit = other.m_jit;
1368 m_gpr = other.m_gpr;
1369 other.m_jit = 0;
1370 other.m_gpr = InvalidGPRReg;
1371}
1372
1373FPRTemporary::FPRTemporary(FPRTemporary&& other)
1374{
1375 ASSERT(other.m_jit);
1376 ASSERT(other.m_fpr != InvalidFPRReg);
1377 m_jit = other.m_jit;
1378 m_fpr = other.m_fpr;
1379
1380 other.m_jit = nullptr;
1381}
1382
1383FPRTemporary::FPRTemporary(SpeculativeJIT* jit)
1384 : m_jit(jit)
1385 , m_fpr(InvalidFPRReg)
1386{
1387 m_fpr = m_jit->fprAllocate();
1388}
1389
1390FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1)
1391 : m_jit(jit)
1392 , m_fpr(InvalidFPRReg)
1393{
1394 if (m_jit->canReuse(op1.node()))
1395 m_fpr = m_jit->reuse(op1.fpr());
1396 else
1397 m_fpr = m_jit->fprAllocate();
1398}
1399
1400FPRTemporary::FPRTemporary(SpeculativeJIT* jit, SpeculateDoubleOperand& op1, SpeculateDoubleOperand& op2)
1401 : m_jit(jit)
1402 , m_fpr(InvalidFPRReg)
1403{
1404 if (m_jit->canReuse(op1.node()))
1405 m_fpr = m_jit->reuse(op1.fpr());
1406 else if (m_jit->canReuse(op2.node()))
1407 m_fpr = m_jit->reuse(op2.fpr());
1408 else if (m_jit->canReuse(op1.node(), op2.node()) && op1.fpr() == op2.fpr())
1409 m_fpr = m_jit->reuse(op1.fpr());
1410 else
1411 m_fpr = m_jit->fprAllocate();
1412}
1413
1414#if USE(JSVALUE32_64)
1415FPRTemporary::FPRTemporary(SpeculativeJIT* jit, JSValueOperand& op1)
1416 : m_jit(jit)
1417 , m_fpr(InvalidFPRReg)
1418{
1419 if (op1.isDouble() && m_jit->canReuse(op1.node()))
1420 m_fpr = m_jit->reuse(op1.fpr());
1421 else
1422 m_fpr = m_jit->fprAllocate();
1423}
1424#endif
1425
1426void SpeculativeJIT::compilePeepHoleDoubleBranch(Node* node, Node* branchNode, JITCompiler::DoubleCondition condition)
1427{
1428 BasicBlock* taken = branchNode->branchData()->taken.block;
1429 BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1430
1431 if (taken == nextBlock()) {
1432 condition = MacroAssembler::invert(condition);
1433 std::swap(taken, notTaken);
1434 }
1435
1436 SpeculateDoubleOperand op1(this, node->child1());
1437 SpeculateDoubleOperand op2(this, node->child2());
1438
1439 branchDouble(condition, op1.fpr(), op2.fpr(), taken);
1440 jump(notTaken);
1441}
1442
1443void SpeculativeJIT::compilePeepHoleObjectEquality(Node* node, Node* branchNode)
1444{
1445 BasicBlock* taken = branchNode->branchData()->taken.block;
1446 BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1447
1448 MacroAssembler::RelationalCondition condition = MacroAssembler::Equal;
1449
1450 if (taken == nextBlock()) {
1451 condition = MacroAssembler::NotEqual;
1452 BasicBlock* tmp = taken;
1453 taken = notTaken;
1454 notTaken = tmp;
1455 }
1456
1457 SpeculateCellOperand op1(this, node->child1());
1458 SpeculateCellOperand op2(this, node->child2());
1459
1460 GPRReg op1GPR = op1.gpr();
1461 GPRReg op2GPR = op2.gpr();
1462
1463 if (masqueradesAsUndefinedWatchpointIsStillValid()) {
1464 if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1465 speculationCheck(
1466 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchIfNotObject(op1GPR));
1467 }
1468 if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1469 speculationCheck(
1470 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchIfNotObject(op2GPR));
1471 }
1472 } else {
1473 if (m_state.forNode(node->child1()).m_type & ~SpecObject) {
1474 speculationCheck(
1475 BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1476 m_jit.branchIfNotObject(op1GPR));
1477 }
1478 speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1479 m_jit.branchTest8(
1480 MacroAssembler::NonZero,
1481 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()),
1482 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1483
1484 if (m_state.forNode(node->child2()).m_type & ~SpecObject) {
1485 speculationCheck(
1486 BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1487 m_jit.branchIfNotObject(op2GPR));
1488 }
1489 speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1490 m_jit.branchTest8(
1491 MacroAssembler::NonZero,
1492 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()),
1493 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1494 }
1495
1496 branchPtr(condition, op1GPR, op2GPR, taken);
1497 jump(notTaken);
1498}
1499
1500void SpeculativeJIT::compilePeepHoleBooleanBranch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1501{
1502 BasicBlock* taken = branchNode->branchData()->taken.block;
1503 BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1504
1505 // The branch instruction will branch to the taken block.
1506 // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1507 if (taken == nextBlock()) {
1508 condition = JITCompiler::invert(condition);
1509 BasicBlock* tmp = taken;
1510 taken = notTaken;
1511 notTaken = tmp;
1512 }
1513
1514 if (node->child1()->isInt32Constant()) {
1515 int32_t imm = node->child1()->asInt32();
1516 SpeculateBooleanOperand op2(this, node->child2());
1517 branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1518 } else if (node->child2()->isInt32Constant()) {
1519 SpeculateBooleanOperand op1(this, node->child1());
1520 int32_t imm = node->child2()->asInt32();
1521 branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1522 } else {
1523 SpeculateBooleanOperand op1(this, node->child1());
1524 SpeculateBooleanOperand op2(this, node->child2());
1525 branch32(condition, op1.gpr(), op2.gpr(), taken);
1526 }
1527
1528 jump(notTaken);
1529}
1530
1531void SpeculativeJIT::compileStringSlice(Node* node)
1532{
1533 SpeculateCellOperand string(this, node->child1());
1534
1535 GPRReg stringGPR = string.gpr();
1536
1537 speculateString(node->child1(), stringGPR);
1538
1539 SpeculateInt32Operand start(this, node->child2());
1540 GPRReg startGPR = start.gpr();
1541
1542 Optional<SpeculateInt32Operand> end;
1543 Optional<GPRReg> endGPR;
1544 if (node->child3()) {
1545 end.emplace(this, node->child3());
1546 endGPR.emplace(end->gpr());
1547 }
1548
1549 GPRTemporary temp(this);
1550 GPRTemporary temp2(this);
1551 GPRTemporary startIndex(this);
1552
1553 GPRReg tempGPR = temp.gpr();
1554 GPRReg temp2GPR = temp2.gpr();
1555 GPRReg startIndexGPR = startIndex.gpr();
1556
1557 m_jit.loadPtr(CCallHelpers::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1558 auto isRope = m_jit.branchIfRopeStringImpl(tempGPR);
1559 {
1560 m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), temp2GPR);
1561
1562 emitPopulateSliceIndex(node->child2(), startGPR, temp2GPR, startIndexGPR);
1563
1564 if (node->child3())
1565 emitPopulateSliceIndex(node->child3(), endGPR.value(), temp2GPR, tempGPR);
1566 else
1567 m_jit.move(temp2GPR, tempGPR);
1568 }
1569
1570 CCallHelpers::JumpList doneCases;
1571 CCallHelpers::JumpList slowCases;
1572
1573 auto nonEmptyCase = m_jit.branch32(MacroAssembler::Below, startIndexGPR, tempGPR);
1574 m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&vm())), tempGPR);
1575 doneCases.append(m_jit.jump());
1576
1577 nonEmptyCase.link(&m_jit);
1578 m_jit.sub32(startIndexGPR, tempGPR); // the size of the sliced string.
1579 slowCases.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, TrustedImm32(1)));
1580
1581 // Refill StringImpl* here.
1582 m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), temp2GPR);
1583 m_jit.loadPtr(MacroAssembler::Address(temp2GPR, StringImpl::dataOffset()), tempGPR);
1584
1585 // Load the character into scratchReg
1586 m_jit.zeroExtend32ToPtr(startIndexGPR, startIndexGPR);
1587 auto is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(temp2GPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
1588
1589 m_jit.load8(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesOne, 0), tempGPR);
1590 auto cont8Bit = m_jit.jump();
1591
1592 is16Bit.link(&m_jit);
1593 m_jit.load16(MacroAssembler::BaseIndex(tempGPR, startIndexGPR, MacroAssembler::TimesTwo, 0), tempGPR);
1594
1595 auto bigCharacter = m_jit.branch32(MacroAssembler::Above, tempGPR, TrustedImm32(maxSingleCharacterString));
1596
1597 // 8 bit string values don't need the isASCII check.
1598 cont8Bit.link(&m_jit);
1599
1600 m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), tempGPR);
1601 m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), tempGPR);
1602 m_jit.loadPtr(tempGPR, tempGPR);
1603
1604 addSlowPathGenerator(slowPathCall(bigCharacter, this, operationSingleCharacterString, tempGPR, tempGPR));
1605
1606 addSlowPathGenerator(slowPathCall(slowCases, this, operationStringSubstr, tempGPR, stringGPR, startIndexGPR, tempGPR));
1607
1608 if (endGPR)
1609 addSlowPathGenerator(slowPathCall(isRope, this, operationStringSlice, tempGPR, stringGPR, startGPR, *endGPR));
1610 else
1611 addSlowPathGenerator(slowPathCall(isRope, this, operationStringSlice, tempGPR, stringGPR, startGPR, TrustedImm32(std::numeric_limits<int32_t>::max())));
1612
1613 doneCases.link(&m_jit);
1614 cellResult(tempGPR, node);
1615}
1616
1617void SpeculativeJIT::compileToLowerCase(Node* node)
1618{
1619 ASSERT(node->op() == ToLowerCase);
1620 SpeculateCellOperand string(this, node->child1());
1621 GPRTemporary temp(this);
1622 GPRTemporary index(this);
1623 GPRTemporary charReg(this);
1624 GPRTemporary length(this);
1625
1626 GPRReg stringGPR = string.gpr();
1627 GPRReg tempGPR = temp.gpr();
1628 GPRReg indexGPR = index.gpr();
1629 GPRReg charGPR = charReg.gpr();
1630 GPRReg lengthGPR = length.gpr();
1631
1632 speculateString(node->child1(), stringGPR);
1633
1634 CCallHelpers::JumpList slowPath;
1635
1636 m_jit.move(TrustedImmPtr(nullptr), indexGPR);
1637
1638 m_jit.loadPtr(MacroAssembler::Address(stringGPR, JSString::offsetOfValue()), tempGPR);
1639 slowPath.append(m_jit.branchIfRopeStringImpl(tempGPR));
1640 slowPath.append(m_jit.branchTest32(
1641 MacroAssembler::Zero, MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
1642 MacroAssembler::TrustedImm32(StringImpl::flagIs8Bit())));
1643 m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
1644 m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), tempGPR);
1645
1646 auto loopStart = m_jit.label();
1647 auto loopDone = m_jit.branch32(CCallHelpers::AboveOrEqual, indexGPR, lengthGPR);
1648 m_jit.load8(MacroAssembler::BaseIndex(tempGPR, indexGPR, MacroAssembler::TimesOne), charGPR);
1649 slowPath.append(m_jit.branchTest32(CCallHelpers::NonZero, charGPR, TrustedImm32(~0x7F)));
1650 m_jit.sub32(TrustedImm32('A'), charGPR);
1651 slowPath.append(m_jit.branch32(CCallHelpers::BelowOrEqual, charGPR, TrustedImm32('Z' - 'A')));
1652
1653 m_jit.add32(TrustedImm32(1), indexGPR);
1654 m_jit.jump().linkTo(loopStart, &m_jit);
1655
1656 slowPath.link(&m_jit);
1657 silentSpillAllRegisters(lengthGPR);
1658 callOperation(operationToLowerCase, lengthGPR, stringGPR, indexGPR);
1659 silentFillAllRegisters();
1660 m_jit.exceptionCheck();
1661 auto done = m_jit.jump();
1662
1663 loopDone.link(&m_jit);
1664 m_jit.move(stringGPR, lengthGPR);
1665
1666 done.link(&m_jit);
1667 cellResult(lengthGPR, node);
1668}
1669
1670void SpeculativeJIT::compilePeepHoleInt32Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition)
1671{
1672 BasicBlock* taken = branchNode->branchData()->taken.block;
1673 BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
1674
1675 // The branch instruction will branch to the taken block.
1676 // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
1677 if (taken == nextBlock()) {
1678 condition = JITCompiler::invert(condition);
1679 BasicBlock* tmp = taken;
1680 taken = notTaken;
1681 notTaken = tmp;
1682 }
1683
1684 if (node->child1()->isInt32Constant()) {
1685 int32_t imm = node->child1()->asInt32();
1686 SpeculateInt32Operand op2(this, node->child2());
1687 branch32(condition, JITCompiler::Imm32(imm), op2.gpr(), taken);
1688 } else if (node->child2()->isInt32Constant()) {
1689 SpeculateInt32Operand op1(this, node->child1());
1690 int32_t imm = node->child2()->asInt32();
1691 branch32(condition, op1.gpr(), JITCompiler::Imm32(imm), taken);
1692 } else {
1693 SpeculateInt32Operand op1(this, node->child1());
1694 SpeculateInt32Operand op2(this, node->child2());
1695 branch32(condition, op1.gpr(), op2.gpr(), taken);
1696 }
1697
1698 jump(notTaken);
1699}
1700
1701// Returns true if the compare is fused with a subsequent branch.
1702bool SpeculativeJIT::compilePeepHoleBranch(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
1703{
1704 // Fused compare & branch.
1705 unsigned branchIndexInBlock = detectPeepHoleBranch();
1706 if (branchIndexInBlock != UINT_MAX) {
1707 Node* branchNode = m_block->at(branchIndexInBlock);
1708
1709 // detectPeepHoleBranch currently only permits the branch to be the very next node,
1710 // so can be no intervening nodes to also reference the compare.
1711 ASSERT(node->adjustedRefCount() == 1);
1712
1713 if (node->isBinaryUseKind(Int32Use))
1714 compilePeepHoleInt32Branch(node, branchNode, condition);
1715#if USE(JSVALUE64)
1716 else if (node->isBinaryUseKind(Int52RepUse))
1717 compilePeepHoleInt52Branch(node, branchNode, condition);
1718#endif // USE(JSVALUE64)
1719 else if (node->isBinaryUseKind(StringUse) || node->isBinaryUseKind(StringIdentUse)) {
1720 // Use non-peephole comparison, for now.
1721 return false;
1722 } else if (node->isBinaryUseKind(DoubleRepUse))
1723 compilePeepHoleDoubleBranch(node, branchNode, doubleCondition);
1724 else if (node->op() == CompareEq) {
1725 if (node->isBinaryUseKind(BooleanUse))
1726 compilePeepHoleBooleanBranch(node, branchNode, condition);
1727 else if (node->isBinaryUseKind(SymbolUse))
1728 compilePeepHoleSymbolEquality(node, branchNode);
1729 else if (node->isBinaryUseKind(ObjectUse))
1730 compilePeepHoleObjectEquality(node, branchNode);
1731 else if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse))
1732 compilePeepHoleObjectToObjectOrOtherEquality(node->child1(), node->child2(), branchNode);
1733 else if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse))
1734 compilePeepHoleObjectToObjectOrOtherEquality(node->child2(), node->child1(), branchNode);
1735 else if (!needsTypeCheck(node->child1(), SpecOther))
1736 nonSpeculativePeepholeBranchNullOrUndefined(node->child2(), branchNode);
1737 else if (!needsTypeCheck(node->child2(), SpecOther))
1738 nonSpeculativePeepholeBranchNullOrUndefined(node->child1(), branchNode);
1739 else {
1740 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1741 return true;
1742 }
1743 } else {
1744 nonSpeculativePeepholeBranch(node, branchNode, condition, operation);
1745 return true;
1746 }
1747
1748 use(node->child1());
1749 use(node->child2());
1750 m_indexInBlock = branchIndexInBlock;
1751 m_currentNode = branchNode;
1752 return true;
1753 }
1754 return false;
1755}
1756
1757void SpeculativeJIT::noticeOSRBirth(Node* node)
1758{
1759 if (!node->hasVirtualRegister())
1760 return;
1761
1762 VirtualRegister virtualRegister = node->virtualRegister();
1763 GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
1764
1765 info.noticeOSRBirth(*m_stream, node, virtualRegister);
1766}
1767
1768void SpeculativeJIT::compileMovHint(Node* node)
1769{
1770 ASSERT(node->containsMovHint() && node->op() != ZombieHint);
1771
1772 Node* child = node->child1().node();
1773 noticeOSRBirth(child);
1774
1775 m_stream->appendAndLog(VariableEvent::movHint(MinifiedID(child), node->unlinkedLocal()));
1776}
1777
1778void SpeculativeJIT::bail(AbortReason reason)
1779{
1780 if (verboseCompilationEnabled())
1781 dataLog("Bailing compilation.\n");
1782 m_compileOkay = true;
1783 m_jit.abortWithReason(reason, m_lastGeneratedNode);
1784 clearGenerationInfo();
1785}
1786
1787void SpeculativeJIT::compileCurrentBlock()
1788{
1789 ASSERT(m_compileOkay);
1790
1791 if (!m_block)
1792 return;
1793
1794 ASSERT(m_block->isReachable);
1795
1796 m_jit.blockHeads()[m_block->index] = m_jit.label();
1797
1798 if (!m_block->intersectionOfCFAHasVisited) {
1799 // Don't generate code for basic blocks that are unreachable according to CFA.
1800 // But to be sure that nobody has generated a jump to this block, drop in a
1801 // breakpoint here.
1802 m_jit.abortWithReason(DFGUnreachableBasicBlock);
1803 return;
1804 }
1805
1806 if (m_block->isCatchEntrypoint) {
1807 m_jit.addPtr(CCallHelpers::TrustedImm32(-(m_jit.graph().frameRegisterCount() * sizeof(Register))), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
1808 if (Options::zeroStackFrame())
1809 m_jit.clearStackFrame(GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister, GPRInfo::regT0, m_jit.graph().frameRegisterCount() * sizeof(Register));
1810 m_jit.emitSaveCalleeSaves();
1811 m_jit.emitMaterializeTagCheckRegisters();
1812 m_jit.emitPutToCallFrameHeader(m_jit.codeBlock(), CallFrameSlot::codeBlock);
1813 }
1814
1815 m_stream->appendAndLog(VariableEvent::reset());
1816
1817 m_jit.jitAssertHasValidCallFrame();
1818 m_jit.jitAssertTagsInPlace();
1819 m_jit.jitAssertArgumentCountSane();
1820
1821 m_state.reset();
1822 m_state.beginBasicBlock(m_block);
1823
1824 for (size_t i = m_block->variablesAtHead.size(); i--;) {
1825 int operand = m_block->variablesAtHead.operandForIndex(i);
1826 Node* node = m_block->variablesAtHead[i];
1827 if (!node)
1828 continue; // No need to record dead SetLocal's.
1829
1830 VariableAccessData* variable = node->variableAccessData();
1831 DataFormat format;
1832 if (!node->refCount())
1833 continue; // No need to record dead SetLocal's.
1834 format = dataFormatFor(variable->flushFormat());
1835 m_stream->appendAndLog(
1836 VariableEvent::setLocal(
1837 VirtualRegister(operand),
1838 variable->machineLocal(),
1839 format));
1840 }
1841
1842 m_origin = NodeOrigin();
1843
1844 for (m_indexInBlock = 0; m_indexInBlock < m_block->size(); ++m_indexInBlock) {
1845 m_currentNode = m_block->at(m_indexInBlock);
1846
1847 // We may have hit a contradiction that the CFA was aware of but that the JIT
1848 // didn't cause directly.
1849 if (!m_state.isValid()) {
1850 bail(DFGBailedAtTopOfBlock);
1851 return;
1852 }
1853
1854 m_interpreter.startExecuting();
1855 m_interpreter.executeKnownEdgeTypes(m_currentNode);
1856 m_jit.setForNode(m_currentNode);
1857 m_origin = m_currentNode->origin;
1858 m_lastGeneratedNode = m_currentNode->op();
1859
1860 ASSERT(m_currentNode->shouldGenerate());
1861
1862 if (verboseCompilationEnabled()) {
1863 dataLogF(
1864 "SpeculativeJIT generating Node @%d (bc#%u) at JIT offset 0x%x",
1865 (int)m_currentNode->index(),
1866 m_currentNode->origin.semantic.bytecodeIndex(), m_jit.debugOffset());
1867 dataLog("\n");
1868 }
1869
1870 if (Options::validateDFGExceptionHandling() && (mayExit(m_jit.graph(), m_currentNode) != DoesNotExit || m_currentNode->isTerminal()))
1871 m_jit.jitReleaseAssertNoException(*m_jit.vm());
1872
1873 m_jit.pcToCodeOriginMapBuilder().appendItem(m_jit.labelIgnoringWatchpoints(), m_origin.semantic);
1874
1875 compile(m_currentNode);
1876
1877 if (belongsInMinifiedGraph(m_currentNode->op()))
1878 m_minifiedGraph->append(MinifiedNode::fromNode(m_currentNode));
1879
1880#if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1881 m_jit.clearRegisterAllocationOffsets();
1882#endif
1883
1884 if (!m_compileOkay) {
1885 bail(DFGBailedAtEndOfNode);
1886 return;
1887 }
1888
1889 // Make sure that the abstract state is rematerialized for the next node.
1890 m_interpreter.executeEffects(m_indexInBlock);
1891 }
1892
1893 // Perform the most basic verification that children have been used correctly.
1894 if (!ASSERT_DISABLED) {
1895 for (auto& info : m_generationInfo)
1896 RELEASE_ASSERT(!info.alive());
1897 }
1898}
1899
1900// If we are making type predictions about our arguments then
1901// we need to check that they are correct on function entry.
1902void SpeculativeJIT::checkArgumentTypes()
1903{
1904 ASSERT(!m_currentNode);
1905 m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true);
1906
1907 auto& arguments = m_jit.graph().m_rootToArguments.find(m_jit.graph().block(0))->value;
1908 for (int i = 0; i < m_jit.codeBlock()->numParameters(); ++i) {
1909 Node* node = arguments[i];
1910 if (!node) {
1911 // The argument is dead. We don't do any checks for such arguments.
1912 continue;
1913 }
1914
1915 ASSERT(node->op() == SetArgument);
1916 ASSERT(node->shouldGenerate());
1917
1918 VariableAccessData* variableAccessData = node->variableAccessData();
1919 FlushFormat format = variableAccessData->flushFormat();
1920
1921 if (format == FlushedJSValue)
1922 continue;
1923
1924 VirtualRegister virtualRegister = variableAccessData->local();
1925
1926 JSValueSource valueSource = JSValueSource(JITCompiler::addressFor(virtualRegister));
1927
1928#if USE(JSVALUE64)
1929 switch (format) {
1930 case FlushedInt32: {
1931 speculationCheck(BadType, valueSource, node, m_jit.branch64(MacroAssembler::Below, JITCompiler::addressFor(virtualRegister), GPRInfo::tagTypeNumberRegister));
1932 break;
1933 }
1934 case FlushedBoolean: {
1935 GPRTemporary temp(this);
1936 m_jit.load64(JITCompiler::addressFor(virtualRegister), temp.gpr());
1937 m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), temp.gpr());
1938 speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, temp.gpr(), TrustedImm32(static_cast<int32_t>(~1))));
1939 break;
1940 }
1941 case FlushedCell: {
1942 speculationCheck(BadType, valueSource, node, m_jit.branchTest64(MacroAssembler::NonZero, JITCompiler::addressFor(virtualRegister), GPRInfo::tagMaskRegister));
1943 break;
1944 }
1945 default:
1946 RELEASE_ASSERT_NOT_REACHED();
1947 break;
1948 }
1949#else
1950 switch (format) {
1951 case FlushedInt32: {
1952 speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
1953 break;
1954 }
1955 case FlushedBoolean: {
1956 speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1957 break;
1958 }
1959 case FlushedCell: {
1960 speculationCheck(BadType, valueSource, node, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1961 break;
1962 }
1963 default:
1964 RELEASE_ASSERT_NOT_REACHED();
1965 break;
1966 }
1967#endif
1968 }
1969
1970 m_origin = NodeOrigin();
1971}
1972
1973bool SpeculativeJIT::compile()
1974{
1975 checkArgumentTypes();
1976
1977 ASSERT(!m_currentNode);
1978 for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1979 m_jit.setForBlockIndex(blockIndex);
1980 m_block = m_jit.graph().block(blockIndex);
1981 compileCurrentBlock();
1982 }
1983 linkBranches();
1984 return true;
1985}
1986
1987void SpeculativeJIT::createOSREntries()
1988{
1989 for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
1990 BasicBlock* block = m_jit.graph().block(blockIndex);
1991 if (!block)
1992 continue;
1993 if (block->isOSRTarget || block->isCatchEntrypoint) {
1994 // Currently we don't have OSR entry trampolines. We could add them
1995 // here if need be.
1996 m_osrEntryHeads.append(m_jit.blockHeads()[blockIndex]);
1997 }
1998 }
1999}
2000
2001void SpeculativeJIT::linkOSREntries(LinkBuffer& linkBuffer)
2002{
2003 unsigned osrEntryIndex = 0;
2004 for (BlockIndex blockIndex = 0; blockIndex < m_jit.graph().numBlocks(); ++blockIndex) {
2005 BasicBlock* block = m_jit.graph().block(blockIndex);
2006 if (!block)
2007 continue;
2008 if (!block->isOSRTarget && !block->isCatchEntrypoint)
2009 continue;
2010 if (block->isCatchEntrypoint) {
2011 auto& argumentsVector = m_jit.graph().m_rootToArguments.find(block)->value;
2012 Vector<FlushFormat> argumentFormats;
2013 argumentFormats.reserveInitialCapacity(argumentsVector.size());
2014 for (Node* setArgument : argumentsVector) {
2015 if (setArgument) {
2016 FlushFormat flushFormat = setArgument->variableAccessData()->flushFormat();
2017 ASSERT(flushFormat == FlushedInt32 || flushFormat == FlushedCell || flushFormat == FlushedBoolean || flushFormat == FlushedJSValue);
2018 argumentFormats.uncheckedAppend(flushFormat);
2019 } else
2020 argumentFormats.uncheckedAppend(DeadFlush);
2021 }
2022 m_jit.noticeCatchEntrypoint(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer, WTFMove(argumentFormats));
2023 } else {
2024 ASSERT(block->isOSRTarget);
2025 m_jit.noticeOSREntry(*block, m_osrEntryHeads[osrEntryIndex++], linkBuffer);
2026 }
2027 }
2028
2029 m_jit.jitCode()->finalizeOSREntrypoints();
2030 m_jit.jitCode()->common.finalizeCatchEntrypoints();
2031
2032 ASSERT(osrEntryIndex == m_osrEntryHeads.size());
2033
2034 if (verboseCompilationEnabled()) {
2035 DumpContext dumpContext;
2036 dataLog("OSR Entries:\n");
2037 for (OSREntryData& entryData : m_jit.jitCode()->osrEntry)
2038 dataLog(" ", inContext(entryData, &dumpContext), "\n");
2039 if (!dumpContext.isEmpty())
2040 dumpContext.dump(WTF::dataFile());
2041 }
2042}
2043
2044void SpeculativeJIT::compileCheckTraps(Node* node)
2045{
2046 ASSERT(Options::usePollingTraps());
2047 GPRTemporary unused(this);
2048 GPRReg unusedGPR = unused.gpr();
2049
2050 JITCompiler::Jump needTrapHandling = m_jit.branchTest8(JITCompiler::NonZero,
2051 JITCompiler::AbsoluteAddress(m_jit.vm()->needTrapHandlingAddress()));
2052
2053 addSlowPathGenerator(slowPathCall(needTrapHandling, this, operationHandleTraps, unusedGPR));
2054 noResult(node);
2055}
2056
2057void SpeculativeJIT::compileDoublePutByVal(Node* node, SpeculateCellOperand& base, SpeculateStrictInt32Operand& property)
2058{
2059 Edge child3 = m_jit.graph().varArgChild(node, 2);
2060 Edge child4 = m_jit.graph().varArgChild(node, 3);
2061
2062 ArrayMode arrayMode = node->arrayMode();
2063
2064 GPRReg baseReg = base.gpr();
2065 GPRReg propertyReg = property.gpr();
2066
2067 SpeculateDoubleOperand value(this, child3);
2068
2069 FPRReg valueReg = value.fpr();
2070
2071 DFG_TYPE_CHECK(
2072 JSValueRegs(), child3, SpecFullRealNumber,
2073 m_jit.branchIfNaN(valueReg));
2074
2075 if (!m_compileOkay)
2076 return;
2077
2078 StorageOperand storage(this, child4);
2079 GPRReg storageReg = storage.gpr();
2080
2081 if (node->op() == PutByValAlias) {
2082 // Store the value to the array.
2083 GPRReg propertyReg = property.gpr();
2084 FPRReg valueReg = value.fpr();
2085 m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2086
2087 noResult(m_currentNode);
2088 return;
2089 }
2090
2091 GPRTemporary temporary;
2092 GPRReg temporaryReg = temporaryRegisterForPutByVal(temporary, node);
2093
2094 MacroAssembler::Jump slowCase;
2095
2096 if (arrayMode.isInBounds()) {
2097 speculationCheck(
2098 OutOfBounds, JSValueRegs(), 0,
2099 m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2100 } else {
2101 MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2102
2103 slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
2104
2105 if (!arrayMode.isOutOfBounds())
2106 speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
2107
2108 m_jit.add32(TrustedImm32(1), propertyReg, temporaryReg);
2109 m_jit.store32(temporaryReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
2110
2111 inBounds.link(&m_jit);
2112 }
2113
2114 m_jit.storeDouble(valueReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight));
2115
2116 base.use();
2117 property.use();
2118 value.use();
2119 storage.use();
2120
2121 if (arrayMode.isOutOfBounds()) {
2122 addSlowPathGenerator(
2123 slowPathCall(
2124 slowCase, this,
2125 m_jit.isStrictModeFor(node->origin.semantic)
2126 ? (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsStrict)
2127 : (node->op() == PutByValDirect ? operationPutDoubleByValDirectBeyondArrayBoundsNonStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict),
2128 NoResult, baseReg, propertyReg, valueReg));
2129 }
2130
2131 noResult(m_currentNode, UseChildrenCalledExplicitly);
2132}
2133
2134void SpeculativeJIT::compileGetCharCodeAt(Node* node)
2135{
2136 SpeculateCellOperand string(this, node->child1());
2137 SpeculateStrictInt32Operand index(this, node->child2());
2138 StorageOperand storage(this, node->child3());
2139
2140 GPRReg stringReg = string.gpr();
2141 GPRReg indexReg = index.gpr();
2142 GPRReg storageReg = storage.gpr();
2143
2144 ASSERT(speculationChecked(m_state.forNode(node->child1()).m_type, SpecString));
2145
2146 GPRTemporary scratch(this);
2147 GPRReg scratchReg = scratch.gpr();
2148
2149 m_jit.loadPtr(MacroAssembler::Address(stringReg, JSString::offsetOfValue()), scratchReg);
2150
2151 // unsigned comparison so we can filter out negative indices and indices that are too large
2152 speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, indexReg, CCallHelpers::Address(scratchReg, StringImpl::lengthMemoryOffset())));
2153
2154 // Load the character into scratchReg
2155 JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2156
2157 m_jit.load8(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesOne, 0), scratchReg);
2158 JITCompiler::Jump cont8Bit = m_jit.jump();
2159
2160 is16Bit.link(&m_jit);
2161
2162 m_jit.load16(MacroAssembler::BaseIndex(storageReg, indexReg, MacroAssembler::TimesTwo, 0), scratchReg);
2163
2164 cont8Bit.link(&m_jit);
2165
2166 int32Result(scratchReg, m_currentNode);
2167}
2168
2169void SpeculativeJIT::compileGetByValOnString(Node* node)
2170{
2171 SpeculateCellOperand base(this, m_graph.child(node, 0));
2172 SpeculateStrictInt32Operand property(this, m_graph.child(node, 1));
2173 StorageOperand storage(this, m_graph.child(node, 2));
2174 GPRReg baseReg = base.gpr();
2175 GPRReg propertyReg = property.gpr();
2176 GPRReg storageReg = storage.gpr();
2177
2178 GPRTemporary scratch(this);
2179 GPRReg scratchReg = scratch.gpr();
2180#if USE(JSVALUE32_64)
2181 GPRTemporary resultTag;
2182 GPRReg resultTagReg = InvalidGPRReg;
2183 if (node->arrayMode().isOutOfBounds()) {
2184 GPRTemporary realResultTag(this);
2185 resultTag.adopt(realResultTag);
2186 resultTagReg = resultTag.gpr();
2187 }
2188#endif
2189
2190 ASSERT(ArrayMode(Array::String, Array::Read).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.child(node, 0))));
2191
2192 // unsigned comparison so we can filter out negative indices and indices that are too large
2193 m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), scratchReg);
2194 JITCompiler::Jump outOfBounds = m_jit.branch32(
2195 MacroAssembler::AboveOrEqual, propertyReg,
2196 MacroAssembler::Address(scratchReg, StringImpl::lengthMemoryOffset()));
2197 if (node->arrayMode().isInBounds())
2198 speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds);
2199
2200 // Load the character into scratchReg
2201 JITCompiler::Jump is16Bit = m_jit.branchTest32(MacroAssembler::Zero, MacroAssembler::Address(scratchReg, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIs8Bit()));
2202
2203 m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne, 0), scratchReg);
2204 JITCompiler::Jump cont8Bit = m_jit.jump();
2205
2206 is16Bit.link(&m_jit);
2207
2208 m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo, 0), scratchReg);
2209
2210 JITCompiler::Jump bigCharacter =
2211 m_jit.branch32(MacroAssembler::Above, scratchReg, TrustedImm32(maxSingleCharacterString));
2212
2213 // 8 bit string values don't need the isASCII check.
2214 cont8Bit.link(&m_jit);
2215
2216 m_jit.lshift32(MacroAssembler::TrustedImm32(sizeof(void*) == 4 ? 2 : 3), scratchReg);
2217 m_jit.addPtr(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), scratchReg);
2218 m_jit.loadPtr(scratchReg, scratchReg);
2219
2220 addSlowPathGenerator(
2221 slowPathCall(
2222 bigCharacter, this, operationSingleCharacterString, scratchReg, scratchReg));
2223
2224 if (node->arrayMode().isOutOfBounds()) {
2225#if USE(JSVALUE32_64)
2226 m_jit.move(TrustedImm32(JSValue::CellTag), resultTagReg);
2227#endif
2228
2229 JSGlobalObject* globalObject = m_jit.globalObjectFor(node->origin.semantic);
2230 bool prototypeChainIsSane = false;
2231 if (globalObject->stringPrototypeChainIsSane()) {
2232 // FIXME: This could be captured using a Speculation mode that means "out-of-bounds
2233 // loads return a trivial value". Something like SaneChainOutOfBounds. This should
2234 // speculate that we don't take negative out-of-bounds, or better yet, it should rely
2235 // on a stringPrototypeChainIsSane() guaranteeing that the prototypes have no negative
2236 // indexed properties either.
2237 // https://bugs.webkit.org/show_bug.cgi?id=144668
2238 m_jit.graph().registerAndWatchStructureTransition(globalObject->stringPrototype()->structure(*m_jit.vm()));
2239 m_jit.graph().registerAndWatchStructureTransition(globalObject->objectPrototype()->structure(*m_jit.vm()));
2240 prototypeChainIsSane = globalObject->stringPrototypeChainIsSane();
2241 }
2242 if (prototypeChainIsSane) {
2243#if USE(JSVALUE64)
2244 addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2245 outOfBounds, this, JSValueRegs(scratchReg), baseReg, propertyReg));
2246#else
2247 addSlowPathGenerator(std::make_unique<SaneStringGetByValSlowPathGenerator>(
2248 outOfBounds, this, JSValueRegs(resultTagReg, scratchReg),
2249 baseReg, propertyReg));
2250#endif
2251 } else {
2252#if USE(JSVALUE64)
2253 addSlowPathGenerator(
2254 slowPathCall(
2255 outOfBounds, this, operationGetByValStringInt,
2256 scratchReg, baseReg, propertyReg));
2257#else
2258 addSlowPathGenerator(
2259 slowPathCall(
2260 outOfBounds, this, operationGetByValStringInt,
2261 JSValueRegs(resultTagReg, scratchReg), baseReg, propertyReg));
2262#endif
2263 }
2264
2265#if USE(JSVALUE64)
2266 jsValueResult(scratchReg, m_currentNode);
2267#else
2268 jsValueResult(resultTagReg, scratchReg, m_currentNode);
2269#endif
2270 } else
2271 cellResult(scratchReg, m_currentNode);
2272}
2273
2274void SpeculativeJIT::compileFromCharCode(Node* node)
2275{
2276 Edge& child = node->child1();
2277 if (child.useKind() == UntypedUse) {
2278 JSValueOperand opr(this, child);
2279 JSValueRegs oprRegs = opr.jsValueRegs();
2280
2281 flushRegisters();
2282 JSValueRegsFlushedCallResult result(this);
2283 JSValueRegs resultRegs = result.regs();
2284 callOperation(operationStringFromCharCodeUntyped, resultRegs, oprRegs);
2285 m_jit.exceptionCheck();
2286
2287 jsValueResult(resultRegs, node);
2288 return;
2289 }
2290
2291 SpeculateStrictInt32Operand property(this, child);
2292 GPRReg propertyReg = property.gpr();
2293 GPRTemporary smallStrings(this);
2294 GPRTemporary scratch(this);
2295 GPRReg scratchReg = scratch.gpr();
2296 GPRReg smallStringsReg = smallStrings.gpr();
2297
2298 JITCompiler::JumpList slowCases;
2299 slowCases.append(m_jit.branch32(MacroAssembler::Above, propertyReg, TrustedImm32(maxSingleCharacterString)));
2300 m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.singleCharacterStrings()), smallStringsReg);
2301 m_jit.loadPtr(MacroAssembler::BaseIndex(smallStringsReg, propertyReg, MacroAssembler::ScalePtr, 0), scratchReg);
2302
2303 slowCases.append(m_jit.branchTest32(MacroAssembler::Zero, scratchReg));
2304 addSlowPathGenerator(slowPathCall(slowCases, this, operationStringFromCharCode, scratchReg, propertyReg));
2305 cellResult(scratchReg, m_currentNode);
2306}
2307
2308GeneratedOperandType SpeculativeJIT::checkGeneratedTypeForToInt32(Node* node)
2309{
2310 VirtualRegister virtualRegister = node->virtualRegister();
2311 GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2312
2313 switch (info.registerFormat()) {
2314 case DataFormatStorage:
2315 RELEASE_ASSERT_NOT_REACHED();
2316
2317 case DataFormatBoolean:
2318 case DataFormatCell:
2319 terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2320 return GeneratedOperandTypeUnknown;
2321
2322 case DataFormatNone:
2323 case DataFormatJSCell:
2324 case DataFormatJS:
2325 case DataFormatJSBoolean:
2326 case DataFormatJSDouble:
2327 return GeneratedOperandJSValue;
2328
2329 case DataFormatJSInt32:
2330 case DataFormatInt32:
2331 return GeneratedOperandInteger;
2332
2333 default:
2334 RELEASE_ASSERT_NOT_REACHED();
2335 return GeneratedOperandTypeUnknown;
2336 }
2337}
2338
2339void SpeculativeJIT::compileValueToInt32(Node* node)
2340{
2341 switch (node->child1().useKind()) {
2342#if USE(JSVALUE64)
2343 case Int52RepUse: {
2344 SpeculateStrictInt52Operand op1(this, node->child1());
2345 GPRTemporary result(this, Reuse, op1);
2346 GPRReg op1GPR = op1.gpr();
2347 GPRReg resultGPR = result.gpr();
2348 m_jit.zeroExtend32ToPtr(op1GPR, resultGPR);
2349 int32Result(resultGPR, node, DataFormatInt32);
2350 return;
2351 }
2352#endif // USE(JSVALUE64)
2353
2354 case DoubleRepUse: {
2355 GPRTemporary result(this);
2356 SpeculateDoubleOperand op1(this, node->child1());
2357 FPRReg fpr = op1.fpr();
2358 GPRReg gpr = result.gpr();
2359#if CPU(ARM64)
2360 if (MacroAssemblerARM64::supportsDoubleToInt32ConversionUsingJavaScriptSemantics())
2361 m_jit.convertDoubleToInt32UsingJavaScriptSemantics(fpr, gpr);
2362 else
2363#endif
2364 {
2365 JITCompiler::Jump notTruncatedToInteger = m_jit.branchTruncateDoubleToInt32(fpr, gpr, JITCompiler::BranchIfTruncateFailed);
2366 addSlowPathGenerator(slowPathCall(notTruncatedToInteger, this,
2367 hasSensibleDoubleToInt() ? operationToInt32SensibleSlow : operationToInt32, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, gpr, fpr));
2368 }
2369 int32Result(gpr, node);
2370 return;
2371 }
2372
2373 case NumberUse:
2374 case NotCellUse: {
2375 switch (checkGeneratedTypeForToInt32(node->child1().node())) {
2376 case GeneratedOperandInteger: {
2377 SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2378 GPRTemporary result(this, Reuse, op1);
2379 m_jit.move(op1.gpr(), result.gpr());
2380 int32Result(result.gpr(), node, op1.format());
2381 return;
2382 }
2383 case GeneratedOperandJSValue: {
2384 GPRTemporary result(this);
2385#if USE(JSVALUE64)
2386 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2387
2388 GPRReg gpr = op1.gpr();
2389 GPRReg resultGpr = result.gpr();
2390 FPRTemporary tempFpr(this);
2391 FPRReg fpr = tempFpr.fpr();
2392
2393 JITCompiler::Jump isInteger = m_jit.branchIfInt32(gpr);
2394 JITCompiler::JumpList converted;
2395
2396 if (node->child1().useKind() == NumberUse) {
2397 DFG_TYPE_CHECK(
2398 JSValueRegs(gpr), node->child1(), SpecBytecodeNumber,
2399 m_jit.branchIfNotNumber(gpr));
2400 } else {
2401 JITCompiler::Jump isNumber = m_jit.branchIfNumber(gpr);
2402
2403 DFG_TYPE_CHECK(
2404 JSValueRegs(gpr), node->child1(), ~SpecCellCheck, m_jit.branchIfCell(JSValueRegs(gpr)));
2405
2406 // It's not a cell: so true turns into 1 and all else turns into 0.
2407 m_jit.compare64(JITCompiler::Equal, gpr, TrustedImm32(ValueTrue), resultGpr);
2408 converted.append(m_jit.jump());
2409
2410 isNumber.link(&m_jit);
2411 }
2412
2413 // First, if we get here we have a double encoded as a JSValue
2414 unboxDouble(gpr, resultGpr, fpr);
2415#if CPU(ARM64)
2416 if (MacroAssemblerARM64::supportsDoubleToInt32ConversionUsingJavaScriptSemantics())
2417 m_jit.convertDoubleToInt32UsingJavaScriptSemantics(fpr, resultGpr);
2418 else
2419#endif
2420 {
2421 silentSpillAllRegisters(resultGpr);
2422 callOperation(operationToInt32, resultGpr, fpr);
2423 silentFillAllRegisters();
2424 }
2425
2426 converted.append(m_jit.jump());
2427
2428 isInteger.link(&m_jit);
2429 m_jit.zeroExtend32ToPtr(gpr, resultGpr);
2430
2431 converted.link(&m_jit);
2432#else
2433 Node* childNode = node->child1().node();
2434 VirtualRegister virtualRegister = childNode->virtualRegister();
2435 GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister);
2436
2437 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2438
2439 GPRReg payloadGPR = op1.payloadGPR();
2440 GPRReg resultGpr = result.gpr();
2441
2442 JITCompiler::JumpList converted;
2443
2444 if (info.registerFormat() == DataFormatJSInt32)
2445 m_jit.move(payloadGPR, resultGpr);
2446 else {
2447 GPRReg tagGPR = op1.tagGPR();
2448 FPRTemporary tempFpr(this);
2449 FPRReg fpr = tempFpr.fpr();
2450 FPRTemporary scratch(this);
2451
2452 JITCompiler::Jump isInteger = m_jit.branchIfInt32(tagGPR);
2453
2454 if (node->child1().useKind() == NumberUse) {
2455 DFG_TYPE_CHECK(
2456 op1.jsValueRegs(), node->child1(), SpecBytecodeNumber,
2457 m_jit.branch32(
2458 MacroAssembler::AboveOrEqual, tagGPR,
2459 TrustedImm32(JSValue::LowestTag)));
2460 } else {
2461 JITCompiler::Jump isNumber = m_jit.branch32(MacroAssembler::Below, tagGPR, TrustedImm32(JSValue::LowestTag));
2462
2463 DFG_TYPE_CHECK(
2464 op1.jsValueRegs(), node->child1(), ~SpecCell,
2465 m_jit.branchIfCell(op1.jsValueRegs()));
2466
2467 // It's not a cell: so true turns into 1 and all else turns into 0.
2468 JITCompiler::Jump isBoolean = m_jit.branchIfBoolean(tagGPR, InvalidGPRReg);
2469 m_jit.move(TrustedImm32(0), resultGpr);
2470 converted.append(m_jit.jump());
2471
2472 isBoolean.link(&m_jit);
2473 m_jit.move(payloadGPR, resultGpr);
2474 converted.append(m_jit.jump());
2475
2476 isNumber.link(&m_jit);
2477 }
2478
2479 unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
2480
2481 silentSpillAllRegisters(resultGpr);
2482 callOperation(operationToInt32, resultGpr, fpr);
2483 silentFillAllRegisters();
2484
2485 converted.append(m_jit.jump());
2486
2487 isInteger.link(&m_jit);
2488 m_jit.move(payloadGPR, resultGpr);
2489
2490 converted.link(&m_jit);
2491 }
2492#endif
2493 int32Result(resultGpr, node);
2494 return;
2495 }
2496 case GeneratedOperandTypeUnknown:
2497 RELEASE_ASSERT(!m_compileOkay);
2498 return;
2499 }
2500 RELEASE_ASSERT_NOT_REACHED();
2501 return;
2502 }
2503
2504 default:
2505 ASSERT(!m_compileOkay);
2506 return;
2507 }
2508}
2509
2510void SpeculativeJIT::compileUInt32ToNumber(Node* node)
2511{
2512 if (doesOverflow(node->arithMode())) {
2513 if (enableInt52()) {
2514 SpeculateInt32Operand op1(this, node->child1());
2515 GPRTemporary result(this, Reuse, op1);
2516 m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2517 strictInt52Result(result.gpr(), node);
2518 return;
2519 }
2520 SpeculateInt32Operand op1(this, node->child1());
2521 FPRTemporary result(this);
2522
2523 GPRReg inputGPR = op1.gpr();
2524 FPRReg outputFPR = result.fpr();
2525
2526 m_jit.convertInt32ToDouble(inputGPR, outputFPR);
2527
2528 JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, inputGPR, TrustedImm32(0));
2529 m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), outputFPR);
2530 positive.link(&m_jit);
2531
2532 doubleResult(outputFPR, node);
2533 return;
2534 }
2535
2536 RELEASE_ASSERT(node->arithMode() == Arith::CheckOverflow);
2537
2538 SpeculateInt32Operand op1(this, node->child1());
2539 GPRTemporary result(this);
2540
2541 m_jit.move(op1.gpr(), result.gpr());
2542
2543 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, result.gpr(), TrustedImm32(0)));
2544
2545 int32Result(result.gpr(), node, op1.format());
2546}
2547
2548void SpeculativeJIT::compileDoubleAsInt32(Node* node)
2549{
2550 SpeculateDoubleOperand op1(this, node->child1());
2551 FPRTemporary scratch(this);
2552 GPRTemporary result(this);
2553
2554 FPRReg valueFPR = op1.fpr();
2555 FPRReg scratchFPR = scratch.fpr();
2556 GPRReg resultGPR = result.gpr();
2557
2558 JITCompiler::JumpList failureCases;
2559 RELEASE_ASSERT(shouldCheckOverflow(node->arithMode()));
2560 m_jit.branchConvertDoubleToInt32(
2561 valueFPR, resultGPR, failureCases, scratchFPR,
2562 shouldCheckNegativeZero(node->arithMode()));
2563 speculationCheck(Overflow, JSValueRegs(), 0, failureCases);
2564
2565 int32Result(resultGPR, node);
2566}
2567
2568void SpeculativeJIT::compileDoubleRep(Node* node)
2569{
2570 switch (node->child1().useKind()) {
2571 case RealNumberUse: {
2572 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2573 FPRTemporary result(this);
2574
2575 JSValueRegs op1Regs = op1.jsValueRegs();
2576 FPRReg resultFPR = result.fpr();
2577
2578#if USE(JSVALUE64)
2579 GPRTemporary temp(this);
2580 GPRReg tempGPR = temp.gpr();
2581 m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
2582#else
2583 FPRTemporary temp(this);
2584 FPRReg tempFPR = temp.fpr();
2585 unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
2586#endif
2587
2588 JITCompiler::Jump done = m_jit.branchIfNotNaN(resultFPR);
2589
2590 DFG_TYPE_CHECK(
2591 op1Regs, node->child1(), SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
2592 m_jit.convertInt32ToDouble(op1Regs.payloadGPR(), resultFPR);
2593
2594 done.link(&m_jit);
2595
2596 doubleResult(resultFPR, node);
2597 return;
2598 }
2599
2600 case NotCellUse:
2601 case NumberUse: {
2602 SpeculatedType possibleTypes = m_state.forNode(node->child1()).m_type;
2603 if (isInt32Speculation(possibleTypes)) {
2604 SpeculateInt32Operand op1(this, node->child1(), ManualOperandSpeculation);
2605 FPRTemporary result(this);
2606 m_jit.convertInt32ToDouble(op1.gpr(), result.fpr());
2607 doubleResult(result.fpr(), node);
2608 return;
2609 }
2610
2611 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
2612 FPRTemporary result(this);
2613
2614#if USE(JSVALUE64)
2615 GPRTemporary temp(this);
2616
2617 GPRReg op1GPR = op1.gpr();
2618 GPRReg tempGPR = temp.gpr();
2619 FPRReg resultFPR = result.fpr();
2620 JITCompiler::JumpList done;
2621
2622 JITCompiler::Jump isInteger = m_jit.branchIfInt32(op1GPR);
2623
2624 if (node->child1().useKind() == NotCellUse) {
2625 JITCompiler::Jump isNumber = m_jit.branchIfNumber(op1GPR);
2626 JITCompiler::Jump isUndefined = m_jit.branchIfUndefined(op1GPR);
2627
2628 static const double zero = 0;
2629 m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2630
2631 JITCompiler::Jump isNull = m_jit.branchIfNull(op1GPR);
2632 done.append(isNull);
2633
2634 DFG_TYPE_CHECK(JSValueRegs(op1GPR), node->child1(), ~SpecCellCheck,
2635 m_jit.branchTest64(JITCompiler::Zero, op1GPR, TrustedImm32(static_cast<int32_t>(TagBitBool))));
2636
2637 JITCompiler::Jump isFalse = m_jit.branch64(JITCompiler::Equal, op1GPR, TrustedImm64(ValueFalse));
2638 static const double one = 1;
2639 m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2640 done.append(m_jit.jump());
2641 done.append(isFalse);
2642
2643 isUndefined.link(&m_jit);
2644 static const double NaN = PNaN;
2645 m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2646 done.append(m_jit.jump());
2647
2648 isNumber.link(&m_jit);
2649 } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2650 typeCheck(
2651 JSValueRegs(op1GPR), node->child1(), SpecBytecodeNumber,
2652 m_jit.branchIfNotNumber(op1GPR));
2653 }
2654
2655 unboxDouble(op1GPR, tempGPR, resultFPR);
2656 done.append(m_jit.jump());
2657
2658 isInteger.link(&m_jit);
2659 m_jit.convertInt32ToDouble(op1GPR, resultFPR);
2660 done.link(&m_jit);
2661#else // USE(JSVALUE64) -> this is the 32_64 case
2662 FPRTemporary temp(this);
2663
2664 GPRReg op1TagGPR = op1.tagGPR();
2665 GPRReg op1PayloadGPR = op1.payloadGPR();
2666 FPRReg tempFPR = temp.fpr();
2667 FPRReg resultFPR = result.fpr();
2668 JITCompiler::JumpList done;
2669
2670 JITCompiler::Jump isInteger = m_jit.branchIfInt32(op1TagGPR);
2671
2672 if (node->child1().useKind() == NotCellUse) {
2673 JITCompiler::Jump isNumber = m_jit.branch32(JITCompiler::Below, op1TagGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
2674 JITCompiler::Jump isUndefined = m_jit.branchIfUndefined(op1TagGPR);
2675
2676 static const double zero = 0;
2677 m_jit.loadDouble(TrustedImmPtr(&zero), resultFPR);
2678
2679 JITCompiler::Jump isNull = m_jit.branchIfNull(op1TagGPR);
2680 done.append(isNull);
2681
2682 DFG_TYPE_CHECK(JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), ~SpecCell, m_jit.branchIfNotBoolean(op1TagGPR, InvalidGPRReg));
2683
2684 JITCompiler::Jump isFalse = m_jit.branchTest32(JITCompiler::Zero, op1PayloadGPR, TrustedImm32(1));
2685 static const double one = 1;
2686 m_jit.loadDouble(TrustedImmPtr(&one), resultFPR);
2687 done.append(m_jit.jump());
2688 done.append(isFalse);
2689
2690 isUndefined.link(&m_jit);
2691 static const double NaN = PNaN;
2692 m_jit.loadDouble(TrustedImmPtr(&NaN), resultFPR);
2693 done.append(m_jit.jump());
2694
2695 isNumber.link(&m_jit);
2696 } else if (needsTypeCheck(node->child1(), SpecBytecodeNumber)) {
2697 // This check fails with Int32Tag, but it is OK since Int32 case is already excluded.
2698 typeCheck(
2699 JSValueRegs(op1TagGPR, op1PayloadGPR), node->child1(), SpecBytecodeNumber,
2700 m_jit.branch32(MacroAssembler::AboveOrEqual, op1TagGPR, TrustedImm32(JSValue::LowestTag)));
2701 }
2702
2703 unboxDouble(op1TagGPR, op1PayloadGPR, resultFPR, tempFPR);
2704 done.append(m_jit.jump());
2705
2706 isInteger.link(&m_jit);
2707 m_jit.convertInt32ToDouble(op1PayloadGPR, resultFPR);
2708 done.link(&m_jit);
2709#endif // USE(JSVALUE64)
2710
2711 doubleResult(resultFPR, node);
2712 return;
2713 }
2714
2715#if USE(JSVALUE64)
2716 case Int52RepUse: {
2717 SpeculateStrictInt52Operand value(this, node->child1());
2718 FPRTemporary result(this);
2719
2720 GPRReg valueGPR = value.gpr();
2721 FPRReg resultFPR = result.fpr();
2722
2723 m_jit.convertInt64ToDouble(valueGPR, resultFPR);
2724
2725 doubleResult(resultFPR, node);
2726 return;
2727 }
2728#endif // USE(JSVALUE64)
2729
2730 default:
2731 RELEASE_ASSERT_NOT_REACHED();
2732 return;
2733 }
2734}
2735
2736void SpeculativeJIT::compileValueRep(Node* node)
2737{
2738 switch (node->child1().useKind()) {
2739 case DoubleRepUse: {
2740 SpeculateDoubleOperand value(this, node->child1());
2741 JSValueRegsTemporary result(this);
2742
2743 FPRReg valueFPR = value.fpr();
2744 JSValueRegs resultRegs = result.regs();
2745
2746 // It's very tempting to in-place filter the value to indicate that it's not impure NaN
2747 // anymore. Unfortunately, this would be unsound. If it's a GetLocal or if the value was
2748 // subject to a prior SetLocal, filtering the value would imply that the corresponding
2749 // local was purified.
2750 if (needsTypeCheck(node->child1(), ~SpecDoubleImpureNaN))
2751 m_jit.purifyNaN(valueFPR);
2752
2753 boxDouble(valueFPR, resultRegs);
2754
2755 jsValueResult(resultRegs, node);
2756 return;
2757 }
2758
2759#if USE(JSVALUE64)
2760 case Int52RepUse: {
2761 SpeculateStrictInt52Operand value(this, node->child1());
2762 GPRTemporary result(this);
2763
2764 GPRReg valueGPR = value.gpr();
2765 GPRReg resultGPR = result.gpr();
2766
2767 boxInt52(valueGPR, resultGPR, DataFormatStrictInt52);
2768
2769 jsValueResult(resultGPR, node);
2770 return;
2771 }
2772#endif // USE(JSVALUE64)
2773
2774 default:
2775 RELEASE_ASSERT_NOT_REACHED();
2776 return;
2777 }
2778}
2779
2780static double clampDoubleToByte(double d)
2781{
2782 d += 0.5;
2783 if (!(d > 0))
2784 d = 0;
2785 else if (d > 255)
2786 d = 255;
2787 return d;
2788}
2789
2790static void compileClampIntegerToByte(JITCompiler& jit, GPRReg result)
2791{
2792 MacroAssembler::Jump inBounds = jit.branch32(MacroAssembler::BelowOrEqual, result, JITCompiler::TrustedImm32(0xff));
2793 MacroAssembler::Jump tooBig = jit.branch32(MacroAssembler::GreaterThan, result, JITCompiler::TrustedImm32(0xff));
2794 jit.xorPtr(result, result);
2795 MacroAssembler::Jump clamped = jit.jump();
2796 tooBig.link(&jit);
2797 jit.move(JITCompiler::TrustedImm32(255), result);
2798 clamped.link(&jit);
2799 inBounds.link(&jit);
2800}
2801
2802static void compileClampDoubleToByte(JITCompiler& jit, GPRReg result, FPRReg source, FPRReg scratch)
2803{
2804 // Unordered compare so we pick up NaN
2805 static const double zero = 0;
2806 static const double byteMax = 255;
2807 static const double half = 0.5;
2808 jit.loadDouble(JITCompiler::TrustedImmPtr(&zero), scratch);
2809 MacroAssembler::Jump tooSmall = jit.branchDouble(MacroAssembler::DoubleLessThanOrEqualOrUnordered, source, scratch);
2810 jit.loadDouble(JITCompiler::TrustedImmPtr(&byteMax), scratch);
2811 MacroAssembler::Jump tooBig = jit.branchDouble(MacroAssembler::DoubleGreaterThan, source, scratch);
2812
2813 jit.loadDouble(JITCompiler::TrustedImmPtr(&half), scratch);
2814 // FIXME: This should probably just use a floating point round!
2815 // https://bugs.webkit.org/show_bug.cgi?id=72054
2816 jit.addDouble(source, scratch);
2817 jit.truncateDoubleToInt32(scratch, result);
2818 MacroAssembler::Jump truncatedInt = jit.jump();
2819
2820 tooSmall.link(&jit);
2821 jit.xorPtr(result, result);
2822 MacroAssembler::Jump zeroed = jit.jump();
2823
2824 tooBig.link(&jit);
2825 jit.move(JITCompiler::TrustedImm32(255), result);
2826
2827 truncatedInt.link(&jit);
2828 zeroed.link(&jit);
2829
2830}
2831
2832JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayOutOfBounds(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2833{
2834 if (node->op() == PutByValAlias)
2835 return JITCompiler::Jump();
2836 JSArrayBufferView* view = m_jit.graph().tryGetFoldableView(
2837 m_state.forNode(m_jit.graph().child(node, 0)).m_value, node->arrayMode());
2838 if (view) {
2839 uint32_t length = view->length();
2840 Node* indexNode = m_jit.graph().child(node, 1).node();
2841 if (indexNode->isInt32Constant() && indexNode->asUInt32() < length)
2842 return JITCompiler::Jump();
2843 return m_jit.branch32(
2844 MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Imm32(length));
2845 }
2846 return m_jit.branch32(
2847 MacroAssembler::AboveOrEqual, indexGPR,
2848 MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()));
2849}
2850
2851void SpeculativeJIT::emitTypedArrayBoundsCheck(Node* node, GPRReg baseGPR, GPRReg indexGPR)
2852{
2853 JITCompiler::Jump jump = jumpForTypedArrayOutOfBounds(node, baseGPR, indexGPR);
2854 if (!jump.isSet())
2855 return;
2856 speculationCheck(OutOfBounds, JSValueRegs(), 0, jump);
2857}
2858
2859JITCompiler::Jump SpeculativeJIT::jumpForTypedArrayIsNeuteredIfOutOfBounds(Node* node, GPRReg base, JITCompiler::Jump outOfBounds)
2860{
2861 JITCompiler::Jump done;
2862 if (outOfBounds.isSet()) {
2863 done = m_jit.jump();
2864 if (node->arrayMode().isInBounds())
2865 speculationCheck(OutOfBounds, JSValueSource(), 0, outOfBounds);
2866 else {
2867 outOfBounds.link(&m_jit);
2868
2869 JITCompiler::Jump notWasteful = m_jit.branch32(
2870 MacroAssembler::NotEqual,
2871 MacroAssembler::Address(base, JSArrayBufferView::offsetOfMode()),
2872 TrustedImm32(WastefulTypedArray));
2873
2874 JITCompiler::Jump hasNullVector = m_jit.branchTestPtr(
2875 MacroAssembler::Zero,
2876 MacroAssembler::Address(base, JSArrayBufferView::offsetOfVector()));
2877 speculationCheck(Uncountable, JSValueSource(), node, hasNullVector);
2878 notWasteful.link(&m_jit);
2879 }
2880 }
2881 return done;
2882}
2883
2884void SpeculativeJIT::loadFromIntTypedArray(GPRReg storageReg, GPRReg propertyReg, GPRReg resultReg, TypedArrayType type)
2885{
2886 switch (elementSize(type)) {
2887 case 1:
2888 if (isSigned(type))
2889 m_jit.load8SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2890 else
2891 m_jit.load8(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesOne), resultReg);
2892 break;
2893 case 2:
2894 if (isSigned(type))
2895 m_jit.load16SignedExtendTo32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2896 else
2897 m_jit.load16(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesTwo), resultReg);
2898 break;
2899 case 4:
2900 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
2901 break;
2902 default:
2903 CRASH();
2904 }
2905}
2906
2907void SpeculativeJIT::setIntTypedArrayLoadResult(Node* node, GPRReg resultReg, TypedArrayType type, bool canSpeculate)
2908{
2909 if (elementSize(type) < 4 || isSigned(type)) {
2910 int32Result(resultReg, node);
2911 return;
2912 }
2913
2914 ASSERT(elementSize(type) == 4 && !isSigned(type));
2915 if (node->shouldSpeculateInt32() && canSpeculate) {
2916 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, TrustedImm32(0)));
2917 int32Result(resultReg, node);
2918 return;
2919 }
2920
2921#if USE(JSVALUE64)
2922 if (node->shouldSpeculateInt52()) {
2923 ASSERT(enableInt52());
2924 m_jit.zeroExtend32ToPtr(resultReg, resultReg);
2925 strictInt52Result(resultReg, node);
2926 return;
2927 }
2928#endif
2929
2930 FPRTemporary fresult(this);
2931 m_jit.convertInt32ToDouble(resultReg, fresult.fpr());
2932 JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, resultReg, TrustedImm32(0));
2933 m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), fresult.fpr());
2934 positive.link(&m_jit);
2935 doubleResult(fresult.fpr(), node);
2936}
2937
2938void SpeculativeJIT::compileGetByValOnIntTypedArray(Node* node, TypedArrayType type)
2939{
2940 ASSERT(isInt(type));
2941
2942 SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
2943 SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
2944 StorageOperand storage(this, m_graph.varArgChild(node, 2));
2945
2946 GPRReg baseReg = base.gpr();
2947 GPRReg propertyReg = property.gpr();
2948 GPRReg storageReg = storage.gpr();
2949
2950 GPRTemporary result(this);
2951 GPRReg resultReg = result.gpr();
2952
2953 ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
2954
2955 emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
2956 loadFromIntTypedArray(storageReg, propertyReg, resultReg, type);
2957 bool canSpeculate = true;
2958 setIntTypedArrayLoadResult(node, resultReg, type, canSpeculate);
2959}
2960
2961bool SpeculativeJIT::getIntTypedArrayStoreOperand(
2962 GPRTemporary& value,
2963 GPRReg property,
2964#if USE(JSVALUE32_64)
2965 GPRTemporary& propertyTag,
2966 GPRTemporary& valueTag,
2967#endif
2968 Edge valueUse, JITCompiler::JumpList& slowPathCases, bool isClamped)
2969{
2970 bool isAppropriateConstant = false;
2971 if (valueUse->isConstant()) {
2972 JSValue jsValue = valueUse->asJSValue();
2973 SpeculatedType expectedType = typeFilterFor(valueUse.useKind());
2974 SpeculatedType actualType = speculationFromValue(jsValue);
2975 isAppropriateConstant = (expectedType | actualType) == expectedType;
2976 }
2977
2978 if (isAppropriateConstant) {
2979 JSValue jsValue = valueUse->asJSValue();
2980 if (!jsValue.isNumber()) {
2981 terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
2982 return false;
2983 }
2984 double d = jsValue.asNumber();
2985 if (isClamped)
2986 d = clampDoubleToByte(d);
2987 GPRTemporary scratch(this);
2988 GPRReg scratchReg = scratch.gpr();
2989 m_jit.move(Imm32(toInt32(d)), scratchReg);
2990 value.adopt(scratch);
2991 } else {
2992 switch (valueUse.useKind()) {
2993 case Int32Use: {
2994 SpeculateInt32Operand valueOp(this, valueUse);
2995 GPRTemporary scratch(this);
2996 GPRReg scratchReg = scratch.gpr();
2997 m_jit.move(valueOp.gpr(), scratchReg);
2998 if (isClamped)
2999 compileClampIntegerToByte(m_jit, scratchReg);
3000 value.adopt(scratch);
3001 break;
3002 }
3003
3004#if USE(JSVALUE64)
3005 case Int52RepUse: {
3006 SpeculateStrictInt52Operand valueOp(this, valueUse);
3007 GPRTemporary scratch(this);
3008 GPRReg scratchReg = scratch.gpr();
3009 m_jit.move(valueOp.gpr(), scratchReg);
3010 if (isClamped) {
3011 MacroAssembler::Jump inBounds = m_jit.branch64(
3012 MacroAssembler::BelowOrEqual, scratchReg, JITCompiler::TrustedImm64(0xff));
3013 MacroAssembler::Jump tooBig = m_jit.branch64(
3014 MacroAssembler::GreaterThan, scratchReg, JITCompiler::TrustedImm64(0xff));
3015 m_jit.move(TrustedImm32(0), scratchReg);
3016 MacroAssembler::Jump clamped = m_jit.jump();
3017 tooBig.link(&m_jit);
3018 m_jit.move(JITCompiler::TrustedImm32(255), scratchReg);
3019 clamped.link(&m_jit);
3020 inBounds.link(&m_jit);
3021 }
3022 value.adopt(scratch);
3023 break;
3024 }
3025#endif // USE(JSVALUE64)
3026
3027 case DoubleRepUse: {
3028 RELEASE_ASSERT(!isAtomicsIntrinsic(m_currentNode->op()));
3029 if (isClamped) {
3030 SpeculateDoubleOperand valueOp(this, valueUse);
3031 GPRTemporary result(this);
3032 FPRTemporary floatScratch(this);
3033 FPRReg fpr = valueOp.fpr();
3034 GPRReg gpr = result.gpr();
3035 compileClampDoubleToByte(m_jit, gpr, fpr, floatScratch.fpr());
3036 value.adopt(result);
3037 } else {
3038#if USE(JSVALUE32_64)
3039 GPRTemporary realPropertyTag(this);
3040 propertyTag.adopt(realPropertyTag);
3041 GPRReg propertyTagGPR = propertyTag.gpr();
3042
3043 GPRTemporary realValueTag(this);
3044 valueTag.adopt(realValueTag);
3045 GPRReg valueTagGPR = valueTag.gpr();
3046#endif
3047 SpeculateDoubleOperand valueOp(this, valueUse);
3048 GPRTemporary result(this);
3049 FPRReg fpr = valueOp.fpr();
3050 GPRReg gpr = result.gpr();
3051 MacroAssembler::Jump notNaN = m_jit.branchIfNotNaN(fpr);
3052 m_jit.xorPtr(gpr, gpr);
3053 MacroAssembler::JumpList fixed(m_jit.jump());
3054 notNaN.link(&m_jit);
3055
3056 fixed.append(m_jit.branchTruncateDoubleToInt32(
3057 fpr, gpr, MacroAssembler::BranchIfTruncateSuccessful));
3058
3059#if USE(JSVALUE64)
3060 m_jit.or64(GPRInfo::tagTypeNumberRegister, property);
3061 boxDouble(fpr, gpr);
3062#else
3063 UNUSED_PARAM(property);
3064 m_jit.move(TrustedImm32(JSValue::Int32Tag), propertyTagGPR);
3065 boxDouble(fpr, valueTagGPR, gpr);
3066#endif
3067 slowPathCases.append(m_jit.jump());
3068
3069 fixed.link(&m_jit);
3070 value.adopt(result);
3071 }
3072 break;
3073 }
3074
3075 default:
3076 RELEASE_ASSERT_NOT_REACHED();
3077 break;
3078 }
3079 }
3080 return true;
3081}
3082
3083void SpeculativeJIT::compilePutByValForIntTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3084{
3085 ASSERT(isInt(type));
3086
3087 StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3088 GPRReg storageReg = storage.gpr();
3089
3090 Edge valueUse = m_jit.graph().varArgChild(node, 2);
3091
3092 GPRTemporary value;
3093#if USE(JSVALUE32_64)
3094 GPRTemporary propertyTag;
3095 GPRTemporary valueTag;
3096#endif
3097
3098 JITCompiler::JumpList slowPathCases;
3099
3100 bool result = getIntTypedArrayStoreOperand(
3101 value, property,
3102#if USE(JSVALUE32_64)
3103 propertyTag, valueTag,
3104#endif
3105 valueUse, slowPathCases, isClamped(type));
3106 if (!result) {
3107 noResult(node);
3108 return;
3109 }
3110
3111 GPRReg valueGPR = value.gpr();
3112#if USE(JSVALUE32_64)
3113 GPRReg propertyTagGPR = propertyTag.gpr();
3114 GPRReg valueTagGPR = valueTag.gpr();
3115#endif
3116
3117 ASSERT_UNUSED(valueGPR, valueGPR != property);
3118 ASSERT(valueGPR != base);
3119 ASSERT(valueGPR != storageReg);
3120 JITCompiler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3121
3122 switch (elementSize(type)) {
3123 case 1:
3124 m_jit.store8(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesOne));
3125 break;
3126 case 2:
3127 m_jit.store16(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesTwo));
3128 break;
3129 case 4:
3130 m_jit.store32(value.gpr(), MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3131 break;
3132 default:
3133 CRASH();
3134 }
3135
3136 JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3137 if (done.isSet())
3138 done.link(&m_jit);
3139
3140 if (!slowPathCases.empty()) {
3141#if USE(JSVALUE64)
3142 if (node->op() == PutByValDirect) {
3143 addSlowPathGenerator(slowPathCall(
3144 slowPathCases, this,
3145 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict,
3146 NoResult, base, property, valueGPR));
3147 } else {
3148 addSlowPathGenerator(slowPathCall(
3149 slowPathCases, this,
3150 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict,
3151 NoResult, base, property, valueGPR));
3152 }
3153#else // not USE(JSVALUE64)
3154 if (node->op() == PutByValDirect) {
3155 addSlowPathGenerator(slowPathCall(
3156 slowPathCases, this,
3157 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict,
3158 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3159 } else {
3160 addSlowPathGenerator(slowPathCall(
3161 slowPathCases, this,
3162 m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStrict : operationPutByValCellNonStrict,
3163 NoResult, base, JSValueRegs(propertyTagGPR, property), JSValueRegs(valueTagGPR, valueGPR)));
3164 }
3165#endif
3166 }
3167
3168 noResult(node);
3169}
3170
3171void SpeculativeJIT::compileGetByValOnFloatTypedArray(Node* node, TypedArrayType type)
3172{
3173 ASSERT(isFloat(type));
3174
3175 SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
3176 SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
3177 StorageOperand storage(this, m_graph.varArgChild(node, 2));
3178
3179 GPRReg baseReg = base.gpr();
3180 GPRReg propertyReg = property.gpr();
3181 GPRReg storageReg = storage.gpr();
3182
3183 ASSERT(node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
3184
3185 FPRTemporary result(this);
3186 FPRReg resultReg = result.fpr();
3187 emitTypedArrayBoundsCheck(node, baseReg, propertyReg);
3188 switch (elementSize(type)) {
3189 case 4:
3190 m_jit.loadFloat(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesFour), resultReg);
3191 m_jit.convertFloatToDouble(resultReg, resultReg);
3192 break;
3193 case 8: {
3194 m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), resultReg);
3195 break;
3196 }
3197 default:
3198 RELEASE_ASSERT_NOT_REACHED();
3199 }
3200
3201 doubleResult(resultReg, node);
3202}
3203
3204void SpeculativeJIT::compilePutByValForFloatTypedArray(GPRReg base, GPRReg property, Node* node, TypedArrayType type)
3205{
3206 ASSERT(isFloat(type));
3207
3208 StorageOperand storage(this, m_jit.graph().varArgChild(node, 3));
3209 GPRReg storageReg = storage.gpr();
3210
3211 Edge baseUse = m_jit.graph().varArgChild(node, 0);
3212 Edge valueUse = m_jit.graph().varArgChild(node, 2);
3213
3214 SpeculateDoubleOperand valueOp(this, valueUse);
3215 FPRTemporary scratch(this);
3216 FPRReg valueFPR = valueOp.fpr();
3217 FPRReg scratchFPR = scratch.fpr();
3218
3219 ASSERT_UNUSED(baseUse, node->arrayMode().alreadyChecked(m_jit.graph(), node, m_state.forNode(baseUse)));
3220
3221 MacroAssembler::Jump outOfBounds = jumpForTypedArrayOutOfBounds(node, base, property);
3222
3223 switch (elementSize(type)) {
3224 case 4: {
3225 m_jit.moveDouble(valueFPR, scratchFPR);
3226 m_jit.convertDoubleToFloat(valueFPR, scratchFPR);
3227 m_jit.storeFloat(scratchFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesFour));
3228 break;
3229 }
3230 case 8:
3231 m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageReg, property, MacroAssembler::TimesEight));
3232 break;
3233 default:
3234 RELEASE_ASSERT_NOT_REACHED();
3235 }
3236
3237 JITCompiler::Jump done = jumpForTypedArrayIsNeuteredIfOutOfBounds(node, base, outOfBounds);
3238 if (done.isSet())
3239 done.link(&m_jit);
3240 noResult(node);
3241}
3242
3243void SpeculativeJIT::compileGetByValForObjectWithString(Node* node)
3244{
3245 SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3246 SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3247
3248 GPRReg arg1GPR = arg1.gpr();
3249 GPRReg arg2GPR = arg2.gpr();
3250
3251 speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3252 speculateString(m_graph.varArgChild(node, 1), arg2GPR);
3253
3254 flushRegisters();
3255 JSValueRegsFlushedCallResult result(this);
3256 JSValueRegs resultRegs = result.regs();
3257 callOperation(operationGetByValObjectString, resultRegs, arg1GPR, arg2GPR);
3258 m_jit.exceptionCheck();
3259
3260 jsValueResult(resultRegs, node);
3261}
3262
3263void SpeculativeJIT::compileGetByValForObjectWithSymbol(Node* node)
3264{
3265 SpeculateCellOperand arg1(this, m_graph.varArgChild(node, 0));
3266 SpeculateCellOperand arg2(this, m_graph.varArgChild(node, 1));
3267
3268 GPRReg arg1GPR = arg1.gpr();
3269 GPRReg arg2GPR = arg2.gpr();
3270
3271 speculateObject(m_graph.varArgChild(node, 0), arg1GPR);
3272 speculateSymbol(m_graph.varArgChild(node, 1), arg2GPR);
3273
3274 flushRegisters();
3275 JSValueRegsFlushedCallResult result(this);
3276 JSValueRegs resultRegs = result.regs();
3277 callOperation(operationGetByValObjectSymbol, resultRegs, arg1GPR, arg2GPR);
3278 m_jit.exceptionCheck();
3279
3280 jsValueResult(resultRegs, node);
3281}
3282
3283void SpeculativeJIT::compilePutByValForCellWithString(Node* node, Edge& child1, Edge& child2, Edge& child3)
3284{
3285 SpeculateCellOperand arg1(this, child1);
3286 SpeculateCellOperand arg2(this, child2);
3287 JSValueOperand arg3(this, child3);
3288
3289 GPRReg arg1GPR = arg1.gpr();
3290 GPRReg arg2GPR = arg2.gpr();
3291 JSValueRegs arg3Regs = arg3.jsValueRegs();
3292
3293 speculateString(child2, arg2GPR);
3294
3295 flushRegisters();
3296 callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellStringStrict : operationPutByValCellStringNonStrict, arg1GPR, arg2GPR, arg3Regs);
3297 m_jit.exceptionCheck();
3298
3299 noResult(node);
3300}
3301
3302void SpeculativeJIT::compilePutByValForCellWithSymbol(Node* node, Edge& child1, Edge& child2, Edge& child3)
3303{
3304 SpeculateCellOperand arg1(this, child1);
3305 SpeculateCellOperand arg2(this, child2);
3306 JSValueOperand arg3(this, child3);
3307
3308 GPRReg arg1GPR = arg1.gpr();
3309 GPRReg arg2GPR = arg2.gpr();
3310 JSValueRegs arg3Regs = arg3.jsValueRegs();
3311
3312 speculateSymbol(child2, arg2GPR);
3313
3314 flushRegisters();
3315 callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValCellSymbolStrict : operationPutByValCellSymbolNonStrict, arg1GPR, arg2GPR, arg3Regs);
3316 m_jit.exceptionCheck();
3317
3318 noResult(node);
3319}
3320
3321void SpeculativeJIT::compileGetByValWithThis(Node* node)
3322{
3323 JSValueOperand base(this, node->child1());
3324 JSValueRegs baseRegs = base.jsValueRegs();
3325 JSValueOperand thisValue(this, node->child2());
3326 JSValueRegs thisValueRegs = thisValue.jsValueRegs();
3327 JSValueOperand subscript(this, node->child3());
3328 JSValueRegs subscriptRegs = subscript.jsValueRegs();
3329
3330 flushRegisters();
3331 JSValueRegsFlushedCallResult result(this);
3332 JSValueRegs resultRegs = result.regs();
3333 callOperation(operationGetByValWithThis, resultRegs, baseRegs, thisValueRegs, subscriptRegs);
3334 m_jit.exceptionCheck();
3335
3336 jsValueResult(resultRegs, node);
3337}
3338
3339void SpeculativeJIT::compileCheckTypeInfoFlags(Node* node)
3340{
3341 SpeculateCellOperand base(this, node->child1());
3342
3343 GPRReg baseGPR = base.gpr();
3344
3345 // FIXME: This only works for checking if a single bit is set. If we want to check more
3346 // than one bit at once, we'll need to fix this:
3347 // https://bugs.webkit.org/show_bug.cgi?id=185705
3348 speculationCheck(BadTypeInfoFlags, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(node->typeInfoOperand())));
3349
3350 noResult(node);
3351}
3352
3353void SpeculativeJIT::compileParseInt(Node* node)
3354{
3355 RELEASE_ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == StringUse);
3356 if (node->child2()) {
3357 SpeculateInt32Operand radix(this, node->child2());
3358 GPRReg radixGPR = radix.gpr();
3359 if (node->child1().useKind() == UntypedUse) {
3360 JSValueOperand value(this, node->child1());
3361 JSValueRegs valueRegs = value.jsValueRegs();
3362
3363 flushRegisters();
3364 JSValueRegsFlushedCallResult result(this);
3365 JSValueRegs resultRegs = result.regs();
3366 callOperation(operationParseIntGeneric, resultRegs, valueRegs, radixGPR);
3367 m_jit.exceptionCheck();
3368 jsValueResult(resultRegs, node);
3369 return;
3370 }
3371
3372 SpeculateCellOperand value(this, node->child1());
3373 GPRReg valueGPR = value.gpr();
3374 speculateString(node->child1(), valueGPR);
3375
3376 flushRegisters();
3377 JSValueRegsFlushedCallResult result(this);
3378 JSValueRegs resultRegs = result.regs();
3379 callOperation(operationParseIntString, resultRegs, valueGPR, radixGPR);
3380 m_jit.exceptionCheck();
3381 jsValueResult(resultRegs, node);
3382 return;
3383 }
3384
3385 if (node->child1().useKind() == UntypedUse) {
3386 JSValueOperand value(this, node->child1());
3387 JSValueRegs valueRegs = value.jsValueRegs();
3388
3389 flushRegisters();
3390 JSValueRegsFlushedCallResult result(this);
3391 JSValueRegs resultRegs = result.regs();
3392 callOperation(operationParseIntNoRadixGeneric, resultRegs, valueRegs);
3393 m_jit.exceptionCheck();
3394 jsValueResult(resultRegs, node);
3395 return;
3396 }
3397
3398 SpeculateCellOperand value(this, node->child1());
3399 GPRReg valueGPR = value.gpr();
3400 speculateString(node->child1(), valueGPR);
3401
3402 flushRegisters();
3403 JSValueRegsFlushedCallResult result(this);
3404 JSValueRegs resultRegs = result.regs();
3405 callOperation(operationParseIntStringNoRadix, resultRegs, valueGPR);
3406 m_jit.exceptionCheck();
3407 jsValueResult(resultRegs, node);
3408}
3409
3410void SpeculativeJIT::compileOverridesHasInstance(Node* node)
3411{
3412 Node* hasInstanceValueNode = node->child2().node();
3413 JSFunction* defaultHasInstanceFunction = jsCast<JSFunction*>(node->cellOperand()->value());
3414
3415 MacroAssembler::JumpList notDefault;
3416 SpeculateCellOperand base(this, node->child1());
3417 JSValueOperand hasInstanceValue(this, node->child2());
3418 GPRTemporary result(this);
3419
3420 GPRReg baseGPR = base.gpr();
3421 GPRReg resultGPR = result.gpr();
3422
3423 // It would be great if constant folding handled automatically the case where we knew the hasInstance function
3424 // was a constant. Unfortunately, the folding rule for OverridesHasInstance is in the strength reduction phase
3425 // since it relies on OSR information. https://bugs.webkit.org/show_bug.cgi?id=154832
3426 if (!hasInstanceValueNode->isCellConstant() || defaultHasInstanceFunction != hasInstanceValueNode->asCell()) {
3427 JSValueRegs hasInstanceValueRegs = hasInstanceValue.jsValueRegs();
3428#if USE(JSVALUE64)
3429 notDefault.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.gpr(), TrustedImmPtr(node->cellOperand())));
3430#else
3431 notDefault.append(m_jit.branchIfNotCell(hasInstanceValueRegs));
3432 notDefault.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.payloadGPR(), TrustedImmPtr(node->cellOperand())));
3433#endif
3434 }
3435
3436 // Check that base 'ImplementsDefaultHasInstance'.
3437 m_jit.test8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance), resultGPR);
3438 MacroAssembler::Jump done = m_jit.jump();
3439
3440 if (!notDefault.empty()) {
3441 notDefault.link(&m_jit);
3442 m_jit.move(TrustedImm32(1), resultGPR);
3443 }
3444
3445 done.link(&m_jit);
3446 unblessedBooleanResult(resultGPR, node);
3447}
3448
3449void SpeculativeJIT::compileInstanceOfForCells(Node* node, JSValueRegs valueRegs, JSValueRegs prototypeRegs, GPRReg resultGPR, GPRReg scratchGPR, GPRReg scratch2GPR, JITCompiler::Jump slowCase)
3450{
3451 CallSiteIndex callSiteIndex = m_jit.addCallSite(node->origin.semantic);
3452
3453 JITInstanceOfGenerator gen(
3454 m_jit.codeBlock(), node->origin.semantic, callSiteIndex, usedRegisters(), resultGPR,
3455 valueRegs.payloadGPR(), prototypeRegs.payloadGPR(), scratchGPR, scratch2GPR,
3456 m_state.forNode(node->child2()).isType(SpecObject | ~SpecCell));
3457 gen.generateFastPath(m_jit);
3458
3459 JITCompiler::JumpList slowCases;
3460 slowCases.append(slowCase);
3461
3462 std::unique_ptr<SlowPathGenerator> slowPath = slowPathCall(
3463 slowCases, this, operationInstanceOfOptimize, resultGPR, gen.stubInfo(), valueRegs,
3464 prototypeRegs);
3465
3466 m_jit.addInstanceOf(gen, slowPath.get());
3467 addSlowPathGenerator(WTFMove(slowPath));
3468}
3469
3470void SpeculativeJIT::compileInstanceOf(Node* node)
3471{
3472#if USE(JSVALUE64)
3473 if (node->child1().useKind() == CellUse
3474 && node->child2().useKind() == CellUse) {
3475 SpeculateCellOperand value(this, node->child1());
3476 SpeculateCellOperand prototype(this, node->child2());
3477
3478 GPRTemporary result(this);
3479 GPRTemporary scratch(this);
3480 GPRTemporary scratch2(this);
3481
3482 GPRReg valueGPR = value.gpr();
3483 GPRReg prototypeGPR = prototype.gpr();
3484 GPRReg resultGPR = result.gpr();
3485 GPRReg scratchGPR = scratch.gpr();
3486 GPRReg scratch2GPR = scratch2.gpr();
3487
3488 compileInstanceOfForCells(node, JSValueRegs(valueGPR), JSValueRegs(prototypeGPR), resultGPR, scratchGPR, scratch2GPR);
3489
3490 blessedBooleanResult(resultGPR, node);
3491 return;
3492 }
3493#endif
3494
3495 DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse);
3496 DFG_ASSERT(m_jit.graph(), node, node->child2().useKind() == UntypedUse);
3497
3498 JSValueOperand value(this, node->child1());
3499 JSValueOperand prototype(this, node->child2());
3500
3501 GPRTemporary result(this);
3502 GPRTemporary scratch(this);
3503
3504 JSValueRegs valueRegs = value.jsValueRegs();
3505 JSValueRegs prototypeRegs = prototype.jsValueRegs();
3506
3507 GPRReg resultGPR = result.gpr();
3508 GPRReg scratchGPR = scratch.gpr();
3509
3510 JITCompiler::Jump isCell = m_jit.branchIfCell(valueRegs);
3511 moveFalseTo(resultGPR);
3512
3513 JITCompiler::Jump done = m_jit.jump();
3514
3515 isCell.link(&m_jit);
3516
3517 JITCompiler::Jump slowCase = m_jit.branchIfNotCell(prototypeRegs);
3518
3519 compileInstanceOfForCells(node, valueRegs, prototypeRegs, resultGPR, scratchGPR, InvalidGPRReg, slowCase);
3520
3521 done.link(&m_jit);
3522 blessedBooleanResult(resultGPR, node);
3523 return;
3524}
3525
3526void SpeculativeJIT::compileValueBitNot(Node* node)
3527{
3528 Edge& child1 = node->child1();
3529
3530 if (child1.useKind() == BigIntUse) {
3531 SpeculateCellOperand operand(this, child1);
3532 GPRReg operandGPR = operand.gpr();
3533
3534 speculateBigInt(child1, operandGPR);
3535
3536 flushRegisters();
3537 GPRFlushedCallResult result(this);
3538 GPRReg resultGPR = result.gpr();
3539
3540 callOperation(operationBitNotBigInt, resultGPR, operandGPR);
3541 m_jit.exceptionCheck();
3542 cellResult(resultGPR, node);
3543
3544 return;
3545 }
3546
3547 JSValueOperand operand(this, child1);
3548 JSValueRegs operandRegs = operand.jsValueRegs();
3549
3550 flushRegisters();
3551 JSValueRegsFlushedCallResult result(this);
3552 JSValueRegs resultRegs = result.regs();
3553 callOperation(operationValueBitNot, resultRegs, operandRegs);
3554 m_jit.exceptionCheck();
3555
3556 jsValueResult(resultRegs, node);
3557}
3558
3559void SpeculativeJIT::compileBitwiseNot(Node* node)
3560{
3561 Edge& child1 = node->child1();
3562
3563 SpeculateInt32Operand operand(this, child1);
3564 GPRTemporary result(this);
3565 GPRReg resultGPR = result.gpr();
3566
3567 m_jit.move(operand.gpr(), resultGPR);
3568
3569 m_jit.not32(resultGPR);
3570
3571 int32Result(resultGPR, node);
3572}
3573
3574template<typename SnippetGenerator, J_JITOperation_EJJ snippetSlowPathFunction>
3575void SpeculativeJIT::emitUntypedBitOp(Node* node)
3576{
3577 Edge& leftChild = node->child1();
3578 Edge& rightChild = node->child2();
3579
3580 if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3581 JSValueOperand left(this, leftChild);
3582 JSValueOperand right(this, rightChild);
3583 JSValueRegs leftRegs = left.jsValueRegs();
3584 JSValueRegs rightRegs = right.jsValueRegs();
3585
3586 flushRegisters();
3587 JSValueRegsFlushedCallResult result(this);
3588 JSValueRegs resultRegs = result.regs();
3589 callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3590 m_jit.exceptionCheck();
3591
3592 jsValueResult(resultRegs, node);
3593 return;
3594 }
3595
3596 Optional<JSValueOperand> left;
3597 Optional<JSValueOperand> right;
3598
3599 JSValueRegs leftRegs;
3600 JSValueRegs rightRegs;
3601
3602#if USE(JSVALUE64)
3603 GPRTemporary result(this);
3604 JSValueRegs resultRegs = JSValueRegs(result.gpr());
3605 GPRTemporary scratch(this);
3606 GPRReg scratchGPR = scratch.gpr();
3607#else
3608 GPRTemporary resultTag(this);
3609 GPRTemporary resultPayload(this);
3610 JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3611 GPRReg scratchGPR = resultTag.gpr();
3612#endif
3613
3614 SnippetOperand leftOperand;
3615 SnippetOperand rightOperand;
3616
3617 // The snippet generator does not support both operands being constant. If the left
3618 // operand is already const, we'll ignore the right operand's constness.
3619 if (leftChild->isInt32Constant())
3620 leftOperand.setConstInt32(leftChild->asInt32());
3621 else if (rightChild->isInt32Constant())
3622 rightOperand.setConstInt32(rightChild->asInt32());
3623
3624 RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3625
3626 if (!leftOperand.isConst()) {
3627 left.emplace(this, leftChild);
3628 leftRegs = left->jsValueRegs();
3629 }
3630 if (!rightOperand.isConst()) {
3631 right.emplace(this, rightChild);
3632 rightRegs = right->jsValueRegs();
3633 }
3634
3635 SnippetGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, scratchGPR);
3636 gen.generateFastPath(m_jit);
3637
3638 ASSERT(gen.didEmitFastPath());
3639 gen.endJumpList().append(m_jit.jump());
3640
3641 gen.slowPathJumpList().link(&m_jit);
3642 silentSpillAllRegisters(resultRegs);
3643
3644 if (leftOperand.isConst()) {
3645 leftRegs = resultRegs;
3646 m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3647 } else if (rightOperand.isConst()) {
3648 rightRegs = resultRegs;
3649 m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3650 }
3651
3652 callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3653
3654 silentFillAllRegisters();
3655 m_jit.exceptionCheck();
3656
3657 gen.endJumpList().link(&m_jit);
3658 jsValueResult(resultRegs, node);
3659}
3660
3661void SpeculativeJIT::compileValueBitwiseOp(Node* node)
3662{
3663 NodeType op = node->op();
3664 Edge& leftChild = node->child1();
3665 Edge& rightChild = node->child2();
3666
3667 if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3668 switch (op) {
3669 case ValueBitAnd:
3670 emitUntypedBitOp<JITBitAndGenerator, operationValueBitAnd>(node);
3671 return;
3672 case ValueBitXor:
3673 emitUntypedBitOp<JITBitXorGenerator, operationValueBitXor>(node);
3674 return;
3675 case ValueBitOr:
3676 emitUntypedBitOp<JITBitOrGenerator, operationValueBitOr>(node);
3677 return;
3678 default:
3679 RELEASE_ASSERT_NOT_REACHED();
3680 }
3681 }
3682
3683 ASSERT(leftChild.useKind() == BigIntUse && rightChild.useKind() == BigIntUse);
3684
3685 SpeculateCellOperand left(this, node->child1());
3686 SpeculateCellOperand right(this, node->child2());
3687 GPRReg leftGPR = left.gpr();
3688 GPRReg rightGPR = right.gpr();
3689
3690 speculateBigInt(leftChild, leftGPR);
3691 speculateBigInt(rightChild, rightGPR);
3692
3693 flushRegisters();
3694 GPRFlushedCallResult result(this);
3695 GPRReg resultGPR = result.gpr();
3696
3697 switch (op) {
3698 case ValueBitAnd:
3699 callOperation(operationBitAndBigInt, resultGPR, leftGPR, rightGPR);
3700 break;
3701 case ValueBitXor:
3702 callOperation(operationBitXorBigInt, resultGPR, leftGPR, rightGPR);
3703 break;
3704 case ValueBitOr:
3705 callOperation(operationBitOrBigInt, resultGPR, leftGPR, rightGPR);
3706 break;
3707 default:
3708 RELEASE_ASSERT_NOT_REACHED();
3709 }
3710
3711 m_jit.exceptionCheck();
3712 cellResult(resultGPR, node);
3713}
3714
3715void SpeculativeJIT::compileBitwiseOp(Node* node)
3716{
3717 NodeType op = node->op();
3718 Edge& leftChild = node->child1();
3719 Edge& rightChild = node->child2();
3720
3721 if (leftChild->isInt32Constant()) {
3722 SpeculateInt32Operand op2(this, rightChild);
3723 GPRTemporary result(this, Reuse, op2);
3724
3725 bitOp(op, leftChild->asInt32(), op2.gpr(), result.gpr());
3726
3727 int32Result(result.gpr(), node);
3728 return;
3729 }
3730
3731 if (rightChild->isInt32Constant()) {
3732 SpeculateInt32Operand op1(this, leftChild);
3733 GPRTemporary result(this, Reuse, op1);
3734
3735 bitOp(op, rightChild->asInt32(), op1.gpr(), result.gpr());
3736
3737 int32Result(result.gpr(), node);
3738 return;
3739 }
3740
3741 SpeculateInt32Operand op1(this, leftChild);
3742 SpeculateInt32Operand op2(this, rightChild);
3743 GPRTemporary result(this, Reuse, op1, op2);
3744
3745 GPRReg reg1 = op1.gpr();
3746 GPRReg reg2 = op2.gpr();
3747 bitOp(op, reg1, reg2, result.gpr());
3748
3749 int32Result(result.gpr(), node);
3750}
3751
3752void SpeculativeJIT::emitUntypedRightShiftBitOp(Node* node)
3753{
3754 J_JITOperation_EJJ snippetSlowPathFunction = node->op() == BitRShift
3755 ? operationValueBitRShift : operationValueBitURShift;
3756 JITRightShiftGenerator::ShiftType shiftType = node->op() == BitRShift
3757 ? JITRightShiftGenerator::SignedShift : JITRightShiftGenerator::UnsignedShift;
3758
3759 Edge& leftChild = node->child1();
3760 Edge& rightChild = node->child2();
3761
3762 if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3763 JSValueOperand left(this, leftChild);
3764 JSValueOperand right(this, rightChild);
3765 JSValueRegs leftRegs = left.jsValueRegs();
3766 JSValueRegs rightRegs = right.jsValueRegs();
3767
3768 flushRegisters();
3769 JSValueRegsFlushedCallResult result(this);
3770 JSValueRegs resultRegs = result.regs();
3771 callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3772 m_jit.exceptionCheck();
3773
3774 jsValueResult(resultRegs, node);
3775 return;
3776 }
3777
3778 Optional<JSValueOperand> left;
3779 Optional<JSValueOperand> right;
3780
3781 JSValueRegs leftRegs;
3782 JSValueRegs rightRegs;
3783
3784 FPRTemporary leftNumber(this);
3785 FPRReg leftFPR = leftNumber.fpr();
3786
3787#if USE(JSVALUE64)
3788 GPRTemporary result(this);
3789 JSValueRegs resultRegs = JSValueRegs(result.gpr());
3790 GPRTemporary scratch(this);
3791 GPRReg scratchGPR = scratch.gpr();
3792 FPRReg scratchFPR = InvalidFPRReg;
3793#else
3794 GPRTemporary resultTag(this);
3795 GPRTemporary resultPayload(this);
3796 JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
3797 GPRReg scratchGPR = resultTag.gpr();
3798 FPRTemporary fprScratch(this);
3799 FPRReg scratchFPR = fprScratch.fpr();
3800#endif
3801
3802 SnippetOperand leftOperand;
3803 SnippetOperand rightOperand;
3804
3805 // The snippet generator does not support both operands being constant. If the left
3806 // operand is already const, we'll ignore the right operand's constness.
3807 if (leftChild->isInt32Constant())
3808 leftOperand.setConstInt32(leftChild->asInt32());
3809 else if (rightChild->isInt32Constant())
3810 rightOperand.setConstInt32(rightChild->asInt32());
3811
3812 RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
3813
3814 if (!leftOperand.isConst()) {
3815 left.emplace(this, leftChild);
3816 leftRegs = left->jsValueRegs();
3817 }
3818 if (!rightOperand.isConst()) {
3819 right.emplace(this, rightChild);
3820 rightRegs = right->jsValueRegs();
3821 }
3822
3823 JITRightShiftGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
3824 leftFPR, scratchGPR, scratchFPR, shiftType);
3825 gen.generateFastPath(m_jit);
3826
3827 ASSERT(gen.didEmitFastPath());
3828 gen.endJumpList().append(m_jit.jump());
3829
3830 gen.slowPathJumpList().link(&m_jit);
3831 silentSpillAllRegisters(resultRegs);
3832
3833 if (leftOperand.isConst()) {
3834 leftRegs = resultRegs;
3835 m_jit.moveValue(leftChild->asJSValue(), leftRegs);
3836 } else if (rightOperand.isConst()) {
3837 rightRegs = resultRegs;
3838 m_jit.moveValue(rightChild->asJSValue(), rightRegs);
3839 }
3840
3841 callOperation(snippetSlowPathFunction, resultRegs, leftRegs, rightRegs);
3842
3843 silentFillAllRegisters();
3844 m_jit.exceptionCheck();
3845
3846 gen.endJumpList().link(&m_jit);
3847 jsValueResult(resultRegs, node);
3848 return;
3849}
3850
3851void SpeculativeJIT::compileShiftOp(Node* node)
3852{
3853 NodeType op = node->op();
3854 Edge& leftChild = node->child1();
3855 Edge& rightChild = node->child2();
3856
3857 if (leftChild.useKind() == UntypedUse || rightChild.useKind() == UntypedUse) {
3858 switch (op) {
3859 case BitLShift:
3860 emitUntypedBitOp<JITLeftShiftGenerator, operationValueBitLShift>(node);
3861 return;
3862 case BitRShift:
3863 case BitURShift:
3864 emitUntypedRightShiftBitOp(node);
3865 return;
3866 default:
3867 RELEASE_ASSERT_NOT_REACHED();
3868 }
3869 }
3870
3871 if (rightChild->isInt32Constant()) {
3872 SpeculateInt32Operand op1(this, leftChild);
3873 GPRTemporary result(this, Reuse, op1);
3874
3875 shiftOp(op, op1.gpr(), rightChild->asInt32() & 0x1f, result.gpr());
3876
3877 int32Result(result.gpr(), node);
3878 } else {
3879 // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
3880 SpeculateInt32Operand op1(this, leftChild);
3881 SpeculateInt32Operand op2(this, rightChild);
3882 GPRTemporary result(this, Reuse, op1);
3883
3884 GPRReg reg1 = op1.gpr();
3885 GPRReg reg2 = op2.gpr();
3886 shiftOp(op, reg1, reg2, result.gpr());
3887
3888 int32Result(result.gpr(), node);
3889 }
3890}
3891
3892void SpeculativeJIT::compileValueAdd(Node* node)
3893{
3894 Edge& leftChild = node->child1();
3895 Edge& rightChild = node->child2();
3896
3897 if (node->isBinaryUseKind(BigIntUse)) {
3898 SpeculateCellOperand left(this, node->child1());
3899 SpeculateCellOperand right(this, node->child2());
3900 GPRReg leftGPR = left.gpr();
3901 GPRReg rightGPR = right.gpr();
3902
3903 speculateBigInt(leftChild, leftGPR);
3904 speculateBigInt(rightChild, rightGPR);
3905
3906 flushRegisters();
3907 GPRFlushedCallResult result(this);
3908 GPRReg resultGPR = result.gpr();
3909 callOperation(operationAddBigInt, resultGPR, leftGPR, rightGPR);
3910 m_jit.exceptionCheck();
3911
3912 cellResult(resultGPR, node);
3913 return;
3914 }
3915
3916 if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
3917 JSValueOperand left(this, leftChild);
3918 JSValueOperand right(this, rightChild);
3919 JSValueRegs leftRegs = left.jsValueRegs();
3920 JSValueRegs rightRegs = right.jsValueRegs();
3921
3922 flushRegisters();
3923 JSValueRegsFlushedCallResult result(this);
3924 JSValueRegs resultRegs = result.regs();
3925 callOperation(operationValueAddNotNumber, resultRegs, leftRegs, rightRegs);
3926 m_jit.exceptionCheck();
3927
3928 jsValueResult(resultRegs, node);
3929 return;
3930 }
3931
3932#if USE(JSVALUE64)
3933 bool needsScratchGPRReg = true;
3934 bool needsScratchFPRReg = false;
3935#else
3936 bool needsScratchGPRReg = true;
3937 bool needsScratchFPRReg = true;
3938#endif
3939
3940 CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
3941 unsigned bytecodeIndex = node->origin.semantic.bytecodeIndex();
3942 ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
3943 const Instruction* instruction = baselineCodeBlock->instructions().at(bytecodeIndex).ptr();
3944 JITAddIC* addIC = m_jit.codeBlock()->addJITAddIC(arithProfile, instruction);
3945 auto repatchingFunction = operationValueAddOptimize;
3946 auto nonRepatchingFunction = operationValueAdd;
3947
3948 compileMathIC(node, addIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3949}
3950
3951void SpeculativeJIT::compileValueSub(Node* node)
3952{
3953 Edge& leftChild = node->child1();
3954 Edge& rightChild = node->child2();
3955
3956 if (node->binaryUseKind() == UntypedUse) {
3957#if USE(JSVALUE64)
3958 bool needsScratchGPRReg = true;
3959 bool needsScratchFPRReg = false;
3960#else
3961 bool needsScratchGPRReg = true;
3962 bool needsScratchFPRReg = true;
3963#endif
3964
3965 CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
3966 unsigned bytecodeIndex = node->origin.semantic.bytecodeIndex();
3967 ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
3968 const Instruction* instruction = baselineCodeBlock->instructions().at(bytecodeIndex).ptr();
3969 JITSubIC* subIC = m_jit.codeBlock()->addJITSubIC(arithProfile, instruction);
3970 auto repatchingFunction = operationValueSubOptimize;
3971 auto nonRepatchingFunction = operationValueSub;
3972
3973 compileMathIC(node, subIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
3974 return;
3975 }
3976
3977 ASSERT(leftChild.useKind() == BigIntUse && rightChild.useKind() == BigIntUse);
3978
3979 SpeculateCellOperand left(this, node->child1());
3980 SpeculateCellOperand right(this, node->child2());
3981 GPRReg leftGPR = left.gpr();
3982 GPRReg rightGPR = right.gpr();
3983
3984 speculateBigInt(leftChild, leftGPR);
3985 speculateBigInt(rightChild, rightGPR);
3986
3987 flushRegisters();
3988 GPRFlushedCallResult result(this);
3989 GPRReg resultGPR = result.gpr();
3990
3991 callOperation(operationSubBigInt, resultGPR, leftGPR, rightGPR);
3992
3993 m_jit.exceptionCheck();
3994 cellResult(resultGPR, node);
3995}
3996
3997template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
3998void SpeculativeJIT::compileMathIC(Node* node, JITBinaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, bool needsScratchFPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
3999{
4000 Edge& leftChild = node->child1();
4001 Edge& rightChild = node->child2();
4002
4003 Optional<JSValueOperand> left;
4004 Optional<JSValueOperand> right;
4005
4006 JSValueRegs leftRegs;
4007 JSValueRegs rightRegs;
4008
4009 FPRTemporary leftNumber(this);
4010 FPRTemporary rightNumber(this);
4011 FPRReg leftFPR = leftNumber.fpr();
4012 FPRReg rightFPR = rightNumber.fpr();
4013
4014 GPRReg scratchGPR = InvalidGPRReg;
4015 FPRReg scratchFPR = InvalidFPRReg;
4016
4017 Optional<FPRTemporary> fprScratch;
4018 if (needsScratchFPRReg) {
4019 fprScratch.emplace(this);
4020 scratchFPR = fprScratch->fpr();
4021 }
4022
4023#if USE(JSVALUE64)
4024 Optional<GPRTemporary> gprScratch;
4025 if (needsScratchGPRReg) {
4026 gprScratch.emplace(this);
4027 scratchGPR = gprScratch->gpr();
4028 }
4029 GPRTemporary result(this);
4030 JSValueRegs resultRegs = JSValueRegs(result.gpr());
4031#else
4032 GPRTemporary resultTag(this);
4033 GPRTemporary resultPayload(this);
4034 JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
4035 if (needsScratchGPRReg)
4036 scratchGPR = resultRegs.tagGPR();
4037#endif
4038
4039 SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
4040 SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
4041
4042 // The snippet generator does not support both operands being constant. If the left
4043 // operand is already const, we'll ignore the right operand's constness.
4044 if (leftChild->isInt32Constant())
4045 leftOperand.setConstInt32(leftChild->asInt32());
4046 else if (rightChild->isInt32Constant())
4047 rightOperand.setConstInt32(rightChild->asInt32());
4048
4049 ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
4050 ASSERT(!(Generator::isLeftOperandValidConstant(leftOperand) && Generator::isRightOperandValidConstant(rightOperand)));
4051
4052 if (!Generator::isLeftOperandValidConstant(leftOperand)) {
4053 left.emplace(this, leftChild);
4054 leftRegs = left->jsValueRegs();
4055 }
4056 if (!Generator::isRightOperandValidConstant(rightOperand)) {
4057 right.emplace(this, rightChild);
4058 rightRegs = right->jsValueRegs();
4059 }
4060
4061#if ENABLE(MATH_IC_STATS)
4062 auto inlineStart = m_jit.label();
4063#endif
4064
4065 Box<MathICGenerationState> addICGenerationState = Box<MathICGenerationState>::create();
4066 mathIC->m_generator = Generator(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs, leftFPR, rightFPR, scratchGPR, scratchFPR);
4067
4068 bool shouldEmitProfiling = false;
4069 bool generatedInline = mathIC->generateInline(m_jit, *addICGenerationState, shouldEmitProfiling);
4070 if (generatedInline) {
4071 ASSERT(!addICGenerationState->slowPathJumps.empty());
4072
4073 Vector<SilentRegisterSavePlan> savePlans;
4074 silentSpillAllRegistersImpl(false, savePlans, resultRegs);
4075
4076 auto done = m_jit.label();
4077
4078 addSlowPathGeneratorLambda([=, savePlans = WTFMove(savePlans)] () {
4079 addICGenerationState->slowPathJumps.link(&m_jit);
4080 addICGenerationState->slowPathStart = m_jit.label();
4081#if ENABLE(MATH_IC_STATS)
4082 auto slowPathStart = m_jit.label();
4083#endif
4084
4085 silentSpill(savePlans);
4086
4087 auto innerLeftRegs = leftRegs;
4088 auto innerRightRegs = rightRegs;
4089 if (Generator::isLeftOperandValidConstant(leftOperand)) {
4090 innerLeftRegs = resultRegs;
4091 m_jit.moveValue(leftChild->asJSValue(), innerLeftRegs);
4092 } else if (Generator::isRightOperandValidConstant(rightOperand)) {
4093 innerRightRegs = resultRegs;
4094 m_jit.moveValue(rightChild->asJSValue(), innerRightRegs);
4095 }
4096
4097 if (addICGenerationState->shouldSlowPathRepatch)
4098 addICGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJJMic>(repatchingFunction), resultRegs, innerLeftRegs, innerRightRegs, TrustedImmPtr(mathIC));
4099 else
4100 addICGenerationState->slowPathCall = callOperation(nonRepatchingFunction, resultRegs, innerLeftRegs, innerRightRegs);
4101
4102 silentFill(savePlans);
4103 m_jit.exceptionCheck();
4104 m_jit.jump().linkTo(done, &m_jit);
4105
4106 m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4107 mathIC->finalizeInlineCode(*addICGenerationState, linkBuffer);
4108 });
4109
4110#if ENABLE(MATH_IC_STATS)
4111 auto slowPathEnd = m_jit.label();
4112 m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4113 size_t size = static_cast<char*>(linkBuffer.locationOf(slowPathEnd).executableAddress()) - static_cast<char*>(linkBuffer.locationOf(slowPathStart).executableAddress());
4114 mathIC->m_generatedCodeSize += size;
4115 });
4116#endif
4117
4118 });
4119 } else {
4120 if (Generator::isLeftOperandValidConstant(leftOperand)) {
4121 left.emplace(this, leftChild);
4122 leftRegs = left->jsValueRegs();
4123 } else if (Generator::isRightOperandValidConstant(rightOperand)) {
4124 right.emplace(this, rightChild);
4125 rightRegs = right->jsValueRegs();
4126 }
4127
4128 flushRegisters();
4129 callOperation(nonRepatchingFunction, resultRegs, leftRegs, rightRegs);
4130 m_jit.exceptionCheck();
4131 }
4132
4133#if ENABLE(MATH_IC_STATS)
4134 auto inlineEnd = m_jit.label();
4135 m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4136 size_t size = static_cast<char*>(linkBuffer.locationOf(inlineEnd).executableAddress()) - static_cast<char*>(linkBuffer.locationOf(inlineStart).executableAddress());
4137 mathIC->m_generatedCodeSize += size;
4138 });
4139#endif
4140
4141 jsValueResult(resultRegs, node);
4142 return;
4143}
4144
4145void SpeculativeJIT::compileInstanceOfCustom(Node* node)
4146{
4147 // We could do something smarter here but this case is currently super rare and unless
4148 // Symbol.hasInstance becomes popular will likely remain that way.
4149
4150 JSValueOperand value(this, node->child1());
4151 SpeculateCellOperand constructor(this, node->child2());
4152 JSValueOperand hasInstanceValue(this, node->child3());
4153 GPRTemporary result(this);
4154
4155 JSValueRegs valueRegs = value.jsValueRegs();
4156 GPRReg constructorGPR = constructor.gpr();
4157 JSValueRegs hasInstanceRegs = hasInstanceValue.jsValueRegs();
4158 GPRReg resultGPR = result.gpr();
4159
4160 MacroAssembler::Jump slowCase = m_jit.jump();
4161
4162 addSlowPathGenerator(slowPathCall(slowCase, this, operationInstanceOfCustom, resultGPR, valueRegs, constructorGPR, hasInstanceRegs));
4163
4164 unblessedBooleanResult(resultGPR, node);
4165}
4166
4167void SpeculativeJIT::compileIsCellWithType(Node* node)
4168{
4169 switch (node->child1().useKind()) {
4170 case UntypedUse: {
4171 JSValueOperand value(this, node->child1());
4172 GPRTemporary result(this, Reuse, value, PayloadWord);
4173
4174 JSValueRegs valueRegs = value.jsValueRegs();
4175 GPRReg resultGPR = result.gpr();
4176
4177 JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(valueRegs);
4178
4179 m_jit.compare8(JITCompiler::Equal,
4180 JITCompiler::Address(valueRegs.payloadGPR(), JSCell::typeInfoTypeOffset()),
4181 TrustedImm32(node->queriedType()),
4182 resultGPR);
4183 blessBoolean(resultGPR);
4184 JITCompiler::Jump done = m_jit.jump();
4185
4186 isNotCell.link(&m_jit);
4187 moveFalseTo(resultGPR);
4188
4189 done.link(&m_jit);
4190 blessedBooleanResult(resultGPR, node);
4191 return;
4192 }
4193
4194 case CellUse: {
4195 SpeculateCellOperand cell(this, node->child1());
4196 GPRTemporary result(this, Reuse, cell);
4197
4198 GPRReg cellGPR = cell.gpr();
4199 GPRReg resultGPR = result.gpr();
4200
4201 m_jit.compare8(JITCompiler::Equal,
4202 JITCompiler::Address(cellGPR, JSCell::typeInfoTypeOffset()),
4203 TrustedImm32(node->queriedType()),
4204 resultGPR);
4205 blessBoolean(resultGPR);
4206 blessedBooleanResult(resultGPR, node);
4207 return;
4208 }
4209
4210 default:
4211 RELEASE_ASSERT_NOT_REACHED();
4212 break;
4213 }
4214}
4215
4216void SpeculativeJIT::compileIsTypedArrayView(Node* node)
4217{
4218 JSValueOperand value(this, node->child1());
4219 GPRTemporary result(this, Reuse, value, PayloadWord);
4220
4221 JSValueRegs valueRegs = value.jsValueRegs();
4222 GPRReg resultGPR = result.gpr();
4223
4224 JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(valueRegs);
4225
4226 m_jit.load8(JITCompiler::Address(valueRegs.payloadGPR(), JSCell::typeInfoTypeOffset()), resultGPR);
4227 m_jit.sub32(TrustedImm32(FirstTypedArrayType), resultGPR);
4228 m_jit.compare32(JITCompiler::Below,
4229 resultGPR,
4230 TrustedImm32(NumberOfTypedArrayTypesExcludingDataView),
4231 resultGPR);
4232 blessBoolean(resultGPR);
4233 JITCompiler::Jump done = m_jit.jump();
4234
4235 isNotCell.link(&m_jit);
4236 moveFalseTo(resultGPR);
4237
4238 done.link(&m_jit);
4239 blessedBooleanResult(resultGPR, node);
4240}
4241
4242void SpeculativeJIT::compileToObjectOrCallObjectConstructor(Node* node)
4243{
4244 RELEASE_ASSERT(node->child1().useKind() == UntypedUse);
4245
4246 JSValueOperand value(this, node->child1());
4247 GPRTemporary result(this, Reuse, value, PayloadWord);
4248
4249 JSValueRegs valueRegs = value.jsValueRegs();
4250 GPRReg resultGPR = result.gpr();
4251
4252 MacroAssembler::JumpList slowCases;
4253 slowCases.append(m_jit.branchIfNotCell(valueRegs));
4254 slowCases.append(m_jit.branchIfNotObject(valueRegs.payloadGPR()));
4255 m_jit.move(valueRegs.payloadGPR(), resultGPR);
4256
4257 if (node->op() == ToObject)
4258 addSlowPathGenerator(slowPathCall(slowCases, this, operationToObject, resultGPR, m_jit.graph().globalObjectFor(node->origin.semantic), valueRegs, identifierUID(node->identifierNumber())));
4259 else
4260 addSlowPathGenerator(slowPathCall(slowCases, this, operationCallObjectConstructor, resultGPR, TrustedImmPtr(node->cellOperand()), valueRegs));
4261
4262 cellResult(resultGPR, node);
4263}
4264
4265void SpeculativeJIT::compileArithAdd(Node* node)
4266{
4267 switch (node->binaryUseKind()) {
4268 case Int32Use: {
4269 ASSERT(!shouldCheckNegativeZero(node->arithMode()));
4270
4271 if (node->child2()->isInt32Constant()) {
4272 SpeculateInt32Operand op1(this, node->child1());
4273 GPRTemporary result(this, Reuse, op1);
4274
4275 GPRReg gpr1 = op1.gpr();
4276 int32_t imm2 = node->child2()->asInt32();
4277 GPRReg gprResult = result.gpr();
4278
4279 if (!shouldCheckOverflow(node->arithMode())) {
4280 m_jit.add32(Imm32(imm2), gpr1, gprResult);
4281 int32Result(gprResult, node);
4282 return;
4283 }
4284
4285 MacroAssembler::Jump check = m_jit.branchAdd32(MacroAssembler::Overflow, gpr1, Imm32(imm2), gprResult);
4286 if (gpr1 == gprResult) {
4287 speculationCheck(Overflow, JSValueRegs(), 0, check,
4288 SpeculationRecovery(SpeculativeAddImmediate, gpr1, imm2));
4289 } else
4290 speculationCheck(Overflow, JSValueRegs(), 0, check);
4291
4292 int32Result(gprResult, node);
4293 return;
4294 }
4295
4296 SpeculateInt32Operand op1(this, node->child1());
4297 SpeculateInt32Operand op2(this, node->child2());
4298 GPRTemporary result(this, Reuse, op1, op2);
4299
4300 GPRReg gpr1 = op1.gpr();
4301 GPRReg gpr2 = op2.gpr();
4302 GPRReg gprResult = result.gpr();
4303
4304 if (!shouldCheckOverflow(node->arithMode()))
4305 m_jit.add32(gpr1, gpr2, gprResult);
4306 else {
4307 MacroAssembler::Jump check = m_jit.branchAdd32(MacroAssembler::Overflow, gpr1, gpr2, gprResult);
4308
4309 if (gpr1 == gprResult && gpr2 == gprResult)
4310 speculationCheck(Overflow, JSValueRegs(), 0, check, SpeculationRecovery(SpeculativeAddSelf, gprResult, gpr2));
4311 else if (gpr1 == gprResult)
4312 speculationCheck(Overflow, JSValueRegs(), 0, check, SpeculationRecovery(SpeculativeAdd, gprResult, gpr2));
4313 else if (gpr2 == gprResult)
4314 speculationCheck(Overflow, JSValueRegs(), 0, check, SpeculationRecovery(SpeculativeAdd, gprResult, gpr1));
4315 else
4316 speculationCheck(Overflow, JSValueRegs(), 0, check);
4317 }
4318
4319 int32Result(gprResult, node);
4320 return;
4321 }
4322
4323#if USE(JSVALUE64)
4324 case Int52RepUse: {
4325 ASSERT(shouldCheckOverflow(node->arithMode()));
4326 ASSERT(!shouldCheckNegativeZero(node->arithMode()));
4327
4328 // Will we need an overflow check? If we can prove that neither input can be
4329 // Int52 then the overflow check will not be necessary.
4330 if (!m_state.forNode(node->child1()).couldBeType(SpecNonInt32AsInt52)
4331 && !m_state.forNode(node->child2()).couldBeType(SpecNonInt32AsInt52)) {
4332 SpeculateWhicheverInt52Operand op1(this, node->child1());
4333 SpeculateWhicheverInt52Operand op2(this, node->child2(), op1);
4334 GPRTemporary result(this, Reuse, op1);
4335 m_jit.add64(op1.gpr(), op2.gpr(), result.gpr());
4336 int52Result(result.gpr(), node, op1.format());
4337 return;
4338 }
4339
4340 SpeculateInt52Operand op1(this, node->child1());
4341 SpeculateInt52Operand op2(this, node->child2());
4342 GPRTemporary result(this);
4343 m_jit.move(op1.gpr(), result.gpr());
4344 speculationCheck(
4345 Int52Overflow, JSValueRegs(), 0,
4346 m_jit.branchAdd64(MacroAssembler::Overflow, op2.gpr(), result.gpr()));
4347 int52Result(result.gpr(), node);
4348 return;
4349 }
4350#endif // USE(JSVALUE64)
4351
4352 case DoubleRepUse: {
4353 SpeculateDoubleOperand op1(this, node->child1());
4354 SpeculateDoubleOperand op2(this, node->child2());
4355 FPRTemporary result(this, op1, op2);
4356
4357 FPRReg reg1 = op1.fpr();
4358 FPRReg reg2 = op2.fpr();
4359 m_jit.addDouble(reg1, reg2, result.fpr());
4360
4361 doubleResult(result.fpr(), node);
4362 return;
4363 }
4364
4365 default:
4366 RELEASE_ASSERT_NOT_REACHED();
4367 break;
4368 }
4369}
4370
4371void SpeculativeJIT::compileArithAbs(Node* node)
4372{
4373 switch (node->child1().useKind()) {
4374 case Int32Use: {
4375 SpeculateStrictInt32Operand op1(this, node->child1());
4376 GPRTemporary result(this, Reuse, op1);
4377 GPRTemporary scratch(this);
4378
4379 m_jit.move(op1.gpr(), result.gpr());
4380 m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr());
4381 m_jit.add32(scratch.gpr(), result.gpr());
4382 m_jit.xor32(scratch.gpr(), result.gpr());
4383 if (shouldCheckOverflow(node->arithMode()))
4384 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Signed, result.gpr()));
4385 int32Result(result.gpr(), node);
4386 break;
4387 }
4388
4389 case DoubleRepUse: {
4390 SpeculateDoubleOperand op1(this, node->child1());
4391 FPRTemporary result(this);
4392
4393 m_jit.absDouble(op1.fpr(), result.fpr());
4394 doubleResult(result.fpr(), node);
4395 break;
4396 }
4397
4398 default: {
4399 DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse, node->child1().useKind());
4400 JSValueOperand op1(this, node->child1());
4401 JSValueRegs op1Regs = op1.jsValueRegs();
4402 flushRegisters();
4403 FPRResult result(this);
4404 callOperation(operationArithAbs, result.fpr(), op1Regs);
4405 m_jit.exceptionCheck();
4406 doubleResult(result.fpr(), node);
4407 break;
4408 }
4409 }
4410}
4411
4412void SpeculativeJIT::compileArithClz32(Node* node)
4413{
4414 if (node->child1().useKind() == Int32Use || node->child1().useKind() == KnownInt32Use) {
4415 SpeculateInt32Operand value(this, node->child1());
4416 GPRTemporary result(this, Reuse, value);
4417 GPRReg valueReg = value.gpr();
4418 GPRReg resultReg = result.gpr();
4419 m_jit.countLeadingZeros32(valueReg, resultReg);
4420 int32Result(resultReg, node);
4421 return;
4422 }
4423 JSValueOperand op1(this, node->child1());
4424 JSValueRegs op1Regs = op1.jsValueRegs();
4425 GPRTemporary result(this);
4426 GPRReg resultReg = result.gpr();
4427 flushRegisters();
4428 callOperation(operationArithClz32, resultReg, op1Regs);
4429 m_jit.exceptionCheck();
4430 int32Result(resultReg, node);
4431}
4432
4433void SpeculativeJIT::compileArithDoubleUnaryOp(Node* node, double (*doubleFunction)(double), double (*operation)(ExecState*, EncodedJSValue))
4434{
4435 if (node->child1().useKind() == DoubleRepUse) {
4436 SpeculateDoubleOperand op1(this, node->child1());
4437 FPRReg op1FPR = op1.fpr();
4438
4439 flushRegisters();
4440
4441 FPRResult result(this);
4442 callOperation(doubleFunction, result.fpr(), op1FPR);
4443
4444 doubleResult(result.fpr(), node);
4445 return;
4446 }
4447
4448 JSValueOperand op1(this, node->child1());
4449 JSValueRegs op1Regs = op1.jsValueRegs();
4450 flushRegisters();
4451 FPRResult result(this);
4452 callOperation(operation, result.fpr(), op1Regs);
4453 m_jit.exceptionCheck();
4454 doubleResult(result.fpr(), node);
4455}
4456
4457void SpeculativeJIT::compileArithSub(Node* node)
4458{
4459 switch (node->binaryUseKind()) {
4460 case Int32Use: {
4461 ASSERT(!shouldCheckNegativeZero(node->arithMode()));
4462
4463 if (node->child2()->isInt32Constant()) {
4464 SpeculateInt32Operand op1(this, node->child1());
4465 int32_t imm2 = node->child2()->asInt32();
4466 GPRTemporary result(this);
4467
4468 if (!shouldCheckOverflow(node->arithMode())) {
4469 m_jit.move(op1.gpr(), result.gpr());
4470 m_jit.sub32(Imm32(imm2), result.gpr());
4471 } else {
4472 GPRTemporary scratch(this);
4473 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), Imm32(imm2), result.gpr(), scratch.gpr()));
4474 }
4475
4476 int32Result(result.gpr(), node);
4477 return;
4478 }
4479
4480 if (node->child1()->isInt32Constant()) {
4481 int32_t imm1 = node->child1()->asInt32();
4482 SpeculateInt32Operand op2(this, node->child2());
4483 GPRTemporary result(this);
4484
4485 m_jit.move(Imm32(imm1), result.gpr());
4486 if (!shouldCheckOverflow(node->arithMode()))
4487 m_jit.sub32(op2.gpr(), result.gpr());
4488 else
4489 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op2.gpr(), result.gpr()));
4490
4491 int32Result(result.gpr(), node);
4492 return;
4493 }
4494
4495 SpeculateInt32Operand op1(this, node->child1());
4496 SpeculateInt32Operand op2(this, node->child2());
4497 GPRTemporary result(this);
4498
4499 if (!shouldCheckOverflow(node->arithMode())) {
4500 m_jit.move(op1.gpr(), result.gpr());
4501 m_jit.sub32(op2.gpr(), result.gpr());
4502 } else
4503 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchSub32(MacroAssembler::Overflow, op1.gpr(), op2.gpr(), result.gpr()));
4504
4505 int32Result(result.gpr(), node);
4506 return;
4507 }
4508
4509#if USE(JSVALUE64)
4510 case Int52RepUse: {
4511 ASSERT(shouldCheckOverflow(node->arithMode()));
4512 ASSERT(!shouldCheckNegativeZero(node->arithMode()));
4513
4514 // Will we need an overflow check? If we can prove that neither input can be
4515 // Int52 then the overflow check will not be necessary.
4516 if (!m_state.forNode(node->child1()).couldBeType(SpecNonInt32AsInt52)
4517 && !m_state.forNode(node->child2()).couldBeType(SpecNonInt32AsInt52)) {
4518 SpeculateWhicheverInt52Operand op1(this, node->child1());
4519 SpeculateWhicheverInt52Operand op2(this, node->child2(), op1);
4520 GPRTemporary result(this, Reuse, op1);
4521 m_jit.move(op1.gpr(), result.gpr());
4522 m_jit.sub64(op2.gpr(), result.gpr());
4523 int52Result(result.gpr(), node, op1.format());
4524 return;
4525 }
4526
4527 SpeculateInt52Operand op1(this, node->child1());
4528 SpeculateInt52Operand op2(this, node->child2());
4529 GPRTemporary result(this);
4530 m_jit.move(op1.gpr(), result.gpr());
4531 speculationCheck(
4532 Int52Overflow, JSValueRegs(), 0,
4533 m_jit.branchSub64(MacroAssembler::Overflow, op2.gpr(), result.gpr()));
4534 int52Result(result.gpr(), node);
4535 return;
4536 }
4537#endif // USE(JSVALUE64)
4538
4539 case DoubleRepUse: {
4540 SpeculateDoubleOperand op1(this, node->child1());
4541 SpeculateDoubleOperand op2(this, node->child2());
4542 FPRTemporary result(this, op1);
4543
4544 FPRReg reg1 = op1.fpr();
4545 FPRReg reg2 = op2.fpr();
4546 m_jit.subDouble(reg1, reg2, result.fpr());
4547
4548 doubleResult(result.fpr(), node);
4549 return;
4550 }
4551
4552 default:
4553 RELEASE_ASSERT_NOT_REACHED();
4554 return;
4555 }
4556}
4557
4558void SpeculativeJIT::compileValueNegate(Node* node)
4559{
4560 CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
4561 unsigned bytecodeIndex = node->origin.semantic.bytecodeIndex();
4562 ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
4563 const Instruction* instruction = baselineCodeBlock->instructions().at(bytecodeIndex).ptr();
4564 JITNegIC* negIC = m_jit.codeBlock()->addJITNegIC(arithProfile, instruction);
4565 auto repatchingFunction = operationArithNegateOptimize;
4566 auto nonRepatchingFunction = operationArithNegate;
4567 bool needsScratchGPRReg = true;
4568 compileMathIC(node, negIC, needsScratchGPRReg, repatchingFunction, nonRepatchingFunction);
4569}
4570
4571void SpeculativeJIT::compileArithNegate(Node* node)
4572{
4573 switch (node->child1().useKind()) {
4574 case Int32Use: {
4575 SpeculateInt32Operand op1(this, node->child1());
4576 GPRTemporary result(this);
4577
4578 m_jit.move(op1.gpr(), result.gpr());
4579
4580 // Note: there is no notion of being not used as a number, but someone
4581 // caring about negative zero.
4582
4583 if (!shouldCheckOverflow(node->arithMode()))
4584 m_jit.neg32(result.gpr());
4585 else if (!shouldCheckNegativeZero(node->arithMode()))
4586 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchNeg32(MacroAssembler::Overflow, result.gpr()));
4587 else {
4588 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Zero, result.gpr(), TrustedImm32(0x7fffffff)));
4589 m_jit.neg32(result.gpr());
4590 }
4591
4592 int32Result(result.gpr(), node);
4593 return;
4594 }
4595
4596#if USE(JSVALUE64)
4597 case Int52RepUse: {
4598 ASSERT(shouldCheckOverflow(node->arithMode()));
4599
4600 if (!m_state.forNode(node->child1()).couldBeType(SpecNonInt32AsInt52)) {
4601 SpeculateWhicheverInt52Operand op1(this, node->child1());
4602 GPRTemporary result(this);
4603 GPRReg op1GPR = op1.gpr();
4604 GPRReg resultGPR = result.gpr();
4605 m_jit.move(op1GPR, resultGPR);
4606 m_jit.neg64(resultGPR);
4607 if (shouldCheckNegativeZero(node->arithMode())) {
4608 speculationCheck(
4609 NegativeZero, JSValueRegs(), 0,
4610 m_jit.branchTest64(MacroAssembler::Zero, resultGPR));
4611 }
4612 int52Result(resultGPR, node, op1.format());
4613 return;
4614 }
4615
4616 SpeculateInt52Operand op1(this, node->child1());
4617 GPRTemporary result(this);
4618 GPRReg op1GPR = op1.gpr();
4619 GPRReg resultGPR = result.gpr();
4620 m_jit.move(op1GPR, resultGPR);
4621 speculationCheck(
4622 Int52Overflow, JSValueRegs(), 0,
4623 m_jit.branchNeg64(MacroAssembler::Overflow, resultGPR));
4624 if (shouldCheckNegativeZero(node->arithMode())) {
4625 speculationCheck(
4626 NegativeZero, JSValueRegs(), 0,
4627 m_jit.branchTest64(MacroAssembler::Zero, resultGPR));
4628 }
4629 int52Result(resultGPR, node);
4630 return;
4631 }
4632#endif // USE(JSVALUE64)
4633
4634 case DoubleRepUse: {
4635 SpeculateDoubleOperand op1(this, node->child1());
4636 FPRTemporary result(this);
4637
4638 m_jit.negateDouble(op1.fpr(), result.fpr());
4639
4640 doubleResult(result.fpr(), node);
4641 return;
4642 }
4643
4644 default: {
4645 RELEASE_ASSERT_NOT_REACHED();
4646 }
4647 }
4648}
4649
4650template <typename Generator, typename RepatchingFunction, typename NonRepatchingFunction>
4651void SpeculativeJIT::compileMathIC(Node* node, JITUnaryMathIC<Generator>* mathIC, bool needsScratchGPRReg, RepatchingFunction repatchingFunction, NonRepatchingFunction nonRepatchingFunction)
4652{
4653 GPRReg scratchGPR = InvalidGPRReg;
4654 Optional<GPRTemporary> gprScratch;
4655 if (needsScratchGPRReg) {
4656 gprScratch.emplace(this);
4657 scratchGPR = gprScratch->gpr();
4658 }
4659 JSValueOperand childOperand(this, node->child1());
4660 JSValueRegs childRegs = childOperand.jsValueRegs();
4661#if USE(JSVALUE64)
4662 GPRTemporary result(this, Reuse, childOperand);
4663 JSValueRegs resultRegs(result.gpr());
4664#else
4665 GPRTemporary resultTag(this);
4666 GPRTemporary resultPayload(this);
4667 JSValueRegs resultRegs(resultPayload.gpr(), resultTag.gpr());
4668#endif
4669
4670#if ENABLE(MATH_IC_STATS)
4671 auto inlineStart = m_jit.label();
4672#endif
4673
4674 Box<MathICGenerationState> icGenerationState = Box<MathICGenerationState>::create();
4675 mathIC->m_generator = Generator(resultRegs, childRegs, scratchGPR);
4676
4677 bool shouldEmitProfiling = false;
4678 bool generatedInline = mathIC->generateInline(m_jit, *icGenerationState, shouldEmitProfiling);
4679 if (generatedInline) {
4680 ASSERT(!icGenerationState->slowPathJumps.empty());
4681
4682 Vector<SilentRegisterSavePlan> savePlans;
4683 silentSpillAllRegistersImpl(false, savePlans, resultRegs);
4684
4685 auto done = m_jit.label();
4686
4687 addSlowPathGeneratorLambda([=, savePlans = WTFMove(savePlans)] () {
4688 icGenerationState->slowPathJumps.link(&m_jit);
4689 icGenerationState->slowPathStart = m_jit.label();
4690#if ENABLE(MATH_IC_STATS)
4691 auto slowPathStart = m_jit.label();
4692#endif
4693
4694 silentSpill(savePlans);
4695
4696 if (icGenerationState->shouldSlowPathRepatch)
4697 icGenerationState->slowPathCall = callOperation(bitwise_cast<J_JITOperation_EJMic>(repatchingFunction), resultRegs, childRegs, TrustedImmPtr(mathIC));
4698 else
4699 icGenerationState->slowPathCall = callOperation(nonRepatchingFunction, resultRegs, childRegs);
4700
4701 silentFill(savePlans);
4702 m_jit.exceptionCheck();
4703 m_jit.jump().linkTo(done, &m_jit);
4704
4705 m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4706 mathIC->finalizeInlineCode(*icGenerationState, linkBuffer);
4707 });
4708
4709#if ENABLE(MATH_IC_STATS)
4710 auto slowPathEnd = m_jit.label();
4711 m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4712 size_t size = static_cast<char*>(linkBuffer.locationOf(slowPathEnd).executableAddress()) - static_cast<char*>(linkBuffer.locationOf(slowPathStart).executableAddress());
4713 mathIC->m_generatedCodeSize += size;
4714 });
4715#endif
4716
4717 });
4718 } else {
4719 flushRegisters();
4720 callOperation(nonRepatchingFunction, resultRegs, childRegs);
4721 m_jit.exceptionCheck();
4722 }
4723
4724#if ENABLE(MATH_IC_STATS)
4725 auto inlineEnd = m_jit.label();
4726 m_jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
4727 size_t size = static_cast<char*>(linkBuffer.locationOf(inlineEnd).executableAddress()) - static_cast<char*>(linkBuffer.locationOf(inlineStart).executableAddress());
4728 mathIC->m_generatedCodeSize += size;
4729 });
4730#endif
4731
4732 jsValueResult(resultRegs, node);
4733 return;
4734}
4735
4736void SpeculativeJIT::compileValueMul(Node* node)
4737{
4738 Edge& leftChild = node->child1();
4739 Edge& rightChild = node->child2();
4740
4741 if (leftChild.useKind() == BigIntUse && rightChild.useKind() == BigIntUse) {
4742 SpeculateCellOperand left(this, leftChild);
4743 SpeculateCellOperand right(this, rightChild);
4744 GPRReg leftGPR = left.gpr();
4745 GPRReg rightGPR = right.gpr();
4746
4747 speculateBigInt(leftChild, leftGPR);
4748 speculateBigInt(rightChild, rightGPR);
4749
4750 flushRegisters();
4751 GPRFlushedCallResult result(this);
4752 GPRReg resultGPR = result.gpr();
4753
4754 callOperation(operationMulBigInt, resultGPR, leftGPR, rightGPR);
4755
4756 m_jit.exceptionCheck();
4757 cellResult(resultGPR, node);
4758 return;
4759 }
4760
4761 if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
4762 JSValueOperand left(this, leftChild);
4763 JSValueOperand right(this, rightChild);
4764 JSValueRegs leftRegs = left.jsValueRegs();
4765 JSValueRegs rightRegs = right.jsValueRegs();
4766
4767 flushRegisters();
4768 JSValueRegsFlushedCallResult result(this);
4769 JSValueRegs resultRegs = result.regs();
4770 callOperation(operationValueMul, resultRegs, leftRegs, rightRegs);
4771 m_jit.exceptionCheck();
4772
4773 jsValueResult(resultRegs, node);
4774 return;
4775 }
4776
4777 bool needsScratchGPRReg = true;
4778#if USE(JSVALUE64)
4779 bool needsScratchFPRReg = false;
4780#else
4781 bool needsScratchFPRReg = true;
4782#endif
4783
4784 CodeBlock* baselineCodeBlock = m_jit.graph().baselineCodeBlockFor(node->origin.semantic);
4785 unsigned bytecodeIndex = node->origin.semantic.bytecodeIndex();
4786 ArithProfile* arithProfile = baselineCodeBlock->arithProfileForBytecodeOffset(bytecodeIndex);
4787 const Instruction* instruction = baselineCodeBlock->instructions().at(bytecodeIndex).ptr();
4788 JITMulIC* mulIC = m_jit.codeBlock()->addJITMulIC(arithProfile, instruction);
4789 auto repatchingFunction = operationValueMulOptimize;
4790 auto nonRepatchingFunction = operationValueMul;
4791
4792 compileMathIC(node, mulIC, needsScratchGPRReg, needsScratchFPRReg, repatchingFunction, nonRepatchingFunction);
4793}
4794
4795void SpeculativeJIT::compileArithMul(Node* node)
4796{
4797 switch (node->binaryUseKind()) {
4798 case Int32Use: {
4799 if (node->child2()->isInt32Constant()) {
4800 SpeculateInt32Operand op1(this, node->child1());
4801 GPRTemporary result(this);
4802
4803 int32_t imm = node->child2()->asInt32();
4804 GPRReg op1GPR = op1.gpr();
4805 GPRReg resultGPR = result.gpr();
4806
4807 if (!shouldCheckOverflow(node->arithMode()))
4808 m_jit.mul32(Imm32(imm), op1GPR, resultGPR);
4809 else {
4810 speculationCheck(Overflow, JSValueRegs(), 0,
4811 m_jit.branchMul32(MacroAssembler::Overflow, op1GPR, Imm32(imm), resultGPR));
4812 }
4813
4814 // The only way to create negative zero with a constant is:
4815 // -negative-op1 * 0.
4816 // -zero-op1 * negative constant.
4817 if (shouldCheckNegativeZero(node->arithMode())) {
4818 if (!imm)
4819 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Signed, op1GPR));
4820 else if (imm < 0) {
4821 if (shouldCheckOverflow(node->arithMode()))
4822 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Zero, resultGPR));
4823 else
4824 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Zero, op1GPR));
4825 }
4826 }
4827
4828 int32Result(resultGPR, node);
4829 return;
4830 }
4831 SpeculateInt32Operand op1(this, node->child1());
4832 SpeculateInt32Operand op2(this, node->child2());
4833 GPRTemporary result(this);
4834
4835 GPRReg reg1 = op1.gpr();
4836 GPRReg reg2 = op2.gpr();
4837
4838 // We can perform truncated multiplications if we get to this point, because if the
4839 // fixup phase could not prove that it would be safe, it would have turned us into
4840 // a double multiplication.
4841 if (!shouldCheckOverflow(node->arithMode()))
4842 m_jit.mul32(reg1, reg2, result.gpr());
4843 else {
4844 speculationCheck(
4845 Overflow, JSValueRegs(), 0,
4846 m_jit.branchMul32(MacroAssembler::Overflow, reg1, reg2, result.gpr()));
4847 }
4848
4849 // Check for negative zero, if the users of this node care about such things.
4850 if (shouldCheckNegativeZero(node->arithMode())) {
4851 MacroAssembler::Jump resultNonZero = m_jit.branchTest32(MacroAssembler::NonZero, result.gpr());
4852 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Signed, reg1));
4853 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(MacroAssembler::Signed, reg2));
4854 resultNonZero.link(&m_jit);
4855 }
4856
4857 int32Result(result.gpr(), node);
4858 return;
4859 }
4860
4861#if USE(JSVALUE64)
4862 case Int52RepUse: {
4863 ASSERT(shouldCheckOverflow(node->arithMode()));
4864
4865 // This is super clever. We want to do an int52 multiplication and check the
4866 // int52 overflow bit. There is no direct hardware support for this, but we do
4867 // have the ability to do an int64 multiplication and check the int64 overflow
4868 // bit. We leverage that. Consider that a, b are int52 numbers inside int64
4869 // registers, with the high 12 bits being sign-extended. We can do:
4870 //
4871 // (a * (b << 12))
4872 //
4873 // This will give us a left-shifted int52 (value is in high 52 bits, low 16
4874 // bits are zero) plus the int52 overflow bit. I.e. whether this 64-bit
4875 // multiplication overflows is identical to whether the 'a * b' 52-bit
4876 // multiplication overflows.
4877 //
4878 // In our nomenclature, this is:
4879 //
4880 // strictInt52(a) * int52(b) => int52
4881 //
4882 // That is "strictInt52" means unshifted and "int52" means left-shifted by 16
4883 // bits.
4884 //
4885 // We don't care which of op1 or op2 serves as the left-shifted operand, so
4886 // we just do whatever is more convenient for op1 and have op2 do the
4887 // opposite. This ensures that we do at most one shift.
4888
4889 SpeculateWhicheverInt52Operand op1(this, node->child1());
4890 SpeculateWhicheverInt52Operand op2(this, node->child2(), OppositeShift, op1);
4891 GPRTemporary result(this);
4892
4893 GPRReg op1GPR = op1.gpr();
4894 GPRReg op2GPR = op2.gpr();
4895 GPRReg resultGPR = result.gpr();
4896
4897 m_jit.move(op1GPR, resultGPR);
4898 speculationCheck(
4899 Int52Overflow, JSValueRegs(), 0,
4900 m_jit.branchMul64(MacroAssembler::Overflow, op2GPR, resultGPR));
4901
4902 if (shouldCheckNegativeZero(node->arithMode())) {
4903 MacroAssembler::Jump resultNonZero = m_jit.branchTest64(
4904 MacroAssembler::NonZero, resultGPR);
4905 speculationCheck(
4906 NegativeZero, JSValueRegs(), 0,
4907 m_jit.branch64(MacroAssembler::LessThan, op1GPR, TrustedImm32(0)));
4908 speculationCheck(
4909 NegativeZero, JSValueRegs(), 0,
4910 m_jit.branch64(MacroAssembler::LessThan, op2GPR, TrustedImm32(0)));
4911 resultNonZero.link(&m_jit);
4912 }
4913
4914 int52Result(resultGPR, node);
4915 return;
4916 }
4917#endif // USE(JSVALUE64)
4918
4919 case DoubleRepUse: {
4920 SpeculateDoubleOperand op1(this, node->child1());
4921 SpeculateDoubleOperand op2(this, node->child2());
4922 FPRTemporary result(this, op1, op2);
4923
4924 FPRReg reg1 = op1.fpr();
4925 FPRReg reg2 = op2.fpr();
4926
4927 m_jit.mulDouble(reg1, reg2, result.fpr());
4928
4929 doubleResult(result.fpr(), node);
4930 return;
4931 }
4932
4933 default:
4934 RELEASE_ASSERT_NOT_REACHED();
4935 return;
4936 }
4937}
4938
4939void SpeculativeJIT::compileValueDiv(Node* node)
4940{
4941 Edge& leftChild = node->child1();
4942 Edge& rightChild = node->child2();
4943
4944 if (leftChild.useKind() == BigIntUse && rightChild.useKind() == BigIntUse) {
4945 SpeculateCellOperand left(this, leftChild);
4946 SpeculateCellOperand right(this, rightChild);
4947 GPRReg leftGPR = left.gpr();
4948 GPRReg rightGPR = right.gpr();
4949
4950 speculateBigInt(leftChild, leftGPR);
4951 speculateBigInt(rightChild, rightGPR);
4952
4953 flushRegisters();
4954 GPRFlushedCallResult result(this);
4955 GPRReg resultGPR = result.gpr();
4956
4957 callOperation(operationDivBigInt, resultGPR, leftGPR, rightGPR);
4958
4959 m_jit.exceptionCheck();
4960 cellResult(resultGPR, node);
4961 return;
4962 }
4963
4964 if (isKnownNotNumber(leftChild.node()) || isKnownNotNumber(rightChild.node())) {
4965 JSValueOperand left(this, leftChild);
4966 JSValueOperand right(this, rightChild);
4967 JSValueRegs leftRegs = left.jsValueRegs();
4968 JSValueRegs rightRegs = right.jsValueRegs();
4969
4970 flushRegisters();
4971 JSValueRegsFlushedCallResult result(this);
4972 JSValueRegs resultRegs = result.regs();
4973 callOperation(operationValueDiv, resultRegs, leftRegs, rightRegs);
4974 m_jit.exceptionCheck();
4975
4976 jsValueResult(resultRegs, node);
4977 return;
4978 }
4979
4980 Optional<JSValueOperand> left;
4981 Optional<JSValueOperand> right;
4982
4983 JSValueRegs leftRegs;
4984 JSValueRegs rightRegs;
4985
4986 FPRTemporary leftNumber(this);
4987 FPRTemporary rightNumber(this);
4988 FPRReg leftFPR = leftNumber.fpr();
4989 FPRReg rightFPR = rightNumber.fpr();
4990 FPRTemporary fprScratch(this);
4991 FPRReg scratchFPR = fprScratch.fpr();
4992
4993#if USE(JSVALUE64)
4994 GPRTemporary result(this);
4995 JSValueRegs resultRegs = JSValueRegs(result.gpr());
4996 GPRTemporary scratch(this);
4997 GPRReg scratchGPR = scratch.gpr();
4998#else
4999 GPRTemporary resultTag(this);
5000 GPRTemporary resultPayload(this);
5001 JSValueRegs resultRegs = JSValueRegs(resultPayload.gpr(), resultTag.gpr());
5002 GPRReg scratchGPR = resultTag.gpr();
5003#endif
5004
5005 SnippetOperand leftOperand(m_state.forNode(leftChild).resultType());
5006 SnippetOperand rightOperand(m_state.forNode(rightChild).resultType());
5007
5008 if (leftChild->isInt32Constant())
5009 leftOperand.setConstInt32(leftChild->asInt32());
5010#if USE(JSVALUE64)
5011 else if (leftChild->isDoubleConstant())
5012 leftOperand.setConstDouble(leftChild->asNumber());
5013#endif
5014
5015 if (leftOperand.isConst()) {
5016 // The snippet generator only supports 1 argument as a constant.
5017 // Ignore the rightChild's const-ness.
5018 } else if (rightChild->isInt32Constant())
5019 rightOperand.setConstInt32(rightChild->asInt32());
5020#if USE(JSVALUE64)
5021 else if (rightChild->isDoubleConstant())
5022 rightOperand.setConstDouble(rightChild->asNumber());
5023#endif
5024
5025 RELEASE_ASSERT(!leftOperand.isConst() || !rightOperand.isConst());
5026
5027 if (!leftOperand.isConst()) {
5028 left.emplace(this, leftChild);
5029 leftRegs = left->jsValueRegs();
5030 }
5031 if (!rightOperand.isConst()) {
5032 right.emplace(this, rightChild);
5033 rightRegs = right->jsValueRegs();
5034 }
5035
5036 JITDivGenerator gen(leftOperand, rightOperand, resultRegs, leftRegs, rightRegs,
5037 leftFPR, rightFPR, scratchGPR, scratchFPR);
5038 gen.generateFastPath(m_jit);
5039
5040 ASSERT(gen.didEmitFastPath());
5041 gen.endJumpList().append(m_jit.jump());
5042
5043 gen.slowPathJumpList().link(&m_jit);
5044 silentSpillAllRegisters(resultRegs);
5045
5046 if (leftOperand.isConst()) {
5047 leftRegs = resultRegs;
5048 m_jit.moveValue(leftChild->asJSValue(), leftRegs);
5049 }
5050 if (rightOperand.isConst()) {
5051 rightRegs = resultRegs;
5052 m_jit.moveValue(rightChild->asJSValue(), rightRegs);
5053 }
5054
5055 callOperation(operationValueDiv, resultRegs, leftRegs, rightRegs);
5056
5057 silentFillAllRegisters();
5058 m_jit.exceptionCheck();
5059
5060 gen.endJumpList().link(&m_jit);
5061 jsValueResult(resultRegs, node);
5062}
5063
5064void SpeculativeJIT::compileArithDiv(Node* node)
5065{
5066 switch (node->binaryUseKind()) {
5067 case Int32Use: {
5068#if CPU(X86) || CPU(X86_64)
5069 SpeculateInt32Operand op1(this, node->child1());
5070 SpeculateInt32Operand op2(this, node->child2());
5071 GPRTemporary eax(this, X86Registers::eax);
5072 GPRTemporary edx(this, X86Registers::edx);
5073 GPRReg op1GPR = op1.gpr();
5074 GPRReg op2GPR = op2.gpr();
5075
5076 GPRReg op2TempGPR;
5077 GPRReg temp;
5078 if (op2GPR == X86Registers::eax || op2GPR == X86Registers::edx) {
5079 op2TempGPR = allocate();
5080 temp = op2TempGPR;
5081 } else {
5082 op2TempGPR = InvalidGPRReg;
5083 if (op1GPR == X86Registers::eax)
5084 temp = X86Registers::edx;
5085 else
5086 temp = X86Registers::eax;
5087 }
5088
5089 ASSERT(temp != op1GPR);
5090 ASSERT(temp != op2GPR);
5091
5092 m_jit.add32(JITCompiler::TrustedImm32(1), op2GPR, temp);
5093
5094 JITCompiler::Jump safeDenominator = m_jit.branch32(JITCompiler::Above, temp, JITCompiler::TrustedImm32(1));
5095
5096 JITCompiler::JumpList done;
5097 if (shouldCheckOverflow(node->arithMode())) {
5098 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, op2GPR));
5099 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1)));
5100 } else {
5101 // This is the case where we convert the result to an int after we're done, and we
5102 // already know that the denominator is either -1 or 0. So, if the denominator is
5103 // zero, then the result should be zero. If the denominator is not zero (i.e. it's
5104 // -1) and the numerator is -2^31 then the result should be -2^31. Otherwise we
5105 // are happy to fall through to a normal division, since we're just dividing
5106 // something by negative 1.
5107
5108 JITCompiler::Jump notZero = m_jit.branchTest32(JITCompiler::NonZero, op2GPR);
5109 m_jit.move(TrustedImm32(0), eax.gpr());
5110 done.append(m_jit.jump());
5111
5112 notZero.link(&m_jit);
5113 JITCompiler::Jump notNeg2ToThe31 =
5114 m_jit.branch32(JITCompiler::NotEqual, op1GPR, TrustedImm32(-2147483647-1));
5115 m_jit.zeroExtend32ToPtr(op1GPR, eax.gpr());
5116 done.append(m_jit.jump());
5117
5118 notNeg2ToThe31.link(&m_jit);
5119 }
5120
5121 safeDenominator.link(&m_jit);
5122
5123 // If the user cares about negative zero, then speculate that we're not about
5124 // to produce negative zero.
5125 if (shouldCheckNegativeZero(node->arithMode())) {
5126 MacroAssembler::Jump numeratorNonZero = m_jit.branchTest32(MacroAssembler::NonZero, op1GPR);
5127 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, op2GPR, TrustedImm32(0)));
5128 numeratorNonZero.link(&m_jit);
5129 }
5130
5131 if (op2TempGPR != InvalidGPRReg) {
5132 m_jit.move(op2GPR, op2TempGPR);
5133 op2GPR = op2TempGPR;
5134 }
5135
5136 m_jit.move(op1GPR, eax.gpr());
5137 m_jit.x86ConvertToDoubleWord32();
5138 m_jit.x86Div32(op2GPR);
5139
5140 if (op2TempGPR != InvalidGPRReg)
5141 unlock(op2TempGPR);
5142
5143 // Check that there was no remainder. If there had been, then we'd be obligated to
5144 // produce a double result instead.
5145 if (shouldCheckOverflow(node->arithMode()))
5146 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::NonZero, edx.gpr()));
5147
5148 done.link(&m_jit);
5149 int32Result(eax.gpr(), node);
5150#elif HAVE(ARM_IDIV_INSTRUCTIONS) || CPU(ARM64)
5151 SpeculateInt32Operand op1(this, node->child1());
5152 SpeculateInt32Operand op2(this, node->child2());
5153 GPRReg op1GPR = op1.gpr();
5154 GPRReg op2GPR = op2.gpr();
5155 GPRTemporary quotient(this);
5156 GPRTemporary multiplyAnswer(this);
5157
5158 // If the user cares about negative zero, then speculate that we're not about
5159 // to produce negative zero.
5160 if (shouldCheckNegativeZero(node->arithMode())) {
5161 MacroAssembler::Jump numeratorNonZero = m_jit.branchTest32(MacroAssembler::NonZero, op1GPR);
5162 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, op2GPR, TrustedImm32(0)));
5163 numeratorNonZero.link(&m_jit);
5164 }
5165
5166 if (shouldCheckOverflow(node->arithMode()))
5167 speculationCheck(Overflow, JSValueRegs(), nullptr, m_jit.branchTest32(MacroAssembler::Zero, op2GPR));
5168
5169 m_jit.assembler().sdiv<32>(quotient.gpr(), op1GPR, op2GPR);
5170
5171 // Check that there was no remainder. If there had been, then we'd be obligated to
5172 // produce a double result instead.
5173 if (shouldCheckOverflow(node->arithMode())) {
5174 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchMul32(JITCompiler::Overflow, quotient.gpr(), op2GPR, multiplyAnswer.gpr()));
5175 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::NotEqual, multiplyAnswer.gpr(), op1GPR));
5176 }
5177
5178 int32Result(quotient.gpr(), node);
5179#else
5180 RELEASE_ASSERT_NOT_REACHED();
5181#endif
5182 break;
5183 }
5184
5185 case DoubleRepUse: {
5186 SpeculateDoubleOperand op1(this, node->child1());
5187 SpeculateDoubleOperand op2(this, node->child2());
5188 FPRTemporary result(this, op1);
5189
5190 FPRReg reg1 = op1.fpr();
5191 FPRReg reg2 = op2.fpr();
5192 m_jit.divDouble(reg1, reg2, result.fpr());
5193
5194 doubleResult(result.fpr(), node);
5195 break;
5196 }
5197
5198 default:
5199 RELEASE_ASSERT_NOT_REACHED();
5200 break;
5201 }
5202}
5203
5204void SpeculativeJIT::compileArithFRound(Node* node)
5205{
5206 if (node->child1().useKind() == DoubleRepUse) {
5207 SpeculateDoubleOperand op1(this, node->child1());
5208 FPRTemporary result(this, op1);
5209 m_jit.convertDoubleToFloat(op1.fpr(), result.fpr());
5210 m_jit.convertFloatToDouble(result.fpr(), result.fpr());
5211 doubleResult(result.fpr(), node);
5212 return;
5213 }
5214
5215 JSValueOperand op1(this, node->child1());
5216 JSValueRegs op1Regs = op1.jsValueRegs();
5217 flushRegisters();
5218 FPRResult result(this);
5219 callOperation(operationArithFRound, result.fpr(), op1Regs);
5220 m_jit.exceptionCheck();
5221 doubleResult(result.fpr(), node);
5222}
5223
5224void SpeculativeJIT::compileArithMod(Node* node)
5225{
5226 switch (node->binaryUseKind()) {
5227 case Int32Use: {
5228 // In the fast path, the dividend value could be the final result
5229 // (in case of |dividend| < |divisor|), so we speculate it as strict int32.
5230 SpeculateStrictInt32Operand op1(this, node->child1());
5231
5232 if (node->child2()->isInt32Constant()) {
5233 int32_t divisor = node->child2()->asInt32();
5234 if (divisor > 1 && hasOneBitSet(divisor)) {
5235 unsigned logarithm = WTF::fastLog2(static_cast<uint32_t>(divisor));
5236 GPRReg dividendGPR = op1.gpr();
5237 GPRTemporary result(this);
5238 GPRReg resultGPR = result.gpr();
5239
5240 // This is what LLVM generates. It's pretty crazy. Here's my
5241 // attempt at understanding it.
5242
5243 // First, compute either divisor - 1, or 0, depending on whether
5244 // the dividend is negative:
5245 //
5246 // If dividend < 0: resultGPR = divisor - 1
5247 // If dividend >= 0: resultGPR = 0
5248 m_jit.move(dividendGPR, resultGPR);
5249 m_jit.rshift32(TrustedImm32(31), resultGPR);
5250 m_jit.urshift32(TrustedImm32(32 - logarithm), resultGPR);
5251
5252 // Add in the dividend, so that:
5253 //
5254 // If dividend < 0: resultGPR = dividend + divisor - 1
5255 // If dividend >= 0: resultGPR = dividend
5256 m_jit.add32(dividendGPR, resultGPR);
5257
5258 // Mask so as to only get the *high* bits. This rounds down
5259 // (towards negative infinity) resultGPR to the nearest multiple
5260 // of divisor, so that:
5261 //
5262 // If dividend < 0: resultGPR = floor((dividend + divisor - 1) / divisor)
5263 // If dividend >= 0: resultGPR = floor(dividend / divisor)
5264 //
5265 // Note that this can be simplified to:
5266 //
5267 // If dividend < 0: resultGPR = ceil(dividend / divisor)
5268 // If dividend >= 0: resultGPR = floor(dividend / divisor)
5269 //
5270 // Note that if the dividend is negative, resultGPR will also be negative.
5271 // Regardless of the sign of dividend, resultGPR will be rounded towards
5272 // zero, because of how things are conditionalized.
5273 m_jit.and32(TrustedImm32(-divisor), resultGPR);
5274
5275 // Subtract resultGPR from dividendGPR, which yields the remainder:
5276 //
5277 // resultGPR = dividendGPR - resultGPR
5278 m_jit.neg32(resultGPR);
5279 m_jit.add32(dividendGPR, resultGPR);
5280
5281 if (shouldCheckNegativeZero(node->arithMode())) {
5282 // Check that we're not about to create negative zero.
5283 JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, dividendGPR, TrustedImm32(0));
5284 speculationCheck(NegativeZero, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, resultGPR));
5285 numeratorPositive.link(&m_jit);
5286 }
5287
5288 int32Result(resultGPR, node);
5289 return;
5290 }
5291 }
5292
5293#if CPU(X86) || CPU(X86_64)
5294 if (node->child2()->isInt32Constant()) {
5295 int32_t divisor = node->child2()->asInt32();
5296 if (divisor && divisor != -1) {
5297 GPRReg op1Gpr = op1.gpr();
5298
5299 GPRTemporary eax(this, X86Registers::eax);
5300 GPRTemporary edx(this, X86Registers::edx);
5301 GPRTemporary scratch(this);
5302 GPRReg scratchGPR = scratch.gpr();
5303
5304 GPRReg op1SaveGPR;
5305 if (op1Gpr == X86Registers::eax || op1Gpr == X86Registers::edx) {
5306 op1SaveGPR = allocate();
5307 ASSERT(op1Gpr != op1SaveGPR);
5308 m_jit.move(op1Gpr, op1SaveGPR);
5309 } else
5310 op1SaveGPR = op1Gpr;
5311 ASSERT(op1SaveGPR != X86Registers::eax);
5312 ASSERT(op1SaveGPR != X86Registers::edx);
5313
5314 m_jit.move(op1Gpr, eax.gpr());
5315 m_jit.move(TrustedImm32(divisor), scratchGPR);
5316 m_jit.x86ConvertToDoubleWord32();
5317 m_jit.x86Div32(scratchGPR);
5318 if (shouldCheckNegativeZero(node->arithMode())) {
5319 JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1SaveGPR, TrustedImm32(0));
5320 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, edx.gpr()));
5321 numeratorPositive.link(&m_jit);
5322 }
5323
5324 if (op1SaveGPR != op1Gpr)
5325 unlock(op1SaveGPR);
5326
5327 int32Result(edx.gpr(), node);
5328 return;
5329 }
5330 }
5331#endif
5332
5333 SpeculateInt32Operand op2(this, node->child2());
5334#if CPU(X86) || CPU(X86_64)
5335 GPRTemporary eax(this, X86Registers::eax);
5336 GPRTemporary edx(this, X86Registers::edx);
5337 GPRReg op1GPR = op1.gpr();
5338 GPRReg op2GPR = op2.gpr();
5339
5340 GPRReg op2TempGPR;
5341 GPRReg temp;
5342 GPRReg op1SaveGPR;
5343
5344 if (op2GPR == X86Registers::eax || op2GPR == X86Registers::edx) {
5345 op2TempGPR = allocate();
5346 temp = op2TempGPR;
5347 } else {
5348 op2TempGPR = InvalidGPRReg;
5349 if (op1GPR == X86Registers::eax)
5350 temp = X86Registers::edx;
5351 else
5352 temp = X86Registers::eax;
5353 }
5354
5355 if (op1GPR == X86Registers::eax || op1GPR == X86Registers::edx) {
5356 op1SaveGPR = allocate();
5357 ASSERT(op1GPR != op1SaveGPR);
5358 m_jit.move(op1GPR, op1SaveGPR);
5359 } else
5360 op1SaveGPR = op1GPR;
5361
5362 ASSERT(temp != op1GPR);
5363 ASSERT(temp != op2GPR);
5364 ASSERT(op1SaveGPR != X86Registers::eax);
5365 ASSERT(op1SaveGPR != X86Registers::edx);
5366
5367 m_jit.add32(JITCompiler::TrustedImm32(1), op2GPR, temp);
5368
5369 JITCompiler::Jump safeDenominator = m_jit.branch32(JITCompiler::Above, temp, JITCompiler::TrustedImm32(1));
5370
5371 JITCompiler::JumpList done;
5372
5373 // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
5374 // separate case for that. But it probably doesn't matter so much.
5375 if (shouldCheckOverflow(node->arithMode())) {
5376 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, op2GPR));
5377 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(JITCompiler::Equal, op1GPR, TrustedImm32(-2147483647-1)));
5378 } else {
5379 // This is the case where we convert the result to an int after we're done, and we
5380 // already know that the denominator is either -1 or 0. So, if the denominator is
5381 // zero, then the result should be zero. If the denominator is not zero (i.e. it's
5382 // -1) and the numerator is -2^31 then the result should be 0. Otherwise we are
5383 // happy to fall through to a normal division, since we're just dividing something
5384 // by negative 1.
5385
5386 JITCompiler::Jump notZero = m_jit.branchTest32(JITCompiler::NonZero, op2GPR);
5387 m_jit.move(TrustedImm32(0), edx.gpr());
5388 done.append(m_jit.jump());
5389
5390 notZero.link(&m_jit);
5391 JITCompiler::Jump notNeg2ToThe31 =
5392 m_jit.branch32(JITCompiler::NotEqual, op1GPR, TrustedImm32(-2147483647-1));
5393 m_jit.move(TrustedImm32(0), edx.gpr());
5394 done.append(m_jit.jump());
5395
5396 notNeg2ToThe31.link(&m_jit);
5397 }
5398
5399 safeDenominator.link(&m_jit);
5400
5401 if (op2TempGPR != InvalidGPRReg) {
5402 m_jit.move(op2GPR, op2TempGPR);
5403 op2GPR = op2TempGPR;
5404 }
5405
5406 m_jit.move(op1GPR, eax.gpr());
5407 m_jit.x86ConvertToDoubleWord32();
5408 m_jit.x86Div32(op2GPR);
5409
5410 if (op2TempGPR != InvalidGPRReg)
5411 unlock(op2TempGPR);
5412
5413 // Check that we're not about to create negative zero.
5414 if (shouldCheckNegativeZero(node->arithMode())) {
5415 JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, op1SaveGPR, TrustedImm32(0));
5416 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, edx.gpr()));
5417 numeratorPositive.link(&m_jit);
5418 }
5419
5420 if (op1SaveGPR != op1GPR)
5421 unlock(op1SaveGPR);
5422
5423 done.link(&m_jit);
5424 int32Result(edx.gpr(), node);
5425
5426#elif HAVE(ARM_IDIV_INSTRUCTIONS) || CPU(ARM64)
5427 GPRTemporary temp(this);
5428 GPRTemporary quotientThenRemainder(this);
5429 GPRTemporary multiplyAnswer(this);
5430 GPRReg dividendGPR = op1.gpr();
5431 GPRReg divisorGPR = op2.gpr();
5432 GPRReg quotientThenRemainderGPR = quotientThenRemainder.gpr();
5433 GPRReg multiplyAnswerGPR = multiplyAnswer.gpr();
5434
5435 JITCompiler::JumpList done;
5436
5437 if (shouldCheckOverflow(node->arithMode()))
5438 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, divisorGPR));
5439 else {
5440 JITCompiler::Jump denominatorNotZero = m_jit.branchTest32(JITCompiler::NonZero, divisorGPR);
5441 // We know that the low 32-bit of divisorGPR is 0, but we don't know if the high bits are.
5442 // So, use TrustedImm32(0) on ARM instead because done expects the result to be in DataFormatInt32.
5443 // Using an immediate 0 doesn't cost anything extra on ARM.
5444 m_jit.move(TrustedImm32(0), quotientThenRemainderGPR);
5445 done.append(m_jit.jump());
5446 denominatorNotZero.link(&m_jit);
5447 }
5448
5449 m_jit.assembler().sdiv<32>(quotientThenRemainderGPR, dividendGPR, divisorGPR);
5450 // FIXME: It seems like there are cases where we don't need this? What if we have
5451 // arithMode() == Arith::Unchecked?
5452 // https://bugs.webkit.org/show_bug.cgi?id=126444
5453 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchMul32(JITCompiler::Overflow, quotientThenRemainderGPR, divisorGPR, multiplyAnswerGPR));
5454#if HAVE(ARM_IDIV_INSTRUCTIONS)
5455 m_jit.assembler().sub(quotientThenRemainderGPR, dividendGPR, multiplyAnswerGPR);
5456#else
5457 m_jit.assembler().sub<32>(quotientThenRemainderGPR, dividendGPR, multiplyAnswerGPR);
5458#endif
5459
5460 // If the user cares about negative zero, then speculate that we're not about
5461 // to produce negative zero.
5462 if (shouldCheckNegativeZero(node->arithMode())) {
5463 // Check that we're not about to create negative zero.
5464 JITCompiler::Jump numeratorPositive = m_jit.branch32(JITCompiler::GreaterThanOrEqual, dividendGPR, TrustedImm32(0));
5465 speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branchTest32(JITCompiler::Zero, quotientThenRemainderGPR));
5466 numeratorPositive.link(&m_jit);
5467 }
5468
5469 done.link(&m_jit);
5470
5471 int32Result(quotientThenRemainderGPR, node);
5472#else // not architecture that can do integer division
5473 RELEASE_ASSERT_NOT_REACHED();
5474#endif
5475 return;
5476 }
5477
5478 case DoubleRepUse: {
5479 SpeculateDoubleOperand op1(this, node->child1());
5480 SpeculateDoubleOperand op2(this, node->child2());
5481
5482 FPRReg op1FPR = op1.fpr();
5483 FPRReg op2FPR = op2.fpr();
5484
5485 flushRegisters();
5486
5487 FPRResult result(this);
5488
5489 using OperationType = D_JITOperation_DD;
5490 callOperation<OperationType>(jsMod, result.fpr(), op1FPR, op2FPR);
5491
5492 doubleResult(result.fpr(), node);
5493 return;
5494 }
5495
5496 default:
5497 RELEASE_ASSERT_NOT_REACHED();
5498 return;
5499 }
5500}
5501
5502void SpeculativeJIT::compileArithRounding(Node* node)
5503{
5504 if (node->child1().useKind() == DoubleRepUse) {
5505 SpeculateDoubleOperand value(this, node->child1());
5506 FPRReg valueFPR = value.fpr();
5507
5508 auto setResult = [&] (FPRReg resultFPR) {
5509 if (producesInteger(node->arithRoundingMode())) {
5510 GPRTemporary roundedResultAsInt32(this);
5511 FPRTemporary scratch(this);
5512 FPRReg scratchFPR = scratch.fpr();
5513 GPRReg resultGPR = roundedResultAsInt32.gpr();
5514 JITCompiler::JumpList failureCases;
5515 m_jit.branchConvertDoubleToInt32(resultFPR, resultGPR, failureCases, scratchFPR, shouldCheckNegativeZero(node->arithRoundingMode()));
5516 speculationCheck(Overflow, JSValueRegs(), node, failureCases);
5517
5518 int32Result(resultGPR, node);
5519 } else
5520 doubleResult(resultFPR, node);
5521 };
5522
5523 if (m_jit.supportsFloatingPointRounding()) {
5524 switch (node->op()) {
5525 case ArithRound: {
5526 FPRTemporary result(this);
5527 FPRReg resultFPR = result.fpr();
5528 if (producesInteger(node->arithRoundingMode()) && !shouldCheckNegativeZero(node->arithRoundingMode())) {
5529 static const double halfConstant = 0.5;
5530 m_jit.loadDouble(TrustedImmPtr(&halfConstant), resultFPR);
5531 m_jit.addDouble(valueFPR, resultFPR);
5532 m_jit.floorDouble(resultFPR, resultFPR);
5533 } else {
5534 m_jit.ceilDouble(valueFPR, resultFPR);
5535 FPRTemporary realPart(this);
5536 FPRReg realPartFPR = realPart.fpr();
5537 m_jit.subDouble(resultFPR, valueFPR, realPartFPR);
5538
5539 FPRTemporary scratch(this);
5540 FPRReg scratchFPR = scratch.fpr();
5541 static const double halfConstant = 0.5;
5542 m_jit.loadDouble(TrustedImmPtr(&halfConstant), scratchFPR);
5543
5544 JITCompiler::Jump shouldUseCeiled = m_jit.branchDouble(JITCompiler::DoubleLessThanOrEqual, realPartFPR, scratchFPR);
5545 static const double oneConstant = -1.0;
5546 m_jit.loadDouble(TrustedImmPtr(&oneConstant), scratchFPR);
5547 m_jit.addDouble(scratchFPR, resultFPR);
5548 shouldUseCeiled.link(&m_jit);
5549 }
5550 setResult(resultFPR);
5551 return;
5552 }
5553
5554 case ArithFloor: {
5555 FPRTemporary rounded(this);
5556 FPRReg resultFPR = rounded.fpr();
5557 m_jit.floorDouble(valueFPR, resultFPR);
5558 setResult(resultFPR);
5559 return;
5560 }
5561
5562 case ArithCeil: {
5563 FPRTemporary rounded(this);
5564 FPRReg resultFPR = rounded.fpr();
5565 m_jit.ceilDouble(valueFPR, resultFPR);
5566 setResult(resultFPR);
5567 return;
5568 }
5569
5570 case ArithTrunc: {
5571 FPRTemporary rounded(this);
5572 FPRReg resultFPR = rounded.fpr();
5573 m_jit.roundTowardZeroDouble(valueFPR, resultFPR);
5574 setResult(resultFPR);
5575 return;
5576 }
5577
5578 default:
5579 RELEASE_ASSERT_NOT_REACHED();
5580 }
5581 } else {
5582 flushRegisters();
5583 FPRResult roundedResultAsDouble(this);
5584 FPRReg resultFPR = roundedResultAsDouble.fpr();
5585 using OperationType = D_JITOperation_D;
5586 if (node->op() == ArithRound)
5587 callOperation<OperationType>(jsRound, resultFPR, valueFPR);
5588 else if (node->op() == ArithFloor)
5589 callOperation<OperationType>(floor, resultFPR, valueFPR);
5590 else if (node->op() == ArithCeil)
5591 callOperation<OperationType>(ceil, resultFPR, valueFPR);
5592 else {
5593 ASSERT(node->op() == ArithTrunc);
5594 callOperation<OperationType>(trunc, resultFPR, valueFPR);
5595 }
5596 setResult(resultFPR);
5597 }
5598 return;
5599 }
5600
5601 DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse, node->child1().useKind());
5602
5603 JSValueOperand argument(this, node->child1());
5604 JSValueRegs argumentRegs = argument.jsValueRegs();
5605
5606 flushRegisters();
5607 JSValueRegsFlushedCallResult result(this);
5608 JSValueRegs resultRegs = result.regs();
5609 J_JITOperation_EJ operation = nullptr;
5610 if (node->op() == ArithRound)
5611 operation = operationArithRound;
5612 else if (node->op() == ArithFloor)
5613 operation = operationArithFloor;
5614 else if (node->op() == ArithCeil)
5615 operation = operationArithCeil;
5616 else {
5617 ASSERT(node->op() == ArithTrunc);
5618 operation = operationArithTrunc;
5619 }
5620 callOperation(operation, resultRegs, argumentRegs);
5621 m_jit.exceptionCheck();
5622 jsValueResult(resultRegs, node);
5623}
5624
5625void SpeculativeJIT::compileArithUnary(Node* node)
5626{
5627 compileArithDoubleUnaryOp(node, arithUnaryFunction(node->arithUnaryType()), arithUnaryOperation(node->arithUnaryType()));
5628}
5629
5630void SpeculativeJIT::compileArithSqrt(Node* node)
5631{
5632 if (node->child1().useKind() == DoubleRepUse) {
5633 SpeculateDoubleOperand op1(this, node->child1());
5634 FPRReg op1FPR = op1.fpr();
5635
5636 if (!MacroAssembler::supportsFloatingPointSqrt() || !Options::useArchitectureSpecificOptimizations()) {
5637 flushRegisters();
5638 FPRResult result(this);
5639 callOperation<D_JITOperation_D>(sqrt, result.fpr(), op1FPR);
5640 doubleResult(result.fpr(), node);
5641 } else {
5642 FPRTemporary result(this, op1);
5643 m_jit.sqrtDouble(op1.fpr(), result.fpr());
5644 doubleResult(result.fpr(), node);
5645 }
5646 return;
5647 }
5648
5649 JSValueOperand op1(this, node->child1());
5650 JSValueRegs op1Regs = op1.jsValueRegs();
5651 flushRegisters();
5652 FPRResult result(this);
5653 callOperation(operationArithSqrt, result.fpr(), op1Regs);
5654 m_jit.exceptionCheck();
5655 doubleResult(result.fpr(), node);
5656}
5657
5658void SpeculativeJIT::compileArithMinMax(Node* node)
5659{
5660 switch (node->binaryUseKind()) {
5661 case Int32Use: {
5662 SpeculateStrictInt32Operand op1(this, node->child1());
5663 SpeculateStrictInt32Operand op2(this, node->child2());
5664 GPRTemporary result(this, Reuse, op1);
5665
5666 GPRReg op1GPR = op1.gpr();
5667 GPRReg op2GPR = op2.gpr();
5668 GPRReg resultGPR = result.gpr();
5669
5670 MacroAssembler::Jump op1Less = m_jit.branch32(node->op() == ArithMin ? MacroAssembler::LessThan : MacroAssembler::GreaterThan, op1GPR, op2GPR);
5671 m_jit.move(op2GPR, resultGPR);
5672 if (op1GPR != resultGPR) {
5673 MacroAssembler::Jump done = m_jit.jump();
5674 op1Less.link(&m_jit);
5675 m_jit.move(op1GPR, resultGPR);
5676 done.link(&m_jit);
5677 } else
5678 op1Less.link(&m_jit);
5679
5680 int32Result(resultGPR, node);
5681 break;
5682 }
5683
5684 case DoubleRepUse: {
5685 SpeculateDoubleOperand op1(this, node->child1());
5686 SpeculateDoubleOperand op2(this, node->child2());
5687 FPRTemporary result(this, op1);
5688
5689 FPRReg op1FPR = op1.fpr();
5690 FPRReg op2FPR = op2.fpr();
5691 FPRReg resultFPR = result.fpr();
5692
5693 MacroAssembler::JumpList done;
5694
5695 MacroAssembler::Jump op1Less = m_jit.branchDouble(node->op() == ArithMin ? MacroAssembler::DoubleLessThan : MacroAssembler::DoubleGreaterThan, op1FPR, op2FPR);
5696
5697 // op2 is eather the lesser one or one of then is NaN
5698 MacroAssembler::Jump op2Less = m_jit.branchDouble(node->op() == ArithMin ? MacroAssembler::DoubleGreaterThanOrEqual : MacroAssembler::DoubleLessThanOrEqual, op1FPR, op2FPR);
5699
5700 // Unordered case. We don't know which of op1, op2 is NaN. Manufacture NaN by adding
5701 // op1 + op2 and putting it into result.
5702 m_jit.addDouble(op1FPR, op2FPR, resultFPR);
5703 done.append(m_jit.jump());
5704
5705 op2Less.link(&m_jit);
5706 m_jit.moveDouble(op2FPR, resultFPR);
5707
5708 if (op1FPR != resultFPR) {
5709 done.append(m_jit.jump());
5710
5711 op1Less.link(&m_jit);
5712 m_jit.moveDouble(op1FPR, resultFPR);
5713 } else
5714 op1Less.link(&m_jit);
5715
5716 done.link(&m_jit);
5717
5718 doubleResult(resultFPR, node);
5719 break;
5720 }
5721
5722 default:
5723 DFG_CRASH(m_jit.graph(), node, "Bad use kind");
5724 break;
5725 }
5726}
5727
5728// For small positive integers , it is worth doing a tiny inline loop to exponentiate the base.
5729// Every register is clobbered by this helper.
5730static MacroAssembler::Jump compileArithPowIntegerFastPath(JITCompiler& assembler, FPRReg xOperand, GPRReg yOperand, FPRReg result)
5731{
5732 MacroAssembler::JumpList skipFastPath;
5733 skipFastPath.append(assembler.branch32(MacroAssembler::Above, yOperand, MacroAssembler::TrustedImm32(maxExponentForIntegerMathPow)));
5734
5735 static const double oneConstant = 1.0;
5736 assembler.loadDouble(MacroAssembler::TrustedImmPtr(&oneConstant), result);
5737
5738 MacroAssembler::Label startLoop(assembler.label());
5739 MacroAssembler::Jump exponentIsEven = assembler.branchTest32(MacroAssembler::Zero, yOperand, MacroAssembler::TrustedImm32(1));
5740 assembler.mulDouble(xOperand, result);
5741 exponentIsEven.link(&assembler);
5742 assembler.mulDouble(xOperand, xOperand);
5743 assembler.rshift32(MacroAssembler::TrustedImm32(1), yOperand);
5744 assembler.branchTest32(MacroAssembler::NonZero, yOperand).linkTo(startLoop, &assembler);
5745
5746 MacroAssembler::Jump skipSlowPath = assembler.jump();
5747 skipFastPath.link(&assembler);
5748
5749 return skipSlowPath;
5750}
5751
5752void SpeculativeJIT::compileArithPow(Node* node)
5753{
5754 if (node->child2().useKind() == Int32Use) {
5755 SpeculateDoubleOperand xOperand(this, node->child1());
5756 SpeculateInt32Operand yOperand(this, node->child2());
5757 FPRReg xOperandfpr = xOperand.fpr();
5758 GPRReg yOperandGpr = yOperand.gpr();
5759 FPRTemporary yOperandfpr(this);
5760
5761 flushRegisters();
5762
5763 FPRResult result(this);
5764 FPRReg resultFpr = result.fpr();
5765
5766 FPRTemporary xOperandCopy(this);
5767 FPRReg xOperandCopyFpr = xOperandCopy.fpr();
5768 m_jit.moveDouble(xOperandfpr, xOperandCopyFpr);
5769
5770 GPRTemporary counter(this);
5771 GPRReg counterGpr = counter.gpr();
5772 m_jit.move(yOperandGpr, counterGpr);
5773
5774 MacroAssembler::Jump skipFallback = compileArithPowIntegerFastPath(m_jit, xOperandCopyFpr, counterGpr, resultFpr);
5775 m_jit.convertInt32ToDouble(yOperandGpr, yOperandfpr.fpr());
5776 callOperation(operationMathPow, resultFpr, xOperandfpr, yOperandfpr.fpr());
5777
5778 skipFallback.link(&m_jit);
5779 doubleResult(resultFpr, node);
5780 return;
5781 }
5782
5783 if (node->child2()->isDoubleConstant()) {
5784 double exponent = node->child2()->asNumber();
5785 static const double infinityConstant = std::numeric_limits<double>::infinity();
5786 static const double minusInfinityConstant = -std::numeric_limits<double>::infinity();
5787 if (exponent == 0.5) {
5788 SpeculateDoubleOperand xOperand(this, node->child1());
5789 FPRTemporary result(this);
5790 FPRReg xOperandFpr = xOperand.fpr();
5791 FPRReg resultFpr = result.fpr();
5792
5793 m_jit.moveZeroToDouble(resultFpr);
5794 MacroAssembler::Jump xIsZeroOrNegativeZero = m_jit.branchDouble(MacroAssembler::DoubleEqual, xOperandFpr, resultFpr);
5795
5796 m_jit.loadDouble(TrustedImmPtr(&minusInfinityConstant), resultFpr);
5797 MacroAssembler::Jump xIsMinusInfinity = m_jit.branchDouble(MacroAssembler::DoubleEqual, xOperandFpr, resultFpr);
5798 m_jit.sqrtDouble(xOperandFpr, resultFpr);
5799 MacroAssembler::Jump doneWithSqrt = m_jit.jump();
5800
5801 xIsMinusInfinity.link(&m_jit);
5802 if (isX86())
5803 m_jit.loadDouble(TrustedImmPtr(&infinityConstant), resultFpr);
5804 else
5805 m_jit.absDouble(resultFpr, resultFpr);
5806
5807 xIsZeroOrNegativeZero.link(&m_jit);
5808 doneWithSqrt.link(&m_jit);
5809 doubleResult(resultFpr, node);
5810 return;
5811 }
5812 if (exponent == -0.5) {
5813 SpeculateDoubleOperand xOperand(this, node->child1());
5814 FPRTemporary scratch(this);
5815 FPRTemporary result(this);
5816 FPRReg xOperandFpr = xOperand.fpr();
5817 FPRReg scratchFPR = scratch.fpr();
5818 FPRReg resultFpr = result.fpr();
5819
5820 m_jit.moveZeroToDouble(resultFpr);
5821 MacroAssembler::Jump xIsZeroOrNegativeZero = m_jit.branchDouble(MacroAssembler::DoubleEqual, xOperandFpr, resultFpr);
5822
5823 m_jit.loadDouble(TrustedImmPtr(&minusInfinityConstant), resultFpr);
5824 MacroAssembler::Jump xIsMinusInfinity = m_jit.branchDouble(MacroAssembler::DoubleEqual, xOperandFpr, resultFpr);
5825
5826 static const double oneConstant = 1.;
5827 m_jit.loadDouble(TrustedImmPtr(&oneConstant), resultFpr);
5828 m_jit.sqrtDouble(xOperandFpr, scratchFPR);
5829 m_jit.divDouble(resultFpr, scratchFPR, resultFpr);
5830 MacroAssembler::Jump doneWithSqrt = m_jit.jump();
5831
5832 xIsZeroOrNegativeZero.link(&m_jit);
5833 m_jit.loadDouble(TrustedImmPtr(&infinityConstant), resultFpr);
5834 MacroAssembler::Jump doneWithBaseZero = m_jit.jump();
5835
5836 xIsMinusInfinity.link(&m_jit);
5837 m_jit.moveZeroToDouble(resultFpr);
5838
5839 doneWithBaseZero.link(&m_jit);
5840 doneWithSqrt.link(&m_jit);
5841 doubleResult(resultFpr, node);
5842 return;
5843 }
5844 }
5845
5846 SpeculateDoubleOperand xOperand(this, node->child1());
5847 SpeculateDoubleOperand yOperand(this, node->child2());
5848 FPRReg xOperandfpr = xOperand.fpr();
5849 FPRReg yOperandfpr = yOperand.fpr();
5850
5851 flushRegisters();
5852
5853 FPRResult result(this);
5854 FPRReg resultFpr = result.fpr();
5855
5856 FPRTemporary xOperandCopy(this);
5857 FPRReg xOperandCopyFpr = xOperandCopy.fpr();
5858
5859 FPRTemporary scratch(this);
5860 FPRReg scratchFpr = scratch.fpr();
5861
5862 GPRTemporary yOperandInteger(this);
5863 GPRReg yOperandIntegerGpr = yOperandInteger.gpr();
5864 MacroAssembler::JumpList failedExponentConversionToInteger;
5865 m_jit.branchConvertDoubleToInt32(yOperandfpr, yOperandIntegerGpr, failedExponentConversionToInteger, scratchFpr, false);
5866
5867 m_jit.moveDouble(xOperandfpr, xOperandCopyFpr);
5868 MacroAssembler::Jump skipFallback = compileArithPowIntegerFastPath(m_jit, xOperandCopyFpr, yOperandInteger.gpr(), resultFpr);
5869 failedExponentConversionToInteger.link(&m_jit);
5870
5871 callOperation(operationMathPow, resultFpr, xOperandfpr, yOperandfpr);
5872 skipFallback.link(&m_jit);
5873 doubleResult(resultFpr, node);
5874}
5875
5876// Returns true if the compare is fused with a subsequent branch.
5877bool SpeculativeJIT::compare(Node* node, MacroAssembler::RelationalCondition condition, MacroAssembler::DoubleCondition doubleCondition, S_JITOperation_EJJ operation)
5878{
5879 if (compilePeepHoleBranch(node, condition, doubleCondition, operation))
5880 return true;
5881
5882 if (node->isBinaryUseKind(Int32Use)) {
5883 compileInt32Compare(node, condition);
5884 return false;
5885 }
5886
5887#if USE(JSVALUE64)
5888 if (node->isBinaryUseKind(Int52RepUse)) {
5889 compileInt52Compare(node, condition);
5890 return false;
5891 }
5892#endif // USE(JSVALUE64)
5893
5894 if (node->isBinaryUseKind(DoubleRepUse)) {
5895 compileDoubleCompare(node, doubleCondition);
5896 return false;
5897 }
5898
5899 if (node->isBinaryUseKind(StringUse)) {
5900 if (node->op() == CompareEq)
5901 compileStringEquality(node);
5902 else
5903 compileStringCompare(node, condition);
5904 return false;
5905 }
5906
5907 if (node->isBinaryUseKind(StringIdentUse)) {
5908 if (node->op() == CompareEq)
5909 compileStringIdentEquality(node);
5910 else
5911 compileStringIdentCompare(node, condition);
5912 return false;
5913 }
5914
5915 if (node->op() == CompareEq) {
5916 if (node->isBinaryUseKind(BooleanUse)) {
5917 compileBooleanCompare(node, condition);
5918 return false;
5919 }
5920
5921 if (node->isBinaryUseKind(SymbolUse)) {
5922 compileSymbolEquality(node);
5923 return false;
5924 }
5925
5926 if (node->isBinaryUseKind(ObjectUse)) {
5927 compileObjectEquality(node);
5928 return false;
5929 }
5930
5931 if (node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) {
5932 compileObjectToObjectOrOtherEquality(node->child1(), node->child2());
5933 return false;
5934 }
5935
5936 if (node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) {
5937 compileObjectToObjectOrOtherEquality(node->child2(), node->child1());
5938 return false;
5939 }
5940
5941 if (!needsTypeCheck(node->child1(), SpecOther)) {
5942 nonSpeculativeNonPeepholeCompareNullOrUndefined(node->child2());
5943 return false;
5944 }
5945
5946 if (!needsTypeCheck(node->child2(), SpecOther)) {
5947 nonSpeculativeNonPeepholeCompareNullOrUndefined(node->child1());
5948 return false;
5949 }
5950 }
5951
5952 nonSpeculativeNonPeepholeCompare(node, condition, operation);
5953 return false;
5954}
5955
5956void SpeculativeJIT::compileCompareUnsigned(Node* node, MacroAssembler::RelationalCondition condition)
5957{
5958 compileInt32Compare(node, condition);
5959}
5960
5961bool SpeculativeJIT::compileStrictEq(Node* node)
5962{
5963 if (node->isBinaryUseKind(BooleanUse)) {
5964 unsigned branchIndexInBlock = detectPeepHoleBranch();
5965 if (branchIndexInBlock != UINT_MAX) {
5966 Node* branchNode = m_block->at(branchIndexInBlock);
5967 compilePeepHoleBooleanBranch(node, branchNode, MacroAssembler::Equal);
5968 use(node->child1());
5969 use(node->child2());
5970 m_indexInBlock = branchIndexInBlock;
5971 m_currentNode = branchNode;
5972 return true;
5973 }
5974 compileBooleanCompare(node, MacroAssembler::Equal);
5975 return false;
5976 }
5977
5978 if (node->isBinaryUseKind(Int32Use)) {
5979 unsigned branchIndexInBlock = detectPeepHoleBranch();
5980 if (branchIndexInBlock != UINT_MAX) {
5981 Node* branchNode = m_block->at(branchIndexInBlock);
5982 compilePeepHoleInt32Branch(node, branchNode, MacroAssembler::Equal);
5983 use(node->child1());
5984 use(node->child2());
5985 m_indexInBlock = branchIndexInBlock;
5986 m_currentNode = branchNode;
5987 return true;
5988 }
5989 compileInt32Compare(node, MacroAssembler::Equal);
5990 return false;
5991 }
5992
5993#if USE(JSVALUE64)
5994 if (node->isBinaryUseKind(Int52RepUse)) {
5995 unsigned branchIndexInBlock = detectPeepHoleBranch();
5996 if (branchIndexInBlock != UINT_MAX) {
5997 Node* branchNode = m_block->at(branchIndexInBlock);
5998 compilePeepHoleInt52Branch(node, branchNode, MacroAssembler::Equal);
5999 use(node->child1());
6000 use(node->child2());
6001 m_indexInBlock = branchIndexInBlock;
6002 m_currentNode = branchNode;
6003 return true;
6004 }
6005 compileInt52Compare(node, MacroAssembler::Equal);
6006 return false;
6007 }
6008#endif // USE(JSVALUE64)
6009
6010 if (node->isBinaryUseKind(DoubleRepUse)) {
6011 unsigned branchIndexInBlock = detectPeepHoleBranch();
6012 if (branchIndexInBlock != UINT_MAX) {
6013 Node* branchNode = m_block->at(branchIndexInBlock);
6014 compilePeepHoleDoubleBranch(node, branchNode, MacroAssembler::DoubleEqual);
6015 use(node->child1());
6016 use(node->child2());
6017 m_indexInBlock = branchIndexInBlock;
6018 m_currentNode = branchNode;
6019 return true;
6020 }
6021 compileDoubleCompare(node, MacroAssembler::DoubleEqual);
6022 return false;
6023 }
6024
6025 if (node->isBinaryUseKind(SymbolUse)) {
6026 unsigned branchIndexInBlock = detectPeepHoleBranch();
6027 if (branchIndexInBlock != UINT_MAX) {
6028 Node* branchNode = m_block->at(branchIndexInBlock);
6029 compilePeepHoleSymbolEquality(node, branchNode);
6030 use(node->child1());
6031 use(node->child2());
6032 m_indexInBlock = branchIndexInBlock;
6033 m_currentNode = branchNode;
6034 return true;
6035 }
6036 compileSymbolEquality(node);
6037 return false;
6038 }
6039
6040 if (node->isBinaryUseKind(BigIntUse)) {
6041 compileBigIntEquality(node);
6042 return false;
6043 }
6044
6045 if (node->isBinaryUseKind(SymbolUse, UntypedUse)) {
6046 compileSymbolUntypedEquality(node, node->child1(), node->child2());
6047 return false;
6048 }
6049
6050 if (node->isBinaryUseKind(UntypedUse, SymbolUse)) {
6051 compileSymbolUntypedEquality(node, node->child2(), node->child1());
6052 return false;
6053 }
6054
6055 if (node->isBinaryUseKind(StringUse)) {
6056 compileStringEquality(node);
6057 return false;
6058 }
6059
6060 if (node->isBinaryUseKind(StringIdentUse)) {
6061 compileStringIdentEquality(node);
6062 return false;
6063 }
6064
6065 if (node->isBinaryUseKind(ObjectUse, UntypedUse)) {
6066 unsigned branchIndexInBlock = detectPeepHoleBranch();
6067 if (branchIndexInBlock != UINT_MAX) {
6068 Node* branchNode = m_block->at(branchIndexInBlock);
6069 compilePeepHoleObjectStrictEquality(node->child1(), node->child2(), branchNode);
6070 use(node->child1());
6071 use(node->child2());
6072 m_indexInBlock = branchIndexInBlock;
6073 m_currentNode = branchNode;
6074 return true;
6075 }
6076 compileObjectStrictEquality(node->child1(), node->child2());
6077 return false;
6078 }
6079
6080 if (node->isBinaryUseKind(UntypedUse, ObjectUse)) {
6081 unsigned branchIndexInBlock = detectPeepHoleBranch();
6082 if (branchIndexInBlock != UINT_MAX) {
6083 Node* branchNode = m_block->at(branchIndexInBlock);
6084 compilePeepHoleObjectStrictEquality(node->child2(), node->child1(), branchNode);
6085 use(node->child1());
6086 use(node->child2());
6087 m_indexInBlock = branchIndexInBlock;
6088 m_currentNode = branchNode;
6089 return true;
6090 }
6091 compileObjectStrictEquality(node->child2(), node->child1());
6092 return false;
6093 }
6094
6095 if (node->isBinaryUseKind(ObjectUse)) {
6096 unsigned branchIndexInBlock = detectPeepHoleBranch();
6097 if (branchIndexInBlock != UINT_MAX) {
6098 Node* branchNode = m_block->at(branchIndexInBlock);
6099 compilePeepHoleObjectEquality(node, branchNode);
6100 use(node->child1());
6101 use(node->child2());
6102 m_indexInBlock = branchIndexInBlock;
6103 m_currentNode = branchNode;
6104 return true;
6105 }
6106 compileObjectEquality(node);
6107 return false;
6108 }
6109
6110 if (node->isBinaryUseKind(MiscUse, UntypedUse)
6111 || node->isBinaryUseKind(UntypedUse, MiscUse)) {
6112 compileMiscStrictEq(node);
6113 return false;
6114 }
6115
6116 if (node->isBinaryUseKind(StringIdentUse, NotStringVarUse)) {
6117 compileStringIdentToNotStringVarEquality(node, node->child1(), node->child2());
6118 return false;
6119 }
6120
6121 if (node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) {
6122 compileStringIdentToNotStringVarEquality(node, node->child2(), node->child1());
6123 return false;
6124 }
6125
6126 if (node->isBinaryUseKind(StringUse, UntypedUse)) {
6127 compileStringToUntypedEquality(node, node->child1(), node->child2());
6128 return false;
6129 }
6130
6131 if (node->isBinaryUseKind(UntypedUse, StringUse)) {
6132 compileStringToUntypedEquality(node, node->child2(), node->child1());
6133 return false;
6134 }
6135
6136 RELEASE_ASSERT(node->isBinaryUseKind(UntypedUse));
6137 return nonSpeculativeStrictEq(node);
6138}
6139
6140void SpeculativeJIT::compileBooleanCompare(Node* node, MacroAssembler::RelationalCondition condition)
6141{
6142 SpeculateBooleanOperand op1(this, node->child1());
6143 SpeculateBooleanOperand op2(this, node->child2());
6144 GPRTemporary result(this);
6145
6146 m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr());
6147
6148 unblessedBooleanResult(result.gpr(), node);
6149}
6150
6151void SpeculativeJIT::compileInt32Compare(Node* node, MacroAssembler::RelationalCondition condition)
6152{
6153 if (node->child1()->isInt32Constant()) {
6154 SpeculateInt32Operand op2(this, node->child2());
6155 GPRTemporary result(this, Reuse, op2);
6156 int32_t imm = node->child1()->asInt32();
6157 m_jit.compare32(condition, JITCompiler::Imm32(imm), op2.gpr(), result.gpr());
6158
6159 unblessedBooleanResult(result.gpr(), node);
6160 } else if (node->child2()->isInt32Constant()) {
6161 SpeculateInt32Operand op1(this, node->child1());
6162 GPRTemporary result(this, Reuse, op1);
6163 int32_t imm = node->child2()->asInt32();
6164 m_jit.compare32(condition, op1.gpr(), JITCompiler::Imm32(imm), result.gpr());
6165
6166 unblessedBooleanResult(result.gpr(), node);
6167 } else {
6168 SpeculateInt32Operand op1(this, node->child1());
6169 SpeculateInt32Operand op2(this, node->child2());
6170 GPRTemporary result(this, Reuse, op1, op2);
6171 m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr());
6172
6173 unblessedBooleanResult(result.gpr(), node);
6174 }
6175}
6176
6177void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCondition condition)
6178{
6179 SpeculateDoubleOperand op1(this, node->child1());
6180 SpeculateDoubleOperand op2(this, node->child2());
6181 GPRTemporary result(this);
6182
6183 FPRReg op1FPR = op1.fpr();
6184 FPRReg op2FPR = op2.fpr();
6185 GPRReg resultGPR = result.gpr();
6186
6187 m_jit.compareDouble(condition, op1FPR, op2FPR, resultGPR);
6188
6189 unblessedBooleanResult(resultGPR, node);
6190}
6191
6192void SpeculativeJIT::compileObjectEquality(Node* node)
6193{
6194 SpeculateCellOperand op1(this, node->child1());
6195 SpeculateCellOperand op2(this, node->child2());
6196 GPRTemporary result(this, Reuse, op1);
6197
6198 GPRReg op1GPR = op1.gpr();
6199 GPRReg op2GPR = op2.gpr();
6200 GPRReg resultGPR = result.gpr();
6201
6202 if (masqueradesAsUndefinedWatchpointIsStillValid()) {
6203 DFG_TYPE_CHECK(
6204 JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR));
6205 DFG_TYPE_CHECK(
6206 JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR));
6207 } else {
6208 DFG_TYPE_CHECK(
6209 JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR));
6210 speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
6211 m_jit.branchTest8(
6212 MacroAssembler::NonZero,
6213 MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()),
6214 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
6215
6216 DFG_TYPE_CHECK(
6217 JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR));
6218 speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
6219 m_jit.branchTest8(
6220 MacroAssembler::NonZero,
6221 MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()),
6222 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
6223 }
6224
6225 m_jit.comparePtr(MacroAssembler::Equal, op1GPR, op2GPR, resultGPR);
6226 unblessedBooleanResult(resultGPR, node);
6227}
6228
6229void SpeculativeJIT::compileSymbolEquality(Node* node)
6230{
6231 SpeculateCellOperand left(this, node->child1());
6232 SpeculateCellOperand right(this, node->child2());
6233 GPRTemporary result(this, Reuse, left, right);
6234
6235 GPRReg leftGPR = left.gpr();
6236 GPRReg rightGPR = right.gpr();
6237 GPRReg resultGPR = result.gpr();
6238
6239 speculateSymbol(node->child1(), leftGPR);
6240 speculateSymbol(node->child2(), rightGPR);
6241
6242 m_jit.comparePtr(JITCompiler::Equal, leftGPR, rightGPR, resultGPR);
6243 unblessedBooleanResult(resultGPR, node);
6244}
6245
6246void SpeculativeJIT::compilePeepHoleSymbolEquality(Node* node, Node* branchNode)
6247{
6248 SpeculateCellOperand left(this, node->child1());
6249 SpeculateCellOperand right(this, node->child2());
6250
6251 GPRReg leftGPR = left.gpr();
6252 GPRReg rightGPR = right.gpr();
6253
6254 speculateSymbol(node->child1(), leftGPR);
6255 speculateSymbol(node->child2(), rightGPR);
6256
6257 BasicBlock* taken = branchNode->branchData()->taken.block;
6258 BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
6259
6260 if (taken == nextBlock()) {
6261 branchPtr(JITCompiler::NotEqual, leftGPR, rightGPR, notTaken);
6262 jump(taken);
6263 } else {
6264 branchPtr(JITCompiler::Equal, leftGPR, rightGPR, taken);
6265 jump(notTaken);
6266 }
6267}
6268
6269void SpeculativeJIT::compileStringEquality(
6270 Node* node, GPRReg leftGPR, GPRReg rightGPR, GPRReg lengthGPR, GPRReg leftTempGPR,
6271 GPRReg rightTempGPR, GPRReg leftTemp2GPR, GPRReg rightTemp2GPR,
6272 const JITCompiler::JumpList& fastTrue, const JITCompiler::JumpList& fastFalse)
6273{
6274 JITCompiler::JumpList trueCase;
6275 JITCompiler::JumpList falseCase;
6276 JITCompiler::JumpList slowCase;
6277
6278 trueCase.append(fastTrue);
6279 falseCase.append(fastFalse);
6280
6281 m_jit.loadPtr(MacroAssembler::Address(leftGPR, JSString::offsetOfValue()), leftTempGPR);
6282 m_jit.loadPtr(MacroAssembler::Address(rightGPR, JSString::offsetOfValue()), rightTempGPR);
6283
6284 slowCase.append(m_jit.branchIfRopeStringImpl(leftTempGPR));
6285 slowCase.append(m_jit.branchIfRopeStringImpl(rightTempGPR));
6286
6287 m_jit.load32(MacroAssembler::Address(leftTempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
6288
6289 falseCase.append(m_jit.branch32(
6290 MacroAssembler::NotEqual,
6291 MacroAssembler::Address(rightTempGPR, StringImpl::lengthMemoryOffset()),
6292 lengthGPR));
6293
6294 trueCase.append(m_jit.branchTest32(MacroAssembler::Zero, lengthGPR));
6295
6296 slowCase.append(m_jit.branchTest32(
6297 MacroAssembler::Zero,
6298 MacroAssembler::Address(leftTempGPR, StringImpl::flagsOffset()),
6299 TrustedImm32(StringImpl::flagIs8Bit())));
6300 slowCase.append(m_jit.branchTest32(
6301 MacroAssembler::Zero,
6302 MacroAssembler::Address(rightTempGPR, StringImpl::flagsOffset()),
6303 TrustedImm32(StringImpl::flagIs8Bit())));
6304
6305 m_jit.loadPtr(MacroAssembler::Address(leftTempGPR, StringImpl::dataOffset()), leftTempGPR);
6306 m_jit.loadPtr(MacroAssembler::Address(rightTempGPR, StringImpl::dataOffset()), rightTempGPR);
6307
6308 MacroAssembler::Label loop = m_jit.label();
6309
6310 m_jit.sub32(TrustedImm32(1), lengthGPR);
6311
6312 // This isn't going to generate the best code on x86. But that's OK, it's still better
6313 // than not inlining.
6314 m_jit.load8(MacroAssembler::BaseIndex(leftTempGPR, lengthGPR, MacroAssembler::TimesOne), leftTemp2GPR);
6315 m_jit.load8(MacroAssembler::BaseIndex(rightTempGPR, lengthGPR, MacroAssembler::TimesOne), rightTemp2GPR);
6316 falseCase.append(m_jit.branch32(MacroAssembler::NotEqual, leftTemp2GPR, rightTemp2GPR));
6317
6318 m_jit.branchTest32(MacroAssembler::NonZero, lengthGPR).linkTo(loop, &m_jit);
6319
6320 trueCase.link(&m_jit);
6321 moveTrueTo(leftTempGPR);
6322
6323 JITCompiler::Jump done = m_jit.jump();
6324
6325 falseCase.link(&m_jit);
6326 moveFalseTo(leftTempGPR);
6327
6328 done.link(&m_jit);
6329 addSlowPathGenerator(
6330 slowPathCall(
6331 slowCase, this, operationCompareStringEq, leftTempGPR, leftGPR, rightGPR));
6332
6333 blessedBooleanResult(leftTempGPR, node);
6334}
6335
6336void SpeculativeJIT::compileStringEquality(Node* node)
6337{
6338 SpeculateCellOperand left(this, node->child1());
6339 SpeculateCellOperand right(this, node->child2());
6340 GPRTemporary length(this);
6341 GPRTemporary leftTemp(this);
6342 GPRTemporary rightTemp(this);
6343 GPRTemporary leftTemp2(this, Reuse, left);
6344 GPRTemporary rightTemp2(this, Reuse, right);
6345
6346 GPRReg leftGPR = left.gpr();
6347 GPRReg rightGPR = right.gpr();
6348 GPRReg lengthGPR = length.gpr();
6349 GPRReg leftTempGPR = leftTemp.gpr();
6350 GPRReg rightTempGPR = rightTemp.gpr();
6351 GPRReg leftTemp2GPR = leftTemp2.gpr();
6352 GPRReg rightTemp2GPR = rightTemp2.gpr();
6353
6354 speculateString(node->child1(), leftGPR);
6355
6356 // It's safe to branch around the type check below, since proving that the values are
6357 // equal does indeed prove that the right value is a string.
6358 JITCompiler::Jump fastTrue = m_jit.branchPtr(MacroAssembler::Equal, leftGPR, rightGPR);
6359
6360 speculateString(node->child2(), rightGPR);
6361
6362 compileStringEquality(
6363 node, leftGPR, rightGPR, lengthGPR, leftTempGPR, rightTempGPR, leftTemp2GPR,
6364 rightTemp2GPR, fastTrue, JITCompiler::Jump());
6365}
6366
6367void SpeculativeJIT::compileStringToUntypedEquality(Node* node, Edge stringEdge, Edge untypedEdge)
6368{
6369 SpeculateCellOperand left(this, stringEdge);
6370 JSValueOperand right(this, untypedEdge, ManualOperandSpeculation);
6371 GPRTemporary length(this);
6372 GPRTemporary leftTemp(this);
6373 GPRTemporary rightTemp(this);
6374 GPRTemporary leftTemp2(this, Reuse, left);
6375 GPRTemporary rightTemp2(this);
6376
6377 GPRReg leftGPR = left.gpr();
6378 JSValueRegs rightRegs = right.jsValueRegs();
6379 GPRReg lengthGPR = length.gpr();
6380 GPRReg leftTempGPR = leftTemp.gpr();
6381 GPRReg rightTempGPR = rightTemp.gpr();
6382 GPRReg leftTemp2GPR = leftTemp2.gpr();
6383 GPRReg rightTemp2GPR = rightTemp2.gpr();
6384
6385 speculateString(stringEdge, leftGPR);
6386
6387 JITCompiler::JumpList fastTrue;
6388 JITCompiler::JumpList fastFalse;
6389
6390 fastFalse.append(m_jit.branchIfNotCell(rightRegs));
6391
6392 // It's safe to branch around the type check below, since proving that the values are
6393 // equal does indeed prove that the right value is a string.
6394 fastTrue.append(m_jit.branchPtr(
6395 MacroAssembler::Equal, leftGPR, rightRegs.payloadGPR()));
6396
6397 fastFalse.append(m_jit.branchIfNotString(rightRegs.payloadGPR()));
6398
6399 compileStringEquality(
6400 node, leftGPR, rightRegs.payloadGPR(), lengthGPR, leftTempGPR, rightTempGPR, leftTemp2GPR,
6401 rightTemp2GPR, fastTrue, fastFalse);
6402}
6403
6404void SpeculativeJIT::compileStringIdentEquality(Node* node)
6405{
6406 SpeculateCellOperand left(this, node->child1());
6407 SpeculateCellOperand right(this, node->child2());
6408 GPRTemporary leftTemp(this);
6409 GPRTemporary rightTemp(this);
6410
6411 GPRReg leftGPR = left.gpr();
6412 GPRReg rightGPR = right.gpr();
6413 GPRReg leftTempGPR = leftTemp.gpr();
6414 GPRReg rightTempGPR = rightTemp.gpr();
6415
6416 speculateString(node->child1(), leftGPR);
6417 speculateString(node->child2(), rightGPR);
6418
6419 speculateStringIdentAndLoadStorage(node->child1(), leftGPR, leftTempGPR);
6420 speculateStringIdentAndLoadStorage(node->child2(), rightGPR, rightTempGPR);
6421
6422 m_jit.comparePtr(MacroAssembler::Equal, leftTempGPR, rightTempGPR, leftTempGPR);
6423
6424 unblessedBooleanResult(leftTempGPR, node);
6425}
6426
6427void SpeculativeJIT::compileStringIdentToNotStringVarEquality(
6428 Node* node, Edge stringEdge, Edge notStringVarEdge)
6429{
6430 SpeculateCellOperand left(this, stringEdge);
6431 JSValueOperand right(this, notStringVarEdge, ManualOperandSpeculation);
6432 GPRTemporary leftTemp(this);
6433 GPRTemporary rightTemp(this);
6434 GPRReg leftTempGPR = leftTemp.gpr();
6435 GPRReg rightTempGPR = rightTemp.gpr();
6436 GPRReg leftGPR = left.gpr();
6437 JSValueRegs rightRegs = right.jsValueRegs();
6438
6439 speculateString(stringEdge, leftGPR);
6440 speculateStringIdentAndLoadStorage(stringEdge, leftGPR, leftTempGPR);
6441
6442 moveFalseTo(rightTempGPR);
6443 JITCompiler::JumpList notString;
6444 notString.append(m_jit.branchIfNotCell(rightRegs));
6445 notString.append(m_jit.branchIfNotString(rightRegs.payloadGPR()));
6446
6447 speculateStringIdentAndLoadStorage(notStringVarEdge, rightRegs.payloadGPR(), rightTempGPR);
6448
6449 m_jit.comparePtr(MacroAssembler::Equal, leftTempGPR, rightTempGPR, rightTempGPR);
6450 notString.link(&m_jit);
6451
6452 unblessedBooleanResult(rightTempGPR, node);
6453}
6454
6455void SpeculativeJIT::compileStringCompare(Node* node, MacroAssembler::RelationalCondition condition)
6456{
6457 SpeculateCellOperand left(this, node->child1());
6458 SpeculateCellOperand right(this, node->child2());
6459 GPRReg leftGPR = left.gpr();
6460 GPRReg rightGPR = right.gpr();
6461
6462 speculateString(node->child1(), leftGPR);
6463 speculateString(node->child2(), rightGPR);
6464
6465 C_JITOperation_B_EJssJss compareFunction = nullptr;
6466 if (condition == MacroAssembler::LessThan)
6467 compareFunction = operationCompareStringLess;
6468 else if (condition == MacroAssembler::LessThanOrEqual)
6469 compareFunction = operationCompareStringLessEq;
6470 else if (condition == MacroAssembler::GreaterThan)
6471 compareFunction = operationCompareStringGreater;
6472 else if (condition == MacroAssembler::GreaterThanOrEqual)
6473 compareFunction = operationCompareStringGreaterEq;
6474 else
6475 RELEASE_ASSERT_NOT_REACHED();
6476
6477 GPRFlushedCallResult result(this);
6478 GPRReg resultGPR = result.gpr();
6479
6480 flushRegisters();
6481 callOperation(compareFunction, resultGPR, leftGPR, rightGPR);
6482 m_jit.exceptionCheck();
6483
6484 unblessedBooleanResult(resultGPR, node);
6485}
6486
6487void SpeculativeJIT::compileStringIdentCompare(Node* node, MacroAssembler::RelationalCondition condition)
6488{
6489 SpeculateCellOperand left(this, node->child1());
6490 SpeculateCellOperand right(this, node->child2());
6491 GPRFlushedCallResult result(this);
6492 GPRTemporary leftTemp(this);
6493 GPRTemporary rightTemp(this);
6494
6495 GPRReg leftGPR = left.gpr();
6496 GPRReg rightGPR = right.gpr();
6497 GPRReg resultGPR = result.gpr();
6498 GPRReg leftTempGPR = leftTemp.gpr();
6499 GPRReg rightTempGPR = rightTemp.gpr();
6500
6501 speculateString(node->child1(), leftGPR);
6502 speculateString(node->child2(), rightGPR);
6503
6504 C_JITOperation_TT compareFunction = nullptr;
6505 if (condition == MacroAssembler::LessThan)
6506 compareFunction = operationCompareStringImplLess;
6507 else if (condition == MacroAssembler::LessThanOrEqual)
6508 compareFunction = operationCompareStringImplLessEq;
6509 else if (condition == MacroAssembler::GreaterThan)
6510 compareFunction = operationCompareStringImplGreater;
6511 else if (condition == MacroAssembler::GreaterThanOrEqual)
6512 compareFunction = operationCompareStringImplGreaterEq;
6513 else
6514 RELEASE_ASSERT_NOT_REACHED();
6515
6516 speculateStringIdentAndLoadStorage(node->child1(), leftGPR, leftTempGPR);
6517 speculateStringIdentAndLoadStorage(node->child2(), rightGPR, rightTempGPR);
6518
6519 flushRegisters();
6520 callOperation(compareFunction, resultGPR, leftTempGPR, rightTempGPR);
6521
6522 unblessedBooleanResult(resultGPR, node);
6523}
6524
6525void SpeculativeJIT::compileSameValue(Node* node)
6526{
6527 if (node->isBinaryUseKind(DoubleRepUse)) {
6528 SpeculateDoubleOperand arg1(this, node->child1());
6529 SpeculateDoubleOperand arg2(this, node->child2());
6530 GPRTemporary result(this);
6531 GPRTemporary temp(this);
6532 GPRTemporary temp2(this);
6533
6534 FPRReg arg1FPR = arg1.fpr();
6535 FPRReg arg2FPR = arg2.fpr();
6536 GPRReg resultGPR = result.gpr();
6537 GPRReg tempGPR = temp.gpr();
6538 GPRReg temp2GPR = temp2.gpr();
6539
6540#if USE(JSVALUE64)
6541 m_jit.moveDoubleTo64(arg1FPR, tempGPR);
6542 m_jit.moveDoubleTo64(arg2FPR, temp2GPR);
6543 auto trueCase = m_jit.branch64(CCallHelpers::Equal, tempGPR, temp2GPR);
6544#else
6545 GPRTemporary temp3(this);
6546 GPRReg temp3GPR = temp3.gpr();
6547
6548 m_jit.moveDoubleToInts(arg1FPR, tempGPR, temp2GPR);
6549 m_jit.moveDoubleToInts(arg2FPR, temp3GPR, resultGPR);
6550 auto notEqual = m_jit.branch32(CCallHelpers::NotEqual, tempGPR, temp3GPR);
6551 auto trueCase = m_jit.branch32(CCallHelpers::Equal, temp2GPR, resultGPR);
6552 notEqual.link(&m_jit);
6553#endif
6554
6555 m_jit.compareDouble(CCallHelpers::DoubleNotEqualOrUnordered, arg1FPR, arg1FPR, tempGPR);
6556 m_jit.compareDouble(CCallHelpers::DoubleNotEqualOrUnordered, arg2FPR, arg2FPR, temp2GPR);
6557 m_jit.and32(tempGPR, temp2GPR, resultGPR);
6558 auto done = m_jit.jump();
6559
6560 trueCase.link(&m_jit);
6561 m_jit.move(CCallHelpers::TrustedImm32(1), resultGPR);
6562 done.link(&m_jit);
6563
6564 unblessedBooleanResult(resultGPR, node);
6565 return;
6566 }
6567
6568 ASSERT(node->isBinaryUseKind(UntypedUse));
6569
6570 JSValueOperand arg1(this, node->child1());
6571 JSValueOperand arg2(this, node->child2());
6572 JSValueRegs arg1Regs = arg1.jsValueRegs();
6573 JSValueRegs arg2Regs = arg2.jsValueRegs();
6574
6575 arg1.use();
6576 arg2.use();
6577
6578 flushRegisters();
6579
6580 GPRFlushedCallResult result(this);
6581 GPRReg resultGPR = result.gpr();
6582 callOperation(operationSameValue, resultGPR, arg1Regs, arg2Regs);
6583 m_jit.exceptionCheck();
6584
6585 unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
6586}
6587
6588void SpeculativeJIT::compileStringZeroLength(Node* node)
6589{
6590 SpeculateCellOperand str(this, node->child1());
6591 GPRReg strGPR = str.gpr();
6592
6593 // Make sure that this is a string.
6594 speculateString(node->child1(), strGPR);
6595
6596 GPRTemporary eq(this);
6597 GPRReg eqGPR = eq.gpr();
6598
6599 m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(m_jit.vm())), eqGPR);
6600 m_jit.comparePtr(CCallHelpers::Equal, strGPR, eqGPR, eqGPR);
6601 unblessedBooleanResult(eqGPR, node);
6602}
6603
6604void SpeculativeJIT::compileLogicalNotStringOrOther(Node* node)
6605{
6606 JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
6607 GPRTemporary temp(this);
6608 JSValueRegs valueRegs = value.jsValueRegs();
6609 GPRReg tempGPR = temp.gpr();
6610
6611 JITCompiler::Jump notCell = m_jit.branchIfNotCell(valueRegs);
6612 GPRReg cellGPR = valueRegs.payloadGPR();
6613 DFG_TYPE_CHECK(
6614 valueRegs, node->child1(), (~SpecCellCheck) | SpecString, m_jit.branchIfNotString(cellGPR));
6615
6616 m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(m_jit.vm())), tempGPR);
6617 m_jit.comparePtr(CCallHelpers::Equal, cellGPR, tempGPR, tempGPR);
6618 auto done = m_jit.jump();
6619
6620 notCell.link(&m_jit);
6621 DFG_TYPE_CHECK(
6622 valueRegs, node->child1(), SpecCellCheck | SpecOther, m_jit.branchIfNotOther(valueRegs, tempGPR));
6623 m_jit.move(TrustedImm32(1), tempGPR);
6624
6625 done.link(&m_jit);
6626 unblessedBooleanResult(tempGPR, node);
6627
6628}
6629
6630void SpeculativeJIT::emitStringBranch(Edge nodeUse, BasicBlock* taken, BasicBlock* notTaken)
6631{
6632 SpeculateCellOperand str(this, nodeUse);
6633
6634 GPRReg strGPR = str.gpr();
6635
6636 speculateString(nodeUse, strGPR);
6637
6638 branchPtr(CCallHelpers::Equal, strGPR, TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(m_jit.vm())), notTaken);
6639 jump(taken);
6640
6641 noResult(m_currentNode);
6642}
6643
6644void SpeculativeJIT::emitStringOrOtherBranch(Edge nodeUse, BasicBlock* taken, BasicBlock* notTaken)
6645{
6646 JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
6647 GPRTemporary temp(this);
6648 JSValueRegs valueRegs = value.jsValueRegs();
6649 GPRReg tempGPR = temp.gpr();
6650
6651 JITCompiler::Jump notCell = m_jit.branchIfNotCell(valueRegs);
6652 GPRReg cellGPR = valueRegs.payloadGPR();
6653 DFG_TYPE_CHECK(valueRegs, nodeUse, (~SpecCellCheck) | SpecString, m_jit.branchIfNotString(cellGPR));
6654
6655 branchPtr(CCallHelpers::Equal, cellGPR, TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(m_jit.vm())), notTaken);
6656 jump(taken, ForceJump);
6657
6658 notCell.link(&m_jit);
6659 DFG_TYPE_CHECK(
6660 valueRegs, nodeUse, SpecCellCheck | SpecOther, m_jit.branchIfNotOther(valueRegs, tempGPR));
6661 jump(notTaken);
6662 noResult(m_currentNode);
6663}
6664
6665void SpeculativeJIT::compileConstantStoragePointer(Node* node)
6666{
6667 GPRTemporary storage(this);
6668 GPRReg storageGPR = storage.gpr();
6669 m_jit.move(TrustedImmPtr(node->storagePointer()), storageGPR);
6670 storageResult(storageGPR, node);
6671}
6672
6673void SpeculativeJIT::cageTypedArrayStorage(GPRReg storageReg)
6674{
6675#if GIGACAGE_ENABLED
6676 if (!Gigacage::shouldBeEnabled())
6677 return;
6678
6679 if (Gigacage::canPrimitiveGigacageBeDisabled()) {
6680 if (m_jit.vm()->primitiveGigacageEnabled().isStillValid())
6681 m_jit.graph().watchpoints().addLazily(m_jit.vm()->primitiveGigacageEnabled());
6682 else
6683 return;
6684 }
6685
6686 m_jit.cage(Gigacage::Primitive, storageReg);
6687#else
6688 UNUSED_PARAM(storageReg);
6689#endif
6690}
6691
6692void SpeculativeJIT::compileGetIndexedPropertyStorage(Node* node)
6693{
6694 SpeculateCellOperand base(this, node->child1());
6695 GPRReg baseReg = base.gpr();
6696
6697 GPRTemporary storage(this);
6698 GPRReg storageReg = storage.gpr();
6699
6700 switch (node->arrayMode().type()) {
6701 case Array::String:
6702 m_jit.loadPtr(MacroAssembler::Address(baseReg, JSString::offsetOfValue()), storageReg);
6703
6704 addSlowPathGenerator(
6705 slowPathCall(
6706 m_jit.branchIfRopeStringImpl(storageReg),
6707 this, operationResolveRope, storageReg, baseReg));
6708
6709 m_jit.loadPtr(MacroAssembler::Address(storageReg, StringImpl::dataOffset()), storageReg);
6710 break;
6711
6712 default:
6713 auto typedArrayType = node->arrayMode().typedArrayType();
6714 ASSERT_UNUSED(typedArrayType, isTypedView(typedArrayType));
6715
6716 m_jit.loadPtr(JITCompiler::Address(baseReg, JSArrayBufferView::offsetOfVector()), storageReg);
6717 cageTypedArrayStorage(storageReg);
6718 break;
6719 }
6720
6721 storageResult(storageReg, node);
6722}
6723
6724void SpeculativeJIT::compileGetTypedArrayByteOffset(Node* node)
6725{
6726 SpeculateCellOperand base(this, node->child1());
6727 GPRTemporary vector(this);
6728 GPRTemporary data(this);
6729
6730 GPRReg baseGPR = base.gpr();
6731 GPRReg vectorGPR = vector.gpr();
6732 GPRReg dataGPR = data.gpr();
6733 ASSERT(baseGPR != vectorGPR);
6734 ASSERT(baseGPR != dataGPR);
6735 ASSERT(vectorGPR != dataGPR);
6736
6737 GPRReg arrayBufferGPR = dataGPR;
6738
6739 JITCompiler::Jump emptyByteOffset = m_jit.branch32(
6740 MacroAssembler::NotEqual,
6741 MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfMode()),
6742 TrustedImm32(WastefulTypedArray));
6743
6744 m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfVector()), vectorGPR);
6745 JITCompiler::Jump nullVector = m_jit.branchTestPtr(JITCompiler::Zero, vectorGPR);
6746
6747 m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), dataGPR);
6748 m_jit.cage(Gigacage::JSValue, dataGPR);
6749
6750 cageTypedArrayStorage(vectorGPR);
6751
6752 m_jit.loadPtr(MacroAssembler::Address(dataGPR, Butterfly::offsetOfArrayBuffer()), arrayBufferGPR);
6753 // FIXME: This needs caging.
6754 // https://bugs.webkit.org/show_bug.cgi?id=175515
6755 m_jit.loadPtr(MacroAssembler::Address(arrayBufferGPR, ArrayBuffer::offsetOfData()), dataGPR);
6756 m_jit.subPtr(dataGPR, vectorGPR);
6757
6758 JITCompiler::Jump done = m_jit.jump();
6759
6760 emptyByteOffset.link(&m_jit);
6761 m_jit.move(TrustedImmPtr(nullptr), vectorGPR);
6762
6763 done.link(&m_jit);
6764 nullVector.link(&m_jit);
6765
6766 int32Result(vectorGPR, node);
6767}
6768
6769void SpeculativeJIT::compileGetByValOnDirectArguments(Node* node)
6770{
6771 SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
6772 SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
6773 JSValueRegsTemporary result(this);
6774 GPRTemporary scratch(this);
6775
6776 GPRReg baseReg = base.gpr();
6777 GPRReg propertyReg = property.gpr();
6778 JSValueRegs resultRegs = result.regs();
6779 GPRReg scratchReg = scratch.gpr();
6780
6781 if (!m_compileOkay)
6782 return;
6783
6784 ASSERT(ArrayMode(Array::DirectArguments, Array::Read).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
6785
6786 speculationCheck(
6787 ExoticObjectMode, JSValueSource(), 0,
6788 m_jit.branchTestPtr(
6789 MacroAssembler::NonZero,
6790 MacroAssembler::Address(baseReg, DirectArguments::offsetOfMappedArguments())));
6791
6792 m_jit.load32(CCallHelpers::Address(baseReg, DirectArguments::offsetOfLength()), scratchReg);
6793 auto isOutOfBounds = m_jit.branch32(CCallHelpers::AboveOrEqual, propertyReg, scratchReg);
6794 if (node->arrayMode().isInBounds())
6795 speculationCheck(OutOfBounds, JSValueSource(), 0, isOutOfBounds);
6796
6797 m_jit.loadValue(
6798 MacroAssembler::BaseIndex(
6799 baseReg, propertyReg, MacroAssembler::TimesEight, DirectArguments::storageOffset()),
6800 resultRegs);
6801
6802 if (!node->arrayMode().isInBounds()) {
6803 addSlowPathGenerator(
6804 slowPathCall(
6805 isOutOfBounds, this, operationGetByValObjectInt,
6806 extractResult(resultRegs), baseReg, propertyReg));
6807 }
6808
6809 jsValueResult(resultRegs, node);
6810}
6811
6812void SpeculativeJIT::compileGetByValOnScopedArguments(Node* node)
6813{
6814 SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
6815 SpeculateStrictInt32Operand property(this, m_graph.varArgChild(node, 1));
6816 JSValueRegsTemporary result(this);
6817 GPRTemporary scratch(this);
6818 GPRTemporary scratch2(this);
6819 GPRTemporary indexMask(this);
6820
6821 GPRReg baseReg = base.gpr();
6822 GPRReg propertyReg = property.gpr();
6823 JSValueRegs resultRegs = result.regs();
6824 GPRReg scratchReg = scratch.gpr();
6825 GPRReg scratch2Reg = scratch2.gpr();
6826 GPRReg indexMaskReg = indexMask.gpr();
6827
6828 if (!m_compileOkay)
6829 return;
6830
6831 ASSERT(ArrayMode(Array::ScopedArguments, Array::Read).alreadyChecked(m_jit.graph(), node, m_state.forNode(m_graph.varArgChild(node, 0))));
6832
6833 m_jit.loadPtr(
6834 MacroAssembler::Address(baseReg, ScopedArguments::offsetOfStorage()), resultRegs.payloadGPR());
6835 m_jit.load32(
6836 MacroAssembler::Address(resultRegs.payloadGPR(), ScopedArguments::offsetOfTotalLengthInStorage()),
6837 scratchReg);
6838
6839 speculationCheck(
6840 ExoticObjectMode, JSValueSource(), nullptr,
6841 m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, scratchReg));
6842
6843 m_jit.emitPreparePreciseIndexMask32(propertyReg, scratchReg, indexMaskReg);
6844
6845 m_jit.loadPtr(MacroAssembler::Address(baseReg, ScopedArguments::offsetOfTable()), scratchReg);
6846 m_jit.load32(
6847 MacroAssembler::Address(scratchReg, ScopedArgumentsTable::offsetOfLength()), scratch2Reg);
6848
6849 MacroAssembler::Jump overflowArgument = m_jit.branch32(
6850 MacroAssembler::AboveOrEqual, propertyReg, scratch2Reg);
6851
6852 m_jit.loadPtr(MacroAssembler::Address(baseReg, ScopedArguments::offsetOfScope()), scratch2Reg);
6853
6854 m_jit.loadPtr(
6855 MacroAssembler::Address(scratchReg, ScopedArgumentsTable::offsetOfArguments()),
6856 scratchReg);
6857 m_jit.load32(
6858 MacroAssembler::BaseIndex(scratchReg, propertyReg, MacroAssembler::TimesFour),
6859 scratchReg);
6860
6861 speculationCheck(
6862 ExoticObjectMode, JSValueSource(), nullptr,
6863 m_jit.branch32(
6864 MacroAssembler::Equal, scratchReg, TrustedImm32(ScopeOffset::invalidOffset)));
6865
6866 m_jit.loadValue(
6867 MacroAssembler::BaseIndex(
6868 scratch2Reg, propertyReg, MacroAssembler::TimesEight,
6869 JSLexicalEnvironment::offsetOfVariables()),
6870 resultRegs);
6871
6872 MacroAssembler::Jump done = m_jit.jump();
6873 overflowArgument.link(&m_jit);
6874
6875 m_jit.sub32(propertyReg, scratch2Reg);
6876 m_jit.neg32(scratch2Reg);
6877
6878 m_jit.loadValue(
6879 MacroAssembler::BaseIndex(
6880 resultRegs.payloadGPR(), scratch2Reg, MacroAssembler::TimesEight),
6881 resultRegs);
6882 speculationCheck(ExoticObjectMode, JSValueSource(), nullptr, m_jit.branchIfEmpty(resultRegs));
6883
6884 done.link(&m_jit);
6885
6886 m_jit.andPtr(indexMaskReg, resultRegs.payloadGPR());
6887
6888 jsValueResult(resultRegs, node);
6889}
6890
6891void SpeculativeJIT::compileGetScope(Node* node)
6892{
6893 SpeculateCellOperand function(this, node->child1());
6894 GPRTemporary result(this, Reuse, function);
6895 m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr());
6896 cellResult(result.gpr(), node);
6897}
6898
6899void SpeculativeJIT::compileSkipScope(Node* node)
6900{
6901 SpeculateCellOperand scope(this, node->child1());
6902 GPRTemporary result(this, Reuse, scope);
6903 m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr());
6904 cellResult(result.gpr(), node);
6905}
6906
6907void SpeculativeJIT::compileGetGlobalObject(Node* node)
6908{
6909 SpeculateCellOperand object(this, node->child1());
6910 GPRTemporary result(this);
6911 GPRTemporary scratch(this);
6912 m_jit.emitLoadStructure(*m_jit.vm(), object.gpr(), result.gpr(), scratch.gpr());
6913 m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), result.gpr());
6914 cellResult(result.gpr(), node);
6915}
6916
6917void SpeculativeJIT::compileGetGlobalThis(Node* node)
6918{
6919 GPRTemporary result(this);
6920 GPRReg resultGPR = result.gpr();
6921 auto* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
6922 m_jit.loadPtr(globalObject->addressOfGlobalThis(), resultGPR);
6923 cellResult(resultGPR, node);
6924}
6925
6926bool SpeculativeJIT::canBeRope(Edge& edge)
6927{
6928 if (m_state.forNode(edge).isType(SpecStringIdent))
6929 return false;
6930 // If this value is LazyValue, it will be converted to JSString, and the result must be non-rope string.
6931 String string = edge->tryGetString(m_graph);
6932 if (!string.isNull())
6933 return false;
6934 return true;
6935}
6936
6937void SpeculativeJIT::compileGetArrayLength(Node* node)
6938{
6939 switch (node->arrayMode().type()) {
6940 case Array::Undecided:
6941 case Array::Int32:
6942 case Array::Double:
6943 case Array::Contiguous: {
6944 StorageOperand storage(this, node->child2());
6945 GPRTemporary result(this, Reuse, storage);
6946 GPRReg storageReg = storage.gpr();
6947 GPRReg resultReg = result.gpr();
6948 m_jit.load32(MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()), resultReg);
6949
6950 int32Result(resultReg, node);
6951 break;
6952 }
6953 case Array::ArrayStorage:
6954 case Array::SlowPutArrayStorage: {
6955 StorageOperand storage(this, node->child2());
6956 GPRTemporary result(this, Reuse, storage);
6957 GPRReg storageReg = storage.gpr();
6958 GPRReg resultReg = result.gpr();
6959 m_jit.load32(MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()), resultReg);
6960
6961 speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::LessThan, resultReg, MacroAssembler::TrustedImm32(0)));
6962
6963 int32Result(resultReg, node);
6964 break;
6965 }
6966 case Array::String: {
6967 SpeculateCellOperand base(this, node->child1());
6968 GPRTemporary result(this, Reuse, base);
6969 GPRTemporary temp(this);
6970 GPRReg baseGPR = base.gpr();
6971 GPRReg resultGPR = result.gpr();
6972 GPRReg tempGPR = temp.gpr();
6973
6974 bool needsRopeCase = canBeRope(node->child1());
6975
6976 m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSString::offsetOfValue()), tempGPR);
6977 CCallHelpers::Jump isRope;
6978 if (needsRopeCase)
6979 isRope = m_jit.branchIfRopeStringImpl(tempGPR);
6980 m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), resultGPR);
6981 if (needsRopeCase) {
6982 auto done = m_jit.jump();
6983
6984 isRope.link(&m_jit);
6985 m_jit.load32(CCallHelpers::Address(baseGPR, JSRopeString::offsetOfLength()), resultGPR);
6986
6987 done.link(&m_jit);
6988 }
6989 int32Result(resultGPR, node);
6990 break;
6991 }
6992 case Array::DirectArguments: {
6993 SpeculateCellOperand base(this, node->child1());
6994 GPRTemporary result(this, Reuse, base);
6995
6996 GPRReg baseReg = base.gpr();
6997 GPRReg resultReg = result.gpr();
6998
6999 if (!m_compileOkay)
7000 return;
7001
7002 ASSERT(ArrayMode(Array::DirectArguments, Array::Read).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
7003
7004 speculationCheck(
7005 ExoticObjectMode, JSValueSource(), 0,
7006 m_jit.branchTestPtr(
7007 MacroAssembler::NonZero,
7008 MacroAssembler::Address(baseReg, DirectArguments::offsetOfMappedArguments())));
7009
7010 m_jit.load32(
7011 MacroAssembler::Address(baseReg, DirectArguments::offsetOfLength()), resultReg);
7012
7013 int32Result(resultReg, node);
7014 break;
7015 }
7016 case Array::ScopedArguments: {
7017 SpeculateCellOperand base(this, node->child1());
7018 GPRTemporary result(this);
7019
7020 GPRReg baseReg = base.gpr();
7021 GPRReg resultReg = result.gpr();
7022
7023 if (!m_compileOkay)
7024 return;
7025
7026 ASSERT(ArrayMode(Array::ScopedArguments, Array::Read).alreadyChecked(m_jit.graph(), node, m_state.forNode(node->child1())));
7027
7028 m_jit.loadPtr(
7029 MacroAssembler::Address(baseReg, ScopedArguments::offsetOfStorage()), resultReg);
7030
7031 speculationCheck(
7032 ExoticObjectMode, JSValueSource(), 0,
7033 m_jit.branchTest8(
7034 MacroAssembler::NonZero,
7035 MacroAssembler::Address(resultReg, ScopedArguments::offsetOfOverrodeThingsInStorage())));
7036
7037 m_jit.load32(
7038 MacroAssembler::Address(resultReg, ScopedArguments::offsetOfTotalLengthInStorage()), resultReg);
7039
7040 int32Result(resultReg, node);
7041 break;
7042 }
7043 default: {
7044 ASSERT(node->arrayMode().isSomeTypedArrayView());
7045 SpeculateCellOperand base(this, node->child1());
7046 GPRTemporary result(this, Reuse, base);
7047 GPRReg baseGPR = base.gpr();
7048 GPRReg resultGPR = result.gpr();
7049 m_jit.load32(MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()), resultGPR);
7050 int32Result(resultGPR, node);
7051 break;
7052 } }
7053}
7054
7055void SpeculativeJIT::compileCheckStringIdent(Node* node)
7056{
7057 SpeculateCellOperand string(this, node->child1());
7058 GPRTemporary storage(this);
7059
7060 GPRReg stringGPR = string.gpr();
7061 GPRReg storageGPR = storage.gpr();
7062
7063 speculateString(node->child1(), stringGPR);
7064 speculateStringIdentAndLoadStorage(node->child1(), stringGPR, storageGPR);
7065
7066 UniquedStringImpl* uid = node->uidOperand();
7067 speculationCheck(
7068 BadIdent, JSValueSource(), nullptr,
7069 m_jit.branchPtr(JITCompiler::NotEqual, storageGPR, TrustedImmPtr(uid)));
7070 noResult(node);
7071}
7072
7073template <typename ClassType>
7074void SpeculativeJIT::compileNewFunctionCommon(GPRReg resultGPR, RegisteredStructure structure, GPRReg scratch1GPR, GPRReg scratch2GPR, GPRReg scopeGPR, MacroAssembler::JumpList& slowPath, size_t size, FunctionExecutable* executable)
7075{
7076 auto butterfly = TrustedImmPtr(nullptr);
7077 emitAllocateJSObjectWithKnownSize<ClassType>(resultGPR, TrustedImmPtr(structure), butterfly, scratch1GPR, scratch2GPR, slowPath, size);
7078
7079 m_jit.storePtr(scopeGPR, JITCompiler::Address(resultGPR, JSFunction::offsetOfScopeChain()));
7080 m_jit.storePtr(TrustedImmPtr::weakPointer(m_jit.graph(), executable), JITCompiler::Address(resultGPR, JSFunction::offsetOfExecutable()));
7081 m_jit.storePtr(TrustedImmPtr(nullptr), JITCompiler::Address(resultGPR, JSFunction::offsetOfRareData()));
7082
7083 m_jit.mutatorFence(*m_jit.vm());
7084}
7085
7086void SpeculativeJIT::compileNewFunction(Node* node)
7087{
7088 NodeType nodeType = node->op();
7089 ASSERT(nodeType == NewFunction || nodeType == NewGeneratorFunction || nodeType == NewAsyncFunction || nodeType == NewAsyncGeneratorFunction);
7090
7091 SpeculateCellOperand scope(this, node->child1());
7092 GPRReg scopeGPR = scope.gpr();
7093
7094 FunctionExecutable* executable = node->castOperand<FunctionExecutable*>();
7095
7096 if (executable->singletonFunction()->isStillValid()) {
7097 GPRFlushedCallResult result(this);
7098 GPRReg resultGPR = result.gpr();
7099
7100 flushRegisters();
7101
7102 if (nodeType == NewGeneratorFunction)
7103 callOperation(operationNewGeneratorFunction, resultGPR, scopeGPR, executable);
7104 else if (nodeType == NewAsyncFunction)
7105 callOperation(operationNewAsyncFunction, resultGPR, scopeGPR, executable);
7106 else if (nodeType == NewAsyncGeneratorFunction)
7107 callOperation(operationNewAsyncGeneratorFunction, resultGPR, scopeGPR, executable);
7108 else
7109 callOperation(operationNewFunction, resultGPR, scopeGPR, executable);
7110 m_jit.exceptionCheck();
7111 cellResult(resultGPR, node);
7112 return;
7113 }
7114
7115 RegisteredStructure structure = m_jit.graph().registerStructure(
7116 [&] () {
7117 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
7118 switch (nodeType) {
7119 case NewGeneratorFunction:
7120 return globalObject->generatorFunctionStructure();
7121 case NewAsyncFunction:
7122 return globalObject->asyncFunctionStructure();
7123 case NewAsyncGeneratorFunction:
7124 return globalObject->asyncGeneratorFunctionStructure();
7125 case NewFunction:
7126 return JSFunction::selectStructureForNewFuncExp(globalObject, node->castOperand<FunctionExecutable*>());
7127 default:
7128 RELEASE_ASSERT_NOT_REACHED();
7129 }
7130 }());
7131
7132 GPRTemporary result(this);
7133 GPRTemporary scratch1(this);
7134 GPRTemporary scratch2(this);
7135
7136 GPRReg resultGPR = result.gpr();
7137 GPRReg scratch1GPR = scratch1.gpr();
7138 GPRReg scratch2GPR = scratch2.gpr();
7139
7140 JITCompiler::JumpList slowPath;
7141
7142 if (nodeType == NewFunction) {
7143 compileNewFunctionCommon<JSFunction>(resultGPR, structure, scratch1GPR, scratch2GPR, scopeGPR, slowPath, JSFunction::allocationSize(0), executable);
7144
7145 addSlowPathGenerator(slowPathCall(slowPath, this, operationNewFunctionWithInvalidatedReallocationWatchpoint, resultGPR, scopeGPR, executable));
7146 }
7147
7148 if (nodeType == NewGeneratorFunction) {
7149 compileNewFunctionCommon<JSGeneratorFunction>(resultGPR, structure, scratch1GPR, scratch2GPR, scopeGPR, slowPath, JSGeneratorFunction::allocationSize(0), executable);
7150
7151 addSlowPathGenerator(slowPathCall(slowPath, this, operationNewGeneratorFunctionWithInvalidatedReallocationWatchpoint, resultGPR, scopeGPR, executable));
7152 }
7153
7154 if (nodeType == NewAsyncFunction) {
7155 compileNewFunctionCommon<JSAsyncFunction>(resultGPR, structure, scratch1GPR, scratch2GPR, scopeGPR, slowPath, JSAsyncFunction::allocationSize(0), executable);
7156
7157 addSlowPathGenerator(slowPathCall(slowPath, this, operationNewAsyncFunctionWithInvalidatedReallocationWatchpoint, resultGPR, scopeGPR, executable));
7158 }
7159
7160 if (nodeType == NewAsyncGeneratorFunction) {
7161 compileNewFunctionCommon<JSAsyncGeneratorFunction>(resultGPR, structure, scratch1GPR, scratch2GPR, scopeGPR, slowPath, JSAsyncGeneratorFunction::allocationSize(0), executable);
7162
7163 addSlowPathGenerator(slowPathCall(slowPath, this, operationNewAsyncGeneratorFunctionWithInvalidatedReallocationWatchpoint, resultGPR, scopeGPR, executable));
7164 }
7165
7166 cellResult(resultGPR, node);
7167}
7168
7169void SpeculativeJIT::compileSetFunctionName(Node* node)
7170{
7171 SpeculateCellOperand func(this, node->child1());
7172 GPRReg funcGPR = func.gpr();
7173 JSValueOperand nameValue(this, node->child2());
7174 JSValueRegs nameValueRegs = nameValue.jsValueRegs();
7175
7176 flushRegisters();
7177 callOperation(operationSetFunctionName, funcGPR, nameValueRegs);
7178 m_jit.exceptionCheck();
7179
7180 noResult(node);
7181}
7182
7183void SpeculativeJIT::compileLoadVarargs(Node* node)
7184{
7185 LoadVarargsData* data = node->loadVarargsData();
7186
7187 JSValueRegs argumentsRegs;
7188 {
7189 JSValueOperand arguments(this, node->child1());
7190 argumentsRegs = arguments.jsValueRegs();
7191 flushRegisters();
7192 }
7193
7194 callOperation(operationSizeOfVarargs, GPRInfo::returnValueGPR, argumentsRegs, data->offset);
7195 m_jit.exceptionCheck();
7196
7197 lock(GPRInfo::returnValueGPR);
7198 {
7199 JSValueOperand arguments(this, node->child1());
7200 argumentsRegs = arguments.jsValueRegs();
7201 flushRegisters();
7202 }
7203 unlock(GPRInfo::returnValueGPR);
7204
7205 // FIXME: There is a chance that we will call an effectful length property twice. This is safe
7206 // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance
7207 // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right
7208 // past the sizing.
7209 // https://bugs.webkit.org/show_bug.cgi?id=141448
7210
7211 GPRReg argCountIncludingThisGPR =
7212 JITCompiler::selectScratchGPR(GPRInfo::returnValueGPR, argumentsRegs);
7213
7214 m_jit.add32(TrustedImm32(1), GPRInfo::returnValueGPR, argCountIncludingThisGPR);
7215
7216 speculationCheck(
7217 VarargsOverflow, JSValueSource(), Edge(), m_jit.branch32(
7218 MacroAssembler::Above,
7219 GPRInfo::returnValueGPR,
7220 argCountIncludingThisGPR));
7221
7222 speculationCheck(
7223 VarargsOverflow, JSValueSource(), Edge(), m_jit.branch32(
7224 MacroAssembler::Above,
7225 argCountIncludingThisGPR,
7226 TrustedImm32(data->limit)));
7227
7228 m_jit.store32(argCountIncludingThisGPR, JITCompiler::payloadFor(data->machineCount));
7229
7230 callOperation(operationLoadVarargs, data->machineStart.offset(), argumentsRegs, data->offset, GPRInfo::returnValueGPR, data->mandatoryMinimum);
7231 m_jit.exceptionCheck();
7232
7233 noResult(node);
7234}
7235
7236void SpeculativeJIT::compileForwardVarargs(Node* node)
7237{
7238 LoadVarargsData* data = node->loadVarargsData();
7239 InlineCallFrame* inlineCallFrame;
7240 if (node->child1())
7241 inlineCallFrame = node->child1()->origin.semantic.inlineCallFrame();
7242 else
7243 inlineCallFrame = node->origin.semantic.inlineCallFrame();
7244
7245 GPRTemporary length(this);
7246 JSValueRegsTemporary temp(this);
7247 GPRReg lengthGPR = length.gpr();
7248 JSValueRegs tempRegs = temp.regs();
7249
7250 emitGetLength(inlineCallFrame, lengthGPR, /* includeThis = */ true);
7251 if (data->offset)
7252 m_jit.sub32(TrustedImm32(data->offset), lengthGPR);
7253
7254 speculationCheck(
7255 VarargsOverflow, JSValueSource(), Edge(), m_jit.branch32(
7256 MacroAssembler::Above,
7257 lengthGPR, TrustedImm32(data->limit)));
7258
7259 m_jit.store32(lengthGPR, JITCompiler::payloadFor(data->machineCount));
7260
7261 VirtualRegister sourceStart = JITCompiler::argumentsStart(inlineCallFrame) + data->offset;
7262 VirtualRegister targetStart = data->machineStart;
7263
7264 m_jit.sub32(TrustedImm32(1), lengthGPR);
7265
7266 // First have a loop that fills in the undefined slots in case of an arity check failure.
7267 m_jit.move(TrustedImm32(data->mandatoryMinimum), tempRegs.payloadGPR());
7268 JITCompiler::Jump done = m_jit.branch32(JITCompiler::BelowOrEqual, tempRegs.payloadGPR(), lengthGPR);
7269
7270 JITCompiler::Label loop = m_jit.label();
7271 m_jit.sub32(TrustedImm32(1), tempRegs.payloadGPR());
7272 m_jit.storeTrustedValue(
7273 jsUndefined(),
7274 JITCompiler::BaseIndex(
7275 GPRInfo::callFrameRegister, tempRegs.payloadGPR(), JITCompiler::TimesEight,
7276 targetStart.offset() * sizeof(EncodedJSValue)));
7277 m_jit.branch32(JITCompiler::Above, tempRegs.payloadGPR(), lengthGPR).linkTo(loop, &m_jit);
7278 done.link(&m_jit);
7279
7280 // And then fill in the actual argument values.
7281 done = m_jit.branchTest32(JITCompiler::Zero, lengthGPR);
7282
7283 loop = m_jit.label();
7284 m_jit.sub32(TrustedImm32(1), lengthGPR);
7285 m_jit.loadValue(
7286 JITCompiler::BaseIndex(
7287 GPRInfo::callFrameRegister, lengthGPR, JITCompiler::TimesEight,
7288 sourceStart.offset() * sizeof(EncodedJSValue)),
7289 tempRegs);
7290 m_jit.storeValue(
7291 tempRegs,
7292 JITCompiler::BaseIndex(
7293 GPRInfo::callFrameRegister, lengthGPR, JITCompiler::TimesEight,
7294 targetStart.offset() * sizeof(EncodedJSValue)));
7295 m_jit.branchTest32(JITCompiler::NonZero, lengthGPR).linkTo(loop, &m_jit);
7296
7297 done.link(&m_jit);
7298
7299 noResult(node);
7300}
7301
7302void SpeculativeJIT::compileCreateActivation(Node* node)
7303{
7304 SymbolTable* table = node->castOperand<SymbolTable*>();
7305 RegisteredStructure structure = m_jit.graph().registerStructure(m_jit.graph().globalObjectFor(
7306 node->origin.semantic)->activationStructure());
7307
7308 SpeculateCellOperand scope(this, node->child1());
7309 GPRReg scopeGPR = scope.gpr();
7310 JSValue initializationValue = node->initializationValueForActivation();
7311 ASSERT(initializationValue == jsUndefined() || initializationValue == jsTDZValue());
7312
7313 if (table->singletonScope()->isStillValid()) {
7314 GPRFlushedCallResult result(this);
7315 GPRReg resultGPR = result.gpr();
7316
7317#if USE(JSVALUE32_64)
7318 JSValueRegsTemporary initialization(this);
7319 JSValueRegs initializationRegs = initialization.regs();
7320 m_jit.moveTrustedValue(initializationValue, initializationRegs);
7321#endif
7322
7323 flushRegisters();
7324
7325#if USE(JSVALUE64)
7326 callOperation(operationCreateActivationDirect,
7327 resultGPR, structure, scopeGPR, table, TrustedImm64(JSValue::encode(initializationValue)));
7328#else
7329 callOperation(operationCreateActivationDirect,
7330 resultGPR, structure, scopeGPR, table, initializationRegs);
7331#endif
7332 m_jit.exceptionCheck();
7333 cellResult(resultGPR, node);
7334 return;
7335 }
7336
7337 GPRTemporary result(this);
7338 GPRTemporary scratch1(this);
7339 GPRTemporary scratch2(this);
7340 GPRReg resultGPR = result.gpr();
7341 GPRReg scratch1GPR = scratch1.gpr();
7342 GPRReg scratch2GPR = scratch2.gpr();
7343
7344#if USE(JSVALUE32_64)
7345 JSValueRegsTemporary initialization(this);
7346 JSValueRegs initializationRegs = initialization.regs();
7347 m_jit.moveTrustedValue(initializationValue, initializationRegs);
7348#endif
7349
7350 JITCompiler::JumpList slowPath;
7351 auto butterfly = TrustedImmPtr(nullptr);
7352 emitAllocateJSObjectWithKnownSize<JSLexicalEnvironment>(
7353 resultGPR, TrustedImmPtr(structure), butterfly, scratch1GPR, scratch2GPR,
7354 slowPath, JSLexicalEnvironment::allocationSize(table));
7355
7356 // Don't need a memory barriers since we just fast-created the activation, so the
7357 // activation must be young.
7358 m_jit.storePtr(scopeGPR, JITCompiler::Address(resultGPR, JSScope::offsetOfNext()));
7359 m_jit.storePtr(
7360 TrustedImmPtr(node->cellOperand()),
7361 JITCompiler::Address(resultGPR, JSLexicalEnvironment::offsetOfSymbolTable()));
7362
7363 // Must initialize all members to undefined or the TDZ empty value.
7364 for (unsigned i = 0; i < table->scopeSize(); ++i) {
7365 m_jit.storeTrustedValue(
7366 initializationValue,
7367 JITCompiler::Address(
7368 resultGPR, JSLexicalEnvironment::offsetOfVariable(ScopeOffset(i))));
7369 }
7370
7371 m_jit.mutatorFence(*m_jit.vm());
7372
7373#if USE(JSVALUE64)
7374 addSlowPathGenerator(
7375 slowPathCall(
7376 slowPath, this, operationCreateActivationDirect, resultGPR, structure, scopeGPR, table, TrustedImm64(JSValue::encode(initializationValue))));
7377#else
7378 addSlowPathGenerator(
7379 slowPathCall(
7380 slowPath, this, operationCreateActivationDirect, resultGPR, structure, scopeGPR, table, initializationRegs));
7381#endif
7382
7383 cellResult(resultGPR, node);
7384}
7385
7386void SpeculativeJIT::compileCreateDirectArguments(Node* node)
7387{
7388 // FIXME: A more effective way of dealing with the argument count and callee is to have
7389 // them be explicit arguments to this node.
7390 // https://bugs.webkit.org/show_bug.cgi?id=142207
7391
7392 GPRTemporary result(this);
7393 GPRTemporary scratch1(this);
7394 GPRTemporary scratch2(this);
7395 GPRTemporary length;
7396 GPRReg resultGPR = result.gpr();
7397 GPRReg scratch1GPR = scratch1.gpr();
7398 GPRReg scratch2GPR = scratch2.gpr();
7399 GPRReg lengthGPR = InvalidGPRReg;
7400 JSValueRegs valueRegs = JSValueRegs::withTwoAvailableRegs(scratch1GPR, scratch2GPR);
7401
7402 unsigned minCapacity = m_jit.graph().baselineCodeBlockFor(node->origin.semantic)->numParameters() - 1;
7403
7404 unsigned knownLength;
7405 bool lengthIsKnown; // if false, lengthGPR will have the length.
7406 auto* inlineCallFrame = node->origin.semantic.inlineCallFrame();
7407 if (inlineCallFrame
7408 && !inlineCallFrame->isVarargs()) {
7409 knownLength = inlineCallFrame->argumentCountIncludingThis - 1;
7410 lengthIsKnown = true;
7411 } else {
7412 knownLength = UINT_MAX;
7413 lengthIsKnown = false;
7414
7415 GPRTemporary realLength(this);
7416 length.adopt(realLength);
7417 lengthGPR = length.gpr();
7418
7419 VirtualRegister argumentCountRegister = m_jit.argumentCount(node->origin.semantic);
7420 m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), lengthGPR);
7421 m_jit.sub32(TrustedImm32(1), lengthGPR);
7422 }
7423
7424 RegisteredStructure structure =
7425 m_jit.graph().registerStructure(m_jit.graph().globalObjectFor(node->origin.semantic)->directArgumentsStructure());
7426
7427 // Use a different strategy for allocating the object depending on whether we know its
7428 // size statically.
7429 JITCompiler::JumpList slowPath;
7430 if (lengthIsKnown) {
7431 auto butterfly = TrustedImmPtr(nullptr);
7432 emitAllocateJSObjectWithKnownSize<DirectArguments>(
7433 resultGPR, TrustedImmPtr(structure), butterfly, scratch1GPR, scratch2GPR,
7434 slowPath, DirectArguments::allocationSize(std::max(knownLength, minCapacity)));
7435
7436 m_jit.store32(
7437 TrustedImm32(knownLength),
7438 JITCompiler::Address(resultGPR, DirectArguments::offsetOfLength()));
7439 } else {
7440 JITCompiler::Jump tooFewArguments;
7441 if (minCapacity) {
7442 tooFewArguments =
7443 m_jit.branch32(JITCompiler::Below, lengthGPR, TrustedImm32(minCapacity));
7444 }
7445 m_jit.lshift32(lengthGPR, TrustedImm32(3), scratch1GPR);
7446 m_jit.add32(TrustedImm32(DirectArguments::storageOffset()), scratch1GPR);
7447 if (minCapacity) {
7448 JITCompiler::Jump done = m_jit.jump();
7449 tooFewArguments.link(&m_jit);
7450 m_jit.move(TrustedImm32(DirectArguments::allocationSize(minCapacity)), scratch1GPR);
7451 done.link(&m_jit);
7452 }
7453
7454 emitAllocateVariableSizedJSObject<DirectArguments>(
7455 resultGPR, TrustedImmPtr(structure), scratch1GPR, scratch1GPR, scratch2GPR,
7456 slowPath);
7457
7458 m_jit.store32(
7459 lengthGPR, JITCompiler::Address(resultGPR, DirectArguments::offsetOfLength()));
7460 }
7461
7462 m_jit.store32(
7463 TrustedImm32(minCapacity),
7464 JITCompiler::Address(resultGPR, DirectArguments::offsetOfMinCapacity()));
7465
7466 m_jit.storePtr(
7467 TrustedImmPtr(nullptr), JITCompiler::Address(resultGPR, DirectArguments::offsetOfMappedArguments()));
7468
7469 m_jit.storePtr(
7470 TrustedImmPtr(nullptr), JITCompiler::Address(resultGPR, DirectArguments::offsetOfModifiedArgumentsDescriptor()));
7471
7472 if (lengthIsKnown) {
7473 addSlowPathGenerator(
7474 slowPathCall(
7475 slowPath, this, operationCreateDirectArguments, resultGPR, structure,
7476 knownLength, minCapacity));
7477 } else {
7478 auto generator = std::make_unique<CallCreateDirectArgumentsSlowPathGenerator>(
7479 slowPath, this, resultGPR, structure, lengthGPR, minCapacity);
7480 addSlowPathGenerator(WTFMove(generator));
7481 }
7482
7483 if (inlineCallFrame) {
7484 if (inlineCallFrame->isClosureCall) {
7485 m_jit.loadPtr(
7486 JITCompiler::addressFor(
7487 inlineCallFrame->calleeRecovery.virtualRegister()),
7488 scratch1GPR);
7489 } else {
7490 m_jit.move(
7491 TrustedImmPtr::weakPointer(
7492 m_jit.graph(), inlineCallFrame->calleeRecovery.constant().asCell()),
7493 scratch1GPR);
7494 }
7495 } else
7496 m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), scratch1GPR);
7497
7498 // Don't need a memory barriers since we just fast-created the activation, so the
7499 // activation must be young.
7500 m_jit.storePtr(
7501 scratch1GPR, JITCompiler::Address(resultGPR, DirectArguments::offsetOfCallee()));
7502
7503 VirtualRegister start = m_jit.argumentsStart(node->origin.semantic);
7504 if (lengthIsKnown) {
7505 for (unsigned i = 0; i < std::max(knownLength, minCapacity); ++i) {
7506 m_jit.loadValue(JITCompiler::addressFor(start + i), valueRegs);
7507 m_jit.storeValue(
7508 valueRegs, JITCompiler::Address(resultGPR, DirectArguments::offsetOfSlot(i)));
7509 }
7510 } else {
7511 JITCompiler::Jump done;
7512 if (minCapacity) {
7513 JITCompiler::Jump startLoop = m_jit.branch32(
7514 JITCompiler::AboveOrEqual, lengthGPR, TrustedImm32(minCapacity));
7515 m_jit.move(TrustedImm32(minCapacity), lengthGPR);
7516 startLoop.link(&m_jit);
7517 } else
7518 done = m_jit.branchTest32(MacroAssembler::Zero, lengthGPR);
7519 JITCompiler::Label loop = m_jit.label();
7520 m_jit.sub32(TrustedImm32(1), lengthGPR);
7521 m_jit.loadValue(
7522 JITCompiler::BaseIndex(
7523 GPRInfo::callFrameRegister, lengthGPR, JITCompiler::TimesEight,
7524 start.offset() * static_cast<int>(sizeof(Register))),
7525 valueRegs);
7526 m_jit.storeValue(
7527 valueRegs,
7528 JITCompiler::BaseIndex(
7529 resultGPR, lengthGPR, JITCompiler::TimesEight,
7530 DirectArguments::storageOffset()));
7531 m_jit.branchTest32(MacroAssembler::NonZero, lengthGPR).linkTo(loop, &m_jit);
7532 if (done.isSet())
7533 done.link(&m_jit);
7534 }
7535
7536 m_jit.mutatorFence(*m_jit.vm());
7537
7538 cellResult(resultGPR, node);
7539}
7540
7541void SpeculativeJIT::compileGetFromArguments(Node* node)
7542{
7543 SpeculateCellOperand arguments(this, node->child1());
7544 JSValueRegsTemporary result(this);
7545
7546 GPRReg argumentsGPR = arguments.gpr();
7547 JSValueRegs resultRegs = result.regs();
7548
7549 m_jit.loadValue(JITCompiler::Address(argumentsGPR, DirectArguments::offsetOfSlot(node->capturedArgumentsOffset().offset())), resultRegs);
7550 jsValueResult(resultRegs, node);
7551}
7552
7553void SpeculativeJIT::compilePutToArguments(Node* node)
7554{
7555 SpeculateCellOperand arguments(this, node->child1());
7556 JSValueOperand value(this, node->child2());
7557
7558 GPRReg argumentsGPR = arguments.gpr();
7559 JSValueRegs valueRegs = value.jsValueRegs();
7560
7561 m_jit.storeValue(valueRegs, JITCompiler::Address(argumentsGPR, DirectArguments::offsetOfSlot(node->capturedArgumentsOffset().offset())));
7562 noResult(node);
7563}
7564
7565void SpeculativeJIT::compileGetArgument(Node* node)
7566{
7567 GPRTemporary argumentCount(this);
7568 JSValueRegsTemporary result(this);
7569 GPRReg argumentCountGPR = argumentCount.gpr();
7570 JSValueRegs resultRegs = result.regs();
7571 m_jit.load32(CCallHelpers::payloadFor(m_jit.argumentCount(node->origin.semantic)), argumentCountGPR);
7572 auto argumentOutOfBounds = m_jit.branch32(CCallHelpers::LessThanOrEqual, argumentCountGPR, CCallHelpers::TrustedImm32(node->argumentIndex()));
7573 m_jit.loadValue(CCallHelpers::addressFor(CCallHelpers::argumentsStart(node->origin.semantic) + node->argumentIndex() - 1), resultRegs);
7574 auto done = m_jit.jump();
7575
7576 argumentOutOfBounds.link(&m_jit);
7577 m_jit.moveValue(jsUndefined(), resultRegs);
7578
7579 done.link(&m_jit);
7580 jsValueResult(resultRegs, node);
7581}
7582
7583void SpeculativeJIT::compileCreateScopedArguments(Node* node)
7584{
7585 SpeculateCellOperand scope(this, node->child1());
7586 GPRReg scopeGPR = scope.gpr();
7587
7588 GPRFlushedCallResult result(this);
7589 GPRReg resultGPR = result.gpr();
7590 flushRegisters();
7591
7592 // We set up the arguments ourselves, because we have the whole register file and we can
7593 // set them up directly into the argument registers. This also means that we don't have to
7594 // invent a four-argument-register shuffle.
7595
7596 // Arguments: 0:exec, 1:structure, 2:start, 3:length, 4:callee, 5:scope
7597
7598 // Do the scopeGPR first, since it might alias an argument register.
7599 m_jit.setupArgument(5, [&] (GPRReg destGPR) { m_jit.move(scopeGPR, destGPR); });
7600
7601 // These other things could be done in any order.
7602 m_jit.setupArgument(4, [&] (GPRReg destGPR) { emitGetCallee(node->origin.semantic, destGPR); });
7603 m_jit.setupArgument(3, [&] (GPRReg destGPR) { emitGetLength(node->origin.semantic, destGPR); });
7604 m_jit.setupArgument(2, [&] (GPRReg destGPR) { emitGetArgumentStart(node->origin.semantic, destGPR); });
7605 m_jit.setupArgument(
7606 1, [&] (GPRReg destGPR) {
7607 m_jit.move(
7608 TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.globalObjectFor(node->origin.semantic)->scopedArgumentsStructure()),
7609 destGPR);
7610 });
7611 m_jit.setupArgument(0, [&] (GPRReg destGPR) { m_jit.move(GPRInfo::callFrameRegister, destGPR); });
7612
7613 appendCallSetResult(operationCreateScopedArguments, resultGPR);
7614 m_jit.exceptionCheck();
7615
7616 cellResult(resultGPR, node);
7617}
7618
7619void SpeculativeJIT::compileCreateClonedArguments(Node* node)
7620{
7621 GPRFlushedCallResult result(this);
7622 GPRReg resultGPR = result.gpr();
7623 flushRegisters();
7624
7625 // We set up the arguments ourselves, because we have the whole register file and we can
7626 // set them up directly into the argument registers.
7627
7628 // Arguments: 0:exec, 1:structure, 2:start, 3:length, 4:callee
7629 m_jit.setupArgument(4, [&] (GPRReg destGPR) { emitGetCallee(node->origin.semantic, destGPR); });
7630 m_jit.setupArgument(3, [&] (GPRReg destGPR) { emitGetLength(node->origin.semantic, destGPR); });
7631 m_jit.setupArgument(2, [&] (GPRReg destGPR) { emitGetArgumentStart(node->origin.semantic, destGPR); });
7632 m_jit.setupArgument(
7633 1, [&] (GPRReg destGPR) {
7634 m_jit.move(
7635 TrustedImmPtr::weakPointer(
7636 m_jit.graph(), m_jit.globalObjectFor(node->origin.semantic)->clonedArgumentsStructure()),
7637 destGPR);
7638 });
7639 m_jit.setupArgument(0, [&] (GPRReg destGPR) { m_jit.move(GPRInfo::callFrameRegister, destGPR); });
7640
7641 appendCallSetResult(operationCreateClonedArguments, resultGPR);
7642 m_jit.exceptionCheck();
7643
7644 cellResult(resultGPR, node);
7645}
7646
7647void SpeculativeJIT::compileCreateRest(Node* node)
7648{
7649 ASSERT(node->op() == CreateRest);
7650
7651#if !CPU(X86)
7652 if (m_jit.graph().isWatchingHavingABadTimeWatchpoint(node)) {
7653 SpeculateStrictInt32Operand arrayLength(this, node->child1());
7654 GPRTemporary arrayResult(this);
7655
7656 GPRReg arrayLengthGPR = arrayLength.gpr();
7657 GPRReg arrayResultGPR = arrayResult.gpr();
7658
7659 // We can tell compileAllocateNewArrayWithSize() that it does not need to check
7660 // for large arrays and use ArrayStorage structure because arrayLength here will
7661 // always be bounded by stack size. Realistically, we won't be able to push enough
7662 // arguments to have arrayLength exceed MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH.
7663 bool shouldAllowForArrayStorageStructureForLargeArrays = false;
7664 ASSERT(m_jit.graph().globalObjectFor(node->origin.semantic)->restParameterStructure()->indexingMode() == ArrayWithContiguous || m_jit.graph().globalObjectFor(node->origin.semantic)->isHavingABadTime());
7665 compileAllocateNewArrayWithSize(m_jit.graph().globalObjectFor(node->origin.semantic), arrayResultGPR, arrayLengthGPR, ArrayWithContiguous, shouldAllowForArrayStorageStructureForLargeArrays);
7666
7667 GPRTemporary argumentsStart(this);
7668 GPRReg argumentsStartGPR = argumentsStart.gpr();
7669
7670 emitGetArgumentStart(node->origin.semantic, argumentsStartGPR);
7671
7672 GPRTemporary butterfly(this);
7673 GPRTemporary currentLength(this);
7674 JSValueRegsTemporary value(this);
7675
7676 JSValueRegs valueRegs = value.regs();
7677 GPRReg currentLengthGPR = currentLength.gpr();
7678 GPRReg butterflyGPR = butterfly.gpr();
7679
7680 m_jit.loadPtr(MacroAssembler::Address(arrayResultGPR, JSObject::butterflyOffset()), butterflyGPR);
7681
7682 CCallHelpers::Jump skipLoop = m_jit.branch32(MacroAssembler::Equal, arrayLengthGPR, TrustedImm32(0));
7683 m_jit.zeroExtend32ToPtr(arrayLengthGPR, currentLengthGPR);
7684 m_jit.addPtr(Imm32(sizeof(Register) * node->numberOfArgumentsToSkip()), argumentsStartGPR);
7685
7686 auto loop = m_jit.label();
7687 m_jit.sub32(TrustedImm32(1), currentLengthGPR);
7688 m_jit.loadValue(JITCompiler::BaseIndex(argumentsStartGPR, currentLengthGPR, MacroAssembler::TimesEight), valueRegs);
7689 m_jit.storeValue(valueRegs, MacroAssembler::BaseIndex(butterflyGPR, currentLengthGPR, MacroAssembler::TimesEight));
7690 m_jit.branch32(MacroAssembler::NotEqual, currentLengthGPR, TrustedImm32(0)).linkTo(loop, &m_jit);
7691
7692 skipLoop.link(&m_jit);
7693 cellResult(arrayResultGPR, node);
7694 return;
7695 }
7696#endif // !CPU(X86)
7697
7698 SpeculateStrictInt32Operand arrayLength(this, node->child1());
7699 GPRTemporary argumentsStart(this);
7700 GPRTemporary numberOfArgumentsToSkip(this);
7701
7702 GPRReg arrayLengthGPR = arrayLength.gpr();
7703 GPRReg argumentsStartGPR = argumentsStart.gpr();
7704
7705 emitGetArgumentStart(node->origin.semantic, argumentsStartGPR);
7706
7707 flushRegisters();
7708
7709 GPRFlushedCallResult result(this);
7710 GPRReg resultGPR = result.gpr();
7711 callOperation(operationCreateRest, resultGPR, argumentsStartGPR, Imm32(node->numberOfArgumentsToSkip()), arrayLengthGPR);
7712 m_jit.exceptionCheck();
7713
7714 cellResult(resultGPR, node);
7715}
7716
7717void SpeculativeJIT::compileSpread(Node* node)
7718{
7719 ASSERT(node->op() == Spread);
7720
7721 SpeculateCellOperand operand(this, node->child1());
7722 GPRReg argument = operand.gpr();
7723
7724 if (node->child1().useKind() == ArrayUse)
7725 speculateArray(node->child1(), argument);
7726
7727 if (m_jit.graph().canDoFastSpread(node, m_state.forNode(node->child1()))) {
7728#if USE(JSVALUE64)
7729 GPRTemporary result(this);
7730 GPRTemporary scratch1(this);
7731 GPRTemporary scratch2(this);
7732 GPRTemporary length(this);
7733 FPRTemporary doubleRegister(this);
7734
7735 GPRReg resultGPR = result.gpr();
7736 GPRReg scratch1GPR = scratch1.gpr();
7737 GPRReg scratch2GPR = scratch2.gpr();
7738 GPRReg lengthGPR = length.gpr();
7739 FPRReg doubleFPR = doubleRegister.fpr();
7740
7741 MacroAssembler::JumpList slowPath;
7742
7743 m_jit.load8(MacroAssembler::Address(argument, JSCell::indexingTypeAndMiscOffset()), scratch1GPR);
7744 m_jit.and32(TrustedImm32(IndexingShapeMask), scratch1GPR);
7745 m_jit.sub32(TrustedImm32(Int32Shape), scratch1GPR);
7746
7747 slowPath.append(m_jit.branch32(MacroAssembler::Above, scratch1GPR, TrustedImm32(ContiguousShape - Int32Shape)));
7748
7749 m_jit.loadPtr(MacroAssembler::Address(argument, JSObject::butterflyOffset()), lengthGPR);
7750 m_jit.load32(MacroAssembler::Address(lengthGPR, Butterfly::offsetOfPublicLength()), lengthGPR);
7751 static_assert(sizeof(JSValue) == 8 && 1 << 3 == 8, "This is strongly assumed in the code below.");
7752 m_jit.move(lengthGPR, scratch1GPR);
7753 m_jit.lshift32(TrustedImm32(3), scratch1GPR);
7754 m_jit.add32(TrustedImm32(JSFixedArray::offsetOfData()), scratch1GPR);
7755
7756 m_jit.emitAllocateVariableSizedCell<JSFixedArray>(*m_jit.vm(), resultGPR, TrustedImmPtr(m_jit.graph().registerStructure(m_jit.graph().m_vm.fixedArrayStructure.get())), scratch1GPR, scratch1GPR, scratch2GPR, slowPath);
7757 m_jit.store32(lengthGPR, MacroAssembler::Address(resultGPR, JSFixedArray::offsetOfSize()));
7758
7759 m_jit.loadPtr(MacroAssembler::Address(argument, JSObject::butterflyOffset()), scratch1GPR);
7760
7761 MacroAssembler::JumpList done;
7762
7763 m_jit.load8(MacroAssembler::Address(argument, JSCell::indexingTypeAndMiscOffset()), scratch2GPR);
7764 m_jit.and32(TrustedImm32(IndexingShapeMask), scratch2GPR);
7765 auto isDoubleArray = m_jit.branch32(MacroAssembler::Equal, scratch2GPR, TrustedImm32(DoubleShape));
7766
7767 {
7768 done.append(m_jit.branchTest32(MacroAssembler::Zero, lengthGPR));
7769 auto loopStart = m_jit.label();
7770 m_jit.sub32(TrustedImm32(1), lengthGPR);
7771 m_jit.load64(MacroAssembler::BaseIndex(scratch1GPR, lengthGPR, MacroAssembler::TimesEight), scratch2GPR);
7772 auto notEmpty = m_jit.branchIfNotEmpty(scratch2GPR);
7773 m_jit.move(TrustedImm64(JSValue::encode(jsUndefined())), scratch2GPR);
7774 notEmpty.link(&m_jit);
7775 m_jit.store64(scratch2GPR, MacroAssembler::BaseIndex(resultGPR, lengthGPR, MacroAssembler::TimesEight, JSFixedArray::offsetOfData()));
7776 m_jit.branchTest32(MacroAssembler::NonZero, lengthGPR).linkTo(loopStart, &m_jit);
7777 done.append(m_jit.jump());
7778 }
7779
7780 isDoubleArray.link(&m_jit);
7781 {
7782 done.append(m_jit.branchTest32(MacroAssembler::Zero, lengthGPR));
7783 auto loopStart = m_jit.label();
7784 m_jit.sub32(TrustedImm32(1), lengthGPR);
7785 m_jit.loadDouble(MacroAssembler::BaseIndex(scratch1GPR, lengthGPR, MacroAssembler::TimesEight), doubleFPR);
7786 auto notEmpty = m_jit.branchIfNotNaN(doubleFPR);
7787 m_jit.move(TrustedImm64(JSValue::encode(jsUndefined())), scratch2GPR);
7788 auto doStore = m_jit.jump();
7789 notEmpty.link(&m_jit);
7790 m_jit.boxDouble(doubleFPR, scratch2GPR);
7791 doStore.link(&m_jit);
7792 m_jit.store64(scratch2GPR, MacroAssembler::BaseIndex(resultGPR, lengthGPR, MacroAssembler::TimesEight, JSFixedArray::offsetOfData()));
7793 m_jit.branchTest32(MacroAssembler::NonZero, lengthGPR).linkTo(loopStart, &m_jit);
7794 done.append(m_jit.jump());
7795 }
7796
7797 m_jit.mutatorFence(*m_jit.vm());
7798
7799 slowPath.link(&m_jit);
7800 addSlowPathGenerator(slowPathCall(m_jit.jump(), this, operationSpreadFastArray, resultGPR, argument));
7801
7802 done.link(&m_jit);
7803 cellResult(resultGPR, node);
7804#else
7805 flushRegisters();
7806
7807 GPRFlushedCallResult result(this);
7808 GPRReg resultGPR = result.gpr();
7809 callOperation(operationSpreadFastArray, resultGPR, argument);
7810 m_jit.exceptionCheck();
7811 cellResult(resultGPR, node);
7812#endif // USE(JSVALUE64)
7813 } else {
7814 flushRegisters();
7815
7816 GPRFlushedCallResult result(this);
7817 GPRReg resultGPR = result.gpr();
7818 callOperation(operationSpreadGeneric, resultGPR, argument);
7819 m_jit.exceptionCheck();
7820 cellResult(resultGPR, node);
7821 }
7822}
7823
7824void SpeculativeJIT::compileNewArray(Node* node)
7825{
7826 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
7827 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) {
7828 RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()));
7829 DFG_ASSERT(m_jit.graph(), node, structure->indexingType() == node->indexingType(), structure->indexingType(), node->indexingType());
7830 ASSERT(
7831 hasUndecided(structure->indexingType())
7832 || hasInt32(structure->indexingType())
7833 || hasDouble(structure->indexingType())
7834 || hasContiguous(structure->indexingType()));
7835
7836 unsigned numElements = node->numChildren();
7837 unsigned vectorLengthHint = node->vectorLengthHint();
7838 ASSERT(vectorLengthHint >= numElements);
7839
7840 GPRTemporary result(this);
7841 GPRTemporary storage(this);
7842
7843 GPRReg resultGPR = result.gpr();
7844 GPRReg storageGPR = storage.gpr();
7845
7846 emitAllocateRawObject(resultGPR, structure, storageGPR, numElements, vectorLengthHint);
7847
7848 // At this point, one way or another, resultGPR and storageGPR have pointers to
7849 // the JSArray and the Butterfly, respectively.
7850
7851 ASSERT(!hasUndecided(structure->indexingType()) || !node->numChildren());
7852
7853 for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
7854 Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
7855 switch (node->indexingType()) {
7856 case ALL_BLANK_INDEXING_TYPES:
7857 case ALL_UNDECIDED_INDEXING_TYPES:
7858 CRASH();
7859 break;
7860 case ALL_DOUBLE_INDEXING_TYPES: {
7861 SpeculateDoubleOperand operand(this, use);
7862 FPRReg opFPR = operand.fpr();
7863 DFG_TYPE_CHECK(
7864 JSValueRegs(), use, SpecDoubleReal,
7865 m_jit.branchIfNaN(opFPR));
7866 m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx));
7867 break;
7868 }
7869 case ALL_INT32_INDEXING_TYPES:
7870 case ALL_CONTIGUOUS_INDEXING_TYPES: {
7871 JSValueOperand operand(this, use, ManualOperandSpeculation);
7872 JSValueRegs operandRegs = operand.jsValueRegs();
7873 if (hasInt32(node->indexingType())) {
7874 DFG_TYPE_CHECK(
7875 operandRegs, use, SpecInt32Only,
7876 m_jit.branchIfNotInt32(operandRegs));
7877 }
7878 m_jit.storeValue(operandRegs, MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx));
7879 break;
7880 }
7881 default:
7882 CRASH();
7883 break;
7884 }
7885 }
7886
7887 // Yuck, we should *really* have a way of also returning the storageGPR. But
7888 // that's the least of what's wrong with this code. We really shouldn't be
7889 // allocating the array after having computed - and probably spilled to the
7890 // stack - all of the things that will go into the array. The solution to that
7891 // bigger problem will also likely fix the redundancy in reloading the storage
7892 // pointer that we currently have.
7893
7894 cellResult(resultGPR, node);
7895 return;
7896 }
7897
7898 if (!node->numChildren()) {
7899 flushRegisters();
7900 GPRFlushedCallResult result(this);
7901 callOperation(operationNewEmptyArray, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())));
7902 m_jit.exceptionCheck();
7903 cellResult(result.gpr(), node);
7904 return;
7905 }
7906
7907 size_t scratchSize = sizeof(EncodedJSValue) * node->numChildren();
7908 ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
7909 EncodedJSValue* buffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : nullptr;
7910
7911 for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
7912 // Need to perform the speculations that this node promises to perform. If we're
7913 // emitting code here and the indexing type is not array storage then there is
7914 // probably something hilarious going on and we're already failing at all the
7915 // things, but at least we're going to be sound.
7916 Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
7917 switch (node->indexingType()) {
7918 case ALL_BLANK_INDEXING_TYPES:
7919 case ALL_UNDECIDED_INDEXING_TYPES:
7920 CRASH();
7921 break;
7922 case ALL_DOUBLE_INDEXING_TYPES: {
7923 SpeculateDoubleOperand operand(this, use);
7924 FPRReg opFPR = operand.fpr();
7925 DFG_TYPE_CHECK(
7926 JSValueRegs(), use, SpecDoubleReal,
7927 m_jit.branchIfNaN(opFPR));
7928#if USE(JSVALUE64)
7929 JSValueRegsTemporary scratch(this);
7930 JSValueRegs scratchRegs = scratch.regs();
7931 m_jit.boxDouble(opFPR, scratchRegs);
7932 m_jit.storeValue(scratchRegs, buffer + operandIdx);
7933#else
7934 m_jit.storeDouble(opFPR, TrustedImmPtr(buffer + operandIdx));
7935#endif
7936 operand.use();
7937 break;
7938 }
7939 case ALL_INT32_INDEXING_TYPES:
7940 case ALL_CONTIGUOUS_INDEXING_TYPES:
7941 case ALL_ARRAY_STORAGE_INDEXING_TYPES: {
7942 JSValueOperand operand(this, use, ManualOperandSpeculation);
7943 JSValueRegs operandRegs = operand.jsValueRegs();
7944 if (hasInt32(node->indexingType())) {
7945 DFG_TYPE_CHECK(
7946 operandRegs, use, SpecInt32Only,
7947 m_jit.branchIfNotInt32(operandRegs));
7948 }
7949 m_jit.storeValue(operandRegs, buffer + operandIdx);
7950 operand.use();
7951 break;
7952 }
7953 default:
7954 CRASH();
7955 break;
7956 }
7957 }
7958
7959 flushRegisters();
7960
7961 if (scratchSize) {
7962 GPRTemporary scratch(this);
7963
7964 // Tell GC mark phase how much of the scratch buffer is active during call.
7965 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), scratch.gpr());
7966 m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr());
7967 }
7968
7969 GPRFlushedCallResult result(this);
7970
7971 callOperation(
7972 operationNewArray, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())),
7973 static_cast<void*>(buffer), size_t(node->numChildren()));
7974 m_jit.exceptionCheck();
7975
7976 if (scratchSize) {
7977 GPRTemporary scratch(this);
7978
7979 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), scratch.gpr());
7980 m_jit.storePtr(TrustedImmPtr(nullptr), scratch.gpr());
7981 }
7982
7983 cellResult(result.gpr(), node, UseChildrenCalledExplicitly);
7984}
7985
7986void SpeculativeJIT::compileNewArrayWithSpread(Node* node)
7987{
7988 ASSERT(node->op() == NewArrayWithSpread);
7989
7990#if USE(JSVALUE64)
7991 if (m_jit.graph().isWatchingHavingABadTimeWatchpoint(node)) {
7992 GPRTemporary result(this);
7993 GPRReg resultGPR = result.gpr();
7994
7995 BitVector* bitVector = node->bitVector();
7996 {
7997 unsigned startLength = 0;
7998 for (unsigned i = 0; i < node->numChildren(); ++i) {
7999 if (!bitVector->get(i))
8000 ++startLength;
8001 }
8002
8003 GPRTemporary length(this);
8004 GPRReg lengthGPR = length.gpr();
8005 m_jit.move(TrustedImm32(startLength), lengthGPR);
8006
8007 for (unsigned i = 0; i < node->numChildren(); ++i) {
8008 if (bitVector->get(i)) {
8009 Edge use = m_jit.graph().varArgChild(node, i);
8010 SpeculateCellOperand fixedArray(this, use);
8011 GPRReg fixedArrayGPR = fixedArray.gpr();
8012 speculationCheck(Overflow, JSValueRegs(), nullptr, m_jit.branchAdd32(MacroAssembler::Overflow, MacroAssembler::Address(fixedArrayGPR, JSFixedArray::offsetOfSize()), lengthGPR));
8013 }
8014 }
8015
8016 speculationCheck(Overflow, JSValueRegs(), nullptr, m_jit.branch32(MacroAssembler::AboveOrEqual, lengthGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)));
8017
8018 // We can tell compileAllocateNewArrayWithSize() that it does not need to
8019 // check for large arrays and use ArrayStorage structure because we already
8020 // ensured above that the spread array length will definitely fit in a
8021 // non-ArrayStorage shaped array.
8022 bool shouldAllowForArrayStorageStructureForLargeArrays = false;
8023 ASSERT(m_jit.graph().globalObjectFor(node->origin.semantic)->restParameterStructure()->indexingType() == ArrayWithContiguous || m_jit.graph().globalObjectFor(node->origin.semantic)->isHavingABadTime());
8024 compileAllocateNewArrayWithSize(m_jit.graph().globalObjectFor(node->origin.semantic), resultGPR, lengthGPR, ArrayWithContiguous, shouldAllowForArrayStorageStructureForLargeArrays);
8025 }
8026
8027 GPRTemporary index(this);
8028 GPRReg indexGPR = index.gpr();
8029
8030 GPRTemporary storage(this);
8031 GPRReg storageGPR = storage.gpr();
8032
8033 m_jit.move(TrustedImm32(0), indexGPR);
8034 m_jit.loadPtr(MacroAssembler::Address(resultGPR, JSObject::butterflyOffset()), storageGPR);
8035
8036 for (unsigned i = 0; i < node->numChildren(); ++i) {
8037 Edge use = m_jit.graph().varArgChild(node, i);
8038 if (bitVector->get(i)) {
8039 SpeculateCellOperand fixedArray(this, use);
8040 GPRReg fixedArrayGPR = fixedArray.gpr();
8041
8042 GPRTemporary fixedIndex(this);
8043 GPRReg fixedIndexGPR = fixedIndex.gpr();
8044
8045 GPRTemporary item(this);
8046 GPRReg itemGPR = item.gpr();
8047
8048 GPRTemporary fixedLength(this);
8049 GPRReg fixedLengthGPR = fixedLength.gpr();
8050
8051 m_jit.load32(MacroAssembler::Address(fixedArrayGPR, JSFixedArray::offsetOfSize()), fixedLengthGPR);
8052 m_jit.move(TrustedImm32(0), fixedIndexGPR);
8053 auto done = m_jit.branchPtr(MacroAssembler::AboveOrEqual, fixedIndexGPR, fixedLengthGPR);
8054 auto loopStart = m_jit.label();
8055 m_jit.load64(
8056 MacroAssembler::BaseIndex(fixedArrayGPR, fixedIndexGPR, MacroAssembler::TimesEight, JSFixedArray::offsetOfData()),
8057 itemGPR);
8058
8059 m_jit.store64(itemGPR, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight));
8060 m_jit.addPtr(TrustedImm32(1), fixedIndexGPR);
8061 m_jit.addPtr(TrustedImm32(1), indexGPR);
8062 m_jit.branchPtr(MacroAssembler::Below, fixedIndexGPR, fixedLengthGPR).linkTo(loopStart, &m_jit);
8063
8064 done.link(&m_jit);
8065 } else {
8066 JSValueOperand item(this, use);
8067 GPRReg itemGPR = item.gpr();
8068 m_jit.store64(itemGPR, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight));
8069 m_jit.addPtr(TrustedImm32(1), indexGPR);
8070 }
8071 }
8072
8073 cellResult(resultGPR, node);
8074 return;
8075 }
8076#endif // USE(JSVALUE64)
8077
8078 ASSERT(node->numChildren());
8079 size_t scratchSize = sizeof(EncodedJSValue) * node->numChildren();
8080 ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
8081 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
8082
8083 BitVector* bitVector = node->bitVector();
8084 for (unsigned i = 0; i < node->numChildren(); ++i) {
8085 Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + i];
8086 if (bitVector->get(i)) {
8087 SpeculateCellOperand fixedArray(this, use);
8088 GPRReg arrayGPR = fixedArray.gpr();
8089#if USE(JSVALUE64)
8090 m_jit.store64(arrayGPR, &buffer[i]);
8091#else
8092 char* pointer = static_cast<char*>(static_cast<void*>(&buffer[i]));
8093 m_jit.store32(arrayGPR, pointer + PayloadOffset);
8094 m_jit.store32(TrustedImm32(JSValue::CellTag), pointer + TagOffset);
8095#endif
8096 } else {
8097 JSValueOperand input(this, use);
8098 JSValueRegs inputRegs = input.jsValueRegs();
8099 m_jit.storeValue(inputRegs, &buffer[i]);
8100 }
8101 }
8102
8103 {
8104 GPRTemporary scratch(this);
8105 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), scratch.gpr());
8106 m_jit.storePtr(TrustedImmPtr(scratchSize), MacroAssembler::Address(scratch.gpr()));
8107 }
8108
8109 flushRegisters();
8110
8111 GPRFlushedCallResult result(this);
8112 GPRReg resultGPR = result.gpr();
8113
8114 callOperation(operationNewArrayWithSpreadSlow, resultGPR, buffer, node->numChildren());
8115 m_jit.exceptionCheck();
8116 {
8117 GPRTemporary scratch(this);
8118 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), scratch.gpr());
8119 m_jit.storePtr(TrustedImmPtr(nullptr), MacroAssembler::Address(scratch.gpr()));
8120 }
8121
8122 cellResult(resultGPR, node);
8123}
8124
8125void SpeculativeJIT::compileGetRestLength(Node* node)
8126{
8127 ASSERT(node->op() == GetRestLength);
8128
8129 GPRTemporary result(this);
8130 GPRReg resultGPR = result.gpr();
8131
8132 emitGetLength(node->origin.semantic, resultGPR);
8133 CCallHelpers::Jump hasNonZeroLength = m_jit.branch32(MacroAssembler::Above, resultGPR, Imm32(node->numberOfArgumentsToSkip()));
8134 m_jit.move(TrustedImm32(0), resultGPR);
8135 CCallHelpers::Jump done = m_jit.jump();
8136 hasNonZeroLength.link(&m_jit);
8137 if (node->numberOfArgumentsToSkip())
8138 m_jit.sub32(TrustedImm32(node->numberOfArgumentsToSkip()), resultGPR);
8139 done.link(&m_jit);
8140 int32Result(resultGPR, node);
8141}
8142
8143void SpeculativeJIT::emitPopulateSliceIndex(Edge& target, Optional<GPRReg> indexGPR, GPRReg lengthGPR, GPRReg resultGPR)
8144{
8145 if (target->isInt32Constant()) {
8146 int32_t value = target->asInt32();
8147 if (value == 0) {
8148 m_jit.move(TrustedImm32(0), resultGPR);
8149 return;
8150 }
8151
8152 MacroAssembler::JumpList done;
8153 if (value > 0) {
8154 m_jit.move(TrustedImm32(value), resultGPR);
8155 done.append(m_jit.branch32(MacroAssembler::BelowOrEqual, resultGPR, lengthGPR));
8156 m_jit.move(lengthGPR, resultGPR);
8157 } else {
8158 ASSERT(value != 0);
8159 m_jit.move(lengthGPR, resultGPR);
8160 done.append(m_jit.branchAdd32(MacroAssembler::PositiveOrZero, TrustedImm32(value), resultGPR));
8161 m_jit.move(TrustedImm32(0), resultGPR);
8162 }
8163 done.link(&m_jit);
8164 return;
8165 }
8166
8167 Optional<SpeculateInt32Operand> index;
8168 if (!indexGPR) {
8169 index.emplace(this, target);
8170 indexGPR = index->gpr();
8171 }
8172 MacroAssembler::JumpList done;
8173
8174 auto isPositive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, indexGPR.value(), TrustedImm32(0));
8175 m_jit.move(lengthGPR, resultGPR);
8176 done.append(m_jit.branchAdd32(MacroAssembler::PositiveOrZero, indexGPR.value(), resultGPR));
8177 m_jit.move(TrustedImm32(0), resultGPR);
8178 done.append(m_jit.jump());
8179
8180 isPositive.link(&m_jit);
8181 m_jit.move(indexGPR.value(), resultGPR);
8182 done.append(m_jit.branch32(MacroAssembler::BelowOrEqual, resultGPR, lengthGPR));
8183 m_jit.move(lengthGPR, resultGPR);
8184
8185 done.link(&m_jit);
8186}
8187
8188void SpeculativeJIT::compileArraySlice(Node* node)
8189{
8190 ASSERT(node->op() == ArraySlice);
8191
8192 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
8193
8194 GPRTemporary temp(this);
8195 StorageOperand storage(this, m_jit.graph().varArgChild(node, node->numChildren() - 1));
8196 GPRTemporary result(this);
8197
8198 GPRReg storageGPR = storage.gpr();
8199 GPRReg resultGPR = result.gpr();
8200 GPRReg tempGPR = temp.gpr();
8201
8202 if (node->numChildren() == 2)
8203 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), tempGPR);
8204 else {
8205 ASSERT(node->numChildren() == 3 || node->numChildren() == 4);
8206 GPRTemporary tempLength(this);
8207 GPRReg lengthGPR = tempLength.gpr();
8208 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), lengthGPR);
8209
8210 if (node->numChildren() == 4)
8211 emitPopulateSliceIndex(m_jit.graph().varArgChild(node, 2), WTF::nullopt, lengthGPR, tempGPR);
8212 else
8213 m_jit.move(lengthGPR, tempGPR);
8214
8215 if (m_jit.graph().varArgChild(node, 1)->isInt32Constant() && m_jit.graph().varArgChild(node, 1)->asInt32() == 0) {
8216 // Do nothing for array.slice(0, end) or array.slice(0) cases.
8217 // `tempGPR` already points to the size of a newly created array.
8218 } else {
8219 GPRTemporary tempStartIndex(this);
8220 GPRReg startGPR = tempStartIndex.gpr();
8221 emitPopulateSliceIndex(m_jit.graph().varArgChild(node, 1), WTF::nullopt, lengthGPR, startGPR);
8222
8223 auto tooBig = m_jit.branch32(MacroAssembler::Above, startGPR, tempGPR);
8224 m_jit.sub32(startGPR, tempGPR); // the size of the array we'll make.
8225 auto done = m_jit.jump();
8226
8227 tooBig.link(&m_jit);
8228 m_jit.move(TrustedImm32(0), tempGPR);
8229 done.link(&m_jit);
8230 }
8231 }
8232
8233
8234 GPRTemporary temp3(this);
8235 GPRReg tempValue = temp3.gpr();
8236 {
8237 SpeculateCellOperand cell(this, m_jit.graph().varArgChild(node, 0));
8238 m_jit.load8(MacroAssembler::Address(cell.gpr(), JSCell::indexingTypeAndMiscOffset()), tempValue);
8239 // We can ignore the writability of the cell since we won't write to the source.
8240 m_jit.and32(TrustedImm32(AllWritableArrayTypesAndHistory), tempValue);
8241 }
8242
8243 {
8244 JSValueRegsTemporary emptyValue(this);
8245 JSValueRegs emptyValueRegs = emptyValue.regs();
8246
8247 GPRTemporary storage(this);
8248 GPRReg storageResultGPR = storage.gpr();
8249
8250 GPRReg sizeGPR = tempGPR;
8251
8252 CCallHelpers::JumpList done;
8253
8254 auto emitMoveEmptyValue = [&] (JSValue v) {
8255 m_jit.moveValue(v, emptyValueRegs);
8256 };
8257
8258 auto isContiguous = m_jit.branch32(MacroAssembler::Equal, tempValue, TrustedImm32(ArrayWithContiguous));
8259 auto isInt32 = m_jit.branch32(MacroAssembler::Equal, tempValue, TrustedImm32(ArrayWithInt32));
8260 // When we emit an ArraySlice, we dominate the use of the array by a CheckStructure
8261 // to ensure the incoming array is one to be one of the original array structures
8262 // with one of the following indexing shapes: Int32, Contiguous, Double. Therefore,
8263 // we're a double array here.
8264 m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithDouble))), tempValue);
8265 emitMoveEmptyValue(jsNaN());
8266 done.append(m_jit.jump());
8267
8268 isContiguous.link(&m_jit);
8269 m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithContiguous))), tempValue);
8270 emitMoveEmptyValue(JSValue());
8271 done.append(m_jit.jump());
8272
8273 isInt32.link(&m_jit);
8274 m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithInt32))), tempValue);
8275 emitMoveEmptyValue(JSValue());
8276
8277 done.link(&m_jit);
8278
8279 MacroAssembler::JumpList slowCases;
8280 m_jit.move(TrustedImmPtr(nullptr), storageResultGPR);
8281 // Enable the fast case on 64-bit platforms, where a sufficient amount of GP registers should be available.
8282 // Other platforms could support the same approach with custom code, but that is not currently worth the extra code maintenance.
8283 if (is64Bit()) {
8284 GPRTemporary scratch(this);
8285 GPRTemporary scratch2(this);
8286 GPRReg scratchGPR = scratch.gpr();
8287 GPRReg scratch2GPR = scratch2.gpr();
8288
8289 emitAllocateButterfly(storageResultGPR, sizeGPR, scratchGPR, scratch2GPR, resultGPR, slowCases);
8290 emitInitializeButterfly(storageResultGPR, sizeGPR, emptyValueRegs, scratchGPR);
8291 emitAllocateJSObject<JSArray>(resultGPR, tempValue, storageResultGPR, scratchGPR, scratch2GPR, slowCases);
8292 m_jit.mutatorFence(*m_jit.vm());
8293 } else {
8294 slowCases.append(m_jit.jump());
8295 }
8296
8297 addSlowPathGenerator(std::make_unique<CallArrayAllocatorWithVariableStructureVariableSizeSlowPathGenerator>(
8298 slowCases, this, operationNewArrayWithSize, resultGPR, tempValue, sizeGPR, storageResultGPR));
8299 }
8300
8301 GPRTemporary temp4(this);
8302 GPRReg loadIndex = temp4.gpr();
8303
8304 if (node->numChildren() == 2) {
8305 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), tempGPR);
8306 m_jit.move(TrustedImm32(0), loadIndex);
8307 } else {
8308 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), tempValue);
8309 if (node->numChildren() == 4)
8310 emitPopulateSliceIndex(m_jit.graph().varArgChild(node, 2), WTF::nullopt, tempValue, tempGPR);
8311 else
8312 m_jit.move(tempValue, tempGPR);
8313 emitPopulateSliceIndex(m_jit.graph().varArgChild(node, 1), WTF::nullopt, tempValue, loadIndex);
8314 }
8315
8316 GPRTemporary temp5(this);
8317 GPRReg storeIndex = temp5.gpr();
8318 m_jit.move(TrustedImmPtr(nullptr), storeIndex);
8319
8320 GPRTemporary temp2(this);
8321 GPRReg resultButterfly = temp2.gpr();
8322
8323 m_jit.loadPtr(MacroAssembler::Address(resultGPR, JSObject::butterflyOffset()), resultButterfly);
8324 m_jit.zeroExtend32ToPtr(tempGPR, tempGPR);
8325 m_jit.zeroExtend32ToPtr(loadIndex, loadIndex);
8326 auto done = m_jit.branchPtr(MacroAssembler::AboveOrEqual, loadIndex, tempGPR);
8327
8328 auto loop = m_jit.label();
8329#if USE(JSVALUE64)
8330 m_jit.load64(
8331 MacroAssembler::BaseIndex(storageGPR, loadIndex, MacroAssembler::TimesEight), tempValue);
8332 m_jit.store64(
8333 tempValue, MacroAssembler::BaseIndex(resultButterfly, storeIndex, MacroAssembler::TimesEight));
8334#else
8335 m_jit.load32(
8336 MacroAssembler::BaseIndex(storageGPR, loadIndex, MacroAssembler::TimesEight, PayloadOffset), tempValue);
8337 m_jit.store32(
8338 tempValue, MacroAssembler::BaseIndex(resultButterfly, storeIndex, MacroAssembler::TimesEight, PayloadOffset));
8339 m_jit.load32(
8340 MacroAssembler::BaseIndex(storageGPR, loadIndex, MacroAssembler::TimesEight, TagOffset), tempValue);
8341 m_jit.store32(
8342 tempValue, MacroAssembler::BaseIndex(resultButterfly, storeIndex, MacroAssembler::TimesEight, TagOffset));
8343#endif // USE(JSVALUE64)
8344 m_jit.addPtr(TrustedImm32(1), loadIndex);
8345 m_jit.addPtr(TrustedImm32(1), storeIndex);
8346 m_jit.branchPtr(MacroAssembler::Below, loadIndex, tempGPR).linkTo(loop, &m_jit);
8347
8348 done.link(&m_jit);
8349 cellResult(resultGPR, node);
8350}
8351
8352void SpeculativeJIT::compileArrayIndexOf(Node* node)
8353{
8354 ASSERT(node->op() == ArrayIndexOf);
8355
8356 StorageOperand storage(this, m_jit.graph().varArgChild(node, node->numChildren() == 3 ? 2 : 3));
8357 GPRTemporary index(this);
8358 GPRTemporary tempLength(this);
8359
8360 GPRReg storageGPR = storage.gpr();
8361 GPRReg indexGPR = index.gpr();
8362 GPRReg lengthGPR = tempLength.gpr();
8363
8364 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), lengthGPR);
8365
8366 if (node->numChildren() == 4)
8367 emitPopulateSliceIndex(m_jit.graph().varArgChild(node, 2), WTF::nullopt, lengthGPR, indexGPR);
8368 else
8369 m_jit.move(TrustedImm32(0), indexGPR);
8370
8371 Edge& searchElementEdge = m_jit.graph().varArgChild(node, 1);
8372 switch (searchElementEdge.useKind()) {
8373 case Int32Use:
8374 case ObjectUse:
8375 case SymbolUse:
8376 case OtherUse: {
8377 auto emitLoop = [&] (auto emitCompare) {
8378#if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
8379 m_jit.clearRegisterAllocationOffsets();
8380#endif
8381
8382 m_jit.zeroExtend32ToPtr(lengthGPR, lengthGPR);
8383 m_jit.zeroExtend32ToPtr(indexGPR, indexGPR);
8384
8385 auto loop = m_jit.label();
8386 auto notFound = m_jit.branch32(CCallHelpers::Equal, indexGPR, lengthGPR);
8387
8388 auto found = emitCompare();
8389
8390 m_jit.add32(TrustedImm32(1), indexGPR);
8391 m_jit.jump().linkTo(loop, &m_jit);
8392
8393 notFound.link(&m_jit);
8394 m_jit.move(TrustedImm32(-1), indexGPR);
8395 found.link(&m_jit);
8396 int32Result(indexGPR, node);
8397 };
8398
8399 if (searchElementEdge.useKind() == Int32Use) {
8400 ASSERT(node->arrayMode().type() == Array::Int32);
8401#if USE(JSVALUE64)
8402 JSValueOperand searchElement(this, searchElementEdge, ManualOperandSpeculation);
8403 JSValueRegs searchElementRegs = searchElement.jsValueRegs();
8404 speculateInt32(searchElementEdge, searchElementRegs);
8405 GPRReg searchElementGPR = searchElementRegs.payloadGPR();
8406#else
8407 SpeculateInt32Operand searchElement(this, searchElementEdge);
8408 GPRReg searchElementGPR = searchElement.gpr();
8409
8410 GPRTemporary temp(this);
8411 GPRReg tempGPR = temp.gpr();
8412#endif
8413 emitLoop([&] () {
8414#if USE(JSVALUE64)
8415 auto found = m_jit.branch64(CCallHelpers::Equal, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), searchElementGPR);
8416#else
8417 auto skip = m_jit.branch32(CCallHelpers::NotEqual, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, TagOffset), TrustedImm32(JSValue::Int32Tag));
8418 m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, PayloadOffset), tempGPR);
8419 auto found = m_jit.branch32(CCallHelpers::Equal, tempGPR, searchElementGPR);
8420 skip.link(&m_jit);
8421#endif
8422 return found;
8423 });
8424 return;
8425 }
8426
8427 if (searchElementEdge.useKind() == OtherUse) {
8428 ASSERT(node->arrayMode().type() == Array::Contiguous);
8429 JSValueOperand searchElement(this, searchElementEdge, ManualOperandSpeculation);
8430 GPRTemporary temp(this);
8431
8432 JSValueRegs searchElementRegs = searchElement.jsValueRegs();
8433 GPRReg tempGPR = temp.gpr();
8434 speculateOther(searchElementEdge, searchElementRegs, tempGPR);
8435
8436 emitLoop([&] () {
8437#if USE(JSVALUE64)
8438 auto found = m_jit.branch64(CCallHelpers::Equal, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), searchElementRegs.payloadGPR());
8439#else
8440 m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, TagOffset), tempGPR);
8441 auto found = m_jit.branch32(CCallHelpers::Equal, tempGPR, searchElementRegs.tagGPR());
8442#endif
8443 return found;
8444 });
8445 return;
8446 }
8447
8448 ASSERT(node->arrayMode().type() == Array::Contiguous);
8449 SpeculateCellOperand searchElement(this, searchElementEdge);
8450 GPRReg searchElementGPR = searchElement.gpr();
8451
8452 if (searchElementEdge.useKind() == ObjectUse)
8453 speculateObject(searchElementEdge, searchElementGPR);
8454 else {
8455 ASSERT(searchElementEdge.useKind() == SymbolUse);
8456 speculateSymbol(searchElementEdge, searchElementGPR);
8457 }
8458
8459#if USE(JSVALUE32_64)
8460 GPRTemporary temp(this);
8461 GPRReg tempGPR = temp.gpr();
8462#endif
8463
8464 emitLoop([&] () {
8465#if USE(JSVALUE64)
8466 auto found = m_jit.branch64(CCallHelpers::Equal, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), searchElementGPR);
8467#else
8468 auto skip = m_jit.branch32(CCallHelpers::NotEqual, MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, TagOffset), TrustedImm32(JSValue::CellTag));
8469 m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, PayloadOffset), tempGPR);
8470 auto found = m_jit.branch32(CCallHelpers::Equal, tempGPR, searchElementGPR);
8471 skip.link(&m_jit);
8472#endif
8473 return found;
8474 });
8475 return;
8476 }
8477
8478 case DoubleRepUse: {
8479 ASSERT(node->arrayMode().type() == Array::Double);
8480 SpeculateDoubleOperand searchElement(this, searchElementEdge);
8481 FPRTemporary tempDouble(this);
8482
8483 FPRReg searchElementFPR = searchElement.fpr();
8484 FPRReg tempFPR = tempDouble.fpr();
8485
8486#if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
8487 m_jit.clearRegisterAllocationOffsets();
8488#endif
8489
8490 m_jit.zeroExtend32ToPtr(lengthGPR, lengthGPR);
8491 m_jit.zeroExtend32ToPtr(indexGPR, indexGPR);
8492
8493 auto loop = m_jit.label();
8494 auto notFound = m_jit.branch32(CCallHelpers::Equal, indexGPR, lengthGPR);
8495 m_jit.loadDouble(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), tempFPR);
8496 auto found = m_jit.branchDouble(CCallHelpers::DoubleEqual, tempFPR, searchElementFPR);
8497 m_jit.add32(TrustedImm32(1), indexGPR);
8498 m_jit.jump().linkTo(loop, &m_jit);
8499
8500 notFound.link(&m_jit);
8501 m_jit.move(TrustedImm32(-1), indexGPR);
8502 found.link(&m_jit);
8503 int32Result(indexGPR, node);
8504 return;
8505 }
8506
8507 case StringUse: {
8508 ASSERT(node->arrayMode().type() == Array::Contiguous);
8509 SpeculateCellOperand searchElement(this, searchElementEdge);
8510
8511 GPRReg searchElementGPR = searchElement.gpr();
8512
8513 speculateString(searchElementEdge, searchElementGPR);
8514
8515 flushRegisters();
8516
8517 callOperation(operationArrayIndexOfString, lengthGPR, storageGPR, searchElementGPR, indexGPR);
8518 m_jit.exceptionCheck();
8519
8520 int32Result(lengthGPR, node);
8521 return;
8522 }
8523
8524 case UntypedUse: {
8525 JSValueOperand searchElement(this, searchElementEdge);
8526
8527 JSValueRegs searchElementRegs = searchElement.jsValueRegs();
8528
8529 flushRegisters();
8530 switch (node->arrayMode().type()) {
8531 case Array::Double:
8532 callOperation(operationArrayIndexOfValueDouble, lengthGPR, storageGPR, searchElementRegs, indexGPR);
8533 break;
8534 case Array::Int32:
8535 case Array::Contiguous:
8536 callOperation(operationArrayIndexOfValueInt32OrContiguous, lengthGPR, storageGPR, searchElementRegs, indexGPR);
8537 break;
8538 default:
8539 RELEASE_ASSERT_NOT_REACHED();
8540 break;
8541 }
8542 m_jit.exceptionCheck();
8543
8544 int32Result(lengthGPR, node);
8545 return;
8546 }
8547
8548 default:
8549 RELEASE_ASSERT_NOT_REACHED();
8550 return;
8551 }
8552}
8553
8554void SpeculativeJIT::compileArrayPush(Node* node)
8555{
8556 ASSERT(node->arrayMode().isJSArray());
8557
8558 Edge& storageEdge = m_jit.graph().varArgChild(node, 0);
8559 Edge& arrayEdge = m_jit.graph().varArgChild(node, 1);
8560
8561 SpeculateCellOperand base(this, arrayEdge);
8562 GPRTemporary storageLength(this);
8563
8564 GPRReg baseGPR = base.gpr();
8565 GPRReg storageLengthGPR = storageLength.gpr();
8566
8567 StorageOperand storage(this, storageEdge);
8568 GPRReg storageGPR = storage.gpr();
8569 unsigned elementOffset = 2;
8570 unsigned elementCount = node->numChildren() - elementOffset;
8571
8572#if USE(JSVALUE32_64)
8573 GPRTemporary tag(this);
8574 GPRReg tagGPR = tag.gpr();
8575 JSValueRegs resultRegs { tagGPR, storageLengthGPR };
8576#else
8577 JSValueRegs resultRegs { storageLengthGPR };
8578#endif
8579
8580 auto getStorageBufferAddress = [&] (GPRReg storageGPR, GPRReg indexGPR, int32_t offset, GPRReg bufferGPR) {
8581 static_assert(sizeof(JSValue) == 8 && 1 << 3 == 8, "This is strongly assumed in the code below.");
8582 m_jit.getEffectiveAddress(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, offset), bufferGPR);
8583 };
8584
8585 switch (node->arrayMode().type()) {
8586 case Array::Int32:
8587 case Array::Contiguous: {
8588 if (elementCount == 1) {
8589 Edge& element = m_jit.graph().varArgChild(node, elementOffset);
8590 if (node->arrayMode().type() == Array::Int32) {
8591 ASSERT(element.useKind() == Int32Use);
8592 speculateInt32(element);
8593 }
8594 JSValueOperand value(this, element, ManualOperandSpeculation);
8595 JSValueRegs valueRegs = value.jsValueRegs();
8596
8597 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
8598 MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
8599 m_jit.storeValue(valueRegs, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
8600 m_jit.add32(TrustedImm32(1), storageLengthGPR);
8601 m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
8602 m_jit.boxInt32(storageLengthGPR, resultRegs);
8603
8604 addSlowPathGenerator(
8605 slowPathCall(slowPath, this, operationArrayPush, resultRegs, valueRegs, baseGPR));
8606
8607 jsValueResult(resultRegs, node);
8608 return;
8609 }
8610
8611 if (node->arrayMode().type() == Array::Int32) {
8612 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
8613 Edge element = m_jit.graph().varArgChild(node, elementIndex + elementOffset);
8614 ASSERT(element.useKind() == Int32Use);
8615 speculateInt32(element);
8616 }
8617 }
8618
8619 GPRTemporary buffer(this);
8620 GPRReg bufferGPR = buffer.gpr();
8621
8622 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
8623 m_jit.move(storageLengthGPR, bufferGPR);
8624 m_jit.add32(TrustedImm32(elementCount), bufferGPR);
8625 MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::Above, bufferGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
8626
8627 m_jit.store32(bufferGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
8628 getStorageBufferAddress(storageGPR, storageLengthGPR, 0, bufferGPR);
8629 m_jit.add32(TrustedImm32(elementCount), storageLengthGPR);
8630 m_jit.boxInt32(storageLengthGPR, resultRegs);
8631 auto storageDone = m_jit.jump();
8632
8633 slowPath.link(&m_jit);
8634
8635 size_t scratchSize = sizeof(EncodedJSValue) * elementCount;
8636 ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
8637 m_jit.move(TrustedImmPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())), bufferGPR);
8638 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), storageLengthGPR);
8639 m_jit.storePtr(TrustedImmPtr(scratchSize), MacroAssembler::Address(storageLengthGPR));
8640
8641 storageDone.link(&m_jit);
8642 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
8643 Edge& element = m_jit.graph().varArgChild(node, elementIndex + elementOffset);
8644 JSValueOperand value(this, element, ManualOperandSpeculation); // We did type checks above.
8645 JSValueRegs valueRegs = value.jsValueRegs();
8646
8647 m_jit.storeValue(valueRegs, MacroAssembler::Address(bufferGPR, sizeof(EncodedJSValue) * elementIndex));
8648 value.use();
8649 }
8650
8651 MacroAssembler::Jump fastPath = m_jit.branchPtr(MacroAssembler::NotEqual, bufferGPR, TrustedImmPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())));
8652
8653 addSlowPathGenerator(slowPathCall(m_jit.jump(), this, operationArrayPushMultiple, resultRegs, baseGPR, bufferGPR, TrustedImm32(elementCount)));
8654
8655 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), bufferGPR);
8656 m_jit.storePtr(TrustedImmPtr(nullptr), MacroAssembler::Address(bufferGPR));
8657
8658 base.use();
8659 storage.use();
8660
8661 fastPath.link(&m_jit);
8662 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
8663 return;
8664 }
8665
8666 case Array::Double: {
8667 if (elementCount == 1) {
8668 Edge& element = m_jit.graph().varArgChild(node, elementOffset);
8669 speculate(node, element);
8670 SpeculateDoubleOperand value(this, element);
8671 FPRReg valueFPR = value.fpr();
8672
8673 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
8674 MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
8675 m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
8676 m_jit.add32(TrustedImm32(1), storageLengthGPR);
8677 m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
8678 m_jit.boxInt32(storageLengthGPR, resultRegs);
8679
8680 addSlowPathGenerator(
8681 slowPathCall(slowPath, this, operationArrayPushDouble, resultRegs, valueFPR, baseGPR));
8682
8683 jsValueResult(resultRegs, node);
8684 return;
8685 }
8686
8687 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
8688 Edge element = m_jit.graph().varArgChild(node, elementIndex + elementOffset);
8689 ASSERT(element.useKind() == DoubleRepRealUse);
8690 speculate(node, element);
8691 }
8692
8693 GPRTemporary buffer(this);
8694 GPRReg bufferGPR = buffer.gpr();
8695
8696 m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
8697 m_jit.move(storageLengthGPR, bufferGPR);
8698 m_jit.add32(TrustedImm32(elementCount), bufferGPR);
8699 MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::Above, bufferGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
8700
8701 m_jit.store32(bufferGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
8702 getStorageBufferAddress(storageGPR, storageLengthGPR, 0, bufferGPR);
8703 m_jit.add32(TrustedImm32(elementCount), storageLengthGPR);
8704 m_jit.boxInt32(storageLengthGPR, resultRegs);
8705 auto storageDone = m_jit.jump();
8706
8707 slowPath.link(&m_jit);
8708
8709 size_t scratchSize = sizeof(double) * elementCount;
8710 ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
8711 m_jit.move(TrustedImmPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())), bufferGPR);
8712 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), storageLengthGPR);
8713 m_jit.storePtr(TrustedImmPtr(scratchSize), MacroAssembler::Address(storageLengthGPR));
8714
8715 storageDone.link(&m_jit);
8716 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
8717 Edge& element = m_jit.graph().varArgChild(node, elementIndex + elementOffset);
8718 SpeculateDoubleOperand value(this, element);
8719 FPRReg valueFPR = value.fpr();
8720
8721 m_jit.storeDouble(valueFPR, MacroAssembler::Address(bufferGPR, sizeof(double) * elementIndex));
8722 value.use();
8723 }
8724
8725 MacroAssembler::Jump fastPath = m_jit.branchPtr(MacroAssembler::NotEqual, bufferGPR, TrustedImmPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())));
8726
8727 addSlowPathGenerator(slowPathCall(m_jit.jump(), this, operationArrayPushDoubleMultiple, resultRegs, baseGPR, bufferGPR, TrustedImm32(elementCount)));
8728
8729 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), bufferGPR);
8730 m_jit.storePtr(TrustedImmPtr(nullptr), MacroAssembler::Address(bufferGPR));
8731
8732 base.use();
8733 storage.use();
8734
8735 fastPath.link(&m_jit);
8736 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
8737 return;
8738 }
8739
8740 case Array::ArrayStorage: {
8741 // This ensures that the result of ArrayPush is Int32 in AI.
8742 int32_t largestPositiveInt32Length = 0x7fffffff - elementCount;
8743 if (elementCount == 1) {
8744 Edge& element = m_jit.graph().varArgChild(node, elementOffset);
8745 JSValueOperand value(this, element);
8746 JSValueRegs valueRegs = value.jsValueRegs();
8747
8748 m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
8749
8750 // Refuse to handle bizarre lengths.
8751 speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Above, storageLengthGPR, TrustedImm32(largestPositiveInt32Length)));
8752
8753 MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
8754
8755 m_jit.storeValue(valueRegs, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, ArrayStorage::vectorOffset()));
8756
8757 m_jit.add32(TrustedImm32(1), storageLengthGPR);
8758 m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
8759 m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
8760 m_jit.boxInt32(storageLengthGPR, resultRegs);
8761
8762 addSlowPathGenerator(
8763 slowPathCall(slowPath, this, operationArrayPush, resultRegs, valueRegs, baseGPR));
8764
8765 jsValueResult(resultRegs, node);
8766 return;
8767 }
8768
8769 GPRTemporary buffer(this);
8770 GPRReg bufferGPR = buffer.gpr();
8771
8772 m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
8773
8774 // Refuse to handle bizarre lengths.
8775 speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Above, storageLengthGPR, TrustedImm32(largestPositiveInt32Length)));
8776
8777 m_jit.move(storageLengthGPR, bufferGPR);
8778 m_jit.add32(TrustedImm32(elementCount), bufferGPR);
8779 MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::Above, bufferGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
8780
8781 m_jit.store32(bufferGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
8782 getStorageBufferAddress(storageGPR, storageLengthGPR, ArrayStorage::vectorOffset(), bufferGPR);
8783 m_jit.add32(TrustedImm32(elementCount), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
8784 m_jit.add32(TrustedImm32(elementCount), storageLengthGPR);
8785 m_jit.boxInt32(storageLengthGPR, resultRegs);
8786 auto storageDone = m_jit.jump();
8787
8788 slowPath.link(&m_jit);
8789
8790 size_t scratchSize = sizeof(EncodedJSValue) * elementCount;
8791 ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
8792 m_jit.move(TrustedImmPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())), bufferGPR);
8793 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), storageLengthGPR);
8794 m_jit.storePtr(TrustedImmPtr(scratchSize), MacroAssembler::Address(storageLengthGPR));
8795
8796 storageDone.link(&m_jit);
8797 for (unsigned elementIndex = 0; elementIndex < elementCount; ++elementIndex) {
8798 Edge& element = m_jit.graph().varArgChild(node, elementIndex + elementOffset);
8799 JSValueOperand value(this, element);
8800 JSValueRegs valueRegs = value.jsValueRegs();
8801
8802 m_jit.storeValue(valueRegs, MacroAssembler::Address(bufferGPR, sizeof(EncodedJSValue) * elementIndex));
8803 value.use();
8804 }
8805
8806 MacroAssembler::Jump fastPath = m_jit.branchPtr(MacroAssembler::NotEqual, bufferGPR, TrustedImmPtr(static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer())));
8807
8808 addSlowPathGenerator(
8809 slowPathCall(m_jit.jump(), this, operationArrayPushMultiple, resultRegs, baseGPR, bufferGPR, TrustedImm32(elementCount)));
8810
8811 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), bufferGPR);
8812 m_jit.storePtr(TrustedImmPtr(nullptr), MacroAssembler::Address(bufferGPR));
8813
8814 base.use();
8815 storage.use();
8816
8817 fastPath.link(&m_jit);
8818 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
8819 return;
8820 }
8821
8822 default:
8823 RELEASE_ASSERT_NOT_REACHED();
8824 }
8825}
8826
8827void SpeculativeJIT::compileNotifyWrite(Node* node)
8828{
8829 WatchpointSet* set = node->watchpointSet();
8830
8831 JITCompiler::Jump slowCase = m_jit.branch8(
8832 JITCompiler::NotEqual,
8833 JITCompiler::AbsoluteAddress(set->addressOfState()),
8834 TrustedImm32(IsInvalidated));
8835
8836 addSlowPathGenerator(
8837 slowPathCall(slowCase, this, operationNotifyWrite, NeedToSpill, ExceptionCheckRequirement::CheckNotNeeded, NoResult, set));
8838
8839 noResult(node);
8840}
8841
8842void SpeculativeJIT::compileIsObject(Node* node)
8843{
8844 JSValueOperand value(this, node->child1());
8845 GPRTemporary result(this, Reuse, value, TagWord);
8846
8847 JSValueRegs valueRegs = value.jsValueRegs();
8848 GPRReg resultGPR = result.gpr();
8849
8850 JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(valueRegs);
8851
8852 m_jit.compare8(JITCompiler::AboveOrEqual,
8853 JITCompiler::Address(valueRegs.payloadGPR(), JSCell::typeInfoTypeOffset()),
8854 TrustedImm32(ObjectType),
8855 resultGPR);
8856 JITCompiler::Jump done = m_jit.jump();
8857
8858 isNotCell.link(&m_jit);
8859 m_jit.move(TrustedImm32(0), resultGPR);
8860
8861 done.link(&m_jit);
8862 unblessedBooleanResult(resultGPR, node);
8863}
8864
8865void SpeculativeJIT::compileIsObjectOrNull(Node* node)
8866{
8867 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
8868
8869 JSValueOperand value(this, node->child1());
8870 JSValueRegs valueRegs = value.jsValueRegs();
8871
8872 GPRTemporary result(this);
8873 GPRReg resultGPR = result.gpr();
8874
8875 JITCompiler::Jump isCell = m_jit.branchIfCell(valueRegs);
8876
8877 JITCompiler::Jump isNull = m_jit.branchIfEqual(valueRegs, jsNull());
8878 JITCompiler::Jump isNonNullNonCell = m_jit.jump();
8879
8880 isCell.link(&m_jit);
8881 JITCompiler::Jump isFunction = m_jit.branchIfFunction(valueRegs.payloadGPR());
8882 JITCompiler::Jump notObject = m_jit.branchIfNotObject(valueRegs.payloadGPR());
8883
8884 JITCompiler::Jump slowPath = m_jit.branchTest8(
8885 JITCompiler::NonZero,
8886 JITCompiler::Address(valueRegs.payloadGPR(), JSCell::typeInfoFlagsOffset()),
8887 TrustedImm32(MasqueradesAsUndefined | OverridesGetCallData));
8888
8889 isNull.link(&m_jit);
8890 m_jit.move(TrustedImm32(1), resultGPR);
8891 JITCompiler::Jump done = m_jit.jump();
8892
8893 isNonNullNonCell.link(&m_jit);
8894 isFunction.link(&m_jit);
8895 notObject.link(&m_jit);
8896 m_jit.move(TrustedImm32(0), resultGPR);
8897
8898 addSlowPathGenerator(
8899 slowPathCall(
8900 slowPath, this, operationObjectIsObject, resultGPR, globalObject,
8901 valueRegs.payloadGPR()));
8902
8903 done.link(&m_jit);
8904
8905 unblessedBooleanResult(resultGPR, node);
8906}
8907
8908void SpeculativeJIT::compileIsFunction(Node* node)
8909{
8910 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
8911
8912 JSValueOperand value(this, node->child1());
8913 JSValueRegs valueRegs = value.jsValueRegs();
8914
8915 GPRTemporary result(this);
8916 GPRReg resultGPR = result.gpr();
8917
8918 JITCompiler::Jump notCell = m_jit.branchIfNotCell(valueRegs);
8919 JITCompiler::Jump isFunction = m_jit.branchIfFunction(valueRegs.payloadGPR());
8920 JITCompiler::Jump notObject = m_jit.branchIfNotObject(valueRegs.payloadGPR());
8921
8922 JITCompiler::Jump slowPath = m_jit.branchTest8(
8923 JITCompiler::NonZero,
8924 JITCompiler::Address(valueRegs.payloadGPR(), JSCell::typeInfoFlagsOffset()),
8925 TrustedImm32(MasqueradesAsUndefined | OverridesGetCallData));
8926
8927 notCell.link(&m_jit);
8928 notObject.link(&m_jit);
8929 m_jit.move(TrustedImm32(0), resultGPR);
8930 JITCompiler::Jump done = m_jit.jump();
8931
8932 isFunction.link(&m_jit);
8933 m_jit.move(TrustedImm32(1), resultGPR);
8934
8935 addSlowPathGenerator(
8936 slowPathCall(
8937 slowPath, this, operationObjectIsFunction, resultGPR, globalObject,
8938 valueRegs.payloadGPR()));
8939
8940 done.link(&m_jit);
8941
8942 unblessedBooleanResult(resultGPR, node);
8943}
8944
8945void SpeculativeJIT::compileTypeOf(Node* node)
8946{
8947 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
8948
8949 JSValueOperand value(this, node->child1());
8950 JSValueRegs valueRegs = value.jsValueRegs();
8951
8952 GPRTemporary result(this);
8953 GPRReg resultGPR = result.gpr();
8954
8955 JITCompiler::JumpList done;
8956 JITCompiler::Jump slowPath;
8957 m_jit.emitTypeOf(
8958 valueRegs, resultGPR,
8959 [&] (TypeofType type, bool fallsThrough) {
8960 m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.vm()->smallStrings.typeString(type)), resultGPR);
8961 if (!fallsThrough)
8962 done.append(m_jit.jump());
8963 },
8964 [&] (JITCompiler::Jump theSlowPath) {
8965 slowPath = theSlowPath;
8966 });
8967 done.link(&m_jit);
8968
8969 addSlowPathGenerator(
8970 slowPathCall(
8971 slowPath, this, operationTypeOfObject, resultGPR, globalObject,
8972 valueRegs.payloadGPR()));
8973
8974 cellResult(resultGPR, node);
8975}
8976
8977void SpeculativeJIT::emitStructureCheck(Node* node, GPRReg cellGPR, GPRReg tempGPR)
8978{
8979 ASSERT(node->structureSet().size());
8980
8981 if (node->structureSet().size() == 1) {
8982 speculationCheck(
8983 BadCache, JSValueSource::unboxedCell(cellGPR), 0,
8984 m_jit.branchWeakStructure(
8985 JITCompiler::NotEqual,
8986 JITCompiler::Address(cellGPR, JSCell::structureIDOffset()),
8987 node->structureSet()[0]));
8988 } else {
8989 std::unique_ptr<GPRTemporary> structure;
8990 GPRReg structureGPR;
8991
8992 if (tempGPR == InvalidGPRReg) {
8993 structure = std::make_unique<GPRTemporary>(this);
8994 structureGPR = structure->gpr();
8995 } else
8996 structureGPR = tempGPR;
8997
8998 m_jit.load32(JITCompiler::Address(cellGPR, JSCell::structureIDOffset()), structureGPR);
8999
9000 JITCompiler::JumpList done;
9001
9002 for (size_t i = 0; i < node->structureSet().size() - 1; ++i) {
9003 done.append(
9004 m_jit.branchWeakStructure(JITCompiler::Equal, structureGPR, node->structureSet()[i]));
9005 }
9006
9007 speculationCheck(
9008 BadCache, JSValueSource::unboxedCell(cellGPR), 0,
9009 m_jit.branchWeakStructure(
9010 JITCompiler::NotEqual, structureGPR, node->structureSet().last()));
9011
9012 done.link(&m_jit);
9013 }
9014}
9015
9016void SpeculativeJIT::compileCheckCell(Node* node)
9017{
9018 SpeculateCellOperand cell(this, node->child1());
9019 speculationCheck(BadCell, JSValueSource::unboxedCell(cell.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, cell.gpr(), node->cellOperand()->cell()));
9020 noResult(node);
9021}
9022
9023void SpeculativeJIT::compileCheckNotEmpty(Node* node)
9024{
9025 JSValueOperand operand(this, node->child1());
9026 JSValueRegs regs = operand.jsValueRegs();
9027 speculationCheck(TDZFailure, JSValueSource(), nullptr, m_jit.branchIfEmpty(regs));
9028 noResult(node);
9029}
9030
9031void SpeculativeJIT::compileCheckStructure(Node* node)
9032{
9033 switch (node->child1().useKind()) {
9034 case CellUse:
9035 case KnownCellUse: {
9036 SpeculateCellOperand cell(this, node->child1());
9037 emitStructureCheck(node, cell.gpr(), InvalidGPRReg);
9038 noResult(node);
9039 return;
9040 }
9041
9042 case CellOrOtherUse: {
9043 JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
9044 GPRTemporary temp(this);
9045
9046 JSValueRegs valueRegs = value.jsValueRegs();
9047 GPRReg tempGPR = temp.gpr();
9048
9049 JITCompiler::Jump cell = m_jit.branchIfCell(valueRegs);
9050 DFG_TYPE_CHECK(
9051 valueRegs, node->child1(), SpecCell | SpecOther,
9052 m_jit.branchIfNotOther(valueRegs, tempGPR));
9053 JITCompiler::Jump done = m_jit.jump();
9054 cell.link(&m_jit);
9055 emitStructureCheck(node, valueRegs.payloadGPR(), tempGPR);
9056 done.link(&m_jit);
9057 noResult(node);
9058 return;
9059 }
9060
9061 default:
9062 DFG_CRASH(m_jit.graph(), node, "Bad use kind");
9063 return;
9064 }
9065}
9066
9067void SpeculativeJIT::compileAllocatePropertyStorage(Node* node)
9068{
9069 ASSERT(!node->transition()->previous->outOfLineCapacity());
9070 ASSERT(initialOutOfLineCapacity == node->transition()->next->outOfLineCapacity());
9071
9072 size_t size = initialOutOfLineCapacity * sizeof(JSValue);
9073
9074 Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(size, AllocatorForMode::AllocatorIfExists);
9075
9076 if (!allocator || node->transition()->previous->couldHaveIndexingHeader()) {
9077 SpeculateCellOperand base(this, node->child1());
9078
9079 GPRReg baseGPR = base.gpr();
9080
9081 flushRegisters();
9082
9083 GPRFlushedCallResult result(this);
9084 callOperation(operationAllocateComplexPropertyStorageWithInitialCapacity, result.gpr(), baseGPR);
9085 m_jit.exceptionCheck();
9086
9087 storageResult(result.gpr(), node);
9088 return;
9089 }
9090
9091 GPRTemporary scratch1(this);
9092 GPRTemporary scratch2(this);
9093 GPRTemporary scratch3(this);
9094
9095 GPRReg scratchGPR1 = scratch1.gpr();
9096 GPRReg scratchGPR2 = scratch2.gpr();
9097 GPRReg scratchGPR3 = scratch3.gpr();
9098
9099 JITCompiler::JumpList slowPath;
9100 m_jit.emitAllocate(scratchGPR1, JITAllocator::constant(allocator), scratchGPR2, scratchGPR3, slowPath);
9101 m_jit.addPtr(JITCompiler::TrustedImm32(size + sizeof(IndexingHeader)), scratchGPR1);
9102
9103 addSlowPathGenerator(
9104 slowPathCall(slowPath, this, operationAllocateSimplePropertyStorageWithInitialCapacity, scratchGPR1));
9105
9106 for (ptrdiff_t offset = 0; offset < static_cast<ptrdiff_t>(size); offset += sizeof(void*))
9107 m_jit.storePtr(TrustedImmPtr(nullptr), JITCompiler::Address(scratchGPR1, -(offset + sizeof(JSValue) + sizeof(void*))));
9108
9109 storageResult(scratchGPR1, node);
9110}
9111
9112void SpeculativeJIT::compileReallocatePropertyStorage(Node* node)
9113{
9114 size_t oldSize = node->transition()->previous->outOfLineCapacity() * sizeof(JSValue);
9115 size_t newSize = oldSize * outOfLineGrowthFactor;
9116 ASSERT(newSize == node->transition()->next->outOfLineCapacity() * sizeof(JSValue));
9117
9118 Allocator allocator = m_jit.vm()->jsValueGigacageAuxiliarySpace.allocatorForNonVirtual(newSize, AllocatorForMode::AllocatorIfExists);
9119
9120 if (!allocator || node->transition()->previous->couldHaveIndexingHeader()) {
9121 SpeculateCellOperand base(this, node->child1());
9122
9123 GPRReg baseGPR = base.gpr();
9124
9125 flushRegisters();
9126
9127 GPRFlushedCallResult result(this);
9128 callOperation(operationAllocateComplexPropertyStorage, result.gpr(), baseGPR, newSize / sizeof(JSValue));
9129 m_jit.exceptionCheck();
9130
9131 storageResult(result.gpr(), node);
9132 return;
9133 }
9134
9135 StorageOperand oldStorage(this, node->child2());
9136 GPRTemporary scratch1(this);
9137 GPRTemporary scratch2(this);
9138 GPRTemporary scratch3(this);
9139
9140 GPRReg oldStorageGPR = oldStorage.gpr();
9141 GPRReg scratchGPR1 = scratch1.gpr();
9142 GPRReg scratchGPR2 = scratch2.gpr();
9143 GPRReg scratchGPR3 = scratch3.gpr();
9144
9145 JITCompiler::JumpList slowPath;
9146 m_jit.emitAllocate(scratchGPR1, JITAllocator::constant(allocator), scratchGPR2, scratchGPR3, slowPath);
9147
9148 m_jit.addPtr(JITCompiler::TrustedImm32(newSize + sizeof(IndexingHeader)), scratchGPR1);
9149
9150 addSlowPathGenerator(
9151 slowPathCall(slowPath, this, operationAllocateSimplePropertyStorage, scratchGPR1, newSize / sizeof(JSValue)));
9152
9153 for (ptrdiff_t offset = oldSize; offset < static_cast<ptrdiff_t>(newSize); offset += sizeof(void*))
9154 m_jit.storePtr(TrustedImmPtr(nullptr), JITCompiler::Address(scratchGPR1, -(offset + sizeof(JSValue) + sizeof(void*))));
9155
9156 // We have scratchGPR1 = new storage, scratchGPR2 = scratch
9157 for (ptrdiff_t offset = 0; offset < static_cast<ptrdiff_t>(oldSize); offset += sizeof(void*)) {
9158 m_jit.loadPtr(JITCompiler::Address(oldStorageGPR, -(offset + sizeof(JSValue) + sizeof(void*))), scratchGPR2);
9159 m_jit.storePtr(scratchGPR2, JITCompiler::Address(scratchGPR1, -(offset + sizeof(JSValue) + sizeof(void*))));
9160 }
9161
9162 storageResult(scratchGPR1, node);
9163}
9164
9165void SpeculativeJIT::compileNukeStructureAndSetButterfly(Node* node)
9166{
9167 SpeculateCellOperand base(this, node->child1());
9168 StorageOperand storage(this, node->child2());
9169
9170 GPRReg baseGPR = base.gpr();
9171 GPRReg storageGPR = storage.gpr();
9172
9173 m_jit.nukeStructureAndStoreButterfly(*m_jit.vm(), storageGPR, baseGPR);
9174
9175 noResult(node);
9176}
9177
9178void SpeculativeJIT::compileGetButterfly(Node* node)
9179{
9180 SpeculateCellOperand base(this, node->child1());
9181 GPRTemporary result(this, Reuse, base);
9182
9183 GPRReg baseGPR = base.gpr();
9184 GPRReg resultGPR = result.gpr();
9185
9186 m_jit.loadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR);
9187
9188 storageResult(resultGPR, node);
9189}
9190
9191static void allocateTemporaryRegistersForSnippet(SpeculativeJIT* jit, Vector<GPRTemporary>& gpHolders, Vector<FPRTemporary>& fpHolders, Vector<GPRReg>& gpScratch, Vector<FPRReg>& fpScratch, Snippet& snippet)
9192{
9193 for (unsigned i = 0; i < snippet.numGPScratchRegisters; ++i) {
9194 GPRTemporary temporary(jit);
9195 gpScratch.append(temporary.gpr());
9196 gpHolders.append(WTFMove(temporary));
9197 }
9198
9199 for (unsigned i = 0; i < snippet.numFPScratchRegisters; ++i) {
9200 FPRTemporary temporary(jit);
9201 fpScratch.append(temporary.fpr());
9202 fpHolders.append(WTFMove(temporary));
9203 }
9204}
9205
9206void SpeculativeJIT::compileCallDOM(Node* node)
9207{
9208 const DOMJIT::Signature* signature = node->signature();
9209
9210 // FIXME: We should have a way to call functions with the vector of registers.
9211 // https://bugs.webkit.org/show_bug.cgi?id=163099
9212 Vector<Variant<SpeculateCellOperand, SpeculateInt32Operand, SpeculateBooleanOperand>, JSC_DOMJIT_SIGNATURE_MAX_ARGUMENTS_INCLUDING_THIS> operands;
9213 Vector<GPRReg, JSC_DOMJIT_SIGNATURE_MAX_ARGUMENTS_INCLUDING_THIS> regs;
9214
9215 auto appendCell = [&](Edge& edge) {
9216 SpeculateCellOperand operand(this, edge);
9217 regs.append(operand.gpr());
9218 operands.append(WTFMove(operand));
9219 };
9220
9221 auto appendString = [&](Edge& edge) {
9222 SpeculateCellOperand operand(this, edge);
9223 GPRReg gpr = operand.gpr();
9224 regs.append(gpr);
9225 speculateString(edge, gpr);
9226 operands.append(WTFMove(operand));
9227 };
9228
9229 auto appendInt32 = [&](Edge& edge) {
9230 SpeculateInt32Operand operand(this, edge);
9231 regs.append(operand.gpr());
9232 operands.append(WTFMove(operand));
9233 };
9234
9235 auto appendBoolean = [&](Edge& edge) {
9236 SpeculateBooleanOperand operand(this, edge);
9237 regs.append(operand.gpr());
9238 operands.append(WTFMove(operand));
9239 };
9240
9241 unsigned index = 0;
9242 m_jit.graph().doToChildren(node, [&](Edge edge) {
9243 if (!index)
9244 appendCell(edge);
9245 else {
9246 switch (signature->arguments[index - 1]) {
9247 case SpecString:
9248 appendString(edge);
9249 break;
9250 case SpecInt32Only:
9251 appendInt32(edge);
9252 break;
9253 case SpecBoolean:
9254 appendBoolean(edge);
9255 break;
9256 default:
9257 RELEASE_ASSERT_NOT_REACHED();
9258 break;
9259 }
9260 }
9261 ++index;
9262 });
9263
9264 JSValueRegsTemporary result(this);
9265 JSValueRegs resultRegs = result.regs();
9266
9267 flushRegisters();
9268 assertIsTaggedWith(reinterpret_cast<void*>(signature->unsafeFunction), CFunctionPtrTag);
9269 unsigned argumentCountIncludingThis = signature->argumentCount + 1;
9270 switch (argumentCountIncludingThis) {
9271 case 1:
9272 callOperation(reinterpret_cast<J_JITOperation_EP>(signature->unsafeFunction), extractResult(resultRegs), regs[0]);
9273 break;
9274 case 2:
9275 callOperation(reinterpret_cast<J_JITOperation_EPP>(signature->unsafeFunction), extractResult(resultRegs), regs[0], regs[1]);
9276 break;
9277 case 3:
9278 callOperation(reinterpret_cast<J_JITOperation_EPPP>(signature->unsafeFunction), extractResult(resultRegs), regs[0], regs[1], regs[2]);
9279 break;
9280 default:
9281 RELEASE_ASSERT_NOT_REACHED();
9282 break;
9283 }
9284
9285 m_jit.exceptionCheck();
9286 jsValueResult(resultRegs, node);
9287}
9288
9289void SpeculativeJIT::compileCallDOMGetter(Node* node)
9290{
9291 DOMJIT::CallDOMGetterSnippet* snippet = node->callDOMGetterData()->snippet;
9292 if (!snippet) {
9293 FunctionPtr<OperationPtrTag> getter = node->callDOMGetterData()->customAccessorGetter;
9294 SpeculateCellOperand base(this, node->child1());
9295 JSValueRegsTemporary result(this);
9296
9297 JSValueRegs resultRegs = result.regs();
9298 GPRReg baseGPR = base.gpr();
9299
9300 flushRegisters();
9301 m_jit.setupArguments<J_JITOperation_EJI>(CCallHelpers::CellValue(baseGPR), identifierUID(node->callDOMGetterData()->identifierNumber));
9302 m_jit.storePtr(GPRInfo::callFrameRegister, &m_jit.vm()->topCallFrame);
9303 m_jit.emitStoreCodeOrigin(m_currentNode->origin.semantic);
9304 m_jit.appendCall(getter.retagged<CFunctionPtrTag>());
9305 m_jit.setupResults(resultRegs);
9306
9307 m_jit.exceptionCheck();
9308 jsValueResult(resultRegs, node);
9309 return;
9310 }
9311
9312 Vector<GPRReg> gpScratch;
9313 Vector<FPRReg> fpScratch;
9314 Vector<SnippetParams::Value> regs;
9315
9316 JSValueRegsTemporary result(this);
9317 regs.append(result.regs());
9318
9319 Edge& baseEdge = node->child1();
9320 SpeculateCellOperand base(this, baseEdge);
9321 regs.append(SnippetParams::Value(base.gpr(), m_state.forNode(baseEdge).value()));
9322
9323 Optional<SpeculateCellOperand> globalObject;
9324 if (snippet->requireGlobalObject) {
9325 Edge& globalObjectEdge = node->child2();
9326 globalObject.emplace(this, globalObjectEdge);
9327 regs.append(SnippetParams::Value(globalObject->gpr(), m_state.forNode(globalObjectEdge).value()));
9328 }
9329
9330 Vector<GPRTemporary> gpTempraries;
9331 Vector<FPRTemporary> fpTempraries;
9332 allocateTemporaryRegistersForSnippet(this, gpTempraries, fpTempraries, gpScratch, fpScratch, *snippet);
9333 SnippetParams params(this, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch));
9334 snippet->generator()->run(m_jit, params);
9335 jsValueResult(result.regs(), node);
9336}
9337
9338void SpeculativeJIT::compileCheckSubClass(Node* node)
9339{
9340 const ClassInfo* classInfo = node->classInfo();
9341 if (!classInfo->checkSubClassSnippet) {
9342 SpeculateCellOperand base(this, node->child1());
9343 GPRTemporary other(this);
9344 GPRTemporary specified(this);
9345
9346 GPRReg baseGPR = base.gpr();
9347 GPRReg otherGPR = other.gpr();
9348 GPRReg specifiedGPR = specified.gpr();
9349
9350 m_jit.emitLoadStructure(*m_jit.vm(), baseGPR, otherGPR, specifiedGPR);
9351 m_jit.loadPtr(CCallHelpers::Address(otherGPR, Structure::classInfoOffset()), otherGPR);
9352 m_jit.move(CCallHelpers::TrustedImmPtr(node->classInfo()), specifiedGPR);
9353
9354 CCallHelpers::Label loop = m_jit.label();
9355 auto done = m_jit.branchPtr(CCallHelpers::Equal, otherGPR, specifiedGPR);
9356 m_jit.loadPtr(CCallHelpers::Address(otherGPR, ClassInfo::offsetOfParentClass()), otherGPR);
9357 m_jit.branchTestPtr(CCallHelpers::NonZero, otherGPR).linkTo(loop, &m_jit);
9358 speculationCheck(BadType, JSValueSource::unboxedCell(baseGPR), node->child1(), m_jit.jump());
9359 done.link(&m_jit);
9360 noResult(node);
9361 return;
9362 }
9363
9364 Ref<Snippet> snippet = classInfo->checkSubClassSnippet();
9365
9366 Vector<GPRReg> gpScratch;
9367 Vector<FPRReg> fpScratch;
9368 Vector<SnippetParams::Value> regs;
9369
9370 SpeculateCellOperand base(this, node->child1());
9371 GPRReg baseGPR = base.gpr();
9372 regs.append(SnippetParams::Value(baseGPR, m_state.forNode(node->child1()).value()));
9373
9374 Vector<GPRTemporary> gpTempraries;
9375 Vector<FPRTemporary> fpTempraries;
9376 allocateTemporaryRegistersForSnippet(this, gpTempraries, fpTempraries, gpScratch, fpScratch, snippet.get());
9377
9378 SnippetParams params(this, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch));
9379 CCallHelpers::JumpList failureCases = snippet->generator()->run(m_jit, params);
9380 speculationCheck(BadType, JSValueSource::unboxedCell(baseGPR), node->child1(), failureCases);
9381 noResult(node);
9382}
9383
9384GPRReg SpeculativeJIT::temporaryRegisterForPutByVal(GPRTemporary& temporary, ArrayMode arrayMode)
9385{
9386 if (!putByValWillNeedExtraRegister(arrayMode))
9387 return InvalidGPRReg;
9388
9389 GPRTemporary realTemporary(this);
9390 temporary.adopt(realTemporary);
9391 return temporary.gpr();
9392}
9393
9394void SpeculativeJIT::compileToStringOrCallStringConstructorOrStringValueOf(Node* node)
9395{
9396 ASSERT(node->op() != StringValueOf || node->child1().useKind() == UntypedUse);
9397 switch (node->child1().useKind()) {
9398 case NotCellUse: {
9399 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
9400 JSValueRegs op1Regs = op1.jsValueRegs();
9401
9402 GPRFlushedCallResult result(this);
9403 GPRReg resultGPR = result.gpr();
9404
9405 speculateNotCell(node->child1(), op1Regs);
9406
9407 flushRegisters();
9408
9409 if (node->op() == ToString)
9410 callOperation(operationToString, resultGPR, op1Regs);
9411 else {
9412 ASSERT(node->op() == CallStringConstructor);
9413 callOperation(operationCallStringConstructor, resultGPR, op1Regs);
9414 }
9415 m_jit.exceptionCheck();
9416 cellResult(resultGPR, node);
9417 return;
9418 }
9419
9420 case UntypedUse: {
9421 JSValueOperand op1(this, node->child1());
9422 JSValueRegs op1Regs = op1.jsValueRegs();
9423 GPRReg op1PayloadGPR = op1Regs.payloadGPR();
9424
9425 GPRFlushedCallResult result(this);
9426 GPRReg resultGPR = result.gpr();
9427
9428 flushRegisters();
9429
9430 JITCompiler::Jump done;
9431 if (node->child1()->prediction() & SpecString) {
9432 JITCompiler::Jump slowPath1 = m_jit.branchIfNotCell(op1.jsValueRegs());
9433 JITCompiler::Jump slowPath2 = m_jit.branchIfNotString(op1PayloadGPR);
9434 m_jit.move(op1PayloadGPR, resultGPR);
9435 done = m_jit.jump();
9436 slowPath1.link(&m_jit);
9437 slowPath2.link(&m_jit);
9438 }
9439 if (node->op() == ToString)
9440 callOperation(operationToString, resultGPR, op1Regs);
9441 else if (node->op() == StringValueOf)
9442 callOperation(operationStringValueOf, resultGPR, op1Regs);
9443 else {
9444 ASSERT(node->op() == CallStringConstructor);
9445 callOperation(operationCallStringConstructor, resultGPR, op1Regs);
9446 }
9447 m_jit.exceptionCheck();
9448 if (done.isSet())
9449 done.link(&m_jit);
9450 cellResult(resultGPR, node);
9451 return;
9452 }
9453
9454 case Int32Use:
9455 case Int52RepUse:
9456 case DoubleRepUse:
9457 compileNumberToStringWithValidRadixConstant(node, 10);
9458 return;
9459
9460 default:
9461 break;
9462 }
9463
9464 SpeculateCellOperand op1(this, node->child1());
9465 GPRReg op1GPR = op1.gpr();
9466
9467 switch (node->child1().useKind()) {
9468 case StringObjectUse: {
9469 GPRTemporary result(this);
9470 GPRReg resultGPR = result.gpr();
9471
9472 speculateStringObject(node->child1(), op1GPR);
9473
9474 m_jit.loadPtr(JITCompiler::Address(op1GPR, JSWrapperObject::internalValueCellOffset()), resultGPR);
9475 cellResult(resultGPR, node);
9476 break;
9477 }
9478
9479 case StringOrStringObjectUse: {
9480 GPRTemporary result(this);
9481 GPRReg resultGPR = result.gpr();
9482
9483 m_jit.load8(JITCompiler::Address(op1GPR, JSCell::typeInfoTypeOffset()), resultGPR);
9484 JITCompiler::Jump isString = m_jit.branch32(JITCompiler::Equal, resultGPR, TrustedImm32(StringType));
9485
9486 speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1().node(), m_jit.branch32(JITCompiler::NotEqual, resultGPR, TrustedImm32(StringObjectType)));
9487 m_jit.loadPtr(JITCompiler::Address(op1GPR, JSWrapperObject::internalValueCellOffset()), resultGPR);
9488 JITCompiler::Jump done = m_jit.jump();
9489
9490 isString.link(&m_jit);
9491 m_jit.move(op1GPR, resultGPR);
9492 done.link(&m_jit);
9493
9494 m_interpreter.filter(node->child1(), SpecString | SpecStringObject);
9495
9496 cellResult(resultGPR, node);
9497 break;
9498 }
9499
9500 case CellUse: {
9501 GPRFlushedCallResult result(this);
9502 GPRReg resultGPR = result.gpr();
9503
9504 // We flush registers instead of silent spill/fill because in this mode we
9505 // believe that most likely the input is not a string, and we need to take
9506 // slow path.
9507 flushRegisters();
9508 JITCompiler::Jump done;
9509 if (node->child1()->prediction() & SpecString) {
9510 JITCompiler::Jump needCall = m_jit.branchIfNotString(op1GPR);
9511 m_jit.move(op1GPR, resultGPR);
9512 done = m_jit.jump();
9513 needCall.link(&m_jit);
9514 }
9515 if (node->op() == ToString)
9516 callOperation(operationToStringOnCell, resultGPR, op1GPR);
9517 else {
9518 ASSERT(node->op() == CallStringConstructor);
9519 callOperation(operationCallStringConstructorOnCell, resultGPR, op1GPR);
9520 }
9521 m_jit.exceptionCheck();
9522 if (done.isSet())
9523 done.link(&m_jit);
9524 cellResult(resultGPR, node);
9525 break;
9526 }
9527
9528 default:
9529 RELEASE_ASSERT_NOT_REACHED();
9530 }
9531}
9532
9533void SpeculativeJIT::compileNumberToStringWithValidRadixConstant(Node* node)
9534{
9535 compileNumberToStringWithValidRadixConstant(node, node->validRadixConstant());
9536}
9537
9538void SpeculativeJIT::compileNumberToStringWithValidRadixConstant(Node* node, int32_t radix)
9539{
9540 auto callToString = [&] (auto operation, GPRReg resultGPR, auto valueReg) {
9541 flushRegisters();
9542 callOperation(operation, resultGPR, valueReg, TrustedImm32(radix));
9543 m_jit.exceptionCheck();
9544 cellResult(resultGPR, node);
9545 };
9546
9547 switch (node->child1().useKind()) {
9548 case Int32Use: {
9549 SpeculateStrictInt32Operand value(this, node->child1());
9550 GPRFlushedCallResult result(this);
9551 callToString(operationInt32ToStringWithValidRadix, result.gpr(), value.gpr());
9552 break;
9553 }
9554
9555#if USE(JSVALUE64)
9556 case Int52RepUse: {
9557 SpeculateStrictInt52Operand value(this, node->child1());
9558 GPRFlushedCallResult result(this);
9559 callToString(operationInt52ToStringWithValidRadix, result.gpr(), value.gpr());
9560 break;
9561 }
9562#endif
9563
9564 case DoubleRepUse: {
9565 SpeculateDoubleOperand value(this, node->child1());
9566 GPRFlushedCallResult result(this);
9567 callToString(operationDoubleToStringWithValidRadix, result.gpr(), value.fpr());
9568 break;
9569 }
9570
9571 default:
9572 RELEASE_ASSERT_NOT_REACHED();
9573 }
9574}
9575
9576void SpeculativeJIT::compileNumberToStringWithRadix(Node* node)
9577{
9578 bool validRadixIsGuaranteed = false;
9579 if (node->child2()->isInt32Constant()) {
9580 int32_t radix = node->child2()->asInt32();
9581 if (radix >= 2 && radix <= 36)
9582 validRadixIsGuaranteed = true;
9583 }
9584
9585 auto callToString = [&] (auto operation, GPRReg resultGPR, auto valueReg, GPRReg radixGPR) {
9586 flushRegisters();
9587 callOperation(operation, resultGPR, valueReg, radixGPR);
9588 m_jit.exceptionCheck();
9589 cellResult(resultGPR, node);
9590 };
9591
9592 switch (node->child1().useKind()) {
9593 case Int32Use: {
9594 SpeculateStrictInt32Operand value(this, node->child1());
9595 SpeculateStrictInt32Operand radix(this, node->child2());
9596 GPRFlushedCallResult result(this);
9597 callToString(validRadixIsGuaranteed ? operationInt32ToStringWithValidRadix : operationInt32ToString, result.gpr(), value.gpr(), radix.gpr());
9598 break;
9599 }
9600
9601#if USE(JSVALUE64)
9602 case Int52RepUse: {
9603 SpeculateStrictInt52Operand value(this, node->child1());
9604 SpeculateStrictInt32Operand radix(this, node->child2());
9605 GPRFlushedCallResult result(this);
9606 callToString(validRadixIsGuaranteed ? operationInt52ToStringWithValidRadix : operationInt52ToString, result.gpr(), value.gpr(), radix.gpr());
9607 break;
9608 }
9609#endif
9610
9611 case DoubleRepUse: {
9612 SpeculateDoubleOperand value(this, node->child1());
9613 SpeculateStrictInt32Operand radix(this, node->child2());
9614 GPRFlushedCallResult result(this);
9615 callToString(validRadixIsGuaranteed ? operationDoubleToStringWithValidRadix : operationDoubleToString, result.gpr(), value.fpr(), radix.gpr());
9616 break;
9617 }
9618
9619 default:
9620 RELEASE_ASSERT_NOT_REACHED();
9621 }
9622}
9623
9624void SpeculativeJIT::compileNewStringObject(Node* node)
9625{
9626 SpeculateCellOperand operand(this, node->child1());
9627
9628 GPRTemporary result(this);
9629 GPRTemporary scratch1(this);
9630 GPRTemporary scratch2(this);
9631
9632 GPRReg operandGPR = operand.gpr();
9633 GPRReg resultGPR = result.gpr();
9634 GPRReg scratch1GPR = scratch1.gpr();
9635 GPRReg scratch2GPR = scratch2.gpr();
9636
9637 JITCompiler::JumpList slowPath;
9638
9639 auto butterfly = TrustedImmPtr(nullptr);
9640 emitAllocateJSObject<StringObject>(
9641 resultGPR, TrustedImmPtr(node->structure()), butterfly, scratch1GPR, scratch2GPR,
9642 slowPath);
9643
9644 m_jit.storePtr(
9645 TrustedImmPtr(StringObject::info()),
9646 JITCompiler::Address(resultGPR, JSDestructibleObject::classInfoOffset()));
9647#if USE(JSVALUE64)
9648 m_jit.store64(
9649 operandGPR, JITCompiler::Address(resultGPR, JSWrapperObject::internalValueOffset()));
9650#else
9651 m_jit.store32(
9652 TrustedImm32(JSValue::CellTag),
9653 JITCompiler::Address(resultGPR, JSWrapperObject::internalValueOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
9654 m_jit.store32(
9655 operandGPR,
9656 JITCompiler::Address(resultGPR, JSWrapperObject::internalValueOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
9657#endif
9658
9659 m_jit.mutatorFence(*m_jit.vm());
9660
9661 addSlowPathGenerator(slowPathCall(
9662 slowPath, this, operationNewStringObject, resultGPR, operandGPR, node->structure()));
9663
9664 cellResult(resultGPR, node);
9665}
9666
9667void SpeculativeJIT::compileNewSymbol(Node* node)
9668{
9669 if (!node->child1()) {
9670 flushRegisters();
9671 GPRFlushedCallResult result(this);
9672 GPRReg resultGPR = result.gpr();
9673 callOperation(operationNewSymbol, resultGPR);
9674 m_jit.exceptionCheck();
9675 cellResult(resultGPR, node);
9676 return;
9677 }
9678
9679
9680 ASSERT(node->child1().useKind() == KnownStringUse);
9681 SpeculateCellOperand operand(this, node->child1());
9682
9683 GPRReg stringGPR = operand.gpr();
9684
9685 flushRegisters();
9686 GPRFlushedCallResult result(this);
9687 GPRReg resultGPR = result.gpr();
9688 callOperation(operationNewSymbolWithDescription, resultGPR, stringGPR);
9689 m_jit.exceptionCheck();
9690 cellResult(resultGPR, node);
9691}
9692
9693void SpeculativeJIT::compileNewTypedArrayWithSize(Node* node)
9694{
9695 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
9696 auto typedArrayType = node->typedArrayType();
9697 RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->typedArrayStructureConcurrently(typedArrayType));
9698 RELEASE_ASSERT(structure.get());
9699
9700 SpeculateInt32Operand size(this, node->child1());
9701 GPRReg sizeGPR = size.gpr();
9702
9703 GPRTemporary result(this);
9704 GPRTemporary storage(this);
9705 GPRTemporary scratch(this);
9706 GPRTemporary scratch2(this);
9707 GPRReg resultGPR = result.gpr();
9708 GPRReg storageGPR = storage.gpr();
9709 GPRReg scratchGPR = scratch.gpr();
9710 GPRReg scratchGPR2 = scratch2.gpr();
9711
9712 JITCompiler::JumpList slowCases;
9713
9714 m_jit.move(TrustedImmPtr(nullptr), storageGPR);
9715
9716 slowCases.append(m_jit.branch32(
9717 MacroAssembler::Above, sizeGPR, TrustedImm32(JSArrayBufferView::fastSizeLimit)));
9718
9719 m_jit.move(sizeGPR, scratchGPR);
9720 m_jit.lshift32(TrustedImm32(logElementSize(typedArrayType)), scratchGPR);
9721 if (elementSize(typedArrayType) < 8) {
9722 m_jit.add32(TrustedImm32(7), scratchGPR);
9723 m_jit.and32(TrustedImm32(~7), scratchGPR);
9724 }
9725 m_jit.emitAllocateVariableSized(
9726 storageGPR, m_jit.vm()->primitiveGigacageAuxiliarySpace, scratchGPR, scratchGPR,
9727 scratchGPR2, slowCases);
9728
9729 MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, sizeGPR);
9730 m_jit.move(sizeGPR, scratchGPR);
9731 if (elementSize(typedArrayType) != 4) {
9732 if (elementSize(typedArrayType) > 4)
9733 m_jit.lshift32(TrustedImm32(logElementSize(typedArrayType) - 2), scratchGPR);
9734 else {
9735 if (elementSize(typedArrayType) > 1)
9736 m_jit.lshift32(TrustedImm32(logElementSize(typedArrayType)), scratchGPR);
9737 m_jit.add32(TrustedImm32(3), scratchGPR);
9738 m_jit.urshift32(TrustedImm32(2), scratchGPR);
9739 }
9740 }
9741 MacroAssembler::Label loop = m_jit.label();
9742 m_jit.sub32(TrustedImm32(1), scratchGPR);
9743 m_jit.store32(
9744 TrustedImm32(0),
9745 MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesFour));
9746 m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit);
9747 done.link(&m_jit);
9748
9749 auto butterfly = TrustedImmPtr(nullptr);
9750 emitAllocateJSObject<JSArrayBufferView>(
9751 resultGPR, TrustedImmPtr(structure), butterfly, scratchGPR, scratchGPR2,
9752 slowCases);
9753
9754 m_jit.storePtr(
9755 storageGPR,
9756 MacroAssembler::Address(resultGPR, JSArrayBufferView::offsetOfVector()));
9757 m_jit.store32(
9758 sizeGPR,
9759 MacroAssembler::Address(resultGPR, JSArrayBufferView::offsetOfLength()));
9760 m_jit.store32(
9761 TrustedImm32(FastTypedArray),
9762 MacroAssembler::Address(resultGPR, JSArrayBufferView::offsetOfMode()));
9763
9764 m_jit.mutatorFence(*m_jit.vm());
9765
9766 addSlowPathGenerator(slowPathCall(
9767 slowCases, this, operationNewTypedArrayWithSizeForType(typedArrayType),
9768 resultGPR, structure, sizeGPR, storageGPR));
9769
9770 cellResult(resultGPR, node);
9771}
9772
9773void SpeculativeJIT::compileNewRegexp(Node* node)
9774{
9775 RegExp* regexp = node->castOperand<RegExp*>();
9776
9777 GPRTemporary result(this);
9778 GPRTemporary scratch1(this);
9779 GPRTemporary scratch2(this);
9780 JSValueOperand lastIndex(this, node->child1());
9781
9782 GPRReg resultGPR = result.gpr();
9783 GPRReg scratch1GPR = scratch1.gpr();
9784 GPRReg scratch2GPR = scratch2.gpr();
9785 JSValueRegs lastIndexRegs = lastIndex.jsValueRegs();
9786
9787 JITCompiler::JumpList slowPath;
9788
9789 auto structure = m_jit.graph().registerStructure(m_jit.graph().globalObjectFor(node->origin.semantic)->regExpStructure());
9790 auto butterfly = TrustedImmPtr(nullptr);
9791 emitAllocateJSObject<RegExpObject>(resultGPR, TrustedImmPtr(structure), butterfly, scratch1GPR, scratch2GPR, slowPath);
9792
9793 m_jit.storePtr(
9794 TrustedImmPtr(node->cellOperand()),
9795 CCallHelpers::Address(resultGPR, RegExpObject::offsetOfRegExpAndLastIndexIsNotWritableFlag()));
9796 m_jit.storeValue(lastIndexRegs, CCallHelpers::Address(resultGPR, RegExpObject::offsetOfLastIndex()));
9797 m_jit.mutatorFence(*m_jit.vm());
9798
9799 addSlowPathGenerator(slowPathCall(slowPath, this, operationNewRegexpWithLastIndex, resultGPR, regexp, lastIndexRegs));
9800
9801 cellResult(resultGPR, node);
9802}
9803
9804void SpeculativeJIT::speculateCellTypeWithoutTypeFiltering(
9805 Edge edge, GPRReg cellGPR, JSType jsType)
9806{
9807 speculationCheck(
9808 BadType, JSValueSource::unboxedCell(cellGPR), edge,
9809 m_jit.branchIfNotType(cellGPR, jsType));
9810}
9811
9812void SpeculativeJIT::speculateCellType(
9813 Edge edge, GPRReg cellGPR, SpeculatedType specType, JSType jsType)
9814{
9815 DFG_TYPE_CHECK(
9816 JSValueSource::unboxedCell(cellGPR), edge, specType,
9817 m_jit.branchIfNotType(cellGPR, jsType));
9818}
9819
9820void SpeculativeJIT::speculateInt32(Edge edge)
9821{
9822 if (!needsTypeCheck(edge, SpecInt32Only))
9823 return;
9824
9825 (SpeculateInt32Operand(this, edge)).gpr();
9826}
9827
9828void SpeculativeJIT::speculateNumber(Edge edge)
9829{
9830 if (!needsTypeCheck(edge, SpecBytecodeNumber))
9831 return;
9832
9833 JSValueOperand value(this, edge, ManualOperandSpeculation);
9834#if USE(JSVALUE64)
9835 GPRReg gpr = value.gpr();
9836 typeCheck(
9837 JSValueRegs(gpr), edge, SpecBytecodeNumber,
9838 m_jit.branchIfNotNumber(gpr));
9839#else
9840 IGNORE_WARNINGS_BEGIN("enum-compare")
9841 static_assert(JSValue::Int32Tag >= JSValue::LowestTag, "Int32Tag is included in >= JSValue::LowestTag range.");
9842 IGNORE_WARNINGS_END
9843 GPRReg tagGPR = value.tagGPR();
9844 DFG_TYPE_CHECK(
9845 value.jsValueRegs(), edge, ~SpecInt32Only,
9846 m_jit.branchIfInt32(tagGPR));
9847 DFG_TYPE_CHECK(
9848 value.jsValueRegs(), edge, SpecBytecodeNumber,
9849 m_jit.branch32(MacroAssembler::AboveOrEqual, tagGPR, TrustedImm32(JSValue::LowestTag)));
9850#endif
9851}
9852
9853void SpeculativeJIT::speculateRealNumber(Edge edge)
9854{
9855 if (!needsTypeCheck(edge, SpecBytecodeRealNumber))
9856 return;
9857
9858 JSValueOperand op1(this, edge, ManualOperandSpeculation);
9859 FPRTemporary result(this);
9860
9861 JSValueRegs op1Regs = op1.jsValueRegs();
9862 FPRReg resultFPR = result.fpr();
9863
9864#if USE(JSVALUE64)
9865 GPRTemporary temp(this);
9866 GPRReg tempGPR = temp.gpr();
9867 m_jit.unboxDoubleWithoutAssertions(op1Regs.gpr(), tempGPR, resultFPR);
9868#else
9869 FPRTemporary temp(this);
9870 FPRReg tempFPR = temp.fpr();
9871 unboxDouble(op1Regs.tagGPR(), op1Regs.payloadGPR(), resultFPR, tempFPR);
9872#endif
9873
9874 JITCompiler::Jump done = m_jit.branchIfNotNaN(resultFPR);
9875
9876 typeCheck(op1Regs, edge, SpecBytecodeRealNumber, m_jit.branchIfNotInt32(op1Regs));
9877
9878 done.link(&m_jit);
9879}
9880
9881void SpeculativeJIT::speculateDoubleRepReal(Edge edge)
9882{
9883 if (!needsTypeCheck(edge, SpecDoubleReal))
9884 return;
9885
9886 SpeculateDoubleOperand operand(this, edge);
9887 FPRReg fpr = operand.fpr();
9888 typeCheck(
9889 JSValueRegs(), edge, SpecDoubleReal,
9890 m_jit.branchIfNaN(fpr));
9891}
9892
9893void SpeculativeJIT::speculateBoolean(Edge edge)
9894{
9895 if (!needsTypeCheck(edge, SpecBoolean))
9896 return;
9897
9898 (SpeculateBooleanOperand(this, edge)).gpr();
9899}
9900
9901void SpeculativeJIT::speculateCell(Edge edge)
9902{
9903 if (!needsTypeCheck(edge, SpecCellCheck))
9904 return;
9905
9906 (SpeculateCellOperand(this, edge)).gpr();
9907}
9908
9909void SpeculativeJIT::speculateCellOrOther(Edge edge)
9910{
9911 if (!needsTypeCheck(edge, SpecCellCheck | SpecOther))
9912 return;
9913
9914 JSValueOperand operand(this, edge, ManualOperandSpeculation);
9915 GPRTemporary temp(this);
9916 GPRReg tempGPR = temp.gpr();
9917
9918 MacroAssembler::Jump ok = m_jit.branchIfCell(operand.jsValueRegs());
9919 DFG_TYPE_CHECK(
9920 operand.jsValueRegs(), edge, SpecCellCheck | SpecOther,
9921 m_jit.branchIfNotOther(operand.jsValueRegs(), tempGPR));
9922 ok.link(&m_jit);
9923}
9924
9925void SpeculativeJIT::speculateObject(Edge edge, GPRReg cell)
9926{
9927 DFG_TYPE_CHECK(JSValueSource::unboxedCell(cell), edge, SpecObject, m_jit.branchIfNotObject(cell));
9928}
9929
9930void SpeculativeJIT::speculateObject(Edge edge)
9931{
9932 if (!needsTypeCheck(edge, SpecObject))
9933 return;
9934
9935 SpeculateCellOperand operand(this, edge);
9936 speculateObject(edge, operand.gpr());
9937}
9938
9939void SpeculativeJIT::speculateFunction(Edge edge, GPRReg cell)
9940{
9941 speculateCellType(edge, cell, SpecFunction, JSFunctionType);
9942}
9943
9944void SpeculativeJIT::speculateFunction(Edge edge)
9945{
9946 if (!needsTypeCheck(edge, SpecFunction))
9947 return;
9948
9949 SpeculateCellOperand operand(this, edge);
9950 speculateFunction(edge, operand.gpr());
9951}
9952
9953void SpeculativeJIT::speculateFinalObject(Edge edge, GPRReg cell)
9954{
9955 speculateCellType(edge, cell, SpecFinalObject, FinalObjectType);
9956}
9957
9958void SpeculativeJIT::speculateFinalObject(Edge edge)
9959{
9960 if (!needsTypeCheck(edge, SpecFinalObject))
9961 return;
9962
9963 SpeculateCellOperand operand(this, edge);
9964 speculateFinalObject(edge, operand.gpr());
9965}
9966
9967void SpeculativeJIT::speculateRegExpObject(Edge edge, GPRReg cell)
9968{
9969 speculateCellType(edge, cell, SpecRegExpObject, RegExpObjectType);
9970}
9971
9972void SpeculativeJIT::speculateRegExpObject(Edge edge)
9973{
9974 if (!needsTypeCheck(edge, SpecRegExpObject))
9975 return;
9976
9977 SpeculateCellOperand operand(this, edge);
9978 speculateRegExpObject(edge, operand.gpr());
9979}
9980
9981void SpeculativeJIT::speculateArray(Edge edge, GPRReg cell)
9982{
9983 speculateCellType(edge, cell, SpecArray, ArrayType);
9984}
9985
9986void SpeculativeJIT::speculateArray(Edge edge)
9987{
9988 if (!needsTypeCheck(edge, SpecArray))
9989 return;
9990
9991 SpeculateCellOperand operand(this, edge);
9992 speculateArray(edge, operand.gpr());
9993}
9994
9995void SpeculativeJIT::speculateProxyObject(Edge edge, GPRReg cell)
9996{
9997 speculateCellType(edge, cell, SpecProxyObject, ProxyObjectType);
9998}
9999
10000void SpeculativeJIT::speculateProxyObject(Edge edge)
10001{
10002 if (!needsTypeCheck(edge, SpecProxyObject))
10003 return;
10004
10005 SpeculateCellOperand operand(this, edge);
10006 speculateProxyObject(edge, operand.gpr());
10007}
10008
10009void SpeculativeJIT::speculateDerivedArray(Edge edge, GPRReg cell)
10010{
10011 speculateCellType(edge, cell, SpecDerivedArray, DerivedArrayType);
10012}
10013
10014void SpeculativeJIT::speculateDerivedArray(Edge edge)
10015{
10016 if (!needsTypeCheck(edge, SpecDerivedArray))
10017 return;
10018
10019 SpeculateCellOperand operand(this, edge);
10020 speculateDerivedArray(edge, operand.gpr());
10021}
10022
10023void SpeculativeJIT::speculateMapObject(Edge edge, GPRReg cell)
10024{
10025 speculateCellType(edge, cell, SpecMapObject, JSMapType);
10026}
10027
10028void SpeculativeJIT::speculateMapObject(Edge edge)
10029{
10030 if (!needsTypeCheck(edge, SpecMapObject))
10031 return;
10032
10033 SpeculateCellOperand operand(this, edge);
10034 speculateMapObject(edge, operand.gpr());
10035}
10036
10037void SpeculativeJIT::speculateSetObject(Edge edge, GPRReg cell)
10038{
10039 speculateCellType(edge, cell, SpecSetObject, JSSetType);
10040}
10041
10042void SpeculativeJIT::speculateSetObject(Edge edge)
10043{
10044 if (!needsTypeCheck(edge, SpecSetObject))
10045 return;
10046
10047 SpeculateCellOperand operand(this, edge);
10048 speculateSetObject(edge, operand.gpr());
10049}
10050
10051void SpeculativeJIT::speculateWeakMapObject(Edge edge, GPRReg cell)
10052{
10053 speculateCellType(edge, cell, SpecWeakMapObject, JSWeakMapType);
10054}
10055
10056void SpeculativeJIT::speculateWeakMapObject(Edge edge)
10057{
10058 if (!needsTypeCheck(edge, SpecWeakMapObject))
10059 return;
10060
10061 SpeculateCellOperand operand(this, edge);
10062 speculateWeakMapObject(edge, operand.gpr());
10063}
10064
10065void SpeculativeJIT::speculateWeakSetObject(Edge edge, GPRReg cell)
10066{
10067 speculateCellType(edge, cell, SpecWeakSetObject, JSWeakSetType);
10068}
10069
10070void SpeculativeJIT::speculateWeakSetObject(Edge edge)
10071{
10072 if (!needsTypeCheck(edge, SpecWeakSetObject))
10073 return;
10074
10075 SpeculateCellOperand operand(this, edge);
10076 speculateWeakSetObject(edge, operand.gpr());
10077}
10078
10079void SpeculativeJIT::speculateDataViewObject(Edge edge, GPRReg cell)
10080{
10081 speculateCellType(edge, cell, SpecDataViewObject, DataViewType);
10082}
10083
10084void SpeculativeJIT::speculateDataViewObject(Edge edge)
10085{
10086 if (!needsTypeCheck(edge, SpecDataViewObject))
10087 return;
10088
10089 SpeculateCellOperand operand(this, edge);
10090 speculateDataViewObject(edge, operand.gpr());
10091}
10092
10093void SpeculativeJIT::speculateObjectOrOther(Edge edge)
10094{
10095 if (!needsTypeCheck(edge, SpecObject | SpecOther))
10096 return;
10097
10098 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10099 GPRTemporary temp(this);
10100 GPRReg tempGPR = temp.gpr();
10101 MacroAssembler::Jump notCell = m_jit.branchIfNotCell(operand.jsValueRegs());
10102 GPRReg gpr = operand.jsValueRegs().payloadGPR();
10103 DFG_TYPE_CHECK(
10104 operand.jsValueRegs(), edge, (~SpecCellCheck) | SpecObject, m_jit.branchIfNotObject(gpr));
10105 MacroAssembler::Jump done = m_jit.jump();
10106 notCell.link(&m_jit);
10107 DFG_TYPE_CHECK(
10108 operand.jsValueRegs(), edge, SpecCellCheck | SpecOther,
10109 m_jit.branchIfNotOther(operand.jsValueRegs(), tempGPR));
10110 done.link(&m_jit);
10111}
10112
10113void SpeculativeJIT::speculateString(Edge edge, GPRReg cell)
10114{
10115 DFG_TYPE_CHECK(
10116 JSValueSource::unboxedCell(cell), edge, SpecString | ~SpecCellCheck, m_jit.branchIfNotString(cell));
10117}
10118
10119void SpeculativeJIT::speculateStringOrOther(Edge edge, JSValueRegs regs, GPRReg scratch)
10120{
10121 JITCompiler::Jump notCell = m_jit.branchIfNotCell(regs);
10122 GPRReg cell = regs.payloadGPR();
10123 DFG_TYPE_CHECK(regs, edge, (~SpecCellCheck) | SpecString, m_jit.branchIfNotString(cell));
10124 JITCompiler::Jump done = m_jit.jump();
10125 notCell.link(&m_jit);
10126 DFG_TYPE_CHECK(regs, edge, SpecCellCheck | SpecOther, m_jit.branchIfNotOther(regs, scratch));
10127 done.link(&m_jit);
10128}
10129
10130void SpeculativeJIT::speculateStringOrOther(Edge edge)
10131{
10132 if (!needsTypeCheck(edge, SpecString | SpecOther))
10133 return;
10134
10135 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10136 GPRTemporary temp(this);
10137 JSValueRegs regs = operand.jsValueRegs();
10138 GPRReg tempGPR = temp.gpr();
10139 speculateStringOrOther(edge, regs, tempGPR);
10140}
10141
10142void SpeculativeJIT::speculateStringIdentAndLoadStorage(Edge edge, GPRReg string, GPRReg storage)
10143{
10144 m_jit.loadPtr(MacroAssembler::Address(string, JSString::offsetOfValue()), storage);
10145
10146 if (!needsTypeCheck(edge, SpecStringIdent | ~SpecString))
10147 return;
10148
10149 speculationCheck(
10150 BadType, JSValueSource::unboxedCell(string), edge,
10151 m_jit.branchIfRopeStringImpl(storage));
10152 speculationCheck(
10153 BadType, JSValueSource::unboxedCell(string), edge, m_jit.branchTest32(
10154 MacroAssembler::Zero,
10155 MacroAssembler::Address(storage, StringImpl::flagsOffset()),
10156 MacroAssembler::TrustedImm32(StringImpl::flagIsAtomic())));
10157
10158 m_interpreter.filter(edge, SpecStringIdent | ~SpecString);
10159}
10160
10161void SpeculativeJIT::speculateStringIdent(Edge edge, GPRReg string)
10162{
10163 if (!needsTypeCheck(edge, SpecStringIdent))
10164 return;
10165
10166 GPRTemporary temp(this);
10167 speculateStringIdentAndLoadStorage(edge, string, temp.gpr());
10168}
10169
10170void SpeculativeJIT::speculateStringIdent(Edge edge)
10171{
10172 if (!needsTypeCheck(edge, SpecStringIdent))
10173 return;
10174
10175 SpeculateCellOperand operand(this, edge);
10176 GPRReg gpr = operand.gpr();
10177 speculateString(edge, gpr);
10178 speculateStringIdent(edge, gpr);
10179}
10180
10181void SpeculativeJIT::speculateString(Edge edge)
10182{
10183 if (!needsTypeCheck(edge, SpecString))
10184 return;
10185
10186 SpeculateCellOperand operand(this, edge);
10187 speculateString(edge, operand.gpr());
10188}
10189
10190void SpeculativeJIT::speculateStringObject(Edge edge, GPRReg cellGPR)
10191{
10192 DFG_TYPE_CHECK(JSValueSource::unboxedCell(cellGPR), edge, ~SpecCellCheck | SpecStringObject, m_jit.branchIfNotType(cellGPR, StringObjectType));
10193}
10194
10195void SpeculativeJIT::speculateStringObject(Edge edge)
10196{
10197 if (!needsTypeCheck(edge, SpecStringObject))
10198 return;
10199
10200 SpeculateCellOperand operand(this, edge);
10201 GPRReg gpr = operand.gpr();
10202 speculateStringObject(edge, gpr);
10203}
10204
10205void SpeculativeJIT::speculateStringOrStringObject(Edge edge)
10206{
10207 if (!needsTypeCheck(edge, SpecString | SpecStringObject))
10208 return;
10209
10210 SpeculateCellOperand operand(this, edge);
10211 GPRReg gpr = operand.gpr();
10212 if (!needsTypeCheck(edge, SpecString | SpecStringObject))
10213 return;
10214
10215 GPRTemporary typeTemp(this);
10216 GPRReg typeGPR = typeTemp.gpr();
10217
10218 m_jit.load8(JITCompiler::Address(gpr, JSCell::typeInfoTypeOffset()), typeGPR);
10219
10220 JITCompiler::Jump isString = m_jit.branch32(JITCompiler::Equal, typeGPR, TrustedImm32(StringType));
10221 speculationCheck(BadType, JSValueSource::unboxedCell(gpr), edge.node(), m_jit.branch32(JITCompiler::NotEqual, typeGPR, TrustedImm32(StringObjectType)));
10222 isString.link(&m_jit);
10223
10224 m_interpreter.filter(edge, SpecString | SpecStringObject);
10225}
10226
10227void SpeculativeJIT::speculateNotStringVar(Edge edge)
10228{
10229 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10230 GPRTemporary temp(this);
10231 GPRReg tempGPR = temp.gpr();
10232
10233 JITCompiler::Jump notCell = m_jit.branchIfNotCell(operand.jsValueRegs());
10234 GPRReg cell = operand.jsValueRegs().payloadGPR();
10235
10236 JITCompiler::Jump notString = m_jit.branchIfNotString(cell);
10237
10238 speculateStringIdentAndLoadStorage(edge, cell, tempGPR);
10239
10240 notString.link(&m_jit);
10241 notCell.link(&m_jit);
10242}
10243
10244void SpeculativeJIT::speculateNotSymbol(Edge edge)
10245{
10246 if (!needsTypeCheck(edge, ~SpecSymbol))
10247 return;
10248
10249 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10250 auto valueRegs = operand.jsValueRegs();
10251 GPRReg value = valueRegs.payloadGPR();
10252 JITCompiler::Jump notCell;
10253
10254 bool needsCellCheck = needsTypeCheck(edge, SpecCell);
10255 if (needsCellCheck)
10256 notCell = m_jit.branchIfNotCell(valueRegs);
10257
10258 speculationCheck(BadType, JSValueSource::unboxedCell(value), edge.node(), m_jit.branchIfSymbol(value));
10259
10260 if (needsCellCheck)
10261 notCell.link(&m_jit);
10262
10263 m_interpreter.filter(edge, ~SpecSymbol);
10264}
10265
10266void SpeculativeJIT::speculateSymbol(Edge edge, GPRReg cell)
10267{
10268 DFG_TYPE_CHECK(JSValueSource::unboxedCell(cell), edge, ~SpecCellCheck | SpecSymbol, m_jit.branchIfNotSymbol(cell));
10269}
10270
10271void SpeculativeJIT::speculateSymbol(Edge edge)
10272{
10273 if (!needsTypeCheck(edge, SpecSymbol))
10274 return;
10275
10276 SpeculateCellOperand operand(this, edge);
10277 speculateSymbol(edge, operand.gpr());
10278}
10279
10280void SpeculativeJIT::speculateBigInt(Edge edge, GPRReg cell)
10281{
10282 DFG_TYPE_CHECK(JSValueSource::unboxedCell(cell), edge, ~SpecCellCheck | SpecBigInt, m_jit.branchIfNotBigInt(cell));
10283}
10284
10285void SpeculativeJIT::speculateBigInt(Edge edge)
10286{
10287 if (!needsTypeCheck(edge, SpecBigInt))
10288 return;
10289
10290 SpeculateCellOperand operand(this, edge);
10291 speculateBigInt(edge, operand.gpr());
10292}
10293
10294void SpeculativeJIT::speculateNotCell(Edge edge, JSValueRegs regs)
10295{
10296 DFG_TYPE_CHECK(regs, edge, ~SpecCellCheck, m_jit.branchIfCell(regs));
10297}
10298
10299void SpeculativeJIT::speculateNotCell(Edge edge)
10300{
10301 if (!needsTypeCheck(edge, ~SpecCellCheck))
10302 return;
10303
10304 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10305 speculateNotCell(edge, operand.jsValueRegs());
10306}
10307
10308void SpeculativeJIT::speculateOther(Edge edge, JSValueRegs regs, GPRReg tempGPR)
10309{
10310 DFG_TYPE_CHECK(regs, edge, SpecOther, m_jit.branchIfNotOther(regs, tempGPR));
10311}
10312
10313void SpeculativeJIT::speculateOther(Edge edge, JSValueRegs regs)
10314{
10315 if (!needsTypeCheck(edge, SpecOther))
10316 return;
10317
10318 GPRTemporary temp(this);
10319 GPRReg tempGPR = temp.gpr();
10320 speculateOther(edge, regs, tempGPR);
10321}
10322
10323void SpeculativeJIT::speculateOther(Edge edge)
10324{
10325 if (!needsTypeCheck(edge, SpecOther))
10326 return;
10327
10328 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10329 speculateOther(edge, operand.jsValueRegs());
10330}
10331
10332void SpeculativeJIT::speculateMisc(Edge edge, JSValueRegs regs)
10333{
10334#if USE(JSVALUE64)
10335 DFG_TYPE_CHECK(
10336 regs, edge, SpecMisc,
10337 m_jit.branch64(MacroAssembler::Above, regs.gpr(), MacroAssembler::TrustedImm64(TagBitTypeOther | TagBitBool | TagBitUndefined)));
10338#else
10339 IGNORE_WARNINGS_BEGIN("enum-compare")
10340 static_assert(JSValue::Int32Tag >= JSValue::UndefinedTag, "Int32Tag is included in >= JSValue::UndefinedTag range.");
10341 IGNORE_WARNINGS_END
10342 DFG_TYPE_CHECK(
10343 regs, edge, ~SpecInt32Only,
10344 m_jit.branchIfInt32(regs.tagGPR()));
10345 DFG_TYPE_CHECK(
10346 regs, edge, SpecMisc,
10347 m_jit.branch32(MacroAssembler::Below, regs.tagGPR(), MacroAssembler::TrustedImm32(JSValue::UndefinedTag)));
10348#endif
10349}
10350
10351void SpeculativeJIT::speculateMisc(Edge edge)
10352{
10353 if (!needsTypeCheck(edge, SpecMisc))
10354 return;
10355
10356 JSValueOperand operand(this, edge, ManualOperandSpeculation);
10357 speculateMisc(edge, operand.jsValueRegs());
10358}
10359
10360void SpeculativeJIT::speculate(Node*, Edge edge)
10361{
10362 switch (edge.useKind()) {
10363 case UntypedUse:
10364 break;
10365 case DoubleRepUse:
10366 case Int52RepUse:
10367 case KnownInt32Use:
10368 case KnownCellUse:
10369 case KnownStringUse:
10370 case KnownPrimitiveUse:
10371 case KnownOtherUse:
10372 case KnownBooleanUse:
10373 ASSERT(!m_interpreter.needsTypeCheck(edge));
10374 break;
10375 case Int32Use:
10376 speculateInt32(edge);
10377 break;
10378 case NumberUse:
10379 speculateNumber(edge);
10380 break;
10381 case RealNumberUse:
10382 speculateRealNumber(edge);
10383 break;
10384 case DoubleRepRealUse:
10385 speculateDoubleRepReal(edge);
10386 break;
10387#if USE(JSVALUE64)
10388 case AnyIntUse:
10389 speculateAnyInt(edge);
10390 break;
10391 case DoubleRepAnyIntUse:
10392 speculateDoubleRepAnyInt(edge);
10393 break;
10394#endif
10395 case BooleanUse:
10396 speculateBoolean(edge);
10397 break;
10398 case CellUse:
10399 speculateCell(edge);
10400 break;
10401 case CellOrOtherUse:
10402 speculateCellOrOther(edge);
10403 break;
10404 case ObjectUse:
10405 speculateObject(edge);
10406 break;
10407 case FunctionUse:
10408 speculateFunction(edge);
10409 break;
10410 case ArrayUse:
10411 speculateArray(edge);
10412 break;
10413 case FinalObjectUse:
10414 speculateFinalObject(edge);
10415 break;
10416 case RegExpObjectUse:
10417 speculateRegExpObject(edge);
10418 break;
10419 case ProxyObjectUse:
10420 speculateProxyObject(edge);
10421 break;
10422 case DerivedArrayUse:
10423 speculateDerivedArray(edge);
10424 break;
10425 case MapObjectUse:
10426 speculateMapObject(edge);
10427 break;
10428 case SetObjectUse:
10429 speculateSetObject(edge);
10430 break;
10431 case WeakMapObjectUse:
10432 speculateWeakMapObject(edge);
10433 break;
10434 case WeakSetObjectUse:
10435 speculateWeakSetObject(edge);
10436 break;
10437 case DataViewObjectUse:
10438 speculateDataViewObject(edge);
10439 break;
10440 case ObjectOrOtherUse:
10441 speculateObjectOrOther(edge);
10442 break;
10443 case StringIdentUse:
10444 speculateStringIdent(edge);
10445 break;
10446 case StringUse:
10447 speculateString(edge);
10448 break;
10449 case StringOrOtherUse:
10450 speculateStringOrOther(edge);
10451 break;
10452 case SymbolUse:
10453 speculateSymbol(edge);
10454 break;
10455 case BigIntUse:
10456 speculateBigInt(edge);
10457 break;
10458 case StringObjectUse:
10459 speculateStringObject(edge);
10460 break;
10461 case StringOrStringObjectUse:
10462 speculateStringOrStringObject(edge);
10463 break;
10464 case NotStringVarUse:
10465 speculateNotStringVar(edge);
10466 break;
10467 case NotSymbolUse:
10468 speculateNotSymbol(edge);
10469 break;
10470 case NotCellUse:
10471 speculateNotCell(edge);
10472 break;
10473 case OtherUse:
10474 speculateOther(edge);
10475 break;
10476 case MiscUse:
10477 speculateMisc(edge);
10478 break;
10479 default:
10480 RELEASE_ASSERT_NOT_REACHED();
10481 break;
10482 }
10483}
10484
10485void SpeculativeJIT::emitSwitchIntJump(
10486 SwitchData* data, GPRReg value, GPRReg scratch)
10487{
10488 SimpleJumpTable& table = m_jit.codeBlock()->switchJumpTable(data->switchTableIndex);
10489 table.ensureCTITable();
10490 m_jit.sub32(Imm32(table.min), value);
10491 addBranch(
10492 m_jit.branch32(JITCompiler::AboveOrEqual, value, Imm32(table.ctiOffsets.size())),
10493 data->fallThrough.block);
10494 m_jit.move(TrustedImmPtr(table.ctiOffsets.begin()), scratch);
10495 m_jit.loadPtr(JITCompiler::BaseIndex(scratch, value, JITCompiler::timesPtr()), scratch);
10496
10497 m_jit.jump(scratch, JSSwitchPtrTag);
10498 data->didUseJumpTable = true;
10499}
10500
10501void SpeculativeJIT::emitSwitchImm(Node* node, SwitchData* data)
10502{
10503 switch (node->child1().useKind()) {
10504 case Int32Use: {
10505 SpeculateInt32Operand value(this, node->child1());
10506 GPRTemporary temp(this);
10507 emitSwitchIntJump(data, value.gpr(), temp.gpr());
10508 noResult(node);
10509 break;
10510 }
10511
10512 case UntypedUse: {
10513 JSValueOperand value(this, node->child1());
10514 GPRTemporary temp(this);
10515 JSValueRegs valueRegs = value.jsValueRegs();
10516 GPRReg scratch = temp.gpr();
10517
10518 value.use();
10519
10520 auto notInt32 = m_jit.branchIfNotInt32(valueRegs);
10521 emitSwitchIntJump(data, valueRegs.payloadGPR(), scratch);
10522 notInt32.link(&m_jit);
10523 addBranch(m_jit.branchIfNotNumber(valueRegs, scratch), data->fallThrough.block);
10524 silentSpillAllRegisters(scratch);
10525 callOperation(operationFindSwitchImmTargetForDouble, scratch, valueRegs, data->switchTableIndex);
10526 silentFillAllRegisters();
10527
10528 m_jit.jump(scratch, JSSwitchPtrTag);
10529 noResult(node, UseChildrenCalledExplicitly);
10530 break;
10531 }
10532
10533 default:
10534 RELEASE_ASSERT_NOT_REACHED();
10535 break;
10536 }
10537}
10538
10539void SpeculativeJIT::emitSwitchCharStringJump(
10540 SwitchData* data, GPRReg value, GPRReg scratch)
10541{
10542 m_jit.loadPtr(MacroAssembler::Address(value, JSString::offsetOfValue()), scratch);
10543 auto isRope = m_jit.branchIfRopeStringImpl(scratch);
10544
10545 addBranch(
10546 m_jit.branch32(
10547 MacroAssembler::NotEqual,
10548 MacroAssembler::Address(scratch, StringImpl::lengthMemoryOffset()),
10549 TrustedImm32(1)),
10550 data->fallThrough.block);
10551
10552 addSlowPathGenerator(slowPathCall(isRope, this, operationResolveRope, scratch, value));
10553
10554 m_jit.loadPtr(MacroAssembler::Address(scratch, StringImpl::dataOffset()), value);
10555
10556 JITCompiler::Jump is8Bit = m_jit.branchTest32(
10557 MacroAssembler::NonZero,
10558 MacroAssembler::Address(scratch, StringImpl::flagsOffset()),
10559 TrustedImm32(StringImpl::flagIs8Bit()));
10560
10561 m_jit.load16(MacroAssembler::Address(value), scratch);
10562
10563 JITCompiler::Jump ready = m_jit.jump();
10564
10565 is8Bit.link(&m_jit);
10566 m_jit.load8(MacroAssembler::Address(value), scratch);
10567
10568 ready.link(&m_jit);
10569 emitSwitchIntJump(data, scratch, value);
10570}
10571
10572void SpeculativeJIT::emitSwitchChar(Node* node, SwitchData* data)
10573{
10574 switch (node->child1().useKind()) {
10575 case StringUse: {
10576 SpeculateCellOperand op1(this, node->child1());
10577 GPRTemporary temp(this);
10578
10579 GPRReg op1GPR = op1.gpr();
10580 GPRReg tempGPR = temp.gpr();
10581
10582 op1.use();
10583
10584 speculateString(node->child1(), op1GPR);
10585 emitSwitchCharStringJump(data, op1GPR, tempGPR);
10586 noResult(node, UseChildrenCalledExplicitly);
10587 break;
10588 }
10589
10590 case UntypedUse: {
10591 JSValueOperand op1(this, node->child1());
10592 GPRTemporary temp(this);
10593
10594 JSValueRegs op1Regs = op1.jsValueRegs();
10595 GPRReg tempGPR = temp.gpr();
10596
10597 op1.use();
10598
10599 addBranch(m_jit.branchIfNotCell(op1Regs), data->fallThrough.block);
10600
10601 addBranch(m_jit.branchIfNotString(op1Regs.payloadGPR()), data->fallThrough.block);
10602
10603 emitSwitchCharStringJump(data, op1Regs.payloadGPR(), tempGPR);
10604 noResult(node, UseChildrenCalledExplicitly);
10605 break;
10606 }
10607
10608 default:
10609 RELEASE_ASSERT_NOT_REACHED();
10610 break;
10611 }
10612}
10613
10614namespace {
10615
10616struct CharacterCase {
10617 bool operator<(const CharacterCase& other) const
10618 {
10619 return character < other.character;
10620 }
10621
10622 LChar character;
10623 unsigned begin;
10624 unsigned end;
10625};
10626
10627} // anonymous namespace
10628
10629void SpeculativeJIT::emitBinarySwitchStringRecurse(
10630 SwitchData* data, const Vector<SpeculativeJIT::StringSwitchCase>& cases,
10631 unsigned numChecked, unsigned begin, unsigned end, GPRReg buffer, GPRReg length,
10632 GPRReg temp, unsigned alreadyCheckedLength, bool checkedExactLength)
10633{
10634 static const bool verbose = false;
10635
10636 if (verbose) {
10637 dataLog("We're down to the following cases, alreadyCheckedLength = ", alreadyCheckedLength, ":\n");
10638 for (unsigned i = begin; i < end; ++i) {
10639 dataLog(" ", cases[i].string, "\n");
10640 }
10641 }
10642
10643 if (begin == end) {
10644 jump(data->fallThrough.block, ForceJump);
10645 return;
10646 }
10647
10648 unsigned minLength = cases[begin].string->length();
10649 unsigned commonChars = minLength;
10650 bool allLengthsEqual = true;
10651 for (unsigned i = begin + 1; i < end; ++i) {
10652 unsigned myCommonChars = numChecked;
10653 for (unsigned j = numChecked;
10654 j < std::min(cases[begin].string->length(), cases[i].string->length());
10655 ++j) {
10656 if (cases[begin].string->at(j) != cases[i].string->at(j)) {
10657 if (verbose)
10658 dataLog("string(", cases[i].string, ")[", j, "] != string(", cases[begin].string, ")[", j, "]\n");
10659 break;
10660 }
10661 myCommonChars++;
10662 }
10663 commonChars = std::min(commonChars, myCommonChars);
10664 if (minLength != cases[i].string->length())
10665 allLengthsEqual = false;
10666 minLength = std::min(minLength, cases[i].string->length());
10667 }
10668
10669 if (checkedExactLength) {
10670 RELEASE_ASSERT(alreadyCheckedLength == minLength);
10671 RELEASE_ASSERT(allLengthsEqual);
10672 }
10673
10674 RELEASE_ASSERT(minLength >= commonChars);
10675
10676 if (verbose)
10677 dataLog("length = ", minLength, ", commonChars = ", commonChars, ", allLengthsEqual = ", allLengthsEqual, "\n");
10678
10679 if (!allLengthsEqual && alreadyCheckedLength < minLength)
10680 branch32(MacroAssembler::Below, length, Imm32(minLength), data->fallThrough.block);
10681 if (allLengthsEqual && (alreadyCheckedLength < minLength || !checkedExactLength))
10682 branch32(MacroAssembler::NotEqual, length, Imm32(minLength), data->fallThrough.block);
10683
10684 for (unsigned i = numChecked; i < commonChars; ++i) {
10685 branch8(
10686 MacroAssembler::NotEqual, MacroAssembler::Address(buffer, i),
10687 TrustedImm32(cases[begin].string->at(i)), data->fallThrough.block);
10688 }
10689
10690 if (minLength == commonChars) {
10691 // This is the case where one of the cases is a prefix of all of the other cases.
10692 // We've already checked that the input string is a prefix of all of the cases,
10693 // so we just check length to jump to that case.
10694
10695 if (!ASSERT_DISABLED) {
10696 ASSERT(cases[begin].string->length() == commonChars);
10697 for (unsigned i = begin + 1; i < end; ++i)
10698 ASSERT(cases[i].string->length() > commonChars);
10699 }
10700
10701 if (allLengthsEqual) {
10702 RELEASE_ASSERT(end == begin + 1);
10703 jump(cases[begin].target, ForceJump);
10704 return;
10705 }
10706
10707 branch32(MacroAssembler::Equal, length, Imm32(commonChars), cases[begin].target);
10708
10709 // We've checked if the length is >= minLength, and then we checked if the
10710 // length is == commonChars. We get to this point if it is >= minLength but not
10711 // == commonChars. Hence we know that it now must be > minLength, i.e., that
10712 // it's >= minLength + 1.
10713 emitBinarySwitchStringRecurse(
10714 data, cases, commonChars, begin + 1, end, buffer, length, temp, minLength + 1, false);
10715 return;
10716 }
10717
10718 // At this point we know that the string is longer than commonChars, and we've only
10719 // verified commonChars. Use a binary switch on the next unchecked character, i.e.
10720 // string[commonChars].
10721
10722 RELEASE_ASSERT(end >= begin + 2);
10723
10724 m_jit.load8(MacroAssembler::Address(buffer, commonChars), temp);
10725
10726 Vector<CharacterCase> characterCases;
10727 CharacterCase currentCase;
10728 currentCase.character = cases[begin].string->at(commonChars);
10729 currentCase.begin = begin;
10730 currentCase.end = begin + 1;
10731 for (unsigned i = begin + 1; i < end; ++i) {
10732 if (cases[i].string->at(commonChars) != currentCase.character) {
10733 if (verbose)
10734 dataLog("string(", cases[i].string, ")[", commonChars, "] != string(", cases[begin].string, ")[", commonChars, "]\n");
10735 currentCase.end = i;
10736 characterCases.append(currentCase);
10737 currentCase.character = cases[i].string->at(commonChars);
10738 currentCase.begin = i;
10739 currentCase.end = i + 1;
10740 } else
10741 currentCase.end = i + 1;
10742 }
10743 characterCases.append(currentCase);
10744
10745 Vector<int64_t> characterCaseValues;
10746 for (unsigned i = 0; i < characterCases.size(); ++i)
10747 characterCaseValues.append(characterCases[i].character);
10748
10749 BinarySwitch binarySwitch(temp, characterCaseValues, BinarySwitch::Int32);
10750 while (binarySwitch.advance(m_jit)) {
10751 const CharacterCase& myCase = characterCases[binarySwitch.caseIndex()];
10752 emitBinarySwitchStringRecurse(
10753 data, cases, commonChars + 1, myCase.begin, myCase.end, buffer, length,
10754 temp, minLength, allLengthsEqual);
10755 }
10756
10757 addBranch(binarySwitch.fallThrough(), data->fallThrough.block);
10758}
10759
10760void SpeculativeJIT::emitSwitchStringOnString(SwitchData* data, GPRReg string)
10761{
10762 data->didUseJumpTable = true;
10763
10764 bool canDoBinarySwitch = true;
10765 unsigned totalLength = 0;
10766
10767 for (unsigned i = data->cases.size(); i--;) {
10768 StringImpl* string = data->cases[i].value.stringImpl();
10769 if (!string->is8Bit()) {
10770 canDoBinarySwitch = false;
10771 break;
10772 }
10773 if (string->length() > Options::maximumBinaryStringSwitchCaseLength()) {
10774 canDoBinarySwitch = false;
10775 break;
10776 }
10777 totalLength += string->length();
10778 }
10779
10780 if (!canDoBinarySwitch || totalLength > Options::maximumBinaryStringSwitchTotalLength()) {
10781 flushRegisters();
10782 callOperation(
10783 operationSwitchString, string, static_cast<size_t>(data->switchTableIndex), string);
10784 m_jit.exceptionCheck();
10785 m_jit.jump(string, JSSwitchPtrTag);
10786 return;
10787 }
10788
10789 GPRTemporary length(this);
10790 GPRTemporary temp(this);
10791
10792 GPRReg lengthGPR = length.gpr();
10793 GPRReg tempGPR = temp.gpr();
10794
10795 MacroAssembler::JumpList slowCases;
10796 m_jit.loadPtr(MacroAssembler::Address(string, JSString::offsetOfValue()), tempGPR);
10797 slowCases.append(m_jit.branchIfRopeStringImpl(tempGPR));
10798 m_jit.load32(MacroAssembler::Address(tempGPR, StringImpl::lengthMemoryOffset()), lengthGPR);
10799
10800 slowCases.append(m_jit.branchTest32(
10801 MacroAssembler::Zero,
10802 MacroAssembler::Address(tempGPR, StringImpl::flagsOffset()),
10803 TrustedImm32(StringImpl::flagIs8Bit())));
10804
10805 m_jit.loadPtr(MacroAssembler::Address(tempGPR, StringImpl::dataOffset()), string);
10806
10807 Vector<StringSwitchCase> cases;
10808 for (unsigned i = 0; i < data->cases.size(); ++i) {
10809 cases.append(
10810 StringSwitchCase(data->cases[i].value.stringImpl(), data->cases[i].target.block));
10811 }
10812
10813 std::sort(cases.begin(), cases.end());
10814
10815 emitBinarySwitchStringRecurse(
10816 data, cases, 0, 0, cases.size(), string, lengthGPR, tempGPR, 0, false);
10817
10818 slowCases.link(&m_jit);
10819 silentSpillAllRegisters(string);
10820 callOperation(operationSwitchString, string, static_cast<size_t>(data->switchTableIndex), string);
10821 silentFillAllRegisters();
10822 m_jit.exceptionCheck();
10823 m_jit.jump(string, JSSwitchPtrTag);
10824}
10825
10826void SpeculativeJIT::emitSwitchString(Node* node, SwitchData* data)
10827{
10828 switch (node->child1().useKind()) {
10829 case StringIdentUse: {
10830 SpeculateCellOperand op1(this, node->child1());
10831 GPRTemporary temp(this);
10832
10833 GPRReg op1GPR = op1.gpr();
10834 GPRReg tempGPR = temp.gpr();
10835
10836 speculateString(node->child1(), op1GPR);
10837 speculateStringIdentAndLoadStorage(node->child1(), op1GPR, tempGPR);
10838
10839 Vector<int64_t> identifierCaseValues;
10840 for (unsigned i = 0; i < data->cases.size(); ++i) {
10841 identifierCaseValues.append(
10842 static_cast<int64_t>(bitwise_cast<intptr_t>(data->cases[i].value.stringImpl())));
10843 }
10844
10845 BinarySwitch binarySwitch(tempGPR, identifierCaseValues, BinarySwitch::IntPtr);
10846 while (binarySwitch.advance(m_jit))
10847 jump(data->cases[binarySwitch.caseIndex()].target.block, ForceJump);
10848 addBranch(binarySwitch.fallThrough(), data->fallThrough.block);
10849
10850 noResult(node);
10851 break;
10852 }
10853
10854 case StringUse: {
10855 SpeculateCellOperand op1(this, node->child1());
10856
10857 GPRReg op1GPR = op1.gpr();
10858
10859 op1.use();
10860
10861 speculateString(node->child1(), op1GPR);
10862 emitSwitchStringOnString(data, op1GPR);
10863 noResult(node, UseChildrenCalledExplicitly);
10864 break;
10865 }
10866
10867 case UntypedUse: {
10868 JSValueOperand op1(this, node->child1());
10869
10870 JSValueRegs op1Regs = op1.jsValueRegs();
10871
10872 op1.use();
10873
10874 addBranch(m_jit.branchIfNotCell(op1Regs), data->fallThrough.block);
10875
10876 addBranch(m_jit.branchIfNotString(op1Regs.payloadGPR()), data->fallThrough.block);
10877
10878 emitSwitchStringOnString(data, op1Regs.payloadGPR());
10879 noResult(node, UseChildrenCalledExplicitly);
10880 break;
10881 }
10882
10883 default:
10884 RELEASE_ASSERT_NOT_REACHED();
10885 break;
10886 }
10887}
10888
10889void SpeculativeJIT::emitSwitch(Node* node)
10890{
10891 SwitchData* data = node->switchData();
10892 switch (data->kind) {
10893 case SwitchImm: {
10894 emitSwitchImm(node, data);
10895 return;
10896 }
10897 case SwitchChar: {
10898 emitSwitchChar(node, data);
10899 return;
10900 }
10901 case SwitchString: {
10902 emitSwitchString(node, data);
10903 return;
10904 }
10905 case SwitchCell: {
10906 DFG_CRASH(m_jit.graph(), node, "Bad switch kind");
10907 return;
10908 } }
10909 RELEASE_ASSERT_NOT_REACHED();
10910}
10911
10912void SpeculativeJIT::addBranch(const MacroAssembler::JumpList& jump, BasicBlock* destination)
10913{
10914 for (unsigned i = jump.jumps().size(); i--;)
10915 addBranch(jump.jumps()[i], destination);
10916}
10917
10918void SpeculativeJIT::linkBranches()
10919{
10920 for (auto& branch : m_branches)
10921 branch.jump.linkTo(m_jit.blockHeads()[branch.destination->index], &m_jit);
10922}
10923
10924void SpeculativeJIT::compileStoreBarrier(Node* node)
10925{
10926 ASSERT(node->op() == StoreBarrier || node->op() == FencedStoreBarrier);
10927
10928 bool isFenced = node->op() == FencedStoreBarrier;
10929
10930 SpeculateCellOperand base(this, node->child1());
10931 GPRTemporary scratch1(this);
10932
10933 GPRReg baseGPR = base.gpr();
10934 GPRReg scratch1GPR = scratch1.gpr();
10935
10936 JITCompiler::JumpList ok;
10937
10938 if (isFenced) {
10939 ok.append(m_jit.barrierBranch(*m_jit.vm(), baseGPR, scratch1GPR));
10940
10941 JITCompiler::Jump noFence = m_jit.jumpIfMutatorFenceNotNeeded(*m_jit.vm());
10942 m_jit.memoryFence();
10943 ok.append(m_jit.barrierBranchWithoutFence(baseGPR));
10944 noFence.link(&m_jit);
10945 } else
10946 ok.append(m_jit.barrierBranchWithoutFence(baseGPR));
10947
10948 silentSpillAllRegisters(InvalidGPRReg);
10949 callOperation(operationWriteBarrierSlowPath, baseGPR);
10950 silentFillAllRegisters();
10951
10952 ok.link(&m_jit);
10953
10954 noResult(node);
10955}
10956
10957void SpeculativeJIT::compilePutAccessorById(Node* node)
10958{
10959 SpeculateCellOperand base(this, node->child1());
10960 SpeculateCellOperand accessor(this, node->child2());
10961
10962 GPRReg baseGPR = base.gpr();
10963 GPRReg accessorGPR = accessor.gpr();
10964
10965 flushRegisters();
10966 callOperation(node->op() == PutGetterById ? operationPutGetterById : operationPutSetterById, NoResult, baseGPR, identifierUID(node->identifierNumber()), node->accessorAttributes(), accessorGPR);
10967 m_jit.exceptionCheck();
10968
10969 noResult(node);
10970}
10971
10972void SpeculativeJIT::compilePutGetterSetterById(Node* node)
10973{
10974 SpeculateCellOperand base(this, node->child1());
10975 JSValueOperand getter(this, node->child2());
10976 JSValueOperand setter(this, node->child3());
10977
10978#if USE(JSVALUE64)
10979 GPRReg baseGPR = base.gpr();
10980 GPRReg getterGPR = getter.gpr();
10981 GPRReg setterGPR = setter.gpr();
10982
10983 flushRegisters();
10984 callOperation(operationPutGetterSetter, NoResult, baseGPR, identifierUID(node->identifierNumber()), node->accessorAttributes(), getterGPR, setterGPR);
10985#else
10986 // These JSValues may be JSUndefined OR JSFunction*.
10987 // At that time,
10988 // 1. If the JSValue is JSUndefined, its payload becomes nullptr.
10989 // 2. If the JSValue is JSFunction*, its payload becomes JSFunction*.
10990 // So extract payload and pass it to operationPutGetterSetter. This hack is used as the same way in baseline JIT.
10991 GPRReg baseGPR = base.gpr();
10992 JSValueRegs getterRegs = getter.jsValueRegs();
10993 JSValueRegs setterRegs = setter.jsValueRegs();
10994
10995 flushRegisters();
10996 callOperation(operationPutGetterSetter, NoResult, baseGPR, identifierUID(node->identifierNumber()), node->accessorAttributes(), getterRegs.payloadGPR(), setterRegs.payloadGPR());
10997#endif
10998 m_jit.exceptionCheck();
10999
11000 noResult(node);
11001}
11002
11003void SpeculativeJIT::compileResolveScope(Node* node)
11004{
11005 SpeculateCellOperand scope(this, node->child1());
11006 GPRReg scopeGPR = scope.gpr();
11007 GPRFlushedCallResult result(this);
11008 GPRReg resultGPR = result.gpr();
11009 flushRegisters();
11010 callOperation(operationResolveScope, resultGPR, scopeGPR, identifierUID(node->identifierNumber()));
11011 m_jit.exceptionCheck();
11012 cellResult(resultGPR, node);
11013}
11014
11015void SpeculativeJIT::compileResolveScopeForHoistingFuncDeclInEval(Node* node)
11016{
11017 SpeculateCellOperand scope(this, node->child1());
11018 GPRReg scopeGPR = scope.gpr();
11019 flushRegisters();
11020 JSValueRegsFlushedCallResult result(this);
11021 JSValueRegs resultRegs = result.regs();
11022 callOperation(operationResolveScopeForHoistingFuncDeclInEval, resultRegs, scopeGPR, identifierUID(node->identifierNumber()));
11023 m_jit.exceptionCheck();
11024 jsValueResult(resultRegs, node);
11025}
11026
11027void SpeculativeJIT::compileGetGlobalVariable(Node* node)
11028{
11029 JSValueRegsTemporary result(this);
11030 JSValueRegs resultRegs = result.regs();
11031 m_jit.loadValue(node->variablePointer(), resultRegs);
11032 jsValueResult(resultRegs, node);
11033}
11034
11035void SpeculativeJIT::compilePutGlobalVariable(Node* node)
11036{
11037 JSValueOperand value(this, node->child2());
11038 JSValueRegs valueRegs = value.jsValueRegs();
11039 m_jit.storeValue(valueRegs, node->variablePointer());
11040 noResult(node);
11041}
11042
11043void SpeculativeJIT::compileGetDynamicVar(Node* node)
11044{
11045 SpeculateCellOperand scope(this, node->child1());
11046 GPRReg scopeGPR = scope.gpr();
11047 flushRegisters();
11048 JSValueRegsFlushedCallResult result(this);
11049 JSValueRegs resultRegs = result.regs();
11050 callOperation(operationGetDynamicVar, resultRegs, scopeGPR, identifierUID(node->identifierNumber()), node->getPutInfo());
11051 m_jit.exceptionCheck();
11052 jsValueResult(resultRegs, node);
11053}
11054
11055void SpeculativeJIT::compilePutDynamicVar(Node* node)
11056{
11057 SpeculateCellOperand scope(this, node->child1());
11058 JSValueOperand value(this, node->child2());
11059
11060 GPRReg scopeGPR = scope.gpr();
11061 JSValueRegs valueRegs = value.jsValueRegs();
11062
11063 flushRegisters();
11064 callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutDynamicVarStrict : operationPutDynamicVarNonStrict, NoResult, scopeGPR, valueRegs, identifierUID(node->identifierNumber()), node->getPutInfo());
11065 m_jit.exceptionCheck();
11066 noResult(node);
11067}
11068
11069void SpeculativeJIT::compileGetClosureVar(Node* node)
11070{
11071 SpeculateCellOperand base(this, node->child1());
11072 JSValueRegsTemporary result(this);
11073
11074 GPRReg baseGPR = base.gpr();
11075 JSValueRegs resultRegs = result.regs();
11076
11077 m_jit.loadValue(JITCompiler::Address(baseGPR, JSLexicalEnvironment::offsetOfVariable(node->scopeOffset())), resultRegs);
11078 jsValueResult(resultRegs, node);
11079}
11080
11081void SpeculativeJIT::compilePutClosureVar(Node* node)
11082{
11083 SpeculateCellOperand base(this, node->child1());
11084 JSValueOperand value(this, node->child2());
11085
11086 GPRReg baseGPR = base.gpr();
11087 JSValueRegs valueRegs = value.jsValueRegs();
11088
11089 m_jit.storeValue(valueRegs, JITCompiler::Address(baseGPR, JSLexicalEnvironment::offsetOfVariable(node->scopeOffset())));
11090 noResult(node);
11091}
11092
11093void SpeculativeJIT::compilePutAccessorByVal(Node* node)
11094{
11095 SpeculateCellOperand base(this, node->child1());
11096 JSValueOperand subscript(this, node->child2());
11097 SpeculateCellOperand accessor(this, node->child3());
11098
11099 auto operation = node->op() == PutGetterByVal ? operationPutGetterByVal : operationPutSetterByVal;
11100
11101 GPRReg baseGPR = base.gpr();
11102 JSValueRegs subscriptRegs = subscript.jsValueRegs();
11103 GPRReg accessorGPR = accessor.gpr();
11104
11105 flushRegisters();
11106 callOperation(operation, NoResult, baseGPR, subscriptRegs, node->accessorAttributes(), accessorGPR);
11107 m_jit.exceptionCheck();
11108
11109 noResult(node);
11110}
11111
11112void SpeculativeJIT::compileGetRegExpObjectLastIndex(Node* node)
11113{
11114 SpeculateCellOperand regExp(this, node->child1());
11115 JSValueRegsTemporary result(this);
11116 GPRReg regExpGPR = regExp.gpr();
11117 JSValueRegs resultRegs = result.regs();
11118 speculateRegExpObject(node->child1(), regExpGPR);
11119 m_jit.loadValue(JITCompiler::Address(regExpGPR, RegExpObject::offsetOfLastIndex()), resultRegs);
11120 jsValueResult(resultRegs, node);
11121}
11122
11123void SpeculativeJIT::compileSetRegExpObjectLastIndex(Node* node)
11124{
11125 SpeculateCellOperand regExp(this, node->child1());
11126 JSValueOperand value(this, node->child2());
11127 GPRReg regExpGPR = regExp.gpr();
11128 JSValueRegs valueRegs = value.jsValueRegs();
11129
11130 if (!node->ignoreLastIndexIsWritable()) {
11131 speculateRegExpObject(node->child1(), regExpGPR);
11132 speculationCheck(
11133 ExoticObjectMode, JSValueRegs(), nullptr,
11134 m_jit.branchTestPtr(
11135 JITCompiler::NonZero,
11136 JITCompiler::Address(regExpGPR, RegExpObject::offsetOfRegExpAndLastIndexIsNotWritableFlag()),
11137 JITCompiler::TrustedImm32(RegExpObject::lastIndexIsNotWritableFlag)));
11138 }
11139
11140 m_jit.storeValue(valueRegs, JITCompiler::Address(regExpGPR, RegExpObject::offsetOfLastIndex()));
11141 noResult(node);
11142}
11143
11144void SpeculativeJIT::compileRegExpExec(Node* node)
11145{
11146 bool sample = false;
11147 if (sample)
11148 m_jit.incrementSuperSamplerCount();
11149
11150 SpeculateCellOperand globalObject(this, node->child1());
11151 GPRReg globalObjectGPR = globalObject.gpr();
11152
11153 if (node->child2().useKind() == RegExpObjectUse) {
11154 if (node->child3().useKind() == StringUse) {
11155 SpeculateCellOperand base(this, node->child2());
11156 SpeculateCellOperand argument(this, node->child3());
11157 GPRReg baseGPR = base.gpr();
11158 GPRReg argumentGPR = argument.gpr();
11159 speculateRegExpObject(node->child2(), baseGPR);
11160 speculateString(node->child3(), argumentGPR);
11161
11162 flushRegisters();
11163 JSValueRegsFlushedCallResult result(this);
11164 JSValueRegs resultRegs = result.regs();
11165 callOperation(operationRegExpExecString, resultRegs, globalObjectGPR, baseGPR, argumentGPR);
11166 m_jit.exceptionCheck();
11167
11168 jsValueResult(resultRegs, node);
11169
11170 if (sample)
11171 m_jit.decrementSuperSamplerCount();
11172 return;
11173 }
11174
11175 SpeculateCellOperand base(this, node->child2());
11176 JSValueOperand argument(this, node->child3());
11177 GPRReg baseGPR = base.gpr();
11178 JSValueRegs argumentRegs = argument.jsValueRegs();
11179 speculateRegExpObject(node->child2(), baseGPR);
11180
11181 flushRegisters();
11182 JSValueRegsFlushedCallResult result(this);
11183 JSValueRegs resultRegs = result.regs();
11184 callOperation(operationRegExpExec, resultRegs, globalObjectGPR, baseGPR, argumentRegs);
11185 m_jit.exceptionCheck();
11186
11187 jsValueResult(resultRegs, node);
11188
11189 if (sample)
11190 m_jit.decrementSuperSamplerCount();
11191 return;
11192 }
11193
11194 JSValueOperand base(this, node->child2());
11195 JSValueOperand argument(this, node->child3());
11196 JSValueRegs baseRegs = base.jsValueRegs();
11197 JSValueRegs argumentRegs = argument.jsValueRegs();
11198
11199 flushRegisters();
11200 JSValueRegsFlushedCallResult result(this);
11201 JSValueRegs resultRegs = result.regs();
11202 callOperation(operationRegExpExecGeneric, resultRegs, globalObjectGPR, baseRegs, argumentRegs);
11203 m_jit.exceptionCheck();
11204
11205 jsValueResult(resultRegs, node);
11206
11207 if (sample)
11208 m_jit.decrementSuperSamplerCount();
11209}
11210
11211void SpeculativeJIT::compileRegExpTest(Node* node)
11212{
11213 SpeculateCellOperand globalObject(this, node->child1());
11214 GPRReg globalObjectGPR = globalObject.gpr();
11215
11216 if (node->child2().useKind() == RegExpObjectUse) {
11217 if (node->child3().useKind() == StringUse) {
11218 SpeculateCellOperand base(this, node->child2());
11219 SpeculateCellOperand argument(this, node->child3());
11220 GPRReg baseGPR = base.gpr();
11221 GPRReg argumentGPR = argument.gpr();
11222 speculateRegExpObject(node->child2(), baseGPR);
11223 speculateString(node->child3(), argumentGPR);
11224
11225 flushRegisters();
11226 GPRFlushedCallResult result(this);
11227 callOperation(operationRegExpTestString, result.gpr(), globalObjectGPR, baseGPR, argumentGPR);
11228 m_jit.exceptionCheck();
11229
11230 unblessedBooleanResult(result.gpr(), node);
11231 return;
11232 }
11233
11234 SpeculateCellOperand base(this, node->child2());
11235 JSValueOperand argument(this, node->child3());
11236 GPRReg baseGPR = base.gpr();
11237 JSValueRegs argumentRegs = argument.jsValueRegs();
11238 speculateRegExpObject(node->child2(), baseGPR);
11239
11240 flushRegisters();
11241 GPRFlushedCallResult result(this);
11242 callOperation(operationRegExpTest, result.gpr(), globalObjectGPR, baseGPR, argumentRegs);
11243 m_jit.exceptionCheck();
11244
11245 unblessedBooleanResult(result.gpr(), node);
11246 return;
11247 }
11248
11249 JSValueOperand base(this, node->child2());
11250 JSValueOperand argument(this, node->child3());
11251 JSValueRegs baseRegs = base.jsValueRegs();
11252 JSValueRegs argumentRegs = argument.jsValueRegs();
11253
11254 flushRegisters();
11255 GPRFlushedCallResult result(this);
11256 callOperation(operationRegExpTestGeneric, result.gpr(), globalObjectGPR, baseRegs, argumentRegs);
11257 m_jit.exceptionCheck();
11258
11259 unblessedBooleanResult(result.gpr(), node);
11260}
11261
11262void SpeculativeJIT::compileStringReplace(Node* node)
11263{
11264 ASSERT(node->op() == StringReplace || node->op() == StringReplaceRegExp);
11265 bool sample = false;
11266 if (sample)
11267 m_jit.incrementSuperSamplerCount();
11268
11269 if (node->child1().useKind() == StringUse
11270 && node->child2().useKind() == RegExpObjectUse
11271 && node->child3().useKind() == StringUse) {
11272 if (JSString* replace = node->child3()->dynamicCastConstant<JSString*>(*m_jit.vm())) {
11273 if (!replace->length()) {
11274 SpeculateCellOperand string(this, node->child1());
11275 SpeculateCellOperand regExp(this, node->child2());
11276 GPRReg stringGPR = string.gpr();
11277 GPRReg regExpGPR = regExp.gpr();
11278 speculateString(node->child1(), stringGPR);
11279 speculateRegExpObject(node->child2(), regExpGPR);
11280
11281 flushRegisters();
11282 GPRFlushedCallResult result(this);
11283 callOperation(operationStringProtoFuncReplaceRegExpEmptyStr, result.gpr(), stringGPR, regExpGPR);
11284 m_jit.exceptionCheck();
11285 cellResult(result.gpr(), node);
11286 if (sample)
11287 m_jit.decrementSuperSamplerCount();
11288 return;
11289 }
11290 }
11291
11292 SpeculateCellOperand string(this, node->child1());
11293 SpeculateCellOperand regExp(this, node->child2());
11294 SpeculateCellOperand replace(this, node->child3());
11295 GPRReg stringGPR = string.gpr();
11296 GPRReg regExpGPR = regExp.gpr();
11297 GPRReg replaceGPR = replace.gpr();
11298 speculateString(node->child1(), stringGPR);
11299 speculateRegExpObject(node->child2(), regExpGPR);
11300 speculateString(node->child3(), replaceGPR);
11301
11302 flushRegisters();
11303 GPRFlushedCallResult result(this);
11304 callOperation(operationStringProtoFuncReplaceRegExpString, result.gpr(), stringGPR, regExpGPR, replaceGPR);
11305 m_jit.exceptionCheck();
11306 cellResult(result.gpr(), node);
11307 if (sample)
11308 m_jit.decrementSuperSamplerCount();
11309 return;
11310 }
11311
11312 // If we fixed up the edge of child2, we inserted a Check(@child2, String).
11313 OperandSpeculationMode child2SpeculationMode = AutomaticOperandSpeculation;
11314 if (node->child2().useKind() == StringUse)
11315 child2SpeculationMode = ManualOperandSpeculation;
11316
11317 JSValueOperand string(this, node->child1());
11318 JSValueOperand search(this, node->child2(), child2SpeculationMode);
11319 JSValueOperand replace(this, node->child3());
11320 JSValueRegs stringRegs = string.jsValueRegs();
11321 JSValueRegs searchRegs = search.jsValueRegs();
11322 JSValueRegs replaceRegs = replace.jsValueRegs();
11323
11324 flushRegisters();
11325 GPRFlushedCallResult result(this);
11326 callOperation(operationStringProtoFuncReplaceGeneric, result.gpr(), stringRegs, searchRegs, replaceRegs);
11327 m_jit.exceptionCheck();
11328 cellResult(result.gpr(), node);
11329 if (sample)
11330 m_jit.decrementSuperSamplerCount();
11331}
11332
11333void SpeculativeJIT::compileRegExpExecNonGlobalOrSticky(Node* node)
11334{
11335 SpeculateCellOperand globalObject(this, node->child1());
11336 SpeculateCellOperand argument(this, node->child2());
11337 GPRReg globalObjectGPR = globalObject.gpr();
11338 GPRReg argumentGPR = argument.gpr();
11339
11340 speculateString(node->child2(), argumentGPR);
11341
11342 flushRegisters();
11343 JSValueRegsFlushedCallResult result(this);
11344 JSValueRegs resultRegs = result.regs();
11345 callOperation(
11346 operationRegExpExecNonGlobalOrSticky, resultRegs,
11347 globalObjectGPR, TrustedImmPtr(node->cellOperand()), argumentGPR);
11348 m_jit.exceptionCheck();
11349
11350 jsValueResult(resultRegs, node);
11351}
11352
11353void SpeculativeJIT::compileRegExpMatchFastGlobal(Node* node)
11354{
11355 SpeculateCellOperand globalObject(this, node->child1());
11356 SpeculateCellOperand argument(this, node->child2());
11357 GPRReg globalObjectGPR = globalObject.gpr();
11358 GPRReg argumentGPR = argument.gpr();
11359
11360 speculateString(node->child2(), argumentGPR);
11361
11362 flushRegisters();
11363 JSValueRegsFlushedCallResult result(this);
11364 JSValueRegs resultRegs = result.regs();
11365 callOperation(
11366 operationRegExpMatchFastGlobalString, resultRegs,
11367 globalObjectGPR, TrustedImmPtr(node->cellOperand()), argumentGPR);
11368 m_jit.exceptionCheck();
11369
11370 jsValueResult(resultRegs, node);
11371}
11372
11373void SpeculativeJIT::compileRegExpMatchFast(Node* node)
11374{
11375 SpeculateCellOperand globalObject(this, node->child1());
11376 SpeculateCellOperand base(this, node->child2());
11377 SpeculateCellOperand argument(this, node->child3());
11378 GPRReg globalObjectGPR = globalObject.gpr();
11379 GPRReg baseGPR = base.gpr();
11380 GPRReg argumentGPR = argument.gpr();
11381 speculateRegExpObject(node->child2(), baseGPR);
11382 speculateString(node->child3(), argumentGPR);
11383
11384 flushRegisters();
11385 JSValueRegsFlushedCallResult result(this);
11386 JSValueRegs resultRegs = result.regs();
11387 callOperation(
11388 operationRegExpMatchFastString, resultRegs,
11389 globalObjectGPR, baseGPR, argumentGPR);
11390 m_jit.exceptionCheck();
11391
11392 jsValueResult(resultRegs, node);
11393}
11394
11395void SpeculativeJIT::compileLazyJSConstant(Node* node)
11396{
11397 JSValueRegsTemporary result(this);
11398 JSValueRegs resultRegs = result.regs();
11399 node->lazyJSValue().emit(m_jit, resultRegs);
11400 jsValueResult(resultRegs, node);
11401}
11402
11403void SpeculativeJIT::compileMaterializeNewObject(Node* node)
11404{
11405 RegisteredStructure structure = node->structureSet().at(0);
11406 ASSERT(m_jit.graph().varArgChild(node, 0)->dynamicCastConstant<Structure*>(*m_jit.vm()) == structure.get());
11407
11408 ObjectMaterializationData& data = node->objectMaterializationData();
11409
11410 IndexingType indexingType = structure->indexingType();
11411 bool hasIndexingHeader = hasIndexedProperties(indexingType);
11412 int32_t publicLength = 0;
11413 int32_t vectorLength = 0;
11414
11415 if (hasIndexingHeader) {
11416 for (unsigned i = data.m_properties.size(); i--;) {
11417 Edge edge = m_jit.graph().varArgChild(node, 1 + i);
11418 switch (data.m_properties[i].kind()) {
11419 case PublicLengthPLoc:
11420 publicLength = edge->asInt32();
11421 break;
11422 case VectorLengthPLoc:
11423 vectorLength = edge->asInt32();
11424 break;
11425 default:
11426 break;
11427 }
11428 }
11429 }
11430
11431 GPRTemporary result(this);
11432 GPRTemporary storage(this);
11433 GPRReg resultGPR = result.gpr();
11434 GPRReg storageGPR = storage.gpr();
11435
11436 emitAllocateRawObject(resultGPR, structure, storageGPR, 0, vectorLength);
11437
11438 m_jit.store32(
11439 JITCompiler::TrustedImm32(publicLength),
11440 JITCompiler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
11441
11442 for (unsigned i = data.m_properties.size(); i--;) {
11443 Edge edge = m_jit.graph().varArgChild(node, 1 + i);
11444 PromotedLocationDescriptor descriptor = data.m_properties[i];
11445 switch (descriptor.kind()) {
11446 case IndexedPropertyPLoc: {
11447 JSValueOperand value(this, edge);
11448 m_jit.storeValue(
11449 value.jsValueRegs(),
11450 JITCompiler::Address(storageGPR, sizeof(EncodedJSValue) * descriptor.info()));
11451 break;
11452 }
11453
11454 case NamedPropertyPLoc: {
11455 StringImpl* uid = m_jit.graph().identifiers()[descriptor.info()];
11456 for (PropertyMapEntry entry : structure->getPropertiesConcurrently()) {
11457 if (uid != entry.key)
11458 continue;
11459
11460 JSValueOperand value(this, edge);
11461 GPRReg baseGPR = isInlineOffset(entry.offset) ? resultGPR : storageGPR;
11462 m_jit.storeValue(
11463 value.jsValueRegs(),
11464 JITCompiler::Address(baseGPR, offsetRelativeToBase(entry.offset)));
11465 }
11466 break;
11467 }
11468
11469 default:
11470 break;
11471 }
11472 }
11473
11474 cellResult(resultGPR, node);
11475}
11476
11477void SpeculativeJIT::compileRecordRegExpCachedResult(Node* node)
11478{
11479 Edge globalObjectEdge = m_jit.graph().varArgChild(node, 0);
11480 Edge regExpEdge = m_jit.graph().varArgChild(node, 1);
11481 Edge stringEdge = m_jit.graph().varArgChild(node, 2);
11482 Edge startEdge = m_jit.graph().varArgChild(node, 3);
11483 Edge endEdge = m_jit.graph().varArgChild(node, 4);
11484
11485 SpeculateCellOperand globalObject(this, globalObjectEdge);
11486 SpeculateCellOperand regExp(this, regExpEdge);
11487 SpeculateCellOperand string(this, stringEdge);
11488 SpeculateInt32Operand start(this, startEdge);
11489 SpeculateInt32Operand end(this, endEdge);
11490
11491 GPRReg globalObjectGPR = globalObject.gpr();
11492 GPRReg regExpGPR = regExp.gpr();
11493 GPRReg stringGPR = string.gpr();
11494 GPRReg startGPR = start.gpr();
11495 GPRReg endGPR = end.gpr();
11496
11497 ptrdiff_t offset = JSGlobalObject::regExpGlobalDataOffset() + RegExpGlobalData::offsetOfCachedResult();
11498
11499 m_jit.storePtr(
11500 regExpGPR,
11501 JITCompiler::Address(globalObjectGPR, offset + RegExpCachedResult::offsetOfLastRegExp()));
11502 m_jit.storePtr(
11503 stringGPR,
11504 JITCompiler::Address(globalObjectGPR, offset + RegExpCachedResult::offsetOfLastInput()));
11505 m_jit.store32(
11506 startGPR,
11507 JITCompiler::Address(
11508 globalObjectGPR,
11509 offset + RegExpCachedResult::offsetOfResult() + OBJECT_OFFSETOF(MatchResult, start)));
11510 m_jit.store32(
11511 endGPR,
11512 JITCompiler::Address(
11513 globalObjectGPR,
11514 offset + RegExpCachedResult::offsetOfResult() + OBJECT_OFFSETOF(MatchResult, end)));
11515 m_jit.store8(
11516 TrustedImm32(0),
11517 JITCompiler::Address(globalObjectGPR, offset + RegExpCachedResult::offsetOfReified()));
11518
11519 noResult(node);
11520}
11521
11522void SpeculativeJIT::compileDefineDataProperty(Node* node)
11523{
11524#if USE(JSVALUE64)
11525 static_assert(GPRInfo::numberOfRegisters >= 5, "We are assuming we have enough registers to make this call without incrementally setting up the arguments.");
11526#else
11527 static_assert(GPRInfo::numberOfRegisters >= 6, "We are assuming we have enough registers to make this call without incrementally setting up the arguments.");
11528#endif
11529
11530 SpeculateCellOperand base(this, m_jit.graph().varArgChild(node, 0));
11531 GPRReg baseGPR = base.gpr();
11532
11533 JSValueOperand value(this, m_jit.graph().varArgChild(node, 2));
11534 JSValueRegs valueRegs = value.jsValueRegs();
11535
11536 SpeculateInt32Operand attributes(this, m_jit.graph().varArgChild(node, 3));
11537 GPRReg attributesGPR = attributes.gpr();
11538
11539 Edge& propertyEdge = m_jit.graph().varArgChild(node, 1);
11540 switch (propertyEdge.useKind()) {
11541 case StringUse: {
11542 SpeculateCellOperand property(this, propertyEdge);
11543 GPRReg propertyGPR = property.gpr();
11544 speculateString(propertyEdge, propertyGPR);
11545
11546 useChildren(node);
11547
11548 flushRegisters();
11549 callOperation(operationDefineDataPropertyString, NoResult, baseGPR, propertyGPR, valueRegs, attributesGPR);
11550 m_jit.exceptionCheck();
11551 break;
11552 }
11553 case StringIdentUse: {
11554 SpeculateCellOperand property(this, propertyEdge);
11555 GPRTemporary ident(this);
11556
11557 GPRReg propertyGPR = property.gpr();
11558 GPRReg identGPR = ident.gpr();
11559
11560 speculateString(propertyEdge, propertyGPR);
11561 speculateStringIdentAndLoadStorage(propertyEdge, propertyGPR, identGPR);
11562
11563 useChildren(node);
11564
11565 flushRegisters();
11566 callOperation(operationDefineDataPropertyStringIdent, NoResult, baseGPR, identGPR, valueRegs, attributesGPR);
11567 m_jit.exceptionCheck();
11568 break;
11569 }
11570 case SymbolUse: {
11571 SpeculateCellOperand property(this, propertyEdge);
11572 GPRReg propertyGPR = property.gpr();
11573 speculateSymbol(propertyEdge, propertyGPR);
11574
11575 useChildren(node);
11576
11577 flushRegisters();
11578 callOperation(operationDefineDataPropertySymbol, NoResult, baseGPR, propertyGPR, valueRegs, attributesGPR);
11579 m_jit.exceptionCheck();
11580 break;
11581 }
11582 case UntypedUse: {
11583 JSValueOperand property(this, propertyEdge);
11584 JSValueRegs propertyRegs = property.jsValueRegs();
11585
11586 useChildren(node);
11587
11588 flushRegisters();
11589 callOperation(operationDefineDataProperty, NoResult, baseGPR, propertyRegs, valueRegs, attributesGPR);
11590 m_jit.exceptionCheck();
11591 break;
11592 }
11593 default:
11594 RELEASE_ASSERT_NOT_REACHED();
11595 }
11596
11597 noResult(node, UseChildrenCalledExplicitly);
11598}
11599
11600void SpeculativeJIT::compileDefineAccessorProperty(Node* node)
11601{
11602#if USE(JSVALUE64)
11603 static_assert(GPRInfo::numberOfRegisters >= 5, "We are assuming we have enough registers to make this call without incrementally setting up the arguments.");
11604#else
11605 static_assert(GPRInfo::numberOfRegisters >= 6, "We are assuming we have enough registers to make this call without incrementally setting up the arguments.");
11606#endif
11607
11608 SpeculateCellOperand base(this, m_jit.graph().varArgChild(node, 0));
11609 GPRReg baseGPR = base.gpr();
11610
11611 SpeculateCellOperand getter(this, m_jit.graph().varArgChild(node, 2));
11612 GPRReg getterGPR = getter.gpr();
11613
11614 SpeculateCellOperand setter(this, m_jit.graph().varArgChild(node, 3));
11615 GPRReg setterGPR = setter.gpr();
11616
11617 SpeculateInt32Operand attributes(this, m_jit.graph().varArgChild(node, 4));
11618 GPRReg attributesGPR = attributes.gpr();
11619
11620 Edge& propertyEdge = m_jit.graph().varArgChild(node, 1);
11621 switch (propertyEdge.useKind()) {
11622 case StringUse: {
11623 SpeculateCellOperand property(this, propertyEdge);
11624 GPRReg propertyGPR = property.gpr();
11625 speculateString(propertyEdge, propertyGPR);
11626
11627 useChildren(node);
11628
11629 flushRegisters();
11630 callOperation(operationDefineAccessorPropertyString, NoResult, baseGPR, propertyGPR, getterGPR, setterGPR, attributesGPR);
11631 m_jit.exceptionCheck();
11632 break;
11633 }
11634 case StringIdentUse: {
11635 SpeculateCellOperand property(this, propertyEdge);
11636 GPRTemporary ident(this);
11637
11638 GPRReg propertyGPR = property.gpr();
11639 GPRReg identGPR = ident.gpr();
11640
11641 speculateString(propertyEdge, propertyGPR);
11642 speculateStringIdentAndLoadStorage(propertyEdge, propertyGPR, identGPR);
11643
11644 useChildren(node);
11645
11646 flushRegisters();
11647 callOperation(operationDefineAccessorPropertyStringIdent, NoResult, baseGPR, identGPR, getterGPR, setterGPR, attributesGPR);
11648 m_jit.exceptionCheck();
11649 break;
11650 }
11651 case SymbolUse: {
11652 SpeculateCellOperand property(this, propertyEdge);
11653 GPRReg propertyGPR = property.gpr();
11654 speculateSymbol(propertyEdge, propertyGPR);
11655
11656 useChildren(node);
11657
11658 flushRegisters();
11659 callOperation(operationDefineAccessorPropertySymbol, NoResult, baseGPR, propertyGPR, getterGPR, setterGPR, attributesGPR);
11660 m_jit.exceptionCheck();
11661 break;
11662 }
11663 case UntypedUse: {
11664 JSValueOperand property(this, propertyEdge);
11665 JSValueRegs propertyRegs = property.jsValueRegs();
11666
11667 useChildren(node);
11668
11669 flushRegisters();
11670 callOperation(operationDefineAccessorProperty, NoResult, baseGPR, propertyRegs, getterGPR, setterGPR, attributesGPR);
11671 m_jit.exceptionCheck();
11672 break;
11673 }
11674 default:
11675 RELEASE_ASSERT_NOT_REACHED();
11676 }
11677
11678 noResult(node, UseChildrenCalledExplicitly);
11679}
11680
11681void SpeculativeJIT::emitAllocateButterfly(GPRReg storageResultGPR, GPRReg sizeGPR, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, MacroAssembler::JumpList& slowCases)
11682{
11683 RELEASE_ASSERT(RegisterSet(storageResultGPR, sizeGPR, scratch1, scratch2, scratch3).numberOfSetGPRs() == 5);
11684 ASSERT((1 << 3) == sizeof(JSValue));
11685 m_jit.zeroExtend32ToPtr(sizeGPR, scratch1);
11686 m_jit.lshift32(TrustedImm32(3), scratch1);
11687 m_jit.add32(TrustedImm32(sizeof(IndexingHeader)), scratch1, scratch2);
11688#if !ASSERT_DISABLED
11689 MacroAssembler::Jump didNotOverflow = m_jit.branch32(MacroAssembler::AboveOrEqual, scratch2, sizeGPR);
11690 m_jit.abortWithReason(UncheckedOverflow);
11691 didNotOverflow.link(&m_jit);
11692#endif
11693 m_jit.emitAllocateVariableSized(
11694 storageResultGPR, m_jit.vm()->jsValueGigacageAuxiliarySpace, scratch2, scratch1, scratch3, slowCases);
11695 m_jit.addPtr(TrustedImm32(sizeof(IndexingHeader)), storageResultGPR);
11696
11697 m_jit.store32(sizeGPR, MacroAssembler::Address(storageResultGPR, Butterfly::offsetOfPublicLength()));
11698 m_jit.store32(sizeGPR, MacroAssembler::Address(storageResultGPR, Butterfly::offsetOfVectorLength()));
11699}
11700
11701void SpeculativeJIT::compileNormalizeMapKey(Node* node)
11702{
11703 ASSERT(node->child1().useKind() == UntypedUse);
11704 JSValueOperand key(this, node->child1());
11705 JSValueRegsTemporary result(this, Reuse, key);
11706 GPRTemporary scratch(this);
11707 FPRTemporary doubleValue(this);
11708 FPRTemporary temp(this);
11709
11710 JSValueRegs keyRegs = key.jsValueRegs();
11711 JSValueRegs resultRegs = result.regs();
11712 GPRReg scratchGPR = scratch.gpr();
11713 FPRReg doubleValueFPR = doubleValue.fpr();
11714 FPRReg tempFPR = temp.fpr();
11715
11716 CCallHelpers::JumpList passThroughCases;
11717
11718 passThroughCases.append(m_jit.branchIfNotNumber(keyRegs, scratchGPR));
11719 passThroughCases.append(m_jit.branchIfInt32(keyRegs));
11720
11721#if USE(JSVALUE64)
11722 m_jit.unboxDoubleWithoutAssertions(keyRegs.gpr(), scratchGPR, doubleValueFPR);
11723#else
11724 unboxDouble(keyRegs.tagGPR(), keyRegs.payloadGPR(), doubleValueFPR, tempFPR);
11725#endif
11726 passThroughCases.append(m_jit.branchIfNaN(doubleValueFPR));
11727
11728 m_jit.truncateDoubleToInt32(doubleValueFPR, scratchGPR);
11729 m_jit.convertInt32ToDouble(scratchGPR, tempFPR);
11730 passThroughCases.append(m_jit.branchDouble(JITCompiler::DoubleNotEqual, doubleValueFPR, tempFPR));
11731
11732 m_jit.boxInt32(scratchGPR, resultRegs);
11733 auto done = m_jit.jump();
11734
11735 passThroughCases.link(&m_jit);
11736 m_jit.moveValueRegs(keyRegs, resultRegs);
11737
11738 done.link(&m_jit);
11739 jsValueResult(resultRegs, node);
11740}
11741
11742void SpeculativeJIT::compileGetMapBucketHead(Node* node)
11743{
11744 SpeculateCellOperand map(this, node->child1());
11745 GPRTemporary bucket(this);
11746
11747 GPRReg mapGPR = map.gpr();
11748 GPRReg bucketGPR = bucket.gpr();
11749
11750 if (node->child1().useKind() == MapObjectUse)
11751 speculateMapObject(node->child1(), mapGPR);
11752 else if (node->child1().useKind() == SetObjectUse)
11753 speculateSetObject(node->child1(), mapGPR);
11754 else
11755 RELEASE_ASSERT_NOT_REACHED();
11756
11757 ASSERT(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfHead() == HashMapImpl<HashMapBucket<HashMapBucketDataKeyValue>>::offsetOfHead());
11758 m_jit.loadPtr(MacroAssembler::Address(mapGPR, HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfHead()), bucketGPR);
11759 cellResult(bucketGPR, node);
11760}
11761
11762void SpeculativeJIT::compileGetMapBucketNext(Node* node)
11763{
11764 SpeculateCellOperand bucket(this, node->child1());
11765 GPRTemporary result(this);
11766
11767 GPRReg bucketGPR = bucket.gpr();
11768 GPRReg resultGPR = result.gpr();
11769
11770 ASSERT(HashMapBucket<HashMapBucketDataKey>::offsetOfNext() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext());
11771 ASSERT(HashMapBucket<HashMapBucketDataKey>::offsetOfKey() == HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey());
11772 m_jit.loadPtr(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext()), resultGPR);
11773
11774 MacroAssembler::Label loop = m_jit.label();
11775 auto notBucket = m_jit.branchTestPtr(MacroAssembler::Zero, resultGPR);
11776#if USE(JSVALUE32_64)
11777 auto done = m_jit.branch32(MacroAssembler::NotEqual, MacroAssembler::Address(resultGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey() + TagOffset), TrustedImm32(JSValue::EmptyValueTag));
11778#else
11779 auto done = m_jit.branchTest64(MacroAssembler::NonZero, MacroAssembler::Address(resultGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey()));
11780#endif
11781 m_jit.loadPtr(MacroAssembler::Address(resultGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfNext()), resultGPR);
11782 m_jit.jump().linkTo(loop, &m_jit);
11783
11784 notBucket.link(&m_jit);
11785 JSCell* sentinel = nullptr;
11786 if (node->bucketOwnerType() == BucketOwnerType::Map)
11787 sentinel = m_jit.vm()->sentinelMapBucket();
11788 else {
11789 ASSERT(node->bucketOwnerType() == BucketOwnerType::Set);
11790 sentinel = m_jit.vm()->sentinelSetBucket();
11791 }
11792 m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), sentinel), resultGPR);
11793 done.link(&m_jit);
11794
11795 cellResult(resultGPR, node);
11796}
11797
11798void SpeculativeJIT::compileLoadKeyFromMapBucket(Node* node)
11799{
11800 SpeculateCellOperand bucket(this, node->child1());
11801 JSValueRegsTemporary result(this);
11802
11803 GPRReg bucketGPR = bucket.gpr();
11804 JSValueRegs resultRegs = result.regs();
11805
11806 m_jit.loadValue(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfKey()), resultRegs);
11807 jsValueResult(resultRegs, node);
11808}
11809
11810void SpeculativeJIT::compileLoadValueFromMapBucket(Node* node)
11811{
11812 SpeculateCellOperand bucket(this, node->child1());
11813 JSValueRegsTemporary result(this);
11814
11815 GPRReg bucketGPR = bucket.gpr();
11816 JSValueRegs resultRegs = result.regs();
11817
11818 m_jit.loadValue(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfValue()), resultRegs);
11819 jsValueResult(resultRegs, node);
11820}
11821
11822void SpeculativeJIT::compileExtractValueFromWeakMapGet(Node* node)
11823{
11824 JSValueOperand value(this, node->child1());
11825 JSValueRegsTemporary result(this, Reuse, value);
11826
11827 JSValueRegs valueRegs = value.jsValueRegs();
11828 JSValueRegs resultRegs = result.regs();
11829
11830#if USE(JSVALUE64)
11831 m_jit.moveValueRegs(valueRegs, resultRegs);
11832 auto done = m_jit.branchTestPtr(CCallHelpers::NonZero, resultRegs.payloadGPR());
11833 m_jit.moveValue(jsUndefined(), resultRegs);
11834 done.link(&m_jit);
11835#else
11836 auto isEmpty = m_jit.branchIfEmpty(valueRegs.tagGPR());
11837 m_jit.moveValueRegs(valueRegs, resultRegs);
11838 auto done = m_jit.jump();
11839
11840 isEmpty.link(&m_jit);
11841 m_jit.moveValue(jsUndefined(), resultRegs);
11842
11843 done.link(&m_jit);
11844#endif
11845
11846 jsValueResult(resultRegs, node, DataFormatJS);
11847}
11848
11849void SpeculativeJIT::compileThrow(Node* node)
11850{
11851 JSValueOperand value(this, node->child1());
11852 JSValueRegs valueRegs = value.jsValueRegs();
11853 flushRegisters();
11854 callOperation(operationThrowDFG, valueRegs);
11855 m_jit.exceptionCheck();
11856 m_jit.breakpoint();
11857 noResult(node);
11858}
11859
11860void SpeculativeJIT::compileThrowStaticError(Node* node)
11861{
11862 SpeculateCellOperand message(this, node->child1());
11863 GPRReg messageGPR = message.gpr();
11864 speculateString(node->child1(), messageGPR);
11865 flushRegisters();
11866 callOperation(operationThrowStaticError, messageGPR, node->errorType());
11867 m_jit.exceptionCheck();
11868 m_jit.breakpoint();
11869 noResult(node);
11870}
11871
11872void SpeculativeJIT::compileGetEnumerableLength(Node* node)
11873{
11874 SpeculateCellOperand enumerator(this, node->child1());
11875 GPRFlushedCallResult result(this);
11876 GPRReg resultGPR = result.gpr();
11877
11878 m_jit.load32(MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::indexedLengthOffset()), resultGPR);
11879 int32Result(resultGPR, node);
11880}
11881
11882void SpeculativeJIT::compileHasGenericProperty(Node* node)
11883{
11884 JSValueOperand base(this, node->child1());
11885 SpeculateCellOperand property(this, node->child2());
11886
11887 JSValueRegs baseRegs = base.jsValueRegs();
11888 GPRReg propertyGPR = property.gpr();
11889
11890 flushRegisters();
11891 JSValueRegsFlushedCallResult result(this);
11892 JSValueRegs resultRegs = result.regs();
11893 callOperation(operationHasGenericProperty, resultRegs, baseRegs, propertyGPR);
11894 m_jit.exceptionCheck();
11895 blessedBooleanResult(resultRegs.payloadGPR(), node);
11896}
11897
11898void SpeculativeJIT::compileToIndexString(Node* node)
11899{
11900 SpeculateInt32Operand index(this, node->child1());
11901 GPRReg indexGPR = index.gpr();
11902
11903 flushRegisters();
11904 GPRFlushedCallResult result(this);
11905 GPRReg resultGPR = result.gpr();
11906 callOperation(operationToIndexString, resultGPR, indexGPR);
11907 m_jit.exceptionCheck();
11908 cellResult(resultGPR, node);
11909}
11910
11911void SpeculativeJIT::compilePutByIdFlush(Node* node)
11912{
11913 SpeculateCellOperand base(this, node->child1());
11914 JSValueOperand value(this, node->child2());
11915 GPRTemporary scratch(this);
11916
11917 GPRReg baseGPR = base.gpr();
11918 JSValueRegs valueRegs = value.jsValueRegs();
11919 GPRReg scratchGPR = scratch.gpr();
11920 flushRegisters();
11921
11922 cachedPutById(node->origin.semantic, baseGPR, valueRegs, scratchGPR, node->identifierNumber(), NotDirect, MacroAssembler::Jump(), DontSpill);
11923
11924 noResult(node);
11925}
11926
11927void SpeculativeJIT::compilePutById(Node* node)
11928{
11929 SpeculateCellOperand base(this, node->child1());
11930 JSValueOperand value(this, node->child2());
11931 GPRTemporary scratch(this);
11932
11933 GPRReg baseGPR = base.gpr();
11934 JSValueRegs valueRegs = value.jsValueRegs();
11935 GPRReg scratchGPR = scratch.gpr();
11936
11937 cachedPutById(node->origin.semantic, baseGPR, valueRegs, scratchGPR, node->identifierNumber(), NotDirect);
11938
11939 noResult(node);
11940}
11941
11942void SpeculativeJIT::compilePutByIdDirect(Node* node)
11943{
11944 SpeculateCellOperand base(this, node->child1());
11945 JSValueOperand value(this, node->child2());
11946 GPRTemporary scratch(this);
11947
11948 GPRReg baseGPR = base.gpr();
11949 JSValueRegs valueRegs = value.jsValueRegs();
11950 GPRReg scratchGPR = scratch.gpr();
11951
11952 cachedPutById(node->origin.semantic, baseGPR, valueRegs, scratchGPR, node->identifierNumber(), Direct);
11953
11954 noResult(node);
11955}
11956
11957void SpeculativeJIT::compilePutByIdWithThis(Node* node)
11958{
11959 JSValueOperand base(this, node->child1());
11960 JSValueRegs baseRegs = base.jsValueRegs();
11961 JSValueOperand thisValue(this, node->child2());
11962 JSValueRegs thisRegs = thisValue.jsValueRegs();
11963 JSValueOperand value(this, node->child3());
11964 JSValueRegs valueRegs = value.jsValueRegs();
11965
11966 flushRegisters();
11967 callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis,
11968 NoResult, baseRegs, thisRegs, valueRegs, identifierUID(node->identifierNumber()));
11969 m_jit.exceptionCheck();
11970
11971 noResult(node);
11972}
11973
11974void SpeculativeJIT::compileGetByOffset(Node* node)
11975{
11976 StorageOperand storage(this, node->child1());
11977 JSValueRegsTemporary result(this, Reuse, storage);
11978
11979 GPRReg storageGPR = storage.gpr();
11980 JSValueRegs resultRegs = result.regs();
11981
11982 StorageAccessData& storageAccessData = node->storageAccessData();
11983
11984 m_jit.loadValue(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset)), resultRegs);
11985
11986 jsValueResult(resultRegs, node);
11987}
11988
11989void SpeculativeJIT::compilePutByOffset(Node* node)
11990{
11991 StorageOperand storage(this, node->child1());
11992 JSValueOperand value(this, node->child3());
11993
11994 GPRReg storageGPR = storage.gpr();
11995 JSValueRegs valueRegs = value.jsValueRegs();
11996
11997 speculate(node, node->child2());
11998
11999 StorageAccessData& storageAccessData = node->storageAccessData();
12000
12001 m_jit.storeValue(valueRegs, JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset)));
12002
12003 noResult(node);
12004}
12005
12006void SpeculativeJIT::compileMatchStructure(Node* node)
12007{
12008 SpeculateCellOperand base(this, node->child1());
12009 GPRTemporary temp(this);
12010 GPRReg baseGPR = base.gpr();
12011 GPRReg tempGPR = temp.gpr();
12012
12013 m_jit.load32(JITCompiler::Address(baseGPR, JSCell::structureIDOffset()), tempGPR);
12014
12015 auto& variants = node->matchStructureData().variants;
12016 Vector<int64_t> cases;
12017 for (MatchStructureVariant& variant : variants)
12018 cases.append(bitwise_cast<int32_t>(variant.structure->id()));
12019
12020 BinarySwitch binarySwitch(tempGPR, cases, BinarySwitch::Int32);
12021 JITCompiler::JumpList done;
12022 while (binarySwitch.advance(m_jit)) {
12023 m_jit.boxBooleanPayload(variants[binarySwitch.caseIndex()].result, tempGPR);
12024 done.append(m_jit.jump());
12025 }
12026 speculationCheck(BadCache, JSValueRegs(), node, binarySwitch.fallThrough());
12027
12028 done.link(&m_jit);
12029
12030 blessedBooleanResult(tempGPR, node);
12031}
12032
12033void SpeculativeJIT::compileHasStructureProperty(Node* node)
12034{
12035 JSValueOperand base(this, node->child1());
12036 SpeculateCellOperand property(this, node->child2());
12037 SpeculateCellOperand enumerator(this, node->child3());
12038 JSValueRegsTemporary result(this);
12039
12040 JSValueRegs baseRegs = base.jsValueRegs();
12041 GPRReg propertyGPR = property.gpr();
12042 JSValueRegs resultRegs = result.regs();
12043
12044 CCallHelpers::JumpList wrongStructure;
12045
12046 wrongStructure.append(m_jit.branchIfNotCell(baseRegs));
12047
12048 m_jit.load32(MacroAssembler::Address(baseRegs.payloadGPR(), JSCell::structureIDOffset()), resultRegs.payloadGPR());
12049 wrongStructure.append(m_jit.branch32(MacroAssembler::NotEqual,
12050 resultRegs.payloadGPR(),
12051 MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::cachedStructureIDOffset())));
12052
12053 moveTrueTo(resultRegs.payloadGPR());
12054 MacroAssembler::Jump done = m_jit.jump();
12055
12056 done.link(&m_jit);
12057
12058 addSlowPathGenerator(slowPathCall(wrongStructure, this, operationHasGenericProperty, resultRegs, baseRegs, propertyGPR));
12059 blessedBooleanResult(resultRegs.payloadGPR(), node);
12060}
12061
12062void SpeculativeJIT::compileGetPropertyEnumerator(Node* node)
12063{
12064 if (node->child1().useKind() == CellUse) {
12065 SpeculateCellOperand base(this, node->child1());
12066 GPRReg baseGPR = base.gpr();
12067
12068 flushRegisters();
12069 GPRFlushedCallResult result(this);
12070 GPRReg resultGPR = result.gpr();
12071 callOperation(operationGetPropertyEnumeratorCell, resultGPR, baseGPR);
12072 m_jit.exceptionCheck();
12073 cellResult(resultGPR, node);
12074 return;
12075 }
12076
12077 JSValueOperand base(this, node->child1());
12078 JSValueRegs baseRegs = base.jsValueRegs();
12079
12080 flushRegisters();
12081 GPRFlushedCallResult result(this);
12082 GPRReg resultGPR = result.gpr();
12083 callOperation(operationGetPropertyEnumerator, resultGPR, baseRegs);
12084 m_jit.exceptionCheck();
12085 cellResult(resultGPR, node);
12086}
12087
12088void SpeculativeJIT::compileGetEnumeratorPname(Node* node)
12089{
12090 ASSERT(node->op() == GetEnumeratorStructurePname || node->op() == GetEnumeratorGenericPname);
12091 SpeculateCellOperand enumerator(this, node->child1());
12092 SpeculateStrictInt32Operand index(this, node->child2());
12093 GPRTemporary scratch(this);
12094 JSValueRegsTemporary result(this);
12095
12096 GPRReg enumeratorGPR = enumerator.gpr();
12097 GPRReg indexGPR = index.gpr();
12098 GPRReg scratchGPR = scratch.gpr();
12099 JSValueRegs resultRegs = result.regs();
12100
12101 MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, indexGPR,
12102 MacroAssembler::Address(enumeratorGPR, (node->op() == GetEnumeratorStructurePname)
12103 ? JSPropertyNameEnumerator::endStructurePropertyIndexOffset()
12104 : JSPropertyNameEnumerator::endGenericPropertyIndexOffset()));
12105
12106 m_jit.moveValue(jsNull(), resultRegs);
12107
12108 MacroAssembler::Jump done = m_jit.jump();
12109 inBounds.link(&m_jit);
12110
12111 m_jit.loadPtr(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedPropertyNamesVectorOffset()), scratchGPR);
12112 m_jit.loadPtr(MacroAssembler::BaseIndex(scratchGPR, indexGPR, MacroAssembler::ScalePtr), resultRegs.payloadGPR());
12113#if USE(JSVALUE32_64)
12114 m_jit.move(MacroAssembler::TrustedImm32(JSValue::CellTag), resultRegs.tagGPR());
12115#endif
12116
12117 done.link(&m_jit);
12118 jsValueResult(resultRegs, node);
12119}
12120
12121void SpeculativeJIT::compileGetExecutable(Node* node)
12122{
12123 SpeculateCellOperand function(this, node->child1());
12124 GPRTemporary result(this, Reuse, function);
12125 GPRReg functionGPR = function.gpr();
12126 GPRReg resultGPR = result.gpr();
12127 speculateCellType(node->child1(), functionGPR, SpecFunction, JSFunctionType);
12128 m_jit.loadPtr(JITCompiler::Address(functionGPR, JSFunction::offsetOfExecutable()), resultGPR);
12129 cellResult(resultGPR, node);
12130}
12131
12132void SpeculativeJIT::compileGetGetter(Node* node)
12133{
12134 SpeculateCellOperand op1(this, node->child1());
12135 GPRTemporary result(this, Reuse, op1);
12136
12137 GPRReg op1GPR = op1.gpr();
12138 GPRReg resultGPR = result.gpr();
12139
12140 m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfGetter()), resultGPR);
12141
12142 cellResult(resultGPR, node);
12143}
12144
12145void SpeculativeJIT::compileGetSetter(Node* node)
12146{
12147 SpeculateCellOperand op1(this, node->child1());
12148 GPRTemporary result(this, Reuse, op1);
12149
12150 GPRReg op1GPR = op1.gpr();
12151 GPRReg resultGPR = result.gpr();
12152
12153 m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfSetter()), resultGPR);
12154
12155 cellResult(resultGPR, node);
12156}
12157
12158void SpeculativeJIT::compileGetCallee(Node* node)
12159{
12160 GPRTemporary result(this);
12161 m_jit.loadPtr(JITCompiler::payloadFor(CallFrameSlot::callee), result.gpr());
12162 cellResult(result.gpr(), node);
12163}
12164
12165void SpeculativeJIT::compileSetCallee(Node* node)
12166{
12167 SpeculateCellOperand callee(this, node->child1());
12168 m_jit.storeCell(callee.gpr(), JITCompiler::payloadFor(CallFrameSlot::callee));
12169 noResult(node);
12170}
12171
12172void SpeculativeJIT::compileGetArgumentCountIncludingThis(Node* node)
12173{
12174 GPRTemporary result(this);
12175 VirtualRegister argumentCountRegister;
12176 if (InlineCallFrame* inlineCallFrame = node->argumentsInlineCallFrame())
12177 argumentCountRegister = inlineCallFrame->argumentCountRegister;
12178 else
12179 argumentCountRegister = VirtualRegister(CallFrameSlot::argumentCount);
12180 m_jit.load32(JITCompiler::payloadFor(argumentCountRegister), result.gpr());
12181 int32Result(result.gpr(), node);
12182}
12183
12184void SpeculativeJIT::compileSetArgumentCountIncludingThis(Node* node)
12185{
12186 m_jit.store32(TrustedImm32(node->argumentCountIncludingThis()), JITCompiler::payloadFor(CallFrameSlot::argumentCount));
12187 noResult(node);
12188}
12189
12190void SpeculativeJIT::compileStrCat(Node* node)
12191{
12192 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
12193 JSValueOperand op2(this, node->child2(), ManualOperandSpeculation);
12194 JSValueOperand op3(this, node->child3(), ManualOperandSpeculation);
12195
12196 JSValueRegs op1Regs = op1.jsValueRegs();
12197 JSValueRegs op2Regs = op2.jsValueRegs();
12198 JSValueRegs op3Regs;
12199
12200 if (node->child3())
12201 op3Regs = op3.jsValueRegs();
12202
12203 flushRegisters();
12204
12205 GPRFlushedCallResult result(this);
12206 if (node->child3())
12207 callOperation(operationStrCat3, result.gpr(), op1Regs, op2Regs, op3Regs);
12208 else
12209 callOperation(operationStrCat2, result.gpr(), op1Regs, op2Regs);
12210 m_jit.exceptionCheck();
12211
12212 cellResult(result.gpr(), node);
12213}
12214
12215void SpeculativeJIT::compileNewArrayBuffer(Node* node)
12216{
12217 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
12218 auto* array = node->castOperand<JSImmutableButterfly*>();
12219
12220 IndexingType indexingMode = node->indexingMode();
12221 RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingMode));
12222
12223 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(indexingMode)) {
12224 GPRTemporary result(this);
12225 GPRTemporary scratch1(this);
12226 GPRTemporary scratch2(this);
12227
12228 GPRReg resultGPR = result.gpr();
12229 GPRReg scratch1GPR = scratch1.gpr();
12230 GPRReg scratch2GPR = scratch2.gpr();
12231
12232 MacroAssembler::JumpList slowCases;
12233
12234 emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), TrustedImmPtr(array->toButterfly()), scratch1GPR, scratch2GPR, slowCases);
12235
12236 addSlowPathGenerator(slowPathCall(slowCases, this, operationNewArrayBuffer, result.gpr(), structure, array));
12237
12238 DFG_ASSERT(m_jit.graph(), node, indexingMode & IsArray, indexingMode);
12239 cellResult(resultGPR, node);
12240 return;
12241 }
12242
12243 flushRegisters();
12244 GPRFlushedCallResult result(this);
12245
12246 callOperation(operationNewArrayBuffer, result.gpr(), structure, TrustedImmPtr(node->cellOperand()));
12247 m_jit.exceptionCheck();
12248
12249 cellResult(result.gpr(), node);
12250}
12251
12252void SpeculativeJIT::compileNewArrayWithSize(Node* node)
12253{
12254 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
12255 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) {
12256 SpeculateStrictInt32Operand size(this, node->child1());
12257 GPRTemporary result(this);
12258
12259 GPRReg sizeGPR = size.gpr();
12260 GPRReg resultGPR = result.gpr();
12261
12262 compileAllocateNewArrayWithSize(globalObject, resultGPR, sizeGPR, node->indexingType());
12263 cellResult(resultGPR, node);
12264 return;
12265 }
12266
12267 SpeculateStrictInt32Operand size(this, node->child1());
12268 GPRReg sizeGPR = size.gpr();
12269 flushRegisters();
12270 GPRFlushedCallResult result(this);
12271 GPRReg resultGPR = result.gpr();
12272 GPRReg structureGPR = AssemblyHelpers::selectScratchGPR(sizeGPR);
12273 MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH));
12274 m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()))), structureGPR);
12275 MacroAssembler::Jump done = m_jit.jump();
12276 bigLength.link(&m_jit);
12277 m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))), structureGPR);
12278 done.link(&m_jit);
12279 callOperation(operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR, nullptr);
12280 m_jit.exceptionCheck();
12281 cellResult(resultGPR, node);
12282}
12283
12284void SpeculativeJIT::compileNewTypedArray(Node* node)
12285{
12286 switch (node->child1().useKind()) {
12287 case Int32Use:
12288 compileNewTypedArrayWithSize(node);
12289 break;
12290 case UntypedUse: {
12291 JSValueOperand argument(this, node->child1());
12292 JSValueRegs argumentRegs = argument.jsValueRegs();
12293
12294 flushRegisters();
12295
12296 GPRFlushedCallResult result(this);
12297 GPRReg resultGPR = result.gpr();
12298
12299 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
12300 callOperation(
12301 operationNewTypedArrayWithOneArgumentForType(node->typedArrayType()),
12302 resultGPR, m_jit.graph().registerStructure(globalObject->typedArrayStructureConcurrently(node->typedArrayType())), argumentRegs);
12303 m_jit.exceptionCheck();
12304
12305 cellResult(resultGPR, node);
12306 break;
12307 }
12308 default:
12309 RELEASE_ASSERT_NOT_REACHED();
12310 break;
12311 }
12312}
12313
12314void SpeculativeJIT::compileToThis(Node* node)
12315{
12316 ASSERT(node->child1().useKind() == UntypedUse);
12317 JSValueOperand thisValue(this, node->child1());
12318 JSValueRegsTemporary temp(this);
12319
12320 JSValueRegs thisValueRegs = thisValue.jsValueRegs();
12321 JSValueRegs tempRegs = temp.regs();
12322
12323 MacroAssembler::JumpList slowCases;
12324 slowCases.append(m_jit.branchIfNotCell(thisValueRegs));
12325 slowCases.append(
12326 m_jit.branchTest8(
12327 MacroAssembler::NonZero,
12328 MacroAssembler::Address(thisValueRegs.payloadGPR(), JSCell::typeInfoFlagsOffset()),
12329 MacroAssembler::TrustedImm32(OverridesToThis)));
12330 m_jit.moveValueRegs(thisValueRegs, tempRegs);
12331
12332 J_JITOperation_EJ function;
12333 if (m_jit.isStrictModeFor(node->origin.semantic))
12334 function = operationToThisStrict;
12335 else
12336 function = operationToThis;
12337 addSlowPathGenerator(slowPathCall(slowCases, this, function, tempRegs, thisValueRegs));
12338
12339 jsValueResult(tempRegs, node);
12340}
12341
12342void SpeculativeJIT::compileObjectKeys(Node* node)
12343{
12344 switch (node->child1().useKind()) {
12345 case ObjectUse: {
12346 if (m_graph.isWatchingHavingABadTimeWatchpoint(node)) {
12347 SpeculateCellOperand object(this, node->child1());
12348 GPRTemporary structure(this);
12349 GPRTemporary scratch(this);
12350 GPRTemporary scratch2(this);
12351 GPRTemporary scratch3(this);
12352 GPRTemporary result(this);
12353
12354 GPRReg objectGPR = object.gpr();
12355 GPRReg structureGPR = structure.gpr();
12356 GPRReg scratchGPR = scratch.gpr();
12357 GPRReg scratch2GPR = scratch2.gpr();
12358 GPRReg scratch3GPR = scratch3.gpr();
12359 GPRReg resultGPR = result.gpr();
12360
12361 speculateObject(node->child1(), objectGPR);
12362
12363 CCallHelpers::JumpList slowCases;
12364 m_jit.emitLoadStructure(*m_jit.vm(), objectGPR, structureGPR, scratchGPR);
12365 m_jit.loadPtr(CCallHelpers::Address(structureGPR, Structure::previousOrRareDataOffset()), scratchGPR);
12366
12367 slowCases.append(m_jit.branchTestPtr(CCallHelpers::Zero, scratchGPR));
12368 slowCases.append(m_jit.branch32(CCallHelpers::Equal, CCallHelpers::Address(scratchGPR, JSCell::structureIDOffset()), TrustedImm32(bitwise_cast<int32_t>(m_jit.vm()->structureStructure->structureID()))));
12369
12370 m_jit.loadPtr(CCallHelpers::Address(scratchGPR, StructureRareData::offsetOfCachedOwnKeys()), scratchGPR);
12371
12372 ASSERT(bitwise_cast<uintptr_t>(StructureRareData::cachedOwnKeysSentinel()) == 1);
12373 slowCases.append(m_jit.branchPtr(CCallHelpers::BelowOrEqual, scratchGPR, TrustedImmPtr(bitwise_cast<void*>(StructureRareData::cachedOwnKeysSentinel()))));
12374
12375 MacroAssembler::JumpList slowButArrayBufferCases;
12376
12377 JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic);
12378 RegisteredStructure arrayStructure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(CopyOnWriteArrayWithContiguous));
12379
12380 m_jit.move(scratchGPR, scratch3GPR);
12381 m_jit.addPtr(TrustedImmPtr(JSImmutableButterfly::offsetOfData()), scratchGPR);
12382
12383 emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(arrayStructure), scratchGPR, structureGPR, scratch2GPR, slowButArrayBufferCases);
12384
12385 addSlowPathGenerator(slowPathCall(slowButArrayBufferCases, this, operationNewArrayBuffer, resultGPR, arrayStructure, scratch3GPR));
12386
12387 addSlowPathGenerator(slowPathCall(slowCases, this, operationObjectKeysObject, resultGPR, objectGPR));
12388
12389 cellResult(resultGPR, node);
12390 break;
12391 }
12392
12393 SpeculateCellOperand object(this, node->child1());
12394
12395 GPRReg objectGPR = object.gpr();
12396
12397 speculateObject(node->child1(), objectGPR);
12398
12399 flushRegisters();
12400 GPRFlushedCallResult result(this);
12401 GPRReg resultGPR = result.gpr();
12402 callOperation(operationObjectKeysObject, resultGPR, objectGPR);
12403 m_jit.exceptionCheck();
12404
12405 cellResult(resultGPR, node);
12406 break;
12407 }
12408
12409 case UntypedUse: {
12410 JSValueOperand object(this, node->child1());
12411
12412 JSValueRegs objectRegs = object.jsValueRegs();
12413
12414 flushRegisters();
12415 GPRFlushedCallResult result(this);
12416 GPRReg resultGPR = result.gpr();
12417 callOperation(operationObjectKeys, resultGPR, objectRegs);
12418 m_jit.exceptionCheck();
12419
12420 cellResult(resultGPR, node);
12421 break;
12422 }
12423
12424 default:
12425 RELEASE_ASSERT_NOT_REACHED();
12426 break;
12427 }
12428}
12429
12430void SpeculativeJIT::compileObjectCreate(Node* node)
12431{
12432 switch (node->child1().useKind()) {
12433 case ObjectUse: {
12434 SpeculateCellOperand prototype(this, node->child1());
12435
12436 GPRReg prototypeGPR = prototype.gpr();
12437
12438 speculateObject(node->child1(), prototypeGPR);
12439
12440 flushRegisters();
12441 GPRFlushedCallResult result(this);
12442 GPRReg resultGPR = result.gpr();
12443 callOperation(operationObjectCreateObject, resultGPR, prototypeGPR);
12444 m_jit.exceptionCheck();
12445
12446 cellResult(resultGPR, node);
12447 break;
12448 }
12449
12450 case UntypedUse: {
12451 JSValueOperand prototype(this, node->child1());
12452
12453 JSValueRegs prototypeRegs = prototype.jsValueRegs();
12454
12455 flushRegisters();
12456 GPRFlushedCallResult result(this);
12457 GPRReg resultGPR = result.gpr();
12458 callOperation(operationObjectCreate, resultGPR, prototypeRegs);
12459 m_jit.exceptionCheck();
12460
12461 cellResult(resultGPR, node);
12462 break;
12463 }
12464
12465 default:
12466 RELEASE_ASSERT_NOT_REACHED();
12467 break;
12468 }
12469}
12470
12471void SpeculativeJIT::compileCreateThis(Node* node)
12472{
12473 // Note that there is not so much profit to speculate here. The only things we
12474 // speculate on are (1) that it's a cell, since that eliminates cell checks
12475 // later if the proto is reused, and (2) if we have a FinalObject prediction
12476 // then we speculate because we want to get recompiled if it isn't (since
12477 // otherwise we'd start taking slow path a lot).
12478
12479 SpeculateCellOperand callee(this, node->child1());
12480 GPRTemporary result(this);
12481 GPRTemporary allocator(this);
12482 GPRTemporary structure(this);
12483 GPRTemporary scratch(this);
12484
12485 GPRReg calleeGPR = callee.gpr();
12486 GPRReg resultGPR = result.gpr();
12487 GPRReg allocatorGPR = allocator.gpr();
12488 GPRReg structureGPR = structure.gpr();
12489 GPRReg scratchGPR = scratch.gpr();
12490 // Rare data is only used to access the allocator & structure
12491 // We can avoid using an additional GPR this way
12492 GPRReg rareDataGPR = structureGPR;
12493 GPRReg inlineCapacityGPR = rareDataGPR;
12494
12495 MacroAssembler::JumpList slowPath;
12496
12497 slowPath.append(m_jit.branchIfNotFunction(calleeGPR));
12498 m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR);
12499 slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, rareDataGPR));
12500 m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR);
12501 m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR);
12502
12503 auto butterfly = TrustedImmPtr(nullptr);
12504 emitAllocateJSObject(resultGPR, JITAllocator::variable(), allocatorGPR, structureGPR, butterfly, scratchGPR, slowPath);
12505
12506 m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR);
12507 m_jit.load32(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfInlineCapacity()), inlineCapacityGPR);
12508 m_jit.emitInitializeInlineStorage(resultGPR, inlineCapacityGPR);
12509 m_jit.mutatorFence(*m_jit.vm());
12510
12511 addSlowPathGenerator(slowPathCall(slowPath, this, operationCreateThis, resultGPR, calleeGPR, node->inlineCapacity()));
12512
12513 cellResult(resultGPR, node);
12514}
12515
12516void SpeculativeJIT::compileNewObject(Node* node)
12517{
12518 GPRTemporary result(this);
12519 GPRTemporary allocator(this);
12520 GPRTemporary scratch(this);
12521
12522 GPRReg resultGPR = result.gpr();
12523 GPRReg allocatorGPR = allocator.gpr();
12524 GPRReg scratchGPR = scratch.gpr();
12525
12526 MacroAssembler::JumpList slowPath;
12527
12528 RegisteredStructure structure = node->structure();
12529 size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
12530 Allocator allocatorValue = allocatorForNonVirtualConcurrently<JSFinalObject>(*m_jit.vm(), allocationSize, AllocatorForMode::AllocatorIfExists);
12531 if (!allocatorValue)
12532 slowPath.append(m_jit.jump());
12533 else {
12534 auto butterfly = TrustedImmPtr(nullptr);
12535 emitAllocateJSObject(resultGPR, JITAllocator::constant(allocatorValue), allocatorGPR, TrustedImmPtr(structure), butterfly, scratchGPR, slowPath);
12536 m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity());
12537 m_jit.mutatorFence(*m_jit.vm());
12538 }
12539
12540 addSlowPathGenerator(slowPathCall(slowPath, this, operationNewObject, resultGPR, structure));
12541
12542 cellResult(resultGPR, node);
12543}
12544
12545void SpeculativeJIT::compileToPrimitive(Node* node)
12546{
12547 DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse, node->child1().useKind());
12548 JSValueOperand argument(this, node->child1());
12549 JSValueRegsTemporary result(this, Reuse, argument);
12550
12551 JSValueRegs argumentRegs = argument.jsValueRegs();
12552 JSValueRegs resultRegs = result.regs();
12553
12554 argument.use();
12555
12556 MacroAssembler::Jump alreadyPrimitive = m_jit.branchIfNotCell(argumentRegs);
12557 MacroAssembler::Jump notPrimitive = m_jit.branchIfObject(argumentRegs.payloadGPR());
12558
12559 alreadyPrimitive.link(&m_jit);
12560 m_jit.moveValueRegs(argumentRegs, resultRegs);
12561
12562 addSlowPathGenerator(slowPathCall(notPrimitive, this, operationToPrimitive, resultRegs, argumentRegs));
12563
12564 jsValueResult(resultRegs, node, DataFormatJS, UseChildrenCalledExplicitly);
12565}
12566
12567void SpeculativeJIT::compileLogShadowChickenPrologue(Node* node)
12568{
12569 flushRegisters();
12570 prepareForExternalCall();
12571 m_jit.emitStoreCodeOrigin(node->origin.semantic);
12572
12573 GPRTemporary scratch1(this, GPRInfo::nonArgGPR0); // This must be a non-argument GPR.
12574 GPRReg scratch1Reg = scratch1.gpr();
12575 GPRTemporary scratch2(this);
12576 GPRReg scratch2Reg = scratch2.gpr();
12577 GPRTemporary shadowPacket(this);
12578 GPRReg shadowPacketReg = shadowPacket.gpr();
12579
12580 m_jit.ensureShadowChickenPacket(*m_jit.vm(), shadowPacketReg, scratch1Reg, scratch2Reg);
12581
12582 SpeculateCellOperand scope(this, node->child1());
12583 GPRReg scopeReg = scope.gpr();
12584
12585 m_jit.logShadowChickenProloguePacket(shadowPacketReg, scratch1Reg, scopeReg);
12586 noResult(node);
12587}
12588
12589void SpeculativeJIT::compileLogShadowChickenTail(Node* node)
12590{
12591 flushRegisters();
12592 prepareForExternalCall();
12593 CallSiteIndex callSiteIndex = m_jit.emitStoreCodeOrigin(node->origin.semantic);
12594
12595 GPRTemporary scratch1(this, GPRInfo::nonArgGPR0); // This must be a non-argument GPR.
12596 GPRReg scratch1Reg = scratch1.gpr();
12597 GPRTemporary scratch2(this);
12598 GPRReg scratch2Reg = scratch2.gpr();
12599 GPRTemporary shadowPacket(this);
12600 GPRReg shadowPacketReg = shadowPacket.gpr();
12601
12602 m_jit.ensureShadowChickenPacket(*m_jit.vm(), shadowPacketReg, scratch1Reg, scratch2Reg);
12603
12604 JSValueOperand thisValue(this, node->child1());
12605 JSValueRegs thisRegs = thisValue.jsValueRegs();
12606 SpeculateCellOperand scope(this, node->child2());
12607 GPRReg scopeReg = scope.gpr();
12608
12609 m_jit.logShadowChickenTailPacket(shadowPacketReg, thisRegs, scopeReg, m_jit.codeBlock(), callSiteIndex);
12610 noResult(node);
12611}
12612
12613void SpeculativeJIT::compileSetAdd(Node* node)
12614{
12615 SpeculateCellOperand set(this, node->child1());
12616 JSValueOperand key(this, node->child2());
12617 SpeculateInt32Operand hash(this, node->child3());
12618
12619 GPRReg setGPR = set.gpr();
12620 JSValueRegs keyRegs = key.jsValueRegs();
12621 GPRReg hashGPR = hash.gpr();
12622
12623 speculateSetObject(node->child1(), setGPR);
12624
12625 flushRegisters();
12626 GPRFlushedCallResult result(this);
12627 GPRReg resultGPR = result.gpr();
12628 callOperation(operationSetAdd, resultGPR, setGPR, keyRegs, hashGPR);
12629 m_jit.exceptionCheck();
12630 cellResult(resultGPR, node);
12631}
12632
12633void SpeculativeJIT::compileMapSet(Node* node)
12634{
12635 SpeculateCellOperand map(this, m_jit.graph().varArgChild(node, 0));
12636 JSValueOperand key(this, m_jit.graph().varArgChild(node, 1));
12637 JSValueOperand value(this, m_jit.graph().varArgChild(node, 2));
12638 SpeculateInt32Operand hash(this, m_jit.graph().varArgChild(node, 3));
12639
12640 GPRReg mapGPR = map.gpr();
12641 JSValueRegs keyRegs = key.jsValueRegs();
12642 JSValueRegs valueRegs = value.jsValueRegs();
12643 GPRReg hashGPR = hash.gpr();
12644
12645 speculateMapObject(m_jit.graph().varArgChild(node, 0), mapGPR);
12646
12647 flushRegisters();
12648 GPRFlushedCallResult result(this);
12649 GPRReg resultGPR = result.gpr();
12650 callOperation(operationMapSet, resultGPR, mapGPR, keyRegs, valueRegs, hashGPR);
12651 m_jit.exceptionCheck();
12652 cellResult(resultGPR, node);
12653}
12654
12655void SpeculativeJIT::compileWeakMapGet(Node* node)
12656{
12657 GPRTemporary mask(this);
12658 GPRTemporary buffer(this);
12659 JSValueRegsTemporary result(this);
12660
12661 GPRReg maskGPR = mask.gpr();
12662 GPRReg bufferGPR = buffer.gpr();
12663 JSValueRegs resultRegs = result.regs();
12664
12665 GPRTemporary index;
12666 GPRReg indexGPR { InvalidGPRReg };
12667 {
12668 SpeculateInt32Operand hash(this, node->child3());
12669 GPRReg hashGPR = hash.gpr();
12670 index = GPRTemporary(this, Reuse, hash);
12671 indexGPR = index.gpr();
12672 m_jit.move(hashGPR, indexGPR);
12673 }
12674
12675 {
12676 SpeculateCellOperand weakMap(this, node->child1());
12677 GPRReg weakMapGPR = weakMap.gpr();
12678 if (node->child1().useKind() == WeakMapObjectUse)
12679 speculateWeakMapObject(node->child1(), weakMapGPR);
12680 else
12681 speculateWeakSetObject(node->child1(), weakMapGPR);
12682
12683 ASSERT(WeakMapImpl<WeakMapBucket<WeakMapBucketDataKey>>::offsetOfCapacity() == WeakMapImpl<WeakMapBucket<WeakMapBucketDataKeyValue>>::offsetOfCapacity());
12684 ASSERT(WeakMapImpl<WeakMapBucket<WeakMapBucketDataKey>>::offsetOfBuffer() == WeakMapImpl<WeakMapBucket<WeakMapBucketDataKeyValue>>::offsetOfBuffer());
12685 m_jit.load32(MacroAssembler::Address(weakMapGPR, WeakMapImpl<WeakMapBucket<WeakMapBucketDataKey>>::offsetOfCapacity()), maskGPR);
12686 m_jit.loadPtr(MacroAssembler::Address(weakMapGPR, WeakMapImpl<WeakMapBucket<WeakMapBucketDataKey>>::offsetOfBuffer()), bufferGPR);
12687 }
12688
12689 SpeculateCellOperand key(this, node->child2());
12690 GPRReg keyGPR = key.gpr();
12691 speculateObject(node->child2(), keyGPR);
12692
12693#if USE(JSVALUE32_64)
12694 GPRReg bucketGPR = resultRegs.tagGPR();
12695#else
12696 GPRTemporary bucket(this);
12697 GPRReg bucketGPR = bucket.gpr();
12698#endif
12699
12700 m_jit.sub32(TrustedImm32(1), maskGPR);
12701
12702 MacroAssembler::Label loop = m_jit.label();
12703 m_jit.and32(maskGPR, indexGPR);
12704 if (node->child1().useKind() == WeakSetObjectUse) {
12705 static_assert(sizeof(WeakMapBucket<WeakMapBucketDataKey>) == sizeof(void*), "");
12706 m_jit.zeroExtend32ToPtr(indexGPR, bucketGPR);
12707 m_jit.lshiftPtr(MacroAssembler::Imm32(sizeof(void*) == 4 ? 2 : 3), bucketGPR);
12708 m_jit.addPtr(bufferGPR, bucketGPR);
12709 } else {
12710 ASSERT(node->child1().useKind() == WeakMapObjectUse);
12711 static_assert(sizeof(WeakMapBucket<WeakMapBucketDataKeyValue>) == 16, "");
12712 m_jit.zeroExtend32ToPtr(indexGPR, bucketGPR);
12713 m_jit.lshiftPtr(MacroAssembler::Imm32(4), bucketGPR);
12714 m_jit.addPtr(bufferGPR, bucketGPR);
12715 }
12716
12717 m_jit.loadPtr(MacroAssembler::Address(bucketGPR, WeakMapBucket<WeakMapBucketDataKeyValue>::offsetOfKey()), resultRegs.payloadGPR());
12718
12719 // They're definitely the same value, we found the bucket we were looking for!
12720 // The deleted key comparison is also done with this.
12721 auto found = m_jit.branchPtr(MacroAssembler::Equal, resultRegs.payloadGPR(), keyGPR);
12722
12723 auto notPresentInTable = m_jit.branchTestPtr(MacroAssembler::Zero, resultRegs.payloadGPR());
12724
12725 m_jit.add32(TrustedImm32(1), indexGPR);
12726 m_jit.jump().linkTo(loop, &m_jit);
12727
12728#if USE(JSVALUE32_64)
12729 notPresentInTable.link(&m_jit);
12730 m_jit.moveValue(JSValue(), resultRegs);
12731 auto notPresentInTableDone = m_jit.jump();
12732
12733 found.link(&m_jit);
12734 if (node->child1().useKind() == WeakSetObjectUse)
12735 m_jit.move(TrustedImm32(JSValue::CellTag), resultRegs.tagGPR());
12736 else
12737 m_jit.loadValue(MacroAssembler::Address(bucketGPR, WeakMapBucket<WeakMapBucketDataKeyValue>::offsetOfValue()), resultRegs);
12738
12739 notPresentInTableDone.link(&m_jit);
12740#else
12741 notPresentInTable.link(&m_jit);
12742 found.link(&m_jit);
12743
12744 // In 64bit environment, Empty bucket has JSEmpty value. Empty key is JSEmpty.
12745 // If empty bucket is found, we can use the same path used for the case of finding a bucket.
12746 if (node->child1().useKind() == WeakMapObjectUse)
12747 m_jit.loadValue(MacroAssembler::Address(bucketGPR, WeakMapBucket<WeakMapBucketDataKeyValue>::offsetOfValue()), resultRegs);
12748#endif
12749
12750 jsValueResult(resultRegs, node);
12751}
12752
12753void SpeculativeJIT::compileWeakSetAdd(Node* node)
12754{
12755 SpeculateCellOperand set(this, node->child1());
12756 SpeculateCellOperand key(this, node->child2());
12757 SpeculateInt32Operand hash(this, node->child3());
12758
12759 GPRReg setGPR = set.gpr();
12760 GPRReg keyGPR = key.gpr();
12761 GPRReg hashGPR = hash.gpr();
12762
12763 speculateWeakSetObject(node->child1(), setGPR);
12764 speculateObject(node->child2(), keyGPR);
12765
12766 flushRegisters();
12767 callOperation(operationWeakSetAdd, setGPR, keyGPR, hashGPR);
12768 m_jit.exceptionCheck();
12769 noResult(node);
12770}
12771
12772void SpeculativeJIT::compileWeakMapSet(Node* node)
12773{
12774 SpeculateCellOperand map(this, m_jit.graph().varArgChild(node, 0));
12775 SpeculateCellOperand key(this, m_jit.graph().varArgChild(node, 1));
12776 JSValueOperand value(this, m_jit.graph().varArgChild(node, 2));
12777 SpeculateInt32Operand hash(this, m_jit.graph().varArgChild(node, 3));
12778
12779 GPRReg mapGPR = map.gpr();
12780 GPRReg keyGPR = key.gpr();
12781 JSValueRegs valueRegs = value.jsValueRegs();
12782 GPRReg hashGPR = hash.gpr();
12783
12784 speculateWeakMapObject(m_jit.graph().varArgChild(node, 0), mapGPR);
12785 speculateObject(m_jit.graph().varArgChild(node, 1), keyGPR);
12786
12787 flushRegisters();
12788 callOperation(operationWeakMapSet, mapGPR, keyGPR, valueRegs, hashGPR);
12789 m_jit.exceptionCheck();
12790 noResult(node);
12791}
12792
12793void SpeculativeJIT::compileGetPrototypeOf(Node* node)
12794{
12795 switch (node->child1().useKind()) {
12796 case ArrayUse:
12797 case FunctionUse:
12798 case FinalObjectUse: {
12799 SpeculateCellOperand object(this, node->child1());
12800 GPRTemporary temp(this);
12801 GPRTemporary temp2(this);
12802
12803 GPRReg objectGPR = object.gpr();
12804 GPRReg tempGPR = temp.gpr();
12805 GPRReg temp2GPR = temp2.gpr();
12806
12807 switch (node->child1().useKind()) {
12808 case ArrayUse:
12809 speculateArray(node->child1(), objectGPR);
12810 break;
12811 case FunctionUse:
12812 speculateFunction(node->child1(), objectGPR);
12813 break;
12814 case FinalObjectUse:
12815 speculateFinalObject(node->child1(), objectGPR);
12816 break;
12817 default:
12818 RELEASE_ASSERT_NOT_REACHED();
12819 break;
12820 }
12821
12822 m_jit.emitLoadStructure(*m_jit.vm(), objectGPR, tempGPR, temp2GPR);
12823
12824 AbstractValue& value = m_state.forNode(node->child1());
12825 if ((value.m_type && !(value.m_type & ~SpecObject)) && value.m_structure.isFinite()) {
12826 bool hasPolyProto = false;
12827 bool hasMonoProto = false;
12828 value.m_structure.forEach([&] (RegisteredStructure structure) {
12829 if (structure->hasPolyProto())
12830 hasPolyProto = true;
12831 else
12832 hasMonoProto = true;
12833 });
12834
12835 if (hasMonoProto && !hasPolyProto) {
12836#if USE(JSVALUE64)
12837 m_jit.load64(MacroAssembler::Address(tempGPR, Structure::prototypeOffset()), tempGPR);
12838 jsValueResult(tempGPR, node);
12839#else
12840 m_jit.load32(MacroAssembler::Address(tempGPR, Structure::prototypeOffset() + TagOffset), temp2GPR);
12841 m_jit.load32(MacroAssembler::Address(tempGPR, Structure::prototypeOffset() + PayloadOffset), tempGPR);
12842 jsValueResult(temp2GPR, tempGPR, node);
12843#endif
12844 return;
12845 }
12846
12847 if (hasPolyProto && !hasMonoProto) {
12848#if USE(JSVALUE64)
12849 m_jit.load64(JITCompiler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset)), tempGPR);
12850 jsValueResult(tempGPR, node);
12851#else
12852 m_jit.load32(JITCompiler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset) + TagOffset), temp2GPR);
12853 m_jit.load32(JITCompiler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset) + PayloadOffset), tempGPR);
12854 jsValueResult(temp2GPR, tempGPR, node);
12855#endif
12856 return;
12857 }
12858 }
12859
12860#if USE(JSVALUE64)
12861 m_jit.load64(MacroAssembler::Address(tempGPR, Structure::prototypeOffset()), tempGPR);
12862 auto hasMonoProto = m_jit.branchIfNotEmpty(tempGPR);
12863 m_jit.load64(JITCompiler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset)), tempGPR);
12864 hasMonoProto.link(&m_jit);
12865 jsValueResult(tempGPR, node);
12866#else
12867 m_jit.load32(MacroAssembler::Address(tempGPR, Structure::prototypeOffset() + TagOffset), temp2GPR);
12868 m_jit.load32(MacroAssembler::Address(tempGPR, Structure::prototypeOffset() + PayloadOffset), tempGPR);
12869 auto hasMonoProto = m_jit.branchIfNotEmpty(temp2GPR);
12870 m_jit.load32(JITCompiler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset) + TagOffset), temp2GPR);
12871 m_jit.load32(JITCompiler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset) + PayloadOffset), tempGPR);
12872 hasMonoProto.link(&m_jit);
12873 jsValueResult(temp2GPR, tempGPR, node);
12874#endif
12875 return;
12876 }
12877 case ObjectUse: {
12878 SpeculateCellOperand value(this, node->child1());
12879 JSValueRegsTemporary result(this);
12880
12881 GPRReg valueGPR = value.gpr();
12882 JSValueRegs resultRegs = result.regs();
12883
12884 speculateObject(node->child1(), valueGPR);
12885
12886 flushRegisters();
12887 callOperation(operationGetPrototypeOfObject, resultRegs, valueGPR);
12888 m_jit.exceptionCheck();
12889 jsValueResult(resultRegs, node);
12890 return;
12891 }
12892 default: {
12893 JSValueOperand value(this, node->child1());
12894 JSValueRegsTemporary result(this);
12895
12896 JSValueRegs valueRegs = value.jsValueRegs();
12897 JSValueRegs resultRegs = result.regs();
12898
12899 flushRegisters();
12900 callOperation(operationGetPrototypeOf, resultRegs, valueRegs);
12901 m_jit.exceptionCheck();
12902 jsValueResult(resultRegs, node);
12903 return;
12904 }
12905 }
12906}
12907
12908void SpeculativeJIT::compileIdentity(Node* node)
12909{
12910 speculate(node, node->child1());
12911 switch (node->child1().useKind()) {
12912#if USE(JSVALUE64)
12913 case DoubleRepAnyIntUse:
12914#endif
12915 case DoubleRepUse:
12916 case DoubleRepRealUse: {
12917 SpeculateDoubleOperand op(this, node->child1());
12918 FPRTemporary scratch(this, op);
12919 m_jit.moveDouble(op.fpr(), scratch.fpr());
12920 doubleResult(scratch.fpr(), node);
12921 break;
12922 }
12923#if USE(JSVALUE64)
12924 case Int52RepUse: {
12925 SpeculateInt52Operand op(this, node->child1());
12926 GPRTemporary result(this, Reuse, op);
12927 m_jit.move(op.gpr(), result.gpr());
12928 int52Result(result.gpr(), node);
12929 break;
12930 }
12931#endif
12932 default: {
12933 JSValueOperand op(this, node->child1(), ManualOperandSpeculation);
12934 JSValueRegsTemporary result(this, Reuse, op);
12935 JSValueRegs opRegs = op.jsValueRegs();
12936 JSValueRegs resultRegs = result.regs();
12937 m_jit.moveValueRegs(opRegs, resultRegs);
12938 jsValueResult(resultRegs, node);
12939 break;
12940 }
12941 }
12942}
12943
12944void SpeculativeJIT::compileMiscStrictEq(Node* node)
12945{
12946 JSValueOperand op1(this, node->child1(), ManualOperandSpeculation);
12947 JSValueOperand op2(this, node->child2(), ManualOperandSpeculation);
12948 GPRTemporary result(this);
12949
12950 if (node->child1().useKind() == MiscUse)
12951 speculateMisc(node->child1(), op1.jsValueRegs());
12952 if (node->child2().useKind() == MiscUse)
12953 speculateMisc(node->child2(), op2.jsValueRegs());
12954
12955#if USE(JSVALUE64)
12956 m_jit.compare64(JITCompiler::Equal, op1.gpr(), op2.gpr(), result.gpr());
12957#else
12958 m_jit.move(TrustedImm32(0), result.gpr());
12959 JITCompiler::Jump notEqual = m_jit.branch32(JITCompiler::NotEqual, op1.tagGPR(), op2.tagGPR());
12960 m_jit.compare32(JITCompiler::Equal, op1.payloadGPR(), op2.payloadGPR(), result.gpr());
12961 notEqual.link(&m_jit);
12962#endif
12963 unblessedBooleanResult(result.gpr(), node);
12964}
12965
12966void SpeculativeJIT::emitInitializeButterfly(GPRReg storageGPR, GPRReg sizeGPR, JSValueRegs emptyValueRegs, GPRReg scratchGPR)
12967{
12968 m_jit.zeroExtend32ToPtr(sizeGPR, scratchGPR);
12969 MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratchGPR);
12970 MacroAssembler::Label loop = m_jit.label();
12971 m_jit.sub32(TrustedImm32(1), scratchGPR);
12972 m_jit.storeValue(emptyValueRegs, MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight));
12973 m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit);
12974 done.link(&m_jit);
12975}
12976
12977void SpeculativeJIT::compileAllocateNewArrayWithSize(JSGlobalObject* globalObject, GPRReg resultGPR, GPRReg sizeGPR, IndexingType indexingType, bool shouldConvertLargeSizeToArrayStorage)
12978{
12979 GPRTemporary storage(this);
12980 GPRTemporary scratch(this);
12981 GPRTemporary scratch2(this);
12982
12983 GPRReg storageGPR = storage.gpr();
12984 GPRReg scratchGPR = scratch.gpr();
12985 GPRReg scratch2GPR = scratch2.gpr();
12986
12987 m_jit.move(TrustedImmPtr(nullptr), storageGPR);
12988
12989 MacroAssembler::JumpList slowCases;
12990 if (shouldConvertLargeSizeToArrayStorage)
12991 slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)));
12992#if !ASSERT_DISABLED
12993 else {
12994 MacroAssembler::Jump lengthIsWithinLimits;
12995 lengthIsWithinLimits = m_jit.branch32(MacroAssembler::Below, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH));
12996 m_jit.abortWithReason(UncheckedOverflow);
12997 lengthIsWithinLimits.link(&m_jit);
12998 }
12999#endif
13000
13001 // We can use resultGPR as a scratch right now.
13002 emitAllocateButterfly(storageGPR, sizeGPR, scratchGPR, scratch2GPR, resultGPR, slowCases);
13003
13004#if USE(JSVALUE64)
13005 JSValueRegs emptyValueRegs(scratchGPR);
13006 if (hasDouble(indexingType))
13007 m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), emptyValueRegs.gpr());
13008 else
13009 m_jit.move(TrustedImm64(JSValue::encode(JSValue())), emptyValueRegs.gpr());
13010#else
13011 JSValueRegs emptyValueRegs(scratchGPR, scratch2GPR);
13012 if (hasDouble(indexingType))
13013 m_jit.moveValue(JSValue(JSValue::EncodeAsDouble, PNaN), emptyValueRegs);
13014 else
13015 m_jit.moveValue(JSValue(), emptyValueRegs);
13016#endif
13017 emitInitializeButterfly(storageGPR, sizeGPR, emptyValueRegs, resultGPR);
13018
13019 RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType));
13020
13021 emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases);
13022
13023 m_jit.mutatorFence(*m_jit.vm());
13024
13025 addSlowPathGenerator(std::make_unique<CallArrayAllocatorWithVariableSizeSlowPathGenerator>(
13026 slowCases, this, operationNewArrayWithSize, resultGPR,
13027 structure,
13028 shouldConvertLargeSizeToArrayStorage ? m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)) : structure,
13029 sizeGPR, storageGPR));
13030}
13031
13032void SpeculativeJIT::compileHasIndexedProperty(Node* node)
13033{
13034 SpeculateCellOperand base(this, m_graph.varArgChild(node, 0));
13035 SpeculateStrictInt32Operand index(this, m_graph.varArgChild(node, 1));
13036 GPRTemporary result(this);
13037
13038 GPRReg baseGPR = base.gpr();
13039 GPRReg indexGPR = index.gpr();
13040 GPRReg resultGPR = result.gpr();
13041
13042 MacroAssembler::JumpList slowCases;
13043 ArrayMode mode = node->arrayMode();
13044 switch (mode.type()) {
13045 case Array::Int32:
13046 case Array::Contiguous: {
13047 ASSERT(!!m_graph.varArgChild(node, 2));
13048 StorageOperand storage(this, m_graph.varArgChild(node, 2));
13049 GPRTemporary scratch(this);
13050
13051 GPRReg storageGPR = storage.gpr();
13052 GPRReg scratchGPR = scratch.gpr();
13053
13054 MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
13055 if (mode.isInBounds())
13056 speculationCheck(OutOfBounds, JSValueRegs(), nullptr, outOfBounds);
13057 else
13058 slowCases.append(outOfBounds);
13059
13060#if USE(JSVALUE64)
13061 m_jit.load64(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchGPR);
13062 slowCases.append(m_jit.branchIfEmpty(scratchGPR));
13063#else
13064 m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), scratchGPR);
13065 slowCases.append(m_jit.branchIfEmpty(scratchGPR));
13066#endif
13067 m_jit.move(TrustedImm32(1), resultGPR);
13068 break;
13069 }
13070 case Array::Double: {
13071 ASSERT(!!m_graph.varArgChild(node, 2));
13072 StorageOperand storage(this, m_graph.varArgChild(node, 2));
13073 FPRTemporary scratch(this);
13074 FPRReg scratchFPR = scratch.fpr();
13075 GPRReg storageGPR = storage.gpr();
13076
13077 MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
13078 if (mode.isInBounds())
13079 speculationCheck(OutOfBounds, JSValueRegs(), nullptr, outOfBounds);
13080 else
13081 slowCases.append(outOfBounds);
13082
13083 m_jit.loadDouble(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchFPR);
13084 slowCases.append(m_jit.branchIfNaN(scratchFPR));
13085 m_jit.move(TrustedImm32(1), resultGPR);
13086 break;
13087 }
13088 case Array::ArrayStorage: {
13089 ASSERT(!!m_graph.varArgChild(node, 2));
13090 StorageOperand storage(this, m_graph.varArgChild(node, 2));
13091 GPRTemporary scratch(this);
13092
13093 GPRReg storageGPR = storage.gpr();
13094 GPRReg scratchGPR = scratch.gpr();
13095
13096 MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
13097 if (mode.isInBounds())
13098 speculationCheck(OutOfBounds, JSValueRegs(), nullptr, outOfBounds);
13099 else
13100 slowCases.append(outOfBounds);
13101
13102#if USE(JSVALUE64)
13103 m_jit.load64(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, ArrayStorage::vectorOffset()), scratchGPR);
13104 slowCases.append(m_jit.branchIfEmpty(scratchGPR));
13105#else
13106 m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, ArrayStorage::vectorOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), scratchGPR);
13107 slowCases.append(m_jit.branchIfEmpty(scratchGPR));
13108#endif
13109 m_jit.move(TrustedImm32(1), resultGPR);
13110 break;
13111 }
13112 default: {
13113 slowCases.append(m_jit.jump());
13114 break;
13115 }
13116 }
13117
13118 addSlowPathGenerator(slowPathCall(slowCases, this, operationHasIndexedPropertyByInt, resultGPR, baseGPR, indexGPR, static_cast<int32_t>(node->internalMethodType())));
13119
13120 unblessedBooleanResult(resultGPR, node);
13121}
13122
13123void SpeculativeJIT::compileGetDirectPname(Node* node)
13124{
13125 Edge& baseEdge = m_jit.graph().varArgChild(node, 0);
13126 Edge& propertyEdge = m_jit.graph().varArgChild(node, 1);
13127 Edge& indexEdge = m_jit.graph().varArgChild(node, 2);
13128
13129 SpeculateCellOperand base(this, baseEdge);
13130 SpeculateCellOperand property(this, propertyEdge);
13131 GPRReg baseGPR = base.gpr();
13132 GPRReg propertyGPR = property.gpr();
13133
13134#if CPU(X86)
13135 // Not enough registers on X86 for this code, so always use the slow path.
13136 speculate(node, indexEdge);
13137 flushRegisters();
13138 JSValueRegsFlushedCallResult result(this);
13139 JSValueRegs resultRegs = result.regs();
13140 callOperation(operationGetByValCell, resultRegs, baseGPR, CCallHelpers::CellValue(propertyGPR));
13141 m_jit.exceptionCheck();
13142 jsValueResult(resultRegs, node);
13143#else
13144 Edge& enumeratorEdge = m_jit.graph().varArgChild(node, 3);
13145 SpeculateStrictInt32Operand index(this, indexEdge);
13146 SpeculateCellOperand enumerator(this, enumeratorEdge);
13147 GPRTemporary scratch(this);
13148 JSValueRegsTemporary result(this);
13149
13150 GPRReg indexGPR = index.gpr();
13151 GPRReg enumeratorGPR = enumerator.gpr();
13152 GPRReg scratchGPR = scratch.gpr();
13153 JSValueRegs resultRegs = result.regs();
13154
13155 MacroAssembler::JumpList slowPath;
13156
13157 // Check the structure
13158 m_jit.load32(MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), scratchGPR);
13159 slowPath.append(
13160 m_jit.branch32(
13161 MacroAssembler::NotEqual,
13162 scratchGPR,
13163 MacroAssembler::Address(
13164 enumeratorGPR, JSPropertyNameEnumerator::cachedStructureIDOffset())));
13165
13166 // Compute the offset
13167 // If index is less than the enumerator's cached inline storage, then it's an inline access
13168 MacroAssembler::Jump outOfLineAccess = m_jit.branch32(MacroAssembler::AboveOrEqual,
13169 indexGPR, MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset()));
13170
13171 m_jit.loadValue(MacroAssembler::BaseIndex(baseGPR, indexGPR, MacroAssembler::TimesEight, JSObject::offsetOfInlineStorage()), resultRegs);
13172
13173 MacroAssembler::Jump done = m_jit.jump();
13174
13175 // Otherwise it's out of line
13176 outOfLineAccess.link(&m_jit);
13177 m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), resultRegs.payloadGPR());
13178 m_jit.move(indexGPR, scratchGPR);
13179 m_jit.sub32(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset()), scratchGPR);
13180 m_jit.neg32(scratchGPR);
13181 m_jit.signExtend32ToPtr(scratchGPR, scratchGPR);
13182 int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue);
13183 m_jit.loadValue(MacroAssembler::BaseIndex(resultRegs.payloadGPR(), scratchGPR, MacroAssembler::TimesEight, offsetOfFirstProperty), resultRegs);
13184
13185 done.link(&m_jit);
13186
13187 addSlowPathGenerator(slowPathCall(slowPath, this, operationGetByValCell, resultRegs, baseGPR, CCallHelpers::CellValue(propertyGPR)));
13188
13189 jsValueResult(resultRegs, node);
13190#endif
13191}
13192
13193void SpeculativeJIT::compileExtractCatchLocal(Node* node)
13194{
13195 JSValueRegsTemporary result(this);
13196 JSValueRegs resultRegs = result.regs();
13197
13198 JSValue* ptr = &reinterpret_cast<JSValue*>(m_jit.jitCode()->common.catchOSREntryBuffer->dataBuffer())[node->catchOSREntryIndex()];
13199 m_jit.loadValue(ptr, resultRegs);
13200 jsValueResult(resultRegs, node);
13201}
13202
13203void SpeculativeJIT::compileClearCatchLocals(Node* node)
13204{
13205 ScratchBuffer* scratchBuffer = m_jit.jitCode()->common.catchOSREntryBuffer;
13206 ASSERT(scratchBuffer);
13207 GPRTemporary scratch(this);
13208 GPRReg scratchGPR = scratch.gpr();
13209 m_jit.move(TrustedImmPtr(scratchBuffer->addressOfActiveLength()), scratchGPR);
13210 m_jit.storePtr(TrustedImmPtr(nullptr), scratchGPR);
13211 noResult(node);
13212}
13213
13214void SpeculativeJIT::compileProfileType(Node* node)
13215{
13216 JSValueOperand value(this, node->child1());
13217 GPRTemporary scratch1(this);
13218 GPRTemporary scratch2(this);
13219 GPRTemporary scratch3(this);
13220
13221 JSValueRegs valueRegs = value.jsValueRegs();
13222 GPRReg scratch1GPR = scratch1.gpr();
13223 GPRReg scratch2GPR = scratch2.gpr();
13224 GPRReg scratch3GPR = scratch3.gpr();
13225
13226 MacroAssembler::JumpList jumpToEnd;
13227
13228 jumpToEnd.append(m_jit.branchIfEmpty(valueRegs));
13229
13230 TypeLocation* cachedTypeLocation = node->typeLocation();
13231 // Compile in a predictive type check, if possible, to see if we can skip writing to the log.
13232 // These typechecks are inlined to match those of the 64-bit JSValue type checks.
13233 if (cachedTypeLocation->m_lastSeenType == TypeUndefined)
13234 jumpToEnd.append(m_jit.branchIfUndefined(valueRegs));
13235 else if (cachedTypeLocation->m_lastSeenType == TypeNull)
13236 jumpToEnd.append(m_jit.branchIfNull(valueRegs));
13237 else if (cachedTypeLocation->m_lastSeenType == TypeBoolean)
13238 jumpToEnd.append(m_jit.branchIfBoolean(valueRegs, scratch1GPR));
13239 else if (cachedTypeLocation->m_lastSeenType == TypeAnyInt)
13240 jumpToEnd.append(m_jit.branchIfInt32(valueRegs));
13241 else if (cachedTypeLocation->m_lastSeenType == TypeNumber)
13242 jumpToEnd.append(m_jit.branchIfNumber(valueRegs, scratch1GPR));
13243 else if (cachedTypeLocation->m_lastSeenType == TypeString) {
13244 MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(valueRegs);
13245 jumpToEnd.append(m_jit.branchIfString(valueRegs.payloadGPR()));
13246 isNotCell.link(&m_jit);
13247 }
13248
13249 // Load the TypeProfilerLog into Scratch2.
13250 TypeProfilerLog* cachedTypeProfilerLog = m_jit.vm()->typeProfilerLog();
13251 m_jit.move(TrustedImmPtr(cachedTypeProfilerLog), scratch2GPR);
13252
13253 // Load the next LogEntry into Scratch1.
13254 m_jit.loadPtr(MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset()), scratch1GPR);
13255
13256 // Store the JSValue onto the log entry.
13257 m_jit.storeValue(valueRegs, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::valueOffset()));
13258
13259 // Store the structureID of the cell if valueRegs is a cell, otherwise, store 0 on the log entry.
13260 MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(valueRegs);
13261 m_jit.load32(MacroAssembler::Address(valueRegs.payloadGPR(), JSCell::structureIDOffset()), scratch3GPR);
13262 m_jit.store32(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset()));
13263 MacroAssembler::Jump skipIsCell = m_jit.jump();
13264 isNotCell.link(&m_jit);
13265 m_jit.store32(TrustedImm32(0), MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset()));
13266 skipIsCell.link(&m_jit);
13267
13268 // Store the typeLocation on the log entry.
13269 m_jit.move(TrustedImmPtr(cachedTypeLocation), scratch3GPR);
13270 m_jit.storePtr(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::locationOffset()));
13271
13272 // Increment the current log entry.
13273 m_jit.addPtr(TrustedImm32(sizeof(TypeProfilerLog::LogEntry)), scratch1GPR);
13274 m_jit.storePtr(scratch1GPR, MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset()));
13275 MacroAssembler::Jump clearLog = m_jit.branchPtr(MacroAssembler::Equal, scratch1GPR, TrustedImmPtr(cachedTypeProfilerLog->logEndPtr()));
13276 addSlowPathGenerator(
13277 slowPathCall(clearLog, this, operationProcessTypeProfilerLogDFG, NoResult));
13278
13279 jumpToEnd.link(&m_jit);
13280
13281 noResult(node);
13282}
13283
13284void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, JSValueRegs valueRegs, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode)
13285{
13286 RegisterSet usedRegisters = this->usedRegisters();
13287 if (spillMode == DontSpill) {
13288 // We've already flushed registers to the stack, we don't need to spill these.
13289 usedRegisters.set(baseGPR, false);
13290 usedRegisters.set(valueRegs, false);
13291 }
13292 CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size());
13293 JITPutByIdGenerator gen(
13294 m_jit.codeBlock(), codeOrigin, callSite, usedRegisters,
13295 JSValueRegs::payloadOnly(baseGPR), valueRegs,
13296 scratchGPR, m_jit.ecmaModeFor(codeOrigin), putKind);
13297
13298 gen.generateFastPath(m_jit);
13299
13300 JITCompiler::JumpList slowCases;
13301 if (slowPathTarget.isSet())
13302 slowCases.append(slowPathTarget);
13303 slowCases.append(gen.slowPathJump());
13304
13305 auto slowPath = slowPathCall(
13306 slowCases, this, gen.slowPathFunction(), NoResult, gen.stubInfo(), valueRegs,
13307 CCallHelpers::CellValue(baseGPR), identifierUID(identifierNumber));
13308
13309 m_jit.addPutById(gen, slowPath.get());
13310 addSlowPathGenerator(WTFMove(slowPath));
13311}
13312
13313void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
13314{
13315 ASSERT(node->isBinaryUseKind(UntypedUse));
13316 JSValueOperand arg1(this, node->child1());
13317 JSValueOperand arg2(this, node->child2());
13318
13319 JSValueRegs arg1Regs = arg1.jsValueRegs();
13320 JSValueRegs arg2Regs = arg2.jsValueRegs();
13321
13322 JITCompiler::JumpList slowPath;
13323
13324 if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
13325 GPRFlushedCallResult result(this);
13326 GPRReg resultGPR = result.gpr();
13327
13328 arg1.use();
13329 arg2.use();
13330
13331 flushRegisters();
13332 callOperation(helperFunction, resultGPR, arg1Regs, arg2Regs);
13333 m_jit.exceptionCheck();
13334
13335 unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
13336 return;
13337 }
13338
13339 GPRTemporary result(this, Reuse, arg1, TagWord);
13340 GPRReg resultGPR = result.gpr();
13341
13342 arg1.use();
13343 arg2.use();
13344
13345 if (!isKnownInteger(node->child1().node()))
13346 slowPath.append(m_jit.branchIfNotInt32(arg1Regs));
13347 if (!isKnownInteger(node->child2().node()))
13348 slowPath.append(m_jit.branchIfNotInt32(arg2Regs));
13349
13350 m_jit.compare32(cond, arg1Regs.payloadGPR(), arg2Regs.payloadGPR(), resultGPR);
13351
13352 if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node()))
13353 addSlowPathGenerator(slowPathCall(slowPath, this, helperFunction, resultGPR, arg1Regs, arg2Regs));
13354
13355 unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
13356}
13357
13358void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction)
13359{
13360 BasicBlock* taken = branchNode->branchData()->taken.block;
13361 BasicBlock* notTaken = branchNode->branchData()->notTaken.block;
13362
13363 JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero;
13364
13365 // The branch instruction will branch to the taken block.
13366 // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
13367 if (taken == nextBlock()) {
13368 cond = JITCompiler::invert(cond);
13369 callResultCondition = JITCompiler::Zero;
13370 BasicBlock* tmp = taken;
13371 taken = notTaken;
13372 notTaken = tmp;
13373 }
13374
13375 JSValueOperand arg1(this, node->child1());
13376 JSValueOperand arg2(this, node->child2());
13377 JSValueRegs arg1Regs = arg1.jsValueRegs();
13378 JSValueRegs arg2Regs = arg2.jsValueRegs();
13379
13380 JITCompiler::JumpList slowPath;
13381
13382 if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
13383 GPRFlushedCallResult result(this);
13384 GPRReg resultGPR = result.gpr();
13385
13386 arg1.use();
13387 arg2.use();
13388
13389 flushRegisters();
13390 callOperation(helperFunction, resultGPR, arg1Regs, arg2Regs);
13391 m_jit.exceptionCheck();
13392
13393 branchTest32(callResultCondition, resultGPR, taken);
13394 } else {
13395 GPRTemporary result(this, Reuse, arg2, TagWord);
13396 GPRReg resultGPR = result.gpr();
13397
13398 arg1.use();
13399 arg2.use();
13400
13401 if (!isKnownInteger(node->child1().node()))
13402 slowPath.append(m_jit.branchIfNotInt32(arg1Regs));
13403 if (!isKnownInteger(node->child2().node()))
13404 slowPath.append(m_jit.branchIfNotInt32(arg2Regs));
13405
13406 branch32(cond, arg1Regs.payloadGPR(), arg2Regs.payloadGPR(), taken);
13407
13408 if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
13409 jump(notTaken, ForceJump);
13410
13411 slowPath.link(&m_jit);
13412
13413 silentSpillAllRegisters(resultGPR);
13414 callOperation(helperFunction, resultGPR, arg1Regs, arg2Regs);
13415 silentFillAllRegisters();
13416 m_jit.exceptionCheck();
13417
13418 branchTest32(callResultCondition, resultGPR, taken);
13419 }
13420 }
13421
13422 jump(notTaken);
13423
13424 m_indexInBlock = m_block->size() - 1;
13425 m_currentNode = branchNode;
13426}
13427
13428void SpeculativeJIT::compileBigIntEquality(Node* node)
13429{
13430 // FIXME: [ESNext][BigInt] Create specialized version of strict equals for BigIntUse
13431 // https://bugs.webkit.org/show_bug.cgi?id=182895
13432 SpeculateCellOperand left(this, node->child1());
13433 SpeculateCellOperand right(this, node->child2());
13434 GPRTemporary result(this, Reuse, left);
13435 GPRReg leftGPR = left.gpr();
13436 GPRReg rightGPR = right.gpr();
13437 GPRReg resultGPR = result.gpr();
13438
13439 left.use();
13440 right.use();
13441
13442 speculateBigInt(node->child1(), leftGPR);
13443 speculateBigInt(node->child2(), rightGPR);
13444
13445 JITCompiler::Jump notEqualCase = m_jit.branchPtr(JITCompiler::NotEqual, leftGPR, rightGPR);
13446
13447 m_jit.move(JITCompiler::TrustedImm32(1), resultGPR);
13448
13449 JITCompiler::Jump done = m_jit.jump();
13450
13451 notEqualCase.link(&m_jit);
13452
13453 silentSpillAllRegisters(resultGPR);
13454 callOperation(operationCompareStrictEqCell, resultGPR, leftGPR, rightGPR);
13455 silentFillAllRegisters();
13456
13457 done.link(&m_jit);
13458
13459 unblessedBooleanResult(resultGPR, node, UseChildrenCalledExplicitly);
13460}
13461
13462void SpeculativeJIT::compileMakeRope(Node* node)
13463{
13464 ASSERT(node->child1().useKind() == KnownStringUse);
13465 ASSERT(node->child2().useKind() == KnownStringUse);
13466 ASSERT(!node->child3() || node->child3().useKind() == KnownStringUse);
13467
13468 SpeculateCellOperand op1(this, node->child1());
13469 SpeculateCellOperand op2(this, node->child2());
13470 SpeculateCellOperand op3(this, node->child3());
13471 GPRReg opGPRs[3];
13472 unsigned numOpGPRs;
13473 opGPRs[0] = op1.gpr();
13474 opGPRs[1] = op2.gpr();
13475 if (node->child3()) {
13476 opGPRs[2] = op3.gpr();
13477 numOpGPRs = 3;
13478 } else {
13479 opGPRs[2] = InvalidGPRReg;
13480 numOpGPRs = 2;
13481 }
13482
13483#if CPU(ADDRESS64)
13484 Edge edges[3] = {
13485 node->child1(),
13486 node->child2(),
13487 node->child3()
13488 };
13489
13490 GPRTemporary result(this);
13491 GPRTemporary allocator(this);
13492 GPRTemporary scratch(this);
13493 GPRTemporary scratch2(this);
13494 GPRReg resultGPR = result.gpr();
13495 GPRReg allocatorGPR = allocator.gpr();
13496 GPRReg scratchGPR = scratch.gpr();
13497 GPRReg scratch2GPR = scratch2.gpr();
13498
13499 CCallHelpers::JumpList slowPath;
13500 Allocator allocatorValue = allocatorForNonVirtualConcurrently<JSRopeString>(*m_jit.vm(), sizeof(JSRopeString), AllocatorForMode::AllocatorIfExists);
13501 emitAllocateJSCell(resultGPR, JITAllocator::constant(allocatorValue), allocatorGPR, TrustedImmPtr(m_jit.graph().registerStructure(m_jit.vm()->stringStructure.get())), scratchGPR, slowPath);
13502
13503 // This puts nullptr for the first fiber. It makes visitChildren safe even if this JSRopeString is discarded due to the speculation failure in the following path.
13504 m_jit.storePtr(TrustedImmPtr(JSString::isRopeInPointer), CCallHelpers::Address(resultGPR, JSRopeString::offsetOfFiber0()));
13505
13506 {
13507 if (JSString* string = edges[0]->dynamicCastConstant<JSString*>(*m_jit.vm())) {
13508 m_jit.move(TrustedImm32(string->is8Bit() ? StringImpl::flagIs8Bit() : 0), scratchGPR);
13509 m_jit.move(TrustedImm32(string->length()), allocatorGPR);
13510 } else {
13511 bool needsRopeCase = canBeRope(edges[0]);
13512 m_jit.loadPtr(CCallHelpers::Address(opGPRs[0], JSString::offsetOfValue()), scratch2GPR);
13513 CCallHelpers::Jump isRope;
13514 if (needsRopeCase)
13515 isRope = m_jit.branchIfRopeStringImpl(scratch2GPR);
13516
13517 m_jit.load32(CCallHelpers::Address(scratch2GPR, StringImpl::flagsOffset()), scratchGPR);
13518 m_jit.load32(CCallHelpers::Address(scratch2GPR, StringImpl::lengthMemoryOffset()), allocatorGPR);
13519
13520 if (needsRopeCase) {
13521 auto done = m_jit.jump();
13522
13523 isRope.link(&m_jit);
13524 m_jit.load32(CCallHelpers::Address(opGPRs[0], JSRopeString::offsetOfFlags()), scratchGPR);
13525 m_jit.load32(CCallHelpers::Address(opGPRs[0], JSRopeString::offsetOfLength()), allocatorGPR);
13526 done.link(&m_jit);
13527 }
13528 }
13529
13530 if (!ASSERT_DISABLED) {
13531 CCallHelpers::Jump ok = m_jit.branch32(
13532 CCallHelpers::GreaterThanOrEqual, allocatorGPR, TrustedImm32(0));
13533 m_jit.abortWithReason(DFGNegativeStringLength);
13534 ok.link(&m_jit);
13535 }
13536 }
13537
13538 for (unsigned i = 1; i < numOpGPRs; ++i) {
13539 if (JSString* string = edges[i]->dynamicCastConstant<JSString*>(*m_jit.vm())) {
13540 m_jit.and32(TrustedImm32(string->is8Bit() ? StringImpl::flagIs8Bit() : 0), scratchGPR);
13541 speculationCheck(
13542 Uncountable, JSValueSource(), nullptr,
13543 m_jit.branchAdd32(
13544 CCallHelpers::Overflow,
13545 TrustedImm32(string->length()), allocatorGPR));
13546 } else {
13547 bool needsRopeCase = canBeRope(edges[i]);
13548 m_jit.loadPtr(CCallHelpers::Address(opGPRs[i], JSString::offsetOfValue()), scratch2GPR);
13549 CCallHelpers::Jump isRope;
13550 if (needsRopeCase)
13551 isRope = m_jit.branchIfRopeStringImpl(scratch2GPR);
13552
13553 m_jit.and32(CCallHelpers::Address(scratch2GPR, StringImpl::flagsOffset()), scratchGPR);
13554 speculationCheck(
13555 Uncountable, JSValueSource(), nullptr,
13556 m_jit.branchAdd32(
13557 CCallHelpers::Overflow,
13558 CCallHelpers::Address(scratch2GPR, StringImpl::lengthMemoryOffset()), allocatorGPR));
13559 if (needsRopeCase) {
13560 auto done = m_jit.jump();
13561
13562 isRope.link(&m_jit);
13563 m_jit.and32(CCallHelpers::Address(opGPRs[i], JSRopeString::offsetOfFlags()), scratchGPR);
13564 m_jit.load32(CCallHelpers::Address(opGPRs[i], JSRopeString::offsetOfLength()), scratch2GPR);
13565 speculationCheck(
13566 Uncountable, JSValueSource(), nullptr,
13567 m_jit.branchAdd32(
13568 CCallHelpers::Overflow, scratch2GPR, allocatorGPR));
13569 done.link(&m_jit);
13570 }
13571 }
13572 }
13573
13574 if (!ASSERT_DISABLED) {
13575 CCallHelpers::Jump ok = m_jit.branch32(
13576 CCallHelpers::GreaterThanOrEqual, allocatorGPR, TrustedImm32(0));
13577 m_jit.abortWithReason(DFGNegativeStringLength);
13578 ok.link(&m_jit);
13579 }
13580
13581 static_assert(StringImpl::flagIs8Bit() == JSRopeString::is8BitInPointer, "");
13582 m_jit.and32(TrustedImm32(StringImpl::flagIs8Bit()), scratchGPR);
13583 m_jit.orPtr(opGPRs[0], scratchGPR);
13584 m_jit.orPtr(TrustedImmPtr(JSString::isRopeInPointer), scratchGPR);
13585 m_jit.storePtr(scratchGPR, CCallHelpers::Address(resultGPR, JSRopeString::offsetOfFiber0()));
13586
13587 m_jit.move(opGPRs[1], scratchGPR);
13588 m_jit.lshiftPtr(TrustedImm32(32), scratchGPR);
13589 m_jit.orPtr(allocatorGPR, scratchGPR);
13590 m_jit.storePtr(scratchGPR, CCallHelpers::Address(resultGPR, JSRopeString::offsetOfFiber1()));
13591
13592 if (numOpGPRs == 2) {
13593 m_jit.move(opGPRs[1], scratchGPR);
13594 m_jit.rshiftPtr(TrustedImm32(32), scratchGPR);
13595 m_jit.storePtr(scratchGPR, CCallHelpers::Address(resultGPR, JSRopeString::offsetOfFiber2()));
13596 } else {
13597 m_jit.move(opGPRs[1], scratchGPR);
13598 m_jit.rshiftPtr(TrustedImm32(32), scratchGPR);
13599 m_jit.move(opGPRs[2], scratch2GPR);
13600 m_jit.lshiftPtr(TrustedImm32(16), scratch2GPR);
13601 m_jit.orPtr(scratch2GPR, scratchGPR);
13602 m_jit.storePtr(scratchGPR, CCallHelpers::Address(resultGPR, JSRopeString::offsetOfFiber2()));
13603 }
13604
13605 auto isNonEmptyString = m_jit.branchTest32(CCallHelpers::NonZero, allocatorGPR);
13606
13607 m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), jsEmptyString(&m_jit.graph().m_vm)), resultGPR);
13608
13609 isNonEmptyString.link(&m_jit);
13610 m_jit.mutatorFence(*m_jit.vm());
13611
13612 switch (numOpGPRs) {
13613 case 2:
13614 addSlowPathGenerator(slowPathCall(
13615 slowPath, this, operationMakeRope2, resultGPR, opGPRs[0], opGPRs[1]));
13616 break;
13617 case 3:
13618 addSlowPathGenerator(slowPathCall(
13619 slowPath, this, operationMakeRope3, resultGPR, opGPRs[0], opGPRs[1], opGPRs[2]));
13620 break;
13621 default:
13622 RELEASE_ASSERT_NOT_REACHED();
13623 break;
13624 }
13625
13626 cellResult(resultGPR, node);
13627#else
13628 flushRegisters();
13629 GPRFlushedCallResult result(this);
13630 GPRReg resultGPR = result.gpr();
13631 switch (numOpGPRs) {
13632 case 2:
13633 callOperation(operationMakeRope2, resultGPR, opGPRs[0], opGPRs[1]);
13634 m_jit.exceptionCheck();
13635 break;
13636 case 3:
13637 callOperation(operationMakeRope3, resultGPR, opGPRs[0], opGPRs[1], opGPRs[2]);
13638 m_jit.exceptionCheck();
13639 break;
13640 default:
13641 RELEASE_ASSERT_NOT_REACHED();
13642 break;
13643 }
13644
13645 cellResult(resultGPR, node);
13646#endif
13647}
13648
13649} } // namespace JSC::DFG
13650
13651#endif
13652