1/*
2 * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#pragma once
27
28#if ENABLE(JIT)
29
30#include "ArithProfile.h"
31#include "CCallHelpers.h"
32#include "JITAddGenerator.h"
33#include "JITMathICInlineResult.h"
34#include "JITMulGenerator.h"
35#include "JITNegGenerator.h"
36#include "JITSubGenerator.h"
37#include "LinkBuffer.h"
38#include "Repatch.h"
39
40namespace JSC {
41
42class LinkBuffer;
43
44struct MathICGenerationState {
45 MacroAssembler::Label fastPathStart;
46 MacroAssembler::Label fastPathEnd;
47 MacroAssembler::Label slowPathStart;
48 MacroAssembler::Call slowPathCall;
49 MacroAssembler::JumpList slowPathJumps;
50 bool shouldSlowPathRepatch;
51};
52
53#define ENABLE_MATH_IC_STATS 0
54
55template <typename GeneratorType, bool(*isProfileEmpty)(ArithProfile&)>
56class JITMathIC {
57 WTF_MAKE_FAST_ALLOCATED;
58public:
59 JITMathIC(ArithProfile* arithProfile, const Instruction* instruction)
60 : m_arithProfile(arithProfile)
61 , m_instruction(instruction)
62 {
63 }
64
65 CodeLocationLabel<JSInternalPtrTag> doneLocation() { return m_inlineEnd; }
66 CodeLocationCall<JSInternalPtrTag> slowPathCallLocation() { return m_slowPathCallLocation; }
67 CodeLocationLabel<JSInternalPtrTag> slowPathStartLocation() { return m_slowPathStartLocation; }
68
69 bool generateInline(CCallHelpers& jit, MathICGenerationState& state, bool shouldEmitProfiling = true)
70 {
71 state.fastPathStart = jit.label();
72 size_t startSize = jit.m_assembler.buffer().codeSize();
73
74 if (m_arithProfile) {
75 if (isProfileEmpty(*m_arithProfile)) {
76 // It looks like the MathIC has yet to execute. We don't want to emit code in this
77 // case for a couple reasons. First, the operation may never execute, so if we don't emit
78 // code, it's a win. Second, if the operation does execute, we can emit better code
79 // once we have an idea about the types.
80 state.slowPathJumps.append(jit.patchableJump());
81 size_t inlineSize = jit.m_assembler.buffer().codeSize() - startSize;
82 ASSERT_UNUSED(inlineSize, static_cast<ptrdiff_t>(inlineSize) <= MacroAssembler::patchableJumpSize());
83 state.shouldSlowPathRepatch = true;
84 state.fastPathEnd = jit.label();
85 ASSERT(!m_generateFastPathOnRepatch); // We should have gathered some observed type info about the types before trying to regenerate again.
86 m_generateFastPathOnRepatch = true;
87 return true;
88 }
89 }
90
91 JITMathICInlineResult result = m_generator.generateInline(jit, state, m_arithProfile);
92
93 switch (result) {
94 case JITMathICInlineResult::GeneratedFastPath: {
95 size_t inlineSize = jit.m_assembler.buffer().codeSize() - startSize;
96 if (static_cast<ptrdiff_t>(inlineSize) < MacroAssembler::patchableJumpSize()) {
97 size_t nopsToEmitInBytes = MacroAssembler::patchableJumpSize() - inlineSize;
98 jit.emitNops(nopsToEmitInBytes);
99 }
100 state.shouldSlowPathRepatch = true;
101 state.fastPathEnd = jit.label();
102 return true;
103 }
104 case JITMathICInlineResult::GenerateFullSnippet: {
105 MacroAssembler::JumpList endJumpList;
106 bool result = m_generator.generateFastPath(jit, endJumpList, state.slowPathJumps, m_arithProfile, shouldEmitProfiling);
107 if (result) {
108 state.fastPathEnd = jit.label();
109 state.shouldSlowPathRepatch = false;
110 endJumpList.link(&jit);
111 return true;
112 }
113 return false;
114 }
115 case JITMathICInlineResult::DontGenerate: {
116 return false;
117 }
118 default:
119 ASSERT_NOT_REACHED();
120 }
121
122 return false;
123 }
124
125 void generateOutOfLine(CodeBlock* codeBlock, FunctionPtr<CFunctionPtrTag> callReplacement)
126 {
127 auto linkJumpToOutOfLineSnippet = [&] () {
128 CCallHelpers jit(codeBlock);
129 auto jump = jit.jump();
130 // We don't need a nop sled here because nobody should be jumping into the middle of an IC.
131 bool needsBranchCompaction = false;
132 RELEASE_ASSERT(jit.m_assembler.buffer().codeSize() <= static_cast<size_t>(MacroAssembler::differenceBetweenCodePtr(m_inlineStart, m_inlineEnd)));
133 LinkBuffer linkBuffer(jit, m_inlineStart, jit.m_assembler.buffer().codeSize(), JITCompilationMustSucceed, needsBranchCompaction);
134 RELEASE_ASSERT(linkBuffer.isValid());
135 linkBuffer.link(jump, CodeLocationLabel<JITStubRoutinePtrTag>(m_code.code()));
136 FINALIZE_CODE(linkBuffer, NoPtrTag, "JITMathIC: linking constant jump to out of line stub");
137 };
138
139 auto replaceCall = [&] () {
140#if COMPILER(MSVC) && !COMPILER(CLANG)
141 ftlThunkAwareRepatchCall(codeBlock, slowPathCallLocation().retagged<JSInternalPtrTag>(), callReplacement);
142#else
143 ftlThunkAwareRepatchCall(codeBlock, slowPathCallLocation().template retagged<JSInternalPtrTag>(), callReplacement);
144#endif
145 };
146
147 bool shouldEmitProfiling = !JITCode::isOptimizingJIT(codeBlock->jitType());
148
149 if (m_generateFastPathOnRepatch) {
150
151 CCallHelpers jit(codeBlock);
152 MathICGenerationState generationState;
153 bool generatedInline = generateInline(jit, generationState, shouldEmitProfiling);
154
155 // We no longer want to try to regenerate the fast path.
156 m_generateFastPathOnRepatch = false;
157
158 if (generatedInline) {
159 auto jumpToDone = jit.jump();
160
161 LinkBuffer linkBuffer(jit, codeBlock, JITCompilationCanFail);
162 if (!linkBuffer.didFailToAllocate()) {
163 linkBuffer.link(generationState.slowPathJumps, slowPathStartLocation());
164 linkBuffer.link(jumpToDone, doneLocation());
165
166 m_code = FINALIZE_CODE_FOR(
167 codeBlock, linkBuffer, JITStubRoutinePtrTag, "JITMathIC: generating out of line fast IC snippet");
168
169 if (!generationState.shouldSlowPathRepatch) {
170 // We won't need to regenerate, so we can wire the slow path call
171 // to a non repatching variant.
172 replaceCall();
173 }
174
175 linkJumpToOutOfLineSnippet();
176
177 return;
178 }
179 }
180
181 // We weren't able to generate an out of line fast path.
182 // We just generate the snippet in its full generality.
183 }
184
185 // We rewire to the alternate regardless of whether or not we can allocate the out of line path
186 // because if we fail allocating the out of line path, we don't want to waste time trying to
187 // allocate it in the future.
188 replaceCall();
189
190 {
191 CCallHelpers jit(codeBlock);
192
193 MacroAssembler::JumpList endJumpList;
194 MacroAssembler::JumpList slowPathJumpList;
195
196 bool emittedFastPath = m_generator.generateFastPath(jit, endJumpList, slowPathJumpList, m_arithProfile, shouldEmitProfiling);
197 if (!emittedFastPath)
198 return;
199 endJumpList.append(jit.jump());
200
201 LinkBuffer linkBuffer(jit, codeBlock, JITCompilationCanFail);
202 if (linkBuffer.didFailToAllocate())
203 return;
204
205 linkBuffer.link(endJumpList, doneLocation());
206 linkBuffer.link(slowPathJumpList, slowPathStartLocation());
207
208 m_code = FINALIZE_CODE_FOR(
209 codeBlock, linkBuffer, JITStubRoutinePtrTag, "JITMathIC: generating out of line IC snippet");
210 }
211
212 linkJumpToOutOfLineSnippet();
213 }
214
215 void finalizeInlineCode(const MathICGenerationState& state, LinkBuffer& linkBuffer)
216 {
217 CodeLocationLabel<JSInternalPtrTag> start = linkBuffer.locationOf<JSInternalPtrTag>(state.fastPathStart);
218 m_inlineStart = start;
219
220 m_inlineEnd = linkBuffer.locationOf<JSInternalPtrTag>(state.fastPathEnd);
221 ASSERT(m_inlineEnd.untaggedExecutableAddress() > m_inlineStart.untaggedExecutableAddress());
222
223 m_slowPathCallLocation = linkBuffer.locationOf<JSInternalPtrTag>(state.slowPathCall);
224 m_slowPathStartLocation = linkBuffer.locationOf<JSInternalPtrTag>(state.slowPathStart);
225 }
226
227 ArithProfile* arithProfile() const { return m_arithProfile; }
228 const Instruction* instruction() const { return m_instruction; }
229
230#if ENABLE(MATH_IC_STATS)
231 size_t m_generatedCodeSize { 0 };
232 size_t codeSize() const
233 {
234 size_t result = m_generatedCodeSize;
235 if (m_code)
236 result += m_code.size();
237 return result;
238 }
239#endif
240
241 ArithProfile* m_arithProfile;
242 const Instruction* m_instruction;
243 MacroAssemblerCodeRef<JITStubRoutinePtrTag> m_code;
244 CodeLocationLabel<JSInternalPtrTag> m_inlineStart;
245 CodeLocationLabel<JSInternalPtrTag> m_inlineEnd;
246 CodeLocationLabel<JSInternalPtrTag> m_slowPathCallLocation;
247 CodeLocationLabel<JSInternalPtrTag> m_slowPathStartLocation;
248 bool m_generateFastPathOnRepatch { false };
249 GeneratorType m_generator;
250};
251
252inline bool isBinaryProfileEmpty(ArithProfile& arithProfile)
253{
254 return arithProfile.lhsObservedType().isEmpty() || arithProfile.rhsObservedType().isEmpty();
255}
256template <typename GeneratorType>
257class JITBinaryMathIC : public JITMathIC<GeneratorType, isBinaryProfileEmpty> {
258public:
259 JITBinaryMathIC(ArithProfile* arithProfile, const Instruction* instruction)
260 : JITMathIC<GeneratorType, isBinaryProfileEmpty>(arithProfile, instruction)
261 {
262 }
263};
264
265typedef JITBinaryMathIC<JITAddGenerator> JITAddIC;
266typedef JITBinaryMathIC<JITMulGenerator> JITMulIC;
267typedef JITBinaryMathIC<JITSubGenerator> JITSubIC;
268
269
270inline bool isUnaryProfileEmpty(ArithProfile& arithProfile)
271{
272 return arithProfile.lhsObservedType().isEmpty();
273}
274template <typename GeneratorType>
275class JITUnaryMathIC : public JITMathIC<GeneratorType, isUnaryProfileEmpty> {
276public:
277 JITUnaryMathIC(ArithProfile* arithProfile, const Instruction* instruction)
278 : JITMathIC<GeneratorType, isUnaryProfileEmpty>(arithProfile, instruction)
279 {
280 }
281};
282
283typedef JITUnaryMathIC<JITNegGenerator> JITNegIC;
284
285} // namespace JSC
286
287#endif // ENABLE(JIT)
288