1/*
2 * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmBBQPlan.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "B3Compilation.h"
32#include "WasmAirIRGenerator.h"
33#include "WasmB3IRGenerator.h"
34#include "WasmBinding.h"
35#include "WasmCallee.h"
36#include "WasmCallingConvention.h"
37#include "WasmFaultSignalHandler.h"
38#include "WasmMachineThreads.h"
39#include "WasmMemory.h"
40#include "WasmSignatureInlines.h"
41#include "WasmTierUpCount.h"
42#include "WasmValidate.h"
43#include <wtf/DataLog.h>
44#include <wtf/Locker.h>
45#include <wtf/MonotonicTime.h>
46#include <wtf/StdLibExtras.h>
47#include <wtf/SystemTracing.h>
48#include <wtf/text/StringConcatenateNumbers.h>
49
50namespace JSC { namespace Wasm {
51
52namespace WasmBBQPlanInternal {
53static constexpr bool verbose = false;
54}
55
56BBQPlan::BBQPlan(Context* context, Ref<ModuleInformation> moduleInformation, uint32_t functionIndex, CodeBlock* codeBlock, CompletionTask&& completionTask)
57 : EntryPlan(context, WTFMove(moduleInformation), WTFMove(completionTask))
58 , m_codeBlock(codeBlock)
59 , m_functionIndex(functionIndex)
60{
61 setMode(m_codeBlock->mode());
62}
63
64bool BBQPlan::prepareImpl()
65{
66 const auto& functions = m_moduleInformation->functions;
67 if (!tryReserveCapacity(m_wasmInternalFunctions, functions.size(), " WebAssembly functions")
68 || !tryReserveCapacity(m_compilationContexts, functions.size(), " compilation contexts")
69 || !tryReserveCapacity(m_tierUpCounts, functions.size(), " tier-up counts"))
70 return false;
71
72 m_wasmInternalFunctions.resize(functions.size());
73 m_compilationContexts.resize(functions.size());
74 m_tierUpCounts.resize(functions.size());
75
76 return true;
77}
78
79void BBQPlan::work(CompilationEffort effort)
80{
81 if (!m_codeBlock) {
82 Base::work(effort);
83 return;
84 }
85
86 CompilationContext context;
87 Vector<UnlinkedWasmToWasmCall> unlinkedWasmToWasmCalls;
88 std::unique_ptr<TierUpCount> tierUp = makeUnique<TierUpCount>();
89 std::unique_ptr<InternalFunction> function = compileFunction(m_functionIndex, context, unlinkedWasmToWasmCalls, tierUp.get());
90
91 LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
92 if (UNLIKELY(linkBuffer.didFailToAllocate())) {
93 Base::fail(holdLock(m_lock), makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
94 return;
95 }
96
97 size_t functionIndexSpace = m_functionIndex + m_moduleInformation->importFunctionCount();
98 SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[m_functionIndex];
99 const Signature& signature = SignatureInformation::get(signatureIndex);
100 function->entrypoint.compilation = makeUnique<B3::Compilation>(
101 FINALIZE_WASM_CODE_FOR_MODE(CompilationMode::BBQMode, linkBuffer, B3CompilationPtrTag, "WebAssembly BBQ function[%i] %s name %s", m_functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
102 WTFMove(context.wasmEntrypointByproducts));
103
104 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
105 {
106 Ref<BBQCallee> callee = BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(tierUp), WTFMove(unlinkedWasmToWasmCalls));
107 MacroAssembler::repatchPointer(function->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
108 ASSERT(!m_codeBlock->m_bbqCallees[m_functionIndex]);
109 entrypoint = callee->entrypoint();
110
111 // We want to make sure we publish our callee at the same time as we link our callsites. This enables us to ensure we
112 // always call the fastest code. Any function linked after us will see our new code and the new callsites, which they
113 // will update. It's also ok if they publish their code before we reset the instruction caches because after we release
114 // the lock our code is ready to be published too.
115 LockHolder holder(m_codeBlock->m_lock);
116 m_codeBlock->m_bbqCallees[m_functionIndex] = callee.copyRef();
117 {
118 LLIntCallee& llintCallee = *m_codeBlock->m_llintCallees[m_functionIndex];
119 auto locker = holdLock(llintCallee.tierUpCounter().m_lock);
120 llintCallee.setReplacement(callee.copyRef());
121 llintCallee.tierUpCounter().m_compilationStatus = LLIntTierUpCounter::CompilationStatus::Compiled;
122 }
123 for (auto& call : callee->wasmToWasmCallsites()) {
124 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
125 if (call.functionIndexSpace < m_moduleInformation->importFunctionCount())
126 entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
127 else
128 entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
129
130 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
131 }
132 }
133
134 // It's important to make sure we do this before we make any of the code we just compiled visible. If we didn't, we could end up
135 // where we are tiering up some function A to A' and we repatch some function B to call A' instead of A. Another CPU could see
136 // the updates to B but still not have reset its cache of A', which would lead to all kinds of badness.
137 resetInstructionCacheOnAllThreads();
138 WTF::storeStoreFence(); // This probably isn't necessary but it's good to be paranoid.
139
140 m_codeBlock->m_wasmIndirectCallEntryPoints[m_functionIndex] = entrypoint;
141 {
142 LockHolder holder(m_codeBlock->m_lock);
143
144 auto repatchCalls = [&] (const Vector<UnlinkedWasmToWasmCall>& callsites) {
145 for (auto& call : callsites) {
146 dataLogLnIf(WasmBBQPlanInternal::verbose, "Considering repatching call at: ", RawPointer(call.callLocation.dataLocation()), " that targets ", call.functionIndexSpace);
147 if (call.functionIndexSpace == functionIndexSpace) {
148 dataLogLnIf(WasmBBQPlanInternal::verbose, "Repatching call at: ", RawPointer(call.callLocation.dataLocation()), " to ", RawPointer(entrypoint.executableAddress()));
149 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
150 }
151 }
152 };
153
154 for (unsigned i = 0; i < m_codeBlock->m_wasmToWasmCallsites.size(); ++i) {
155 repatchCalls(m_codeBlock->m_wasmToWasmCallsites[i]);
156 if (LLIntCallee* llintCallee = m_codeBlock->m_llintCallees[i].get()) {
157 if (JITCallee* replacementCallee = llintCallee->replacement())
158 repatchCalls(replacementCallee->wasmToWasmCallsites());
159 if (OMGForOSREntryCallee* osrEntryCallee = llintCallee->osrEntryCallee())
160 repatchCalls(osrEntryCallee->wasmToWasmCallsites());
161 }
162 if (BBQCallee* bbqCallee = m_codeBlock->m_bbqCallees[i].get()) {
163 if (OMGCallee* replacementCallee = bbqCallee->replacement())
164 repatchCalls(replacementCallee->wasmToWasmCallsites());
165 if (OMGForOSREntryCallee* osrEntryCallee = bbqCallee->osrEntryCallee())
166 repatchCalls(osrEntryCallee->wasmToWasmCallsites());
167 }
168 }
169 }
170
171 dataLogLnIf(WasmBBQPlanInternal::verbose, "Finished BBQ ", m_functionIndex);
172
173
174 auto locker = holdLock(m_lock);
175 moveToState(State::Completed);
176 runCompletionTasks(locker);
177}
178
179void BBQPlan::compileFunction(uint32_t functionIndex)
180{
181 m_unlinkedWasmToWasmCalls[functionIndex] = Vector<UnlinkedWasmToWasmCall>();
182
183 if (Options::useBBQTierUpChecks())
184 m_tierUpCounts[functionIndex] = makeUnique<TierUpCount>();
185 else
186 m_tierUpCounts[functionIndex] = nullptr;
187
188 m_wasmInternalFunctions[functionIndex] = compileFunction(functionIndex, m_compilationContexts[functionIndex], m_unlinkedWasmToWasmCalls[functionIndex], m_tierUpCounts[functionIndex].get());
189
190 if (m_exportedFunctionIndices.contains(functionIndex) || m_moduleInformation->referencedFunctions().contains(functionIndex)) {
191 auto locker = holdLock(m_lock);
192 SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
193 const Signature& signature = SignatureInformation::get(signatureIndex);
194 auto result = m_embedderToWasmInternalFunctions.add(functionIndex, m_createEmbedderWrapper(*m_compilationContexts[functionIndex].embedderEntrypointJIT, signature, &m_unlinkedWasmToWasmCalls[functionIndex], m_moduleInformation.get(), m_mode, functionIndex));
195 ASSERT_UNUSED(result, result.isNewEntry);
196 }
197}
198
199std::unique_ptr<InternalFunction> BBQPlan::compileFunction(uint32_t functionIndex, CompilationContext& context, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, TierUpCount* tierUp)
200{
201 const auto& function = m_moduleInformation->functions[functionIndex];
202 SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
203 const Signature& signature = SignatureInformation::get(signatureIndex);
204 unsigned functionIndexSpace = m_moduleInformation->importFunctionCount() + functionIndex;
205 ASSERT_UNUSED(functionIndexSpace, m_moduleInformation->signatureIndexFromFunctionIndexSpace(functionIndexSpace) == signatureIndex);
206 ASSERT(validateFunction(function, signature, m_moduleInformation.get()));
207 Expected<std::unique_ptr<InternalFunction>, String> parseAndCompileResult;
208 unsigned osrEntryScratchBufferSize = 0;
209
210 // FIXME: Some webpages use very large Wasm module, and it exhausts all executable memory in ARM64 devices since the size of executable memory region is only limited to 128MB.
211 // The long term solution should be to introduce a Wasm interpreter. But as a short term solution, we introduce heuristics to switch back to BBQ B3 at the sacrifice of start-up time,
212 // as BBQ Air bloats such lengthy Wasm code and will consume a large amount of executable memory.
213 bool forceUsingB3 = false;
214 if (Options::webAssemblyBBQAirModeThreshold() && m_moduleInformation->codeSectionSize >= Options::webAssemblyBBQAirModeThreshold())
215 forceUsingB3 = true;
216
217 if (!forceUsingB3 && Options::wasmBBQUsesAir())
218 parseAndCompileResult = parseAndCompileAir(context, function, signature, unlinkedWasmToWasmCalls, m_moduleInformation.get(), m_mode, functionIndex, tierUp, m_throwWasmException);
219 else
220 parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedWasmToWasmCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::BBQMode, functionIndex, UINT32_MAX, tierUp, m_throwWasmException);
221
222 if (UNLIKELY(!parseAndCompileResult)) {
223 auto locker = holdLock(m_lock);
224 if (!m_errorMessage) {
225 // Multiple compiles could fail simultaneously. We arbitrarily choose the first.
226 fail(locker, makeString(parseAndCompileResult.error(), ", in function at index ", String::number(functionIndex))); // FIXME make this an Expected.
227 }
228 m_currentIndex = m_moduleInformation->functions.size();
229 return nullptr;
230 }
231
232 return WTFMove(*parseAndCompileResult);
233}
234
235void BBQPlan::didCompleteCompilation(const AbstractLocker& locker)
236{
237 for (uint32_t functionIndex = 0; functionIndex < m_moduleInformation->functions.size(); functionIndex++) {
238 CompilationContext& context = m_compilationContexts[functionIndex];
239 SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
240 const Signature& signature = SignatureInformation::get(signatureIndex);
241 const uint32_t functionIndexSpace = functionIndex + m_moduleInformation->importFunctionCount();
242 ASSERT(functionIndexSpace < m_moduleInformation->functionIndexSpaceSize());
243 {
244 LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
245 if (UNLIKELY(linkBuffer.didFailToAllocate())) {
246 Base::fail(locker, makeString("Out of executable memory in function at index ", String::number(functionIndex)));
247 return;
248 }
249
250 m_wasmInternalFunctions[functionIndex]->entrypoint.compilation = makeUnique<B3::Compilation>(
251 FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "WebAssembly BBQ function[%i] %s name %s", functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
252 WTFMove(context.wasmEntrypointByproducts));
253 }
254
255 if (const auto& embedderToWasmInternalFunction = m_embedderToWasmInternalFunctions.get(functionIndex)) {
256 LinkBuffer linkBuffer(*context.embedderEntrypointJIT, nullptr, JITCompilationCanFail);
257 if (UNLIKELY(linkBuffer.didFailToAllocate())) {
258 Base::fail(locker, makeString("Out of executable memory in function entrypoint at index ", String::number(functionIndex)));
259 return;
260 }
261
262 embedderToWasmInternalFunction->entrypoint.compilation = makeUnique<B3::Compilation>(
263 FINALIZE_CODE(linkBuffer, B3CompilationPtrTag, "Embedder->WebAssembly entrypoint[%i] %s name %s", functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
264 WTFMove(context.embedderEntrypointByproducts));
265 }
266 }
267
268 for (auto& unlinked : m_unlinkedWasmToWasmCalls) {
269 for (auto& call : unlinked) {
270 MacroAssemblerCodePtr<WasmEntryPtrTag> executableAddress;
271 if (m_moduleInformation->isImportedFunctionFromFunctionIndexSpace(call.functionIndexSpace)) {
272 // FIXME imports could have been linked in B3, instead of generating a patchpoint. This condition should be replaced by a RELEASE_ASSERT. https://bugs.webkit.org/show_bug.cgi?id=166462
273 executableAddress = m_wasmToWasmExitStubs.at(call.functionIndexSpace).code();
274 } else
275 executableAddress = m_wasmInternalFunctions.at(call.functionIndexSpace - m_moduleInformation->importFunctionCount())->entrypoint.compilation->code().retagged<WasmEntryPtrTag>();
276 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(executableAddress));
277 }
278 }
279}
280
281void BBQPlan::initializeCallees(const CalleeInitializer& callback)
282{
283 ASSERT(!failed());
284 for (unsigned internalFunctionIndex = 0; internalFunctionIndex < m_wasmInternalFunctions.size(); ++internalFunctionIndex) {
285
286 RefPtr<Wasm::Callee> embedderEntrypointCallee;
287 if (auto embedderToWasmFunction = m_embedderToWasmInternalFunctions.get(internalFunctionIndex)) {
288 embedderEntrypointCallee = Wasm::EmbedderEntrypointCallee::create(WTFMove(embedderToWasmFunction->entrypoint));
289 MacroAssembler::repatchPointer(embedderToWasmFunction->calleeMoveLocation, CalleeBits::boxWasm(embedderEntrypointCallee.get()));
290 }
291
292 InternalFunction* function = m_wasmInternalFunctions[internalFunctionIndex].get();
293 size_t functionIndexSpace = internalFunctionIndex + m_moduleInformation->importFunctionCount();
294 Ref<Wasm::Callee> wasmEntrypointCallee = Wasm::BBQCallee::create(WTFMove(function->entrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), WTFMove(m_tierUpCounts[internalFunctionIndex]), WTFMove(m_unlinkedWasmToWasmCalls[internalFunctionIndex]));
295 MacroAssembler::repatchPointer(function->calleeMoveLocation, CalleeBits::boxWasm(wasmEntrypointCallee.ptr()));
296
297 callback(internalFunctionIndex, WTFMove(embedderEntrypointCallee), WTFMove(wasmEntrypointCallee));
298 }
299}
300
301} } // namespace JSC::Wasm
302
303#endif // ENABLE(WEBASSEMBLY)
304