1/*
2 * Copyright (C) 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmOMGForOSREntryPlan.h"
28
29#if ENABLE(WEBASSEMBLY)
30
31#include "B3Compilation.h"
32#include "B3OpaqueByproducts.h"
33#include "JSCInlines.h"
34#include "LinkBuffer.h"
35#include "WasmB3IRGenerator.h"
36#include "WasmCallee.h"
37#include "WasmContext.h"
38#include "WasmInstance.h"
39#include "WasmMachineThreads.h"
40#include "WasmMemory.h"
41#include "WasmNameSection.h"
42#include "WasmSignatureInlines.h"
43#include "WasmValidate.h"
44#include "WasmWorklist.h"
45#include <wtf/DataLog.h>
46#include <wtf/Locker.h>
47#include <wtf/MonotonicTime.h>
48#include <wtf/StdLibExtras.h>
49#include <wtf/ThreadMessage.h>
50
51namespace JSC { namespace Wasm {
52
53namespace WasmOMGForOSREntryPlanInternal {
54static constexpr bool verbose = false;
55}
56
57OMGForOSREntryPlan::OMGForOSREntryPlan(Context* context, Ref<Module>&& module, Ref<Callee>&& callee, uint32_t functionIndex, uint32_t loopIndex, MemoryMode mode, CompletionTask&& task)
58 : Base(context, makeRef(const_cast<ModuleInformation&>(module->moduleInformation())), WTFMove(task))
59 , m_module(WTFMove(module))
60 , m_codeBlock(*m_module->codeBlockFor(mode))
61 , m_callee(WTFMove(callee))
62 , m_functionIndex(functionIndex)
63 , m_loopIndex(loopIndex)
64{
65 setMode(mode);
66 ASSERT(m_codeBlock->runnable());
67 ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(m_mode));
68 dataLogLnIf(WasmOMGForOSREntryPlanInternal::verbose, "Starting OMGForOSREntry plan for ", functionIndex, " of module: ", RawPointer(&m_module.get()));
69}
70
71void OMGForOSREntryPlan::work(CompilationEffort)
72{
73 ASSERT(m_codeBlock->runnable());
74 ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(mode()));
75 const FunctionData& function = m_moduleInformation->functions[m_functionIndex];
76
77 const uint32_t functionIndexSpace = m_functionIndex + m_module->moduleInformation().importFunctionCount();
78 ASSERT(functionIndexSpace < m_module->moduleInformation().functionIndexSpaceSize());
79
80 SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[m_functionIndex];
81 const Signature& signature = SignatureInformation::get(signatureIndex);
82 ASSERT(validateFunction(function, signature, m_moduleInformation.get()));
83
84 Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
85 CompilationContext context;
86 unsigned osrEntryScratchBufferSize = 0;
87 auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGForOSREntryMode, m_functionIndex, m_loopIndex);
88
89 if (UNLIKELY(!parseAndCompileResult)) {
90 fail(holdLock(m_lock), makeString(parseAndCompileResult.error(), "when trying to tier up ", String::number(m_functionIndex)));
91 return;
92 }
93
94 Entrypoint omgEntrypoint;
95 LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, JITCompilationCanFail);
96 if (UNLIKELY(linkBuffer.didFailToAllocate())) {
97 Base::fail(holdLock(m_lock), makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
98 return;
99 }
100
101 omgEntrypoint.compilation = makeUnique<B3::Compilation>(
102 FINALIZE_WASM_CODE_FOR_MODE(CompilationMode::OMGForOSREntryMode, linkBuffer, B3CompilationPtrTag, "WebAssembly OMGForOSREntry function[%i] %s name %s", m_functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
103 WTFMove(context.wasmEntrypointByproducts));
104
105 omgEntrypoint.calleeSaveRegisters = WTFMove(parseAndCompileResult.value()->entrypoint.calleeSaveRegisters);
106
107 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
108 ASSERT(m_codeBlock.ptr() == m_module->codeBlockFor(mode()));
109 Ref<OMGForOSREntryCallee> callee = OMGForOSREntryCallee::create(WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), osrEntryScratchBufferSize, m_loopIndex, WTFMove(unlinkedCalls));
110 {
111 MacroAssembler::repatchPointer(parseAndCompileResult.value()->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
112 entrypoint = callee->entrypoint();
113
114 auto locker = holdLock(m_codeBlock->m_lock);
115 for (auto& call : callee->wasmToWasmCallsites()) {
116 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
117 if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
118 entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
119 else
120 entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
121
122 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
123 }
124 }
125 resetInstructionCacheOnAllThreads();
126 WTF::storeStoreFence();
127 {
128 auto locker = holdLock(m_codeBlock->m_lock);
129 {
130 switch (m_callee->compilationMode()) {
131 case CompilationMode::LLIntMode: {
132 LLIntCallee* llintCallee = static_cast<LLIntCallee*>(m_callee.ptr());
133 auto locker = holdLock(llintCallee->tierUpCounter().m_lock);
134 llintCallee->setOSREntryCallee(callee.copyRef());
135 llintCallee->tierUpCounter().m_loopCompilationStatus = LLIntTierUpCounter::CompilationStatus::Compiled;
136 break;
137 }
138 case CompilationMode::BBQMode: {
139 BBQCallee* bbqCallee = static_cast<BBQCallee*>(m_callee.ptr());
140 auto locker = holdLock(bbqCallee->tierUpCount()->getLock());
141 bbqCallee->setOSREntryCallee(callee.copyRef());
142 bbqCallee->tierUpCount()->osrEntryTriggers()[m_loopIndex] = TierUpCount::TriggerReason::CompilationDone;
143 bbqCallee->tierUpCount()->m_compilationStatusForOMGForOSREntry = TierUpCount::CompilationStatus::Compiled;
144 break;
145 }
146 default:
147 RELEASE_ASSERT_NOT_REACHED();
148 }
149 }
150 WTF::storeStoreFence();
151 // It is possible that a new OMG callee is added while we release m_codeBlock->lock.
152 // Until we add OMGForOSREntry callee to BBQCallee's m_osrEntryCallee, this new OMG function linking does not happen for this OMGForOSREntry callee.
153 // We re-link this OMGForOSREntry callee again not to miss that chance.
154 for (auto& call : callee->wasmToWasmCallsites()) {
155 MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
156 if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
157 entrypoint = m_codeBlock->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
158 else
159 entrypoint = m_codeBlock->wasmEntrypointCalleeFromFunctionIndexSpace(call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
160
161 MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
162 }
163 }
164 dataLogLnIf(WasmOMGForOSREntryPlanInternal::verbose, "Finished OMGForOSREntry ", m_functionIndex);
165 complete(holdLock(m_lock));
166}
167
168} } // namespace JSC::Wasm
169
170#endif // ENABLE(WEBASSEMBLY)
171