1/*
2 * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "WasmMemory.h"
28#include "WasmInstance.h"
29
30#if ENABLE(WEBASSEMBLY)
31
32#include "Options.h"
33#include <wtf/DataLog.h>
34#include <wtf/Gigacage.h>
35#include <wtf/Lock.h>
36#include <wtf/OSAllocator.h>
37#include <wtf/PageBlock.h>
38#include <wtf/Platform.h>
39#include <wtf/PrintStream.h>
40#include <wtf/RAMSize.h>
41#include <wtf/Vector.h>
42
43#include <cstring>
44#include <mutex>
45
46namespace JSC { namespace Wasm {
47
48// FIXME: We could be smarter about memset / mmap / madvise. https://bugs.webkit.org/show_bug.cgi?id=170343
49// FIXME: Give up some of the cached fast memories if the GC determines it's easy to get them back, and they haven't been used in a while. https://bugs.webkit.org/show_bug.cgi?id=170773
50// FIXME: Limit slow memory size. https://bugs.webkit.org/show_bug.cgi?id=170825
51
52namespace {
53
54constexpr bool verbose = false;
55
56NEVER_INLINE NO_RETURN_DUE_TO_CRASH void webAssemblyCouldntGetFastMemory() { CRASH(); }
57
58struct MemoryResult {
59 enum Kind {
60 Success,
61 SuccessAndNotifyMemoryPressure,
62 SyncTryToReclaimMemory
63 };
64
65 static const char* toString(Kind kind)
66 {
67 switch (kind) {
68 case Success:
69 return "Success";
70 case SuccessAndNotifyMemoryPressure:
71 return "SuccessAndNotifyMemoryPressure";
72 case SyncTryToReclaimMemory:
73 return "SyncTryToReclaimMemory";
74 }
75 RELEASE_ASSERT_NOT_REACHED();
76 return nullptr;
77 }
78
79 MemoryResult() { }
80
81 MemoryResult(void* basePtr, Kind kind)
82 : basePtr(basePtr)
83 , kind(kind)
84 {
85 }
86
87 void dump(PrintStream& out) const
88 {
89 out.print("{basePtr = ", RawPointer(basePtr), ", kind = ", toString(kind), "}");
90 }
91
92 void* basePtr;
93 Kind kind;
94};
95
96class MemoryManager {
97public:
98 MemoryManager()
99 : m_maxFastMemoryCount(Options::maxNumWebAssemblyFastMemories())
100 {
101 }
102
103 MemoryResult tryAllocateFastMemory()
104 {
105 MemoryResult result = [&] {
106 auto holder = holdLock(m_lock);
107 if (m_fastMemories.size() >= m_maxFastMemoryCount)
108 return MemoryResult(nullptr, MemoryResult::SyncTryToReclaimMemory);
109
110 void* result = Gigacage::tryAllocateZeroedVirtualPages(Gigacage::Primitive, Memory::fastMappedBytes());
111 if (!result)
112 return MemoryResult(nullptr, MemoryResult::SyncTryToReclaimMemory);
113
114 m_fastMemories.append(result);
115
116 return MemoryResult(
117 result,
118 m_fastMemories.size() >= m_maxFastMemoryCount / 2 ? MemoryResult::SuccessAndNotifyMemoryPressure : MemoryResult::Success);
119 }();
120
121 if (Options::logWebAssemblyMemory())
122 dataLog("Allocated virtual: ", result, "; state: ", *this, "\n");
123
124 return result;
125 }
126
127 void freeFastMemory(void* basePtr)
128 {
129 {
130 auto holder = holdLock(m_lock);
131 Gigacage::freeVirtualPages(Gigacage::Primitive, basePtr, Memory::fastMappedBytes());
132 m_fastMemories.removeFirst(basePtr);
133 }
134
135 if (Options::logWebAssemblyMemory())
136 dataLog("Freed virtual; state: ", *this, "\n");
137 }
138
139 bool isAddressInFastMemory(void* address)
140 {
141 // NOTE: This can be called from a signal handler, but only after we proved that we're in JIT code.
142 auto holder = holdLock(m_lock);
143 for (void* memory : m_fastMemories) {
144 char* start = static_cast<char*>(memory);
145 if (start <= address && address <= start + Memory::fastMappedBytes())
146 return true;
147 }
148 return false;
149 }
150
151 // We allow people to "commit" more wasm memory than there is on the system since most of the time
152 // people don't actually write to most of that memory. There is some chance that this gets us
153 // JetSammed but that's possible anyway.
154 inline size_t memoryLimit() const { return ramSize() * 3; }
155
156 // FIXME: Ideally, bmalloc would have this kind of mechanism. Then, we would just forward to that
157 // mechanism here.
158 MemoryResult::Kind tryAllocatePhysicalBytes(size_t bytes)
159 {
160 MemoryResult::Kind result = [&] {
161 auto holder = holdLock(m_lock);
162 if (m_physicalBytes + bytes > memoryLimit())
163 return MemoryResult::SyncTryToReclaimMemory;
164
165 m_physicalBytes += bytes;
166
167 if (m_physicalBytes >= memoryLimit() / 2)
168 return MemoryResult::SuccessAndNotifyMemoryPressure;
169
170 return MemoryResult::Success;
171 }();
172
173 if (Options::logWebAssemblyMemory())
174 dataLog("Allocated physical: ", bytes, ", ", MemoryResult::toString(result), "; state: ", *this, "\n");
175
176 return result;
177 }
178
179 void freePhysicalBytes(size_t bytes)
180 {
181 {
182 auto holder = holdLock(m_lock);
183 m_physicalBytes -= bytes;
184 }
185
186 if (Options::logWebAssemblyMemory())
187 dataLog("Freed physical: ", bytes, "; state: ", *this, "\n");
188 }
189
190 void dump(PrintStream& out) const
191 {
192 out.print("fast memories = ", m_fastMemories.size(), "/", m_maxFastMemoryCount, ", bytes = ", m_physicalBytes, "/", memoryLimit());
193 }
194
195private:
196 Lock m_lock;
197 unsigned m_maxFastMemoryCount { 0 };
198 Vector<void*> m_fastMemories;
199 size_t m_physicalBytes { 0 };
200};
201
202static MemoryManager& memoryManager()
203{
204 static std::once_flag onceFlag;
205 static MemoryManager* manager;
206 std::call_once(
207 onceFlag,
208 [] {
209 manager = new MemoryManager();
210 });
211 return *manager;
212}
213
214template<typename Func>
215bool tryAllocate(const Func& allocate, const WTF::Function<void(Memory::NotifyPressure)>& notifyMemoryPressure, const WTF::Function<void(Memory::SyncTryToReclaim)>& syncTryToReclaimMemory)
216{
217 unsigned numTries = 2;
218 bool done = false;
219 for (unsigned i = 0; i < numTries && !done; ++i) {
220 switch (allocate()) {
221 case MemoryResult::Success:
222 done = true;
223 break;
224 case MemoryResult::SuccessAndNotifyMemoryPressure:
225 if (notifyMemoryPressure)
226 notifyMemoryPressure(Memory::NotifyPressureTag);
227 done = true;
228 break;
229 case MemoryResult::SyncTryToReclaimMemory:
230 if (i + 1 == numTries)
231 break;
232 if (syncTryToReclaimMemory)
233 syncTryToReclaimMemory(Memory::SyncTryToReclaimTag);
234 break;
235 }
236 }
237 return done;
238}
239
240} // anonymous namespace
241
242Memory::Memory()
243{
244}
245
246Memory::Memory(PageCount initial, PageCount maximum, Function<void(NotifyPressure)>&& notifyMemoryPressure, Function<void(SyncTryToReclaim)>&& syncTryToReclaimMemory, WTF::Function<void(GrowSuccess, PageCount, PageCount)>&& growSuccessCallback)
247 : m_initial(initial)
248 , m_maximum(maximum)
249 , m_notifyMemoryPressure(WTFMove(notifyMemoryPressure))
250 , m_syncTryToReclaimMemory(WTFMove(syncTryToReclaimMemory))
251 , m_growSuccessCallback(WTFMove(growSuccessCallback))
252{
253 ASSERT(!initial.bytes());
254 ASSERT(m_mode == MemoryMode::BoundsChecking);
255 dataLogLnIf(verbose, "Memory::Memory allocating ", *this);
256}
257
258Memory::Memory(void* memory, PageCount initial, PageCount maximum, size_t mappedCapacity, MemoryMode mode, Function<void(NotifyPressure)>&& notifyMemoryPressure, Function<void(SyncTryToReclaim)>&& syncTryToReclaimMemory, WTF::Function<void(GrowSuccess, PageCount, PageCount)>&& growSuccessCallback)
259 : m_memory(memory)
260 , m_size(initial.bytes())
261 , m_initial(initial)
262 , m_maximum(maximum)
263 , m_mappedCapacity(mappedCapacity)
264 , m_mode(mode)
265 , m_notifyMemoryPressure(WTFMove(notifyMemoryPressure))
266 , m_syncTryToReclaimMemory(WTFMove(syncTryToReclaimMemory))
267 , m_growSuccessCallback(WTFMove(growSuccessCallback))
268{
269 dataLogLnIf(verbose, "Memory::Memory allocating ", *this);
270}
271
272Ref<Memory> Memory::create()
273{
274 return adoptRef(*new Memory());
275}
276
277RefPtr<Memory> Memory::tryCreate(PageCount initial, PageCount maximum, WTF::Function<void(NotifyPressure)>&& notifyMemoryPressure, WTF::Function<void(SyncTryToReclaim)>&& syncTryToReclaimMemory, WTF::Function<void(GrowSuccess, PageCount, PageCount)>&& growSuccessCallback)
278{
279 ASSERT(initial);
280 RELEASE_ASSERT(!maximum || maximum >= initial); // This should be guaranteed by our caller.
281
282 const size_t initialBytes = initial.bytes();
283 const size_t maximumBytes = maximum ? maximum.bytes() : 0;
284
285 if (initialBytes > MAX_ARRAY_BUFFER_SIZE)
286 return nullptr; // Client will throw OOMError.
287
288 if (maximum && !maximumBytes) {
289 // User specified a zero maximum, initial size must also be zero.
290 RELEASE_ASSERT(!initialBytes);
291 return adoptRef(new Memory(initial, maximum, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
292 }
293
294 bool done = tryAllocate(
295 [&] () -> MemoryResult::Kind {
296 return memoryManager().tryAllocatePhysicalBytes(initialBytes);
297 }, notifyMemoryPressure, syncTryToReclaimMemory);
298 if (!done)
299 return nullptr;
300
301 char* fastMemory = nullptr;
302 if (Options::useWebAssemblyFastMemory()) {
303 tryAllocate(
304 [&] () -> MemoryResult::Kind {
305 auto result = memoryManager().tryAllocateFastMemory();
306 fastMemory = bitwise_cast<char*>(result.basePtr);
307 return result.kind;
308 }, notifyMemoryPressure, syncTryToReclaimMemory);
309 }
310
311 if (fastMemory) {
312
313 if (mprotect(fastMemory + initialBytes, Memory::fastMappedBytes() - initialBytes, PROT_NONE)) {
314 dataLog("mprotect failed: ", strerror(errno), "\n");
315 RELEASE_ASSERT_NOT_REACHED();
316 }
317
318 return adoptRef(new Memory(fastMemory, initial, maximum, Memory::fastMappedBytes(), MemoryMode::Signaling, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
319 }
320
321 if (UNLIKELY(Options::crashIfWebAssemblyCantFastMemory()))
322 webAssemblyCouldntGetFastMemory();
323
324 if (!initialBytes)
325 return adoptRef(new Memory(initial, maximum, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
326
327 void* slowMemory = Gigacage::tryAllocateZeroedVirtualPages(Gigacage::Primitive, initialBytes);
328 if (!slowMemory) {
329 memoryManager().freePhysicalBytes(initialBytes);
330 return nullptr;
331 }
332 return adoptRef(new Memory(slowMemory, initial, maximum, initialBytes, MemoryMode::BoundsChecking, WTFMove(notifyMemoryPressure), WTFMove(syncTryToReclaimMemory), WTFMove(growSuccessCallback)));
333}
334
335Memory::~Memory()
336{
337 if (m_memory) {
338 memoryManager().freePhysicalBytes(m_size);
339 switch (m_mode) {
340 case MemoryMode::Signaling:
341 if (mprotect(m_memory, Memory::fastMappedBytes(), PROT_READ | PROT_WRITE)) {
342 dataLog("mprotect failed: ", strerror(errno), "\n");
343 RELEASE_ASSERT_NOT_REACHED();
344 }
345 memoryManager().freeFastMemory(m_memory);
346 break;
347 case MemoryMode::BoundsChecking:
348 Gigacage::freeVirtualPages(Gigacage::Primitive, m_memory, m_size);
349 break;
350 }
351 }
352}
353
354size_t Memory::fastMappedRedzoneBytes()
355{
356 return static_cast<size_t>(PageCount::pageSize) * Options::webAssemblyFastMemoryRedzonePages();
357}
358
359size_t Memory::fastMappedBytes()
360{
361 static_assert(sizeof(uint64_t) == sizeof(size_t), "We rely on allowing the maximum size of Memory we map to be 2^32 + redzone which is larger than fits in a 32-bit integer that we'd pass to mprotect if this didn't hold.");
362 return static_cast<size_t>(std::numeric_limits<uint32_t>::max()) + fastMappedRedzoneBytes();
363}
364
365bool Memory::addressIsInActiveFastMemory(void* address)
366{
367 return memoryManager().isAddressInFastMemory(address);
368}
369
370Expected<PageCount, Memory::GrowFailReason> Memory::grow(PageCount delta)
371{
372 const Wasm::PageCount oldPageCount = sizeInPages();
373
374 if (!delta.isValid())
375 return makeUnexpected(GrowFailReason::InvalidDelta);
376
377 const Wasm::PageCount newPageCount = oldPageCount + delta;
378 if (!newPageCount || !newPageCount.isValid())
379 return makeUnexpected(GrowFailReason::InvalidGrowSize);
380 if (newPageCount.bytes() > MAX_ARRAY_BUFFER_SIZE)
381 return makeUnexpected(GrowFailReason::OutOfMemory);
382
383 auto success = [&] () {
384 m_growSuccessCallback(GrowSuccessTag, oldPageCount, newPageCount);
385 // Update cache for instance
386 for (auto& instance : m_instances) {
387 if (instance.get() != nullptr)
388 instance.get()->updateCachedMemory();
389 }
390 return oldPageCount;
391 };
392
393 if (delta.pageCount() == 0)
394 return success();
395
396 dataLogLnIf(verbose, "Memory::grow(", delta, ") to ", newPageCount, " from ", *this);
397 RELEASE_ASSERT(newPageCount > PageCount::fromBytes(m_size));
398
399 if (maximum() && newPageCount > maximum())
400 return makeUnexpected(GrowFailReason::WouldExceedMaximum);
401
402 size_t desiredSize = newPageCount.bytes();
403 RELEASE_ASSERT(desiredSize <= MAX_ARRAY_BUFFER_SIZE);
404 RELEASE_ASSERT(desiredSize > m_size);
405 size_t extraBytes = desiredSize - m_size;
406 RELEASE_ASSERT(extraBytes);
407 bool allocationSuccess = tryAllocate(
408 [&] () -> MemoryResult::Kind {
409 return memoryManager().tryAllocatePhysicalBytes(extraBytes);
410 }, m_notifyMemoryPressure, m_syncTryToReclaimMemory);
411 if (!allocationSuccess)
412 return makeUnexpected(GrowFailReason::OutOfMemory);
413
414 switch (mode()) {
415 case MemoryMode::BoundsChecking: {
416 RELEASE_ASSERT(maximum().bytes() != 0);
417
418 void* newMemory = Gigacage::tryAllocateZeroedVirtualPages(Gigacage::Primitive, desiredSize);
419 if (!newMemory)
420 return makeUnexpected(GrowFailReason::OutOfMemory);
421
422 memcpy(newMemory, m_memory, m_size);
423 if (m_memory)
424 Gigacage::freeVirtualPages(Gigacage::Primitive, m_memory, m_size);
425 m_memory = newMemory;
426 m_mappedCapacity = desiredSize;
427 m_size = desiredSize;
428 return success();
429 }
430 case MemoryMode::Signaling: {
431 RELEASE_ASSERT(m_memory);
432 // Signaling memory must have been pre-allocated virtually.
433 uint8_t* startAddress = static_cast<uint8_t*>(m_memory) + m_size;
434
435 dataLogLnIf(verbose, "Marking WebAssembly memory's ", RawPointer(m_memory), " as read+write in range [", RawPointer(startAddress), ", ", RawPointer(startAddress + extraBytes), ")");
436 if (mprotect(startAddress, extraBytes, PROT_READ | PROT_WRITE)) {
437 dataLog("mprotect failed: ", strerror(errno), "\n");
438 RELEASE_ASSERT_NOT_REACHED();
439 }
440 m_size = desiredSize;
441 return success();
442 }
443 }
444
445 RELEASE_ASSERT_NOT_REACHED();
446 return oldPageCount;
447}
448
449void Memory::registerInstance(Instance* instance)
450{
451 size_t count = m_instances.size();
452 for (size_t index = 0; index < count; index++) {
453 if (m_instances.at(index).get() == nullptr) {
454 m_instances.at(index) = makeWeakPtr(*instance);
455 return;
456 }
457 }
458 m_instances.append(makeWeakPtr(*instance));
459}
460
461void Memory::dump(PrintStream& out) const
462{
463 out.print("Memory at ", RawPointer(m_memory), ", size ", m_size, "B capacity ", m_mappedCapacity, "B, initial ", m_initial, " maximum ", m_maximum, " mode ", makeString(m_mode));
464}
465
466} // namespace JSC
467
468} // namespace Wasm
469
470#endif // ENABLE(WEBASSEMBLY)
471