1/*
2 * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "Scavenger.h"
27
28#include "AllIsoHeapsInlines.h"
29#include "AvailableMemory.h"
30#include "BulkDecommit.h"
31#include "Environment.h"
32#include "Heap.h"
33#if BOS(DARWIN)
34#import <dispatch/dispatch.h>
35#import <mach/host_info.h>
36#import <mach/mach.h>
37#import <mach/mach_error.h>
38#endif
39#include <stdio.h>
40#include <thread>
41
42namespace bmalloc {
43
44static constexpr bool verbose = false;
45
46struct PrintTime {
47 PrintTime(const char* str)
48 : string(str)
49 { }
50
51 ~PrintTime()
52 {
53 if (!printed)
54 print();
55 }
56 void print()
57 {
58 if (verbose) {
59 fprintf(stderr, "%s %lfms\n", string, static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start).count()) / 1000);
60 printed = true;
61 }
62 }
63 const char* string;
64 std::chrono::steady_clock::time_point start { std::chrono::steady_clock::now() };
65 bool printed { false };
66};
67
68DEFINE_STATIC_PER_PROCESS_STORAGE(Scavenger);
69
70Scavenger::Scavenger(std::lock_guard<Mutex>&)
71{
72 BASSERT(!Environment::get()->isDebugHeapEnabled());
73
74#if BOS(DARWIN)
75 auto queue = dispatch_queue_create("WebKit Malloc Memory Pressure Handler", DISPATCH_QUEUE_SERIAL);
76 m_pressureHandlerDispatchSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0, DISPATCH_MEMORYPRESSURE_CRITICAL, queue);
77 dispatch_source_set_event_handler(m_pressureHandlerDispatchSource, ^{
78 scavenge();
79 });
80 dispatch_resume(m_pressureHandlerDispatchSource);
81 dispatch_release(queue);
82#endif
83 m_waitTime = std::chrono::milliseconds(10);
84
85 m_thread = std::thread(&threadEntryPoint, this);
86}
87
88void Scavenger::run()
89{
90 std::lock_guard<Mutex> lock(mutex());
91 runHoldingLock();
92}
93
94void Scavenger::runHoldingLock()
95{
96 m_state = State::Run;
97 m_condition.notify_all();
98}
99
100void Scavenger::runSoon()
101{
102 std::lock_guard<Mutex> lock(mutex());
103 runSoonHoldingLock();
104}
105
106void Scavenger::runSoonHoldingLock()
107{
108 if (willRunSoon())
109 return;
110 m_state = State::RunSoon;
111 m_condition.notify_all();
112}
113
114void Scavenger::didStartGrowing()
115{
116 // We don't really need to lock here, since this is just a heuristic.
117 m_isProbablyGrowing = true;
118}
119
120void Scavenger::scheduleIfUnderMemoryPressure(size_t bytes)
121{
122 std::lock_guard<Mutex> lock(mutex());
123 scheduleIfUnderMemoryPressureHoldingLock(bytes);
124}
125
126void Scavenger::scheduleIfUnderMemoryPressureHoldingLock(size_t bytes)
127{
128 m_scavengerBytes += bytes;
129 if (m_scavengerBytes < scavengerBytesPerMemoryPressureCheck)
130 return;
131
132 m_scavengerBytes = 0;
133
134 if (willRun())
135 return;
136
137 if (!isUnderMemoryPressure())
138 return;
139
140 m_isProbablyGrowing = false;
141 runHoldingLock();
142}
143
144void Scavenger::schedule(size_t bytes)
145{
146 std::lock_guard<Mutex> lock(mutex());
147 scheduleIfUnderMemoryPressureHoldingLock(bytes);
148
149 if (willRunSoon())
150 return;
151
152 m_isProbablyGrowing = false;
153 runSoonHoldingLock();
154}
155
156inline void dumpStats()
157{
158 auto dump = [] (auto* string, auto size) {
159 fprintf(stderr, "%s %zuMB\n", string, static_cast<size_t>(size) / 1024 / 1024);
160 };
161
162#if BOS(DARWIN)
163 task_vm_info_data_t vmInfo;
164 mach_msg_type_number_t vmSize = TASK_VM_INFO_COUNT;
165 if (KERN_SUCCESS == task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)(&vmInfo), &vmSize)) {
166 dump("phys_footprint", vmInfo.phys_footprint);
167 dump("internal+compressed", vmInfo.internal + vmInfo.compressed);
168 }
169#endif
170
171 dump("bmalloc-freeable", Scavenger::get()->freeableMemory());
172 dump("bmalloc-footprint", Scavenger::get()->footprint());
173}
174
175std::chrono::milliseconds Scavenger::timeSinceLastFullScavenge()
176{
177 std::unique_lock<Mutex> lock(mutex());
178 return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - m_lastFullScavengeTime);
179}
180
181void Scavenger::enableMiniMode()
182{
183 m_isInMiniMode = true; // We just store to this racily. The scavenger thread will eventually pick up the right value.
184 if (m_state == State::RunSoon)
185 run();
186}
187
188void Scavenger::scavenge()
189{
190 std::unique_lock<Mutex> lock(m_scavengingMutex);
191
192 if (verbose) {
193 fprintf(stderr, "--------------------------------\n");
194 fprintf(stderr, "--before scavenging--\n");
195 dumpStats();
196 }
197
198 {
199 BulkDecommit decommitter;
200
201 {
202 PrintTime printTime("\nfull scavenge under lock time");
203 size_t deferredDecommits = 0;
204 std::lock_guard<Mutex> lock(Heap::mutex());
205 for (unsigned i = numHeaps; i--;) {
206 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
207 continue;
208 PerProcess<PerHeapKind<Heap>>::get()->at(i).scavenge(lock, decommitter, deferredDecommits);
209 }
210 decommitter.processEager();
211
212 if (deferredDecommits)
213 m_state = State::RunSoon;
214 }
215
216 {
217 PrintTime printTime("full scavenge lazy decommit time");
218 decommitter.processLazy();
219 }
220
221 {
222 PrintTime printTime("full scavenge mark all as eligible time");
223 std::lock_guard<Mutex> lock(Heap::mutex());
224 for (unsigned i = numHeaps; i--;) {
225 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
226 continue;
227 PerProcess<PerHeapKind<Heap>>::get()->at(i).markAllLargeAsEligibile(lock);
228 }
229 }
230 }
231
232 {
233 RELEASE_BASSERT(!m_deferredDecommits.size());
234 AllIsoHeaps::get()->forEach(
235 [&] (IsoHeapImplBase& heap) {
236 heap.scavenge(m_deferredDecommits);
237 });
238 IsoHeapImplBase::finishScavenging(m_deferredDecommits);
239 m_deferredDecommits.shrink(0);
240 }
241
242 if (verbose) {
243 fprintf(stderr, "--after scavenging--\n");
244 dumpStats();
245 fprintf(stderr, "--------------------------------\n");
246 }
247
248 {
249 std::unique_lock<Mutex> lock(mutex());
250 m_lastFullScavengeTime = std::chrono::steady_clock::now();
251 }
252}
253
254size_t Scavenger::freeableMemory()
255{
256 size_t result = 0;
257 {
258 std::lock_guard<Mutex> lock(Heap::mutex());
259 for (unsigned i = numHeaps; i--;) {
260 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
261 continue;
262 result += PerProcess<PerHeapKind<Heap>>::get()->at(i).freeableMemory(lock);
263 }
264 }
265
266 AllIsoHeaps::get()->forEach(
267 [&] (IsoHeapImplBase& heap) {
268 result += heap.freeableMemory();
269 });
270
271 return result;
272}
273
274size_t Scavenger::footprint()
275{
276 RELEASE_BASSERT(!Environment::get()->isDebugHeapEnabled());
277
278 size_t result = 0;
279 for (unsigned i = numHeaps; i--;) {
280 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
281 continue;
282 result += PerProcess<PerHeapKind<Heap>>::get()->at(i).footprint();
283 }
284
285 AllIsoHeaps::get()->forEach(
286 [&] (IsoHeapImplBase& heap) {
287 result += heap.footprint();
288 });
289
290 return result;
291}
292
293void Scavenger::threadEntryPoint(Scavenger* scavenger)
294{
295 scavenger->threadRunLoop();
296}
297
298void Scavenger::threadRunLoop()
299{
300 setSelfQOSClass();
301#if BOS(DARWIN)
302 setThreadName("JavaScriptCore bmalloc scavenger");
303#else
304 setThreadName("BMScavenger");
305#endif
306
307 // This loop ratchets downward from most active to least active state. While
308 // we ratchet downward, any other thread may reset our state.
309
310 // We require any state change while we are sleeping to signal to our
311 // condition variable and wake us up.
312
313 while (true) {
314 if (m_state == State::Sleep) {
315 std::unique_lock<Mutex> lock(mutex());
316 m_condition.wait(lock, [&]() { return m_state != State::Sleep; });
317 }
318
319 if (m_state == State::RunSoon) {
320 std::unique_lock<Mutex> lock(mutex());
321 m_condition.wait_for(lock, m_waitTime, [&]() { return m_state != State::RunSoon; });
322 }
323
324 m_state = State::Sleep;
325
326 setSelfQOSClass();
327
328 if (verbose) {
329 fprintf(stderr, "--------------------------------\n");
330 fprintf(stderr, "considering running scavenger\n");
331 dumpStats();
332 fprintf(stderr, "--------------------------------\n");
333 }
334
335 std::chrono::steady_clock::time_point start { std::chrono::steady_clock::now() };
336
337 scavenge();
338
339 auto timeSpentScavenging = std::chrono::steady_clock::now() - start;
340
341 if (verbose) {
342 fprintf(stderr, "time spent scavenging %lfms\n",
343 static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(timeSpentScavenging).count()) / 1000);
344 }
345
346 std::chrono::milliseconds newWaitTime;
347
348 if (m_isInMiniMode) {
349 timeSpentScavenging *= 50;
350 newWaitTime = std::chrono::duration_cast<std::chrono::milliseconds>(timeSpentScavenging);
351 newWaitTime = std::min(std::max(newWaitTime, std::chrono::milliseconds(25)), std::chrono::milliseconds(500));
352 } else {
353 timeSpentScavenging *= 150;
354 newWaitTime = std::chrono::duration_cast<std::chrono::milliseconds>(timeSpentScavenging);
355 m_waitTime = std::min(std::max(newWaitTime, std::chrono::milliseconds(100)), std::chrono::milliseconds(10000));
356 }
357
358 if (verbose)
359 fprintf(stderr, "new wait time %lldms\n", static_cast<long long int>(m_waitTime.count()));
360 }
361}
362
363void Scavenger::setThreadName(const char* name)
364{
365 BUNUSED(name);
366#if BOS(DARWIN)
367 pthread_setname_np(name);
368#elif BOS(LINUX)
369 // Truncate the given name since Linux limits the size of the thread name 16 including null terminator.
370 std::array<char, 16> buf;
371 strncpy(buf.data(), name, buf.size() - 1);
372 buf[buf.size() - 1] = '\0';
373 pthread_setname_np(pthread_self(), buf.data());
374#endif
375}
376
377void Scavenger::setSelfQOSClass()
378{
379#if BOS(DARWIN)
380 pthread_set_qos_class_self_np(requestedScavengerThreadQOSClass(), 0);
381#endif
382}
383
384} // namespace bmalloc
385
386