1/*
2 * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "IsoTLS.h"
27
28#include "Environment.h"
29#include "Gigacage.h"
30#include "IsoTLSEntryInlines.h"
31#include "IsoTLSInlines.h"
32#include "IsoTLSLayout.h"
33
34#include <stdio.h>
35
36namespace bmalloc {
37
38IsoTLS::MallocFallbackState IsoTLS::s_mallocFallbackState;
39
40#if !HAVE_PTHREAD_MACHDEP_H
41bool IsoTLS::s_didInitialize;
42pthread_key_t IsoTLS::s_tlsKey;
43#endif
44
45void IsoTLS::scavenge()
46{
47 if (IsoTLS* tls = get()) {
48 tls->forEachEntry(
49 [&] (IsoTLSEntry* entry, void* data) {
50 entry->scavenge(data);
51 });
52 }
53}
54
55IsoTLS::IsoTLS()
56{
57 BASSERT(!Environment::get()->isDebugHeapEnabled());
58}
59
60IsoTLS* IsoTLS::ensureEntries(unsigned offset)
61{
62 RELEASE_BASSERT(!get() || offset >= get()->m_extent);
63
64 static std::once_flag onceFlag;
65 std::call_once(
66 onceFlag,
67 [] () {
68 setvbuf(stderr, NULL, _IONBF, 0);
69#if HAVE_PTHREAD_MACHDEP_H
70 pthread_key_init_np(tlsKey, destructor);
71#else
72 int error = pthread_key_create(&s_tlsKey, destructor);
73 if (error)
74 BCRASH();
75 s_didInitialize = true;
76#endif
77 });
78
79 IsoTLS* tls = get();
80 IsoTLSLayout& layout = *IsoTLSLayout::get();
81
82 IsoTLSEntry* oldLastEntry = tls ? tls->m_lastEntry : nullptr;
83 RELEASE_BASSERT(!oldLastEntry || oldLastEntry->offset() < offset);
84
85 IsoTLSEntry* startEntry = oldLastEntry ? oldLastEntry : layout.head();
86
87 IsoTLSEntry* targetEntry = startEntry;
88 size_t requiredCapacity = 0;
89 if (startEntry) {
90 for (;;) {
91 RELEASE_BASSERT(targetEntry);
92 RELEASE_BASSERT(targetEntry->offset() <= offset);
93 if (targetEntry->offset() == offset)
94 break;
95 targetEntry = targetEntry->m_next;
96 }
97 RELEASE_BASSERT(targetEntry);
98 requiredCapacity = targetEntry->extent();
99 }
100
101 if (!tls || requiredCapacity > tls->m_capacity) {
102 size_t requiredSize = sizeForCapacity(requiredCapacity);
103 size_t goodSize = roundUpToMultipleOf(vmPageSize(), requiredSize);
104 size_t goodCapacity = capacityForSize(goodSize);
105 void* memory = vmAllocate(goodSize);
106 IsoTLS* newTLS = new (memory) IsoTLS();
107 newTLS->m_capacity = goodCapacity;
108 if (tls) {
109 RELEASE_BASSERT(oldLastEntry);
110 RELEASE_BASSERT(layout.head());
111 layout.head()->walkUpToInclusive(
112 oldLastEntry,
113 [&] (IsoTLSEntry* entry) {
114 void* src = tls->m_data + entry->offset();
115 void* dst = newTLS->m_data + entry->offset();
116 entry->move(src, dst);
117 entry->destruct(src);
118 });
119 vmDeallocate(tls, tls->size());
120 }
121 tls = newTLS;
122 set(tls);
123 }
124
125 if (startEntry) {
126 startEntry->walkUpToInclusive(
127 targetEntry,
128 [&] (IsoTLSEntry* entry) {
129 entry->construct(tls->m_data + entry->offset());
130 });
131
132 tls->m_lastEntry = targetEntry;
133 tls->m_extent = targetEntry->extent();
134 }
135
136 return tls;
137}
138
139void IsoTLS::destructor(void* arg)
140{
141 IsoTLS* tls = static_cast<IsoTLS*>(arg);
142 RELEASE_BASSERT(tls);
143 tls->forEachEntry(
144 [&] (IsoTLSEntry* entry, void* data) {
145 entry->scavenge(data);
146 entry->destruct(data);
147 });
148}
149
150size_t IsoTLS::sizeForCapacity(unsigned capacity)
151{
152 return BOFFSETOF(IsoTLS, m_data) + capacity;
153}
154
155unsigned IsoTLS::capacityForSize(size_t size)
156{
157 return size - sizeForCapacity(0);
158}
159
160size_t IsoTLS::size()
161{
162 return sizeForCapacity(m_capacity);
163}
164
165template<typename Func>
166void IsoTLS::forEachEntry(const Func& func)
167{
168 if (!m_lastEntry)
169 return;
170 IsoTLSLayout::get()->head()->walkUpToInclusive(
171 m_lastEntry,
172 [&] (IsoTLSEntry* entry) {
173 func(entry, m_data + entry->offset());
174 });
175}
176
177void IsoTLS::determineMallocFallbackState()
178{
179 static std::once_flag onceFlag;
180 std::call_once(
181 onceFlag,
182 [] {
183 if (s_mallocFallbackState != MallocFallbackState::Undecided)
184 return;
185
186#if GIGACAGE_ENABLED || BCPU(ARM64)
187#if !BCPU(ARM64)
188 if (!Gigacage::shouldBeEnabled()) {
189 s_mallocFallbackState = MallocFallbackState::FallBackToMalloc;
190 return;
191 }
192#endif
193 const char* env = getenv("bmalloc_IsoHeap");
194 if (env && (!strcasecmp(env, "false") || !strcasecmp(env, "no") || !strcmp(env, "0")))
195 s_mallocFallbackState = MallocFallbackState::FallBackToMalloc;
196 else
197 s_mallocFallbackState = MallocFallbackState::DoNotFallBack;
198#else
199 s_mallocFallbackState = MallocFallbackState::FallBackToMalloc;
200#endif
201 });
202}
203
204} // namespace bmalloc
205
206