1 | /* |
2 | * Copyright (C) 2017-2018 Apple Inc. All rights reserved. |
3 | * |
4 | * Redistribution and use in source and binary forms, with or without |
5 | * modification, are permitted provided that the following conditions |
6 | * are met: |
7 | * 1. Redistributions of source code must retain the above copyright |
8 | * notice, this list of conditions and the following disclaimer. |
9 | * 2. Redistributions in binary form must reproduce the above copyright |
10 | * notice, this list of conditions and the following disclaimer in the |
11 | * documentation and/or other materials provided with the distribution. |
12 | * |
13 | * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY |
14 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
15 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
16 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR |
17 | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
18 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
19 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
20 | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
21 | * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
22 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
23 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
24 | */ |
25 | |
26 | #include "IsoTLS.h" |
27 | |
28 | #include "Environment.h" |
29 | #include "Gigacage.h" |
30 | #include "IsoTLSEntryInlines.h" |
31 | #include "IsoTLSInlines.h" |
32 | #include "IsoTLSLayout.h" |
33 | |
34 | #include <stdio.h> |
35 | |
36 | namespace bmalloc { |
37 | |
38 | IsoTLS::MallocFallbackState IsoTLS::s_mallocFallbackState; |
39 | |
40 | #if !HAVE_PTHREAD_MACHDEP_H |
41 | bool IsoTLS::s_didInitialize; |
42 | pthread_key_t IsoTLS::s_tlsKey; |
43 | #endif |
44 | |
45 | void IsoTLS::scavenge() |
46 | { |
47 | if (IsoTLS* tls = get()) { |
48 | tls->forEachEntry( |
49 | [&] (IsoTLSEntry* entry, void* data) { |
50 | entry->scavenge(data); |
51 | }); |
52 | } |
53 | } |
54 | |
55 | IsoTLS::IsoTLS() |
56 | { |
57 | BASSERT(!Environment::get()->isDebugHeapEnabled()); |
58 | } |
59 | |
60 | IsoTLS* IsoTLS::ensureEntries(unsigned offset) |
61 | { |
62 | RELEASE_BASSERT(!get() || offset >= get()->m_extent); |
63 | |
64 | static std::once_flag onceFlag; |
65 | std::call_once( |
66 | onceFlag, |
67 | [] () { |
68 | setvbuf(stderr, NULL, _IONBF, 0); |
69 | #if HAVE_PTHREAD_MACHDEP_H |
70 | pthread_key_init_np(tlsKey, destructor); |
71 | #else |
72 | int error = pthread_key_create(&s_tlsKey, destructor); |
73 | if (error) |
74 | BCRASH(); |
75 | s_didInitialize = true; |
76 | #endif |
77 | }); |
78 | |
79 | IsoTLS* tls = get(); |
80 | IsoTLSLayout& layout = *IsoTLSLayout::get(); |
81 | |
82 | IsoTLSEntry* oldLastEntry = tls ? tls->m_lastEntry : nullptr; |
83 | RELEASE_BASSERT(!oldLastEntry || oldLastEntry->offset() < offset); |
84 | |
85 | IsoTLSEntry* startEntry = oldLastEntry ? oldLastEntry : layout.head(); |
86 | |
87 | IsoTLSEntry* targetEntry = startEntry; |
88 | size_t requiredCapacity = 0; |
89 | if (startEntry) { |
90 | for (;;) { |
91 | RELEASE_BASSERT(targetEntry); |
92 | RELEASE_BASSERT(targetEntry->offset() <= offset); |
93 | if (targetEntry->offset() == offset) |
94 | break; |
95 | targetEntry = targetEntry->m_next; |
96 | } |
97 | RELEASE_BASSERT(targetEntry); |
98 | requiredCapacity = targetEntry->extent(); |
99 | } |
100 | |
101 | if (!tls || requiredCapacity > tls->m_capacity) { |
102 | size_t requiredSize = sizeForCapacity(requiredCapacity); |
103 | size_t goodSize = roundUpToMultipleOf(vmPageSize(), requiredSize); |
104 | size_t goodCapacity = capacityForSize(goodSize); |
105 | void* memory = vmAllocate(goodSize); |
106 | IsoTLS* newTLS = new (memory) IsoTLS(); |
107 | newTLS->m_capacity = goodCapacity; |
108 | if (tls) { |
109 | RELEASE_BASSERT(oldLastEntry); |
110 | RELEASE_BASSERT(layout.head()); |
111 | layout.head()->walkUpToInclusive( |
112 | oldLastEntry, |
113 | [&] (IsoTLSEntry* entry) { |
114 | void* src = tls->m_data + entry->offset(); |
115 | void* dst = newTLS->m_data + entry->offset(); |
116 | entry->move(src, dst); |
117 | entry->destruct(src); |
118 | }); |
119 | vmDeallocate(tls, tls->size()); |
120 | } |
121 | tls = newTLS; |
122 | set(tls); |
123 | } |
124 | |
125 | if (startEntry) { |
126 | startEntry->walkUpToInclusive( |
127 | targetEntry, |
128 | [&] (IsoTLSEntry* entry) { |
129 | entry->construct(tls->m_data + entry->offset()); |
130 | }); |
131 | |
132 | tls->m_lastEntry = targetEntry; |
133 | tls->m_extent = targetEntry->extent(); |
134 | } |
135 | |
136 | return tls; |
137 | } |
138 | |
139 | void IsoTLS::destructor(void* arg) |
140 | { |
141 | IsoTLS* tls = static_cast<IsoTLS*>(arg); |
142 | RELEASE_BASSERT(tls); |
143 | tls->forEachEntry( |
144 | [&] (IsoTLSEntry* entry, void* data) { |
145 | entry->scavenge(data); |
146 | entry->destruct(data); |
147 | }); |
148 | } |
149 | |
150 | size_t IsoTLS::sizeForCapacity(unsigned capacity) |
151 | { |
152 | return BOFFSETOF(IsoTLS, m_data) + capacity; |
153 | } |
154 | |
155 | unsigned IsoTLS::capacityForSize(size_t size) |
156 | { |
157 | return size - sizeForCapacity(0); |
158 | } |
159 | |
160 | size_t IsoTLS::size() |
161 | { |
162 | return sizeForCapacity(m_capacity); |
163 | } |
164 | |
165 | template<typename Func> |
166 | void IsoTLS::forEachEntry(const Func& func) |
167 | { |
168 | if (!m_lastEntry) |
169 | return; |
170 | IsoTLSLayout::get()->head()->walkUpToInclusive( |
171 | m_lastEntry, |
172 | [&] (IsoTLSEntry* entry) { |
173 | func(entry, m_data + entry->offset()); |
174 | }); |
175 | } |
176 | |
177 | void IsoTLS::determineMallocFallbackState() |
178 | { |
179 | static std::once_flag onceFlag; |
180 | std::call_once( |
181 | onceFlag, |
182 | [] { |
183 | if (s_mallocFallbackState != MallocFallbackState::Undecided) |
184 | return; |
185 | |
186 | #if GIGACAGE_ENABLED || BCPU(ARM64) |
187 | #if !BCPU(ARM64) |
188 | if (!Gigacage::shouldBeEnabled()) { |
189 | s_mallocFallbackState = MallocFallbackState::FallBackToMalloc; |
190 | return; |
191 | } |
192 | #endif |
193 | const char* env = getenv("bmalloc_IsoHeap" ); |
194 | if (env && (!strcasecmp(env, "false" ) || !strcasecmp(env, "no" ) || !strcmp(env, "0" ))) |
195 | s_mallocFallbackState = MallocFallbackState::FallBackToMalloc; |
196 | else |
197 | s_mallocFallbackState = MallocFallbackState::DoNotFallBack; |
198 | #else |
199 | s_mallocFallbackState = MallocFallbackState::FallBackToMalloc; |
200 | #endif |
201 | }); |
202 | } |
203 | |
204 | } // namespace bmalloc |
205 | |
206 | |