1/*
2 * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#pragma once
27
28#include "Environment.h"
29#include "IsoHeapImpl.h"
30#include "IsoTLS.h"
31#include "bmalloc.h"
32
33namespace bmalloc {
34
35template<typename Type>
36void* IsoTLS::allocate(api::IsoHeap<Type>& handle, bool abortOnFailure)
37{
38 return allocateImpl<typename api::IsoHeap<Type>::Config>(handle, abortOnFailure);
39}
40
41template<typename Type>
42void IsoTLS::deallocate(api::IsoHeap<Type>& handle, void* p)
43{
44 if (!p)
45 return;
46 deallocateImpl<typename api::IsoHeap<Type>::Config>(handle, p);
47}
48
49template<typename Type>
50void IsoTLS::scavenge(api::IsoHeap<Type>& handle)
51{
52 IsoTLS* tls = get();
53 if (!tls)
54 return;
55 if (!handle.isInitialized())
56 return;
57 unsigned offset = handle.allocatorOffset();
58 if (offset < tls->m_extent)
59 reinterpret_cast<IsoAllocator<typename api::IsoHeap<Type>::Config>*>(tls->m_data + offset)->scavenge();
60 offset = handle.deallocatorOffset();
61 if (offset < tls->m_extent)
62 reinterpret_cast<IsoDeallocator<typename api::IsoHeap<Type>::Config>*>(tls->m_data + offset)->scavenge();
63 handle.impl().scavengeNow();
64}
65
66template<typename Config, typename Type>
67void* IsoTLS::allocateImpl(api::IsoHeap<Type>& handle, bool abortOnFailure)
68{
69 unsigned offset = handle.allocatorOffset();
70 IsoTLS* tls = get();
71 if (!tls || offset >= tls->m_extent)
72 return allocateSlow<Config>(handle, abortOnFailure);
73 return tls->allocateFast<Config>(offset, abortOnFailure);
74}
75
76template<typename Config>
77void* IsoTLS::allocateFast(unsigned offset, bool abortOnFailure)
78{
79 return reinterpret_cast<IsoAllocator<Config>*>(m_data + offset)->allocate(abortOnFailure);
80}
81
82template<typename Config, typename Type>
83BNO_INLINE void* IsoTLS::allocateSlow(api::IsoHeap<Type>& handle, bool abortOnFailure)
84{
85 for (;;) {
86 switch (s_mallocFallbackState) {
87 case MallocFallbackState::Undecided:
88 determineMallocFallbackState();
89 continue;
90 case MallocFallbackState::FallBackToMalloc:
91 return api::tryMalloc(Config::objectSize);
92 case MallocFallbackState::DoNotFallBack:
93 break;
94 }
95 break;
96 }
97
98 // If debug heap is enabled, s_mallocFallbackState becomes MallocFallbackState::FallBackToMalloc.
99 BASSERT(!Environment::get()->isDebugHeapEnabled());
100
101 IsoTLS* tls = ensureHeapAndEntries(handle);
102
103 return tls->allocateFast<Config>(handle.allocatorOffset(), abortOnFailure);
104}
105
106template<typename Config, typename Type>
107void IsoTLS::deallocateImpl(api::IsoHeap<Type>& handle, void* p)
108{
109 unsigned offset = handle.deallocatorOffset();
110 IsoTLS* tls = get();
111 // Note that this bounds check would be here even if we didn't have to support DebugHeap,
112 // since we don't want unpredictable behavior if offset or m_extent ever got corrupted.
113 if (!tls || offset >= tls->m_extent)
114 deallocateSlow<Config>(handle, p);
115 else
116 tls->deallocateFast<Config>(handle, offset, p);
117}
118
119template<typename Config, typename Type>
120void IsoTLS::deallocateFast(api::IsoHeap<Type>& handle, unsigned offset, void* p)
121{
122 reinterpret_cast<IsoDeallocator<Config>*>(m_data + offset)->deallocate(handle, p);
123}
124
125template<typename Config, typename Type>
126BNO_INLINE void IsoTLS::deallocateSlow(api::IsoHeap<Type>& handle, void* p)
127{
128 for (;;) {
129 switch (s_mallocFallbackState) {
130 case MallocFallbackState::Undecided:
131 determineMallocFallbackState();
132 continue;
133 case MallocFallbackState::FallBackToMalloc:
134 return api::free(p);
135 case MallocFallbackState::DoNotFallBack:
136 break;
137 }
138 break;
139 }
140
141 // If debug heap is enabled, s_mallocFallbackState becomes MallocFallbackState::FallBackToMalloc.
142 BASSERT(!Environment::get()->isDebugHeapEnabled());
143
144 RELEASE_BASSERT(handle.isInitialized());
145
146 IsoTLS* tls = ensureEntries(std::max(handle.allocatorOffset(), handle.deallocatorOffset()));
147
148 tls->deallocateFast<Config>(handle, handle.deallocatorOffset(), p);
149}
150
151inline IsoTLS* IsoTLS::get()
152{
153#if HAVE_PTHREAD_MACHDEP_H
154 return static_cast<IsoTLS*>(_pthread_getspecific_direct(tlsKey));
155#else
156 if (!s_didInitialize)
157 return nullptr;
158 return static_cast<IsoTLS*>(pthread_getspecific(s_tlsKey));
159#endif
160}
161
162inline void IsoTLS::set(IsoTLS* tls)
163{
164#if HAVE_PTHREAD_MACHDEP_H
165 _pthread_setspecific_direct(tlsKey, tls);
166#else
167 pthread_setspecific(s_tlsKey, tls);
168#endif
169}
170
171template<typename Type>
172void IsoTLS::ensureHeap(api::IsoHeap<Type>& handle)
173{
174 if (!handle.isInitialized()) {
175 std::lock_guard<Mutex> locker(handle.m_initializationLock);
176 if (!handle.isInitialized()) {
177 auto* heap = new IsoHeapImpl<typename api::IsoHeap<Type>::Config>();
178 std::atomic_thread_fence(std::memory_order_seq_cst);
179 handle.setAllocatorOffset(heap->allocatorOffset());
180 handle.setDeallocatorOffset(heap->deallocatorOffset());
181 handle.m_impl = heap;
182 }
183 }
184}
185
186template<typename Type>
187BNO_INLINE IsoTLS* IsoTLS::ensureHeapAndEntries(api::IsoHeap<Type>& handle)
188{
189 RELEASE_BASSERT(
190 !get()
191 || handle.allocatorOffset() >= get()->m_extent
192 || handle.deallocatorOffset() >= get()->m_extent);
193 ensureHeap(handle);
194 return ensureEntries(std::max(handle.allocatorOffset(), handle.deallocatorOffset()));
195}
196
197} // namespace bmalloc
198
199