1 | /* |
2 | * Copyright (C) 2010, Google Inc. All rights reserved. |
3 | * |
4 | * Redistribution and use in source and binary forms, with or without |
5 | * modification, are permitted provided that the following conditions |
6 | * are met: |
7 | * 1. Redistributions of source code must retain the above copyright |
8 | * notice, this list of conditions and the following disclaimer. |
9 | * 2. Redistributions in binary form must reproduce the above copyright |
10 | * notice, this list of conditions and the following disclaimer in the |
11 | * documentation and/or other materials provided with the distribution. |
12 | * |
13 | * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY |
14 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
15 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
16 | * DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
17 | * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
18 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
19 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON |
20 | * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
21 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
22 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
23 | */ |
24 | |
25 | #include "config.h" |
26 | |
27 | #if ENABLE(WEB_AUDIO) |
28 | |
29 | #include "RealtimeAnalyser.h" |
30 | |
31 | #include "AudioBus.h" |
32 | #include "AudioUtilities.h" |
33 | #include "VectorMath.h" |
34 | #include <JavaScriptCore/Float32Array.h> |
35 | #include <JavaScriptCore/Uint8Array.h> |
36 | #include <algorithm> |
37 | #include <complex> |
38 | #include <wtf/MainThread.h> |
39 | #include <wtf/MathExtras.h> |
40 | |
41 | namespace WebCore { |
42 | |
43 | const double RealtimeAnalyser::DefaultSmoothingTimeConstant = 0.8; |
44 | const double RealtimeAnalyser::DefaultMinDecibels = -100; |
45 | const double RealtimeAnalyser::DefaultMaxDecibels = -30; |
46 | |
47 | const unsigned RealtimeAnalyser::DefaultFFTSize = 2048; |
48 | // All FFT implementations are expected to handle power-of-two sizes MinFFTSize <= size <= MaxFFTSize. |
49 | const unsigned RealtimeAnalyser::MinFFTSize = 32; |
50 | const unsigned RealtimeAnalyser::MaxFFTSize = 32768; |
51 | const unsigned RealtimeAnalyser::InputBufferSize = RealtimeAnalyser::MaxFFTSize * 2; |
52 | |
53 | RealtimeAnalyser::RealtimeAnalyser() |
54 | : m_inputBuffer(InputBufferSize) |
55 | , m_writeIndex(0) |
56 | , m_fftSize(DefaultFFTSize) |
57 | , m_magnitudeBuffer(DefaultFFTSize / 2) |
58 | , m_smoothingTimeConstant(DefaultSmoothingTimeConstant) |
59 | , m_minDecibels(DefaultMinDecibels) |
60 | , m_maxDecibels(DefaultMaxDecibels) |
61 | { |
62 | m_analysisFrame = std::make_unique<FFTFrame>(DefaultFFTSize); |
63 | } |
64 | |
65 | RealtimeAnalyser::~RealtimeAnalyser() = default; |
66 | |
67 | void RealtimeAnalyser::reset() |
68 | { |
69 | m_writeIndex = 0; |
70 | m_inputBuffer.zero(); |
71 | m_magnitudeBuffer.zero(); |
72 | } |
73 | |
74 | bool RealtimeAnalyser::setFftSize(size_t size) |
75 | { |
76 | ASSERT(isMainThread()); |
77 | |
78 | // Only allow powers of two. |
79 | unsigned log2size = static_cast<unsigned>(log2(size)); |
80 | bool isPOT(1UL << log2size == size); |
81 | |
82 | if (!isPOT || size > MaxFFTSize || size < MinFFTSize) |
83 | return false; |
84 | |
85 | if (m_fftSize != size) { |
86 | m_analysisFrame = std::make_unique<FFTFrame>(size); |
87 | // m_magnitudeBuffer has size = fftSize / 2 because it contains floats reduced from complex values in m_analysisFrame. |
88 | m_magnitudeBuffer.allocate(size / 2); |
89 | m_fftSize = size; |
90 | } |
91 | |
92 | return true; |
93 | } |
94 | |
95 | void RealtimeAnalyser::writeInput(AudioBus* bus, size_t framesToProcess) |
96 | { |
97 | bool isBusGood = bus && bus->numberOfChannels() > 0 && bus->channel(0)->length() >= framesToProcess; |
98 | ASSERT(isBusGood); |
99 | if (!isBusGood) |
100 | return; |
101 | |
102 | // FIXME : allow to work with non-FFTSize divisible chunking |
103 | bool isDestinationGood = m_writeIndex < m_inputBuffer.size() && m_writeIndex + framesToProcess <= m_inputBuffer.size(); |
104 | ASSERT(isDestinationGood); |
105 | if (!isDestinationGood) |
106 | return; |
107 | |
108 | // Perform real-time analysis |
109 | const float* source = bus->channel(0)->data(); |
110 | float* dest = m_inputBuffer.data() + m_writeIndex; |
111 | |
112 | // The source has already been sanity checked with isBusGood above. |
113 | memcpy(dest, source, sizeof(float) * framesToProcess); |
114 | |
115 | // Sum all channels in one if numberOfChannels > 1. |
116 | unsigned numberOfChannels = bus->numberOfChannels(); |
117 | if (numberOfChannels > 1) { |
118 | for (unsigned i = 1; i < numberOfChannels; i++) { |
119 | source = bus->channel(i)->data(); |
120 | VectorMath::vadd(dest, 1, source, 1, dest, 1, framesToProcess); |
121 | } |
122 | const float scale = 1.0 / numberOfChannels; |
123 | VectorMath::vsmul(dest, 1, &scale, dest, 1, framesToProcess); |
124 | } |
125 | |
126 | m_writeIndex += framesToProcess; |
127 | if (m_writeIndex >= InputBufferSize) |
128 | m_writeIndex = 0; |
129 | } |
130 | |
131 | namespace { |
132 | |
133 | void applyWindow(float* p, size_t n) |
134 | { |
135 | ASSERT(isMainThread()); |
136 | |
137 | // Blackman window |
138 | double alpha = 0.16; |
139 | double a0 = 0.5 * (1 - alpha); |
140 | double a1 = 0.5; |
141 | double a2 = 0.5 * alpha; |
142 | |
143 | for (unsigned i = 0; i < n; ++i) { |
144 | double x = static_cast<double>(i) / static_cast<double>(n); |
145 | double window = a0 - a1 * cos(2 * piDouble * x) + a2 * cos(4 * piDouble * x); |
146 | p[i] *= float(window); |
147 | } |
148 | } |
149 | |
150 | } // namespace |
151 | |
152 | void RealtimeAnalyser::doFFTAnalysis() |
153 | { |
154 | ASSERT(isMainThread()); |
155 | |
156 | // Unroll the input buffer into a temporary buffer, where we'll apply an analysis window followed by an FFT. |
157 | size_t fftSize = this->fftSize(); |
158 | |
159 | AudioFloatArray temporaryBuffer(fftSize); |
160 | float* inputBuffer = m_inputBuffer.data(); |
161 | float* tempP = temporaryBuffer.data(); |
162 | |
163 | // Take the previous fftSize values from the input buffer and copy into the temporary buffer. |
164 | unsigned writeIndex = m_writeIndex; |
165 | if (writeIndex < fftSize) { |
166 | memcpy(tempP, inputBuffer + writeIndex - fftSize + InputBufferSize, sizeof(*tempP) * (fftSize - writeIndex)); |
167 | memcpy(tempP + fftSize - writeIndex, inputBuffer, sizeof(*tempP) * writeIndex); |
168 | } else |
169 | memcpy(tempP, inputBuffer + writeIndex - fftSize, sizeof(*tempP) * fftSize); |
170 | |
171 | |
172 | // Window the input samples. |
173 | applyWindow(tempP, fftSize); |
174 | |
175 | // Do the analysis. |
176 | m_analysisFrame->doFFT(tempP); |
177 | |
178 | float* realP = m_analysisFrame->realData(); |
179 | float* imagP = m_analysisFrame->imagData(); |
180 | |
181 | // Blow away the packed nyquist component. |
182 | imagP[0] = 0; |
183 | |
184 | // Normalize so than an input sine wave at 0dBfs registers as 0dBfs (undo FFT scaling factor). |
185 | const double magnitudeScale = 1.0 / fftSize; |
186 | |
187 | // A value of 0 does no averaging with the previous result. Larger values produce slower, but smoother changes. |
188 | double k = m_smoothingTimeConstant; |
189 | k = std::max(0.0, k); |
190 | k = std::min(1.0, k); |
191 | |
192 | // Convert the analysis data from complex to magnitude and average with the previous result. |
193 | float* destination = magnitudeBuffer().data(); |
194 | size_t n = magnitudeBuffer().size(); |
195 | for (size_t i = 0; i < n; ++i) { |
196 | std::complex<double> c(realP[i], imagP[i]); |
197 | double scalarMagnitude = abs(c) * magnitudeScale; |
198 | destination[i] = static_cast<float>(k * destination[i] + (1 - k) * scalarMagnitude); |
199 | } |
200 | } |
201 | |
202 | void RealtimeAnalyser::getFloatFrequencyData(Float32Array* destinationArray) |
203 | { |
204 | ASSERT(isMainThread()); |
205 | |
206 | if (!destinationArray) |
207 | return; |
208 | |
209 | doFFTAnalysis(); |
210 | |
211 | // Convert from linear magnitude to floating-point decibels. |
212 | const double minDecibels = m_minDecibels; |
213 | unsigned sourceLength = magnitudeBuffer().size(); |
214 | size_t len = std::min(sourceLength, destinationArray->length()); |
215 | if (len > 0) { |
216 | const float* source = magnitudeBuffer().data(); |
217 | float* destination = destinationArray->data(); |
218 | |
219 | for (unsigned i = 0; i < len; ++i) { |
220 | float linearValue = source[i]; |
221 | double dbMag = !linearValue ? minDecibels : AudioUtilities::linearToDecibels(linearValue); |
222 | destination[i] = static_cast<float>(dbMag); |
223 | } |
224 | } |
225 | } |
226 | |
227 | void RealtimeAnalyser::getByteFrequencyData(Uint8Array* destinationArray) |
228 | { |
229 | ASSERT(isMainThread()); |
230 | |
231 | if (!destinationArray) |
232 | return; |
233 | |
234 | doFFTAnalysis(); |
235 | |
236 | // Convert from linear magnitude to unsigned-byte decibels. |
237 | unsigned sourceLength = magnitudeBuffer().size(); |
238 | size_t len = std::min(sourceLength, destinationArray->length()); |
239 | if (len > 0) { |
240 | const double rangeScaleFactor = m_maxDecibels == m_minDecibels ? 1 : 1 / (m_maxDecibels - m_minDecibels); |
241 | const double minDecibels = m_minDecibels; |
242 | |
243 | const float* source = magnitudeBuffer().data(); |
244 | unsigned char* destination = destinationArray->data(); |
245 | |
246 | for (unsigned i = 0; i < len; ++i) { |
247 | float linearValue = source[i]; |
248 | double dbMag = !linearValue ? minDecibels : AudioUtilities::linearToDecibels(linearValue); |
249 | |
250 | // The range m_minDecibels to m_maxDecibels will be scaled to byte values from 0 to UCHAR_MAX. |
251 | double scaledValue = UCHAR_MAX * (dbMag - minDecibels) * rangeScaleFactor; |
252 | |
253 | // Clip to valid range. |
254 | if (scaledValue < 0) |
255 | scaledValue = 0; |
256 | if (scaledValue > UCHAR_MAX) |
257 | scaledValue = UCHAR_MAX; |
258 | |
259 | destination[i] = static_cast<unsigned char>(scaledValue); |
260 | } |
261 | } |
262 | } |
263 | |
264 | void RealtimeAnalyser::getByteTimeDomainData(Uint8Array* destinationArray) |
265 | { |
266 | ASSERT(isMainThread()); |
267 | |
268 | if (!destinationArray) |
269 | return; |
270 | |
271 | unsigned fftSize = this->fftSize(); |
272 | size_t len = std::min(fftSize, destinationArray->length()); |
273 | if (len > 0) { |
274 | bool isInputBufferGood = m_inputBuffer.size() == InputBufferSize && m_inputBuffer.size() > fftSize; |
275 | ASSERT(isInputBufferGood); |
276 | if (!isInputBufferGood) |
277 | return; |
278 | |
279 | float* inputBuffer = m_inputBuffer.data(); |
280 | unsigned char* destination = destinationArray->data(); |
281 | |
282 | unsigned writeIndex = m_writeIndex; |
283 | |
284 | for (unsigned i = 0; i < len; ++i) { |
285 | // Buffer access is protected due to modulo operation. |
286 | float value = inputBuffer[(i + writeIndex - fftSize + InputBufferSize) % InputBufferSize]; |
287 | |
288 | // Scale from nominal -1 -> +1 to unsigned byte. |
289 | double scaledValue = 128 * (value + 1); |
290 | |
291 | // Clip to valid range. |
292 | if (scaledValue < 0) |
293 | scaledValue = 0; |
294 | if (scaledValue > UCHAR_MAX) |
295 | scaledValue = UCHAR_MAX; |
296 | |
297 | destination[i] = static_cast<unsigned char>(scaledValue); |
298 | } |
299 | } |
300 | } |
301 | |
302 | } // namespace WebCore |
303 | |
304 | #endif // ENABLE(WEB_AUDIO) |
305 | |