cubeb_resampler_internal.h (23022B)
1 /* 2 * Copyright © 2016 Mozilla Foundation 3 * 4 * This program is made available under an ISC-style license. See the 5 * accompanying file LICENSE for details. 6 */ 7 8 #if !defined(CUBEB_RESAMPLER_INTERNAL) 9 #define CUBEB_RESAMPLER_INTERNAL 10 11 #include <algorithm> 12 #include <cassert> 13 #include <cmath> 14 #include <memory> 15 #ifdef CUBEB_GECKO_BUILD 16 #include "mozilla/UniquePtr.h" 17 // In libc++, symbols such as std::unique_ptr may be defined in std::__1. 18 // The _LIBCPP_BEGIN_NAMESPACE_STD and _LIBCPP_END_NAMESPACE_STD macros 19 // will expand to the correct namespace. 20 #ifdef _LIBCPP_BEGIN_NAMESPACE_STD 21 #define MOZ_BEGIN_STD_NAMESPACE _LIBCPP_BEGIN_NAMESPACE_STD 22 #define MOZ_END_STD_NAMESPACE _LIBCPP_END_NAMESPACE_STD 23 #else 24 #define MOZ_BEGIN_STD_NAMESPACE namespace std { 25 #define MOZ_END_STD_NAMESPACE } 26 #endif 27 MOZ_BEGIN_STD_NAMESPACE 28 using mozilla::DefaultDelete; 29 using mozilla::UniquePtr; 30 #define default_delete DefaultDelete 31 #define unique_ptr UniquePtr 32 MOZ_END_STD_NAMESPACE 33 #endif 34 #include "cubeb-speex-resampler.h" 35 #include "cubeb/cubeb.h" 36 #include "cubeb_log.h" 37 #include "cubeb_resampler.h" 38 #include "cubeb_utils.h" 39 #include <stdio.h> 40 41 /* This header file contains the internal C++ API of the resamplers, for 42 * testing. */ 43 44 // When dropping audio input frames to prevent building 45 // an input delay, this function returns the number of frames 46 // to keep in the buffer. 47 // @parameter sample_rate The sample rate of the stream. 48 // @return A number of frames to keep. 49 uint32_t 50 min_buffered_audio_frame(uint32_t sample_rate); 51 52 int 53 to_speex_quality(cubeb_resampler_quality q); 54 55 struct cubeb_resampler { 56 virtual long fill(void * input_buffer, long * input_frames_count, 57 void * output_buffer, long frames_needed) = 0; 58 virtual long latency() = 0; 59 virtual ~cubeb_resampler() {} 60 }; 61 62 /** Base class for processors. This is just used to share methods for now. */ 63 class processor { 64 public: 65 explicit processor(uint32_t channels) : channels(channels) {} 66 67 protected: 68 size_t frames_to_samples(size_t frames) const { return frames * channels; } 69 size_t samples_to_frames(size_t samples) const 70 { 71 assert(!(samples % channels)); 72 return samples / channels; 73 } 74 /** The number of channel of the audio buffers to be resampled. */ 75 const uint32_t channels; 76 }; 77 78 template <typename T> 79 class passthrough_resampler : public cubeb_resampler, public processor { 80 public: 81 passthrough_resampler(cubeb_stream * s, cubeb_data_callback cb, void * ptr, 82 uint32_t input_channels, uint32_t sample_rate); 83 84 virtual long fill(void * input_buffer, long * input_frames_count, 85 void * output_buffer, long output_frames); 86 87 virtual long latency() { return 0; } 88 89 void drop_audio_if_needed() 90 { 91 uint32_t to_keep = min_buffered_audio_frame(sample_rate); 92 uint32_t available = samples_to_frames(internal_input_buffer.length()); 93 if (available > to_keep) { 94 ALOGV("Dropping %u frames", available - to_keep); 95 internal_input_buffer.pop(nullptr, 96 frames_to_samples(available - to_keep)); 97 } 98 } 99 100 private: 101 cubeb_stream * const stream; 102 const cubeb_data_callback data_callback; 103 void * const user_ptr; 104 /* This allows to buffer some input to account for the fact that we buffer 105 * some inputs. */ 106 auto_array<T> internal_input_buffer; 107 uint32_t sample_rate; 108 }; 109 110 /** Bidirectional resampler, can resample an input and an output stream, or just 111 * an input stream or output stream. In this case a delay is inserted in the 112 * opposite direction to keep the streams synchronized. */ 113 template <typename T, typename InputProcessing, typename OutputProcessing> 114 class cubeb_resampler_speex : public cubeb_resampler { 115 public: 116 cubeb_resampler_speex(InputProcessing * input_processor, 117 OutputProcessing * output_processor, cubeb_stream * s, 118 cubeb_data_callback cb, void * ptr); 119 120 virtual ~cubeb_resampler_speex(); 121 122 virtual long fill(void * input_buffer, long * input_frames_count, 123 void * output_buffer, long output_frames_needed); 124 125 virtual long latency() 126 { 127 if (input_processor && output_processor) { 128 assert(input_processor->latency() == output_processor->latency()); 129 return input_processor->latency(); 130 } else if (input_processor) { 131 return input_processor->latency(); 132 } else { 133 return output_processor->latency(); 134 } 135 } 136 137 private: 138 typedef long (cubeb_resampler_speex::*processing_callback)( 139 T * input_buffer, long * input_frames_count, T * output_buffer, 140 long output_frames_needed); 141 142 long fill_internal_duplex(T * input_buffer, long * input_frames_count, 143 T * output_buffer, long output_frames_needed); 144 long fill_internal_input(T * input_buffer, long * input_frames_count, 145 T * output_buffer, long output_frames_needed); 146 long fill_internal_output(T * input_buffer, long * input_frames_count, 147 T * output_buffer, long output_frames_needed); 148 149 std::unique_ptr<InputProcessing> input_processor; 150 std::unique_ptr<OutputProcessing> output_processor; 151 processing_callback fill_internal; 152 cubeb_stream * const stream; 153 const cubeb_data_callback data_callback; 154 void * const user_ptr; 155 bool draining = false; 156 }; 157 158 /** Handles one way of a (possibly) duplex resampler, working on interleaved 159 * audio buffers of type T. This class is designed so that the number of frames 160 * coming out of the resampler can be precisely controled. It manages its own 161 * input buffer, and can use the caller's output buffer, or allocate its own. */ 162 template <typename T> class cubeb_resampler_speex_one_way : public processor { 163 public: 164 /** The sample type of this resampler, either 16-bit integers or 32-bit 165 * floats. */ 166 typedef T sample_type; 167 /** Construct a resampler resampling from #source_rate to #target_rate, that 168 * can be arbitrary, strictly positive number. 169 * @parameter channels The number of channels this resampler will resample. 170 * @parameter source_rate The sample-rate of the audio input. 171 * @parameter target_rate The sample-rate of the audio output. 172 * @parameter quality A number between 0 (fast, low quality) and 10 (slow, 173 * high quality). */ 174 cubeb_resampler_speex_one_way(uint32_t channels, uint32_t source_rate, 175 uint32_t target_rate, int quality) 176 : processor(channels), 177 resampling_ratio(static_cast<float>(source_rate) / target_rate), 178 source_rate(source_rate), additional_latency(0), leftover_samples(0) 179 { 180 int r; 181 speex_resampler = 182 speex_resampler_init(channels, source_rate, target_rate, quality, &r); 183 assert(r == RESAMPLER_ERR_SUCCESS && "resampler allocation failure"); 184 185 uint32_t input_latency = speex_resampler_get_input_latency(speex_resampler); 186 const size_t LATENCY_SAMPLES = 8192; 187 T input_buffer[LATENCY_SAMPLES] = {}; 188 T output_buffer[LATENCY_SAMPLES] = {}; 189 uint32_t input_frame_count = input_latency; 190 uint32_t output_frame_count = LATENCY_SAMPLES; 191 assert(input_latency * channels <= LATENCY_SAMPLES); 192 speex_resample(input_buffer, &input_frame_count, output_buffer, 193 &output_frame_count); 194 } 195 196 /** Destructor, deallocate the resampler */ 197 virtual ~cubeb_resampler_speex_one_way() 198 { 199 speex_resampler_destroy(speex_resampler); 200 } 201 202 /* Fill the resampler with `input_frame_count` frames. */ 203 void input(T * input_buffer, size_t input_frame_count) 204 { 205 resampling_in_buffer.push(input_buffer, 206 frames_to_samples(input_frame_count)); 207 } 208 209 /** Outputs exactly `output_frame_count` into `output_buffer`. 210 * `output_buffer` has to be at least `output_frame_count` long. */ 211 size_t output(T * output_buffer, size_t output_frame_count) 212 { 213 uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); 214 uint32_t out_len = output_frame_count; 215 216 speex_resample(resampling_in_buffer.data(), &in_len, output_buffer, 217 &out_len); 218 219 /* This shifts back any unresampled samples to the beginning of the input 220 buffer. */ 221 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); 222 223 return out_len; 224 } 225 226 size_t output_for_input(uint32_t input_frames) 227 { 228 return (size_t)floorf( 229 (input_frames + samples_to_frames(resampling_in_buffer.length())) / 230 resampling_ratio); 231 } 232 233 /** Returns a buffer containing exactly `output_frame_count` resampled frames. 234 * The consumer should not hold onto the pointer. */ 235 T * output(size_t output_frame_count, size_t * input_frames_used) 236 { 237 if (resampling_out_buffer.capacity() < 238 frames_to_samples(output_frame_count)) { 239 resampling_out_buffer.reserve(frames_to_samples(output_frame_count)); 240 } 241 242 uint32_t in_len = samples_to_frames(resampling_in_buffer.length()); 243 uint32_t out_len = output_frame_count; 244 245 speex_resample(resampling_in_buffer.data(), &in_len, 246 resampling_out_buffer.data(), &out_len); 247 248 if (out_len < output_frame_count) { 249 LOGV("underrun during resampling: got %u frames, expected %zu", 250 (unsigned)out_len, output_frame_count); 251 // silence the rightmost part 252 T * data = resampling_out_buffer.data(); 253 for (uint32_t i = frames_to_samples(out_len); 254 i < frames_to_samples(output_frame_count); i++) { 255 data[i] = 0; 256 } 257 } 258 259 /* This shifts back any unresampled samples to the beginning of the input 260 buffer. */ 261 resampling_in_buffer.pop(nullptr, frames_to_samples(in_len)); 262 *input_frames_used = in_len; 263 264 return resampling_out_buffer.data(); 265 } 266 267 /** Get the latency of the resampler, in output frames. */ 268 uint32_t latency() const 269 { 270 /* The documentation of the resampler talks about "samples" here, but it 271 * only consider a single channel here so it's the same number of frames. */ 272 int latency = 0; 273 274 latency = speex_resampler_get_output_latency(speex_resampler) + 275 additional_latency; 276 277 assert(latency >= 0); 278 279 return latency; 280 } 281 282 /** Returns the number of frames to pass in the input of the resampler to have 283 * exactly `output_frame_count` resampled frames. This can return a number 284 * slightly bigger than what is strictly necessary, but it guaranteed that the 285 * number of output frames will be exactly equal. */ 286 uint32_t input_needed_for_output(int32_t output_frame_count) const 287 { 288 assert(output_frame_count >= 0); // Check overflow 289 int32_t unresampled_frames_left = 290 samples_to_frames(resampling_in_buffer.length()); 291 int32_t resampled_frames_left = 292 samples_to_frames(resampling_out_buffer.length()); 293 float input_frames_needed = 294 (output_frame_count - unresampled_frames_left) * resampling_ratio - 295 resampled_frames_left; 296 if (input_frames_needed < 0) { 297 return 0; 298 } 299 return (uint32_t)ceilf(input_frames_needed); 300 } 301 302 /** Returns a pointer to the input buffer, that contains empty space for at 303 * least `frame_count` elements. This is useful so that consumer can directly 304 * write into the input buffer of the resampler. The pointer returned is 305 * adjusted so that leftover data are not overwritten. 306 */ 307 T * input_buffer(size_t frame_count) 308 { 309 leftover_samples = resampling_in_buffer.length(); 310 resampling_in_buffer.reserve(leftover_samples + 311 frames_to_samples(frame_count)); 312 return resampling_in_buffer.data() + leftover_samples; 313 } 314 315 /** This method works with `input_buffer`, and allows to inform the processor 316 how much frames have been written in the provided buffer. */ 317 void written(size_t written_frames) 318 { 319 resampling_in_buffer.set_length(leftover_samples + 320 frames_to_samples(written_frames)); 321 } 322 323 void drop_audio_if_needed() 324 { 325 // Keep at most 100ms buffered. 326 uint32_t available = samples_to_frames(resampling_in_buffer.length()); 327 uint32_t to_keep = min_buffered_audio_frame(source_rate); 328 if (available > to_keep) { 329 ALOGV("Dropping %u frames", available - to_keep); 330 resampling_in_buffer.pop(nullptr, frames_to_samples(available - to_keep)); 331 } 332 } 333 334 private: 335 /** Wrapper for the speex resampling functions to have a typed 336 * interface. */ 337 void speex_resample(float * input_buffer, uint32_t * input_frame_count, 338 float * output_buffer, uint32_t * output_frame_count) 339 { 340 #ifndef NDEBUG 341 int rv; 342 rv = 343 #endif 344 speex_resampler_process_interleaved_float( 345 speex_resampler, input_buffer, input_frame_count, output_buffer, 346 output_frame_count); 347 assert(rv == RESAMPLER_ERR_SUCCESS); 348 } 349 350 void speex_resample(short * input_buffer, uint32_t * input_frame_count, 351 short * output_buffer, uint32_t * output_frame_count) 352 { 353 #ifndef NDEBUG 354 int rv; 355 rv = 356 #endif 357 speex_resampler_process_interleaved_int( 358 speex_resampler, input_buffer, input_frame_count, output_buffer, 359 output_frame_count); 360 assert(rv == RESAMPLER_ERR_SUCCESS); 361 } 362 /** The state for the speex resampler used internaly. */ 363 SpeexResamplerState * speex_resampler; 364 /** Source rate / target rate. */ 365 const float resampling_ratio; 366 const uint32_t source_rate; 367 /** Storage for the input frames, to be resampled. Also contains 368 * any unresampled frames after resampling. */ 369 auto_array<T> resampling_in_buffer; 370 /* Storage for the resampled frames, to be passed back to the caller. */ 371 auto_array<T> resampling_out_buffer; 372 /** Additional latency inserted into the pipeline for synchronisation. */ 373 uint32_t additional_latency; 374 /** When `input_buffer` is called, this allows tracking the number of samples 375 that were in the buffer. */ 376 uint32_t leftover_samples; 377 }; 378 379 /** This class allows delaying an audio stream by `frames` frames. */ 380 template <typename T> class delay_line : public processor { 381 public: 382 /** Constructor 383 * @parameter frames the number of frames of delay. 384 * @parameter channels the number of channels of this delay line. 385 * @parameter sample_rate sample-rate of the audio going through this delay 386 * line */ 387 delay_line(uint32_t frames, uint32_t channels, uint32_t sample_rate) 388 : processor(channels), length(frames), leftover_samples(0), 389 sample_rate(sample_rate) 390 { 391 /* Fill the delay line with some silent frames to add latency. */ 392 delay_input_buffer.push_silence(frames * channels); 393 } 394 /** Push some frames into the delay line. 395 * @parameter buffer the frames to push. 396 * @parameter frame_count the number of frames in #buffer. */ 397 void input(T * buffer, uint32_t frame_count) 398 { 399 delay_input_buffer.push(buffer, frames_to_samples(frame_count)); 400 } 401 /** Pop some frames from the internal buffer, into a internal output buffer. 402 * @parameter frames_needed the number of frames to be returned. 403 * @return a buffer containing the delayed frames. The consumer should not 404 * hold onto the pointer. */ 405 T * output(uint32_t frames_needed, size_t * input_frames_used) 406 { 407 if (delay_output_buffer.capacity() < frames_to_samples(frames_needed)) { 408 delay_output_buffer.reserve(frames_to_samples(frames_needed)); 409 } 410 411 delay_output_buffer.clear(); 412 delay_output_buffer.push(delay_input_buffer.data(), 413 frames_to_samples(frames_needed)); 414 delay_input_buffer.pop(nullptr, frames_to_samples(frames_needed)); 415 *input_frames_used = frames_needed; 416 417 return delay_output_buffer.data(); 418 } 419 /** Get a pointer to the first writable location in the input buffer> 420 * @parameter frames_needed the number of frames the user needs to write into 421 * the buffer. 422 * @returns a pointer to a location in the input buffer where #frames_needed 423 * can be writen. */ 424 T * input_buffer(uint32_t frames_needed) 425 { 426 leftover_samples = delay_input_buffer.length(); 427 delay_input_buffer.reserve(leftover_samples + 428 frames_to_samples(frames_needed)); 429 return delay_input_buffer.data() + leftover_samples; 430 } 431 /** This method works with `input_buffer`, and allows to inform the processor 432 how much frames have been written in the provided buffer. */ 433 void written(size_t frames_written) 434 { 435 delay_input_buffer.set_length(leftover_samples + 436 frames_to_samples(frames_written)); 437 } 438 /** Drains the delay line, emptying the buffer. 439 * @parameter output_buffer the buffer in which the frames are written. 440 * @parameter frames_needed the maximum number of frames to write. 441 * @return the actual number of frames written. */ 442 size_t output(T * output_buffer, uint32_t frames_needed) 443 { 444 uint32_t in_len = samples_to_frames(delay_input_buffer.length()); 445 uint32_t out_len = frames_needed; 446 447 uint32_t to_pop = std::min(in_len, out_len); 448 449 delay_input_buffer.pop(output_buffer, frames_to_samples(to_pop)); 450 451 return to_pop; 452 } 453 /** Returns the number of frames one needs to input into the delay line to get 454 * #frames_needed frames back. 455 * @parameter frames_needed the number of frames one want to write into the 456 * delay_line 457 * @returns the number of frames one will get. */ 458 uint32_t input_needed_for_output(int32_t frames_needed) const 459 { 460 assert(frames_needed >= 0); // Check overflow 461 return frames_needed; 462 } 463 /** Returns the number of frames produces for `input_frames` frames in input 464 */ 465 size_t output_for_input(uint32_t input_frames) { return input_frames; } 466 /** The number of frames this delay line delays the stream by. 467 * @returns The number of frames of delay. */ 468 size_t latency() { return length; } 469 470 void drop_audio_if_needed() 471 { 472 size_t available = samples_to_frames(delay_input_buffer.length()); 473 uint32_t to_keep = min_buffered_audio_frame(sample_rate); 474 if (available > to_keep) { 475 ALOGV("Dropping %u frames", available - to_keep); 476 delay_input_buffer.pop(nullptr, frames_to_samples(available - to_keep)); 477 } 478 } 479 480 private: 481 /** The length, in frames, of this delay line */ 482 uint32_t length; 483 /** When `input_buffer` is called, this allows tracking the number of samples 484 that where in the buffer. */ 485 uint32_t leftover_samples; 486 /** The input buffer, where the delay is applied. */ 487 auto_array<T> delay_input_buffer; 488 /** The output buffer. This is only ever used if using the ::output with a 489 * single argument. */ 490 auto_array<T> delay_output_buffer; 491 uint32_t sample_rate; 492 }; 493 494 /** This sits behind the C API and is more typed. */ 495 template <typename T> 496 cubeb_resampler * 497 cubeb_resampler_create_internal(cubeb_stream * stream, 498 cubeb_stream_params * input_params, 499 cubeb_stream_params * output_params, 500 unsigned int target_rate, 501 cubeb_data_callback callback, void * user_ptr, 502 cubeb_resampler_quality quality, 503 cubeb_resampler_reclock reclock) 504 { 505 std::unique_ptr<cubeb_resampler_speex_one_way<T>> input_resampler = nullptr; 506 std::unique_ptr<cubeb_resampler_speex_one_way<T>> output_resampler = nullptr; 507 std::unique_ptr<delay_line<T>> input_delay = nullptr; 508 std::unique_ptr<delay_line<T>> output_delay = nullptr; 509 510 assert((input_params || output_params) && 511 "need at least one valid parameter pointer."); 512 513 /* All the streams we have have a sample rate that matches the target 514 sample rate, use a no-op resampler, that simply forwards the buffers to the 515 callback. */ 516 if (((input_params && input_params->rate == target_rate) && 517 (output_params && output_params->rate == target_rate)) || 518 (input_params && !output_params && (input_params->rate == target_rate)) || 519 (output_params && !input_params && 520 (output_params->rate == target_rate))) { 521 LOG("Input and output sample-rate match, target rate of %dHz", target_rate); 522 return new passthrough_resampler<T>( 523 stream, callback, user_ptr, input_params ? input_params->channels : 0, 524 target_rate); 525 } 526 527 /* Determine if we need to resampler one or both directions, and create the 528 resamplers. */ 529 if (output_params && (output_params->rate != target_rate)) { 530 output_resampler.reset(new cubeb_resampler_speex_one_way<T>( 531 output_params->channels, target_rate, output_params->rate, 532 to_speex_quality(quality))); 533 if (!output_resampler) { 534 return NULL; 535 } 536 } 537 538 if (input_params && (input_params->rate != target_rate)) { 539 input_resampler.reset(new cubeb_resampler_speex_one_way<T>( 540 input_params->channels, input_params->rate, target_rate, 541 to_speex_quality(quality))); 542 if (!input_resampler) { 543 return NULL; 544 } 545 } 546 547 /* If we resample only one direction but we have a duplex stream, insert a 548 * delay line with a length equal to the resampler latency of the 549 * other direction so that the streams are synchronized. */ 550 if (input_resampler && !output_resampler && input_params && output_params) { 551 output_delay.reset(new delay_line<T>(input_resampler->latency(), 552 output_params->channels, 553 output_params->rate)); 554 if (!output_delay) { 555 return NULL; 556 } 557 } else if (output_resampler && !input_resampler && input_params && 558 output_params) { 559 input_delay.reset(new delay_line<T>(output_resampler->latency(), 560 input_params->channels, 561 output_params->rate)); 562 if (!input_delay) { 563 return NULL; 564 } 565 } 566 567 if (input_resampler && output_resampler) { 568 LOG("Resampling input (%d) and output (%d) to target rate of %dHz", 569 input_params->rate, output_params->rate, target_rate); 570 return new cubeb_resampler_speex<T, cubeb_resampler_speex_one_way<T>, 571 cubeb_resampler_speex_one_way<T>>( 572 input_resampler.release(), output_resampler.release(), stream, callback, 573 user_ptr); 574 } else if (input_resampler) { 575 LOG("Resampling input (%d) to target and output rate of %dHz", 576 input_params->rate, target_rate); 577 return new cubeb_resampler_speex<T, cubeb_resampler_speex_one_way<T>, 578 delay_line<T>>(input_resampler.release(), 579 output_delay.release(), 580 stream, callback, user_ptr); 581 } else { 582 LOG("Resampling output (%dHz) to target and input rate of %dHz", 583 output_params->rate, target_rate); 584 return new cubeb_resampler_speex<T, delay_line<T>, 585 cubeb_resampler_speex_one_way<T>>( 586 input_delay.release(), output_resampler.release(), stream, callback, 587 user_ptr); 588 } 589 } 590 591 #endif /* CUBEB_RESAMPLER_INTERNAL */