audio_stream.cpp (34126B)
1 // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com> 2 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) 3 4 #include "audio_stream.h" 5 #include "host.h" 6 7 #include "common/align.h" 8 #include "common/assert.h" 9 #include "common/error.h" 10 #include "common/gsvector.h" 11 #include "common/log.h" 12 #include "common/settings_interface.h" 13 #include "common/timer.h" 14 15 #include "soundtouch/SoundTouch.h" 16 #include "soundtouch/SoundTouchDLL.h" 17 18 #ifndef __ANDROID__ 19 #include "freesurround_decoder.h" 20 #endif 21 22 #include <algorithm> 23 #include <cmath> 24 #include <cstring> 25 #include <limits> 26 27 Log_SetChannel(AudioStream); 28 29 static constexpr bool LOG_TIMESTRETCH_STATS = false; 30 31 static constexpr const std::array<std::pair<u8, u8>, static_cast<size_t>(AudioExpansionMode::Count)> 32 s_expansion_channel_count = {{ 33 {u8(2), u8(2)}, // Disabled 34 {u8(3), u8(3)}, // StereoLFE 35 {u8(5), u8(4)}, // Quadraphonic 36 {u8(5), u8(5)}, // QuadraphonicLFE 37 {u8(6), u8(6)}, // Surround51 38 {u8(8), u8(8)}, // Surround71 39 }}; 40 41 AudioStream::DeviceInfo::DeviceInfo(std::string name_, std::string display_name_, u32 minimum_latency_) 42 : name(std::move(name_)), display_name(std::move(display_name_)), minimum_latency_frames(minimum_latency_) 43 { 44 } 45 46 AudioStream::DeviceInfo::~DeviceInfo() = default; 47 48 AudioStream::AudioStream(u32 sample_rate, const AudioStreamParameters& parameters) 49 : m_sample_rate(sample_rate), m_parameters(parameters), 50 m_internal_channels(s_expansion_channel_count[static_cast<size_t>(parameters.expansion_mode)].first), 51 m_output_channels(s_expansion_channel_count[static_cast<size_t>(parameters.expansion_mode)].second) 52 { 53 } 54 55 AudioStream::~AudioStream() 56 { 57 StretchDestroy(); 58 DestroyBuffer(); 59 } 60 61 std::unique_ptr<AudioStream> AudioStream::CreateNullStream(u32 sample_rate, u32 buffer_ms) 62 { 63 // no point stretching with no output 64 AudioStreamParameters params; 65 params.expansion_mode = AudioExpansionMode::Disabled; 66 params.stretch_mode = AudioStretchMode::Off; 67 params.buffer_ms = static_cast<u16>(buffer_ms); 68 69 std::unique_ptr<AudioStream> stream(new AudioStream(sample_rate, params)); 70 stream->BaseInitialize(&StereoSampleReaderImpl); 71 return stream; 72 } 73 74 std::vector<std::pair<std::string, std::string>> AudioStream::GetDriverNames(AudioBackend backend) 75 { 76 std::vector<std::pair<std::string, std::string>> ret; 77 switch (backend) 78 { 79 #ifndef __ANDROID__ 80 case AudioBackend::Cubeb: 81 ret = GetCubebDriverNames(); 82 break; 83 #endif 84 85 default: 86 break; 87 } 88 89 return ret; 90 } 91 92 std::vector<AudioStream::DeviceInfo> AudioStream::GetOutputDevices(AudioBackend backend, const char* driver, 93 u32 sample_rate) 94 { 95 std::vector<AudioStream::DeviceInfo> ret; 96 switch (backend) 97 { 98 #ifndef __ANDROID__ 99 case AudioBackend::Cubeb: 100 ret = GetCubebOutputDevices(driver, sample_rate); 101 break; 102 #endif 103 104 default: 105 break; 106 } 107 108 return ret; 109 } 110 111 std::unique_ptr<AudioStream> AudioStream::CreateStream(AudioBackend backend, u32 sample_rate, 112 const AudioStreamParameters& parameters, const char* driver_name, 113 const char* device_name, Error* error /* = nullptr */) 114 { 115 switch (backend) 116 { 117 #ifndef __ANDROID__ 118 case AudioBackend::Cubeb: 119 return CreateCubebAudioStream(sample_rate, parameters, driver_name, device_name, error); 120 121 case AudioBackend::SDL: 122 return CreateSDLAudioStream(sample_rate, parameters, error); 123 #else 124 case AudioBackend::AAudio: 125 return CreateAAudioAudioStream(sample_rate, parameters, error); 126 127 case AudioBackend::OpenSLES: 128 return CreateOpenSLESAudioStream(sample_rate, parameters, error); 129 #endif 130 131 case AudioBackend::Null: 132 return CreateNullStream(sample_rate, parameters.buffer_ms); 133 134 default: 135 Error::SetStringView(error, "Unknown audio backend."); 136 return nullptr; 137 } 138 } 139 140 u32 AudioStream::GetAlignedBufferSize(u32 size) 141 { 142 static_assert(Common::IsPow2(CHUNK_SIZE)); 143 return Common::AlignUpPow2(size, CHUNK_SIZE); 144 } 145 146 u32 AudioStream::GetBufferSizeForMS(u32 sample_rate, u32 ms) 147 { 148 return GetAlignedBufferSize((ms * sample_rate) / 1000u); 149 } 150 151 u32 AudioStream::GetMSForBufferSize(u32 sample_rate, u32 buffer_size) 152 { 153 buffer_size = GetAlignedBufferSize(buffer_size); 154 return (buffer_size * 1000u) / sample_rate; 155 } 156 157 static constexpr const std::array s_backend_names = { 158 "Null", 159 #ifndef __ANDROID__ 160 "Cubeb", 161 "SDL", 162 #else 163 "AAudio", 164 "OpenSLES", 165 #endif 166 }; 167 static constexpr const std::array s_backend_display_names = { 168 TRANSLATE_NOOP("AudioStream", "Null (No Output)"), 169 #ifndef __ANDROID__ 170 TRANSLATE_NOOP("AudioStream", "Cubeb"), 171 TRANSLATE_NOOP("AudioStream", "SDL"), 172 #else 173 "AAudio", 174 "OpenSL ES", 175 #endif 176 }; 177 178 std::optional<AudioBackend> AudioStream::ParseBackendName(const char* str) 179 { 180 int index = 0; 181 for (const char* name : s_backend_names) 182 { 183 if (std::strcmp(name, str) == 0) 184 return static_cast<AudioBackend>(index); 185 186 index++; 187 } 188 189 return std::nullopt; 190 } 191 192 const char* AudioStream::GetBackendName(AudioBackend backend) 193 { 194 return s_backend_names[static_cast<int>(backend)]; 195 } 196 197 const char* AudioStream::GetBackendDisplayName(AudioBackend backend) 198 { 199 return Host::TranslateToCString("AudioStream", s_backend_display_names[static_cast<int>(backend)]); 200 } 201 202 static constexpr const std::array s_expansion_mode_names = { 203 "Disabled", "StereoLFE", "Quadraphonic", "QuadraphonicLFE", "Surround51", "Surround71", 204 }; 205 static constexpr const std::array s_expansion_mode_display_names = { 206 TRANSLATE_NOOP("AudioStream", "Disabled (Stereo)"), TRANSLATE_NOOP("AudioStream", "Stereo with LFE"), 207 TRANSLATE_NOOP("AudioStream", "Quadraphonic"), TRANSLATE_NOOP("AudioStream", "Quadraphonic with LFE"), 208 TRANSLATE_NOOP("AudioStream", "5.1 Surround"), TRANSLATE_NOOP("AudioStream", "7.1 Surround"), 209 }; 210 211 const char* AudioStream::GetExpansionModeName(AudioExpansionMode mode) 212 { 213 return (static_cast<u32>(mode) < s_expansion_mode_names.size()) ? s_expansion_mode_names[static_cast<u32>(mode)] : ""; 214 } 215 216 const char* AudioStream::GetExpansionModeDisplayName(AudioExpansionMode mode) 217 { 218 return (static_cast<u32>(mode) < s_expansion_mode_display_names.size()) ? 219 Host::TranslateToCString("AudioStream", s_expansion_mode_display_names[static_cast<u32>(mode)]) : 220 ""; 221 } 222 223 std::optional<AudioExpansionMode> AudioStream::ParseExpansionMode(const char* name) 224 { 225 for (u8 i = 0; i < static_cast<u8>(AudioExpansionMode::Count); i++) 226 { 227 if (std::strcmp(name, s_expansion_mode_names[i]) == 0) 228 return static_cast<AudioExpansionMode>(i); 229 } 230 231 return std::nullopt; 232 } 233 234 static constexpr const std::array s_stretch_mode_names = { 235 "None", 236 "Resample", 237 "TimeStretch", 238 }; 239 static constexpr const std::array s_stretch_mode_display_names = { 240 TRANSLATE_NOOP("AudioStream", "Off (Noisy)"), 241 TRANSLATE_NOOP("AudioStream", "Resampling (Pitch Shift)"), 242 TRANSLATE_NOOP("AudioStream", "Time Stretch (Tempo Change, Best Sound)"), 243 }; 244 245 const char* AudioStream::GetStretchModeName(AudioStretchMode mode) 246 { 247 return (static_cast<u32>(mode) < s_stretch_mode_names.size()) ? s_stretch_mode_names[static_cast<u32>(mode)] : ""; 248 } 249 250 const char* AudioStream::GetStretchModeDisplayName(AudioStretchMode mode) 251 { 252 return (static_cast<u32>(mode) < s_stretch_mode_display_names.size()) ? 253 Host::TranslateToCString("AudioStream", s_stretch_mode_display_names[static_cast<u32>(mode)]) : 254 ""; 255 } 256 257 std::optional<AudioStretchMode> AudioStream::ParseStretchMode(const char* name) 258 { 259 for (u8 i = 0; i < static_cast<u8>(AudioStretchMode::Count); i++) 260 { 261 if (std::strcmp(name, s_stretch_mode_names[i]) == 0) 262 return static_cast<AudioStretchMode>(i); 263 } 264 265 return std::nullopt; 266 } 267 268 u32 AudioStream::GetBufferedFramesRelaxed() const 269 { 270 const u32 rpos = m_rpos.load(std::memory_order_relaxed); 271 const u32 wpos = m_wpos.load(std::memory_order_relaxed); 272 return (wpos + m_buffer_size - rpos) % m_buffer_size; 273 } 274 275 void AudioStream::ReadFrames(SampleType* samples, u32 num_frames) 276 { 277 const u32 available_frames = GetBufferedFramesRelaxed(); 278 u32 frames_to_read = num_frames; 279 u32 silence_frames = 0; 280 281 if (m_filling) 282 { 283 u32 toFill = m_buffer_size / ((m_parameters.stretch_mode != AudioStretchMode::TimeStretch) ? 32 : 400); 284 toFill = GetAlignedBufferSize(toFill); 285 286 if (available_frames < toFill) 287 { 288 silence_frames = num_frames; 289 frames_to_read = 0; 290 } 291 else 292 { 293 m_filling = false; 294 VERBOSE_LOG("Underrun compensation done ({} frames buffered)", toFill); 295 } 296 } 297 298 if (available_frames < frames_to_read) 299 { 300 silence_frames = frames_to_read - available_frames; 301 frames_to_read = available_frames; 302 m_filling = true; 303 304 if (m_parameters.stretch_mode == AudioStretchMode::TimeStretch) 305 StretchUnderrun(); 306 } 307 308 if (frames_to_read > 0) 309 { 310 u32 rpos = m_rpos.load(std::memory_order_acquire); 311 312 u32 end = m_buffer_size - rpos; 313 if (end > frames_to_read) 314 end = frames_to_read; 315 316 // towards the end of the buffer 317 if (end > 0) 318 { 319 m_sample_reader(samples, &m_buffer[rpos * m_internal_channels], end); 320 rpos += end; 321 rpos = (rpos == m_buffer_size) ? 0 : rpos; 322 } 323 324 // after wrapping around 325 const u32 start = frames_to_read - end; 326 if (start > 0) 327 { 328 m_sample_reader(&samples[end * m_output_channels], &m_buffer[0], start); 329 rpos = start; 330 } 331 332 m_rpos.store(rpos, std::memory_order_release); 333 } 334 335 if (silence_frames > 0) 336 { 337 if (frames_to_read > 0) 338 { 339 // super basic resampler - spread the input samples evenly across the output samples. will sound like ass and have 340 // aliasing, but better than popping by inserting silence. 341 const u32 increment = 342 static_cast<u32>(65536.0f * (static_cast<float>(frames_to_read) / static_cast<float>(num_frames))); 343 344 SampleType* resample_ptr = 345 static_cast<SampleType*>(alloca(frames_to_read * m_output_channels * sizeof(SampleType))); 346 std::memcpy(resample_ptr, samples, frames_to_read * m_output_channels * sizeof(SampleType)); 347 348 SampleType* out_ptr = samples; 349 const u32 copy_stride = sizeof(SampleType) * m_output_channels; 350 u32 resample_subpos = 0; 351 for (u32 i = 0; i < num_frames; i++) 352 { 353 std::memcpy(out_ptr, resample_ptr, copy_stride); 354 out_ptr += m_output_channels; 355 356 resample_subpos += increment; 357 resample_ptr += (resample_subpos >> 16) * m_output_channels; 358 resample_subpos %= 65536u; 359 } 360 361 VERBOSE_LOG("Audio buffer underflow, resampled {} frames to {}", frames_to_read, num_frames); 362 } 363 else 364 { 365 // no data, fall back to silence 366 std::memset(samples + (frames_to_read * m_output_channels), 0, silence_frames * m_output_channels * sizeof(s16)); 367 } 368 } 369 370 if (m_volume != 100) 371 { 372 u32 num_samples = num_frames * m_output_channels; 373 374 const u32 aligned_samples = Common::AlignDownPow2(num_samples, 8); 375 num_samples -= aligned_samples; 376 377 const float volume_mult = static_cast<float>(m_volume) / 100.0f; 378 const GSVector4 volume_multv = GSVector4(volume_mult); 379 const SampleType* const aligned_samples_end = samples + aligned_samples; 380 for (; samples != aligned_samples_end; samples += 8) 381 { 382 GSVector4i iv = GSVector4i::load<false>(samples); // [0, 1, 2, 3, 4, 5, 6, 7] 383 GSVector4i iv1 = iv.upl16(iv); // [0, 0, 1, 1, 2, 2, 3, 3] 384 GSVector4i iv2 = iv.uph16(iv); // [4, 4, 5, 5, 6, 6, 7, 7] 385 iv1 = iv1.sra32<16>(); // [0, 1, 2, 3] 386 iv2 = iv2.sra32<16>(); // [4, 5, 6, 7] 387 GSVector4 fv1 = GSVector4(iv1); // [f0, f1, f2, f3] 388 GSVector4 fv2 = GSVector4(iv2); // [f4, f5, f6, f7] 389 fv1 = fv1 * volume_multv; // [f0, f1, f2, f3] 390 fv2 = fv2 * volume_multv; // [f4, f5, f6, f7] 391 iv1 = GSVector4i(fv1); // [0, 1, 2, 3] 392 iv2 = GSVector4i(fv2); // [4, 5, 6, 7] 393 iv = iv1.ps32(iv2); // [0, 1, 2, 3, 4, 5, 6, 7] 394 GSVector4i::store<false>(samples, iv); 395 } 396 397 while (num_samples > 0) 398 { 399 *samples = static_cast<s16>(std::clamp(static_cast<float>(*samples) * volume_mult, -32768.0f, 32767.0f)); 400 samples++; 401 num_samples--; 402 } 403 } 404 } 405 406 void AudioStream::StereoSampleReaderImpl(SampleType* dest, const SampleType* src, u32 num_frames) 407 { 408 std::memcpy(dest, src, num_frames * 2 * sizeof(SampleType)); 409 } 410 411 void AudioStream::InternalWriteFrames(s16* data, u32 num_frames) 412 { 413 const u32 free = m_buffer_size - GetBufferedFramesRelaxed(); 414 if (free <= num_frames) 415 { 416 if (m_parameters.stretch_mode == AudioStretchMode::TimeStretch) 417 { 418 StretchOverrun(); 419 } 420 else 421 { 422 DEBUG_LOG("Buffer overrun, chunk dropped"); 423 return; 424 } 425 } 426 427 u32 wpos = m_wpos.load(std::memory_order_acquire); 428 429 // wrapping around the end of the buffer? 430 if ((m_buffer_size - wpos) <= num_frames) 431 { 432 // needs to be written in two parts 433 const u32 end = m_buffer_size - wpos; 434 const u32 start = num_frames - end; 435 436 // start is zero when this chunk reaches exactly the end 437 std::memcpy(&m_buffer[wpos * m_internal_channels], data, end * m_internal_channels * sizeof(SampleType)); 438 if (start > 0) 439 std::memcpy(&m_buffer[0], data + end * m_internal_channels, start * m_internal_channels * sizeof(SampleType)); 440 441 wpos = start; 442 } 443 else 444 { 445 // no split 446 std::memcpy(&m_buffer[wpos * m_internal_channels], data, num_frames * m_internal_channels * sizeof(SampleType)); 447 wpos += num_frames; 448 } 449 450 m_wpos.store(wpos, std::memory_order_release); 451 } 452 453 void AudioStream::BaseInitialize(SampleReader sample_reader) 454 { 455 m_sample_reader = sample_reader; 456 457 AllocateBuffer(); 458 ExpandAllocate(); 459 StretchAllocate(); 460 } 461 462 void AudioStream::AllocateBuffer() 463 { 464 // use a larger buffer when time stretching, since we need more input 465 // TODO: do we really? it's more the output... 466 const u32 multiplier = (m_parameters.stretch_mode == AudioStretchMode::TimeStretch) ? 467 16 : 468 ((m_parameters.stretch_mode == AudioStretchMode::Off) ? 1 : 2); 469 m_buffer_size = GetAlignedBufferSize(((m_parameters.buffer_ms * multiplier) * m_sample_rate) / 1000); 470 m_target_buffer_size = GetAlignedBufferSize((m_sample_rate * m_parameters.buffer_ms) / 1000u); 471 472 m_buffer = std::make_unique<s16[]>(m_buffer_size * m_internal_channels); 473 m_staging_buffer = std::make_unique<s16[]>(CHUNK_SIZE * m_internal_channels); 474 m_float_buffer = std::make_unique<float[]>(CHUNK_SIZE * m_internal_channels); 475 476 if (IsExpansionEnabled()) 477 m_expand_buffer = std::make_unique<float[]>(m_parameters.expand_block_size * NUM_INPUT_CHANNELS); 478 479 DEV_LOG( 480 "Allocated buffer of {} frames for buffer of {} ms [expansion {} (block size {}), stretch {}, target size {}].", 481 m_buffer_size, m_parameters.buffer_ms, GetExpansionModeName(m_parameters.expansion_mode), 482 m_parameters.expand_block_size, GetStretchModeName(m_parameters.stretch_mode), m_target_buffer_size); 483 } 484 485 void AudioStream::DestroyBuffer() 486 { 487 m_expand_buffer.reset(); 488 m_staging_buffer.reset(); 489 m_float_buffer.reset(); 490 m_buffer.reset(); 491 m_buffer_size = 0; 492 m_wpos.store(0, std::memory_order_release); 493 m_rpos.store(0, std::memory_order_release); 494 } 495 496 void AudioStream::EmptyBuffer() 497 { 498 #ifndef __ANDROID__ 499 if (IsExpansionEnabled()) 500 { 501 m_expander->Flush(); 502 m_expand_output_buffer = nullptr; 503 m_expand_buffer_pos = 0; 504 } 505 #endif 506 507 if (IsStretchEnabled()) 508 { 509 soundtouch_clear(m_soundtouch); 510 if (m_parameters.stretch_mode == AudioStretchMode::TimeStretch) 511 soundtouch_setTempo(m_soundtouch, m_nominal_rate); 512 } 513 514 m_wpos.store(m_rpos.load(std::memory_order_acquire), std::memory_order_release); 515 } 516 517 void AudioStream::SetNominalRate(float tempo) 518 { 519 m_nominal_rate = tempo; 520 if (m_parameters.stretch_mode == AudioStretchMode::Resample) 521 soundtouch_setRate(m_soundtouch, tempo); 522 else if (m_parameters.stretch_mode == AudioStretchMode::TimeStretch && m_stretch_inactive) 523 soundtouch_setTempo(m_soundtouch, tempo); 524 } 525 526 void AudioStream::SetStretchMode(AudioStretchMode mode) 527 { 528 if (m_parameters.stretch_mode == mode) 529 return; 530 531 // can't resize the buffers while paused 532 bool paused = m_paused; 533 if (!paused) 534 SetPaused(true); 535 536 DestroyBuffer(); 537 StretchDestroy(); 538 m_parameters.stretch_mode = mode; 539 540 AllocateBuffer(); 541 if (m_parameters.stretch_mode != AudioStretchMode::Off) 542 StretchAllocate(); 543 544 if (!paused) 545 SetPaused(false); 546 } 547 548 void AudioStream::SetPaused(bool paused) 549 { 550 m_paused = paused; 551 } 552 553 void AudioStream::SetOutputVolume(u32 volume) 554 { 555 m_volume = volume; 556 } 557 558 void AudioStream::BeginWrite(SampleType** buffer_ptr, u32* num_frames) 559 { 560 // TODO: Write directly to buffer when not using stretching. 561 *buffer_ptr = &m_staging_buffer[m_staging_buffer_pos]; 562 *num_frames = CHUNK_SIZE - (m_staging_buffer_pos / NUM_INPUT_CHANNELS); 563 } 564 565 static void S16ChunkToFloat(const s16* src, float* dst, u32 num_samples) 566 { 567 constexpr GSVector4 S16_TO_FLOAT_V = GSVector4::cxpr(1.0f / 32767.0f); 568 569 const u32 iterations = (num_samples + 7) / 8; 570 for (u32 i = 0; i < iterations; i++) 571 { 572 const GSVector4i sv = GSVector4i::load<false>(src); 573 src += 8; 574 575 GSVector4i iv1 = sv.upl16(sv); // [0, 0, 1, 1, 2, 2, 3, 3] 576 GSVector4i iv2 = sv.uph16(sv); // [4, 4, 5, 5, 6, 6, 7, 7] 577 iv1 = iv1.sra32<16>(); // [0, 1, 2, 3] 578 iv2 = iv2.sra32<16>(); // [4, 5, 6, 7] 579 GSVector4 fv1 = GSVector4(iv1); // [f0, f1, f2, f3] 580 GSVector4 fv2 = GSVector4(iv2); // [f4, f5, f6, f7] 581 fv1 = fv1 * S16_TO_FLOAT_V; 582 fv2 = fv2 * S16_TO_FLOAT_V; 583 584 GSVector4::store<false>(dst + 0, fv1); 585 GSVector4::store<false>(dst + 4, fv2); 586 dst += 8; 587 } 588 } 589 590 static void FloatChunkToS16(s16* dst, const float* src, u32 num_samples) 591 { 592 const GSVector4 FLOAT_TO_S16_V = GSVector4::cxpr(32767.0f); 593 594 const u32 iterations = (num_samples + 7) / 8; 595 for (u32 i = 0; i < iterations; i++) 596 { 597 GSVector4 fv1 = GSVector4::load<false>(src + 0); 598 GSVector4 fv2 = GSVector4::load<false>(src + 4); 599 src += 8; 600 601 fv1 = fv1 * FLOAT_TO_S16_V; 602 fv2 = fv2 * FLOAT_TO_S16_V; 603 GSVector4i iv1 = GSVector4i(fv1); 604 GSVector4i iv2 = GSVector4i(fv2); 605 606 const GSVector4i iv = iv1.ps32(iv2); 607 GSVector4i::store<false>(dst, iv); 608 dst += 8; 609 } 610 } 611 612 void AudioStream::ExpandAllocate() 613 { 614 DebugAssert(!m_expander); 615 if (m_parameters.expansion_mode == AudioExpansionMode::Disabled) 616 return; 617 618 #ifndef __ANDROID__ 619 static constexpr std::array<std::pair<FreeSurroundDecoder::ChannelSetup, bool>, 620 static_cast<size_t>(AudioExpansionMode::Count)> 621 channel_setup_mapping = {{ 622 {FreeSurroundDecoder::ChannelSetup::Stereo, false}, // Disabled 623 {FreeSurroundDecoder::ChannelSetup::Stereo, true}, // StereoLFE 624 {FreeSurroundDecoder::ChannelSetup::Surround41, false}, // Quadraphonic 625 {FreeSurroundDecoder::ChannelSetup::Surround41, true}, // QuadraphonicLFE 626 {FreeSurroundDecoder::ChannelSetup::Surround51, true}, // Surround51 627 {FreeSurroundDecoder::ChannelSetup::Surround71, true}, // Surround71 628 }}; 629 630 const auto [fs_setup, fs_lfe] = channel_setup_mapping[static_cast<size_t>(m_parameters.expansion_mode)]; 631 632 m_expander = std::make_unique<FreeSurroundDecoder>(fs_setup, m_parameters.expand_block_size); 633 m_expander->SetBassRedirection(fs_lfe); 634 m_expander->SetCircularWrap(m_parameters.expand_circular_wrap); 635 m_expander->SetShift(m_parameters.expand_shift); 636 m_expander->SetDepth(m_parameters.expand_depth); 637 m_expander->SetFocus(m_parameters.expand_focus); 638 m_expander->SetCenterImage(m_parameters.expand_center_image); 639 m_expander->SetFrontSeparation(m_parameters.expand_front_separation); 640 m_expander->SetRearSeparation(m_parameters.expand_rear_separation); 641 m_expander->SetLowCutoff(static_cast<float>(m_parameters.expand_low_cutoff) / m_sample_rate * 2); 642 m_expander->SetHighCutoff(static_cast<float>(m_parameters.expand_high_cutoff) / m_sample_rate * 2); 643 #else 644 Panic("Attempting to use expansion on Android."); 645 #endif 646 } 647 648 void AudioStream::EndWrite(u32 num_frames) 649 { 650 // don't bother committing anything when muted 651 if (m_volume == 0) 652 return; 653 654 m_staging_buffer_pos += num_frames * NUM_INPUT_CHANNELS; 655 DebugAssert(m_staging_buffer_pos <= (CHUNK_SIZE * NUM_INPUT_CHANNELS)); 656 if ((m_staging_buffer_pos / NUM_INPUT_CHANNELS) < CHUNK_SIZE) 657 return; 658 659 m_staging_buffer_pos = 0; 660 661 if (!IsExpansionEnabled() && !IsStretchEnabled()) 662 { 663 InternalWriteFrames(m_staging_buffer.get(), CHUNK_SIZE); 664 return; 665 } 666 667 #ifndef __ANDROID__ 668 if (IsExpansionEnabled()) 669 { 670 // StretchWriteBlock() overwrites the staging buffer on output, so we need to copy into the expand buffer first. 671 S16ChunkToFloat(m_staging_buffer.get(), m_expand_buffer.get() + m_expand_buffer_pos * NUM_INPUT_CHANNELS, 672 CHUNK_SIZE * NUM_INPUT_CHANNELS); 673 674 // Output the corresponding block. 675 if (m_expand_output_buffer) 676 StretchWriteBlock(m_expand_output_buffer + m_expand_buffer_pos * m_internal_channels); 677 678 // Decode the next block if we buffered enough. 679 m_expand_buffer_pos += CHUNK_SIZE; 680 if (m_expand_buffer_pos == m_parameters.expand_block_size) 681 { 682 m_expand_buffer_pos = 0; 683 m_expand_output_buffer = m_expander->Decode(m_expand_buffer.get()); 684 } 685 } 686 else 687 #endif 688 { 689 S16ChunkToFloat(m_staging_buffer.get(), m_float_buffer.get(), CHUNK_SIZE * NUM_INPUT_CHANNELS); 690 StretchWriteBlock(m_float_buffer.get()); 691 } 692 } 693 694 // Time stretching algorithm based on PCSX2 implementation. 695 696 template<class T> 697 ALWAYS_INLINE static bool IsInRange(const T& val, const T& min, const T& max) 698 { 699 return (min <= val && val <= max); 700 } 701 702 void AudioStream::StretchAllocate() 703 { 704 if (m_parameters.stretch_mode == AudioStretchMode::Off) 705 return; 706 707 m_soundtouch = soundtouch_createInstance(); 708 soundtouch_setSampleRate(m_soundtouch, m_sample_rate); 709 soundtouch_setChannels(m_soundtouch, m_internal_channels); 710 711 soundtouch_setSetting(m_soundtouch, SETTING_USE_QUICKSEEK, m_parameters.stretch_use_quickseek); 712 soundtouch_setSetting(m_soundtouch, SETTING_USE_AA_FILTER, m_parameters.stretch_use_aa_filter); 713 714 soundtouch_setSetting(m_soundtouch, SETTING_SEQUENCE_MS, m_parameters.stretch_sequence_length_ms); 715 soundtouch_setSetting(m_soundtouch, SETTING_SEEKWINDOW_MS, m_parameters.stretch_seekwindow_ms); 716 soundtouch_setSetting(m_soundtouch, SETTING_OVERLAP_MS, m_parameters.stretch_overlap_ms); 717 718 if (m_parameters.stretch_mode == AudioStretchMode::Resample) 719 soundtouch_setRate(m_soundtouch, m_nominal_rate); 720 else 721 soundtouch_setTempo(m_soundtouch, m_nominal_rate); 722 723 m_stretch_reset = STRETCH_RESET_THRESHOLD; 724 m_stretch_inactive = false; 725 m_stretch_ok_count = 0; 726 m_dynamic_target_usage = 0.0f; 727 m_average_position = 0; 728 m_average_available = 0; 729 730 m_staging_buffer_pos = 0; 731 } 732 733 void AudioStream::StretchDestroy() 734 { 735 if (m_soundtouch) 736 { 737 soundtouch_destroyInstance(m_soundtouch); 738 m_soundtouch = nullptr; 739 } 740 } 741 742 void AudioStream::StretchWriteBlock(const float* block) 743 { 744 if (IsStretchEnabled()) 745 { 746 soundtouch_putSamples(m_soundtouch, block, CHUNK_SIZE); 747 748 u32 tempProgress; 749 while (tempProgress = soundtouch_receiveSamples(m_soundtouch, m_float_buffer.get(), CHUNK_SIZE), tempProgress != 0) 750 { 751 FloatChunkToS16(m_staging_buffer.get(), m_float_buffer.get(), tempProgress * m_internal_channels); 752 InternalWriteFrames(m_staging_buffer.get(), tempProgress); 753 } 754 755 if (m_parameters.stretch_mode == AudioStretchMode::TimeStretch) 756 UpdateStretchTempo(); 757 } 758 else 759 { 760 FloatChunkToS16(m_staging_buffer.get(), block, CHUNK_SIZE * m_internal_channels); 761 InternalWriteFrames(m_staging_buffer.get(), CHUNK_SIZE); 762 } 763 } 764 765 float AudioStream::AddAndGetAverageTempo(float val) 766 { 767 if (m_stretch_reset >= STRETCH_RESET_THRESHOLD) 768 m_average_available = 0; 769 if (m_average_available < AVERAGING_BUFFER_SIZE) 770 m_average_available++; 771 772 m_average_fullness[m_average_position] = val; 773 m_average_position = (m_average_position + 1U) % AVERAGING_BUFFER_SIZE; 774 775 const u32 actual_window = std::min<u32>(m_average_available, AVERAGING_WINDOW); 776 const u32 first_index = (m_average_position - actual_window + AVERAGING_BUFFER_SIZE) % AVERAGING_BUFFER_SIZE; 777 778 float sum = 0; 779 for (u32 i = first_index; i < first_index + actual_window; i++) 780 sum += m_average_fullness[i % AVERAGING_BUFFER_SIZE]; 781 sum = sum / actual_window; 782 783 return (sum != 0.0f) ? sum : 1.0f; 784 } 785 786 void AudioStream::UpdateStretchTempo() 787 { 788 static constexpr float MIN_TEMPO = 0.05f; 789 static constexpr float MAX_TEMPO = 50.0f; 790 791 // Which range we will run in 1:1 mode for. 792 static constexpr float INACTIVE_GOOD_FACTOR = 1.04f; 793 static constexpr float INACTIVE_BAD_FACTOR = 1.2f; 794 static constexpr u32 INACTIVE_MIN_OK_COUNT = 50; 795 static constexpr u32 COMPENSATION_DIVIDER = 100; 796 797 float base_target_usage = static_cast<float>(m_target_buffer_size) * m_nominal_rate; 798 799 // state vars 800 if (m_stretch_reset >= STRETCH_RESET_THRESHOLD) 801 { 802 VERBOSE_LOG("___ Stretcher is being reset."); 803 m_stretch_inactive = false; 804 m_stretch_ok_count = 0; 805 m_dynamic_target_usage = base_target_usage; 806 } 807 808 const u32 ibuffer_usage = GetBufferedFramesRelaxed(); 809 float buffer_usage = static_cast<float>(ibuffer_usage); 810 float tempo = buffer_usage / m_dynamic_target_usage; 811 tempo = AddAndGetAverageTempo(tempo); 812 813 // Dampening when we get close to target. 814 if (tempo < 2.0f) 815 tempo = std::sqrt(tempo); 816 817 tempo = std::clamp(tempo, MIN_TEMPO, MAX_TEMPO); 818 819 if (tempo < 1.0f) 820 base_target_usage /= std::sqrt(tempo); 821 822 m_dynamic_target_usage += 823 static_cast<float>(base_target_usage / tempo - m_dynamic_target_usage) / static_cast<float>(COMPENSATION_DIVIDER); 824 if (IsInRange(tempo, 0.9f, 1.1f) && 825 IsInRange(m_dynamic_target_usage, base_target_usage * 0.9f, base_target_usage * 1.1f)) 826 { 827 m_dynamic_target_usage = base_target_usage; 828 } 829 830 if (!m_stretch_inactive) 831 { 832 if (IsInRange(tempo, 1.0f / INACTIVE_GOOD_FACTOR, INACTIVE_GOOD_FACTOR)) 833 m_stretch_ok_count++; 834 else 835 m_stretch_ok_count = 0; 836 837 if (m_stretch_ok_count >= INACTIVE_MIN_OK_COUNT) 838 { 839 VERBOSE_LOG("=== Stretcher is now inactive."); 840 m_stretch_inactive = true; 841 } 842 } 843 else if (!IsInRange(tempo, 1.0f / INACTIVE_BAD_FACTOR, INACTIVE_BAD_FACTOR)) 844 { 845 VERBOSE_LOG("~~~ Stretcher is now active @ tempo {}.", tempo); 846 m_stretch_inactive = false; 847 m_stretch_ok_count = 0; 848 } 849 850 if (m_stretch_inactive) 851 tempo = m_nominal_rate; 852 853 if constexpr (LOG_TIMESTRETCH_STATS) 854 { 855 static int iterations = 0; 856 static u64 last_log_time = 0; 857 858 const u64 now = Common::Timer::GetCurrentValue(); 859 860 if (Common::Timer::ConvertValueToSeconds(now - last_log_time) > 1.0f) 861 { 862 VERBOSE_LOG("buffers: {:4d} ms ({:3.0f}%), tempo: {}, comp: {:2.3f}, iters: {}, reset:{}", 863 (ibuffer_usage * 1000u) / m_sample_rate, 100.0f * buffer_usage / base_target_usage, tempo, 864 m_dynamic_target_usage / base_target_usage, iterations, m_stretch_reset); 865 866 last_log_time = now; 867 iterations = 0; 868 } 869 870 iterations++; 871 } 872 873 soundtouch_setTempo(m_soundtouch, tempo); 874 875 if (m_stretch_reset >= STRETCH_RESET_THRESHOLD) 876 m_stretch_reset = 0; 877 } 878 879 void AudioStream::StretchUnderrun() 880 { 881 // Didn't produce enough frames in time. 882 m_stretch_reset++; 883 } 884 885 void AudioStream::StretchOverrun() 886 { 887 // Produced more frames than can fit in the buffer. 888 m_stretch_reset++; 889 890 // Drop two packets to give the time stretcher a bit more time to slow things down. 891 const u32 discard = CHUNK_SIZE * 2; 892 m_rpos.store((m_rpos.load(std::memory_order_acquire) + discard) % m_buffer_size, std::memory_order_release); 893 } 894 895 void AudioStreamParameters::Load(SettingsInterface& si, const char* section) 896 { 897 stretch_mode = 898 AudioStream::ParseStretchMode( 899 si.GetStringValue(section, "StretchMode", AudioStream::GetStretchModeName(DEFAULT_STRETCH_MODE)).c_str()) 900 .value_or(DEFAULT_STRETCH_MODE); 901 #ifndef __ANDROID__ 902 expansion_mode = 903 AudioStream::ParseExpansionMode( 904 si.GetStringValue(section, "ExpansionMode", AudioStream::GetExpansionModeName(DEFAULT_EXPANSION_MODE)).c_str()) 905 .value_or(DEFAULT_EXPANSION_MODE); 906 #else 907 expansion_mode = AudioExpansionMode::Disabled; 908 #endif 909 output_latency_ms = static_cast<u16>(std::min<u32>( 910 si.GetUIntValue(section, "OutputLatencyMS", DEFAULT_OUTPUT_LATENCY_MS), std::numeric_limits<u16>::max())); 911 output_latency_minimal = si.GetBoolValue(section, "OutputLatencyMinimal", DEFAULT_OUTPUT_LATENCY_MINIMAL); 912 buffer_ms = static_cast<u16>( 913 std::min<u32>(si.GetUIntValue(section, "BufferMS", DEFAULT_BUFFER_MS), std::numeric_limits<u16>::max())); 914 915 stretch_sequence_length_ms = 916 static_cast<u16>(std::min<u32>(si.GetUIntValue(section, "StretchSequenceLengthMS", DEFAULT_STRETCH_SEQUENCE_LENGTH), 917 std::numeric_limits<u16>::max())); 918 stretch_seekwindow_ms = static_cast<u16>(std::min<u32>( 919 si.GetUIntValue(section, "StretchSeekWindowMS", DEFAULT_STRETCH_SEEKWINDOW), std::numeric_limits<u16>::max())); 920 stretch_overlap_ms = static_cast<u16>(std::min<u32>( 921 si.GetUIntValue(section, "StretchOverlapMS", DEFAULT_STRETCH_OVERLAP), std::numeric_limits<u16>::max())); 922 stretch_use_quickseek = si.GetBoolValue(section, "StretchUseQuickSeek", DEFAULT_STRETCH_USE_QUICKSEEK); 923 stretch_use_aa_filter = si.GetBoolValue(section, "StretchUseAAFilter", DEFAULT_STRETCH_USE_AA_FILTER); 924 925 expand_block_size = static_cast<u16>(std::min<u32>( 926 si.GetUIntValue(section, "ExpandBlockSize", DEFAULT_EXPAND_BLOCK_SIZE), std::numeric_limits<u16>::max())); 927 expand_block_size = std::clamp<u16>( 928 Common::IsPow2(expand_block_size) ? expand_block_size : Common::NextPow2(expand_block_size), 128, 8192); 929 expand_circular_wrap = 930 std::clamp(si.GetFloatValue(section, "ExpandCircularWrap", DEFAULT_EXPAND_CIRCULAR_WRAP), 0.0f, 360.0f); 931 expand_shift = std::clamp(si.GetFloatValue(section, "ExpandShift", DEFAULT_EXPAND_SHIFT), -1.0f, 1.0f); 932 expand_depth = std::clamp(si.GetFloatValue(section, "ExpandDepth", DEFAULT_EXPAND_DEPTH), 0.0f, 5.0f); 933 expand_focus = std::clamp(si.GetFloatValue(section, "ExpandFocus", DEFAULT_EXPAND_FOCUS), -1.0f, 1.0f); 934 expand_center_image = 935 std::clamp(si.GetFloatValue(section, "ExpandCenterImage", DEFAULT_EXPAND_CENTER_IMAGE), 0.0f, 1.0f); 936 expand_front_separation = 937 std::clamp(si.GetFloatValue(section, "ExpandFrontSeparation", DEFAULT_EXPAND_FRONT_SEPARATION), 0.0f, 10.0f); 938 expand_rear_separation = 939 std::clamp(si.GetFloatValue(section, "ExpandRearSeparation", DEFAULT_EXPAND_REAR_SEPARATION), 0.0f, 10.0f); 940 expand_low_cutoff = 941 static_cast<u8>(std::min<u32>(si.GetUIntValue(section, "ExpandLowCutoff", DEFAULT_EXPAND_LOW_CUTOFF), 100)); 942 expand_high_cutoff = 943 static_cast<u8>(std::min<u32>(si.GetUIntValue(section, "ExpandHighCutoff", DEFAULT_EXPAND_HIGH_CUTOFF), 100)); 944 } 945 946 void AudioStreamParameters::Save(SettingsInterface& si, const char* section) const 947 { 948 si.SetStringValue(section, "StretchMode", AudioStream::GetStretchModeName(stretch_mode)); 949 si.SetStringValue(section, "ExpansionMode", AudioStream::GetExpansionModeName(expansion_mode)); 950 si.SetUIntValue(section, "BufferMS", buffer_ms); 951 si.SetUIntValue(section, "OutputLatencyMS", output_latency_ms); 952 si.SetBoolValue(section, "OutputLatencyMinimal", output_latency_minimal); 953 954 si.SetUIntValue(section, "StretchSequenceLengthMS", stretch_sequence_length_ms); 955 si.SetUIntValue(section, "StretchSeekWindowMS", stretch_seekwindow_ms); 956 si.SetUIntValue(section, "StretchOverlapMS", stretch_overlap_ms); 957 si.SetBoolValue(section, "StretchUseQuickSeek", stretch_use_quickseek); 958 si.SetBoolValue(section, "StretchUseAAFilter", stretch_use_aa_filter); 959 960 si.SetUIntValue(section, "ExpandBlockSize", expand_block_size); 961 si.SetFloatValue(section, "ExpandCircularWrap", expand_circular_wrap); 962 si.SetFloatValue(section, "ExpandShift", expand_shift); 963 si.SetFloatValue(section, "ExpandDepth", expand_depth); 964 si.SetFloatValue(section, "ExpandFocus", expand_focus); 965 si.SetFloatValue(section, "ExpandCenterImage", expand_center_image); 966 si.SetFloatValue(section, "ExpandFrontSeparation", expand_front_separation); 967 si.SetFloatValue(section, "ExpandRearSeparation", expand_rear_separation); 968 si.SetUIntValue(section, "ExpandLowCutoff", expand_low_cutoff); 969 si.SetUIntValue(section, "ExpandHighCutoff", expand_high_cutoff); 970 } 971 972 void AudioStreamParameters::Clear(SettingsInterface& si, const char* section) 973 { 974 si.DeleteValue(section, "StretchMode"); 975 si.DeleteValue(section, "ExpansionMode"); 976 si.DeleteValue(section, "BufferMS"); 977 si.DeleteValue(section, "OutputLatencyMS"); 978 si.DeleteValue(section, "OutputLatencyMinimal"); 979 980 si.DeleteValue(section, "StretchSequenceLengthMS"); 981 si.DeleteValue(section, "StretchSeekWindowMS"); 982 si.DeleteValue(section, "StretchOverlapMS"); 983 si.DeleteValue(section, "StretchUseQuickSeek"); 984 si.DeleteValue(section, "StretchUseAAFilter"); 985 986 si.DeleteValue(section, "ExpandBlockSize"); 987 si.DeleteValue(section, "ExpandCircularWrap"); 988 si.DeleteValue(section, "ExpandShift"); 989 si.DeleteValue(section, "ExpandDepth"); 990 si.DeleteValue(section, "ExpandFocus"); 991 si.DeleteValue(section, "ExpandCenterImage"); 992 si.DeleteValue(section, "ExpandFrontSeparation"); 993 si.DeleteValue(section, "ExpandRearSeparation"); 994 si.DeleteValue(section, "ExpandLowCutoff"); 995 si.DeleteValue(section, "ExpandHighCutoff"); 996 } 997 998 bool AudioStreamParameters::operator!=(const AudioStreamParameters& rhs) const 999 { 1000 return (std::memcmp(this, &rhs, sizeof(*this)) != 0); 1001 } 1002 1003 bool AudioStreamParameters::operator==(const AudioStreamParameters& rhs) const 1004 { 1005 return (std::memcmp(this, &rhs, sizeof(*this)) == 0); 1006 }