audio_stream.h (13487B)
1 // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com> 2 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) 3 4 #pragma once 5 6 #include "common/types.h" 7 8 #include <array> 9 #include <atomic> 10 #include <memory> 11 #include <optional> 12 #include <string> 13 #include <vector> 14 15 class Error; 16 class SettingsInterface; 17 18 class FreeSurroundDecoder; 19 namespace soundtouch { 20 class SoundTouch; 21 } 22 23 enum class AudioBackend : u8 24 { 25 Null, 26 #ifndef __ANDROID__ 27 Cubeb, 28 SDL, 29 #else 30 AAudio, 31 OpenSLES, 32 #endif 33 Count 34 }; 35 36 enum class AudioStretchMode : u8 37 { 38 Off, 39 Resample, 40 TimeStretch, 41 Count 42 }; 43 44 enum class AudioExpansionMode : u8 45 { 46 Disabled, 47 StereoLFE, 48 Quadraphonic, 49 QuadraphonicLFE, 50 Surround51, 51 Surround71, 52 Count 53 }; 54 55 struct AudioStreamParameters 56 { 57 AudioStretchMode stretch_mode = DEFAULT_STRETCH_MODE; 58 AudioExpansionMode expansion_mode = DEFAULT_EXPANSION_MODE; 59 u16 buffer_ms = DEFAULT_BUFFER_MS; 60 u16 output_latency_ms = DEFAULT_OUTPUT_LATENCY_MS; 61 bool output_latency_minimal = DEFAULT_OUTPUT_LATENCY_MINIMAL; 62 bool pad1 = false; 63 64 u16 stretch_sequence_length_ms = DEFAULT_STRETCH_SEQUENCE_LENGTH; 65 u16 stretch_seekwindow_ms = DEFAULT_STRETCH_SEEKWINDOW; 66 u16 stretch_overlap_ms = DEFAULT_STRETCH_OVERLAP; 67 bool stretch_use_quickseek = DEFAULT_STRETCH_USE_QUICKSEEK; 68 bool stretch_use_aa_filter = DEFAULT_STRETCH_USE_AA_FILTER; 69 70 float expand_circular_wrap = DEFAULT_EXPAND_CIRCULAR_WRAP; 71 float expand_shift = DEFAULT_EXPAND_SHIFT; 72 float expand_depth = DEFAULT_EXPAND_DEPTH; 73 float expand_focus = DEFAULT_EXPAND_FOCUS; 74 float expand_center_image = DEFAULT_EXPAND_CENTER_IMAGE; 75 float expand_front_separation = DEFAULT_EXPAND_FRONT_SEPARATION; 76 float expand_rear_separation = DEFAULT_EXPAND_REAR_SEPARATION; 77 u16 expand_block_size = DEFAULT_EXPAND_BLOCK_SIZE; 78 u8 expand_low_cutoff = DEFAULT_EXPAND_LOW_CUTOFF; 79 u8 expand_high_cutoff = DEFAULT_EXPAND_HIGH_CUTOFF; 80 81 static constexpr AudioStretchMode DEFAULT_STRETCH_MODE = AudioStretchMode::TimeStretch; 82 static constexpr AudioExpansionMode DEFAULT_EXPANSION_MODE = AudioExpansionMode::Disabled; 83 #ifndef __ANDROID__ 84 static constexpr u16 DEFAULT_BUFFER_MS = 50; 85 static constexpr u16 DEFAULT_OUTPUT_LATENCY_MS = 20; 86 #else 87 static constexpr u16 DEFAULT_BUFFER_MS = 100; 88 static constexpr u16 DEFAULT_OUTPUT_LATENCY_MS = 20; 89 #endif 90 static constexpr bool DEFAULT_OUTPUT_LATENCY_MINIMAL = false; 91 static constexpr u16 DEFAULT_EXPAND_BLOCK_SIZE = 2048; 92 static constexpr float DEFAULT_EXPAND_CIRCULAR_WRAP = 90.0f; 93 static constexpr float DEFAULT_EXPAND_SHIFT = 0.0f; 94 static constexpr float DEFAULT_EXPAND_DEPTH = 1.0f; 95 static constexpr float DEFAULT_EXPAND_FOCUS = 0.0f; 96 static constexpr float DEFAULT_EXPAND_CENTER_IMAGE = 1.0f; 97 static constexpr float DEFAULT_EXPAND_FRONT_SEPARATION = 1.0f; 98 static constexpr float DEFAULT_EXPAND_REAR_SEPARATION = 1.0f; 99 static constexpr u8 DEFAULT_EXPAND_LOW_CUTOFF = 40; 100 static constexpr u8 DEFAULT_EXPAND_HIGH_CUTOFF = 90; 101 102 static constexpr u16 DEFAULT_STRETCH_SEQUENCE_LENGTH = 30; 103 static constexpr u16 DEFAULT_STRETCH_SEEKWINDOW = 20; 104 static constexpr u16 DEFAULT_STRETCH_OVERLAP = 10; 105 106 static constexpr bool DEFAULT_STRETCH_USE_QUICKSEEK = false; 107 static constexpr bool DEFAULT_STRETCH_USE_AA_FILTER = false; 108 109 void Load(SettingsInterface& si, const char* section); 110 void Save(SettingsInterface& si, const char* section) const; 111 void Clear(SettingsInterface& si, const char* section); 112 113 bool operator==(const AudioStreamParameters& rhs) const; 114 bool operator!=(const AudioStreamParameters& rhs) const; 115 }; 116 117 class AudioStream 118 { 119 public: 120 using SampleType = s16; 121 122 static constexpr u32 NUM_INPUT_CHANNELS = 2; 123 static constexpr u32 MAX_OUTPUT_CHANNELS = 8; 124 static constexpr u32 CHUNK_SIZE = 64; 125 static constexpr u32 MIN_EXPANSION_BLOCK_SIZE = 256; 126 static constexpr u32 MAX_EXPANSION_BLOCK_SIZE = 4096; 127 128 #ifndef __ANDROID__ 129 static constexpr AudioBackend DEFAULT_BACKEND = AudioBackend::Cubeb; 130 #else 131 static constexpr AudioBackend DEFAULT_BACKEND = AudioBackend::AAudio; 132 #endif 133 134 struct DeviceInfo 135 { 136 std::string name; 137 std::string display_name; 138 u32 minimum_latency_frames; 139 140 DeviceInfo(std::string name_, std::string display_name_, u32 minimum_latency_); 141 ~DeviceInfo(); 142 }; 143 144 public: 145 virtual ~AudioStream(); 146 147 static u32 GetAlignedBufferSize(u32 size); 148 static u32 GetBufferSizeForMS(u32 sample_rate, u32 ms); 149 static u32 GetMSForBufferSize(u32 sample_rate, u32 buffer_size); 150 151 static std::optional<AudioBackend> ParseBackendName(const char* str); 152 static const char* GetBackendName(AudioBackend backend); 153 static const char* GetBackendDisplayName(AudioBackend backend); 154 155 static const char* GetExpansionModeName(AudioExpansionMode mode); 156 static const char* GetExpansionModeDisplayName(AudioExpansionMode mode); 157 static std::optional<AudioExpansionMode> ParseExpansionMode(const char* name); 158 159 static const char* GetStretchModeName(AudioStretchMode mode); 160 static const char* GetStretchModeDisplayName(AudioStretchMode mode); 161 static std::optional<AudioStretchMode> ParseStretchMode(const char* name); 162 163 ALWAYS_INLINE u32 GetSampleRate() const { return m_sample_rate; } 164 ALWAYS_INLINE u32 GetInternalChannels() const { return m_internal_channels; } 165 ALWAYS_INLINE u32 GetOutputChannels() const { return m_internal_channels; } 166 ALWAYS_INLINE u32 GetBufferSize() const { return m_buffer_size; } 167 ALWAYS_INLINE u32 GetTargetBufferSize() const { return m_target_buffer_size; } 168 ALWAYS_INLINE u32 GetOutputVolume() const { return m_volume; } 169 ALWAYS_INLINE float GetNominalTempo() const { return m_nominal_rate; } 170 ALWAYS_INLINE bool IsPaused() const { return m_paused; } 171 172 u32 GetBufferedFramesRelaxed() const; 173 174 /// Temporarily pauses the stream, preventing it from requesting data. 175 virtual void SetPaused(bool paused); 176 177 void SetOutputVolume(u32 volume); 178 179 void BeginWrite(SampleType** buffer_ptr, u32* num_frames); 180 void EndWrite(u32 num_frames); 181 182 void EmptyBuffer(); 183 184 /// Nominal rate is used for both resampling and timestretching, input samples are assumed to be this amount faster 185 /// than the sample rate. 186 void SetNominalRate(float tempo); 187 188 void SetStretchMode(AudioStretchMode mode); 189 190 static std::vector<std::pair<std::string, std::string>> GetDriverNames(AudioBackend backend); 191 static std::vector<DeviceInfo> GetOutputDevices(AudioBackend backend, const char* driver, u32 sample_rate); 192 static std::unique_ptr<AudioStream> CreateStream(AudioBackend backend, u32 sample_rate, 193 const AudioStreamParameters& parameters, const char* driver_name, 194 const char* device_name, Error* error = nullptr); 195 static std::unique_ptr<AudioStream> CreateNullStream(u32 sample_rate, u32 buffer_ms); 196 197 protected: 198 enum ReadChannel : u8 199 { 200 READ_CHANNEL_FRONT_LEFT, 201 READ_CHANNEL_FRONT_CENTER, 202 READ_CHANNEL_FRONT_RIGHT, 203 READ_CHANNEL_SIDE_LEFT, 204 READ_CHANNEL_SIDE_RIGHT, 205 READ_CHANNEL_REAR_LEFT, 206 READ_CHANNEL_REAR_RIGHT, 207 READ_CHANNEL_LFE, 208 READ_CHANNEL_NONE 209 }; 210 211 using SampleReader = void (*)(SampleType* dest, const SampleType* src, u32 num_frames); 212 213 AudioStream(u32 sample_rate, const AudioStreamParameters& parameters); 214 void BaseInitialize(SampleReader sample_reader); 215 216 void ReadFrames(SampleType* samples, u32 num_frames); 217 218 template<AudioExpansionMode mode, ReadChannel c0 = READ_CHANNEL_NONE, ReadChannel c1 = READ_CHANNEL_NONE, 219 ReadChannel c2 = READ_CHANNEL_NONE, ReadChannel c3 = READ_CHANNEL_NONE, ReadChannel c4 = READ_CHANNEL_NONE, 220 ReadChannel c5 = READ_CHANNEL_NONE, ReadChannel c6 = READ_CHANNEL_NONE, ReadChannel c7 = READ_CHANNEL_NONE> 221 static void SampleReaderImpl(SampleType* dest, const SampleType* src, u32 num_frames); 222 static void StereoSampleReaderImpl(SampleType* dest, const SampleType* src, u32 num_frames); 223 224 u32 m_sample_rate = 0; 225 u32 m_volume = 100; 226 AudioStreamParameters m_parameters; 227 u8 m_internal_channels = 0; 228 u8 m_output_channels = 0; 229 bool m_stretch_inactive = false; 230 bool m_filling = false; 231 bool m_paused = false; 232 233 private: 234 static constexpr u32 AVERAGING_BUFFER_SIZE = 256; 235 static constexpr u32 AVERAGING_WINDOW = 50; 236 static constexpr u32 STRETCH_RESET_THRESHOLD = 5; 237 static constexpr u32 TARGET_IPS = 691; 238 239 #ifndef __ANDROID__ 240 static std::vector<std::pair<std::string, std::string>> GetCubebDriverNames(); 241 static std::vector<DeviceInfo> GetCubebOutputDevices(const char* driver, u32 sample_rate); 242 static std::unique_ptr<AudioStream> CreateCubebAudioStream(u32 sample_rate, const AudioStreamParameters& parameters, 243 const char* driver_name, const char* device_name, 244 Error* error); 245 static std::unique_ptr<AudioStream> CreateSDLAudioStream(u32 sample_rate, const AudioStreamParameters& parameters, 246 Error* error); 247 #else 248 static std::unique_ptr<AudioStream> CreateAAudioAudioStream(u32 sample_rate, const AudioStreamParameters& parameters, 249 Error* error); 250 static std::unique_ptr<AudioStream> CreateOpenSLESAudioStream(u32 sample_rate, 251 const AudioStreamParameters& parameters, Error* error); 252 #endif 253 254 ALWAYS_INLINE bool IsExpansionEnabled() const { return m_parameters.expansion_mode != AudioExpansionMode::Disabled; } 255 ALWAYS_INLINE bool IsStretchEnabled() const { return m_parameters.stretch_mode != AudioStretchMode::Off; } 256 257 void AllocateBuffer(); 258 void DestroyBuffer(); 259 260 void InternalWriteFrames(SampleType* samples, u32 num_frames); 261 262 #ifndef __ANDROID__ 263 void ExpandAllocate(); 264 #endif 265 266 void StretchAllocate(); 267 void StretchDestroy(); 268 void StretchWriteBlock(const float* block); 269 void StretchUnderrun(); 270 void StretchOverrun(); 271 272 float AddAndGetAverageTempo(float val); 273 void UpdateStretchTempo(); 274 275 u32 m_buffer_size = 0; 276 std::unique_ptr<s16[]> m_buffer; 277 SampleReader m_sample_reader = nullptr; 278 279 std::atomic<u32> m_rpos{0}; 280 std::atomic<u32> m_wpos{0}; 281 282 void* m_soundtouch = nullptr; 283 284 u32 m_target_buffer_size = 0; 285 u32 m_stretch_reset = STRETCH_RESET_THRESHOLD; 286 287 u32 m_stretch_ok_count = 0; 288 float m_nominal_rate = 1.0f; 289 float m_dynamic_target_usage = 0.0f; 290 291 u32 m_average_position = 0; 292 u32 m_average_available = 0; 293 u32 m_staging_buffer_pos = 0; 294 295 std::array<float, AVERAGING_BUFFER_SIZE> m_average_fullness = {}; 296 297 // temporary staging buffer, used for timestretching 298 std::unique_ptr<s16[]> m_staging_buffer; 299 300 // float buffer, soundtouch only accepts float samples as input 301 std::unique_ptr<float[]> m_float_buffer; 302 303 #ifndef __ANDROID__ 304 std::unique_ptr<FreeSurroundDecoder> m_expander; 305 306 // block buffer for expansion 307 std::unique_ptr<float[]> m_expand_buffer; 308 float* m_expand_output_buffer = nullptr; 309 u32 m_expand_buffer_pos = 0; 310 #endif 311 }; 312 313 template<AudioExpansionMode mode, AudioStream::ReadChannel c0, AudioStream::ReadChannel c1, AudioStream::ReadChannel c2, 314 AudioStream::ReadChannel c3, AudioStream::ReadChannel c4, AudioStream::ReadChannel c5, 315 AudioStream::ReadChannel c6, AudioStream::ReadChannel c7> 316 void AudioStream::SampleReaderImpl(SampleType* dest, const SampleType* src, u32 num_frames) 317 { 318 static_assert(READ_CHANNEL_NONE == MAX_OUTPUT_CHANNELS); 319 static constexpr const std::array<std::pair<std::array<s8, MAX_OUTPUT_CHANNELS>, u8>, 320 static_cast<size_t>(AudioExpansionMode::Count)> 321 luts = {{ 322 // FL FC FR SL SR RL RR LFE 323 {{0, -1, 1, -1, -1, -1, -1, -1}, 2}, // Disabled 324 {{0, -1, 1, -1, -1, -1, -1, 2}, 3}, // StereoLFE 325 {{0, -1, 1, -1, -1, 2, 3, -1}, 5}, // Quadraphonic 326 {{0, -1, 2, -1, -1, 2, 3, 4}, 5}, // QuadraphonicLFE 327 {{0, 1, 2, -1, -1, 3, 4, 5}, 6}, // Surround51 328 {{0, 1, 2, 3, 4, 5, 6, 7}, 8}, // Surround71 329 }}; 330 constexpr const auto& lut = luts[static_cast<size_t>(mode)].first; 331 for (u32 i = 0; i < num_frames; i++) 332 { 333 if constexpr (c0 != READ_CHANNEL_NONE) 334 { 335 static_assert(lut[c0] >= 0 && lut[c0] < MAX_OUTPUT_CHANNELS); 336 *(dest++) = src[lut[c0]]; 337 } 338 if constexpr (c1 != READ_CHANNEL_NONE) 339 { 340 static_assert(lut[c1] >= 0 && lut[c1] < MAX_OUTPUT_CHANNELS); 341 *(dest++) = src[lut[c1]]; 342 } 343 if constexpr (c2 != READ_CHANNEL_NONE) 344 { 345 static_assert(lut[c2] >= 0 && lut[c2] < MAX_OUTPUT_CHANNELS); 346 *(dest++) = src[lut[c2]]; 347 } 348 if constexpr (c3 != READ_CHANNEL_NONE) 349 { 350 static_assert(lut[c3] >= 0 && lut[c3] < MAX_OUTPUT_CHANNELS); 351 *(dest++) = src[lut[c3]]; 352 } 353 if constexpr (c4 != READ_CHANNEL_NONE) 354 { 355 static_assert(lut[c4] >= 0 && lut[c4] < MAX_OUTPUT_CHANNELS); 356 *(dest++) = src[lut[c4]]; 357 } 358 if constexpr (c5 != READ_CHANNEL_NONE) 359 { 360 static_assert(lut[c5] >= 0 && lut[c5] < MAX_OUTPUT_CHANNELS); 361 *(dest++) = src[lut[c5]]; 362 } 363 if constexpr (c6 != READ_CHANNEL_NONE) 364 { 365 static_assert(lut[c6] >= 0 && lut[c6] < MAX_OUTPUT_CHANNELS); 366 *(dest++) = src[lut[c6]]; 367 } 368 if constexpr (c7 != READ_CHANNEL_NONE) 369 { 370 static_assert(lut[c7] >= 0 && lut[c7] < MAX_OUTPUT_CHANNELS); 371 *(dest++) = src[lut[c7]]; 372 } 373 374 src += luts[static_cast<size_t>(mode)].second; 375 } 376 } 377