From 5b81f2a31d044da2cac5ca04a7d9c5dce186ad07 Mon Sep 17 00:00:00 2001 From: MerryMage Date: Mon, 10 Apr 2017 16:58:55 +0100 Subject: Mixer: Return actual number of samples mixed into buffer from MixerFifo::Mix No code in the codebase currently depends on the return value of this function. --- Source/Core/AudioCommon/Mixer.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index bdaaad812c..bdfbbfe5a0 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -88,6 +88,9 @@ unsigned int CMixer::MixerFifo::Mix(short* samples, unsigned int numSamples, m_frac &= 0xffff; } + // Actual number of samples written to the buffer without padding. + unsigned int actual_sample_count = currentSample / 2; + // Padding short s[2]; s[0] = Common::swap16(m_buffer[(indexR - 1) & INDEX_MASK]); @@ -106,7 +109,7 @@ unsigned int CMixer::MixerFifo::Mix(short* samples, unsigned int numSamples, // Flush cached variable m_indexR.store(indexR); - return numSamples; + return actual_sample_count; } unsigned int CMixer::Mix(short* samples, unsigned int num_samples, bool consider_framelimit) -- cgit v1.2.3 From b8c867dd7a14ed9e96dfd8f8bc01170dacea9018 Mon Sep 17 00:00:00 2001 From: MerryMage Date: Mon, 10 Apr 2017 17:44:17 +0100 Subject: Mixer: Implement audio stretching --- Source/Core/AudioCommon/Mixer.cpp | 77 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index bdfbbfe5a0..bebdd2abbe 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -4,6 +4,7 @@ #include "AudioCommon/Mixer.h" +#include #include #include "Common/CommonTypes.h" @@ -15,6 +16,15 @@ CMixer::CMixer(unsigned int BackendSampleRate) : m_sampleRate(BackendSampleRate) { INFO_LOG(AUDIO_INTERFACE, "Mixer is initialized"); + + m_sound_touch.setChannels(2); + m_sound_touch.setSampleRate(BackendSampleRate); + m_sound_touch.setPitch(1.0); + m_sound_touch.setTempo(1.0); + m_sound_touch.setSetting(SETTING_USE_QUICKSEEK, 0); + m_sound_touch.setSetting(SETTING_SEQUENCE_MS, 62); + m_sound_touch.setSetting(SETTING_SEEKWINDOW_MS, 28); + m_sound_touch.setSetting(SETTING_OVERLAP_MS, 8); } CMixer::~CMixer() @@ -119,12 +129,73 @@ unsigned int CMixer::Mix(short* samples, unsigned int num_samples, bool consider memset(samples, 0, num_samples * 2 * sizeof(short)); - m_dma_mixer.Mix(samples, num_samples, consider_framelimit); - m_streaming_mixer.Mix(samples, num_samples, consider_framelimit); - m_wiimote_speaker_mixer.Mix(samples, num_samples, consider_framelimit); + unsigned int actual_samples = m_dma_mixer.Mix(samples, num_samples, false); + m_streaming_mixer.Mix(samples, num_samples, false); + m_wiimote_speaker_mixer.Mix(samples, num_samples, false); + + StretchAudio(samples, actual_samples, num_samples); + return num_samples; } +void CMixer::StretchAudio(short* samples, unsigned int actual_samples, unsigned int num_samples) +{ + const double time_delta = static_cast(num_samples) / m_sampleRate; // seconds + + // We were given actual_samples number of samples, and num_samples were requested from us. + double current_ratio = static_cast(actual_samples) / static_cast(num_samples); + + constexpr double MAXIMUM_LATENCY = 0.080; // seconds + const double max_backlog = m_sampleRate * MAXIMUM_LATENCY; + double backlog_fullness = m_sound_touch.numSamples() / max_backlog; + if (backlog_fullness > 1.0) + { + // Exceeded latency budget: Do not add more samples into FIFO. + actual_samples = 0; + } + + // We ideally want the backlog to be about 50% full. + // This gives some headroom both ways to prevent underflow and overflow. + // We tweak current_ratio to encourage this. + constexpr double tweak_time_scale = 0.1; // seconds + current_ratio *= 1.0 + 2.0 * (backlog_fullness - 0.5) * (time_delta / tweak_time_scale); + + // This low-pass filter smoothes out variance in the calculated stretch ratio. + // The time-scale determines how responsive this filter is. + constexpr double lpf_time_scale = 0.3; // seconds + const double m_lpf_gain = 1.0 - std::exp(-time_delta / lpf_time_scale); + m_stretch_ratio += m_lpf_gain * (current_ratio - m_stretch_ratio); + + // Place a lower limit of 10% speed. When a game boots up, there will be + // many silence samples. These do not need to be timestretched. + m_sound_touch.setTempo(std::max(m_stretch_ratio, 0.1)); + + if (actual_samples != num_samples) + { + DEBUG_LOG(AUDIO, "Audio stretching: samples:%u/%u ratio:%f backlog:%f gain: %f", actual_samples, + num_samples, m_stretch_ratio, backlog_fullness, m_lpf_gain); + } + + m_sound_touch.putSamples(samples, actual_samples); + + memset(samples, 0, num_samples * 2 * sizeof(short)); + + const size_t samples_received = m_sound_touch.receiveSamples(samples, num_samples); + + if (samples_received != 0) + { + m_last_stretched_sample[0] = samples[samples_received * 2 - 2]; + m_last_stretched_sample[1] = samples[samples_received * 2 - 1]; + } + + // Preform padding if we've run out of samples. + for (size_t i = samples_received; i < num_samples; i++) + { + samples[i * 2 + 0] = m_last_stretched_sample[0]; + samples[i * 2 + 1] = m_last_stretched_sample[1]; + } +} + void CMixer::MixerFifo::PushSamples(const short* samples, unsigned int num_samples) { // Cache access in non-volatile variable -- cgit v1.2.3 From 26514358f40fd110d5d55d033fad16064a4db27b Mon Sep 17 00:00:00 2001 From: MerryMage Date: Mon, 10 Apr 2017 17:56:24 +0100 Subject: Add audio stretching as a configuration option --- Source/Core/AudioCommon/Mixer.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index bebdd2abbe..5030e5d492 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -122,18 +122,26 @@ unsigned int CMixer::MixerFifo::Mix(short* samples, unsigned int numSamples, return actual_sample_count; } -unsigned int CMixer::Mix(short* samples, unsigned int num_samples, bool consider_framelimit) +unsigned int CMixer::Mix(short* samples, unsigned int num_samples) { if (!samples) return 0; memset(samples, 0, num_samples * 2 * sizeof(short)); - unsigned int actual_samples = m_dma_mixer.Mix(samples, num_samples, false); - m_streaming_mixer.Mix(samples, num_samples, false); - m_wiimote_speaker_mixer.Mix(samples, num_samples, false); + const bool stretch = SConfig::GetInstance().m_audio_stretch; - StretchAudio(samples, actual_samples, num_samples); + unsigned int actual_samples = m_dma_mixer.Mix(samples, num_samples, !stretch); + m_streaming_mixer.Mix(samples, num_samples, !stretch); + m_wiimote_speaker_mixer.Mix(samples, num_samples, !stretch); + + if (stretch) + { + if (m_is_stretching != stretch) + m_sound_touch.clear(); + StretchAudio(samples, actual_samples, num_samples); + } + m_is_stretching = stretch; return num_samples; } @@ -144,10 +152,10 @@ void CMixer::StretchAudio(short* samples, unsigned int actual_samples, unsigned // We were given actual_samples number of samples, and num_samples were requested from us. double current_ratio = static_cast(actual_samples) / static_cast(num_samples); - - constexpr double MAXIMUM_LATENCY = 0.080; // seconds - const double max_backlog = m_sampleRate * MAXIMUM_LATENCY; - double backlog_fullness = m_sound_touch.numSamples() / max_backlog; + + const double max_latency = SConfig::GetInstance().m_audio_stretch_max_latency; + const double max_backlog = m_sampleRate * max_latency / 1000.0; + const double backlog_fullness = m_sound_touch.numSamples() / max_backlog; if (backlog_fullness > 1.0) { // Exceeded latency budget: Do not add more samples into FIFO. -- cgit v1.2.3 From ac0df5b2dbd40ac58137f13c09719abe24a95caf Mon Sep 17 00:00:00 2001 From: MerryMage Date: Tue, 11 Apr 2017 10:07:21 +0100 Subject: Mixer: Disable frequency shifting when stretching is enabled --- Source/Core/AudioCommon/Mixer.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index 5030e5d492..3e64d7f01f 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -47,26 +47,27 @@ unsigned int CMixer::MixerFifo::Mix(short* samples, unsigned int numSamples, u32 indexR = m_indexR.load(); u32 indexW = m_indexW.load(); - u32 low_waterwark = m_input_sample_rate * SConfig::GetInstance().iTimingVariance / 1000; - low_waterwark = std::min(low_waterwark, MAX_SAMPLES / 2); - - float numLeft = (float)(((indexW - indexR) & INDEX_MASK) / 2); - m_numLeftI = (numLeft + m_numLeftI * (CONTROL_AVG - 1)) / CONTROL_AVG; - float offset = (m_numLeftI - low_waterwark) * CONTROL_FACTOR; - if (offset > MAX_FREQ_SHIFT) - offset = MAX_FREQ_SHIFT; - if (offset < -MAX_FREQ_SHIFT) - offset = -MAX_FREQ_SHIFT; - // render numleft sample pairs to samples[] // advance indexR with sample position // remember fractional offset float emulationspeed = SConfig::GetInstance().m_EmulationSpeed; - float aid_sample_rate = m_input_sample_rate + offset; + float aid_sample_rate = static_cast(m_input_sample_rate); if (consider_framelimit && emulationspeed > 0.0f) { - aid_sample_rate = aid_sample_rate * emulationspeed; + float numLeft = static_cast(((indexW - indexR) & INDEX_MASK) / 2); + + u32 low_waterwark = m_input_sample_rate * SConfig::GetInstance().iTimingVariance / 1000; + low_waterwark = std::min(low_waterwark, MAX_SAMPLES / 2); + + m_numLeftI = (numLeft + m_numLeftI * (CONTROL_AVG - 1)) / CONTROL_AVG; + float offset = (m_numLeftI - low_waterwark) * CONTROL_FACTOR; + if (offset > MAX_FREQ_SHIFT) + offset = MAX_FREQ_SHIFT; + if (offset < -MAX_FREQ_SHIFT) + offset = -MAX_FREQ_SHIFT; + + aid_sample_rate = (aid_sample_rate + offset) * emulationspeed; } const u32 ratio = (u32)(65536.0f * aid_sample_rate / (float)m_mixer->m_sampleRate); -- cgit v1.2.3 From f5018010d39d4aff030c19b13f18f12c04c4a99b Mon Sep 17 00:00:00 2001 From: MerryMage Date: Tue, 11 Apr 2017 12:41:35 +0100 Subject: Mixer: Calculate actual_samples based on availability in all FIFOs --- Source/Core/AudioCommon/Mixer.cpp | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index 3e64d7f01f..0863dd1d7c 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -130,19 +130,30 @@ unsigned int CMixer::Mix(short* samples, unsigned int num_samples) memset(samples, 0, num_samples * 2 * sizeof(short)); - const bool stretch = SConfig::GetInstance().m_audio_stretch; + if (SConfig::GetInstance().m_audio_stretch) + { + unsigned int actual_samples = std::min({ + m_dma_mixer.AvailableSamples(), m_streaming_mixer.AvailableSamples(), num_samples, + }); - unsigned int actual_samples = m_dma_mixer.Mix(samples, num_samples, !stretch); - m_streaming_mixer.Mix(samples, num_samples, !stretch); - m_wiimote_speaker_mixer.Mix(samples, num_samples, !stretch); + m_dma_mixer.Mix(samples, actual_samples, false); + m_streaming_mixer.Mix(samples, actual_samples, false); + m_wiimote_speaker_mixer.Mix(samples, actual_samples, false); - if (stretch) - { - if (m_is_stretching != stretch) + if (!m_is_stretching) + { m_sound_touch.clear(); + m_is_stretching = true; + } StretchAudio(samples, actual_samples, num_samples); } - m_is_stretching = stretch; + else + { + m_dma_mixer.Mix(samples, num_samples, true); + m_streaming_mixer.Mix(samples, num_samples, true); + m_wiimote_speaker_mixer.Mix(samples, num_samples, true); + m_is_stretching = false; + } return num_samples; } @@ -378,3 +389,11 @@ void CMixer::MixerFifo::SetVolume(unsigned int lvolume, unsigned int rvolume) m_LVolume.store(lvolume + (lvolume >> 7)); m_RVolume.store(rvolume + (rvolume >> 7)); } + +unsigned int CMixer::MixerFifo::AvailableSamples() const +{ + unsigned int samples_in_fifo = ((m_indexW.load() - m_indexR.load()) & INDEX_MASK) / 2; + if (samples_in_fifo <= 1) + return 0; // CMixer::MixerFifo::Mix always keeps one sample in the buffer. + return (samples_in_fifo - 1) * m_mixer->m_sampleRate / m_input_sample_rate; +} -- cgit v1.2.3 From 9397fdfe95ad5f3e74a1fce2e049abddcbbbfba5 Mon Sep 17 00:00:00 2001 From: MerryMage Date: Wed, 12 Apr 2017 12:56:07 +0100 Subject: Mixer: Use a temporary buffer when stretching audio --- Source/Core/AudioCommon/Mixer.cpp | 48 ++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 26 deletions(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index 0863dd1d7c..ffb9489246 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -132,20 +132,21 @@ unsigned int CMixer::Mix(short* samples, unsigned int num_samples) if (SConfig::GetInstance().m_audio_stretch) { - unsigned int actual_samples = std::min({ - m_dma_mixer.AvailableSamples(), m_streaming_mixer.AvailableSamples(), num_samples, - }); + unsigned int available_samples = + std::min(m_dma_mixer.AvailableSamples(), m_streaming_mixer.AvailableSamples()); - m_dma_mixer.Mix(samples, actual_samples, false); - m_streaming_mixer.Mix(samples, actual_samples, false); - m_wiimote_speaker_mixer.Mix(samples, actual_samples, false); + m_stretch_buffer.fill(0); + + m_dma_mixer.Mix(m_stretch_buffer.data(), available_samples, false); + m_streaming_mixer.Mix(m_stretch_buffer.data(), available_samples, false); + m_wiimote_speaker_mixer.Mix(m_stretch_buffer.data(), available_samples, false); if (!m_is_stretching) { m_sound_touch.clear(); m_is_stretching = true; } - StretchAudio(samples, actual_samples, num_samples); + StretchAudio(m_stretch_buffer.data(), available_samples, samples, num_samples); } else { @@ -158,12 +159,12 @@ unsigned int CMixer::Mix(short* samples, unsigned int num_samples) return num_samples; } -void CMixer::StretchAudio(short* samples, unsigned int actual_samples, unsigned int num_samples) +void CMixer::StretchAudio(const short* in, unsigned int num_in, short* out, unsigned int num_out) { - const double time_delta = static_cast(num_samples) / m_sampleRate; // seconds + const double time_delta = static_cast(num_out) / m_sampleRate; // seconds // We were given actual_samples number of samples, and num_samples were requested from us. - double current_ratio = static_cast(actual_samples) / static_cast(num_samples); + double current_ratio = static_cast(num_in) / static_cast(num_out); const double max_latency = SConfig::GetInstance().m_audio_stretch_max_latency; const double max_backlog = m_sampleRate * max_latency / 1000.0; @@ -171,7 +172,7 @@ void CMixer::StretchAudio(short* samples, unsigned int actual_samples, unsigned if (backlog_fullness > 1.0) { // Exceeded latency budget: Do not add more samples into FIFO. - actual_samples = 0; + num_in = 0; } // We ideally want the backlog to be about 50% full. @@ -182,7 +183,7 @@ void CMixer::StretchAudio(short* samples, unsigned int actual_samples, unsigned // This low-pass filter smoothes out variance in the calculated stretch ratio. // The time-scale determines how responsive this filter is. - constexpr double lpf_time_scale = 0.3; // seconds + constexpr double lpf_time_scale = 1.0; // seconds const double m_lpf_gain = 1.0 - std::exp(-time_delta / lpf_time_scale); m_stretch_ratio += m_lpf_gain * (current_ratio - m_stretch_ratio); @@ -190,29 +191,24 @@ void CMixer::StretchAudio(short* samples, unsigned int actual_samples, unsigned // many silence samples. These do not need to be timestretched. m_sound_touch.setTempo(std::max(m_stretch_ratio, 0.1)); - if (actual_samples != num_samples) - { - DEBUG_LOG(AUDIO, "Audio stretching: samples:%u/%u ratio:%f backlog:%f gain: %f", actual_samples, - num_samples, m_stretch_ratio, backlog_fullness, m_lpf_gain); - } + DEBUG_LOG(AUDIO, "Audio stretching: samples:%u/%u ratio:%f backlog:%f gain: %f", num_in, num_out, + m_stretch_ratio, backlog_fullness, m_lpf_gain); - m_sound_touch.putSamples(samples, actual_samples); - - memset(samples, 0, num_samples * 2 * sizeof(short)); + m_sound_touch.putSamples(in, num_in); - const size_t samples_received = m_sound_touch.receiveSamples(samples, num_samples); + const size_t samples_received = m_sound_touch.receiveSamples(out, num_out); if (samples_received != 0) { - m_last_stretched_sample[0] = samples[samples_received * 2 - 2]; - m_last_stretched_sample[1] = samples[samples_received * 2 - 1]; + m_last_stretched_sample[0] = out[samples_received * 2 - 2]; + m_last_stretched_sample[1] = out[samples_received * 2 - 1]; } // Preform padding if we've run out of samples. - for (size_t i = samples_received; i < num_samples; i++) + for (size_t i = samples_received; i < num_out; i++) { - samples[i * 2 + 0] = m_last_stretched_sample[0]; - samples[i * 2 + 1] = m_last_stretched_sample[1]; + out[i * 2 + 0] = m_last_stretched_sample[0]; + out[i * 2 + 1] = m_last_stretched_sample[1]; } } -- cgit v1.2.3 From cbaa00457a961b3aa0213da0f5282e13a69c610d Mon Sep 17 00:00:00 2001 From: MerryMage Date: Wed, 12 Apr 2017 16:30:10 +0100 Subject: Mixer: Tweak audio stretch parameters --- Source/Core/AudioCommon/Mixer.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'Source/Core/AudioCommon/Mixer.cpp') diff --git a/Source/Core/AudioCommon/Mixer.cpp b/Source/Core/AudioCommon/Mixer.cpp index ffb9489246..8149da088c 100644 --- a/Source/Core/AudioCommon/Mixer.cpp +++ b/Source/Core/AudioCommon/Mixer.cpp @@ -167,18 +167,18 @@ void CMixer::StretchAudio(const short* in, unsigned int num_in, short* out, unsi double current_ratio = static_cast(num_in) / static_cast(num_out); const double max_latency = SConfig::GetInstance().m_audio_stretch_max_latency; - const double max_backlog = m_sampleRate * max_latency / 1000.0; + const double max_backlog = m_sampleRate * max_latency / 1000.0 / m_stretch_ratio; const double backlog_fullness = m_sound_touch.numSamples() / max_backlog; - if (backlog_fullness > 1.0) + if (backlog_fullness > 5.0) { - // Exceeded latency budget: Do not add more samples into FIFO. + // Too many samples in backlog: Don't push anymore on num_in = 0; } // We ideally want the backlog to be about 50% full. // This gives some headroom both ways to prevent underflow and overflow. // We tweak current_ratio to encourage this. - constexpr double tweak_time_scale = 0.1; // seconds + constexpr double tweak_time_scale = 0.5; // seconds current_ratio *= 1.0 + 2.0 * (backlog_fullness - 0.5) * (time_delta / tweak_time_scale); // This low-pass filter smoothes out variance in the calculated stretch ratio. @@ -189,7 +189,8 @@ void CMixer::StretchAudio(const short* in, unsigned int num_in, short* out, unsi // Place a lower limit of 10% speed. When a game boots up, there will be // many silence samples. These do not need to be timestretched. - m_sound_touch.setTempo(std::max(m_stretch_ratio, 0.1)); + m_stretch_ratio = std::max(m_stretch_ratio, 0.1); + m_sound_touch.setTempo(m_stretch_ratio); DEBUG_LOG(AUDIO, "Audio stretching: samples:%u/%u ratio:%f backlog:%f gain: %f", num_in, num_out, m_stretch_ratio, backlog_fullness, m_lpf_gain); -- cgit v1.2.3