audren: Make use of nodiscard, rework downmixing, release all buffers

Preliminary work for upmixing & general cleanup. Fixes basic issues in games such as Shovel Knight and slightly improves the LEGO games. Upmixing stitll needs to be implemented.

Audio levels in a few games will be fixed as we now use the downmix coefficients when possible instead of supplying our own
master
Chloe Marcec 2020-11-17 14:14:29 +07:00
parent 87f220efff
commit 9a4beac95a
13 changed files with 194 additions and 102 deletions

@ -43,6 +43,10 @@ std::vector<Buffer::Tag> AudioOut::GetTagsAndReleaseBuffers(StreamPtr stream,
return stream->GetTagsAndReleaseBuffers(max_count);
}
std::vector<Buffer::Tag> AudioOut::GetTagsAndReleaseBuffers(StreamPtr stream) {
return stream->GetTagsAndReleaseBuffers();
}
void AudioOut::StartStream(StreamPtr stream) {
stream->Play();
}

@ -31,6 +31,9 @@ public:
/// Returns a vector of recently released buffers specified by tag for the specified stream
std::vector<Buffer::Tag> GetTagsAndReleaseBuffers(StreamPtr stream, std::size_t max_count);
/// Returns a vector of all recently released buffers specified by tag for the specified stream
std::vector<Buffer::Tag> GetTagsAndReleaseBuffers(StreamPtr stream);
/// Starts an audio stream for playback
void StartStream(StreamPtr stream);

@ -2,6 +2,7 @@
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <limits>
#include <vector>
#include "audio_core/audio_out.h"
@ -14,6 +15,59 @@
#include "core/memory.h"
#include "core/settings.h"
namespace {
[[nodiscard]] static constexpr s16 ClampToS16(s32 value) {
return static_cast<s16>(std::clamp(value, static_cast<s32>(std::numeric_limits<s16>::min()),
static_cast<s32>(std::numeric_limits<s16>::max())));
}
[[nodiscard]] static constexpr s16 Mix2To1(s16 l_channel, s16 r_channel) {
// Mix 50% from left and 50% from right channel
constexpr float l_mix_amount = 50.0f / 100.0f;
constexpr float r_mix_amount = 50.0f / 100.0f;
return ClampToS16(static_cast<s32>((static_cast<float>(l_channel) * l_mix_amount) +
(static_cast<float>(r_channel) * r_mix_amount)));
}
[[nodiscard]] static constexpr std::tuple<s16, s16> Mix6To2(s16 fl_channel, s16 fr_channel,
s16 fc_channel,
[[maybe_unused]] s16 lf_channel,
s16 bl_channel, s16 br_channel) {
// Front channels are mixed 36.94%, Center channels are mixed to be 26.12% & the back channels
// are mixed to be 36.94%
constexpr float front_mix_amount = 36.94f / 100.0f;
constexpr float center_mix_amount = 26.12f / 100.0f;
constexpr float back_mix_amount = 36.94f / 100.0f;
// Mix 50% from left and 50% from right channel
const auto left = front_mix_amount * static_cast<float>(fl_channel) +
center_mix_amount * static_cast<float>(fc_channel) +
back_mix_amount * static_cast<float>(bl_channel);
const auto right = front_mix_amount * static_cast<float>(fr_channel) +
center_mix_amount * static_cast<float>(fc_channel) +
back_mix_amount * static_cast<float>(br_channel);
return {ClampToS16(static_cast<s32>(left)), ClampToS16(static_cast<s32>(right))};
}
[[nodiscard]] static constexpr std::tuple<s16, s16> Mix6To2WithCoefficients(
s16 fl_channel, s16 fr_channel, s16 fc_channel, s16 lf_channel, s16 bl_channel, s16 br_channel,
const std::array<float_le, 4>& coeff) {
const auto left =
static_cast<float>(fl_channel) * coeff[0] + static_cast<float>(fc_channel) * coeff[1] +
static_cast<float>(lf_channel) * coeff[2] + static_cast<float>(bl_channel) * coeff[0];
const auto right =
static_cast<float>(fr_channel) * coeff[0] + static_cast<float>(fc_channel) * coeff[1] +
static_cast<float>(lf_channel) * coeff[2] + static_cast<float>(br_channel) * coeff[0];
return {ClampToS16(static_cast<s32>(left)), ClampToS16(static_cast<s32>(right))};
}
} // namespace
namespace AudioCore {
AudioRenderer::AudioRenderer(Core::Timing::CoreTiming& core_timing, Core::Memory::Memory& memory_,
AudioCommon::AudioRendererParameter params,
@ -62,10 +116,6 @@ Stream::State AudioRenderer::GetStreamState() const {
return stream->GetState();
}
static constexpr s16 ClampToS16(s32 value) {
return static_cast<s16>(std::clamp(value, -32768, 32767));
}
ResultCode AudioRenderer::UpdateAudioRenderer(const std::vector<u8>& input_params,
std::vector<u8>& output_params) {
@ -104,8 +154,8 @@ ResultCode AudioRenderer::UpdateAudioRenderer(const std::vector<u8>& input_param
}
}
auto mix_result = info_updater.UpdateMixes(mix_context, worker_params.mix_buffer_count,
splitter_context, effect_context);
const auto mix_result = info_updater.UpdateMixes(mix_context, worker_params.mix_buffer_count,
splitter_context, effect_context);
if (mix_result.IsError()) {
LOG_ERROR(Audio, "Failed to update mix parameters");
@ -194,20 +244,22 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) {
for (std::size_t i = 0; i < BUFFER_SIZE; i++) {
if (channel_count == 1) {
const auto sample = ClampToS16(mix_buffers[0][i]);
buffer[i * stream_channel_count + 0] = sample;
if (stream_channel_count > 1) {
buffer[i * stream_channel_count + 1] = sample;
// Place sample in all channels
for (u32 channel = 0; channel < stream_channel_count; channel++) {
buffer[i * stream_channel_count + channel] = sample;
}
if (stream_channel_count == 6) {
buffer[i * stream_channel_count + 2] = sample;
buffer[i * stream_channel_count + 4] = sample;
buffer[i * stream_channel_count + 5] = sample;
// Output stream has a LF channel, mute it!
buffer[i * stream_channel_count + 3] = 0;
}
} else if (channel_count == 2) {
const auto l_sample = ClampToS16(mix_buffers[0][i]);
const auto r_sample = ClampToS16(mix_buffers[1][i]);
if (stream_channel_count == 1) {
buffer[i * stream_channel_count + 0] = l_sample;
buffer[i * stream_channel_count + 0] = Mix2To1(l_sample, r_sample);
} else if (stream_channel_count == 2) {
buffer[i * stream_channel_count + 0] = l_sample;
buffer[i * stream_channel_count + 1] = r_sample;
@ -215,8 +267,8 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) {
buffer[i * stream_channel_count + 0] = l_sample;
buffer[i * stream_channel_count + 1] = r_sample;
buffer[i * stream_channel_count + 2] =
ClampToS16((static_cast<s32>(l_sample) + static_cast<s32>(r_sample)) / 2);
// Combine both left and right channels to the center channel
buffer[i * stream_channel_count + 2] = Mix2To1(l_sample, r_sample);
buffer[i * stream_channel_count + 4] = l_sample;
buffer[i * stream_channel_count + 5] = r_sample;
@ -231,17 +283,25 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) {
const auto br_sample = ClampToS16(mix_buffers[5][i]);
if (stream_channel_count == 1) {
buffer[i * stream_channel_count + 0] = fc_sample;
// Games seem to ignore the center channel half the time, we use the front left
// and right channel for mixing as that's where majority of the audio goes
buffer[i * stream_channel_count + 0] = Mix2To1(fl_sample, fr_sample);
} else if (stream_channel_count == 2) {
buffer[i * stream_channel_count + 0] =
static_cast<s16>(0.3694f * static_cast<float>(fl_sample) +
0.2612f * static_cast<float>(fc_sample) +
0.3694f * static_cast<float>(bl_sample));
buffer[i * stream_channel_count + 1] =
static_cast<s16>(0.3694f * static_cast<float>(fr_sample) +
0.2612f * static_cast<float>(fc_sample) +
0.3694f * static_cast<float>(br_sample));
// Mix all channels into 2 channels
if (sink_context.HasDownMixingCoefficients()) {
const auto [left, right] = Mix6To2WithCoefficients(
fl_sample, fr_sample, fc_sample, lf_sample, bl_sample, br_sample,
sink_context.GetDownmixCoefficients());
buffer[i * stream_channel_count + 0] = left;
buffer[i * stream_channel_count + 1] = right;
} else {
const auto [left, right] = Mix6To2(fl_sample, fr_sample, fc_sample,
lf_sample, bl_sample, br_sample);
buffer[i * stream_channel_count + 0] = left;
buffer[i * stream_channel_count + 1] = right;
}
} else if (stream_channel_count == 6) {
// Pass through
buffer[i * stream_channel_count + 0] = fl_sample;
buffer[i * stream_channel_count + 1] = fr_sample;
buffer[i * stream_channel_count + 2] = fc_sample;
@ -259,7 +319,7 @@ void AudioRenderer::QueueMixedBuffer(Buffer::Tag tag) {
}
void AudioRenderer::ReleaseAndQueueBuffers() {
const auto released_buffers{audio_out->GetTagsAndReleaseBuffers(stream, 2)};
const auto released_buffers{audio_out->GetTagsAndReleaseBuffers(stream)};
for (const auto& tag : released_buffers) {
QueueMixedBuffer(tag);
}

@ -36,16 +36,10 @@ class Memory;
}
namespace AudioCore {
using DSPStateHolder = std::array<VoiceState*, 6>;
using DSPStateHolder = std::array<VoiceState*, AudioCommon::MAX_CHANNEL_COUNT>;
class AudioOut;
struct RendererInfo {
u64_le elasped_frame_count{};
INSERT_PADDING_WORDS(2);
};
static_assert(sizeof(RendererInfo) == 0x10, "RendererInfo is an invalid size");
class AudioRenderer {
public:
AudioRenderer(Core::Timing::CoreTiming& core_timing, Core::Memory::Memory& memory_,
@ -53,14 +47,14 @@ public:
std::shared_ptr<Kernel::WritableEvent> buffer_event, std::size_t instance_number);
~AudioRenderer();
ResultCode UpdateAudioRenderer(const std::vector<u8>& input_params,
std::vector<u8>& output_params);
[[nodiscard]] ResultCode UpdateAudioRenderer(const std::vector<u8>& input_params,
std::vector<u8>& output_params);
void QueueMixedBuffer(Buffer::Tag tag);
void ReleaseAndQueueBuffers();
u32 GetSampleRate() const;
u32 GetSampleCount() const;
u32 GetMixBufferCount() const;
Stream::State GetStreamState() const;
[[nodiscard]] u32 GetSampleRate() const;
[[nodiscard]] u32 GetSampleCount() const;
[[nodiscard]] u32 GetMixBufferCount() const;
[[nodiscard]] Stream::State GetStreamState() const;
private:
BehaviorInfo behavior_info{};

@ -43,22 +43,22 @@ public:
void ClearError();
void UpdateFlags(u64_le dest_flags);
void SetUserRevision(u32_le revision);
u32_le GetUserRevision() const;
u32_le GetProcessRevision() const;
[[nodiscard]] u32_le GetUserRevision() const;
[[nodiscard]] u32_le GetProcessRevision() const;
bool IsAdpcmLoopContextBugFixed() const;
bool IsSplitterSupported() const;
bool IsLongSizePreDelaySupported() const;
bool IsAudioRendererProcessingTimeLimit80PercentSupported() const;
bool IsAudioRendererProcessingTimeLimit75PercentSupported() const;
bool IsAudioRendererProcessingTimeLimit70PercentSupported() const;
bool IsElapsedFrameCountSupported() const;
bool IsMemoryPoolForceMappingEnabled() const;
bool IsFlushVoiceWaveBuffersSupported() const;
bool IsVoicePlayedSampleCountResetAtLoopPointSupported() const;
bool IsVoicePitchAndSrcSkippedSupported() const;
bool IsMixInParameterDirtyOnlyUpdateSupported() const;
bool IsSplitterBugFixed() const;
[[nodiscard]] bool IsAdpcmLoopContextBugFixed() const;
[[nodiscard]] bool IsSplitterSupported() const;
[[nodiscard]] bool IsLongSizePreDelaySupported() const;
[[nodiscard]] bool IsAudioRendererProcessingTimeLimit80PercentSupported() const;
[[nodiscard]] bool IsAudioRendererProcessingTimeLimit75PercentSupported() const;
[[nodiscard]] bool IsAudioRendererProcessingTimeLimit70PercentSupported() const;
[[nodiscard]] bool IsElapsedFrameCountSupported() const;
[[nodiscard]] bool IsMemoryPoolForceMappingEnabled() const;
[[nodiscard]] bool IsFlushVoiceWaveBuffersSupported() const;
[[nodiscard]] bool IsVoicePlayedSampleCountResetAtLoopPointSupported() const;
[[nodiscard]] bool IsVoicePitchAndSrcSkippedSupported() const;
[[nodiscard]] bool IsMixInParameterDirtyOnlyUpdateSupported() const;
[[nodiscard]] bool IsSplitterBugFixed() const;
void CopyErrorInfo(OutParams& dst);
private:

@ -39,13 +39,13 @@ public:
void PreCommand();
void PostCommand();
s32* GetChannelMixBuffer(s32 channel);
const s32* GetChannelMixBuffer(s32 channel) const;
s32* GetMixBuffer(std::size_t index);
const s32* GetMixBuffer(std::size_t index) const;
std::size_t GetMixChannelBufferOffset(s32 channel) const;
[[nodiscard]] s32* GetChannelMixBuffer(s32 channel);
[[nodiscard]] const s32* GetChannelMixBuffer(s32 channel) const;
[[nodiscard]] s32* GetMixBuffer(std::size_t index);
[[nodiscard]] const s32* GetMixBuffer(std::size_t index) const;
[[nodiscard]] std::size_t GetMixChannelBufferOffset(s32 channel) const;
std::size_t GetTotalMixBufferCount() const;
[[nodiscard]] std::size_t GetTotalMixBufferCount() const;
private:
void GenerateDataSourceCommand(ServerVoiceInfo& voice_info, VoiceState& dsp_state, s32 channel);
@ -73,7 +73,7 @@ private:
void GenerateI3dl2ReverbEffectCommand(s32 mix_buffer_offset, EffectBase* info, bool enabled);
void GenerateBiquadFilterEffectCommand(s32 mix_buffer_offset, EffectBase* info, bool enabled);
void GenerateAuxCommand(s32 mix_buffer_offset, EffectBase* info, bool enabled);
ServerSplitterDestinationData* GetDestinationData(s32 splitter_id, s32 index);
[[nodiscard]] ServerSplitterDestinationData* GetDestinationData(s32 splitter_id, s32 index);
s32 WriteAuxBuffer(AuxInfoDSP& dsp_info, VAddr send_buffer, u32 max_samples, const s32* data,
u32 sample_count, u32 write_offset, u32 write_count);

@ -22,7 +22,7 @@ constexpr std::size_t MAX_CHANNEL_COUNT = 6;
constexpr std::size_t MAX_WAVE_BUFFERS = 4;
constexpr std::size_t MAX_SAMPLE_HISTORY = 4;
constexpr u32 STREAM_SAMPLE_RATE = 48000;
constexpr u32 STREAM_NUM_CHANNELS = 6;
constexpr u32 STREAM_NUM_CHANNELS = 2;
constexpr s32 NO_SPLITTER = -1;
constexpr s32 NO_MIX = 0x7fffffff;
constexpr s32 NO_FINAL_MIX = std::numeric_limits<s32>::min();

@ -189,11 +189,11 @@ public:
virtual void Update(EffectInfo::InParams& in_params) = 0;
virtual void UpdateForCommandGeneration() = 0;
UsageState GetUsage() const;
EffectType GetType() const;
bool IsEnabled() const;
s32 GetMixID() const;
s32 GetProcessingOrder() const;
[[nodiscard]] UsageState GetUsage() const;
[[nodiscard]] EffectType GetType() const;
[[nodiscard]] bool IsEnabled() const;
[[nodiscard]] s32 GetMixID() const;
[[nodiscard]] s32 GetProcessingOrder() const;
protected:
UsageState usage{UsageState::Invalid};
@ -257,10 +257,10 @@ public:
void Update(EffectInfo::InParams& in_params) override;
void UpdateForCommandGeneration() override;
VAddr GetSendInfo() const;
VAddr GetSendBuffer() const;
VAddr GetRecvInfo() const;
VAddr GetRecvBuffer() const;
[[nodiscard]] VAddr GetSendInfo() const;
[[nodiscard]] VAddr GetSendBuffer() const;
[[nodiscard]] VAddr GetRecvInfo() const;
[[nodiscard]] VAddr GetRecvBuffer() const;
private:
VAddr send_info{};
@ -309,10 +309,10 @@ public:
explicit EffectContext(std::size_t effect_count);
~EffectContext();
std::size_t GetCount() const;
EffectBase* GetInfo(std::size_t i);
EffectBase* RetargetEffect(std::size_t i, EffectType effect);
const EffectBase* GetInfo(std::size_t i) const;
[[nodiscard]] std::size_t GetCount() const;
[[nodiscard]] EffectBase* GetInfo(std::size_t i);
[[nodiscard]] EffectBase* RetargetEffect(std::size_t i, EffectType effect);
[[nodiscard]] const EffectBase* GetInfo(std::size_t i) const;
private:
std::size_t effect_count{};

@ -62,17 +62,17 @@ public:
ServerMixInfo();
~ServerMixInfo();
const ServerMixInfo::InParams& GetInParams() const;
ServerMixInfo::InParams& GetInParams();
[[nodiscard]] const ServerMixInfo::InParams& GetInParams() const;
[[nodiscard]] ServerMixInfo::InParams& GetInParams();
bool Update(EdgeMatrix& edge_matrix, const MixInfo::InParams& mix_in,
BehaviorInfo& behavior_info, SplitterContext& splitter_context,
EffectContext& effect_context);
bool HasAnyConnection() const;
[[nodiscard]] bool HasAnyConnection() const;
void Cleanup();
void SetEffectCount(std::size_t count);
void ResetEffectProcessingOrder();
s32 GetEffectOrder(std::size_t i) const;
[[nodiscard]] s32 GetEffectOrder(std::size_t i) const;
private:
std::vector<s32> effect_processing_order;
@ -91,15 +91,15 @@ public:
void SortInfo();
bool TsortInfo(SplitterContext& splitter_context);
std::size_t GetCount() const;
ServerMixInfo& GetInfo(std::size_t i);
const ServerMixInfo& GetInfo(std::size_t i) const;
ServerMixInfo& GetSortedInfo(std::size_t i);
const ServerMixInfo& GetSortedInfo(std::size_t i) const;
ServerMixInfo& GetFinalMixInfo();
const ServerMixInfo& GetFinalMixInfo() const;
EdgeMatrix& GetEdgeMatrix();
const EdgeMatrix& GetEdgeMatrix() const;
[[nodiscard]] std::size_t GetCount() const;
[[nodiscard]] ServerMixInfo& GetInfo(std::size_t i);
[[nodiscard]] const ServerMixInfo& GetInfo(std::size_t i) const;
[[nodiscard]] ServerMixInfo& GetSortedInfo(std::size_t i);
[[nodiscard]] const ServerMixInfo& GetSortedInfo(std::size_t i) const;
[[nodiscard]] ServerMixInfo& GetFinalMixInfo();
[[nodiscard]] const ServerMixInfo& GetFinalMixInfo() const;
[[nodiscard]] EdgeMatrix& GetEdgeMatrix();
[[nodiscard]] const EdgeMatrix& GetEdgeMatrix() const;
private:
void CalcMixBufferOffset();

@ -12,10 +12,16 @@ std::size_t SinkContext::GetCount() const {
return sink_count;
}
void SinkContext::UpdateMainSink(SinkInfo::InParams& in) {
void SinkContext::UpdateMainSink(const SinkInfo::InParams& in) {
ASSERT(in.type == SinkTypes::Device);
downmix = in.device.down_matrix_enabled;
if (downmix) {
downmix_coefficients = in.device.down_matrix_coef;
}
in_use = in.in_use;
use_count = in.device.input_count;
std::memcpy(buffers.data(), in.device.input.data(), AudioCommon::MAX_CHANNEL_COUNT);
buffers = in.device.input;
}
bool SinkContext::InUse() const {
@ -28,4 +34,12 @@ std::vector<u8> SinkContext::OutputBuffers() const {
return buffer_ret;
}
bool SinkContext::HasDownMixingCoefficients() const {
return downmix;
}
const std::array<float_le, 4>& SinkContext::GetDownmixCoefficients() const {
return downmix_coefficients;
}
} // namespace AudioCore

@ -74,16 +74,21 @@ public:
explicit SinkContext(std::size_t sink_count);
~SinkContext();
std::size_t GetCount() const;
[[nodiscard]] std::size_t GetCount() const;
void UpdateMainSink(SinkInfo::InParams& in);
bool InUse() const;
std::vector<u8> OutputBuffers() const;
void UpdateMainSink(const SinkInfo::InParams& in);
[[nodiscard]] bool InUse() const;
[[nodiscard]] std::vector<u8> OutputBuffers() const;
[[nodiscard]] bool HasDownMixingCoefficients() const;
[[nodiscard]] const std::array<float_le, 4>& GetDownmixCoefficients() const;
private:
bool in_use{false};
s32 use_count{};
std::array<u8, AudioCommon::MAX_CHANNEL_COUNT> buffers{};
std::size_t sink_count{};
bool downmix{false};
std::array<float_le, 4> downmix_coefficients{};
};
} // namespace AudioCore

@ -136,4 +136,13 @@ std::vector<Buffer::Tag> Stream::GetTagsAndReleaseBuffers(std::size_t max_count)
return tags;
}
std::vector<Buffer::Tag> Stream::GetTagsAndReleaseBuffers() {
std::vector<Buffer::Tag> tags;
while (!released_buffers.empty()) {
tags.push_back(released_buffers.front()->GetTag());
released_buffers.pop();
}
return tags;
}
} // namespace AudioCore

@ -57,37 +57,40 @@ public:
bool QueueBuffer(BufferPtr&& buffer);
/// Returns true if the audio stream contains a buffer with the specified tag
bool ContainsBuffer(Buffer::Tag tag) const;
[[nodiscard]] bool ContainsBuffer(Buffer::Tag tag) const;
/// Returns a vector of recently released buffers specified by tag
std::vector<Buffer::Tag> GetTagsAndReleaseBuffers(std::size_t max_count);
[[nodiscard]] std::vector<Buffer::Tag> GetTagsAndReleaseBuffers(std::size_t max_count);
/// Returns a vector of all recently released buffers specified by tag
[[nodiscard]] std::vector<Buffer::Tag> GetTagsAndReleaseBuffers();
void SetVolume(float volume);
float GetVolume() const {
[[nodiscard]] float GetVolume() const {
return game_volume;
}
/// Returns true if the stream is currently playing
bool IsPlaying() const {
[[nodiscard]] bool IsPlaying() const {
return state == State::Playing;
}
/// Returns the number of queued buffers
std::size_t GetQueueSize() const {
[[nodiscard]] std::size_t GetQueueSize() const {
return queued_buffers.size();
}
/// Gets the sample rate
u32 GetSampleRate() const {
[[nodiscard]] u32 GetSampleRate() const {
return sample_rate;
}
/// Gets the number of channels
u32 GetNumChannels() const;
[[nodiscard]] u32 GetNumChannels() const;
/// Get the state
State GetState() const;
[[nodiscard]] State GetState() const;
private:
/// Plays the next queued buffer in the audio stream, starting playback if necessary
@ -97,7 +100,7 @@ private:
void ReleaseActiveBuffer(std::chrono::nanoseconds ns_late = {});
/// Gets the number of core cycles when the specified buffer will be released
std::chrono::nanoseconds GetBufferReleaseNS(const Buffer& buffer) const;
[[nodiscard]] std::chrono::nanoseconds GetBufferReleaseNS(const Buffer& buffer) const;
u32 sample_rate; ///< Sample rate of the stream
Format format; ///< Format of the stream