BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture every audio stream into its own ring and mix them on the host

Browse Source 
Games with several concurrent WASAPI render streams (e.g. Spider-Man: Miles
Morales) only had their first ("primary") stream mirrored; the rest kept playing
locally and never reached the guest. Now the render-hook captures + silences EVERY
tracked stream into its own ring (coop_audio_<pid>[_<index>]), each published with
that stream's own detected format (Initialize when caught, else GetMixFormat -- the
per-stream format detection, now actually used per ring rather than only for the
primary). The host creates a ring per stream and mixes the same-format streams with
a soft clip (host/src/audio/audio_mix.hpp); streams whose format differs from the
primary are still silenced (no echo) but skipped from the mix (would need
resampling).

The single-stream case is byte-for-byte unchanged: when only one stream is active
the host passes it through without the mixer, so the common path has no overhead or
fidelity change.

Verified: new audio_mix_test covers the decode/sum/soft-clip/encode math (float32 +
int16); audio_hook_test (x64 + x86) still passes, guarding the primary
capture+silence path against regression; full build x64 + x86 clean; ctest x64
11/11, x86 3/3. Multi-stream mixing against a real multi-stream game needs a live
session to fully confirm.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-21 05:45:47 +02:00
parent dfd2be8c17
commit 784c31a9b5
10 changed files with 412 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
host/src/audio/audio_loopback.cpp
View File

@@ -9,6 +9,7 @@
#include <mmdeviceapi.h>
#include <mmreg.h>
#include "audio/audio_mix.hpp"
#include "audio/process_loopback_capture.hpp"
namespace coop
@@ -151,7 +152,10 @@ void AudioMirror::stop()
CloseHandle(stop_event_);
stop_event_ = nullptr;
}
audio_ring_shm_.reset();
for (auto& shm : audio_ring_shm_)
{
shm.reset();
}
running_.store(false, std::memory_order_release);
source_.store(Source::None, std::memory_order_relaxed);
buffered_ms_.store(0, std::memory_order_relaxed);
@@ -191,15 +195,24 @@ void AudioMirror::thread_main(DWORD pid)
// capture. If the hook is present it publishes a format within ~1 s and we
// consume the ring (no echo); otherwise we fall back to process loopback.
bool handled = false;
if (audio_ring_shm_.create(audio_ring_name(pid), audio_ring_total_size(kAudioRingCapacity)))
AudioRingHeader* rings[kMaxAudioStreams] = {};
bool created_primary = false;
for (unsigned i = 0; i < kMaxAudioStreams; ++i)
{
auto* ring = audio_ring_shm_.as<AudioRingHeader>();
audio_ring_init(*ring, kAudioRingCapacity);
ring->capture_enabled.store(1, std::memory_order_release);
set_status("Waiting for render-hook…");
if (wait_for_format(ring, 1000))
if (audio_ring_shm_[i].create(audio_ring_name(pid, i), audio_ring_total_size(kAudioRingCapacity)))
{
handled = run_hooked(ring);
rings[i] = audio_ring_shm_[i].as<AudioRingHeader>();
audio_ring_init(*rings[i], kAudioRingCapacity);
rings[i]->capture_enabled.store(1, std::memory_order_release);
created_primary = created_primary || (i == 0);
}
}
if (created_primary)
{
set_status("Waiting for render-hook…");
if (wait_for_format(rings[0], 1000))
{
handled = run_hooked(rings);
}
}
@@ -208,7 +221,10 @@ void AudioMirror::thread_main(DWORD pid)
run_loopback(pid);
}
audio_ring_shm_.reset();
for (auto& shm : audio_ring_shm_)
{
shm.reset();
}
if (running_.load(std::memory_order_acquire))
{
@@ -226,17 +242,28 @@ void AudioMirror::thread_main(DWORD pid)
// Consume the render-hook's shared ring and re-render the game's frames. The
// game is silenced locally by the hook, so the operator hears no echo. Returns
// true if it ran to a clean stop; false on setup failure (caller falls back).
bool AudioMirror::run_hooked(AudioRingHeader* ring)
bool AudioMirror::run_hooked(AudioRingHeader* const* rings)
{
AudioRingHeader* primary = rings[0];
auto disable_all = [&] {
for (unsigned i = 0; i < kMaxAudioStreams; ++i)
{
if (rings[i] != nullptr)
{
rings[i]->capture_enabled.store(0, std::memory_order_release);
}
}
};
auto fail_to_loopback = [&] {
ring->capture_enabled.store(0, std::memory_order_release); // let the game play locally again
disable_all(); // let the game play locally again
return false;
};
const unsigned rate = ring->sample_rate;
const unsigned channels = ring->channels;
const unsigned bits = ring->bits;
const unsigned block_align = ring->block_align ? ring->block_align : channels * (bits / 8);
const unsigned rate = primary->sample_rate;
const unsigned channels = primary->channels;
const unsigned bits = primary->bits;
const unsigned tag = primary->format_tag;
const unsigned block_align = primary->block_align ? primary->block_align : channels * (bits / 8);
if (rate == 0 || channels == 0 || block_align == 0)
{
return fail_to_loopback();
@@ -267,12 +294,12 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
wfx.dwChannelMask = (channels >= 32) ? 0xFFFFFFFFu : ((1u << channels) - 1u);
break;
}
wfx.SubFormat = (ring->format_tag == WAVE_FORMAT_IEEE_FLOAT) ? KSDATAFORMAT_SUBTYPE_IEEE_FLOAT
: KSDATAFORMAT_SUBTYPE_PCM;
wfx.SubFormat =
(tag == WAVE_FORMAT_IEEE_FLOAT) ? KSDATAFORMAT_SUBTYPE_IEEE_FLOAT : KSDATAFORMAT_SUBTYPE_PCM;
}
else
{
wfx.Format.wFormatTag = static_cast<WORD>(ring->format_tag ? ring->format_tag : WAVE_FORMAT_PCM);
wfx.Format.wFormatTag = static_cast<WORD>(tag ? tag : WAVE_FORMAT_PCM);
wfx.Format.cbSize = 0;
}
auto* fmt = reinterpret_cast<WAVEFORMATEX*>(&wfx);
@@ -355,6 +382,12 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
const size_t prime_bytes = frame_bytes * (rate * 30 / 1000); // ~30 ms before feeding
bool primed = false;
// Mixing scratch (only used when >1 same-format stream is active): a per-stream
// temp buffer and a float accumulator sized to the render buffer.
const bool mixer_ok = mix_format_supported(tag, bits);
std::vector<BYTE> temp(static_cast<size_t>(render_frames) * frame_bytes);
std::vector<float> acc(static_cast<size_t>(render_frames) * channels);
if (FAILED(hr = render_client->Start()))
{
fail("Render Start", hr);
@@ -380,7 +413,8 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
continue;
}
const UINT32 avail = render_frames - padding;
const std::uint32_t ring_bytes = audio_ring_available(*ring);
// The primary ring is the master clock for priming + how much to write.
const std::uint32_t ring_bytes = audio_ring_available(*primary);
buffered_ms_.store(static_cast<unsigned>(ring_bytes / frame_bytes * 1000 / rate),
std::memory_order_relaxed);
if (!primed && ring_bytes >= prime_bytes)
@@ -393,10 +427,47 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
const UINT32 to_write = std::min(avail, have);
if (to_write > 0)
{
// Active streams = same format as primary (so they can be summed).
// Streams with a different format are still silenced by the hook (no
// echo) but can't be mixed here without resampling -> skipped.
unsigned active[kMaxAudioStreams];
unsigned n_active = 0;
for (unsigned i = 0; i < kMaxAudioStreams; ++i)
{
AudioRingHeader* r = rings[i];
if (r == nullptr)
{
continue;
}
if (i == 0 || (audio_ring_format_ready(*r) && r->sample_rate == rate &&
r->channels == channels && r->bits == bits && r->format_tag == tag))
{
active[n_active++] = i;
}
}
BYTE* dst = nullptr;
if (SUCCEEDED(render->GetBuffer(to_write, &dst)))
{
audio_ring_pop(*ring, dst, to_write * static_cast<std::uint32_t>(frame_bytes));
const std::uint32_t want_bytes = to_write * static_cast<std::uint32_t>(frame_bytes);
if (n_active <= 1 || !mixer_ok)
{
// Single stream (the common case) or an unmixable format:
// passthrough the primary, byte-for-byte (no mixer overhead).
audio_ring_pop(*primary, dst, want_bytes);
}
else
{
const std::uint32_t samples = to_write * channels;
std::fill(acc.begin(), acc.begin() + samples, 0.0f);
for (unsigned k = 0; k < n_active; ++k)
{
std::memset(temp.data(), 0, want_bytes); // zero-fill short reads
audio_ring_pop(*rings[active[k]], temp.data(), want_bytes);
mix_add(acc.data(), temp.data(), samples, tag, bits);
}
mix_store(dst, acc.data(), samples, tag, bits);
}
render->ReleaseBuffer(to_write, 0);
}
}
@@ -410,7 +481,7 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
render_client->Stop();
} while (false);
ring->capture_enabled.store(0, std::memory_order_release); // game audible again on stop
disable_all(); // game audible again on stop
if (render)
{