BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture every audio stream into its own ring and mix them on the host

Browse Source 
Games with several concurrent WASAPI render streams (e.g. Spider-Man: Miles
Morales) only had their first ("primary") stream mirrored; the rest kept playing
locally and never reached the guest. Now the render-hook captures + silences EVERY
tracked stream into its own ring (coop_audio_<pid>[_<index>]), each published with
that stream's own detected format (Initialize when caught, else GetMixFormat -- the
per-stream format detection, now actually used per ring rather than only for the
primary). The host creates a ring per stream and mixes the same-format streams with
a soft clip (host/src/audio/audio_mix.hpp); streams whose format differs from the
primary are still silenced (no echo) but skipped from the mix (would need
resampling).

The single-stream case is byte-for-byte unchanged: when only one stream is active
the host passes it through without the mixer, so the common path has no overhead or
fidelity change.

Verified: new audio_mix_test covers the decode/sum/soft-clip/encode math (float32 +
int16); audio_hook_test (x64 + x86) still passes, guarding the primary
capture+silence path against regression; full build x64 + x86 clean; ctest x64
11/11, x86 3/3. Multi-stream mixing against a real multi-stream game needs a live
session to fully confirm.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-21 05:45:47 +02:00
parent dfd2be8c17
commit 784c31a9b5
10 changed files with 412 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
host/src/audio/audio_loopback.cpp
View File

@@ -9,6 +9,7 @@
#include <mmdeviceapi.h>
#include <mmreg.h>
#include "audio/audio_mix.hpp"
#include "audio/process_loopback_capture.hpp"
namespace coop
@@ -151,7 +152,10 @@ void AudioMirror::stop()
CloseHandle(stop_event_);
stop_event_ = nullptr;
}
audio_ring_shm_.reset();
for (auto& shm : audio_ring_shm_)
{
shm.reset();
}
running_.store(false, std::memory_order_release);
source_.store(Source::None, std::memory_order_relaxed);
buffered_ms_.store(0, std::memory_order_relaxed);
@@ -191,15 +195,24 @@ void AudioMirror::thread_main(DWORD pid)
// capture. If the hook is present it publishes a format within ~1 s and we
// consume the ring (no echo); otherwise we fall back to process loopback.
bool handled = false;
if (audio_ring_shm_.create(audio_ring_name(pid), audio_ring_total_size(kAudioRingCapacity)))
AudioRingHeader* rings[kMaxAudioStreams] = {};
bool created_primary = false;
for (unsigned i = 0; i < kMaxAudioStreams; ++i)
{
auto* ring = audio_ring_shm_.as<AudioRingHeader>();
audio_ring_init(*ring, kAudioRingCapacity);
ring->capture_enabled.store(1, std::memory_order_release);
set_status("Waiting for render-hook…");
if (wait_for_format(ring, 1000))
if (audio_ring_shm_[i].create(audio_ring_name(pid, i), audio_ring_total_size(kAudioRingCapacity)))
{
handled = run_hooked(ring);
rings[i] = audio_ring_shm_[i].as<AudioRingHeader>();
audio_ring_init(*rings[i], kAudioRingCapacity);
rings[i]->capture_enabled.store(1, std::memory_order_release);
created_primary = created_primary || (i == 0);
}
}
if (created_primary)
{
set_status("Waiting for render-hook…");
if (wait_for_format(rings[0], 1000))
{
handled = run_hooked(rings);
}
}
@@ -208,7 +221,10 @@ void AudioMirror::thread_main(DWORD pid)
run_loopback(pid);
}
audio_ring_shm_.reset();
for (auto& shm : audio_ring_shm_)
{
shm.reset();
}
if (running_.load(std::memory_order_acquire))
{
@@ -226,17 +242,28 @@ void AudioMirror::thread_main(DWORD pid)
// Consume the render-hook's shared ring and re-render the game's frames. The
// game is silenced locally by the hook, so the operator hears no echo. Returns
// true if it ran to a clean stop; false on setup failure (caller falls back).
bool AudioMirror::run_hooked(AudioRingHeader* ring)
bool AudioMirror::run_hooked(AudioRingHeader* const* rings)
{
AudioRingHeader* primary = rings[0];
auto disable_all = [&] {
for (unsigned i = 0; i < kMaxAudioStreams; ++i)
{
if (rings[i] != nullptr)
{
rings[i]->capture_enabled.store(0, std::memory_order_release);
}
}
};
auto fail_to_loopback = [&] {
ring->capture_enabled.store(0, std::memory_order_release); // let the game play locally again
disable_all(); // let the game play locally again
return false;
};
const unsigned rate = ring->sample_rate;
const unsigned channels = ring->channels;
const unsigned bits = ring->bits;
const unsigned block_align = ring->block_align ? ring->block_align : channels * (bits / 8);
const unsigned rate = primary->sample_rate;
const unsigned channels = primary->channels;
const unsigned bits = primary->bits;
const unsigned tag = primary->format_tag;
const unsigned block_align = primary->block_align ? primary->block_align : channels * (bits / 8);
if (rate == 0 || channels == 0 || block_align == 0)
{
return fail_to_loopback();
@@ -267,12 +294,12 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
wfx.dwChannelMask = (channels >= 32) ? 0xFFFFFFFFu : ((1u << channels) - 1u);
break;
}
wfx.SubFormat = (ring->format_tag == WAVE_FORMAT_IEEE_FLOAT) ? KSDATAFORMAT_SUBTYPE_IEEE_FLOAT
: KSDATAFORMAT_SUBTYPE_PCM;
wfx.SubFormat =
(tag == WAVE_FORMAT_IEEE_FLOAT) ? KSDATAFORMAT_SUBTYPE_IEEE_FLOAT : KSDATAFORMAT_SUBTYPE_PCM;
}
else
{
wfx.Format.wFormatTag = static_cast<WORD>(ring->format_tag ? ring->format_tag : WAVE_FORMAT_PCM);
wfx.Format.wFormatTag = static_cast<WORD>(tag ? tag : WAVE_FORMAT_PCM);
wfx.Format.cbSize = 0;
}
auto* fmt = reinterpret_cast<WAVEFORMATEX*>(&wfx);
@@ -355,6 +382,12 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
const size_t prime_bytes = frame_bytes * (rate * 30 / 1000); // ~30 ms before feeding
bool primed = false;
// Mixing scratch (only used when >1 same-format stream is active): a per-stream
// temp buffer and a float accumulator sized to the render buffer.
const bool mixer_ok = mix_format_supported(tag, bits);
std::vector<BYTE> temp(static_cast<size_t>(render_frames) * frame_bytes);
std::vector<float> acc(static_cast<size_t>(render_frames) * channels);
if (FAILED(hr = render_client->Start()))
{
fail("Render Start", hr);
@@ -380,7 +413,8 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
continue;
}
const UINT32 avail = render_frames - padding;
const std::uint32_t ring_bytes = audio_ring_available(*ring);
// The primary ring is the master clock for priming + how much to write.
const std::uint32_t ring_bytes = audio_ring_available(*primary);
buffered_ms_.store(static_cast<unsigned>(ring_bytes / frame_bytes * 1000 / rate),
std::memory_order_relaxed);
if (!primed && ring_bytes >= prime_bytes)
@@ -393,10 +427,47 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
const UINT32 to_write = std::min(avail, have);
if (to_write > 0)
{
// Active streams = same format as primary (so they can be summed).
// Streams with a different format are still silenced by the hook (no
// echo) but can't be mixed here without resampling -> skipped.
unsigned active[kMaxAudioStreams];
unsigned n_active = 0;
for (unsigned i = 0; i < kMaxAudioStreams; ++i)
{
AudioRingHeader* r = rings[i];
if (r == nullptr)
{
continue;
}
if (i == 0 || (audio_ring_format_ready(*r) && r->sample_rate == rate &&
r->channels == channels && r->bits == bits && r->format_tag == tag))
{
active[n_active++] = i;
}
}
BYTE* dst = nullptr;
if (SUCCEEDED(render->GetBuffer(to_write, &dst)))
{
audio_ring_pop(*ring, dst, to_write * static_cast<std::uint32_t>(frame_bytes));
const std::uint32_t want_bytes = to_write * static_cast<std::uint32_t>(frame_bytes);
if (n_active <= 1 || !mixer_ok)
{
// Single stream (the common case) or an unmixable format:
// passthrough the primary, byte-for-byte (no mixer overhead).
audio_ring_pop(*primary, dst, want_bytes);
}
else
{
const std::uint32_t samples = to_write * channels;
std::fill(acc.begin(), acc.begin() + samples, 0.0f);
for (unsigned k = 0; k < n_active; ++k)
{
std::memset(temp.data(), 0, want_bytes); // zero-fill short reads
audio_ring_pop(*rings[active[k]], temp.data(), want_bytes);
mix_add(acc.data(), temp.data(), samples, tag, bits);
}
mix_store(dst, acc.data(), samples, tag, bits);
}
render->ReleaseBuffer(to_write, 0);
}
}
@@ -410,7 +481,7 @@ bool AudioMirror::run_hooked(AudioRingHeader* ring)
render_client->Stop();
} while (false);
ring->capture_enabled.store(0, std::memory_order_release); // game audible again on stop
disable_all(); // game audible again on stop
if (render)
{

8
host/src/audio/audio_loopback.hpp
View File

@@ -18,6 +18,7 @@
#include <windows.h>
#include "coop/audio_ring.hpp"
#include "coop/protocol.hpp" // kMaxAudioStreams
#include "coop/shared_memory.hpp"
namespace coop
@@ -97,8 +98,9 @@ public:
private:
void thread_main(DWORD pid);
// Returns true if it owned the session to a clean stop; false if setup failed
// and the caller should fall back to the loopback path.
bool run_hooked(AudioRingHeader* ring);
// and the caller should fall back to the loopback path. `rings[0]` is the primary
// stream; additional non-null rings are mixed in.
bool run_hooked(AudioRingHeader* const* rings);
void run_loopback(DWORD pid);
bool wait_for_format(AudioRingHeader* ring, DWORD timeout_ms);
bool stop_requested() const;
@@ -108,7 +110,7 @@ private:
HANDLE stop_event_ = nullptr;
DWORD pid_ = 0;
SharedMemory audio_ring_shm_; // host-created shared ring (named coop_audio_<pid>)
SharedMemory audio_ring_shm_[kMaxAudioStreams]; // per-stream rings (coop_audio_<pid>[_<i>])
std::atomic<bool> running_{false};
std::atomic<Source> source_{Source::None};

77
host/src/audio/audio_mix.hpp Normal file
View File

@@ -0,0 +1,77 @@
// Pure helpers for mixing several same-format audio streams (float32 or int16)
// into one render buffer, with a soft clip on the sum. Header-only so the host
// mixer and a unit test share the exact same math.
#pragma once
#include <cmath>
#include <cstdint>
namespace coop
{
// WAVE_FORMAT_* values used here (kept local to avoid an mmreg.h dependency).
inline constexpr std::uint32_t kWaveFormatPcm = 1;
inline constexpr std::uint32_t kWaveFormatFloat = 3;
// The mixer decodes/encodes these formats; anything else falls back to a plain
// single-stream passthrough on the host (no mixing).
inline bool mix_format_supported(std::uint32_t format_tag, std::uint32_t bits)
{
return (format_tag == kWaveFormatFloat && bits == 32) || (format_tag == kWaveFormatPcm && bits == 16);
}
// Smoothly limit the summed signal to [-1, 1]; ~identity for small inputs.
inline float soft_clip(float x)
{
return std::tanh(x);
}
// Decode `samples` interleaved samples from `src` (format_tag/bits) and add them
// into the float accumulator `acc`. No-op for unsupported formats.
inline void mix_add(float* acc, const std::uint8_t* src, std::uint32_t samples, std::uint32_t format_tag,
std::uint32_t bits)
{
if (format_tag == kWaveFormatFloat && bits == 32)
{
const auto* f = reinterpret_cast<const float*>(src);
for (std::uint32_t i = 0; i < samples; ++i)
{
acc[i] += f[i];
}
}
else if (format_tag == kWaveFormatPcm && bits == 16)
{
const auto* s = reinterpret_cast<const std::int16_t*>(src);
for (std::uint32_t i = 0; i < samples; ++i)
{
acc[i] += static_cast<float>(s[i]) / 32768.0f;
}
}
}
// Soft-clip `acc` and encode `samples` interleaved samples into `dst` in the given
// format. No-op for unsupported formats.
inline void mix_store(std::uint8_t* dst, const float* acc, std::uint32_t samples, std::uint32_t format_tag,
std::uint32_t bits)
{
if (format_tag == kWaveFormatFloat && bits == 32)
{
auto* f = reinterpret_cast<float*>(dst);
for (std::uint32_t i = 0; i < samples; ++i)
{
f[i] = soft_clip(acc[i]);
}
}
else if (format_tag == kWaveFormatPcm && bits == 16)
{
auto* s = reinterpret_cast<std::int16_t*>(dst);
for (std::uint32_t i = 0; i < samples; ++i)
{
int v = static_cast<int>(soft_clip(acc[i]) * 32767.0f);
v = v > 32767 ? 32767 : (v < -32768 ? -32768 : v);
s[i] = static_cast<std::int16_t>(v);
}
}
}
} // namespace coop