BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Recover a guessed audio stream's rate by correlating hook vs loopback (step a)

Browse Source 
When the host attaches to an already-running game it never saw the stream's
Initialize, so the render-hook assumes the device mix format and measures only
the sample rate from the render cadence -- which a jittery game can make wrong
(intermittent pitch shift). But during the measurement window the game is still
audible, so we have the same audio twice: the hook (pre-mix, unknown format) and
a process-loopback (post-mix, the known device format). Cross-correlating them
pins the true rate from ground truth.

- common/include/coop/audio_correlate.hpp: the pure correlator. Resample the hook
  by each candidate standard rate up to the device rate and score how well it
  aligns with the loopback across the window (drift-detecting). audio_correlation_test
  recovers every rate (score ~1.0 vs ~0.01 for wrong ones), incl. 44100-vs-48000,
  and rejects unrelated signals.
- Hook measurement tap: a host-set verify_capture ring flag makes the hook push a
  still-being-measured (guessed) stream's raw pre-mix bytes WITHOUT silencing, so
  the host can co-capture both signals (a silenced game's loopback is silent).
  Inert by default -- the shipping no-echo path is untouched.
- host/src/audio/audio_format_verifier: co-captures hook + loopback and correlates,
  feeding a correction into the existing override channel. Wired into AudioMirror's
  measurement window (hidden in the gap loopback already covers, so exact streams
  pay nothing). audio_verify_test drives it end-to-end against coop_mock_game.

Rate vs layout are coupled (correlating the waveform needs the right channel
de-interleaving), so this step assumes the hook layout matches the device (the
common stereo-on-stereo case); recovering a different channel count / bit depth
is step b.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-23 02:26:06 +02:00
parent 21c15b162b
commit 00244bcfd7
12 changed files with 961 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
host/src/audio/audio_format_verifier.cpp Normal file
View File

@@ -0,0 +1,187 @@
#include "audio/audio_format_verifier.hpp"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#include <audioclient.h>
#include <mmreg.h>
#include "audio/process_loopback_capture.hpp"
#include "coop/audio_correlate.hpp"
namespace coop
{
namespace
{
// Resolve a (possibly EXTENSIBLE) WAVEFORMATEX to scalar channels / bits / tag.
struct ScalarFormat
{
unsigned rate = 0;
unsigned channels = 0;
unsigned bits = 0;
unsigned tag = 0; // WAVE_FORMAT_PCM / _IEEE_FLOAT
};
ScalarFormat resolve(const WAVEFORMATEX* wfx)
{
ScalarFormat f;
f.rate = wfx->nSamplesPerSec;
f.channels = wfx->nChannels;
f.bits = wfx->wBitsPerSample;
f.tag = wfx->wFormatTag;
if (wfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE && wfx->cbSize >= 22)
{
const auto* ext = reinterpret_cast<const WAVEFORMATEXTENSIBLE*>(wfx);
if (ext->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)
{
f.tag = WAVE_FORMAT_IEEE_FLOAT;
}
else if (ext->SubFormat == KSDATAFORMAT_SUBTYPE_PCM)
{
f.tag = WAVE_FORMAT_PCM;
}
}
return f;
}
// Decode interleaved PCM (`channels`/`bits`/`tag`) into per-channel float samples, then average to
// mono. Handles float32 and 16/32-bit PCM (the formats WASAPI shared-mode streams use).
std::vector<float> to_mono(const std::vector<BYTE>& bytes, const ScalarFormat& fmt)
{
std::vector<float> mono;
const unsigned ch = fmt.channels == 0 ? 1 : fmt.channels;
const unsigned bps = fmt.bits / 8;
if (bps == 0)
{
return mono;
}
const std::size_t frame = static_cast<std::size_t>(ch) * bps;
const std::size_t frames = bytes.size() / frame;
mono.resize(frames);
const bool is_float = fmt.tag == WAVE_FORMAT_IEEE_FLOAT;
for (std::size_t i = 0; i < frames; ++i)
{
double sum = 0.0;
for (unsigned c = 0; c < ch; ++c)
{
const BYTE* p = bytes.data() + i * frame + static_cast<std::size_t>(c) * bps;
float s = 0.0f;
if (is_float && fmt.bits == 32)
{
std::memcpy(&s, p, 4);
}
else if (fmt.bits == 16)
{
std::int16_t v;
std::memcpy(&v, p, 2);
s = v / 32768.0f;
}
else if (fmt.bits == 32)
{
std::int32_t v;
std::memcpy(&v, p, 4);
s = static_cast<float>(v / 2147483648.0);
}
sum += s;
}
mono[i] = static_cast<float>(sum / ch);
}
return mono;
}
void drain_ring(AudioRingHeader& ring, std::vector<BYTE>& scratch)
{
while (audio_ring_pop(ring, scratch.data(), static_cast<std::uint32_t>(scratch.size())) > 0)
{
}
}
} // namespace
FormatVerification verify_stream_format(DWORD pid, AudioRingHeader* ring, unsigned window_ms, bool recover_layout)
{
(void)recover_layout; // step (b) extends this; step (a) recovers the rate only
FormatVerification result;
if (ring == nullptr)
{
return result;
}
WAVEFORMATEX* dev_wfx = default_render_format();
if (dev_wfx == nullptr)
{
return result;
}
const ScalarFormat dev = resolve(dev_wfx);
// Ask the hook to push the guessed stream's pre-mix bytes (no silence) while it measures, and
// clear any stale ring contents so we only collect this window.
std::vector<BYTE> scratch(64 * 1024);
drain_ring(*ring, scratch);
ring->verify_capture.store(1, std::memory_order_release);
// Capture the post-mix loopback in parallel (this is the known-format ground truth).
std::vector<BYTE> loop_bytes;
ProcessLoopbackCapture loop;
const std::uint32_t loop_block = dev_wfx->nBlockAlign;
loop.start(pid, dev_wfx, [&](const BYTE* data, std::uint32_t frames, bool silent) {
if (!silent && data != nullptr)
{
loop_bytes.insert(loop_bytes.end(), data, data + static_cast<std::size_t>(frames) * loop_block);
}
});
// Pull the hook's pre-mix bytes out of the ring across the window.
std::vector<BYTE> hook_bytes;
const DWORD end = GetTickCount() + window_ms;
while (GetTickCount() < end)
{
std::uint32_t n = 0;
while ((n = audio_ring_pop(*ring, scratch.data(), static_cast<std::uint32_t>(scratch.size()))) > 0)
{
hook_bytes.insert(hook_bytes.end(), scratch.data(), scratch.data() + n);
}
Sleep(10);
}
std::uint32_t n = 0;
while ((n = audio_ring_pop(*ring, scratch.data(), static_cast<std::uint32_t>(scratch.size()))) > 0)
{
hook_bytes.insert(hook_bytes.end(), scratch.data(), scratch.data() + n);
}
loop.stop();
ring->verify_capture.store(0, std::memory_order_release);
drain_ring(*ring, scratch); // leave the ring clean for the real capture that follows
// The hook bytes are at the guessed layout = the device channels/bits (the assumption the
// cadence path also makes). Decode both captures with that layout and correlate.
const std::vector<float> hook_mono = to_mono(hook_bytes, dev);
const std::vector<float> loop_mono = to_mono(loop_bytes, dev);
CoTaskMemFree(dev_wfx);
if (const char* dbg = std::getenv("COOP_VERIFY_DEBUG"); dbg != nullptr && dbg[0] == '1')
{
std::fprintf(stderr, "[verify] dev=%uHz/%uch/%ubit tag=%u hook_frames=%zu loop_frames=%zu\n", dev.rate,
dev.channels, dev.bits, dev.tag, hook_mono.size(), loop_mono.size());
}
const std::size_t need = dev.rate / 5; // require >= ~200 ms of usable audio on both sides
if (hook_mono.size() < need || loop_mono.size() < need)
{
return result; // not enough non-silent audio captured (game quiet, or stream wasn't a guess)
}
const RateCorrelation rc = correlate_rate(hook_mono, loop_mono, dev.rate, standard_audio_rates());
result.ok = rc.ok;
result.rate = rc.rate;
result.score = rc.score;
// Channels/bit-depth stay the device assumption here; step (b) recovers them.
result.channels = dev.channels;
result.bits = dev.bits;
result.format_tag = dev.tag;
return result;
}
} // namespace coop