Recover a guessed audio stream's rate by correlating hook vs loopback (step a)
When the host attaches to an already-running game it never saw the stream's Initialize, so the render-hook assumes the device mix format and measures only the sample rate from the render cadence -- which a jittery game can make wrong (intermittent pitch shift). But during the measurement window the game is still audible, so we have the same audio twice: the hook (pre-mix, unknown format) and a process-loopback (post-mix, the known device format). Cross-correlating them pins the true rate from ground truth. - common/include/coop/audio_correlate.hpp: the pure correlator. Resample the hook by each candidate standard rate up to the device rate and score how well it aligns with the loopback across the window (drift-detecting). audio_correlation_test recovers every rate (score ~1.0 vs ~0.01 for wrong ones), incl. 44100-vs-48000, and rejects unrelated signals. - Hook measurement tap: a host-set verify_capture ring flag makes the hook push a still-being-measured (guessed) stream's raw pre-mix bytes WITHOUT silencing, so the host can co-capture both signals (a silenced game's loopback is silent). Inert by default -- the shipping no-echo path is untouched. - host/src/audio/audio_format_verifier: co-captures hook + loopback and correlates, feeding a correction into the existing override channel. Wired into AudioMirror's measurement window (hidden in the gap loopback already covers, so exact streams pay nothing). audio_verify_test drives it end-to-end against coop_mock_game. Rate vs layout are coupled (correlating the waveform needs the right channel de-interleaving), so this step assumes the hook layout matches the device (the common stereo-on-stereo case); recovering a different channel count / bit depth is step b. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
187
host/src/audio/audio_format_verifier.cpp
Normal file
187
host/src/audio/audio_format_verifier.cpp
Normal file
@@ -0,0 +1,187 @@
|
||||
#include "audio/audio_format_verifier.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
#include <audioclient.h>
|
||||
#include <mmreg.h>
|
||||
|
||||
#include "audio/process_loopback_capture.hpp"
|
||||
#include "coop/audio_correlate.hpp"
|
||||
|
||||
namespace coop
|
||||
{
|
||||
namespace
|
||||
{
|
||||
|
||||
// Resolve a (possibly EXTENSIBLE) WAVEFORMATEX to scalar channels / bits / tag.
|
||||
struct ScalarFormat
|
||||
{
|
||||
unsigned rate = 0;
|
||||
unsigned channels = 0;
|
||||
unsigned bits = 0;
|
||||
unsigned tag = 0; // WAVE_FORMAT_PCM / _IEEE_FLOAT
|
||||
};
|
||||
|
||||
ScalarFormat resolve(const WAVEFORMATEX* wfx)
|
||||
{
|
||||
ScalarFormat f;
|
||||
f.rate = wfx->nSamplesPerSec;
|
||||
f.channels = wfx->nChannels;
|
||||
f.bits = wfx->wBitsPerSample;
|
||||
f.tag = wfx->wFormatTag;
|
||||
if (wfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE && wfx->cbSize >= 22)
|
||||
{
|
||||
const auto* ext = reinterpret_cast<const WAVEFORMATEXTENSIBLE*>(wfx);
|
||||
if (ext->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT)
|
||||
{
|
||||
f.tag = WAVE_FORMAT_IEEE_FLOAT;
|
||||
}
|
||||
else if (ext->SubFormat == KSDATAFORMAT_SUBTYPE_PCM)
|
||||
{
|
||||
f.tag = WAVE_FORMAT_PCM;
|
||||
}
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
// Decode interleaved PCM (`channels`/`bits`/`tag`) into per-channel float samples, then average to
|
||||
// mono. Handles float32 and 16/32-bit PCM (the formats WASAPI shared-mode streams use).
|
||||
std::vector<float> to_mono(const std::vector<BYTE>& bytes, const ScalarFormat& fmt)
|
||||
{
|
||||
std::vector<float> mono;
|
||||
const unsigned ch = fmt.channels == 0 ? 1 : fmt.channels;
|
||||
const unsigned bps = fmt.bits / 8;
|
||||
if (bps == 0)
|
||||
{
|
||||
return mono;
|
||||
}
|
||||
const std::size_t frame = static_cast<std::size_t>(ch) * bps;
|
||||
const std::size_t frames = bytes.size() / frame;
|
||||
mono.resize(frames);
|
||||
const bool is_float = fmt.tag == WAVE_FORMAT_IEEE_FLOAT;
|
||||
for (std::size_t i = 0; i < frames; ++i)
|
||||
{
|
||||
double sum = 0.0;
|
||||
for (unsigned c = 0; c < ch; ++c)
|
||||
{
|
||||
const BYTE* p = bytes.data() + i * frame + static_cast<std::size_t>(c) * bps;
|
||||
float s = 0.0f;
|
||||
if (is_float && fmt.bits == 32)
|
||||
{
|
||||
std::memcpy(&s, p, 4);
|
||||
}
|
||||
else if (fmt.bits == 16)
|
||||
{
|
||||
std::int16_t v;
|
||||
std::memcpy(&v, p, 2);
|
||||
s = v / 32768.0f;
|
||||
}
|
||||
else if (fmt.bits == 32)
|
||||
{
|
||||
std::int32_t v;
|
||||
std::memcpy(&v, p, 4);
|
||||
s = static_cast<float>(v / 2147483648.0);
|
||||
}
|
||||
sum += s;
|
||||
}
|
||||
mono[i] = static_cast<float>(sum / ch);
|
||||
}
|
||||
return mono;
|
||||
}
|
||||
|
||||
void drain_ring(AudioRingHeader& ring, std::vector<BYTE>& scratch)
|
||||
{
|
||||
while (audio_ring_pop(ring, scratch.data(), static_cast<std::uint32_t>(scratch.size())) > 0)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
FormatVerification verify_stream_format(DWORD pid, AudioRingHeader* ring, unsigned window_ms, bool recover_layout)
|
||||
{
|
||||
(void)recover_layout; // step (b) extends this; step (a) recovers the rate only
|
||||
FormatVerification result;
|
||||
if (ring == nullptr)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
WAVEFORMATEX* dev_wfx = default_render_format();
|
||||
if (dev_wfx == nullptr)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
const ScalarFormat dev = resolve(dev_wfx);
|
||||
|
||||
// Ask the hook to push the guessed stream's pre-mix bytes (no silence) while it measures, and
|
||||
// clear any stale ring contents so we only collect this window.
|
||||
std::vector<BYTE> scratch(64 * 1024);
|
||||
drain_ring(*ring, scratch);
|
||||
ring->verify_capture.store(1, std::memory_order_release);
|
||||
|
||||
// Capture the post-mix loopback in parallel (this is the known-format ground truth).
|
||||
std::vector<BYTE> loop_bytes;
|
||||
ProcessLoopbackCapture loop;
|
||||
const std::uint32_t loop_block = dev_wfx->nBlockAlign;
|
||||
loop.start(pid, dev_wfx, [&](const BYTE* data, std::uint32_t frames, bool silent) {
|
||||
if (!silent && data != nullptr)
|
||||
{
|
||||
loop_bytes.insert(loop_bytes.end(), data, data + static_cast<std::size_t>(frames) * loop_block);
|
||||
}
|
||||
});
|
||||
|
||||
// Pull the hook's pre-mix bytes out of the ring across the window.
|
||||
std::vector<BYTE> hook_bytes;
|
||||
const DWORD end = GetTickCount() + window_ms;
|
||||
while (GetTickCount() < end)
|
||||
{
|
||||
std::uint32_t n = 0;
|
||||
while ((n = audio_ring_pop(*ring, scratch.data(), static_cast<std::uint32_t>(scratch.size()))) > 0)
|
||||
{
|
||||
hook_bytes.insert(hook_bytes.end(), scratch.data(), scratch.data() + n);
|
||||
}
|
||||
Sleep(10);
|
||||
}
|
||||
std::uint32_t n = 0;
|
||||
while ((n = audio_ring_pop(*ring, scratch.data(), static_cast<std::uint32_t>(scratch.size()))) > 0)
|
||||
{
|
||||
hook_bytes.insert(hook_bytes.end(), scratch.data(), scratch.data() + n);
|
||||
}
|
||||
|
||||
loop.stop();
|
||||
ring->verify_capture.store(0, std::memory_order_release);
|
||||
drain_ring(*ring, scratch); // leave the ring clean for the real capture that follows
|
||||
|
||||
// The hook bytes are at the guessed layout = the device channels/bits (the assumption the
|
||||
// cadence path also makes). Decode both captures with that layout and correlate.
|
||||
const std::vector<float> hook_mono = to_mono(hook_bytes, dev);
|
||||
const std::vector<float> loop_mono = to_mono(loop_bytes, dev);
|
||||
CoTaskMemFree(dev_wfx);
|
||||
|
||||
if (const char* dbg = std::getenv("COOP_VERIFY_DEBUG"); dbg != nullptr && dbg[0] == '1')
|
||||
{
|
||||
std::fprintf(stderr, "[verify] dev=%uHz/%uch/%ubit tag=%u hook_frames=%zu loop_frames=%zu\n", dev.rate,
|
||||
dev.channels, dev.bits, dev.tag, hook_mono.size(), loop_mono.size());
|
||||
}
|
||||
const std::size_t need = dev.rate / 5; // require >= ~200 ms of usable audio on both sides
|
||||
if (hook_mono.size() < need || loop_mono.size() < need)
|
||||
{
|
||||
return result; // not enough non-silent audio captured (game quiet, or stream wasn't a guess)
|
||||
}
|
||||
|
||||
const RateCorrelation rc = correlate_rate(hook_mono, loop_mono, dev.rate, standard_audio_rates());
|
||||
result.ok = rc.ok;
|
||||
result.rate = rc.rate;
|
||||
result.score = rc.score;
|
||||
// Channels/bit-depth stay the device assumption here; step (b) recovers them.
|
||||
result.channels = dev.channels;
|
||||
result.bits = dev.bits;
|
||||
result.format_tag = dev.tag;
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace coop
|
||||
48
host/src/audio/audio_format_verifier.hpp
Normal file
48
host/src/audio/audio_format_verifier.hpp
Normal file
@@ -0,0 +1,48 @@
|
||||
// Two-path audio-format verification: recover a still-being-measured (guessed) render stream's
|
||||
// true format by *correlating* the two capture paths instead of guessing.
|
||||
//
|
||||
// When the host attaches to an already-running game the render-hook never saw the stream's
|
||||
// Initialize, so it assumes the device mix format and measures only the sample rate from the
|
||||
// render cadence -- which a jittery game can make wrong (intermittent pitch shift). During the
|
||||
// measurement window the game is still audible, so we can capture BOTH signals of the same audio:
|
||||
// * the hook (pre-mix, at the unknown format) via the ring's verify_capture tap, and
|
||||
// * a WASAPI process-loopback (post-mix, at the KNOWN device format).
|
||||
// Cross-correlating them (common/include/coop/audio_correlate.hpp) pins the true rate from ground
|
||||
// truth. The result feeds the existing rate path as an operator-style override.
|
||||
//
|
||||
// Step (a) here recovers the sample rate. Step (b) (audio_format_verifier.cpp) extends the same
|
||||
// co-capture to recover channels + bit depth by trying candidate de-interleavings.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#include "coop/audio_ring.hpp"
|
||||
|
||||
namespace coop
|
||||
{
|
||||
|
||||
struct FormatVerification
|
||||
{
|
||||
bool ok = false; // a confident rate correlation was found
|
||||
unsigned rate = 0; // recovered true sample rate (Hz)
|
||||
double score = 0.0; // correlation score of the winning rate, [0,1]
|
||||
|
||||
bool layout_ok = false; // a confident channels/bit-depth correlation was found (step b)
|
||||
unsigned channels = 0; // recovered channel count
|
||||
unsigned bits = 0; // recovered bits per sample
|
||||
unsigned format_tag = 0; // recovered WAVE_FORMAT_PCM / _IEEE_FLOAT
|
||||
};
|
||||
|
||||
// One-shot: co-capture the hook (pre-mix, via the ring's verify_capture tap) and a parallel
|
||||
// process-loopback (post-mix, at the device format) of `pid`'s audio for ~window_ms, then
|
||||
// cross-correlate to recover the stream's true sample rate (and, with recover_layout, its channels
|
||||
// + bit depth). Returns ok=false (a no-op for the caller) when the stream isn't a guess, the game
|
||||
// is silent, or the two captures don't correlate. Requires a COM-initialized thread. Toggles
|
||||
// ring->verify_capture for the duration and drains the ring afterwards so normal capture starts
|
||||
// clean.
|
||||
FormatVerification verify_stream_format(DWORD pid, AudioRingHeader* ring, unsigned window_ms = 800,
|
||||
bool recover_layout = false);
|
||||
|
||||
} // namespace coop
|
||||
@@ -9,6 +9,7 @@
|
||||
#include <mmdeviceapi.h>
|
||||
#include <mmreg.h>
|
||||
|
||||
#include "audio/audio_format_verifier.hpp"
|
||||
#include "audio/audio_mix.hpp"
|
||||
#include "audio/process_loopback_capture.hpp"
|
||||
#include "audio/render_pacer.hpp"
|
||||
@@ -278,6 +279,7 @@ void AudioMirror::thread_main(DWORD pid)
|
||||
// The guessed-rate path takes a few seconds to reach consensus; loopback covers
|
||||
// that gap and the promote hands off seamlessly.
|
||||
constexpr DWORD kHookWaitMs = 1200;
|
||||
bool format_verified = false; // run the two-path correlation verify/correct at most once
|
||||
for (;;)
|
||||
{
|
||||
if (stop_requested())
|
||||
@@ -312,6 +314,23 @@ void AudioMirror::thread_main(DWORD pid)
|
||||
set_fallback_reason(
|
||||
"Render-hook hasn't published a format yet; using loopback (echo) -- will switch to "
|
||||
"hooked automatically once it does.");
|
||||
// No format after the wait -> a guessed late-attach stream is being *measured*. Once,
|
||||
// while it measures and the game is still audible, run the two-path correlation (hook
|
||||
// pre-mix vs a parallel loopback post-mix of the same audio) to recover the true rate
|
||||
// and correct it through the existing override channel -- this hardens the cadence
|
||||
// method's intermittent pitch-shift. It's hidden inside the measurement gap loopback
|
||||
// already covers, so exact streams (format published immediately) never pay for it.
|
||||
if (!format_verified)
|
||||
{
|
||||
format_verified = true;
|
||||
const FormatVerification fv = verify_stream_format(pid, rings[0]);
|
||||
if (fv.ok)
|
||||
{
|
||||
set_status("Verified render-hook sample rate by correlation.");
|
||||
audio_ring_post_op(*rings[0], AudioRingOp_Override, fv.rate, fv.channels, fv.bits,
|
||||
fv.format_tag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enable_capture(rings, false); // game audible locally so loopback can capture it
|
||||
|
||||
Reference in New Issue
Block a user