BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Recover a guessed stream's channels + bit depth by correlation too (step b)

Browse Source 
Extends the two-path correlation from rate-only to the full layout, removing the
"channels/bit-depth assumed = device" limitation. correlate_format tries each
candidate de-interleaving (float32 / int16; mono..7.1) of the hook capture, runs
the rate correlation per layout, and keeps whichever aligns with the loopback;
a wrong de-interleaving is noise and won't.

The catch: the hook can't know a guessed stream's real frame size, so its verify
tap pads each render buffer to the device block -- which over-reads stale staging
bytes for a stream with fewer channels/bits, scrambling the audio. So the tap is
now self-describing: it prefixes each buffer with its frame count
([count][count*device_block bytes]), and the host strips the padding per candidate
layout (take the real count*real_block of each chunk) before de-interleaving.

- audio_correlate.hpp: ChunkedCapture + chunk-aware correlate_format + candidate
  layouts; absolute-margin confidence gate (the true layout scores ~1.0, a truly
  ambiguous alternative within ~0.001 -- 2ch@R == 1ch@2R for identical channels --
  is correctly left unconfident).
- audio_hook.cpp: chunked verify tap (free-space-checked so framing can't tear).
- audio_format_verifier: parse chunks; recover_layout path. AudioMirror now corrects
  the full format.
- audio_correlation_test: layout recovery from padded chunks (stereo float, 16-bit
  PCM, 5.1, mono). audio_verify_test gains scenario (b): 2ch on a multichannel
  endpoint with distinct per-channel content (new env-gated ToneSource mode) ->
  recovers ch=2/32-bit float end-to-end.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-23 02:50:59 +02:00
parent 00244bcfd7
commit 7ada550930
9 changed files with 554 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
tests/audio_correlation_test.cpp
View File

@@ -6,6 +6,7 @@
// rates (plus a capture-latency skew and a little noise), then assert correlate_rate() recovers the
// true rate -- including the hard 44100-vs-48000 case the cadence method can misread. Pure header
// logic, no device.
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
@@ -55,6 +56,118 @@ std::vector<float> capture(unsigned rate, double seconds, double t0, double nois
return out;
}
// Per-channel continuous signal: each channel carries genuinely different content (its own
// frequency set), like a real stereo/surround stream. This is what disambiguates the channel
// count -- with identical channels, 2ch@R and 1ch@2R produce the same bytes and are truly
// indistinguishable (the confidence gate correctly rejects that case).
double multi(double t, unsigned channel)
{
const double k = 1.0 + 0.37 * static_cast<double>(channel); // distinct frequency scale per channel
const double chirp = std::sin(2.0 * kPi * (300.0 * k * t + 140.0 * t * t));
return 0.5 * std::sin(2.0 * kPi * 221.0 * k * t) + 0.28 * std::sin(2.0 * kPi * 437.0 * k * t + 0.6) +
0.22 * chirp;
}
double multi_mono(double t, unsigned channels)
{
double sum = 0.0;
for (unsigned c = 0; c < channels; ++c)
{
sum += multi(t, c);
}
return sum / channels;
}
// Encode the multichannel signal to raw interleaved PCM bytes at the given layout/rate.
std::vector<std::uint8_t> encode(unsigned rate, unsigned channels, unsigned bits, unsigned tag, double seconds)
{
const unsigned bps = bits / 8;
const std::size_t frames = static_cast<std::size_t>(rate * seconds);
std::vector<std::uint8_t> out(frames * channels * bps);
for (std::size_t i = 0; i < frames; ++i)
{
const double t = static_cast<double>(i) / rate;
for (unsigned c = 0; c < channels; ++c)
{
const double s = multi(t, c);
std::uint8_t* p = out.data() + (i * channels + c) * bps;
if (tag == coop::kWaveFormatFloat)
{
const float f = static_cast<float>(s);
std::memcpy(p, &f, 4);
}
else
{
const std::int16_t v = static_cast<std::int16_t>(s * 30000.0);
std::memcpy(p, &v, 2);
}
}
}
return out;
}
// Build the hook's self-describing chunked capture from clean audio: split into ~480-frame chunks
// and pad each frame from real_block up to `stride` with GARBAGE -- exactly what the hook's verify
// tap produces (it over-reads a guessed stream whose real layout has fewer channels/bits than the
// device block). correlate_format must strip the padding per candidate layout.
coop::ChunkedCapture make_chunks(unsigned rate, unsigned ch, unsigned bits, unsigned tag, unsigned stride,
double seconds)
{
coop::ChunkedCapture cap;
cap.stride = stride;
const std::vector<std::uint8_t> clean = encode(rate, ch, bits, tag, seconds);
const unsigned real_block = ch * (bits / 8);
const std::size_t frames = clean.size() / real_block;
std::mt19937 rng(123);
std::uniform_int_distribution<int> garbage(0, 255);
std::size_t f = 0;
while (f < frames)
{
const unsigned count = static_cast<unsigned>(std::min<std::size_t>(480, frames - f));
cap.counts.push_back(count);
// The hook reads count*stride contiguous bytes: the count real frames first
// (count*real_block bytes), then count*(stride-real_block) bytes of stale over-read.
const std::uint8_t* src = clean.data() + f * real_block;
cap.bytes.insert(cap.bytes.end(), src, src + static_cast<std::size_t>(count) * real_block);
for (std::size_t p = 0; p < static_cast<std::size_t>(count) * (stride - real_block); ++p)
{
cap.bytes.push_back(static_cast<std::uint8_t>(garbage(rng)));
}
f += count;
}
return cap;
}
// One layout scenario: the hook bytes are at (true_*) and still being measured; the loopback is the
// post-mix mono of the same audio at device_rate. Assert correlate_format recovers the full layout.
void test_layout(unsigned true_rate, unsigned true_ch, unsigned true_bits, unsigned true_tag,
unsigned device_rate, const char* label)
{
std::printf("== layout: %s (%u Hz / %u ch / %u-bit %s -> device %u Hz) ==\n", label, true_rate, true_ch,
true_bits, true_tag == coop::kWaveFormatFloat ? "float" : "pcm", device_rate);
// Device block 32 (8ch float) is the largest stride; every test layout's real block is <= 32.
const coop::ChunkedCapture hook = make_chunks(true_rate, true_ch, true_bits, true_tag, /*stride=*/32, 0.6);
// Loopback: the post-mix mono of the same audio, at the device rate, started ~18 ms later + noise.
const std::size_t loop_frames = static_cast<std::size_t>(device_rate * 0.6);
std::vector<float> loop(loop_frames);
std::mt19937 rng(5);
std::uniform_real_distribution<float> j(-1.0f, 1.0f);
for (std::size_t m = 0; m < loop_frames; ++m)
{
loop[m] = static_cast<float>(multi_mono(0.018 + static_cast<double>(m) / device_rate, true_ch)) +
0.02f * j(rng);
}
const FormatCorrelation r =
correlate_format(hook, loop, device_rate, standard_audio_rates(), standard_audio_layouts());
std::printf(" picked %u Hz / %u ch / %u-bit %s score=%.3f runner_up=%.3f ok=%d\n", r.rate, r.channels,
r.bits, r.tag == coop::kWaveFormatFloat ? "float" : "pcm", r.score, r.runner_up, r.ok ? 1 : 0);
check(r.ok, "layout pick is confident");
check(r.rate == true_rate, "recovered the true rate");
check(r.channels == true_ch, "recovered the true channel count");
check(r.bits == true_bits && r.tag == true_tag, "recovered the true bit depth / sample format");
}
// One scenario: true hook rate `true_rate` mixed to `device_rate`. Assert the correlator picks
// true_rate confidently and that the runner-up is clearly behind.
void test_case(unsigned true_rate, unsigned device_rate, const char* label)
@@ -84,6 +197,13 @@ int main()
test_case(32000, 44100, "low-rate stream on a 44100 endpoint");
test_case(48000, 44100, "48000 stream on a 44100 endpoint");
// Step (b): recover the full layout (channels + bit depth) when it differs from the device, by
// trying candidate de-interleavings -- the case the rate-only step can't handle.
test_layout(44100, 2, 32, coop::kWaveFormatFloat, 48000, "stereo float, wrong rate");
test_layout(44100, 2, 16, coop::kWaveFormatPcm, 48000, "stereo 16-bit PCM (bit depth differs)");
test_layout(48000, 6, 32, coop::kWaveFormatFloat, 48000, "5.1 float (channels differ)");
test_layout(44100, 1, 32, coop::kWaveFormatFloat, 48000, "mono"); // 2ch@22050 isn't a candidate -> unambiguous
// Downmix sanity: a stereo interleaved buffer collapses to the same mono the scalar path uses.
{
std::printf("== downmix stereo -> mono ==\n");

166
tests/audio_verify_test.cpp
View File

@@ -1,22 +1,23 @@
// Integration test for the two-path audio-format verifier (host/src/audio/audio_format_verifier).
//
// Launches coop_mock_game rendering a tone at a NON-device rate (44100 on a typical 48000 endpoint,
// the Godot/Brotato case), injects coop_hook.dll late (so the stream is a *guess*), then runs the
// real verify_stream_format(): it co-captures the hook (pre-mix, via the ring's verify tap) and a
// parallel process-loopback (post-mix, device format) and cross-correlates them. Asserts it
// recovers the true 44100 Hz rate -- the cadence method's hard case. Skips cleanly without an audio
// endpoint / if Vulkan-free... (only needs WASAPI + a D3D11-capable mock, which the mock always is).
// Launches coop_mock_game rendering a tone via WASAPI AUTOCONVERTPCM, injects coop_hook.dll late
// (so the stream is a *guess*), and runs the real verify_stream_format(): it co-captures the hook
// (pre-mix, via the ring's verify tap) and a parallel process-loopback (post-mix, device format)
// and cross-correlates them. Two scenarios:
// (a) rate: render at the device's channel count but a different rate -> recover the rate.
// (b) layout: render a DIFFERENT channel count than the device, with distinct per-channel content
// -> recover channels + bit depth + rate.
// Skips cleanly without an audio endpoint.
#include <cstdint>
#include <cstdio>
#include <string>
#include <windows.h>
#include <mmreg.h>
#include <objbase.h>
#include <tlhelp32.h>
#include <mmreg.h>
#include "audio/audio_format_verifier.hpp"
#include "audio/process_loopback_capture.hpp" // default_render_format
#include "coop/audio_ring.hpp"
@@ -111,42 +112,42 @@ bool inject_retry(unsigned long pid)
}
return false;
}
} // namespace
int main()
struct Scenario
{
unsigned rate, channels, bits;
bool distinct; // distinct per-channel content (so the channel count is recoverable)
bool recover_layout; // false = rate only (step a); true = full layout (step b)
};
// Launch the mock at the scenario's format, inject the hook late, and run the verifier. `ran` is
// set false when the environment can't support the test (launch/inject failed) so the caller skips.
FormatVerification run(const Scenario& sc, bool& ran)
{
ran = false;
FormatVerification fv;
kill_stray_mock_games();
const bool com = SUCCEEDED(CoInitializeEx(nullptr, COINIT_MULTITHREADED));
// Render the mock at the device's CHANNEL count (so this step-(a) rate test isn't perturbed by
// a channel mismatch -- that's step (b)'s job) but at a DIFFERENT standard rate than the device,
// so the verifier has a real rate to recover. Default to 48000/2ch if we can't read the device.
unsigned dev_rate = 48000, dev_channels = 2;
if (WAVEFORMATEX* dev = default_render_format())
if (sc.distinct)
{
dev_rate = dev->nSamplesPerSec;
dev_channels = dev->nChannels;
CoTaskMemFree(dev);
SetEnvironmentVariableW(L"COOP_TONE_DISTINCT_CH", L"1");
}
const unsigned game_rate = (dev_rate == 44100) ? 48000u : 44100u; // guarantee a rate mismatch
std::printf(" device %u Hz / %u ch -> rendering the mock at %u Hz / %u ch (rate mismatch)\n", dev_rate,
dev_channels, game_rate, dev_channels);
const std::wstring exe = exe_directory() + L"coop_mock_game.exe";
std::wstring cmd = L"\"" + exe + L"\" dx11 30 " + std::to_wstring(game_rate) + L" " +
std::to_wstring(dev_channels) + L" 32 float";
std::wstring cmd = L"\"" + exe + L"\" dx11 30 " + std::to_wstring(sc.rate) + L" " +
std::to_wstring(sc.channels) + L" " + std::to_wstring(sc.bits) + L" " +
(sc.bits == 16 ? L"pcm" : L"float");
STARTUPINFOW si{};
si.cb = sizeof(si);
PROCESS_INFORMATION pi{};
if (!CreateProcessW(exe.c_str(), cmd.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &si, &pi))
const BOOL launched = CreateProcessW(exe.c_str(), cmd.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr,
&si, &pi);
if (sc.distinct)
{
std::printf("Could not launch coop_mock_game -- skipping audio_verify_test.\n");
if (com)
{
CoUninitialize();
}
return 0;
SetEnvironmentVariableW(L"COOP_TONE_DISTINCT_CH", nullptr);
}
if (!launched)
{
return fv;
}
auto cleanup = [&] {
TerminateProcess(pi.hProcess, 0);
@@ -155,73 +156,104 @@ int main()
CloseHandle(pi.hProcess);
kill_stray_mock_games();
};
Sleep(800); // window + audio client up
Sleep(800);
// IPC + the primary audio ring the hook produces into.
SharedMemory shm;
if (!shm.create(shared_memory_name(pi.dwProcessId), sizeof(SharedBlock)))
SharedMemory ring_shm;
if (!shm.create(shared_memory_name(pi.dwProcessId), sizeof(SharedBlock)) ||
!ring_shm.create(audio_ring_name(pi.dwProcessId), audio_ring_total_size(kAudioRingCapacity)))
{
std::printf("Could not create IPC block -- skipping.\n");
cleanup();
return 0;
return fv;
}
auto* block = shm.as<SharedBlock>();
block->version = kProtocolVersion;
block->pad_count = 0;
block->sequence.store(0, std::memory_order_relaxed);
// Only the audio subsystem.
for (std::uint32_t s = 0; s < HookSubsys_Count; ++s)
for (std::uint32_t s = 0; s < HookSubsys_Count; ++s) // audio subsystem only
{
const bool off = s != HookSubsys_Audio;
block->control.subsystem_disabled[s].store(off ? 1u : 0u, std::memory_order_release);
block->control.subsystem_disabled[s].store(s != HookSubsys_Audio ? 1u : 0u, std::memory_order_release);
}
block->magic = kProtocolMagic;
SharedMemory ring_shm;
if (!ring_shm.create(audio_ring_name(pi.dwProcessId), audio_ring_total_size(kAudioRingCapacity)))
{
std::printf("Could not create audio ring -- skipping.\n");
cleanup();
return 0;
}
auto* ring = ring_shm.as<AudioRingHeader>();
audio_ring_init(*ring, kAudioRingCapacity);
// Leave capture_enabled = 0: we want the stream audible (so loopback hears it) and still being
// MEASURED (so the verify tap fires), exactly the window verify_stream_format targets.
audio_ring_init(*ring, kAudioRingCapacity); // capture_enabled stays 0: audible + still measuring
if (!inject_retry(pi.dwProcessId))
{
std::printf("Could not inject coop_hook.dll -- skipping.\n");
cleanup();
return 0;
return fv;
}
Sleep(500); // let the hook attach + the pre-existing render client register as a guess
Sleep(500); // hook attaches + the pre-existing render client registers as a guess
// Run the real verifier: co-capture hook (pre-mix) + loopback (post-mix) and correlate.
const FormatVerification fv = verify_stream_format(pi.dwProcessId, ring, /*window_ms=*/1400);
std::printf(" verify: ok=%d rate=%u score=%.3f\n", fv.ok ? 1 : 0, fv.rate, fv.score);
fv = verify_stream_format(pi.dwProcessId, ring, /*window_ms=*/1400, sc.recover_layout);
ran = true;
cleanup();
return fv;
}
} // namespace
if (!fv.ok && fv.rate == 0 && fv.score == 0.0)
int main()
{
const bool com = SUCCEEDED(CoInitializeEx(nullptr, COINIT_MULTITHREADED));
unsigned dev_rate = 48000, dev_channels = 2;
if (WAVEFORMATEX* dev = default_render_format())
{
// No audio endpoint, or no usable audio captured (e.g. the mock's WASAPI client never
// started on this machine) -> treat as a skip rather than a failure.
std::printf(" no usable co-capture (no endpoint / silent) -- skipping audio_verify_test.\n");
dev_rate = dev->nSamplesPerSec;
dev_channels = dev->nChannels;
CoTaskMemFree(dev);
}
const unsigned mismatched = (dev_rate == 44100) ? 48000u : 44100u; // guarantee a rate mismatch
std::printf("device: %u Hz / %u ch\n", dev_rate, dev_channels);
// (a) Rate: render at the device's channel count (no layout mismatch) but a different rate.
std::printf("== (a) rate recovery: %u Hz / %u ch ==\n", mismatched, dev_channels);
bool ran = false;
FormatVerification a = run({mismatched, dev_channels, 32, /*distinct=*/false, /*recover_layout=*/false}, ran);
if (!ran)
{
std::printf(" environment can't run the mock+inject -- skipping audio_verify_test.\n");
if (com)
{
CoUninitialize();
}
cleanup();
return 0;
}
std::printf(" ok=%d rate=%u score=%.3f\n", a.ok ? 1 : 0, a.rate, a.score);
if (!a.ok && a.rate == 0 && a.score == 0.0)
{
std::printf(" no usable co-capture (no endpoint / silent) -- skipping.\n");
if (com)
{
CoUninitialize();
}
return 0;
}
check(a.ok, "(a) verifier confidently correlated the two capture paths");
check(a.rate == mismatched, "(a) recovered the game's true rate (not the device rate)");
check(fv.ok, "verifier confidently correlated the two capture paths");
check(fv.rate == game_rate, "verifier recovered the game's true rate (not the device rate)");
// (b) Layout: render 2ch with distinct per-channel content -- a layout that differs from a
// multichannel device -- and recover channels + bit depth + rate.
std::printf("== (b) layout recovery: 44100 Hz / 2 ch / 32-bit float (distinct channels) ==\n");
FormatVerification b = run({44100, 2, 32, /*distinct=*/true, /*recover_layout=*/true}, ran);
std::printf(" ok=%d rate=%u ch=%u bits=%u tag=%u score=%.3f\n", b.ok ? 1 : 0, b.rate, b.channels, b.bits,
b.format_tag, b.score);
if (b.ok || b.score > 0.0)
{
check(b.layout_ok, "(b) verifier confidently recovered the layout");
check(b.rate == 44100, "(b) recovered the true rate");
check(b.channels == 2, "(b) recovered the true channel count (2, not the device's)");
check(b.bits == 32 && b.format_tag == 3, "(b) recovered 32-bit float");
}
else
{
std::printf(" no usable co-capture for (b) -- skipping that scenario.\n");
}
if (com)
{
CoUninitialize();
}
cleanup();
if (g_failures == 0)
{