// Recover a pre-existing render stream's true audio format by *correlating* the two capture // paths, instead of guessing. // // When we attach to an already-running game we never saw its IAudioClient::Initialize, so the // render-hook assumes the device mix format and measures only the sample rate from the render // cadence -- which can be wrong on a jittery game (intermittent pitch shift). But during the // measurement window the game is still audible, so we have BOTH signals of the same audio: // * the render-hook capture -- pre-mix, at the *unknown* format, // * the process-loopback capture -- post-mix, at the *known* device format. // The loopback is just the hook signal resampled by WASAPI's AUTOCONVERTPCM from the stream's // true rate to the device rate. So if we resample the hook stream by a candidate rate up to the // device rate and it lines up with the loopback over the whole window (no drift), that candidate // is the truth. A wrong rate time-warps the hook stream, so a single alignment can't hold across // the window and the correlation collapses. // // This header is the pure, headless-testable core (no devices, no WASAPI). The host downmixes the // two captures to mono float, calls correlate_rate(), and feeds the result into the existing rate // path (publish / override). correlate_format() below builds on the same helpers to also recover // channels + bit depth by trying candidate de-interleavings. #pragma once #include #include #include #include namespace coop { // The standard sample rates a shared-mode WASAPI stream realistically uses. Candidates are this // set; a non-standard true rate is out of scope (and would show as low-confidence either way). inline const std::vector& standard_audio_rates() { static const std::vector rates = {32000, 44100, 48000, 88200, 96000}; return rates; } struct RateCorrelation { bool ok = false; // a confident pick was made (winner clears the threshold AND beats the runner-up) unsigned rate = 0; // best candidate rate (Hz) double score = 0.0; // alignment score of the winner, in [0,1] (1 = perfect) double runner_up = 0.0; // score of the second-best candidate (for separation) }; namespace correlate_detail { // Average interleaved float frames down to a single mono channel. inline void downmix(const float* interleaved, std::size_t frames, unsigned channels, std::vector& out) { out.resize(frames); if (channels == 0) { channels = 1; } for (std::size_t i = 0; i < frames; ++i) { float sum = 0.0f; for (unsigned c = 0; c < channels; ++c) { sum += interleaved[i * channels + c]; } out[i] = sum / static_cast(channels); } } // Linear-resample a mono signal from src_rate to dst_rate. inline void resample_linear(const std::vector& in, unsigned src_rate, unsigned dst_rate, std::vector& out) { if (src_rate == 0 || dst_rate == 0 || in.empty()) { out.clear(); return; } if (src_rate == dst_rate) { out = in; return; } const double step = static_cast(src_rate) / static_cast(dst_rate); const std::size_t out_n = static_cast(static_cast(in.size()) / step); out.resize(out_n); for (std::size_t i = 0; i < out_n; ++i) { const double pos = static_cast(i) * step; const std::size_t j = static_cast(pos); const double frac = pos - static_cast(j); const float a = in[j]; const float b = (j + 1 < in.size()) ? in[j + 1] : a; out[i] = a + static_cast(frac) * (b - a); } } // Box-decimate a mono signal from `rate` down to ~corr_rate for a cheap, content-preserving // alignment search (the envelope/content alignment doesn't need full bandwidth). inline void decimate(const std::vector& in, unsigned rate, unsigned corr_rate, std::vector& out) { if (rate <= corr_rate || in.empty()) { out = in; return; } const double factor = static_cast(rate) / static_cast(corr_rate); const std::size_t out_n = static_cast(static_cast(in.size()) / factor); out.resize(out_n); for (std::size_t i = 0; i < out_n; ++i) { const std::size_t lo = static_cast(static_cast(i) * factor); std::size_t hi = static_cast(static_cast(i + 1) * factor); if (hi <= lo) { hi = lo + 1; } if (hi > in.size()) { hi = in.size(); } float sum = 0.0f; for (std::size_t k = lo; k < hi; ++k) { sum += in[k]; } out[i] = sum / static_cast(hi - lo); } } // Zero-mean, unit-norm cross-correlation of a vs b over [start, start+len), with b shifted by lag. // Returns a value in [-1, 1]; out-of-range samples are skipped. ~0 when the two don't align. inline double ncc(const std::vector& a, const std::vector& b, long lag, std::size_t start, std::size_t len) { double sa = 0.0, sb = 0.0; std::size_t n = 0; for (std::size_t i = start; i < start + len && i < a.size(); ++i) { const long bi = static_cast(i) + lag; if (bi < 0 || static_cast(bi) >= b.size()) { continue; } sa += a[i]; sb += b[bi]; ++n; } if (n < 8) { return 0.0; } const double ma = sa / static_cast(n); const double mb = sb / static_cast(n); double num = 0.0, da = 0.0, db = 0.0; for (std::size_t i = start; i < start + len && i < a.size(); ++i) { const long bi = static_cast(i) + lag; if (bi < 0 || static_cast(bi) >= b.size()) { continue; } const double xa = a[i] - ma; const double xb = b[bi] - mb; num += xa * xb; da += xa * xa; db += xb * xb; } if (da < 1e-9 || db < 1e-9) { return 0.0; } return num / std::sqrt(da * db); } // Alignment score of two same-rate mono signals: find the single best lag over the whole window, // then require that lag to hold in BOTH an early and a late segment (drift detection). The score // is the weaker of the two segment correlations, so a rate that only lines up at the start (a // wrong rate, which time-warps and drifts) scores low while the true rate scores high. inline double aligned_score(const std::vector& a, const std::vector& b, unsigned rate) { const std::size_t n = a.size() < b.size() ? a.size() : b.size(); if (n < rate / 10) // need at least ~100 ms of overlap to judge { return 0.0; } const long max_lag = static_cast(rate / 8); // search +/-125 ms of capture-path latency skew // Coarse global lag from the middle half of the window. const std::size_t mid_start = n / 4; const std::size_t mid_len = n / 2; double best = -2.0; long best_lag = 0; for (long lag = -max_lag; lag <= max_lag; ++lag) { const double c = ncc(a, b, lag, mid_start, mid_len); if (c > best) { best = c; best_lag = lag; } } // Re-evaluate that lag in an early and a late third: the true rate holds; a drifting (wrong) // rate does not. const std::size_t third = n / 3; const double early = ncc(a, b, best_lag, 0, third); const double late = ncc(a, b, best_lag, 2 * third, third); const double weaker = early < late ? early : late; return weaker < 0.0 ? 0.0 : weaker; } } // namespace correlate_detail // Determine the hook stream's true sample rate by resampling it by each candidate rate up to the // known device (loopback) rate and scoring how well it aligns with the loopback across the window. // hook_mono / loop_mono are mono float (caller downmixes). `min_score` is the absolute alignment // floor and `separation` the ratio by which the winner must beat the runner-up to be `ok`. inline RateCorrelation correlate_rate(const std::vector& hook_mono, const std::vector& loop_mono, unsigned device_rate, const std::vector& candidates, double min_score = 0.55, double separation = 1.2) { using namespace correlate_detail; RateCorrelation result; if (hook_mono.empty() || loop_mono.empty() || device_rate == 0) { return result; } constexpr unsigned kCorrRate = 8000; // alignment search rate (Nyquist 4 kHz -- plenty for content) std::vector loop_ds; decimate(loop_mono, device_rate, kCorrRate, loop_ds); double best = -1.0, second = -1.0; unsigned best_rate = 0; std::vector resampled, hook_ds; for (unsigned cand : candidates) { resample_linear(hook_mono, cand, device_rate, resampled); // treat hook as sampled at `cand` decimate(resampled, device_rate, kCorrRate, hook_ds); const double s = aligned_score(hook_ds, loop_ds, kCorrRate); if (s > best) { second = best; best = s; best_rate = cand; } else if (s > second) { second = s; } } result.rate = best_rate; result.score = best < 0.0 ? 0.0 : best; result.runner_up = second < 0.0 ? 0.0 : second; result.ok = result.score >= min_score && (result.runner_up <= 1e-6 || result.score >= result.runner_up * separation); return result; } // --- Channels + bit-depth recovery ------------------------------------------------------------- // // The rate step assumes the hook bytes are de-interleaved at the device channel/bit layout. When a // game renders a DIFFERENT layout than the device (e.g. stereo float on a 7.1 endpoint, or 16-bit // PCM), that assumption garbles the waveform and the rate won't lock. We can't measure the layout // (AUTOCONVERTPCM hides the stride), but we can RECOVER it: interpret the raw hook bytes under each // candidate layout, run the rate correlation, and keep whichever (layout, rate) aligns with the // loopback -- a wrong de-interleaving is noise and won't correlate. inline constexpr unsigned kWaveFormatPcm = 1; // WAVE_FORMAT_PCM inline constexpr unsigned kWaveFormatFloat = 3; // WAVE_FORMAT_IEEE_FLOAT struct LayoutCandidate { unsigned channels; unsigned bits; unsigned tag; // kWaveFormatPcm / kWaveFormatFloat }; // Candidate de-interleavings a shared-mode WASAPI render stream realistically uses: float32 and // 16-bit PCM, across the common channel counts. Ordered most-likely-first. inline const std::vector& standard_audio_layouts() { static const std::vector v = { {2, 32, kWaveFormatFloat}, {1, 32, kWaveFormatFloat}, {6, 32, kWaveFormatFloat}, {8, 32, kWaveFormatFloat}, {4, 32, kWaveFormatFloat}, {2, 16, kWaveFormatPcm}, {1, 16, kWaveFormatPcm}, {6, 16, kWaveFormatPcm}, {8, 16, kWaveFormatPcm}, {4, 16, kWaveFormatPcm}, }; return v; } struct FormatCorrelation { bool ok = false; unsigned rate = 0; unsigned channels = 0; unsigned bits = 0; unsigned tag = 0; double score = 0.0; double runner_up = 0.0; }; // The hook can't know a guessed stream's real frame size, so its verify tap pushes each render // buffer padded to the device block (`stride`) and prefixes it with the real frame `count`. That // padding is stale staging-buffer bytes, so the host must extract the real `count*real_block` bytes // per buffer (and concatenate) before de-interleaving -- otherwise the padding scrambles the audio. // This carries that self-describing capture: `bytes` holds counts[i]*stride bytes per chunk. struct ChunkedCapture { unsigned stride = 0; // bytes per frame as pushed (the guessed/device block_align) std::vector counts; // real frame count of each chunk std::vector bytes; // concatenated, counts[i]*stride bytes per chunk }; namespace correlate_detail { // De-interleave raw bytes under (channels/bits/tag) and average to mono float. inline void decode_layout(const std::uint8_t* bytes, std::size_t n, const LayoutCandidate& fmt, std::vector& mono) { mono.clear(); const unsigned ch = fmt.channels == 0 ? 1 : fmt.channels; const unsigned bps = fmt.bits / 8; if (bps == 0) { return; } const std::size_t frame = static_cast(ch) * bps; const std::size_t frames = n / frame; mono.resize(frames); const bool is_float = fmt.tag == kWaveFormatFloat; for (std::size_t i = 0; i < frames; ++i) { double sum = 0.0; for (unsigned c = 0; c < ch; ++c) { const std::uint8_t* p = bytes + i * frame + static_cast(c) * bps; float s = 0.0f; if (is_float && fmt.bits == 32) { std::memcpy(&s, p, 4); } else if (fmt.bits == 16) { std::int16_t v; std::memcpy(&v, p, 2); s = v / 32768.0f; } else if (fmt.bits == 32) { std::int32_t v; std::memcpy(&v, p, 4); s = static_cast(v / 2147483648.0); } sum += s; } mono[i] = static_cast(sum / ch); } } } // namespace correlate_detail // Recover BOTH the layout and the rate of a guessed stream from its (padded, self-describing) hook // capture: for each candidate layout, extract the real count*real_block bytes from each padded // chunk, de-interleave to mono, and run the rate correlation against the known-format loopback, // keeping the (layout, rate) that aligns best. `ok` when the winner clears the alignment floor and // clearly beats the runner-up (so a coincidental partial match is rejected). // // `min_margin` is an ABSOLUTE gap (not a ratio): the true layout scores near-perfectly while a // truly-ambiguous alternative (e.g. 1ch@2R vs 2ch@R when the channels carry the same content) // scores within ~0.001, so requiring the winner to clear the runner-up by a fixed margin cleanly // separates "recovered" from "genuinely ambiguous, don't guess". inline FormatCorrelation correlate_format(const ChunkedCapture& hook, const std::vector& loop_mono, unsigned device_rate, const std::vector& rates, const std::vector& layouts, double min_score = 0.55, double min_margin = 0.04) { FormatCorrelation result; if (hook.stride == 0 || hook.counts.empty() || loop_mono.empty() || device_rate == 0) { return result; } double best = -1.0, second = -1.0; std::vector clean; std::vector hook_mono; for (const LayoutCandidate& layout : layouts) { const unsigned real_block = layout.channels * (layout.bits / 8); if (real_block == 0 || real_block > hook.stride) { continue; // can't extract a frame larger than what was pushed (the guess is the max) } // Pull the real count*real_block bytes out of each padded chunk and concatenate -> contiguous // audio for this candidate layout (the padding, which is stale staging bytes, is dropped). clean.clear(); std::size_t off = 0; for (std::uint32_t count : hook.counts) { const std::size_t chunk_bytes = static_cast(count) * hook.stride; const std::size_t take = static_cast(count) * real_block; if (off + chunk_bytes <= hook.bytes.size()) { clean.insert(clean.end(), hook.bytes.begin() + off, hook.bytes.begin() + off + take); } off += chunk_bytes; } correlate_detail::decode_layout(clean.data(), clean.size(), layout, hook_mono); if (hook_mono.size() < device_rate / 5) { continue; // this layout yields too little audio to judge } const RateCorrelation rc = correlate_rate(hook_mono, loop_mono, device_rate, rates, /*min_score=*/0.0, /*separation=*/1.0); if (rc.score > best) { second = best; best = rc.score; result.rate = rc.rate; result.channels = layout.channels; result.bits = layout.bits; result.tag = layout.tag; } else if (rc.score > second) { second = rc.score; } } result.score = best < 0.0 ? 0.0 : best; result.runner_up = second < 0.0 ? 0.0 : second; result.ok = result.score >= min_score && (result.runner_up <= 1e-6 || result.score - result.runner_up >= min_margin); return result; } } // namespace coop