BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix 32-bit FMOD audio crash: hook WASAPI COM methods via vtable swap

Browse Source 
The stdcall() fix stopped the Present-hook crash but 32-bit games (Slaps
and Beans, FMOD) still crashed the instant audio init ran through the
hook. Root cause: SafetyHook's inline hook relocates the target's
overwritten prologue into a trampoline, but MMDevApi/AudioSes COM methods
on x86 open with `push ebp; mov ebp,esp; and esp,-8` (dynamic stack
alignment) and read arguments EBP-relative. The relocated copy leaves EBP
wrong, so the original runs with garbage arguments and faults (AV writing
*ppInterface inside CEndpointDevice::Activate+0x3d).

Switch all five WASAPI COM hooks (IMMDevice::Activate, IAudioClient::
Initialize/GetService, IAudioRenderClient::GetBuffer/ReleaseBuffer) from
safetyhook::create_inline to a small VtableHook helper: VirtualProtect the
shared vtable slot, overwrite the function pointer, call the saved original
directly. No code patching, no trampoline, pristine stack regardless of
prologue. One swap covers every instance (a coclass shares one vtable), so
the existing shared-vtable strategy is preserved. Inline hooking stays for
Present/SwapBuffers, whose prologues relocate cleanly.

Reproduced in-process with a new x86 build of the audio render-hook test
(audio_hook_test_x86): it installs the hooks, then drives a fresh
IAudioClient through them and renders -- segfaulted before, passes now.
The x64 audio_hook_test passes regardless of the bug, so the 32-bit build
is the regression guard.

ctest: x64 7/7, x86 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-20 20:39:29 +02:00
parent 435ab9d30f
commit 4985239222
3 changed files with 151 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
hook/src/audio_hook.cpp
View File

@@ -11,8 +11,6 @@
#include <mmdeviceapi.h>
#include <mmreg.h>
#include <safetyhook.hpp>
#include "debug_log.hpp"
#include "hook_registry.hpp"
@@ -32,6 +30,76 @@ constexpr unsigned kIdx_IAudioClient_GetService = 14;
constexpr unsigned kIdx_IAudioRenderClient_GetBuffer = 3;
constexpr unsigned kIdx_IAudioRenderClient_ReleaseBuffer = 4;
// Original COM method signatures (all __stdcall via STDMETHODCALLTYPE). We call
// the originals through the saved vtable pointers, so these types must match the
// real interfaces exactly.
using ActivateFn = HRESULT(STDMETHODCALLTYPE*)(IMMDevice*, REFIID, DWORD, PROPVARIANT*, void**);
using InitializeFn = HRESULT(STDMETHODCALLTYPE*)(IAudioClient*, AUDCLNT_SHAREMODE, DWORD, REFERENCE_TIME,
REFERENCE_TIME, const WAVEFORMATEX*, LPCGUID);
using GetServiceFn = HRESULT(STDMETHODCALLTYPE*)(IAudioClient*, REFIID, void**);
using GetBufferFn = HRESULT(STDMETHODCALLTYPE*)(IAudioRenderClient*, UINT32, BYTE**);
using ReleaseBufferFn = HRESULT(STDMETHODCALLTYPE*)(IAudioRenderClient*, UINT32, DWORD);
// Hooks one COM vtable slot by overwriting its function pointer; the original is
// called through the saved pointer. We use this instead of SafetyHook's inline
// hooks for the WASAPI COM methods because, on x86, MMDevApi/AudioSes prologues
// use dynamic stack alignment (`and esp,-8`) with EBP-relative argument access,
// which SafetyHook's trampoline relocation mishandles: the relocated prologue
// leaves EBP wrong, so the original reads garbage arguments and faults (it froze
// 32-bit FMOD games the instant audio init ran through the hook). Swapping the
// vtable entry leaves the original code untouched, so it runs with a pristine
// stack regardless of prologue shape. Every instance of a COM coclass shares one
// vtable, so a single swap intercepts all of them (the same property the old
// inline approach relied on). See the project's stdcall-x86 note.
class VtableHook
{
public:
bool install(void* com_object, unsigned index, void* detour)
{
if (m_vtable != nullptr)
{
return true; // already installed (shared vtable covers every instance)
}
auto** vtable = *reinterpret_cast<void***>(com_object);
DWORD old_protect = 0;
if (!VirtualProtect(&vtable[index], sizeof(void*), PAGE_READWRITE, &old_protect))
{
return false;
}
m_original = vtable[index];
vtable[index] = detour; // aligned pointer store -> atomic vs. a concurrent caller
VirtualProtect(&vtable[index], sizeof(void*), old_protect, &old_protect);
m_vtable = vtable;
m_index = index;
return true;
}
void remove()
{
if (m_vtable == nullptr)
{
return;
}
DWORD old_protect = 0;
if (VirtualProtect(&m_vtable[m_index], sizeof(void*), PAGE_READWRITE, &old_protect))
{
m_vtable[m_index] = m_original;
VirtualProtect(&m_vtable[m_index], sizeof(void*), old_protect, &old_protect);
}
m_vtable = nullptr;
m_original = nullptr;
m_index = 0;
}
template <typename Fn> Fn original() const { return reinterpret_cast<Fn>(m_original); }
explicit operator bool() const { return m_vtable != nullptr; }
private:
void** m_vtable = nullptr;
unsigned m_index = 0;
void* m_original = nullptr;
};
// The scalar audio format we forward; resolved from the game's WAVEFORMATEX.
struct CapturedFormat
{
@@ -49,11 +117,11 @@ std::atomic<AudioRingHeader*> g_ring{nullptr};
std::mutex g_setup_mutex; // guards installs + the format map + stream registration
safetyhook::InlineHook g_hk_activate;
safetyhook::InlineHook g_hk_initialize;
safetyhook::InlineHook g_hk_getservice;
safetyhook::InlineHook g_hk_getbuffer;
safetyhook::InlineHook g_hk_releasebuffer;
VtableHook g_vh_activate;
VtableHook g_vh_initialize;
VtableHook g_vh_getservice;
VtableHook g_vh_getbuffer;
VtableHook g_vh_releasebuffer;
bool g_audioclient_hooked = false;
// Registry ids for the hook list.
@@ -111,11 +179,6 @@ thread_local IAudioRenderClient* t_gb_client = nullptr;
thread_local BYTE* t_gb_data = nullptr;
thread_local UINT32 t_gb_frames = 0;
void* vtable_method(void* obj, unsigned index)
{
return (*reinterpret_cast<void***>(obj))[index];
}
CapturedFormat capture_format(const WAVEFORMATEX* wfx)
{
CapturedFormat cf;
@@ -151,10 +214,7 @@ void try_register_lazy(IAudioRenderClient* rc);
HRESULT STDMETHODCALLTYPE hk_GetBuffer(IAudioRenderClient* self, UINT32 num_frames, BYTE** data)
{
hook_note_call(g_id_getbuffer);
// stdcall(), NOT call(): these are COM methods (__stdcall). SafetyHook's call()
// uses a __cdecl pointer (the x86 default), which double-cleans the stack on
// 32-bit -> ESP imbalance -> Run-Time Check Failure #0 / crash. Harmless on x64.
const HRESULT hr = g_hk_getbuffer.stdcall<HRESULT>(self, num_frames, data);
const HRESULT hr = g_vh_getbuffer.original<GetBufferFn>()(self, num_frames, data);
if (SUCCEEDED(hr) && data != nullptr)
{
t_gb_client = self;
@@ -207,11 +267,12 @@ HRESULT STDMETHODCALLTYPE hk_ReleaseBuffer(IAudioRenderClient* self, UINT32 num_
{
std::memset(t_gb_data, 0, bytes); // belt-and-suspenders vs a driver ignoring SILENT
g_frames_captured.fetch_add(num_frames, std::memory_order_relaxed);
return g_hk_releasebuffer.stdcall<HRESULT>(self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT);
return g_vh_releasebuffer.original<ReleaseBufferFn>()(
self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT);
}
}
}
return g_hk_releasebuffer.stdcall<HRESULT>(self, num_frames, flags);
return g_vh_releasebuffer.original<ReleaseBufferFn>()(self, num_frames, flags);
}
// Registers a newly created render client: assigns it a debug slot, marks the
@@ -314,8 +375,8 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo
const WAVEFORMATEX* format, LPCGUID session)
{
hook_note_call(g_id_initialize);
const HRESULT hr =
g_hk_initialize.stdcall<HRESULT>(self, mode, flags, buffer_duration, periodicity, format, session);
const HRESULT hr = g_vh_initialize.original<InitializeFn>()(self, mode, flags, buffer_duration,
periodicity, format, session);
logf("hk_Initialize: client=%p mode=%d flags=0x%lX hr=0x%08lX fmt=%s", self, mode,
static_cast<unsigned long>(flags), static_cast<unsigned long>(hr), format ? "yes" : "null");
if (SUCCEEDED(hr) && format != nullptr)
@@ -329,7 +390,7 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo
HRESULT STDMETHODCALLTYPE hk_GetService(IAudioClient* self, REFIID riid, void** ppv)
{
hook_note_call(g_id_getservice);
const HRESULT hr = g_hk_getservice.stdcall<HRESULT>(self, riid, ppv);
const HRESULT hr = g_vh_getservice.original<GetServiceFn>()(self, riid, ppv);
const bool is_render = (riid == __uuidof(IAudioRenderClient));
logf("hk_GetService: client=%p hr=0x%08lX render_client=%d", self, static_cast<unsigned long>(hr),
is_render ? 1 : 0);
@@ -374,20 +435,18 @@ void install_audioclient_hooks(IAudioClient* ac)
{
return; // shared vtable: hook the first IAudioClient we see, covers all
}
g_hk_initialize = safetyhook::create_inline(
vtable_method(ac, kIdx_IAudioClient_Initialize), reinterpret_cast<void*>(&hk_Initialize));
g_hk_getservice = safetyhook::create_inline(
vtable_method(ac, kIdx_IAudioClient_GetService), reinterpret_cast<void*>(&hk_GetService));
g_audioclient_hooked = (g_hk_initialize && g_hk_getservice);
g_vh_initialize.install(ac, kIdx_IAudioClient_Initialize, reinterpret_cast<void*>(&hk_Initialize));
g_vh_getservice.install(ac, kIdx_IAudioClient_GetService, reinterpret_cast<void*>(&hk_GetService));
g_audioclient_hooked = (static_cast<bool>(g_vh_initialize) && static_cast<bool>(g_vh_getservice));
logf("install_audioclient_hooks: ac=%p initialize=%d getservice=%d", ac,
static_cast<bool>(g_hk_initialize) ? 1 : 0, static_cast<bool>(g_hk_getservice) ? 1 : 0);
static_cast<bool>(g_vh_initialize) ? 1 : 0, static_cast<bool>(g_vh_getservice) ? 1 : 0);
}
HRESULT STDMETHODCALLTYPE hk_Activate(IMMDevice* self, REFIID riid, DWORD cls_ctx, PROPVARIANT* params,
void** ppv)
{
hook_note_call(g_id_activate);
const HRESULT hr = g_hk_activate.stdcall<HRESULT>(self, riid, cls_ctx, params, ppv);
const HRESULT hr = g_vh_activate.original<ActivateFn>()(self, riid, cls_ctx, params, ppv);
const bool is_audioclient = (riid == __uuidof(IAudioClient) || riid == __uuidof(IAudioClient2) ||
riid == __uuidof(IAudioClient3));
logf("hk_Activate: device=%p hr=0x%08lX audioclient=%d", self, static_cast<unsigned long>(hr),
@@ -406,7 +465,7 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring)
std::scoped_lock lock(g_setup_mutex);
g_ipc = &ipc;
g_ring.store(ring, std::memory_order_release);
if (g_hk_activate)
if (g_vh_activate)
{
return true; // anchor already installed
}
@@ -434,13 +493,12 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring)
return false;
}
// Build our *own* client + render client with RAW calls first (no hook is
// live yet, so these don't re-enter our detours — which would deadlock on the
// setup mutex we already hold). We attach to a game that's usually already
// playing, so its IAudioClient / IAudioRenderClient predate us and we'll never
// see their Activate/GetService; but every instance of each coclass shares one
// vtable, so hooking *our* objects' vtable slots patches the shared vtables and
// intercepts the game's pre-existing objects too.
// Build our *own* client + render client first (no hook is live yet). We attach
// to a game that's usually already playing, so its IAudioClient /
// IAudioRenderClient predate us and we'll never see their Activate/GetService;
// but every instance of each coclass shares one vtable, so hooking the slots on
// *our* objects' vtables patches the shared vtables and intercepts the game's
// pre-existing objects too.
g_self_client = nullptr;
IAudioRenderClient* self_render = nullptr;
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
@@ -471,42 +529,41 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring)
static_cast<unsigned long>(hr));
}
// Now install every hook. Anchor Activate (idx 3) catches streams created
// after us; the inner hooks catch every render client on the shared vtables.
g_hk_activate = safetyhook::create_inline(
vtable_method(device, kIdx_IMMDevice_Activate), reinterpret_cast<void*>(&hk_Activate));
// Now install every hook by swapping vtable slots. Anchor Activate (idx 3)
// catches streams created after us; the inner hooks catch every render client
// on the shared vtables.
g_vh_activate.install(device, kIdx_IMMDevice_Activate, reinterpret_cast<void*>(&hk_Activate));
if (self_render != nullptr)
{
g_self_render.store(self_render, std::memory_order_release);
g_hk_initialize = safetyhook::create_inline(
vtable_method(g_self_client, kIdx_IAudioClient_Initialize), reinterpret_cast<void*>(&hk_Initialize));
g_hk_getservice = safetyhook::create_inline(
vtable_method(g_self_client, kIdx_IAudioClient_GetService), reinterpret_cast<void*>(&hk_GetService));
g_hk_getbuffer = safetyhook::create_inline(
vtable_method(self_render, kIdx_IAudioRenderClient_GetBuffer), reinterpret_cast<void*>(&hk_GetBuffer));
g_hk_releasebuffer = safetyhook::create_inline(
vtable_method(self_render, kIdx_IAudioRenderClient_ReleaseBuffer),
reinterpret_cast<void*>(&hk_ReleaseBuffer));
g_audioclient_hooked = (g_hk_initialize && g_hk_getservice);
g_vh_initialize.install(g_self_client, kIdx_IAudioClient_Initialize,
reinterpret_cast<void*>(&hk_Initialize));
g_vh_getservice.install(g_self_client, kIdx_IAudioClient_GetService,
reinterpret_cast<void*>(&hk_GetService));
g_vh_getbuffer.install(self_render, kIdx_IAudioRenderClient_GetBuffer,
reinterpret_cast<void*>(&hk_GetBuffer));
g_vh_releasebuffer.install(self_render, kIdx_IAudioRenderClient_ReleaseBuffer,
reinterpret_cast<void*>(&hk_ReleaseBuffer));
g_audioclient_hooked = (static_cast<bool>(g_vh_initialize) && static_cast<bool>(g_vh_getservice));
// Keep g_self_client + self_render alive (held in globals) so the vtables
// stay valid; they're released in remove_audio_hooks.
}
hook_set_installed(g_id_activate, static_cast<bool>(g_hk_activate));
hook_set_installed(g_id_initialize, static_cast<bool>(g_hk_initialize));
hook_set_installed(g_id_getservice, static_cast<bool>(g_hk_getservice));
hook_set_installed(g_id_getbuffer, static_cast<bool>(g_hk_getbuffer));
hook_set_installed(g_id_releasebuffer, static_cast<bool>(g_hk_releasebuffer));
hook_set_installed(g_id_activate, static_cast<bool>(g_vh_activate));
hook_set_installed(g_id_initialize, static_cast<bool>(g_vh_initialize));
hook_set_installed(g_id_getservice, static_cast<bool>(g_vh_getservice));
hook_set_installed(g_id_getbuffer, static_cast<bool>(g_vh_getbuffer));
hook_set_installed(g_id_releasebuffer, static_cast<bool>(g_vh_releasebuffer));
logf("install_audio_hooks: activate=%d init=%d getsvc=%d getbuf=%d relbuf=%d (device=%p)",
static_cast<bool>(g_hk_activate) ? 1 : 0, static_cast<bool>(g_hk_initialize) ? 1 : 0,
static_cast<bool>(g_hk_getservice) ? 1 : 0, static_cast<bool>(g_hk_getbuffer) ? 1 : 0,
static_cast<bool>(g_hk_releasebuffer) ? 1 : 0, device);
static_cast<bool>(g_vh_activate) ? 1 : 0, static_cast<bool>(g_vh_initialize) ? 1 : 0,
static_cast<bool>(g_vh_getservice) ? 1 : 0, static_cast<bool>(g_vh_getbuffer) ? 1 : 0,
static_cast<bool>(g_vh_releasebuffer) ? 1 : 0, device);
device->Release(); // vtable lives in the (still-loaded) audio COM module
enumerator->Release();
return static_cast<bool>(g_hk_activate);
return static_cast<bool>(g_vh_activate);
}
void republish_audio_format()
@@ -544,11 +601,11 @@ void set_audio_ring(AudioRingHeader* ring)
void remove_audio_hooks()
{
std::scoped_lock lock(g_setup_mutex);
g_hk_releasebuffer = {};
g_hk_getbuffer = {};
g_hk_getservice = {};
g_hk_initialize = {};
g_hk_activate = {};
g_vh_releasebuffer.remove();
g_vh_getbuffer.remove();
g_vh_getservice.remove();
g_vh_initialize.remove();
g_vh_activate.remove();
g_audioclient_hooked = false;
hook_set_installed(g_id_activate, false);
hook_set_installed(g_id_initialize, false);