Fix 32-bit FMOD audio crash: hook WASAPI COM methods via vtable swap

The stdcall() fix stopped the Present-hook crash but 32-bit games (Slaps
and Beans, FMOD) still crashed the instant audio init ran through the
hook. Root cause: SafetyHook's inline hook relocates the target's
overwritten prologue into a trampoline, but MMDevApi/AudioSes COM methods
on x86 open with `push ebp; mov ebp,esp; and esp,-8` (dynamic stack
alignment) and read arguments EBP-relative. The relocated copy leaves EBP
wrong, so the original runs with garbage arguments and faults (AV writing
*ppInterface inside CEndpointDevice::Activate+0x3d).

Switch all five WASAPI COM hooks (IMMDevice::Activate, IAudioClient::
Initialize/GetService, IAudioRenderClient::GetBuffer/ReleaseBuffer) from
safetyhook::create_inline to a small VtableHook helper: VirtualProtect the
shared vtable slot, overwrite the function pointer, call the saved original
directly. No code patching, no trampoline, pristine stack regardless of
prologue. One swap covers every instance (a coclass shares one vtable), so
the existing shared-vtable strategy is preserved. Inline hooking stays for
Present/SwapBuffers, whose prologues relocate cleanly.

Reproduced in-process with a new x86 build of the audio render-hook test
(audio_hook_test_x86): it installs the hooks, then drives a fresh
IAudioClient through them and renders -- segfaulted before, passes now.
The x64 audio_hook_test passes regardless of the bug, so the 32-bit build
is the regression guard.

ctest: x64 7/7, x86 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

CMakeLists.txt

@@ -69,6 +69,23 @@ if(COOP_X86_HELPER_BUILD)
 	target_include_directories(present_hook_test_x86 PRIVATE hook/src)
 	target_link_libraries(present_hook_test_x86 PRIVATE coop_common safetyhook::safetyhook d3d11 dxgi)
 	add_test(NAME present_hook_test_x86 COMMAND present_hook_test_x86)
+
+	# x86 build of the audio render-hook test. It installs the WASAPI hooks and then
+	# creates a *fresh* IAudioClient/IAudioRenderClient that calls Initialize /
+	# GetService / GetBuffer / ReleaseBuffer back through the SafetyHook trampolines
+	# -- the exact ordering that crashed 32-bit Slaps & Beans (FMOD) when audio hooks
+	# were live as the game initialized its output. The x64 audio_hook_test passes,
+	# so this 32-bit build is the regression guard for any x86-only fault in the
+	# audio setup detours / trampoline relocation of AudioSes.dll prologues.
+	add_executable(audio_hook_test_x86
+		tests/audio_hook_test.cpp
+		hook/src/audio_hook.cpp
+		hook/src/debug_log.cpp
+		hook/src/hook_registry.cpp)
+	target_include_directories(audio_hook_test_x86 PRIVATE hook/src)
+	target_compile_definitions(audio_hook_test_x86 PRIVATE NTDDI_VERSION=0x0A00000B)
+	target_link_libraries(audio_hook_test_x86 PRIVATE coop_common safetyhook::safetyhook ole32 mmdevapi)
+	add_test(NAME audio_hook_test_x86 COMMAND audio_hook_test_x86)
 	return()
 endif()
 

README.md

@@ -312,6 +312,19 @@ Non-obvious things that cost time and constrain the design:
   trampolines with the matching convention — `stdcall()` for these — which is a
   no-op on x64. The XInput/focus hooks dodged it only because they never call the
   trampoline (they return synthesized data).
+- **Hook COM methods by swapping the vtable entry, not by inline-patching the
+  function — on x86.** Inline hooking relocates the target's overwritten prologue
+  into a trampoline. Some x86 prologues defeat that: MMDevApi/AudioSes methods open
+  with `push ebp; mov ebp,esp; and esp,-8` (dynamic stack alignment) and read their
+  arguments **EBP-relative**. SafetyHook's relocated copy leaves EBP wrong, so the
+  original ran with garbage arguments and faulted — this crashed 32-bit FMOD games
+  (Slaps and Beans) the instant audio init flowed through the hook, *after* the
+  `stdcall()` fix above. The robust fix is vtable-entry hooking: `VirtualProtect` the
+  shared vtable slot, overwrite the function pointer, call the saved original
+  directly. No code patching, no trampoline, pristine stack regardless of prologue.
+  The audio hooks use this; inline hooking is fine for `Present`/`SwapBuffers`, whose
+  prologues relocate cleanly. (One swap covers every instance — a coclass shares one
+  vtable.) Guarded by `audio_hook_test_x86`.
 - **Steam Input init suppresses XInput.** Initializing the Steam Input API turns on
   Steam's in-process XInput interception, which hides controllers from
   `XInputGetState` unless they're bound to the running appid's action set —

hook/src/audio_hook.cpp

@@ -11,8 +11,6 @@
 #include <mmdeviceapi.h>
 #include <mmreg.h>
 
-#include <safetyhook.hpp>
-
 #include "debug_log.hpp"
 #include "hook_registry.hpp"
 
@@ -32,6 +30,76 @@ constexpr unsigned kIdx_IAudioClient_GetService = 14;
 constexpr unsigned kIdx_IAudioRenderClient_GetBuffer = 3;
 constexpr unsigned kIdx_IAudioRenderClient_ReleaseBuffer = 4;
 
+// Original COM method signatures (all __stdcall via STDMETHODCALLTYPE). We call
+// the originals through the saved vtable pointers, so these types must match the
+// real interfaces exactly.
+using ActivateFn = HRESULT(STDMETHODCALLTYPE*)(IMMDevice*, REFIID, DWORD, PROPVARIANT*, void**);
+using InitializeFn = HRESULT(STDMETHODCALLTYPE*)(IAudioClient*, AUDCLNT_SHAREMODE, DWORD, REFERENCE_TIME,
+                                                 REFERENCE_TIME, const WAVEFORMATEX*, LPCGUID);
+using GetServiceFn = HRESULT(STDMETHODCALLTYPE*)(IAudioClient*, REFIID, void**);
+using GetBufferFn = HRESULT(STDMETHODCALLTYPE*)(IAudioRenderClient*, UINT32, BYTE**);
+using ReleaseBufferFn = HRESULT(STDMETHODCALLTYPE*)(IAudioRenderClient*, UINT32, DWORD);
+
+// Hooks one COM vtable slot by overwriting its function pointer; the original is
+// called through the saved pointer. We use this instead of SafetyHook's inline
+// hooks for the WASAPI COM methods because, on x86, MMDevApi/AudioSes prologues
+// use dynamic stack alignment (`and esp,-8`) with EBP-relative argument access,
+// which SafetyHook's trampoline relocation mishandles: the relocated prologue
+// leaves EBP wrong, so the original reads garbage arguments and faults (it froze
+// 32-bit FMOD games the instant audio init ran through the hook). Swapping the
+// vtable entry leaves the original code untouched, so it runs with a pristine
+// stack regardless of prologue shape. Every instance of a COM coclass shares one
+// vtable, so a single swap intercepts all of them (the same property the old
+// inline approach relied on). See the project's stdcall-x86 note.
+class VtableHook
+{
+public:
+	bool install(void* com_object, unsigned index, void* detour)
+	{
+		if (m_vtable != nullptr)
+		{
+			return true; // already installed (shared vtable covers every instance)
+		}
+		auto** vtable = *reinterpret_cast<void***>(com_object);
+		DWORD old_protect = 0;
+		if (!VirtualProtect(&vtable[index], sizeof(void*), PAGE_READWRITE, &old_protect))
+		{
+			return false;
+		}
+		m_original = vtable[index];
+		vtable[index] = detour; // aligned pointer store -> atomic vs. a concurrent caller
+		VirtualProtect(&vtable[index], sizeof(void*), old_protect, &old_protect);
+		m_vtable = vtable;
+		m_index = index;
+		return true;
+	}
+
+	void remove()
+	{
+		if (m_vtable == nullptr)
+		{
+			return;
+		}
+		DWORD old_protect = 0;
+		if (VirtualProtect(&m_vtable[m_index], sizeof(void*), PAGE_READWRITE, &old_protect))
+		{
+			m_vtable[m_index] = m_original;
+			VirtualProtect(&m_vtable[m_index], sizeof(void*), old_protect, &old_protect);
+		}
+		m_vtable = nullptr;
+		m_original = nullptr;
+		m_index = 0;
+	}
+
+	template <typename Fn> Fn original() const { return reinterpret_cast<Fn>(m_original); }
+	explicit operator bool() const { return m_vtable != nullptr; }
+
+private:
+	void** m_vtable = nullptr;
+	unsigned m_index = 0;
+	void* m_original = nullptr;
+};
+
 // The scalar audio format we forward; resolved from the game's WAVEFORMATEX.
 struct CapturedFormat
 {
@@ -49,11 +117,11 @@ std::atomic<AudioRingHeader*> g_ring{nullptr};
 
 std::mutex g_setup_mutex; // guards installs + the format map + stream registration
 
-safetyhook::InlineHook g_hk_activate;
+VtableHook g_vh_activate;
-safetyhook::InlineHook g_hk_initialize;
+VtableHook g_vh_initialize;
-safetyhook::InlineHook g_hk_getservice;
+VtableHook g_vh_getservice;
-safetyhook::InlineHook g_hk_getbuffer;
+VtableHook g_vh_getbuffer;
-safetyhook::InlineHook g_hk_releasebuffer;
+VtableHook g_vh_releasebuffer;
 bool g_audioclient_hooked = false;
 
 // Registry ids for the hook list.
@@ -111,11 +179,6 @@ thread_local IAudioRenderClient* t_gb_client = nullptr;
 thread_local BYTE* t_gb_data = nullptr;
 thread_local UINT32 t_gb_frames = 0;
 
-void* vtable_method(void* obj, unsigned index)
-{
-	return (*reinterpret_cast<void***>(obj))[index];
-}
-
 CapturedFormat capture_format(const WAVEFORMATEX* wfx)
 {
 	CapturedFormat cf;
@@ -151,10 +214,7 @@ void try_register_lazy(IAudioRenderClient* rc);
 HRESULT STDMETHODCALLTYPE hk_GetBuffer(IAudioRenderClient* self, UINT32 num_frames, BYTE** data)
 {
 	hook_note_call(g_id_getbuffer);
-	// stdcall(), NOT call(): these are COM methods (__stdcall). SafetyHook's call()
+	const HRESULT hr = g_vh_getbuffer.original<GetBufferFn>()(self, num_frames, data);
-	// uses a __cdecl pointer (the x86 default), which double-cleans the stack on
-	// 32-bit -> ESP imbalance -> Run-Time Check Failure #0 / crash. Harmless on x64.
-	const HRESULT hr = g_hk_getbuffer.stdcall<HRESULT>(self, num_frames, data);
 	if (SUCCEEDED(hr) && data != nullptr)
 	{
 		t_gb_client = self;
@@ -207,11 +267,12 @@ HRESULT STDMETHODCALLTYPE hk_ReleaseBuffer(IAudioRenderClient* self, UINT32 num_
 			{
 				std::memset(t_gb_data, 0, bytes); // belt-and-suspenders vs a driver ignoring SILENT
 				g_frames_captured.fetch_add(num_frames, std::memory_order_relaxed);
-				return g_hk_releasebuffer.stdcall<HRESULT>(self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT);
+				return g_vh_releasebuffer.original<ReleaseBufferFn>()(
+				    self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT);
 			}
 		}
 	}
-	return g_hk_releasebuffer.stdcall<HRESULT>(self, num_frames, flags);
+	return g_vh_releasebuffer.original<ReleaseBufferFn>()(self, num_frames, flags);
 }
 
 // Registers a newly created render client: assigns it a debug slot, marks the
@@ -314,8 +375,8 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo
                                         const WAVEFORMATEX* format, LPCGUID session)
 {
 	hook_note_call(g_id_initialize);
-	const HRESULT hr =
+	const HRESULT hr = g_vh_initialize.original<InitializeFn>()(self, mode, flags, buffer_duration,
-	    g_hk_initialize.stdcall<HRESULT>(self, mode, flags, buffer_duration, periodicity, format, session);
+	                                                            periodicity, format, session);
 	logf("hk_Initialize: client=%p mode=%d flags=0x%lX hr=0x%08lX fmt=%s", self, mode,
 	     static_cast<unsigned long>(flags), static_cast<unsigned long>(hr), format ? "yes" : "null");
 	if (SUCCEEDED(hr) && format != nullptr)
@@ -329,7 +390,7 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo
 HRESULT STDMETHODCALLTYPE hk_GetService(IAudioClient* self, REFIID riid, void** ppv)
 {
 	hook_note_call(g_id_getservice);
-	const HRESULT hr = g_hk_getservice.stdcall<HRESULT>(self, riid, ppv);
+	const HRESULT hr = g_vh_getservice.original<GetServiceFn>()(self, riid, ppv);
 	const bool is_render = (riid == __uuidof(IAudioRenderClient));
 	logf("hk_GetService: client=%p hr=0x%08lX render_client=%d", self, static_cast<unsigned long>(hr),
 	     is_render ? 1 : 0);
@@ -374,20 +435,18 @@ void install_audioclient_hooks(IAudioClient* ac)
 	{
 		return; // shared vtable: hook the first IAudioClient we see, covers all
 	}
-	g_hk_initialize = safetyhook::create_inline(
+	g_vh_initialize.install(ac, kIdx_IAudioClient_Initialize, reinterpret_cast<void*>(&hk_Initialize));
-	    vtable_method(ac, kIdx_IAudioClient_Initialize), reinterpret_cast<void*>(&hk_Initialize));
+	g_vh_getservice.install(ac, kIdx_IAudioClient_GetService, reinterpret_cast<void*>(&hk_GetService));
-	g_hk_getservice = safetyhook::create_inline(
+	g_audioclient_hooked = (static_cast<bool>(g_vh_initialize) && static_cast<bool>(g_vh_getservice));
-	    vtable_method(ac, kIdx_IAudioClient_GetService), reinterpret_cast<void*>(&hk_GetService));
-	g_audioclient_hooked = (g_hk_initialize && g_hk_getservice);
 	logf("install_audioclient_hooks: ac=%p initialize=%d getservice=%d", ac,
-	     static_cast<bool>(g_hk_initialize) ? 1 : 0, static_cast<bool>(g_hk_getservice) ? 1 : 0);
+	     static_cast<bool>(g_vh_initialize) ? 1 : 0, static_cast<bool>(g_vh_getservice) ? 1 : 0);
 }
 
 HRESULT STDMETHODCALLTYPE hk_Activate(IMMDevice* self, REFIID riid, DWORD cls_ctx, PROPVARIANT* params,
                                       void** ppv)
 {
 	hook_note_call(g_id_activate);
-	const HRESULT hr = g_hk_activate.stdcall<HRESULT>(self, riid, cls_ctx, params, ppv);
+	const HRESULT hr = g_vh_activate.original<ActivateFn>()(self, riid, cls_ctx, params, ppv);
 	const bool is_audioclient = (riid == __uuidof(IAudioClient) || riid == __uuidof(IAudioClient2) ||
 	                             riid == __uuidof(IAudioClient3));
 	logf("hk_Activate: device=%p hr=0x%08lX audioclient=%d", self, static_cast<unsigned long>(hr),
@@ -406,7 +465,7 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring)
 	std::scoped_lock lock(g_setup_mutex);
 	g_ipc = &ipc;
 	g_ring.store(ring, std::memory_order_release);
-	if (g_hk_activate)
+	if (g_vh_activate)
 	{
 		return true; // anchor already installed
 	}
@@ -434,13 +493,12 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring)
 		return false;
 	}
 
-	// Build our *own* client + render client with RAW calls first (no hook is
+	// Build our *own* client + render client first (no hook is live yet). We attach
-	// live yet, so these don't re-enter our detours — which would deadlock on the
+	// to a game that's usually already playing, so its IAudioClient /
-	// setup mutex we already hold). We attach to a game that's usually already
+	// IAudioRenderClient predate us and we'll never see their Activate/GetService;
-	// playing, so its IAudioClient / IAudioRenderClient predate us and we'll never
+	// but every instance of each coclass shares one vtable, so hooking the slots on
-	// see their Activate/GetService; but every instance of each coclass shares one
+	// *our* objects' vtables patches the shared vtables and intercepts the game's
-	// vtable, so hooking *our* objects' vtable slots patches the shared vtables and
+	// pre-existing objects too.
-	// intercepts the game's pre-existing objects too.
 	g_self_client = nullptr;
 	IAudioRenderClient* self_render = nullptr;
 	hr = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr,
@@ -471,42 +529,41 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring)
 		     static_cast<unsigned long>(hr));
 	}
 
-	// Now install every hook. Anchor Activate (idx 3) catches streams created
+	// Now install every hook by swapping vtable slots. Anchor Activate (idx 3)
-	// after us; the inner hooks catch every render client on the shared vtables.
+	// catches streams created after us; the inner hooks catch every render client
-	g_hk_activate = safetyhook::create_inline(
+	// on the shared vtables.
-	    vtable_method(device, kIdx_IMMDevice_Activate), reinterpret_cast<void*>(&hk_Activate));
+	g_vh_activate.install(device, kIdx_IMMDevice_Activate, reinterpret_cast<void*>(&hk_Activate));
 
 	if (self_render != nullptr)
 	{
 		g_self_render.store(self_render, std::memory_order_release);
-		g_hk_initialize = safetyhook::create_inline(
+		g_vh_initialize.install(g_self_client, kIdx_IAudioClient_Initialize,
-		    vtable_method(g_self_client, kIdx_IAudioClient_Initialize), reinterpret_cast<void*>(&hk_Initialize));
+		                        reinterpret_cast<void*>(&hk_Initialize));
-		g_hk_getservice = safetyhook::create_inline(
+		g_vh_getservice.install(g_self_client, kIdx_IAudioClient_GetService,
-		    vtable_method(g_self_client, kIdx_IAudioClient_GetService), reinterpret_cast<void*>(&hk_GetService));
+		                        reinterpret_cast<void*>(&hk_GetService));
-		g_hk_getbuffer = safetyhook::create_inline(
+		g_vh_getbuffer.install(self_render, kIdx_IAudioRenderClient_GetBuffer,
-		    vtable_method(self_render, kIdx_IAudioRenderClient_GetBuffer), reinterpret_cast<void*>(&hk_GetBuffer));
+		                       reinterpret_cast<void*>(&hk_GetBuffer));
-		g_hk_releasebuffer = safetyhook::create_inline(
+		g_vh_releasebuffer.install(self_render, kIdx_IAudioRenderClient_ReleaseBuffer,
-		    vtable_method(self_render, kIdx_IAudioRenderClient_ReleaseBuffer),
+		                           reinterpret_cast<void*>(&hk_ReleaseBuffer));
-		    reinterpret_cast<void*>(&hk_ReleaseBuffer));
+		g_audioclient_hooked = (static_cast<bool>(g_vh_initialize) && static_cast<bool>(g_vh_getservice));
-		g_audioclient_hooked = (g_hk_initialize && g_hk_getservice);
 		// Keep g_self_client + self_render alive (held in globals) so the vtables
 		// stay valid; they're released in remove_audio_hooks.
 	}
 
-	hook_set_installed(g_id_activate, static_cast<bool>(g_hk_activate));
+	hook_set_installed(g_id_activate, static_cast<bool>(g_vh_activate));
-	hook_set_installed(g_id_initialize, static_cast<bool>(g_hk_initialize));
+	hook_set_installed(g_id_initialize, static_cast<bool>(g_vh_initialize));
-	hook_set_installed(g_id_getservice, static_cast<bool>(g_hk_getservice));
+	hook_set_installed(g_id_getservice, static_cast<bool>(g_vh_getservice));
-	hook_set_installed(g_id_getbuffer, static_cast<bool>(g_hk_getbuffer));
+	hook_set_installed(g_id_getbuffer, static_cast<bool>(g_vh_getbuffer));
-	hook_set_installed(g_id_releasebuffer, static_cast<bool>(g_hk_releasebuffer));
+	hook_set_installed(g_id_releasebuffer, static_cast<bool>(g_vh_releasebuffer));
 
 	logf("install_audio_hooks: activate=%d init=%d getsvc=%d getbuf=%d relbuf=%d (device=%p)",
-	     static_cast<bool>(g_hk_activate) ? 1 : 0, static_cast<bool>(g_hk_initialize) ? 1 : 0,
+	     static_cast<bool>(g_vh_activate) ? 1 : 0, static_cast<bool>(g_vh_initialize) ? 1 : 0,
-	     static_cast<bool>(g_hk_getservice) ? 1 : 0, static_cast<bool>(g_hk_getbuffer) ? 1 : 0,
+	     static_cast<bool>(g_vh_getservice) ? 1 : 0, static_cast<bool>(g_vh_getbuffer) ? 1 : 0,
-	     static_cast<bool>(g_hk_releasebuffer) ? 1 : 0, device);
+	     static_cast<bool>(g_vh_releasebuffer) ? 1 : 0, device);
 
 	device->Release(); // vtable lives in the (still-loaded) audio COM module
 	enumerator->Release();
-	return static_cast<bool>(g_hk_activate);
+	return static_cast<bool>(g_vh_activate);
 }
 
 void republish_audio_format()
@@ -544,11 +601,11 @@ void set_audio_ring(AudioRingHeader* ring)
 void remove_audio_hooks()
 {
 	std::scoped_lock lock(g_setup_mutex);
-	g_hk_releasebuffer = {};
+	g_vh_releasebuffer.remove();
-	g_hk_getbuffer = {};
+	g_vh_getbuffer.remove();
-	g_hk_getservice = {};
+	g_vh_getservice.remove();
-	g_hk_initialize = {};
+	g_vh_initialize.remove();
-	g_hk_activate = {};
+	g_vh_activate.remove();
 	g_audioclient_hooked = false;
 	hook_set_installed(g_id_activate, false);
 	hook_set_installed(g_id_initialize, false);