diff --git a/CMakeLists.txt b/CMakeLists.txt index b87e9d9..8311fd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,23 @@ if(COOP_X86_HELPER_BUILD) target_include_directories(present_hook_test_x86 PRIVATE hook/src) target_link_libraries(present_hook_test_x86 PRIVATE coop_common safetyhook::safetyhook d3d11 dxgi) add_test(NAME present_hook_test_x86 COMMAND present_hook_test_x86) + + # x86 build of the audio render-hook test. It installs the WASAPI hooks and then + # creates a *fresh* IAudioClient/IAudioRenderClient that calls Initialize / + # GetService / GetBuffer / ReleaseBuffer back through the SafetyHook trampolines + # -- the exact ordering that crashed 32-bit Slaps & Beans (FMOD) when audio hooks + # were live as the game initialized its output. The x64 audio_hook_test passes, + # so this 32-bit build is the regression guard for any x86-only fault in the + # audio setup detours / trampoline relocation of AudioSes.dll prologues. + add_executable(audio_hook_test_x86 + tests/audio_hook_test.cpp + hook/src/audio_hook.cpp + hook/src/debug_log.cpp + hook/src/hook_registry.cpp) + target_include_directories(audio_hook_test_x86 PRIVATE hook/src) + target_compile_definitions(audio_hook_test_x86 PRIVATE NTDDI_VERSION=0x0A00000B) + target_link_libraries(audio_hook_test_x86 PRIVATE coop_common safetyhook::safetyhook ole32 mmdevapi) + add_test(NAME audio_hook_test_x86 COMMAND audio_hook_test_x86) return() endif() diff --git a/README.md b/README.md index 53ad33b..76323bf 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,19 @@ Non-obvious things that cost time and constrain the design: trampolines with the matching convention — `stdcall()` for these — which is a no-op on x64. The XInput/focus hooks dodged it only because they never call the trampoline (they return synthesized data). +- **Hook COM methods by swapping the vtable entry, not by inline-patching the + function — on x86.** Inline hooking relocates the target's overwritten prologue + into a trampoline. Some x86 prologues defeat that: MMDevApi/AudioSes methods open + with `push ebp; mov ebp,esp; and esp,-8` (dynamic stack alignment) and read their + arguments **EBP-relative**. SafetyHook's relocated copy leaves EBP wrong, so the + original ran with garbage arguments and faulted — this crashed 32-bit FMOD games + (Slaps and Beans) the instant audio init flowed through the hook, *after* the + `stdcall()` fix above. The robust fix is vtable-entry hooking: `VirtualProtect` the + shared vtable slot, overwrite the function pointer, call the saved original + directly. No code patching, no trampoline, pristine stack regardless of prologue. + The audio hooks use this; inline hooking is fine for `Present`/`SwapBuffers`, whose + prologues relocate cleanly. (One swap covers every instance — a coclass shares one + vtable.) Guarded by `audio_hook_test_x86`. - **Steam Input init suppresses XInput.** Initializing the Steam Input API turns on Steam's in-process XInput interception, which hides controllers from `XInputGetState` unless they're bound to the running appid's action set — diff --git a/hook/src/audio_hook.cpp b/hook/src/audio_hook.cpp index e542670..640a818 100644 --- a/hook/src/audio_hook.cpp +++ b/hook/src/audio_hook.cpp @@ -11,8 +11,6 @@ #include #include -#include - #include "debug_log.hpp" #include "hook_registry.hpp" @@ -32,6 +30,76 @@ constexpr unsigned kIdx_IAudioClient_GetService = 14; constexpr unsigned kIdx_IAudioRenderClient_GetBuffer = 3; constexpr unsigned kIdx_IAudioRenderClient_ReleaseBuffer = 4; +// Original COM method signatures (all __stdcall via STDMETHODCALLTYPE). We call +// the originals through the saved vtable pointers, so these types must match the +// real interfaces exactly. +using ActivateFn = HRESULT(STDMETHODCALLTYPE*)(IMMDevice*, REFIID, DWORD, PROPVARIANT*, void**); +using InitializeFn = HRESULT(STDMETHODCALLTYPE*)(IAudioClient*, AUDCLNT_SHAREMODE, DWORD, REFERENCE_TIME, + REFERENCE_TIME, const WAVEFORMATEX*, LPCGUID); +using GetServiceFn = HRESULT(STDMETHODCALLTYPE*)(IAudioClient*, REFIID, void**); +using GetBufferFn = HRESULT(STDMETHODCALLTYPE*)(IAudioRenderClient*, UINT32, BYTE**); +using ReleaseBufferFn = HRESULT(STDMETHODCALLTYPE*)(IAudioRenderClient*, UINT32, DWORD); + +// Hooks one COM vtable slot by overwriting its function pointer; the original is +// called through the saved pointer. We use this instead of SafetyHook's inline +// hooks for the WASAPI COM methods because, on x86, MMDevApi/AudioSes prologues +// use dynamic stack alignment (`and esp,-8`) with EBP-relative argument access, +// which SafetyHook's trampoline relocation mishandles: the relocated prologue +// leaves EBP wrong, so the original reads garbage arguments and faults (it froze +// 32-bit FMOD games the instant audio init ran through the hook). Swapping the +// vtable entry leaves the original code untouched, so it runs with a pristine +// stack regardless of prologue shape. Every instance of a COM coclass shares one +// vtable, so a single swap intercepts all of them (the same property the old +// inline approach relied on). See the project's stdcall-x86 note. +class VtableHook +{ +public: + bool install(void* com_object, unsigned index, void* detour) + { + if (m_vtable != nullptr) + { + return true; // already installed (shared vtable covers every instance) + } + auto** vtable = *reinterpret_cast(com_object); + DWORD old_protect = 0; + if (!VirtualProtect(&vtable[index], sizeof(void*), PAGE_READWRITE, &old_protect)) + { + return false; + } + m_original = vtable[index]; + vtable[index] = detour; // aligned pointer store -> atomic vs. a concurrent caller + VirtualProtect(&vtable[index], sizeof(void*), old_protect, &old_protect); + m_vtable = vtable; + m_index = index; + return true; + } + + void remove() + { + if (m_vtable == nullptr) + { + return; + } + DWORD old_protect = 0; + if (VirtualProtect(&m_vtable[m_index], sizeof(void*), PAGE_READWRITE, &old_protect)) + { + m_vtable[m_index] = m_original; + VirtualProtect(&m_vtable[m_index], sizeof(void*), old_protect, &old_protect); + } + m_vtable = nullptr; + m_original = nullptr; + m_index = 0; + } + + template Fn original() const { return reinterpret_cast(m_original); } + explicit operator bool() const { return m_vtable != nullptr; } + +private: + void** m_vtable = nullptr; + unsigned m_index = 0; + void* m_original = nullptr; +}; + // The scalar audio format we forward; resolved from the game's WAVEFORMATEX. struct CapturedFormat { @@ -49,11 +117,11 @@ std::atomic g_ring{nullptr}; std::mutex g_setup_mutex; // guards installs + the format map + stream registration -safetyhook::InlineHook g_hk_activate; -safetyhook::InlineHook g_hk_initialize; -safetyhook::InlineHook g_hk_getservice; -safetyhook::InlineHook g_hk_getbuffer; -safetyhook::InlineHook g_hk_releasebuffer; +VtableHook g_vh_activate; +VtableHook g_vh_initialize; +VtableHook g_vh_getservice; +VtableHook g_vh_getbuffer; +VtableHook g_vh_releasebuffer; bool g_audioclient_hooked = false; // Registry ids for the hook list. @@ -111,11 +179,6 @@ thread_local IAudioRenderClient* t_gb_client = nullptr; thread_local BYTE* t_gb_data = nullptr; thread_local UINT32 t_gb_frames = 0; -void* vtable_method(void* obj, unsigned index) -{ - return (*reinterpret_cast(obj))[index]; -} - CapturedFormat capture_format(const WAVEFORMATEX* wfx) { CapturedFormat cf; @@ -151,10 +214,7 @@ void try_register_lazy(IAudioRenderClient* rc); HRESULT STDMETHODCALLTYPE hk_GetBuffer(IAudioRenderClient* self, UINT32 num_frames, BYTE** data) { hook_note_call(g_id_getbuffer); - // stdcall(), NOT call(): these are COM methods (__stdcall). SafetyHook's call() - // uses a __cdecl pointer (the x86 default), which double-cleans the stack on - // 32-bit -> ESP imbalance -> Run-Time Check Failure #0 / crash. Harmless on x64. - const HRESULT hr = g_hk_getbuffer.stdcall(self, num_frames, data); + const HRESULT hr = g_vh_getbuffer.original()(self, num_frames, data); if (SUCCEEDED(hr) && data != nullptr) { t_gb_client = self; @@ -207,11 +267,12 @@ HRESULT STDMETHODCALLTYPE hk_ReleaseBuffer(IAudioRenderClient* self, UINT32 num_ { std::memset(t_gb_data, 0, bytes); // belt-and-suspenders vs a driver ignoring SILENT g_frames_captured.fetch_add(num_frames, std::memory_order_relaxed); - return g_hk_releasebuffer.stdcall(self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT); + return g_vh_releasebuffer.original()( + self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT); } } } - return g_hk_releasebuffer.stdcall(self, num_frames, flags); + return g_vh_releasebuffer.original()(self, num_frames, flags); } // Registers a newly created render client: assigns it a debug slot, marks the @@ -314,8 +375,8 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo const WAVEFORMATEX* format, LPCGUID session) { hook_note_call(g_id_initialize); - const HRESULT hr = - g_hk_initialize.stdcall(self, mode, flags, buffer_duration, periodicity, format, session); + const HRESULT hr = g_vh_initialize.original()(self, mode, flags, buffer_duration, + periodicity, format, session); logf("hk_Initialize: client=%p mode=%d flags=0x%lX hr=0x%08lX fmt=%s", self, mode, static_cast(flags), static_cast(hr), format ? "yes" : "null"); if (SUCCEEDED(hr) && format != nullptr) @@ -329,7 +390,7 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo HRESULT STDMETHODCALLTYPE hk_GetService(IAudioClient* self, REFIID riid, void** ppv) { hook_note_call(g_id_getservice); - const HRESULT hr = g_hk_getservice.stdcall(self, riid, ppv); + const HRESULT hr = g_vh_getservice.original()(self, riid, ppv); const bool is_render = (riid == __uuidof(IAudioRenderClient)); logf("hk_GetService: client=%p hr=0x%08lX render_client=%d", self, static_cast(hr), is_render ? 1 : 0); @@ -374,20 +435,18 @@ void install_audioclient_hooks(IAudioClient* ac) { return; // shared vtable: hook the first IAudioClient we see, covers all } - g_hk_initialize = safetyhook::create_inline( - vtable_method(ac, kIdx_IAudioClient_Initialize), reinterpret_cast(&hk_Initialize)); - g_hk_getservice = safetyhook::create_inline( - vtable_method(ac, kIdx_IAudioClient_GetService), reinterpret_cast(&hk_GetService)); - g_audioclient_hooked = (g_hk_initialize && g_hk_getservice); + g_vh_initialize.install(ac, kIdx_IAudioClient_Initialize, reinterpret_cast(&hk_Initialize)); + g_vh_getservice.install(ac, kIdx_IAudioClient_GetService, reinterpret_cast(&hk_GetService)); + g_audioclient_hooked = (static_cast(g_vh_initialize) && static_cast(g_vh_getservice)); logf("install_audioclient_hooks: ac=%p initialize=%d getservice=%d", ac, - static_cast(g_hk_initialize) ? 1 : 0, static_cast(g_hk_getservice) ? 1 : 0); + static_cast(g_vh_initialize) ? 1 : 0, static_cast(g_vh_getservice) ? 1 : 0); } HRESULT STDMETHODCALLTYPE hk_Activate(IMMDevice* self, REFIID riid, DWORD cls_ctx, PROPVARIANT* params, void** ppv) { hook_note_call(g_id_activate); - const HRESULT hr = g_hk_activate.stdcall(self, riid, cls_ctx, params, ppv); + const HRESULT hr = g_vh_activate.original()(self, riid, cls_ctx, params, ppv); const bool is_audioclient = (riid == __uuidof(IAudioClient) || riid == __uuidof(IAudioClient2) || riid == __uuidof(IAudioClient3)); logf("hk_Activate: device=%p hr=0x%08lX audioclient=%d", self, static_cast(hr), @@ -406,7 +465,7 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring) std::scoped_lock lock(g_setup_mutex); g_ipc = &ipc; g_ring.store(ring, std::memory_order_release); - if (g_hk_activate) + if (g_vh_activate) { return true; // anchor already installed } @@ -434,13 +493,12 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring) return false; } - // Build our *own* client + render client with RAW calls first (no hook is - // live yet, so these don't re-enter our detours — which would deadlock on the - // setup mutex we already hold). We attach to a game that's usually already - // playing, so its IAudioClient / IAudioRenderClient predate us and we'll never - // see their Activate/GetService; but every instance of each coclass shares one - // vtable, so hooking *our* objects' vtable slots patches the shared vtables and - // intercepts the game's pre-existing objects too. + // Build our *own* client + render client first (no hook is live yet). We attach + // to a game that's usually already playing, so its IAudioClient / + // IAudioRenderClient predate us and we'll never see their Activate/GetService; + // but every instance of each coclass shares one vtable, so hooking the slots on + // *our* objects' vtables patches the shared vtables and intercepts the game's + // pre-existing objects too. g_self_client = nullptr; IAudioRenderClient* self_render = nullptr; hr = device->Activate(__uuidof(IAudioClient), CLSCTX_ALL, nullptr, @@ -471,42 +529,41 @@ bool install_audio_hooks(IpcClient& ipc, AudioRingHeader* ring) static_cast(hr)); } - // Now install every hook. Anchor Activate (idx 3) catches streams created - // after us; the inner hooks catch every render client on the shared vtables. - g_hk_activate = safetyhook::create_inline( - vtable_method(device, kIdx_IMMDevice_Activate), reinterpret_cast(&hk_Activate)); + // Now install every hook by swapping vtable slots. Anchor Activate (idx 3) + // catches streams created after us; the inner hooks catch every render client + // on the shared vtables. + g_vh_activate.install(device, kIdx_IMMDevice_Activate, reinterpret_cast(&hk_Activate)); if (self_render != nullptr) { g_self_render.store(self_render, std::memory_order_release); - g_hk_initialize = safetyhook::create_inline( - vtable_method(g_self_client, kIdx_IAudioClient_Initialize), reinterpret_cast(&hk_Initialize)); - g_hk_getservice = safetyhook::create_inline( - vtable_method(g_self_client, kIdx_IAudioClient_GetService), reinterpret_cast(&hk_GetService)); - g_hk_getbuffer = safetyhook::create_inline( - vtable_method(self_render, kIdx_IAudioRenderClient_GetBuffer), reinterpret_cast(&hk_GetBuffer)); - g_hk_releasebuffer = safetyhook::create_inline( - vtable_method(self_render, kIdx_IAudioRenderClient_ReleaseBuffer), - reinterpret_cast(&hk_ReleaseBuffer)); - g_audioclient_hooked = (g_hk_initialize && g_hk_getservice); + g_vh_initialize.install(g_self_client, kIdx_IAudioClient_Initialize, + reinterpret_cast(&hk_Initialize)); + g_vh_getservice.install(g_self_client, kIdx_IAudioClient_GetService, + reinterpret_cast(&hk_GetService)); + g_vh_getbuffer.install(self_render, kIdx_IAudioRenderClient_GetBuffer, + reinterpret_cast(&hk_GetBuffer)); + g_vh_releasebuffer.install(self_render, kIdx_IAudioRenderClient_ReleaseBuffer, + reinterpret_cast(&hk_ReleaseBuffer)); + g_audioclient_hooked = (static_cast(g_vh_initialize) && static_cast(g_vh_getservice)); // Keep g_self_client + self_render alive (held in globals) so the vtables // stay valid; they're released in remove_audio_hooks. } - hook_set_installed(g_id_activate, static_cast(g_hk_activate)); - hook_set_installed(g_id_initialize, static_cast(g_hk_initialize)); - hook_set_installed(g_id_getservice, static_cast(g_hk_getservice)); - hook_set_installed(g_id_getbuffer, static_cast(g_hk_getbuffer)); - hook_set_installed(g_id_releasebuffer, static_cast(g_hk_releasebuffer)); + hook_set_installed(g_id_activate, static_cast(g_vh_activate)); + hook_set_installed(g_id_initialize, static_cast(g_vh_initialize)); + hook_set_installed(g_id_getservice, static_cast(g_vh_getservice)); + hook_set_installed(g_id_getbuffer, static_cast(g_vh_getbuffer)); + hook_set_installed(g_id_releasebuffer, static_cast(g_vh_releasebuffer)); logf("install_audio_hooks: activate=%d init=%d getsvc=%d getbuf=%d relbuf=%d (device=%p)", - static_cast(g_hk_activate) ? 1 : 0, static_cast(g_hk_initialize) ? 1 : 0, - static_cast(g_hk_getservice) ? 1 : 0, static_cast(g_hk_getbuffer) ? 1 : 0, - static_cast(g_hk_releasebuffer) ? 1 : 0, device); + static_cast(g_vh_activate) ? 1 : 0, static_cast(g_vh_initialize) ? 1 : 0, + static_cast(g_vh_getservice) ? 1 : 0, static_cast(g_vh_getbuffer) ? 1 : 0, + static_cast(g_vh_releasebuffer) ? 1 : 0, device); device->Release(); // vtable lives in the (still-loaded) audio COM module enumerator->Release(); - return static_cast(g_hk_activate); + return static_cast(g_vh_activate); } void republish_audio_format() @@ -544,11 +601,11 @@ void set_audio_ring(AudioRingHeader* ring) void remove_audio_hooks() { std::scoped_lock lock(g_setup_mutex); - g_hk_releasebuffer = {}; - g_hk_getbuffer = {}; - g_hk_getservice = {}; - g_hk_initialize = {}; - g_hk_activate = {}; + g_vh_releasebuffer.remove(); + g_vh_getbuffer.remove(); + g_vh_getservice.remove(); + g_vh_initialize.remove(); + g_vh_activate.remove(); g_audioclient_hooked = false; hook_set_installed(g_id_activate, false); hook_set_installed(g_id_initialize, false);