Fix 32-bit game crash: call hooked __stdcall functions with stdcall()
SafetyHook's InlineHook::call() invokes the trampoline through a __cdecl pointer (the compiler default on x86). The functions we hook are __stdcall (IDXGISwapChain::Present/Present1, the WASAPI render interfaces, and the WINAPI SwapBuffers/wglSwapBuffers), so on 32-bit both sides cleaned the stack -> ESP imbalance -> Run-Time Check Failure #0 and an instant crash. On x64 every convention collapses to one, so it only bit 32-bit games: Slaps and Beans (Unity/Rewired, 32-bit D3D11) froze the moment the Present hook ran. The user's "crashes as soon as a button is pressed" was the Present, not the button. Switch every __stdcall trampoline call to SafetyHook's stdcall() (a no-op on x64). The XInput/focus hooks were unaffected because they never call the trampoline -- they return synthesized data. Reproduction + regression coverage: - tools/input_probe (coop_input_probe): injects, reports a connected pad, toggles a button, and takes a disable_mask to bisect which subsystem affects a game. Isolated the freeze to the video subsystem live. - hook_selftest_x86 + present_hook_test_x86: the x86 sub-build now builds and runs these (the x64 present_hook_test can't see a one-convention bug). present_hook_test_x86 drives a real swapchain through the trampoline -- it would hit RTC #0 before this fix. - hook_selftest strengthened to exercise every loaded xinput DLL's full export set (GetState, ordinal-100 GetStateEx, GetCapabilities, rumble SetState) and to dump the SharedBlock layout. - protocol.hpp: static_asserts lock the cross-bitness front-of-block offsets (verified byte-identical on x86 and x64). README roadmap trimmed (this milestone done) and a lessons-learned note added on the call()/stdcall() convention trap. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -151,7 +151,10 @@ void try_register_lazy(IAudioRenderClient* rc);
|
||||
HRESULT STDMETHODCALLTYPE hk_GetBuffer(IAudioRenderClient* self, UINT32 num_frames, BYTE** data)
|
||||
{
|
||||
hook_note_call(g_id_getbuffer);
|
||||
const HRESULT hr = g_hk_getbuffer.call<HRESULT>(self, num_frames, data);
|
||||
// stdcall(), NOT call(): these are COM methods (__stdcall). SafetyHook's call()
|
||||
// uses a __cdecl pointer (the x86 default), which double-cleans the stack on
|
||||
// 32-bit -> ESP imbalance -> Run-Time Check Failure #0 / crash. Harmless on x64.
|
||||
const HRESULT hr = g_hk_getbuffer.stdcall<HRESULT>(self, num_frames, data);
|
||||
if (SUCCEEDED(hr) && data != nullptr)
|
||||
{
|
||||
t_gb_client = self;
|
||||
@@ -204,11 +207,11 @@ HRESULT STDMETHODCALLTYPE hk_ReleaseBuffer(IAudioRenderClient* self, UINT32 num_
|
||||
{
|
||||
std::memset(t_gb_data, 0, bytes); // belt-and-suspenders vs a driver ignoring SILENT
|
||||
g_frames_captured.fetch_add(num_frames, std::memory_order_relaxed);
|
||||
return g_hk_releasebuffer.call<HRESULT>(self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT);
|
||||
return g_hk_releasebuffer.stdcall<HRESULT>(self, num_frames, flags | AUDCLNT_BUFFERFLAGS_SILENT);
|
||||
}
|
||||
}
|
||||
}
|
||||
return g_hk_releasebuffer.call<HRESULT>(self, num_frames, flags);
|
||||
return g_hk_releasebuffer.stdcall<HRESULT>(self, num_frames, flags);
|
||||
}
|
||||
|
||||
// Registers a newly created render client: assigns it a debug slot, marks the
|
||||
@@ -312,7 +315,7 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo
|
||||
{
|
||||
hook_note_call(g_id_initialize);
|
||||
const HRESULT hr =
|
||||
g_hk_initialize.call<HRESULT>(self, mode, flags, buffer_duration, periodicity, format, session);
|
||||
g_hk_initialize.stdcall<HRESULT>(self, mode, flags, buffer_duration, periodicity, format, session);
|
||||
logf("hk_Initialize: client=%p mode=%d flags=0x%lX hr=0x%08lX fmt=%s", self, mode,
|
||||
static_cast<unsigned long>(flags), static_cast<unsigned long>(hr), format ? "yes" : "null");
|
||||
if (SUCCEEDED(hr) && format != nullptr)
|
||||
@@ -326,7 +329,7 @@ HRESULT STDMETHODCALLTYPE hk_Initialize(IAudioClient* self, AUDCLNT_SHAREMODE mo
|
||||
HRESULT STDMETHODCALLTYPE hk_GetService(IAudioClient* self, REFIID riid, void** ppv)
|
||||
{
|
||||
hook_note_call(g_id_getservice);
|
||||
const HRESULT hr = g_hk_getservice.call<HRESULT>(self, riid, ppv);
|
||||
const HRESULT hr = g_hk_getservice.stdcall<HRESULT>(self, riid, ppv);
|
||||
const bool is_render = (riid == __uuidof(IAudioRenderClient));
|
||||
logf("hk_GetService: client=%p hr=0x%08lX render_client=%d", self, static_cast<unsigned long>(hr),
|
||||
is_render ? 1 : 0);
|
||||
@@ -384,7 +387,7 @@ HRESULT STDMETHODCALLTYPE hk_Activate(IMMDevice* self, REFIID riid, DWORD cls_ct
|
||||
void** ppv)
|
||||
{
|
||||
hook_note_call(g_id_activate);
|
||||
const HRESULT hr = g_hk_activate.call<HRESULT>(self, riid, cls_ctx, params, ppv);
|
||||
const HRESULT hr = g_hk_activate.stdcall<HRESULT>(self, riid, cls_ctx, params, ppv);
|
||||
const bool is_audioclient = (riid == __uuidof(IAudioClient) || riid == __uuidof(IAudioClient2) ||
|
||||
riid == __uuidof(IAudioClient3));
|
||||
logf("hk_Activate: device=%p hr=0x%08lX audioclient=%d", self, static_cast<unsigned long>(hr),
|
||||
|
||||
@@ -242,7 +242,7 @@ BOOL WINAPI hk_SwapBuffers(HDC hdc)
|
||||
t_in_swap = true;
|
||||
capture_gl(hdc);
|
||||
}
|
||||
const BOOL r = g_hk_swapbuffers.call<BOOL>(hdc);
|
||||
const BOOL r = g_hk_swapbuffers.stdcall<BOOL>(hdc); // __stdcall: call() is __cdecl on x86 -> crash
|
||||
if (outer)
|
||||
{
|
||||
t_in_swap = false;
|
||||
@@ -260,7 +260,7 @@ BOOL WINAPI hk_wglSwapBuffers(HDC hdc)
|
||||
t_in_swap = true;
|
||||
capture_gl(hdc);
|
||||
}
|
||||
const BOOL r = g_hk_wglswap.call<BOOL>(hdc);
|
||||
const BOOL r = g_hk_wglswap.stdcall<BOOL>(hdc); // __stdcall: call() is __cdecl on x86 -> crash
|
||||
if (outer)
|
||||
{
|
||||
t_in_swap = false;
|
||||
|
||||
@@ -232,7 +232,12 @@ HRESULT STDMETHODCALLTYPE hk_Present(IDXGISwapChain* sc, UINT sync_interval, UIN
|
||||
{
|
||||
capture_backbuffer(sc);
|
||||
}
|
||||
return g_hk_present.call<HRESULT>(sc, sync_interval, flags);
|
||||
// stdcall(), NOT call(): IDXGISwapChain::Present is __stdcall, but SafetyHook's
|
||||
// call() invokes the trampoline through a __cdecl pointer (the default on x86).
|
||||
// On 32-bit that double-cleans the stack -> ESP imbalance -> Run-Time Check
|
||||
// Failure #0 and an instant crash. On x64 the conventions collapse, so it only
|
||||
// bit 32-bit games (e.g. Slaps and Beans froze the moment it presented).
|
||||
return g_hk_present.stdcall<HRESULT>(sc, sync_interval, flags);
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE hk_Present1(IDXGISwapChain1* sc, UINT sync_interval, UINT flags,
|
||||
@@ -248,7 +253,7 @@ HRESULT STDMETHODCALLTYPE hk_Present1(IDXGISwapChain1* sc, UINT sync_interval, U
|
||||
{
|
||||
capture_backbuffer(sc); // IDXGISwapChain1 derives from IDXGISwapChain
|
||||
}
|
||||
return g_hk_present1.call<HRESULT>(sc, sync_interval, flags, params);
|
||||
return g_hk_present1.stdcall<HRESULT>(sc, sync_interval, flags, params); // __stdcall, see hk_Present
|
||||
}
|
||||
|
||||
// Create a throwaway device + swapchain purely to read IDXGISwapChain::Present
|
||||
|
||||
Reference in New Issue
Block a user