BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix 32-bit game crash: call hooked __stdcall functions with stdcall()

Browse Source 
SafetyHook's InlineHook::call() invokes the trampoline through a __cdecl
pointer (the compiler default on x86). The functions we hook are __stdcall
(IDXGISwapChain::Present/Present1, the WASAPI render interfaces, and the
WINAPI SwapBuffers/wglSwapBuffers), so on 32-bit both sides cleaned the
stack -> ESP imbalance -> Run-Time Check Failure #0 and an instant crash.
On x64 every convention collapses to one, so it only bit 32-bit games:
Slaps and Beans (Unity/Rewired, 32-bit D3D11) froze the moment the Present
hook ran. The user's "crashes as soon as a button is pressed" was the
Present, not the button.

Switch every __stdcall trampoline call to SafetyHook's stdcall() (a no-op
on x64). The XInput/focus hooks were unaffected because they never call
the trampoline -- they return synthesized data.

Reproduction + regression coverage:
- tools/input_probe (coop_input_probe): injects, reports a connected pad,
  toggles a button, and takes a disable_mask to bisect which subsystem
  affects a game. Isolated the freeze to the video subsystem live.
- hook_selftest_x86 + present_hook_test_x86: the x86 sub-build now builds
  and runs these (the x64 present_hook_test can't see a one-convention
  bug). present_hook_test_x86 drives a real swapchain through the
  trampoline -- it would hit RTC #0 before this fix.
- hook_selftest strengthened to exercise every loaded xinput DLL's full
  export set (GetState, ordinal-100 GetStateEx, GetCapabilities, rumble
  SetState) and to dump the SharedBlock layout.
- protocol.hpp: static_asserts lock the cross-bitness front-of-block
  offsets (verified byte-identical on x86 and x64).

README roadmap trimmed (this milestone done) and a lessons-learned note
added on the call()/stdcall() convention trap.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-20 20:07:17 +02:00
parent 9977ad5d5a
commit 435ab9d30f
9 changed files with 528 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tools/input_probe/CMakeLists.txt Normal file
View File

@@ -0,0 +1,7 @@
# Dev harness: creates the hook's IPC block, injects coop_hook.dll into a target
# game by pid, reports one connected pad, and toggles a button so the game's input
# layer processes a real state change. Reproduces the "32-bit game crashes on
# button press" report headlessly, without Steam / RPT / the host UI.
add_executable(coop_input_probe main.cpp)
target_link_libraries(coop_input_probe PRIVATE coop_common)
set_target_properties(coop_input_probe PROPERTIES OUTPUT_NAME "coop_input_probe")

257
tools/input_probe/main.cpp Normal file
View File

@@ -0,0 +1,257 @@
// coop_input_probe -- standalone harness to bring up the injected XInput hook and
// forward synthetic controller input into a real game, without Steam / RPT / the
// host UI. This is the headless reproduction for the "32-bit game crashes as soon
// as a button is pressed" report: it injects coop_hook.dll, reports one connected
// pad, and toggles a button every second so the game's input layer processes a
// real state change.
//
// coop_input_probe <pid> [seconds]
//
// Run from the same directory as coop_hook.dll (i.e. bin/<config>/). For a 32-bit
// (WOW64) target it shells out to coop_inject_x86.exe + coop_hook_x86.dll, exactly
// like the host.
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <string>
#include <windows.h>
#include "coop/log_ring.hpp"
#include "coop/protocol.hpp"
#include "coop/shared_memory.hpp"
namespace
{
std::wstring dll_path_next_to_self()
{
wchar_t exe[MAX_PATH] = {};
GetModuleFileNameW(nullptr, exe, MAX_PATH);
std::wstring path(exe);
const size_t slash = path.find_last_of(L"\\/");
if (slash != std::wstring::npos)
{
path.resize(slash + 1);
}
return path + L"coop_hook.dll";
}
std::wstring sibling_of(const std::wstring& path, const wchar_t* name)
{
const size_t slash = path.find_last_of(L"\\/");
return (slash == std::wstring::npos ? std::wstring() : path.substr(0, slash + 1)) + name;
}
// Inject a 32-bit (WOW64) target via the x86 helper, mirroring the host/audio probe.
bool inject_via_helper(unsigned long pid, const std::wstring& dll_path)
{
const std::wstring helper = sibling_of(dll_path, L"coop_inject_x86.exe");
const std::wstring x86_dll = sibling_of(dll_path, L"coop_hook_x86.dll");
if (GetFileAttributesW(helper.c_str()) == INVALID_FILE_ATTRIBUTES ||
GetFileAttributesW(x86_dll.c_str()) == INVALID_FILE_ATTRIBUTES)
{
std::printf("ERROR: x86 helper/dll missing next to the probe.\n");
return false;
}
std::wstring cmd = L"\"" + helper + L"\" " + std::to_wstring(pid) + L" \"" + x86_dll + L"\"";
std::printf("32-bit target: injecting coop_hook_x86.dll via coop_inject_x86.exe ...\n");
STARTUPINFOW si{};
si.cb = sizeof(si);
PROCESS_INFORMATION pi{};
if (!CreateProcessW(helper.c_str(), cmd.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &si, &pi))
{
std::printf("ERROR: CreateProcess(coop_inject_x86) failed (%lu).\n", GetLastError());
return false;
}
WaitForSingleObject(pi.hProcess, INFINITE);
DWORD code = 1;
GetExitCodeProcess(pi.hProcess, &code);
CloseHandle(pi.hThread);
CloseHandle(pi.hProcess);
if (code != 0)
{
std::printf("ERROR: coop_inject_x86 reported failure (exit %lu).\n", code);
return false;
}
return true;
}
bool inject(unsigned long pid, const std::wstring& dll_path)
{
if (GetFileAttributesW(dll_path.c_str()) == INVALID_FILE_ATTRIBUTES)
{
std::printf("ERROR: coop_hook.dll not found at the probe's directory.\n");
return false;
}
const DWORD access = PROCESS_CREATE_THREAD | PROCESS_QUERY_INFORMATION | PROCESS_VM_OPERATION |
PROCESS_VM_WRITE | PROCESS_VM_READ;
HANDLE process = OpenProcess(access, FALSE, pid);
if (process == nullptr)
{
std::printf("ERROR: OpenProcess(%lu) failed (%lu). Run as administrator?\n", pid, GetLastError());
return false;
}
USHORT proc_machine = IMAGE_FILE_MACHINE_UNKNOWN, native_machine = IMAGE_FILE_MACHINE_UNKNOWN;
if (IsWow64Process2(process, &proc_machine, &native_machine) && proc_machine != IMAGE_FILE_MACHINE_UNKNOWN)
{
CloseHandle(process);
return inject_via_helper(pid, dll_path);
}
const SIZE_T bytes = (dll_path.size() + 1) * sizeof(wchar_t);
void* remote = VirtualAllocEx(process, nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
bool ok = false;
if (remote != nullptr && WriteProcessMemory(process, remote, dll_path.c_str(), bytes, nullptr))
{
auto load_library = reinterpret_cast<LPTHREAD_START_ROUTINE>(
GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "LoadLibraryW"));
HANDLE thread = CreateRemoteThread(process, nullptr, 0, load_library, remote, 0, nullptr);
if (thread != nullptr)
{
WaitForSingleObject(thread, INFINITE);
DWORD exit_code = 0;
GetExitCodeThread(thread, &exit_code);
CloseHandle(thread);
ok = (exit_code != 0);
}
}
if (remote != nullptr)
{
VirtualFreeEx(process, remote, 0, MEM_RELEASE);
}
CloseHandle(process);
if (!ok)
{
std::printf("ERROR: injection failed (%lu).\n", GetLastError());
}
return ok;
}
} // namespace
int wmain(int argc, wchar_t** argv)
{
if (argc < 2)
{
std::printf("usage: coop_input_probe <pid> [seconds] [disable_mask]\n"
" Injects coop_hook.dll, reports one connected pad, and toggles a\n"
" button every second so the game processes a real state change.\n"
" disable_mask (hex): bit per subsystem NOT to install --\n"
" 0x1=input 0x2=focus 0x4=audio 0x8=video (default 0 = all on).\n");
return 1;
}
const unsigned long pid = std::wcstoul(argv[1], nullptr, 10);
const int seconds = (argc >= 3) ? std::max(1, _wtoi(argv[2])) : 30;
const unsigned disable_mask = (argc >= 4) ? std::wcstoul(argv[3], nullptr, 0) : 0u;
if (pid == 0)
{
std::printf("ERROR: invalid pid.\n");
return 1;
}
// 1) Input SharedBlock: report one connected pad up front (buttons still zero),
// so the game sees a controller arrive before we start pressing anything.
coop::SharedMemory ipc;
if (!ipc.create(coop::shared_memory_name(pid), sizeof(coop::SharedBlock)))
{
std::printf("ERROR: create input mapping failed (%lu).\n", GetLastError());
return 1;
}
auto* block = ipc.as<coop::SharedBlock>();
block->version = coop::kProtocolVersion;
block->sequence.store(0, std::memory_order_relaxed);
// Subsystem isolation: skip installing the ones whose bit is set in disable_mask
// (0x1=input 0x2=focus 0x4=audio 0x8=video). Lets us bisect which injected
// subsystem freezes a given game.
static const char* kSubsysNames[] = {"input", "focus", "audio", "video"};
for (std::uint32_t i = 0; i < coop::HookSubsys_Count; ++i)
{
const bool disabled = (disable_mask & (1u << i)) != 0;
block->control.subsystem_disabled[i].store(disabled ? 1u : 0u, std::memory_order_release);
std::printf("subsystem %-6s %s\n", kSubsysNames[i], disabled ? "DISABLED" : "on");
}
coop::CoopPadState pads[coop::kMaxPads] = {};
pads[0].connected = 1;
pads[0].packet = 1;
coop::publish_pads(*block, pads, coop::kMaxPads);
block->magic = coop::kProtocolMagic;
// Log ring (host's role) so the hook streams its trace back to us.
coop::SharedMemory log_shm;
coop::LogRing* log_ring = nullptr;
std::uint64_t log_cursor = 0;
if (log_shm.create(coop::log_ring_name(pid), coop::log_ring_total_size(coop::kLogCapacity)))
{
log_ring = log_shm.as<coop::LogRing>();
coop::log_ring_init(*log_ring, coop::kLogCapacity);
}
// Enable the hook's file trace for this session.
{
wchar_t dir[MAX_PATH] = {};
if (GetTempPathW(MAX_PATH, dir) != 0)
{
const std::wstring sentinel = std::wstring(dir) + L"coop_hook.log.on";
HANDLE h = CreateFileW(sentinel.c_str(), GENERIC_WRITE, FILE_SHARE_READ, nullptr, OPEN_ALWAYS,
FILE_ATTRIBUTE_NORMAL, nullptr);
if (h != INVALID_HANDLE_VALUE)
{
CloseHandle(h);
}
}
}
std::printf("Injecting coop_hook.dll into pid %lu ...\n", pid);
if (!inject(pid, dll_path_next_to_self()))
{
return 1;
}
std::printf("Injected. Reporting pad 0 connected; toggling button A each second for %d s.\n", seconds);
std::printf("Hook trace: %%TEMP%%\\coop_hook.log\n\n");
const coop::HookStatus& status = block->status;
for (int t = 0; t < seconds; ++t)
{
// Toggle A (0x1000) every other second so the game's input layer sees a real
// edge -- this is the "press a button" event the crash report points at.
const bool press = (t % 2) == 1;
pads[0].packet = static_cast<std::uint32_t>(t + 2);
pads[0].buttons = press ? 0x1000 : 0x0000;
pads[0].thumb_lx = press ? 20000 : 0;
coop::publish_pads(*block, pads, coop::kMaxPads);
Sleep(1000);
const std::uint32_t heartbeat = status.heartbeat.load(std::memory_order_relaxed);
const std::uint32_t attached = status.attached;
const std::uint64_t gs0 = status.get_state_calls[0].load(std::memory_order_relaxed);
const std::uint64_t gc0 = status.get_caps_calls[0].load(std::memory_order_relaxed);
std::printf("[%2ds] %s hb=%u attached=%u getstate[0]=%llu getcaps[0]=%llu buttons=0x%04X\n", t + 1,
press ? "A-DOWN" : "A-up ", heartbeat, attached, static_cast<unsigned long long>(gs0),
static_cast<unsigned long long>(gc0), pads[0].buttons);
// Surface the hook's log lines as they arrive (shows where it got to).
if (log_ring != nullptr)
{
coop::log_ring_drain(*log_ring, log_cursor,
[](const coop::LogRecord& rec) { std::printf(" | %s\n", rec.text); });
}
}
std::printf("\nInstalled hooks (%u):\n", status.hook_entry_count);
static const char* kSubsys[] = {"Input", "Focus", "Audio", "Video"};
for (std::uint32_t i = 0; i < status.hook_entry_count && i < coop::kMaxHookEntries; ++i)
{
const coop::HookEntry& e = status.hook_entries[i];
std::printf(" [%-5s] %-34s %s calls=%llu\n", e.subsystem < 4 ? kSubsys[e.subsystem] : "?", e.name,
e.installed ? "ON " : "off", static_cast<unsigned long long>(e.calls));
}
std::printf("\nDone. Leaving the hook loaded in the game.\n");
block->magic = 0;
return 0;
}