Files
CoopAllTheThings/CMakeLists.txt
BlackMark 4985239222 Fix 32-bit FMOD audio crash: hook WASAPI COM methods via vtable swap
The stdcall() fix stopped the Present-hook crash but 32-bit games (Slaps
and Beans, FMOD) still crashed the instant audio init ran through the
hook. Root cause: SafetyHook's inline hook relocates the target's
overwritten prologue into a trampoline, but MMDevApi/AudioSes COM methods
on x86 open with `push ebp; mov ebp,esp; and esp,-8` (dynamic stack
alignment) and read arguments EBP-relative. The relocated copy leaves EBP
wrong, so the original runs with garbage arguments and faults (AV writing
*ppInterface inside CEndpointDevice::Activate+0x3d).

Switch all five WASAPI COM hooks (IMMDevice::Activate, IAudioClient::
Initialize/GetService, IAudioRenderClient::GetBuffer/ReleaseBuffer) from
safetyhook::create_inline to a small VtableHook helper: VirtualProtect the
shared vtable slot, overwrite the function pointer, call the saved original
directly. No code patching, no trampoline, pristine stack regardless of
prologue. One swap covers every instance (a coclass shares one vtable), so
the existing shared-vtable strategy is preserved. Inline hooking stays for
Present/SwapBuffers, whose prologues relocate cleanly.

Reproduced in-process with a new x86 build of the audio render-hook test
(audio_hook_test_x86): it installs the hooks, then drives a fresh
IAudioClient through them and renders -- segfaulted before, passes now.
The x64 audio_hook_test passes regardless of the bug, so the 32-bit build
is the regression guard.

ctest: x64 7/7, x86 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-20 20:39:29 +02:00

134 lines
6.1 KiB
CMake

cmake_minimum_required(VERSION 3.21)
project(CoopAllTheThings
VERSION 0.0.1
DESCRIPTION "Steam Remote Play Together for any XInput game"
LANGUAGES CXX)
if(NOT WIN32)
message(FATAL_ERROR "CoopAllTheThings targets Windows only.")
endif()
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
# Emit compile_commands.json so clangd resolves includes/flags. Ignored by the
# Visual Studio generator, so configure the Ninja build (build-clangd/) for this;
# .clangd points clangd at that compilation database.
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
# Single output dir keeps the donor-launch / inject workflow simple: host exe and
# hook dll land next to each other under /bin (already in .gitignore).
set(COOP_OUTPUT_DIR "${CMAKE_SOURCE_DIR}/bin/$<CONFIG>")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${COOP_OUTPUT_DIR}")
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${COOP_OUTPUT_DIR}")
if(MSVC)
add_compile_options(/W4 /permissive- /Zc:__cplusplus /utf-8 /MP)
add_compile_definitions(UNICODE _UNICODE WIN32_LEAN_AND_MEAN NOMINMAX)
endif()
# --- x86 sub-build (internal, re-entrant) ----------------------------------
# When the ExternalProject below reconfigures this same tree with -A Win32 and
# -DCOOP_X86_HELPER_BUILD=ON, build ONLY the 32-bit hook DLL + injector helper
# (staged alongside the x64 binaries) and stop -- no host, tests, or imgui, and
# crucially no recursive ExternalProject.
option(COOP_X86_HELPER_BUILD "Internal: x86 sub-build (hook DLL + injector helper only)" OFF)
if(COOP_X86_HELPER_BUILD)
add_subdirectory(common)
set(SAFETYHOOK_BUILD_TEST OFF CACHE BOOL "" FORCE)
set(SAFETYHOOK_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(SAFETYHOOK_BUILD_DOCS OFF CACHE BOOL "" FORCE)
add_subdirectory(third_party/safetyhook)
add_subdirectory(hook)
add_subdirectory(tools/inject_helper)
# x86 reproduction of the XInput self-test. The x64 hook_selftest passes, so an
# x86 build of the same logic is how we reproduce (and then guard against) the
# 32-bit input-forwarding crash on real games like Slaps and Beans.
enable_testing()
add_executable(hook_selftest_x86
tests/hook_selftest.cpp
hook/src/xinput_hook.cpp
hook/src/hook_registry.cpp)
target_include_directories(hook_selftest_x86 PRIVATE hook/src)
target_link_libraries(hook_selftest_x86 PRIVATE coop_common safetyhook::safetyhook xinput)
add_test(NAME hook_selftest_x86 COMMAND hook_selftest_x86)
# x86 build of the Present-hook test. It drives a real swapchain through the
# SafetyHook trampoline, so it catches the x86-only calling-convention bug that
# froze 32-bit games (SafetyHook's call() is __cdecl; Present is __stdcall ->
# ESP imbalance / Run-Time Check Failure #0). The x64 present_hook_test can't see
# it (one calling convention), so this 32-bit build is the regression guard.
add_executable(present_hook_test_x86
tests/present_hook_test.cpp
hook/src/present_hook.cpp
hook/src/debug_log.cpp
hook/src/hook_registry.cpp)
target_include_directories(present_hook_test_x86 PRIVATE hook/src)
target_link_libraries(present_hook_test_x86 PRIVATE coop_common safetyhook::safetyhook d3d11 dxgi)
add_test(NAME present_hook_test_x86 COMMAND present_hook_test_x86)
# x86 build of the audio render-hook test. It installs the WASAPI hooks and then
# creates a *fresh* IAudioClient/IAudioRenderClient that calls Initialize /
# GetService / GetBuffer / ReleaseBuffer back through the SafetyHook trampolines
# -- the exact ordering that crashed 32-bit Slaps & Beans (FMOD) when audio hooks
# were live as the game initialized its output. The x64 audio_hook_test passes,
# so this 32-bit build is the regression guard for any x86-only fault in the
# audio setup detours / trampoline relocation of AudioSes.dll prologues.
add_executable(audio_hook_test_x86
tests/audio_hook_test.cpp
hook/src/audio_hook.cpp
hook/src/debug_log.cpp
hook/src/hook_registry.cpp)
target_include_directories(audio_hook_test_x86 PRIVATE hook/src)
target_compile_definitions(audio_hook_test_x86 PRIVATE NTDDI_VERSION=0x0A00000B)
target_link_libraries(audio_hook_test_x86 PRIVATE coop_common safetyhook::safetyhook ole32 mmdevapi)
add_test(NAME audio_hook_test_x86 COMMAND audio_hook_test_x86)
return()
endif()
add_subdirectory(third_party)
add_subdirectory(common)
add_subdirectory(host)
# The injected hook DLL pulls in SafetyHook, which fetches Zydis via FetchContent
# at configure time (allowed for transitive deps).
option(COOP_BUILD_HOOK "Build the injected game-side hook DLL" ON)
if(COOP_BUILD_HOOK)
# SafetyHook's own tests/examples/docs are noise for us.
set(SAFETYHOOK_BUILD_TEST OFF CACHE BOOL "" FORCE)
set(SAFETYHOOK_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(SAFETYHOOK_BUILD_DOCS OFF CACHE BOOL "" FORCE)
add_subdirectory(third_party/safetyhook)
add_subdirectory(hook)
enable_testing()
add_subdirectory(tools/audio_tone) # coop_tone: audio source for the loopback test
add_subdirectory(tools/audio_probe) # coop_audio_probe: inject + diagnose the render-hook
add_subdirectory(tools/input_probe) # coop_input_probe: inject + forward synthetic input
add_subdirectory(tests)
endif()
# --- x86 helper artifacts for 32-bit (WOW64) games -------------------------
# A 64-bit process can't cleanly inject a 32-bit one, so build the x86 hook DLL
# (coop_hook_x86.dll) + a 32-bit injector helper (coop_inject_x86.exe) via a
# nested Win32 configure of this same tree, staged next to the x64 binaries. Only
# the multi-config Visual Studio generator is supported here (the Ninja
# build-clangd/ tree is just for compile_commands and skips this).
option(COOP_BUILD_X86_HELPER "Build the x86 hook DLL + injector helper for 32-bit games" ON)
if(COOP_BUILD_HOOK AND COOP_BUILD_X86_HELPER AND CMAKE_GENERATOR MATCHES "Visual Studio")
include(ExternalProject)
ExternalProject_Add(coop_x86
SOURCE_DIR "${CMAKE_SOURCE_DIR}"
BINARY_DIR "${CMAKE_BINARY_DIR}/x86"
CMAKE_GENERATOR "${CMAKE_GENERATOR}"
CMAKE_GENERATOR_PLATFORM "Win32"
CMAKE_ARGS "-DCOOP_X86_HELPER_BUILD=ON"
BUILD_COMMAND "${CMAKE_COMMAND}" --build <BINARY_DIR> --config $<CONFIG>
INSTALL_COMMAND ""
BUILD_ALWAYS ON
USES_TERMINAL_CONFIGURE ON
USES_TERMINAL_BUILD ON)
endif()