Harden every hook against the install/remove use-after-free

Spamming a subsystem toggle (the "Mirror video" button) could crash the game:
remove_*_hooks freed a hook's shared D3D / Vulkan / IPC state immediately, while a
capture detour was still mid-flight on the game's render thread -> use-after-free.
Only the audio hooks had the safe-unhook drain; the video (Present/D3D9/D3D10/GL/
Vulkan) and XInput/focus/MKB hooks did not.

Test-first: mock_game_test now runs an aggressive hook/unhook storm -- a separate
thread thrashes every subsystem on/off while the game presents, across all backends.
It crashed gl + vk (0xC0000005) and failed dx9 capture-resume before the fix.

Fix (hook/src/hook_guard.hpp, DetourGate): each detour wraps its body in an RAII
active-count Guard; remove_* restores the hook first (so no new detour starts),
drains the in-flight detours to zero, and only then frees the shared state. Vulkan
is special-cased -- the game caches hk_vkQueuePresentKHR, so removal closes an
atomic capture gate (detours then pass through to the real present), drains, then
frees the read-back resources. Storm now passes on every backend.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-23 01:56:36 +02:00
parent 79399e88f1
commit 21c15b162b
10 changed files with 365 additions and 22 deletions

View File

@@ -6,6 +6,7 @@
#include <safetyhook.hpp>
#include "hook_guard.hpp"
#include "hook_registry.hpp"
namespace coop::hook
@@ -14,6 +15,8 @@ namespace coop::hook
namespace
{
DetourGate g_gate; // drains in-flight focus / WNDPROC detours before remove nulls their state
HWND g_game_hwnd = nullptr;
WNDPROC g_orig_proc = nullptr;
bool g_unicode = true;
@@ -74,6 +77,7 @@ HWND find_main_window(DWORD pid)
// Replacement window procedure: convince the game it is never deactivated.
LRESULT CALLBACK subclass_proc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam)
{
DetourGate::Guard guard(g_gate); // keep g_orig_proc / g_unicode valid for this whole dispatch
switch (msg)
{
case WM_ACTIVATE:
@@ -103,6 +107,7 @@ LRESULT CALLBACK subclass_proc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam
HWND WINAPI hk_GetForegroundWindow()
{
DetourGate::Guard guard(g_gate);
hook_note_call(g_id_foreground);
if (g_focus_ipc != nullptr)
{
@@ -113,6 +118,7 @@ HWND WINAPI hk_GetForegroundWindow()
HWND WINAPI hk_GetActiveWindow()
{
DetourGate::Guard guard(g_gate);
hook_note_call(g_id_active);
if (g_focus_ipc != nullptr)
{
@@ -123,6 +129,7 @@ HWND WINAPI hk_GetActiveWindow()
HWND WINAPI hk_GetFocus()
{
DetourGate::Guard guard(g_gate);
hook_note_call(g_id_focus);
if (g_focus_ipc != nullptr)
{
@@ -135,6 +142,7 @@ HWND WINAPI hk_GetFocus()
// the operator's mouse isn't trapped; otherwise honor the game's clip.
BOOL WINAPI hk_ClipCursor(const RECT* rect)
{
DetourGate::Guard guard(g_gate);
hook_note_call(g_id_clipcursor);
const bool allow = g_focus_ipc != nullptr && g_focus_ipc->cursor_clip_allowed();
return g_hk_clipcursor.stdcall<BOOL>(allow ? rect : nullptr);
@@ -144,6 +152,7 @@ BOOL WINAPI hk_ClipCursor(const RECT* rect)
// mouse can move freely (e.g. to reach the overlay); pass it through when clipping.
BOOL WINAPI hk_SetCursorPos(int x, int y)
{
DetourGate::Guard guard(g_gate);
hook_note_call(g_id_setcursorpos);
const bool allow = g_focus_ipc != nullptr && g_focus_ipc->cursor_clip_allowed();
if (!allow)
@@ -290,6 +299,11 @@ void remove_focus_spoof()
hook_set_installed(g_id_wndproc, false);
hook_set_installed(g_id_clipcursor, false);
hook_set_installed(g_id_setcursorpos, false);
// The WNDPROC is restored and the inline hooks reset above, so no NEW detour can start. Drain
// any focus / WNDPROC detour still in-flight on the game's window thread before nulling the
// state they read (g_orig_proc / g_game_hwnd / g_focus_ipc) -- otherwise a dispatch mid-flight
// could call a null original WNDPROC or a dangling IPC pointer.
g_gate.drain();
if (g_focus_ipc != nullptr)
{
g_focus_ipc->mark_focus_spoof(false, 0);