BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden every hook against the install/remove use-after-free

Browse Source 
Spamming a subsystem toggle (the "Mirror video" button) could crash the game:
remove_*_hooks freed a hook's shared D3D / Vulkan / IPC state immediately, while a
capture detour was still mid-flight on the game's render thread -> use-after-free.
Only the audio hooks had the safe-unhook drain; the video (Present/D3D9/D3D10/GL/
Vulkan) and XInput/focus/MKB hooks did not.

Test-first: mock_game_test now runs an aggressive hook/unhook storm -- a separate
thread thrashes every subsystem on/off while the game presents, across all backends.
It crashed gl + vk (0xC0000005) and failed dx9 capture-resume before the fix.

Fix (hook/src/hook_guard.hpp, DetourGate): each detour wraps its body in an RAII
active-count Guard; remove_* restores the hook first (so no new detour starts),
drains the in-flight detours to zero, and only then frees the shared state. Vulkan
is special-cased -- the game caches hk_vkQueuePresentKHR, so removal closes an
atomic capture gate (detours then pass through to the real present), drains, then
frees the read-back resources. Storm now passes on every backend.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-23 01:56:36 +02:00
parent 79399e88f1
commit 21c15b162b
10 changed files with 365 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
tests/mock_game_test.cpp
View File

@@ -9,10 +9,12 @@
// cycles the audio subsystem off/on a few times (hook/unhook stress), and confirms the
// game never freezes/crashes. Skips cleanly without a D3D11 device.
#include <algorithm>
#include <atomic>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <string>
#include <thread>
#include <vector>
#include <windows.h>
@@ -755,6 +757,172 @@ void test_av_and_hook_cycles(ID3D11Device* device)
game.kill();
}
// Aggressively toggle every injected subsystem on/off from a separate thread while the game is
// actively presenting, to provoke an unsafe hook install/remove race. The known failure mode:
// remove_*_hooks frees the hook's shared D3D state (device / context / keyed-mutex texture, and
// the Vulkan read-back resources) while a capture detour on the game's render thread is still
// using it -> use-after-free -> the game crashes. This is the "spamming the Mirror video button
// crashed Brotato" bug; the gentle, audio-only cycles in test_av_and_hook_cycles never exercised
// the video teardown, so they missed it. We storm ALL subsystems (especially video) across every
// backend, then confirm the game never crashed/froze and that capture resumes. vk_early uses the
// suspended-launch + early-inject path (Vulkan caches its present pointer at init, so a late inject
// can't hook it).
void test_hook_storm(const char* backend, ID3D11Device* device, bool vk_early)
{
std::printf("== hook/unhook storm: %s ==\n", backend);
std::wstring wbackend;
for (const char* p = backend; *p != '\0'; ++p) // backend names are ASCII
{
wbackend.push_back(static_cast<wchar_t>(*p));
}
PROCESS_INFORMATION pi{};
STARTUPINFOW si{};
si.cb = sizeof(si);
const std::wstring exe = tool_path(L"coop_mock_game.exe");
std::wstring cmd = L"\"" + exe + L"\" " + wbackend + L" 60";
if (vk_early)
{
SetEnvironmentVariableW(L"COOP_MOCK_VK_EARLY", L"1");
}
const DWORD launch_flags = vk_early ? CREATE_SUSPENDED : 0;
const BOOL launched =
CreateProcessW(exe.c_str(), cmd.data(), nullptr, nullptr, FALSE, launch_flags, nullptr, nullptr, &si, &pi);
if (vk_early)
{
SetEnvironmentVariableW(L"COOP_MOCK_VK_EARLY", nullptr);
}
if (!launched)
{
check(false, "launch mock game (storm)");
return;
}
auto alive = [&] { return WaitForSingleObject(pi.hProcess, 0) == WAIT_TIMEOUT; };
auto exit_code = [&] {
DWORD c = 0;
GetExitCodeProcess(pi.hProcess, &c);
return c;
};
auto cleanup = [&] {
TerminateProcess(pi.hProcess, 0);
WaitForSingleObject(pi.hProcess, 2000);
CloseHandle(pi.hThread);
CloseHandle(pi.hProcess);
};
SharedMemory shm;
SharedBlock* block = make_ipc(shm, pi.dwProcessId, /*disabled=*/0); // all subsystems on
SharedMemory ring_shm;
AudioRingHeader* ring = nullptr;
if (ring_shm.create(audio_ring_name(pi.dwProcessId), audio_ring_total_size(kAudioRingCapacity)))
{
ring = ring_shm.as<AudioRingHeader>();
audio_ring_init(*ring, kAudioRingCapacity);
ring->capture_enabled.store(1, std::memory_order_release);
}
const bool injected = block != nullptr && inject_retry(pi.dwProcessId);
if (vk_early)
{
ResumeThread(pi.hThread); // the mock loads Vulkan + waits, then renders
}
if (!injected)
{
if (vk_early && !alive() && exit_code() == 2)
{
std::printf(" Vulkan unavailable -- skipping storm\n");
}
else
{
check(false, "inject mock game (storm)");
}
cleanup();
return;
}
Sleep(vk_early ? 2500 : 1000); // let the hook attach + the game start presenting
if (vk_early && !alive() && exit_code() == 2)
{
std::printf(" Vulkan unavailable -- skipping storm\n");
cleanup();
return;
}
const std::uint32_t hb_start = block->status.heartbeat.load(std::memory_order_relaxed);
// Storm thread: flip every subsystem on/off as fast as it can while the game presents, so a
// remove lands while a capture detour is mid-flight on the game's render thread.
std::atomic<bool> stop{false};
std::thread storm([&] {
bool off = false;
while (!stop.load(std::memory_order_relaxed))
{
off = !off;
for (std::uint32_t s = 0; s < HookSubsys_Count; ++s)
{
block->control.subsystem_disabled[s].store(off ? 1u : 0u, std::memory_order_release);
}
Sleep(60);
}
});
bool crashed = false;
for (int i = 0; i < 170 && !crashed; ++i) // ~10 s of storming
{
Sleep(60);
if (!alive())
{
crashed = true;
}
}
stop.store(true, std::memory_order_relaxed);
storm.join();
if (crashed)
{
std::printf(" game CRASHED during the storm (exit 0x%08lX)\n", exit_code());
}
check(!crashed, "game survived the hook/unhook storm (no crash)");
if (crashed)
{
cleanup();
return;
}
// Re-enable everything and confirm the game is still alive + the hook still beating.
for (std::uint32_t s = 0; s < HookSubsys_Count; ++s)
{
block->control.subsystem_disabled[s].store(0, std::memory_order_release);
}
Sleep(600);
check(alive(), "game alive after the storm settles");
check(block->status.heartbeat.load(std::memory_order_relaxed) > hb_start,
"hook heartbeat advanced across the storm (no freeze)");
// Capture must resume (the rehook works end-to-end). Vulkan can't re-arm after a toggle (its
// present pointer was cached at init), so only assert resume for the other backends.
if (!vk_early)
{
SharedTextureSource src;
src.init(device);
const std::uint64_t frames0 = src.frames_copied();
bool advanced = false;
for (int i = 0; i < 80 && alive(); ++i) // ~4 s
{
Sleep(50);
const VideoShareView share = read_video_share(block);
if (src.update(share, pi.dwProcessId) && src.frames_copied() > frames0 + 3)
{
advanced = true;
break;
}
}
check(advanced, "video capture resumed after the storm");
}
cleanup();
}
} // namespace
int main()
@@ -787,6 +955,16 @@ int main()
test_av_and_hook_cycles(device);
// Aggressive hook/unhook storm across every backend: a separate thread thrashes every
// subsystem on/off while the game presents, to catch an unsafe install/remove race (the
// "spamming Mirror video crashed Brotato" use-after-free). vk uses the early-load path.
test_hook_storm("gl", device, /*vk_early=*/false);
test_hook_storm("dx9", device, /*vk_early=*/false);
test_hook_storm("dx10", device, /*vk_early=*/false);
test_hook_storm("dx11", device, /*vk_early=*/false);
test_hook_storm("dx12", device, /*vk_early=*/false);
test_hook_storm("vk", device, /*vk_early=*/true);
device->Release();
kill_stray_mock_games(); // belt-and-suspenders: ensure nothing is left running