Files
CoopAllTheThings/hook/src/vtable_hook.hpp
BlackMark e4aafa08db Consolidate the duplicated VtableHook into one shared copy
audio_hook.cpp carried its own VtableHook class identical to the shared
hook/src/vtable_hook.hpp -- two copies of the same delicate vtable-swap unhook
logic to keep in sync. Drop the audio copy and use the shared one (it's in
namespace coop::hook, so the in-file references resolve to it), leaving a short
note on why WASAPI methods are vtable-swapped rather than inline-hooked. Update
the shared header's comment to name both users (audio + DirectInput).

Last item from the review pass -- the Roadmap's Current-work section is now empty
(done work lives in git history); only Future work remains. Validated by
audio_hook_test (the vtable swap still hooks GetService/GetBuffer/ReleaseBuffer
end to end).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-24 02:28:05 +02:00

69 lines
2.3 KiB
C++

// Hook a single COM vtable slot by overwriting its function pointer; the original is called
// through the saved pointer. We use this instead of SafetyHook's inline hooks for COM methods
// because, on x86, MMDevApi/AudioSes/DirectInput prologues use dynamic stack alignment
// (`and esp,-8`) with EBP-relative argument access, which SafetyHook's trampoline relocation
// mishandles -> the original runs with garbage args and faults. Swapping the vtable entry leaves
// the original code untouched, so it runs with a pristine stack regardless of prologue shape.
// Every instance of a COM coclass shares one vtable, so a single swap intercepts all of them.
// See the project's stdcall-x86 note. Shared by the audio render-hook (audio_hook.cpp) and the
// input-side DirectInput hook (mkb_hook.cpp).
#pragma once
#include <windows.h>
namespace coop::hook
{
class VtableHook
{
public:
bool install(void* com_object, unsigned index, void* detour)
{
if (m_vtable != nullptr)
{
return true; // already installed (shared vtable covers every instance)
}
auto** vtable = *reinterpret_cast<void***>(com_object);
DWORD old_protect = 0;
if (!VirtualProtect(&vtable[index], sizeof(void*), PAGE_READWRITE, &old_protect))
{
return false;
}
m_original = vtable[index];
vtable[index] = detour; // aligned pointer store -> atomic vs. a concurrent caller
VirtualProtect(&vtable[index], sizeof(void*), old_protect, &old_protect);
m_vtable = vtable;
m_index = index;
return true;
}
void remove()
{
if (m_vtable == nullptr)
{
return;
}
DWORD old_protect = 0;
if (VirtualProtect(&m_vtable[m_index], sizeof(void*), PAGE_READWRITE, &old_protect))
{
m_vtable[m_index] = m_original;
VirtualProtect(&m_vtable[m_index], sizeof(void*), old_protect, &old_protect);
}
m_vtable = nullptr;
// Keep m_original valid: a detour already running on the game's thread may still call
// original() after we restore the slot. The original lives in the loaded module, so the
// pointer stays valid; a re-install re-reads it.
m_index = 0;
}
template <typename Fn> Fn original() const { return reinterpret_cast<Fn>(m_original); }
explicit operator bool() const { return m_vtable != nullptr; }
private:
void** m_vtable = nullptr;
unsigned m_index = 0;
void* m_original = nullptr;
};
} // namespace coop::hook