Phase 2: Present-hook video path (shared-texture mirror)
Add an injected IDXGISwapChain::Present / Present1 hook as a lower-latency, border-free alternative to WGC. The hook copies the swapchain backbuffer into a shared keyed-mutex texture (coop_video_<pid>); the host opens it by name and samples it. New opt-in HookSubsys_Video (protocol v6 -> v7); the Video mirror panel gains a WGC vs Hooked source toggle that installs/removes the subsystem. Verified by present_hook_test (drives a real D3D11 swapchain end-to-end and reads the rendered pixels back through the shared texture) and against Phantom Brave (D3D9: hook installs cleanly and stays idle, WGC fallback). All 5 tests pass. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
384
hook/src/present_hook.cpp
Normal file
384
hook/src/present_hook.cpp
Normal file
@@ -0,0 +1,384 @@
|
||||
#include "present_hook.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#include <d3d11.h>
|
||||
#include <dxgi1_2.h>
|
||||
|
||||
#include <safetyhook.hpp>
|
||||
|
||||
#include "coop/shared_memory.hpp"
|
||||
#include "debug_log.hpp"
|
||||
#include "hook_registry.hpp"
|
||||
|
||||
namespace coop::hook
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
// IDXGISwapChain vtable layout (frozen ABI). IUnknown 0..2, IDXGIObject 3..6,
|
||||
// IDXGIDeviceSubObject 7, then IDXGISwapChain: Present = 8, GetBuffer = 9.
|
||||
// IDXGISwapChain1 adds methods after IDXGISwapChain (18 methods, 0..17), so
|
||||
// Present1 sits at index 22. Flip-model D3D11/12 games present via Present1.
|
||||
constexpr unsigned kIdx_IDXGISwapChain_Present = 8;
|
||||
constexpr unsigned kIdx_IDXGISwapChain1_Present1 = 22;
|
||||
|
||||
IpcClient* g_ipc = nullptr;
|
||||
unsigned long g_pid = 0;
|
||||
|
||||
safetyhook::InlineHook g_hk_present;
|
||||
safetyhook::InlineHook g_hk_present1;
|
||||
int g_id_present = -1;
|
||||
int g_id_present1 = -1;
|
||||
|
||||
std::atomic<std::uint64_t> g_present_calls{0};
|
||||
std::atomic<std::uint64_t> g_frames_shared{0};
|
||||
|
||||
// The shared backbuffer copy and its keyed mutex, created lazily on the first
|
||||
// Present once we can see the game's device + backbuffer format. Guarded by
|
||||
// g_tex_mutex (touched only on the render thread, but install/remove may race).
|
||||
std::mutex g_tex_mutex;
|
||||
ID3D11Texture2D* g_shared_tex = nullptr;
|
||||
IDXGIKeyedMutex* g_shared_mutex = nullptr;
|
||||
HANDLE g_shared_handle = nullptr;
|
||||
UINT g_share_w = 0;
|
||||
UINT g_share_h = 0;
|
||||
DXGI_FORMAT g_share_fmt = DXGI_FORMAT_UNKNOWN;
|
||||
bool g_unsupported_logged = false;
|
||||
|
||||
void* vtable_method(void* obj, unsigned index)
|
||||
{
|
||||
return (*reinterpret_cast<void***>(obj))[index];
|
||||
}
|
||||
|
||||
// Drop the shared texture/mutex/handle. Caller holds g_tex_mutex.
|
||||
void release_shared_locked()
|
||||
{
|
||||
if (g_shared_mutex != nullptr)
|
||||
{
|
||||
g_shared_mutex->Release();
|
||||
g_shared_mutex = nullptr;
|
||||
}
|
||||
if (g_shared_tex != nullptr)
|
||||
{
|
||||
g_shared_tex->Release();
|
||||
g_shared_tex = nullptr;
|
||||
}
|
||||
if (g_shared_handle != nullptr)
|
||||
{
|
||||
CloseHandle(g_shared_handle);
|
||||
g_shared_handle = nullptr;
|
||||
}
|
||||
g_share_w = g_share_h = 0;
|
||||
g_share_fmt = DXGI_FORMAT_UNKNOWN;
|
||||
}
|
||||
|
||||
// (Re)create the shared keyed-mutex texture for a w x h `fmt` backbuffer on the
|
||||
// game's `device`. Caller holds g_tex_mutex. Returns true if it's ready.
|
||||
bool ensure_shared_texture_locked(ID3D11Device* device, UINT w, UINT h, DXGI_FORMAT fmt)
|
||||
{
|
||||
if (g_shared_tex != nullptr && g_share_w == w && g_share_h == h && g_share_fmt == fmt)
|
||||
{
|
||||
return true; // already matches the current backbuffer
|
||||
}
|
||||
release_shared_locked();
|
||||
|
||||
D3D11_TEXTURE2D_DESC desc{};
|
||||
desc.Width = w;
|
||||
desc.Height = h;
|
||||
desc.MipLevels = 1;
|
||||
desc.ArraySize = 1;
|
||||
desc.Format = fmt;
|
||||
desc.SampleDesc.Count = 1;
|
||||
desc.Usage = D3D11_USAGE_DEFAULT;
|
||||
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET;
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
|
||||
|
||||
ID3D11Texture2D* tex = nullptr;
|
||||
HRESULT hr = device->CreateTexture2D(&desc, nullptr, &tex);
|
||||
if (FAILED(hr) || tex == nullptr)
|
||||
{
|
||||
logf("present: CreateTexture2D(shared) failed hr=0x%08lX (%ux%u fmt=%d)",
|
||||
static_cast<unsigned long>(hr), w, h, static_cast<int>(fmt));
|
||||
return false;
|
||||
}
|
||||
|
||||
IDXGIResource1* res = nullptr;
|
||||
hr = tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast<void**>(&res));
|
||||
if (FAILED(hr) || res == nullptr)
|
||||
{
|
||||
logf("present: QI IDXGIResource1 failed hr=0x%08lX", static_cast<unsigned long>(hr));
|
||||
tex->Release();
|
||||
return false;
|
||||
}
|
||||
const std::wstring name = video_share_name(g_pid);
|
||||
HANDLE handle = nullptr;
|
||||
hr = res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE,
|
||||
name.c_str(), &handle);
|
||||
res->Release();
|
||||
if (FAILED(hr) || handle == nullptr)
|
||||
{
|
||||
logf("present: CreateSharedHandle failed hr=0x%08lX", static_cast<unsigned long>(hr));
|
||||
tex->Release();
|
||||
return false;
|
||||
}
|
||||
|
||||
IDXGIKeyedMutex* mutex = nullptr;
|
||||
hr = tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast<void**>(&mutex));
|
||||
if (FAILED(hr) || mutex == nullptr)
|
||||
{
|
||||
logf("present: QI IDXGIKeyedMutex failed hr=0x%08lX", static_cast<unsigned long>(hr));
|
||||
CloseHandle(handle);
|
||||
tex->Release();
|
||||
return false;
|
||||
}
|
||||
|
||||
g_shared_tex = tex;
|
||||
g_shared_mutex = mutex;
|
||||
g_shared_handle = handle;
|
||||
g_share_w = w;
|
||||
g_share_h = h;
|
||||
g_share_fmt = fmt;
|
||||
logf("present: shared texture ready %ux%u fmt=%d name=%ls", w, h, static_cast<int>(fmt), name.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
// Copy the swapchain's backbuffer into the shared texture and publish it.
|
||||
void capture_backbuffer(IDXGISwapChain* sc)
|
||||
{
|
||||
ID3D11Texture2D* backbuf = nullptr;
|
||||
if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(&backbuf))) ||
|
||||
backbuf == nullptr)
|
||||
{
|
||||
if (!g_unsupported_logged)
|
||||
{
|
||||
logf("present: backbuffer is not an ID3D11Texture2D (D3D12/D3D9?); video hook idle");
|
||||
g_unsupported_logged = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
D3D11_TEXTURE2D_DESC bd{};
|
||||
backbuf->GetDesc(&bd);
|
||||
|
||||
// Multisampled backbuffers would need ResolveSubresource; flip-model swapchains
|
||||
// are single-sampled. Skip the rare MSAA case rather than mis-copy.
|
||||
if (bd.SampleDesc.Count != 1)
|
||||
{
|
||||
backbuf->Release();
|
||||
return;
|
||||
}
|
||||
|
||||
ID3D11Device* device = nullptr;
|
||||
backbuf->GetDevice(&device);
|
||||
ID3D11DeviceContext* ctx = nullptr;
|
||||
if (device != nullptr)
|
||||
{
|
||||
device->GetImmediateContext(&ctx);
|
||||
}
|
||||
|
||||
bool shared = false;
|
||||
if (device != nullptr && ctx != nullptr)
|
||||
{
|
||||
std::scoped_lock lock(g_tex_mutex);
|
||||
if (ensure_shared_texture_locked(device, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr)
|
||||
{
|
||||
// Key 0 on both sides: a plain cross-process mutex on the texture (the
|
||||
// keyed mutex is created released at key 0). Bounded wait so a stalled
|
||||
// host consumer can never hang the game's render thread.
|
||||
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
|
||||
{
|
||||
ctx->CopyResource(g_shared_tex, backbuf);
|
||||
g_shared_mutex->ReleaseSync(kVideoMutexKey);
|
||||
shared = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (shared)
|
||||
{
|
||||
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->publish_video_frame(bd.Width, bd.Height, static_cast<std::uint32_t>(bd.Format));
|
||||
}
|
||||
}
|
||||
|
||||
if (ctx != nullptr)
|
||||
{
|
||||
ctx->Release();
|
||||
}
|
||||
if (device != nullptr)
|
||||
{
|
||||
device->Release();
|
||||
}
|
||||
backbuf->Release();
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE hk_Present(IDXGISwapChain* sc, UINT sync_interval, UINT flags)
|
||||
{
|
||||
hook_note_call(g_id_present);
|
||||
g_present_calls.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->note_present();
|
||||
}
|
||||
// DXGI_PRESENT_TEST presents nothing; don't bother copying for it.
|
||||
if ((flags & DXGI_PRESENT_TEST) == 0)
|
||||
{
|
||||
capture_backbuffer(sc);
|
||||
}
|
||||
return g_hk_present.call<HRESULT>(sc, sync_interval, flags);
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE hk_Present1(IDXGISwapChain1* sc, UINT sync_interval, UINT flags,
|
||||
const DXGI_PRESENT_PARAMETERS* params)
|
||||
{
|
||||
hook_note_call(g_id_present1);
|
||||
g_present_calls.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->note_present();
|
||||
}
|
||||
if ((flags & DXGI_PRESENT_TEST) == 0)
|
||||
{
|
||||
capture_backbuffer(sc); // IDXGISwapChain1 derives from IDXGISwapChain
|
||||
}
|
||||
return g_hk_present1.call<HRESULT>(sc, sync_interval, flags, params);
|
||||
}
|
||||
|
||||
// Create a throwaway device + swapchain purely to read IDXGISwapChain::Present
|
||||
// (and IDXGISwapChain1::Present1) addresses, so we can inline-hook them. The
|
||||
// inline hook patches the function itself, so it catches every swapchain in the
|
||||
// process -- we can release the dummy after. `present1_out` may stay null on
|
||||
// platforms without DXGI 1.2.
|
||||
void* grab_present_address(void** present1_out)
|
||||
{
|
||||
*present1_out = nullptr;
|
||||
WNDCLASSEXW wc{};
|
||||
wc.cbSize = sizeof(wc);
|
||||
wc.lpfnWndProc = DefWindowProcW;
|
||||
wc.hInstance = GetModuleHandleW(nullptr);
|
||||
wc.lpszClassName = L"coop_present_probe";
|
||||
RegisterClassExW(&wc);
|
||||
HWND hwnd = CreateWindowExW(0, wc.lpszClassName, L"", WS_OVERLAPPEDWINDOW, 0, 0, 8, 8, nullptr, nullptr,
|
||||
wc.hInstance, nullptr);
|
||||
if (hwnd == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
DXGI_SWAP_CHAIN_DESC scd{};
|
||||
scd.BufferCount = 1;
|
||||
scd.BufferDesc.Width = 8;
|
||||
scd.BufferDesc.Height = 8;
|
||||
scd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
|
||||
scd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
|
||||
scd.OutputWindow = hwnd;
|
||||
scd.SampleDesc.Count = 1;
|
||||
scd.Windowed = TRUE;
|
||||
scd.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
|
||||
|
||||
IDXGISwapChain* swapchain = nullptr;
|
||||
ID3D11Device* device = nullptr;
|
||||
ID3D11DeviceContext* ctx = nullptr;
|
||||
const HRESULT hr = D3D11CreateDeviceAndSwapChain(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0,
|
||||
D3D11_SDK_VERSION, &scd, &swapchain, &device, nullptr, &ctx);
|
||||
void* present = nullptr;
|
||||
if (SUCCEEDED(hr) && swapchain != nullptr)
|
||||
{
|
||||
present = vtable_method(swapchain, kIdx_IDXGISwapChain_Present);
|
||||
IDXGISwapChain1* sc1 = nullptr;
|
||||
if (SUCCEEDED(swapchain->QueryInterface(__uuidof(IDXGISwapChain1), reinterpret_cast<void**>(&sc1))) &&
|
||||
sc1 != nullptr)
|
||||
{
|
||||
*present1_out = vtable_method(sc1, kIdx_IDXGISwapChain1_Present1);
|
||||
sc1->Release();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
logf("present: D3D11CreateDeviceAndSwapChain(probe) failed hr=0x%08lX", static_cast<unsigned long>(hr));
|
||||
}
|
||||
if (ctx != nullptr)
|
||||
{
|
||||
ctx->Release();
|
||||
}
|
||||
if (device != nullptr)
|
||||
{
|
||||
device->Release();
|
||||
}
|
||||
if (swapchain != nullptr)
|
||||
{
|
||||
swapchain->Release();
|
||||
}
|
||||
DestroyWindow(hwnd);
|
||||
UnregisterClassW(wc.lpszClassName, wc.hInstance);
|
||||
return present;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool install_present_hooks(IpcClient& ipc)
|
||||
{
|
||||
g_ipc = &ipc;
|
||||
g_pid = GetCurrentProcessId();
|
||||
if (g_hk_present)
|
||||
{
|
||||
return true; // already installed
|
||||
}
|
||||
|
||||
g_id_present = hook_register("IDXGISwapChain::Present", HookSubsys_Video);
|
||||
g_id_present1 = hook_register("IDXGISwapChain1::Present1", HookSubsys_Video);
|
||||
|
||||
void* present1 = nullptr;
|
||||
void* present = grab_present_address(&present1);
|
||||
if (present == nullptr)
|
||||
{
|
||||
hook_set_installed(g_id_present, false);
|
||||
hook_set_installed(g_id_present1, false);
|
||||
return false;
|
||||
}
|
||||
g_hk_present = safetyhook::create_inline(present, reinterpret_cast<void*>(&hk_Present));
|
||||
if (present1 != nullptr)
|
||||
{
|
||||
g_hk_present1 = safetyhook::create_inline(present1, reinterpret_cast<void*>(&hk_Present1));
|
||||
}
|
||||
g_unsupported_logged = false;
|
||||
hook_set_installed(g_id_present, static_cast<bool>(g_hk_present));
|
||||
hook_set_installed(g_id_present1, static_cast<bool>(g_hk_present1));
|
||||
logf("install_present_hooks: present=%p hooked=%d present1=%p hooked=%d", present,
|
||||
static_cast<bool>(g_hk_present) ? 1 : 0, present1, static_cast<bool>(g_hk_present1) ? 1 : 0);
|
||||
return static_cast<bool>(g_hk_present);
|
||||
}
|
||||
|
||||
void remove_present_hooks()
|
||||
{
|
||||
g_hk_present = {};
|
||||
g_hk_present1 = {};
|
||||
hook_set_installed(g_id_present, false);
|
||||
hook_set_installed(g_id_present1, false);
|
||||
{
|
||||
std::scoped_lock lock(g_tex_mutex);
|
||||
release_shared_locked();
|
||||
}
|
||||
g_present_calls.store(0, std::memory_order_relaxed);
|
||||
g_frames_shared.store(0, std::memory_order_relaxed);
|
||||
g_ipc = nullptr;
|
||||
}
|
||||
|
||||
std::uint64_t present_calls()
|
||||
{
|
||||
return g_present_calls.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
std::uint64_t present_frames_shared()
|
||||
{
|
||||
return g_frames_shared.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
} // namespace coop::hook
|
||||
Reference in New Issue
Block a user