Files
CoopAllTheThings/hook/src/present_hook.cpp
BlackMark f843c56f5b Fix stale removal comments (persistent, not destroy) + add deterministic install test
- The remove_* comments still described the superseded "disable -> drain -> destroy"
  flow; the code keeps hooks alive (persistent) and re-enables on re-install. Updated
  the comments to match, and corrected the XInput note (its detours return synthesized
  state and never call the trampoline, so destroying its vector is safe -- unlike the
  trampoline-calling present/MKB/focus-cursor hooks).
- hook_install_test: a fast, single-threaded contract test for hook_install.hpp --
  install_inline creates the hook once and reuses the SAME trampoline across 50
  install/remove cycles (never freed -> no stale-detour UAF), toggling enable/disable
  cleanly. Fills the guard the removed (flaky, concurrency-bound) reproducer left, with
  no threads so it can't flake on SafetyHook's enable/disable atomicity.

x64 23/23, x86 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-23 11:51:48 +02:00

1038 lines
35 KiB
C++

#include "present_hook.hpp"
#include <atomic>
#include <mutex>
#include <windows.h>
#include <d3d10_1.h> // before d3d11on12.h (which also pulls it in); gives the ID3D10* types
#include <d3d11.h>
#include <d3d11on12.h>
#include <d3d12.h>
#include <dxgi1_2.h>
#include <dxgi1_4.h>
#include <safetyhook.hpp>
#include "coop/shared_memory.hpp"
#include "debug_log.hpp"
#include "hook_guard.hpp"
#include "hook_install.hpp"
#include "hook_registry.hpp"
namespace coop::hook
{
namespace
{
DetourGate g_gate; // drains in-flight Present/ECL detours before remove frees the shared state
// IDXGISwapChain vtable layout (frozen ABI). IUnknown 0..2, IDXGIObject 3..6,
// IDXGIDeviceSubObject 7, then IDXGISwapChain: Present = 8, GetBuffer = 9.
// IDXGISwapChain1 adds methods after IDXGISwapChain (18 methods, 0..17), so
// Present1 sits at index 22. Flip-model D3D11/12 games present via Present1.
constexpr unsigned kIdx_IDXGISwapChain_Present = 8;
constexpr unsigned kIdx_IDXGISwapChain1_Present1 = 22;
IpcClient* g_ipc = nullptr;
unsigned long g_pid = 0;
safetyhook::InlineHook g_hk_present;
safetyhook::InlineHook g_hk_present1;
int g_id_present = -1;
int g_id_present1 = -1;
std::atomic<std::uint64_t> g_present_calls{0};
std::atomic<std::uint64_t> g_frames_shared{0};
// The shared backbuffer copy and its keyed mutex, created lazily on the first
// Present once we can see the game's device + backbuffer format. Guarded by
// g_tex_mutex (touched only on the render thread, but install/remove may race).
std::mutex g_tex_mutex;
ID3D11Texture2D* g_shared_tex = nullptr;
IDXGIKeyedMutex* g_shared_mutex = nullptr;
HANDLE g_shared_handle = nullptr;
UINT g_share_w = 0;
UINT g_share_h = 0;
DXGI_FORMAT g_share_fmt = DXGI_FORMAT_UNKNOWN;
bool g_unsupported_logged = false;
// D3D11On12 bridge for D3D12 games: we build our own D3D11 device on top of the
// game's D3D12 device (with a queue we create on it) so we can wrap the D3D12
// backbuffer as a D3D11 resource and CopyResource it into the shared texture.
// Created lazily on the first D3D12 Present; guarded by g_tex_mutex.
ID3D11On12Device* g_on12 = nullptr;
ID3D11Device* g_on12_d3d11 = nullptr;
ID3D11DeviceContext* g_on12_ctx = nullptr;
ID3D12CommandQueue* g_on12_queue = nullptr; // our OWN copy queue (never the game's)
ID3D12Device* g_on12_d3d12 = nullptr;
// Cross-queue ordering fence, created on the game's device. The game's present queue
// signals it after the frame's render (a near-free operation); our own copy queue waits
// on it before copying. This orders our copy correctly *after* the frame WITHOUT putting
// any copy or keyed-mutex work on the game's present queue -- doing that stalled the
// game's own presents (GPU back-pressure, plus the keyed mutex is a CPU-blocking acquire
// on the render thread). Lifetime tied to the bridge; guarded by g_tex_mutex.
ID3D12Fence* g_copy_fence = nullptr;
UINT64 g_copy_fence_val = 0;
// D3D10 path state. A pure D3D10 game's backbuffer can be QI'd to ID3D11Texture2D, but that
// D3D11 view does NOT carry the content the D3D10 device rendered, so it must be read through
// the game's own D3D10 device: copy it into a D3D10 staging texture, Map it (which blocks until
// the GPU copy completes -> no race), and UpdateSubresource it into the shared keyed-mutex
// texture, which lives on a hook-owned D3D11 device (g_aux_d3d11) since the game has no usable
// D3D11 device of its own. The aux device is reusable by later GL/Vulkan paths. Guarded by
// g_tex_mutex.
ID3D11Device* g_aux_d3d11 = nullptr;
ID3D11DeviceContext* g_aux_ctx = nullptr;
ID3D10Texture2D* g_d3d10_staging = nullptr; // staging on the game's D3D10 device
ID3D10Device* g_d3d10_dev = nullptr; // the game device the staging belongs to
UINT g_d3d10_w = 0;
UINT g_d3d10_h = 0;
DXGI_FORMAT g_d3d10_fmt = DXGI_FORMAT_UNKNOWN;
bool g_force_d3d10 = false; // sticky: the game device rejected the shared texture -> D3D10 read-back
// ExecuteCommandLists hook: recovers the game's D3D12 *present* command queue so we can
// signal the ordering fence (above) on it. We hook the per-frame method rather than
// swapchain/queue creation, so it works for late injection (the queue already exists).
// ID3D12CommandQueue method index 10: IUnknown 0-2, ID3D12Object 3-6, ID3D12DeviceChild
// 7 (ID3D12Pageable adds none), then UpdateTileMappings 8, CopyTileMappings 9,
// ExecuteCommandLists 10.
constexpr unsigned kIdx_ID3D12CommandQueue_ExecuteCommandLists = 10;
safetyhook::InlineHook g_hk_ecl;
int g_id_ecl = -1;
// Last DIRECT (graphics) queue seen executing command lists. Present and its queue's
// ExecuteCommandLists run on the same render thread, so the thread-local is the most
// reliable match on engines with multiple DIRECT queues; the atomic is a cross-thread
// fallback (e.g. the first D3D12 Present beating any ExecuteCommandLists on its thread).
thread_local ID3D12CommandQueue* t_present_queue = nullptr;
std::atomic<ID3D12CommandQueue*> g_present_queue{nullptr};
// Present-pattern inventory. The first time we see each swapchain -- and each distinct
// present-flags value per swapchain -- we log it once, mapping how a game drives DXGI:
// how many swapchains it presents, their size/format, and how often it issues no-op
// DXGI_PRESENT_TEST presents (occlusion probes that draw nothing). That last point
// explains games whose raw Present count outpaces the frames actually captured, since a
// test present is counted but produces no frame. Render-thread only; small fixed tables.
constexpr int kMaxLoggedPresents = 16;
constexpr int kMaxLoggedSwapchains = 8;
struct LoggedPresent
{
void* swapchain;
UINT flags;
};
LoggedPresent g_logged_presents[kMaxLoggedPresents] = {};
int g_logged_presents_n = 0;
void* g_logged_swapchains[kMaxLoggedSwapchains] = {};
int g_logged_swapchains_n = 0;
// True the first time this (swapchain, flags) pair is presented, so the caller logs once.
bool first_present_with_flags(void* swapchain, UINT flags)
{
for (int i = 0; i < g_logged_presents_n; ++i)
{
if (g_logged_presents[i].swapchain == swapchain && g_logged_presents[i].flags == flags)
{
return false;
}
}
if (g_logged_presents_n >= kMaxLoggedPresents)
{
return false;
}
g_logged_presents[g_logged_presents_n++] = {swapchain, flags};
return true;
}
// True the first time this swapchain feeds the capture, so the caller logs it once.
bool first_capture_from(void* swapchain)
{
for (int i = 0; i < g_logged_swapchains_n; ++i)
{
if (g_logged_swapchains[i] == swapchain)
{
return false;
}
}
if (g_logged_swapchains_n >= kMaxLoggedSwapchains)
{
return false;
}
g_logged_swapchains[g_logged_swapchains_n++] = swapchain;
return true;
}
void* vtable_method(void* obj, unsigned index)
{
return (*reinterpret_cast<void***>(obj))[index];
}
// Drop the shared texture/mutex/handle. Caller holds g_tex_mutex.
void release_shared_locked()
{
if (g_shared_mutex != nullptr)
{
g_shared_mutex->Release();
g_shared_mutex = nullptr;
}
if (g_shared_tex != nullptr)
{
g_shared_tex->Release();
g_shared_tex = nullptr;
}
if (g_shared_handle != nullptr)
{
CloseHandle(g_shared_handle);
g_shared_handle = nullptr;
}
g_share_w = g_share_h = 0;
g_share_fmt = DXGI_FORMAT_UNKNOWN;
}
// (Re)create the shared keyed-mutex texture for a w x h `fmt` backbuffer on the
// game's `device`. Caller holds g_tex_mutex. Returns true if it's ready.
bool ensure_shared_texture_locked(ID3D11Device* device, UINT w, UINT h, DXGI_FORMAT fmt)
{
if (g_shared_tex != nullptr && g_share_w == w && g_share_h == h && g_share_fmt == fmt)
{
return true; // already matches the current backbuffer
}
release_shared_locked();
D3D11_TEXTURE2D_DESC desc{};
desc.Width = w;
desc.Height = h;
desc.MipLevels = 1;
desc.ArraySize = 1;
desc.Format = fmt;
desc.SampleDesc.Count = 1;
desc.Usage = D3D11_USAGE_DEFAULT;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET;
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
ID3D11Texture2D* tex = nullptr;
HRESULT hr = device->CreateTexture2D(&desc, nullptr, &tex);
if (FAILED(hr) || tex == nullptr)
{
logf("present: CreateTexture2D(shared) failed hr=0x%08lX (%ux%u fmt=%d)",
static_cast<unsigned long>(hr), w, h, static_cast<int>(fmt));
return false;
}
IDXGIResource1* res = nullptr;
hr = tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast<void**>(&res));
if (FAILED(hr) || res == nullptr)
{
logf("present: QI IDXGIResource1 failed hr=0x%08lX", static_cast<unsigned long>(hr));
tex->Release();
return false;
}
const std::wstring name = video_share_name(g_pid);
HANDLE handle = nullptr;
hr = res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE,
name.c_str(), &handle);
res->Release();
if (FAILED(hr) || handle == nullptr)
{
logf("present: CreateSharedHandle failed hr=0x%08lX", static_cast<unsigned long>(hr));
tex->Release();
return false;
}
IDXGIKeyedMutex* mutex = nullptr;
hr = tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast<void**>(&mutex));
if (FAILED(hr) || mutex == nullptr)
{
logf("present: QI IDXGIKeyedMutex failed hr=0x%08lX", static_cast<unsigned long>(hr));
CloseHandle(handle);
tex->Release();
return false;
}
g_shared_tex = tex;
g_shared_mutex = mutex;
g_shared_handle = handle;
g_share_w = w;
g_share_h = h;
g_share_fmt = fmt;
logf("present: shared texture ready %ux%u fmt=%d name=%ls", w, h, static_cast<int>(fmt), name.c_str());
return true;
}
// Drop the D3D11On12 bridge. Caller holds g_tex_mutex.
void release_on12_locked()
{
if (g_on12_ctx != nullptr)
{
g_on12_ctx->Release();
g_on12_ctx = nullptr;
}
if (g_on12 != nullptr)
{
g_on12->Release();
g_on12 = nullptr;
}
if (g_on12_d3d11 != nullptr)
{
g_on12_d3d11->Release();
g_on12_d3d11 = nullptr;
}
if (g_on12_queue != nullptr)
{
g_on12_queue->Release();
g_on12_queue = nullptr;
}
if (g_on12_d3d12 != nullptr)
{
g_on12_d3d12->Release();
g_on12_d3d12 = nullptr;
}
if (g_copy_fence != nullptr)
{
g_copy_fence->Release();
g_copy_fence = nullptr;
}
g_copy_fence_val = 0;
}
// Build the D3D11On12 device for the game's D3D12 `dev` on a copy queue of our own, plus
// a fence on `dev` for ordering our copy after the game's frame (signaled by the game's
// present queue in capture_backbuffer_d3d12, waited by our queue). Keeping the copy on
// our own queue is what stops our work from stalling the game's present queue. Caller
// holds g_tex_mutex. Returns true when the bridge is ready.
bool ensure_on12_locked(ID3D12Device* dev)
{
if (g_on12 != nullptr && g_on12_d3d12 == dev)
{
return true;
}
release_on12_locked();
D3D12_COMMAND_QUEUE_DESC qd{};
qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
ID3D12CommandQueue* queue = nullptr;
HRESULT hr = dev->CreateCommandQueue(&qd, __uuidof(ID3D12CommandQueue), reinterpret_cast<void**>(&queue));
if (FAILED(hr) || queue == nullptr)
{
logf("present(d3d12): CreateCommandQueue failed hr=0x%08lX", static_cast<unsigned long>(hr));
return false;
}
IUnknown* queues[] = {queue};
ID3D11Device* d11 = nullptr;
ID3D11DeviceContext* ctx = nullptr;
hr = D3D11On12CreateDevice(dev, 0, nullptr, 0, queues, 1, 0, &d11, &ctx, nullptr);
if (FAILED(hr) || d11 == nullptr)
{
logf("present(d3d12): D3D11On12CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
queue->Release();
return false;
}
ID3D11On12Device* on12 = nullptr;
hr = d11->QueryInterface(__uuidof(ID3D11On12Device), reinterpret_cast<void**>(&on12));
if (FAILED(hr) || on12 == nullptr)
{
logf("present(d3d12): QI ID3D11On12Device failed hr=0x%08lX", static_cast<unsigned long>(hr));
if (ctx != nullptr)
{
ctx->Release();
}
d11->Release();
queue->Release();
return false;
}
// Ordering fence on the game's device. Non-fatal if it fails -- we simply lose the
// cross-queue ordering (the copy can then race the frame, the pre-fence behavior).
ID3D12Fence* fence = nullptr;
hr = dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), reinterpret_cast<void**>(&fence));
if (FAILED(hr) || fence == nullptr)
{
logf("present(d3d12): CreateFence failed hr=0x%08lX (copy will be unordered)",
static_cast<unsigned long>(hr));
fence = nullptr;
}
g_on12 = on12;
g_on12_d3d11 = d11;
g_on12_ctx = ctx;
g_on12_queue = queue;
g_on12_d3d12 = dev;
g_copy_fence = fence;
g_copy_fence_val = 0;
dev->AddRef(); // we hold a reference for the lifetime of the bridge
logf("present(d3d12): D3D11On12 bridge ready (own copy queue, fence=%d)", fence != nullptr ? 1 : 0);
return true;
}
// D3D12 backbuffer path: wrap the game's D3D12 swapchain buffer as a D3D11 resource
// via the On12 bridge and CopyResource it into the shared texture.
void capture_backbuffer_d3d12(IDXGISwapChain* sc)
{
// CRITICAL: D3D12 rotates back buffers explicitly -- the game rendered into the
// buffer at GetCurrentBackBufferIndex(), and that index advances every Present.
// (Unlike D3D11 flip-model, where DXGI keeps GetBuffer(0) pointing at the live
// back buffer.) Grabbing buffer 0 unconditionally copies a stale buffer on N-1 of
// every N frames, so the mirror silently runs at refresh/N with duplicate frames
// in between -- even though the Present counter, published FPS, generation bump,
// and latency all read full rate (they count Presents, not unique content). We're
// called before the trampoline Present, so the current index is the just-rendered
// buffer. Query IDXGISwapChain3 for it; fall back to 0 only if unavailable.
UINT bb_index = 0;
IDXGISwapChain3* sc3 = nullptr;
if (SUCCEEDED(sc->QueryInterface(__uuidof(IDXGISwapChain3), reinterpret_cast<void**>(&sc3))) && sc3 != nullptr)
{
bb_index = sc3->GetCurrentBackBufferIndex();
sc3->Release();
}
ID3D12Resource* bb = nullptr;
if (FAILED(sc->GetBuffer(bb_index, __uuidof(ID3D12Resource), reinterpret_cast<void**>(&bb))) || bb == nullptr)
{
if (!g_unsupported_logged)
{
logf("present: backbuffer is neither ID3D11Texture2D nor ID3D12Resource (D3D9/Vulkan?); idle");
g_unsupported_logged = true;
}
return;
}
ID3D12Device* dev = nullptr;
bb->GetDevice(__uuidof(ID3D12Device), reinterpret_cast<void**>(&dev));
bool shared = false;
bool dropped = false;
UINT w = 0, h = 0;
DXGI_FORMAT fmt = DXGI_FORMAT_UNKNOWN;
// The game's present queue, preferring the one seen on this (the render) thread.
ID3D12CommandQueue* game_queue = t_present_queue;
if (game_queue == nullptr)
{
game_queue = g_present_queue.load(std::memory_order_relaxed);
}
// Log each distinct swapchain feeding the capture once (size/format/buffer index).
if (first_capture_from(sc))
{
const D3D12_RESOURCE_DESC rd = bb->GetDesc();
logf("present: swapchain=%p capturing D3D12 backbuffer %llux%u fmt=%d samples=%u bufferindex=%u queue=%s",
sc, static_cast<unsigned long long>(rd.Width), rd.Height, static_cast<int>(rd.Format),
rd.SampleDesc.Count, bb_index, game_queue != nullptr ? "known" : "unknown");
}
if (dev != nullptr)
{
std::scoped_lock lock(g_tex_mutex);
if (ensure_on12_locked(dev))
{
// DX12 capture is pricier than DX11/OpenGL (measured ~0.38 ms vs ~0.05/0.09 ms of
// present-thread overhead) because it goes through the D3D11On12 bridge. The cost is NOT
// CreateWrappedResource (measured ~0.012 ms) -- it's the CopyResource issued on the 11On12
// immediate context (~0.13 ms) plus the mandatory Flush to make the shared copy visible to
// the host (~0.06 ms), neither of which the native-D3D11 path pays. Eliminating it needs a
// native-D3D12 copy-queue path into a D3D12-shared texture, but the host consumes the
// shared surface via an IDXGIKeyedMutex (a D3D11 concept), so that also means switching the
// DX12 producer<->host sync to a shared ID3D12Fence -- a cross-API rewrite. Deferred: the
// overhead is ~5% of a 144 Hz frame and the capture is correct; the bridge stays for now.
//
// Order our copy after the game's frame without burdening the game's queue: the
// game queue signals the fence (cheap), our copy queue waits on it. Skipped if the
// queue isn't captured yet or the fence is missing (one possibly-early frame).
if (game_queue != nullptr && g_copy_fence != nullptr)
{
const UINT64 fence_val = ++g_copy_fence_val;
game_queue->Signal(g_copy_fence, fence_val);
g_on12_queue->Wait(g_copy_fence, fence_val);
}
D3D11_RESOURCE_FLAGS rf{};
rf.BindFlags = D3D11_BIND_RENDER_TARGET;
ID3D11Resource* wrapped = nullptr;
HRESULT hr = g_on12->CreateWrappedResource(bb, &rf, D3D12_RESOURCE_STATE_PRESENT,
D3D12_RESOURCE_STATE_PRESENT, __uuidof(ID3D11Resource),
reinterpret_cast<void**>(&wrapped));
if (SUCCEEDED(hr) && wrapped != nullptr)
{
g_on12->AcquireWrappedResources(&wrapped, 1);
ID3D11Texture2D* wtex = nullptr;
if (SUCCEEDED(wrapped->QueryInterface(__uuidof(ID3D11Texture2D),
reinterpret_cast<void**>(&wtex))) &&
wtex != nullptr)
{
D3D11_TEXTURE2D_DESC d{};
wtex->GetDesc(&d);
w = d.Width;
h = d.Height;
fmt = d.Format;
if (d.SampleDesc.Count == 1 && ensure_shared_texture_locked(g_on12_d3d11, w, h, fmt) &&
g_shared_mutex != nullptr)
{
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 0) == S_OK)
{
g_on12_ctx->CopyResource(g_shared_tex, wtex);
g_shared_mutex->ReleaseSync(kVideoMutexKey);
shared = true;
}
else
{
dropped = true; // host held the mutex -> this frame never reaches the mirror
}
}
wtex->Release();
}
g_on12->ReleaseWrappedResources(&wrapped, 1);
g_on12_ctx->Flush();
wrapped->Release();
}
else if (!g_unsupported_logged)
{
logf("present(d3d12): CreateWrappedResource failed hr=0x%08lX", static_cast<unsigned long>(hr));
g_unsupported_logged = true;
}
}
}
if (shared)
{
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->publish_video_frame(w, h, static_cast<std::uint32_t>(fmt));
}
}
else if (dropped && g_ipc != nullptr)
{
g_ipc->note_video_dropped();
}
if (dev != nullptr)
{
dev->Release();
}
bb->Release();
}
// Drop the hook-owned D3D11 device and the D3D10 staging texture. Caller holds g_tex_mutex.
void release_aux_locked()
{
if (g_d3d10_staging != nullptr)
{
g_d3d10_staging->Release();
g_d3d10_staging = nullptr;
}
if (g_d3d10_dev != nullptr)
{
g_d3d10_dev->Release();
g_d3d10_dev = nullptr;
}
g_d3d10_w = g_d3d10_h = 0;
g_d3d10_fmt = DXGI_FORMAT_UNKNOWN;
g_force_d3d10 = false;
if (g_aux_ctx != nullptr)
{
g_aux_ctx->Release();
g_aux_ctx = nullptr;
}
if (g_aux_d3d11 != nullptr)
{
g_aux_d3d11->Release();
g_aux_d3d11 = nullptr;
}
}
// Create the hook-owned D3D11 device that backs the shared texture for non-D3D11 games
// (the game has no D3D11 device of its own). Caller holds g_tex_mutex.
bool ensure_aux_d3d11_locked()
{
if (g_aux_d3d11 != nullptr)
{
return true;
}
const D3D_FEATURE_LEVEL levels[] = {D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_10_0};
HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, levels,
static_cast<UINT>(std::size(levels)), D3D11_SDK_VERSION, &g_aux_d3d11, nullptr,
&g_aux_ctx);
if (FAILED(hr) || g_aux_d3d11 == nullptr)
{
logf("present(d3d10): aux D3D11CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
g_aux_d3d11 = nullptr;
g_aux_ctx = nullptr;
return false;
}
return true;
}
// D3D10 backbuffer path: read it back through a D3D10 staging texture (Map blocks until the
// game's GPU copy completes -> no race) and UpdateSubresource it into the shared keyed-mutex
// texture on the hook-owned D3D11 device. Reached by trying ID3D10Texture2D *before*
// ID3D11Texture2D, because a D3D10 backbuffer's ID3D11 view doesn't carry the rendered content.
void capture_backbuffer_d3d10(IDXGISwapChain* sc, ID3D10Texture2D* backbuf)
{
D3D10_TEXTURE2D_DESC bd{};
backbuf->GetDesc(&bd);
if (first_capture_from(sc))
{
logf("present: swapchain=%p capturing D3D10 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width, bd.Height,
static_cast<int>(bd.Format), bd.SampleDesc.Count);
}
if (bd.SampleDesc.Count != 1)
{
return; // MSAA: would need ResolveSubresource; skip rather than mis-copy
}
ID3D10Device* gdev = nullptr;
backbuf->GetDevice(&gdev);
if (gdev == nullptr)
{
return;
}
bool shared = false;
bool dropped = false;
{
std::scoped_lock lock(g_tex_mutex);
if (!(g_d3d10_staging != nullptr && g_d3d10_dev == gdev && g_d3d10_w == bd.Width &&
g_d3d10_h == bd.Height && g_d3d10_fmt == bd.Format))
{
if (g_d3d10_staging != nullptr)
{
g_d3d10_staging->Release();
g_d3d10_staging = nullptr;
}
if (g_d3d10_dev != nullptr)
{
g_d3d10_dev->Release();
g_d3d10_dev = nullptr;
}
D3D10_TEXTURE2D_DESC sd{};
sd.Width = bd.Width;
sd.Height = bd.Height;
sd.MipLevels = 1;
sd.ArraySize = 1;
sd.Format = bd.Format;
sd.SampleDesc.Count = 1;
sd.Usage = D3D10_USAGE_STAGING;
sd.CPUAccessFlags = D3D10_CPU_ACCESS_READ;
if (SUCCEEDED(gdev->CreateTexture2D(&sd, nullptr, &g_d3d10_staging)) && g_d3d10_staging != nullptr)
{
g_d3d10_dev = gdev;
gdev->AddRef();
g_d3d10_w = bd.Width;
g_d3d10_h = bd.Height;
g_d3d10_fmt = bd.Format;
}
}
if (g_d3d10_staging != nullptr && ensure_aux_d3d11_locked() &&
ensure_shared_texture_locked(g_aux_d3d11, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr)
{
gdev->CopyResource(g_d3d10_staging, backbuf);
D3D10_MAPPED_TEXTURE2D m{};
if (SUCCEEDED(g_d3d10_staging->Map(0, D3D10_MAP_READ, 0, &m)) && m.pData != nullptr)
{
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
{
g_aux_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, m.pData, m.RowPitch, 0);
g_shared_mutex->ReleaseSync(kVideoMutexKey);
shared = true;
}
else
{
dropped = true;
}
g_d3d10_staging->Unmap(0);
}
}
}
if (shared)
{
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->publish_video_frame(bd.Width, bd.Height, static_cast<std::uint32_t>(bd.Format));
}
}
else if (dropped && g_ipc != nullptr)
{
g_ipc->note_video_dropped();
}
gdev->Release();
}
// Copy the swapchain's backbuffer into the shared texture and publish it.
void capture_backbuffer(IDXGISwapChain* sc)
{
// Fast path: a normal D3D11 (feature level 11.1+) game, whose device can host the shared
// texture, so a single CopyResource publishes the frame. Note a D3D10 backbuffer ALSO QIs to
// ID3D11Texture2D (so we can't discriminate by GetBuffer), but its feature-level-10 device
// rejects the share flags -- so a failed shared-texture creation is the signal to switch
// (sticky) to the D3D10 read-back path, which reads through the game's own D3D10 device.
if (!g_force_d3d10)
{
ID3D11Texture2D* backbuf = nullptr;
if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(&backbuf))) ||
backbuf == nullptr)
{
capture_backbuffer_d3d12(sc); // D3D12 game: bridge via D3D11On12 (or idle if neither)
return;
}
D3D11_TEXTURE2D_DESC bd{};
backbuf->GetDesc(&bd);
bool shared = false;
bool dropped = false;
bool cant_host = false;
if (bd.SampleDesc.Count != 1)
{
backbuf->Release(); // MSAA would need ResolveSubresource; skip rather than mis-copy
return;
}
ID3D11Device* device = nullptr;
backbuf->GetDevice(&device);
ID3D11DeviceContext* ctx = nullptr;
if (device != nullptr)
{
device->GetImmediateContext(&ctx);
}
if (device != nullptr && ctx != nullptr)
{
std::scoped_lock lock(g_tex_mutex);
if (ensure_shared_texture_locked(device, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr)
{
if (first_capture_from(sc))
{
logf("present: swapchain=%p capturing D3D11 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width,
bd.Height, static_cast<int>(bd.Format), bd.SampleDesc.Count);
}
// Key 0 on both sides: a plain cross-process mutex on the texture (created
// released at key 0). Bounded wait so a stalled host consumer can never hang
// the game's render thread.
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
{
ctx->CopyResource(g_shared_tex, backbuf);
g_shared_mutex->ReleaseSync(kVideoMutexKey);
shared = true;
}
else
{
dropped = true; // host held the mutex past the wait -> frame lost (rare on D3D11)
}
}
else
{
cant_host = true; // device can't host the shared texture -> try the D3D10 path
release_shared_locked();
}
}
if (ctx != nullptr)
{
ctx->Release();
}
if (device != nullptr)
{
device->Release();
}
backbuf->Release();
if (shared)
{
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->publish_video_frame(bd.Width, bd.Height, static_cast<std::uint32_t>(bd.Format));
}
return;
}
if (!cant_host)
{
if (dropped && g_ipc != nullptr)
{
g_ipc->note_video_dropped();
}
return; // captured-or-dropped on the D3D11 path; nothing else to try this frame
}
g_force_d3d10 = true; // switch (sticky) to the D3D10 read-back path
logf("present: game device can't host the shared texture -> D3D10 read-back path");
// fall through to the D3D10 path below
}
// D3D10 game: its backbuffer must be read through its own D3D10 device.
ID3D10Texture2D* bb10 = nullptr;
if (SUCCEEDED(sc->GetBuffer(0, __uuidof(ID3D10Texture2D), reinterpret_cast<void**>(&bb10))) && bb10 != nullptr)
{
capture_backbuffer_d3d10(sc, bb10);
bb10->Release();
}
}
void STDMETHODCALLTYPE hk_ExecuteCommandLists(ID3D12CommandQueue* queue, UINT num_lists,
ID3D12CommandList* const* lists)
{
DetourGate::Guard guard(g_gate); // keep g_present_queue/g_hk_ecl alive for this detour
// Record the graphics queue; compute/copy queues never present, so skip them and
// keep the last DIRECT one (the present queue on single-graphics-queue engines).
if (queue != nullptr && queue->GetDesc().Type == D3D12_COMMAND_LIST_TYPE_DIRECT)
{
t_present_queue = queue;
g_present_queue.store(queue, std::memory_order_relaxed);
hook_note_call(g_id_ecl);
}
g_hk_ecl.stdcall<void>(queue, num_lists, lists); // __stdcall, see hk_Present
}
// Create a throwaway D3D12 device + command queue to read the address of
// ID3D12CommandQueue::ExecuteCommandLists, so we can inline-hook it (catching the
// game's pre-existing queues regardless of when we injected). Resolves D3D12CreateDevice
// dynamically: only D3D12 games have d3d12.dll loaded, and we never want to force-load
// it into a D3D11 game. Returns null when D3D12 isn't present.
void* grab_execute_command_lists_address()
{
HMODULE d3d12 = GetModuleHandleW(L"d3d12.dll");
if (d3d12 == nullptr)
{
return nullptr; // not a D3D12 game -> nothing to capture
}
using PFN_D3D12_CREATE_DEVICE = HRESULT(WINAPI*)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**);
auto create = reinterpret_cast<PFN_D3D12_CREATE_DEVICE>(GetProcAddress(d3d12, "D3D12CreateDevice"));
if (create == nullptr)
{
return nullptr;
}
ID3D12Device* dev = nullptr;
if (FAILED(create(nullptr, D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), reinterpret_cast<void**>(&dev))) ||
dev == nullptr)
{
return nullptr;
}
D3D12_COMMAND_QUEUE_DESC qd{};
qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
ID3D12CommandQueue* queue = nullptr;
void* addr = nullptr;
if (SUCCEEDED(dev->CreateCommandQueue(&qd, __uuidof(ID3D12CommandQueue), reinterpret_cast<void**>(&queue))) &&
queue != nullptr)
{
addr = vtable_method(queue, kIdx_ID3D12CommandQueue_ExecuteCommandLists);
queue->Release();
}
dev->Release();
return addr;
}
HRESULT STDMETHODCALLTYPE hk_Present(IDXGISwapChain* sc, UINT sync_interval, UINT flags)
{
DetourGate::Guard guard(g_gate); // keep the shared texture / On12 bridge alive for this detour
hook_note_call(g_id_present);
g_present_calls.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->note_present();
}
// Log each distinct (swapchain, flags) once. A DXGI_PRESENT_TEST present draws nothing
// (it only probes occlusion), so those inflate the Present count without producing a
// frame -- which is why some games show more presents than captured frames.
if (first_present_with_flags(sc, flags))
{
logf("present: swapchain=%p Present flags=0x%08X%s", sc, flags,
(flags & DXGI_PRESENT_TEST) ? " (DXGI_PRESENT_TEST: occlusion probe, no frame drawn)" : "");
}
// DXGI_PRESENT_TEST presents nothing; don't bother copying for it.
if ((flags & DXGI_PRESENT_TEST) == 0)
{
capture_backbuffer(sc);
}
// stdcall(), NOT call(): IDXGISwapChain::Present is __stdcall, but SafetyHook's
// call() invokes the trampoline through a __cdecl pointer (the default on x86).
// On 32-bit that double-cleans the stack -> ESP imbalance -> Run-Time Check
// Failure #0 and an instant crash. On x64 the conventions collapse, so it only
// bit 32-bit games (e.g. Slaps and Beans froze the moment it presented).
return g_hk_present.stdcall<HRESULT>(sc, sync_interval, flags);
}
HRESULT STDMETHODCALLTYPE hk_Present1(IDXGISwapChain1* sc, UINT sync_interval, UINT flags,
const DXGI_PRESENT_PARAMETERS* params)
{
DetourGate::Guard guard(g_gate); // keep the shared texture / On12 bridge alive for this detour
hook_note_call(g_id_present1);
g_present_calls.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->note_present();
}
if (first_present_with_flags(sc, flags)) // see hk_Present
{
logf("present: swapchain=%p Present1 flags=0x%08X%s", sc, flags,
(flags & DXGI_PRESENT_TEST) ? " (DXGI_PRESENT_TEST: occlusion probe, no frame drawn)" : "");
}
if ((flags & DXGI_PRESENT_TEST) == 0)
{
capture_backbuffer(sc); // IDXGISwapChain1 derives from IDXGISwapChain
}
return g_hk_present1.stdcall<HRESULT>(sc, sync_interval, flags, params); // __stdcall, see hk_Present
}
// Create a throwaway device + swapchain purely to read IDXGISwapChain::Present
// (and IDXGISwapChain1::Present1) addresses, so we can inline-hook them. The
// inline hook patches the function itself, so it catches every swapchain in the
// process -- we can release the dummy after. `present1_out` may stay null on
// platforms without DXGI 1.2.
void* grab_present_address(void** present1_out)
{
*present1_out = nullptr;
WNDCLASSEXW wc{};
wc.cbSize = sizeof(wc);
wc.lpfnWndProc = DefWindowProcW;
wc.hInstance = GetModuleHandleW(nullptr);
wc.lpszClassName = L"coop_present_probe";
RegisterClassExW(&wc);
HWND hwnd = CreateWindowExW(0, wc.lpszClassName, L"", WS_OVERLAPPEDWINDOW, 0, 0, 8, 8, nullptr, nullptr,
wc.hInstance, nullptr);
if (hwnd == nullptr)
{
return nullptr;
}
DXGI_SWAP_CHAIN_DESC scd{};
scd.BufferCount = 1;
scd.BufferDesc.Width = 8;
scd.BufferDesc.Height = 8;
scd.BufferDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
scd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
scd.OutputWindow = hwnd;
scd.SampleDesc.Count = 1;
scd.Windowed = TRUE;
scd.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
IDXGISwapChain* swapchain = nullptr;
ID3D11Device* device = nullptr;
ID3D11DeviceContext* ctx = nullptr;
const HRESULT hr = D3D11CreateDeviceAndSwapChain(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0,
D3D11_SDK_VERSION, &scd, &swapchain, &device, nullptr, &ctx);
void* present = nullptr;
if (SUCCEEDED(hr) && swapchain != nullptr)
{
present = vtable_method(swapchain, kIdx_IDXGISwapChain_Present);
IDXGISwapChain1* sc1 = nullptr;
if (SUCCEEDED(swapchain->QueryInterface(__uuidof(IDXGISwapChain1), reinterpret_cast<void**>(&sc1))) &&
sc1 != nullptr)
{
*present1_out = vtable_method(sc1, kIdx_IDXGISwapChain1_Present1);
sc1->Release();
}
}
else
{
logf("present: D3D11CreateDeviceAndSwapChain(probe) failed hr=0x%08lX", static_cast<unsigned long>(hr));
}
if (ctx != nullptr)
{
ctx->Release();
}
if (device != nullptr)
{
device->Release();
}
if (swapchain != nullptr)
{
swapchain->Release();
}
DestroyWindow(hwnd);
UnregisterClassW(wc.lpszClassName, wc.hInstance);
return present;
}
} // namespace
bool install_present_hooks(IpcClient& ipc)
{
g_ipc = &ipc;
g_pid = GetCurrentProcessId();
if (g_hk_present.enabled())
{
return true; // already installed (persistent hook; the re-install path below re-enables it)
}
g_id_present = hook_register("IDXGISwapChain::Present", HookSubsys_Video);
g_id_present1 = hook_register("IDXGISwapChain1::Present1", HookSubsys_Video);
g_id_ecl = hook_register("ID3D12CommandQueue::ExecuteCommandLists", HookSubsys_Video);
void* present1 = nullptr;
void* present = grab_present_address(&present1);
if (present == nullptr)
{
hook_set_installed(g_id_present, false);
hook_set_installed(g_id_present1, false);
return false;
}
install_inline(g_hk_present, present, &hk_Present);
if (present1 != nullptr)
{
install_inline(g_hk_present1, present1, &hk_Present1);
}
g_unsupported_logged = false;
hook_set_installed(g_id_present, static_cast<bool>(g_hk_present));
hook_set_installed(g_id_present1, static_cast<bool>(g_hk_present1));
logf("install_present_hooks: present=%p hooked=%d present1=%p hooked=%d", present,
static_cast<bool>(g_hk_present) ? 1 : 0, present1, static_cast<bool>(g_hk_present1) ? 1 : 0);
// Capture the game's D3D12 present queue (D3D12 games only; null otherwise). Done
// here at injection time -- d3d12.dll is already loaded in a running D3D12 game --
// so the queue is recovered even though we attached after it was created.
void* ecl = grab_execute_command_lists_address();
if (ecl != nullptr)
{
install_inline(g_hk_ecl, ecl, &hk_ExecuteCommandLists);
hook_set_installed(g_id_ecl, static_cast<bool>(g_hk_ecl));
logf("install_present_hooks: d3d12 ExecuteCommandLists=%p hooked=%d", ecl,
static_cast<bool>(g_hk_ecl) ? 1 : 0);
}
else
{
hook_set_installed(g_id_ecl, false); // not a D3D12 game; On12 path uses its own queue
}
return static_cast<bool>(g_hk_present);
}
void remove_present_hooks()
{
// DISABLE (persistent model -- never destroy during the session): this restores the
// Present/Present1/ECL bytes under thread suspension so no NEW detour starts, while KEEPING the
// trampolines alive -- an in-flight detour about to call g_hk_present.stdcall() (the trampoline)
// must never have it freed under it (destroying = {} would, an AV the thousands/s storm hits
// reliably). So: disable -> drain (in-flight detours finish on the live trampoline) -> leave the
// hooks alive (re-install re-enables them; see hook_install.hpp).
disable_for_removal(g_hk_present);
disable_for_removal(g_hk_present1);
disable_for_removal(g_hk_ecl);
hook_set_installed(g_id_present, false);
hook_set_installed(g_id_present1, false);
hook_set_installed(g_id_ecl, false);
g_gate.drain();
// Persistent hooks: keep g_hk_present/present1/ecl ALIVE (disabled), so the trampoline a stale
// detour may still call is never freed -- re-install re-enables them (see hook_install.hpp).
g_present_queue.store(nullptr, std::memory_order_relaxed);
g_logged_presents_n = 0; // let a fresh injection re-log the present pattern
g_logged_swapchains_n = 0;
{
std::scoped_lock lock(g_tex_mutex);
release_shared_locked();
release_on12_locked();
release_aux_locked();
}
g_present_calls.store(0, std::memory_order_relaxed);
g_frames_shared.store(0, std::memory_order_relaxed);
g_ipc = nullptr;
}
std::uint64_t present_calls()
{
return g_present_calls.load(std::memory_order_relaxed);
}
std::uint64_t present_frames_shared()
{
return g_frames_shared.load(std::memory_order_relaxed);
}
} // namespace coop::hook