BlackMark/CoopAllTheThings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add DX12 hooked capture via a D3D11On12 bridge

Browse Source 
D3D12 games (e.g. Spider-Man: Miles Morales) fire the Present hook -- the DXGI
swapchain's Present is the same vtable function for D3D11 and D3D12 -- but
GetBuffer(0) as ID3D11Texture2D fails, so the hook used to idle. Now, when the D3D11
GetBuffer fails, present_hook bridges via D3D11On12: it gets the game's ID3D12Device
from the backbuffer, creates its own DIRECT command queue on it (no need to hook the
game's ExecuteCommandLists), builds an ID3D11On12Device, CreateWrappedResource's the
D3D12 backbuffer, and CopyResource's it into the existing shared keyed-mutex texture
-- so the host side is unchanged. The bridge is created lazily and torn down with the
hook.

Verified with a new in-process dx12_present_hook_test: it drives a real D3D12
swapchain (clears a backbuffer, Presents) and asserts present fired, the backbuffer
was bridged into the shared texture, and a second device reads the exact color back
by name -- {51,102,153,255}. Full build x64 + x86 clean (the x86 hook compiles the
D3D12 path too); ctest x64 10/10, x86 3/3.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
BlackMark
2026-06-21 05:32:15 +02:00
parent 5e05d38be8
commit dfd2be8c17
4 changed files with 513 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
hook/src/present_hook.cpp
View File

@@ -6,6 +6,8 @@
#include <windows.h>
#include <d3d11.h>
#include <d3d11on12.h>
#include <d3d12.h>
#include <dxgi1_2.h>
#include <safetyhook.hpp>
@@ -50,6 +52,16 @@ UINT g_share_h = 0;
DXGI_FORMAT g_share_fmt = DXGI_FORMAT_UNKNOWN;
bool g_unsupported_logged = false;
// D3D11On12 bridge for D3D12 games: we build our own D3D11 device on top of the
// game's D3D12 device (with a queue we create on it) so we can wrap the D3D12
// backbuffer as a D3D11 resource and CopyResource it into the shared texture.
// Created lazily on the first D3D12 Present; guarded by g_tex_mutex.
ID3D11On12Device* g_on12 = nullptr;
ID3D11Device* g_on12_d3d11 = nullptr;
ID3D11DeviceContext* g_on12_ctx = nullptr;
ID3D12CommandQueue* g_on12_queue = nullptr;
ID3D12Device* g_on12_d3d12 = nullptr;
void* vtable_method(void* obj, unsigned index)
{
return (*reinterpret_cast<void***>(obj))[index];
@@ -147,6 +159,174 @@ bool ensure_shared_texture_locked(ID3D11Device* device, UINT w, UINT h, DXGI_FOR
return true;
}
// Drop the D3D11On12 bridge. Caller holds g_tex_mutex.
void release_on12_locked()
{
if (g_on12_ctx != nullptr)
{
g_on12_ctx->Release();
g_on12_ctx = nullptr;
}
if (g_on12 != nullptr)
{
g_on12->Release();
g_on12 = nullptr;
}
if (g_on12_d3d11 != nullptr)
{
g_on12_d3d11->Release();
g_on12_d3d11 = nullptr;
}
if (g_on12_queue != nullptr)
{
g_on12_queue->Release();
g_on12_queue = nullptr;
}
if (g_on12_d3d12 != nullptr)
{
g_on12_d3d12->Release();
g_on12_d3d12 = nullptr;
}
}
// Build the D3D11On12 device for the game's D3D12 `dev` (creating our own DIRECT
// queue on it). Caller holds g_tex_mutex. Returns true when the bridge is ready.
bool ensure_on12_locked(ID3D12Device* dev)
{
if (g_on12 != nullptr && g_on12_d3d12 == dev)
{
return true;
}
release_on12_locked();
D3D12_COMMAND_QUEUE_DESC qd{};
qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
ID3D12CommandQueue* queue = nullptr;
HRESULT hr = dev->CreateCommandQueue(&qd, __uuidof(ID3D12CommandQueue), reinterpret_cast<void**>(&queue));
if (FAILED(hr) || queue == nullptr)
{
logf("present(d3d12): CreateCommandQueue failed hr=0x%08lX", static_cast<unsigned long>(hr));
return false;
}
IUnknown* queues[] = {queue};
ID3D11Device* d11 = nullptr;
ID3D11DeviceContext* ctx = nullptr;
hr = D3D11On12CreateDevice(dev, 0, nullptr, 0, queues, 1, 0, &d11, &ctx, nullptr);
if (FAILED(hr) || d11 == nullptr)
{
logf("present(d3d12): D3D11On12CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
queue->Release();
return false;
}
ID3D11On12Device* on12 = nullptr;
hr = d11->QueryInterface(__uuidof(ID3D11On12Device), reinterpret_cast<void**>(&on12));
if (FAILED(hr) || on12 == nullptr)
{
logf("present(d3d12): QI ID3D11On12Device failed hr=0x%08lX", static_cast<unsigned long>(hr));
if (ctx != nullptr)
{
ctx->Release();
}
d11->Release();
queue->Release();
return false;
}
g_on12 = on12;
g_on12_d3d11 = d11;
g_on12_ctx = ctx;
g_on12_queue = queue;
g_on12_d3d12 = dev;
dev->AddRef(); // we hold a reference for the lifetime of the bridge
logf("present(d3d12): D3D11On12 bridge ready");
return true;
}
// D3D12 backbuffer path: wrap the game's D3D12 swapchain buffer as a D3D11 resource
// via the On12 bridge and CopyResource it into the shared texture.
void capture_backbuffer_d3d12(IDXGISwapChain* sc)
{
ID3D12Resource* bb = nullptr;
if (FAILED(sc->GetBuffer(0, __uuidof(ID3D12Resource), reinterpret_cast<void**>(&bb))) || bb == nullptr)
{
if (!g_unsupported_logged)
{
logf("present: backbuffer is neither ID3D11Texture2D nor ID3D12Resource (D3D9/Vulkan?); idle");
g_unsupported_logged = true;
}
return;
}
ID3D12Device* dev = nullptr;
bb->GetDevice(__uuidof(ID3D12Device), reinterpret_cast<void**>(&dev));
bool shared = false;
UINT w = 0, h = 0;
DXGI_FORMAT fmt = DXGI_FORMAT_UNKNOWN;
if (dev != nullptr)
{
std::scoped_lock lock(g_tex_mutex);
if (ensure_on12_locked(dev))
{
D3D11_RESOURCE_FLAGS rf{};
rf.BindFlags = D3D11_BIND_RENDER_TARGET;
ID3D11Resource* wrapped = nullptr;
HRESULT hr = g_on12->CreateWrappedResource(bb, &rf, D3D12_RESOURCE_STATE_PRESENT,
D3D12_RESOURCE_STATE_PRESENT, __uuidof(ID3D11Resource),
reinterpret_cast<void**>(&wrapped));
if (SUCCEEDED(hr) && wrapped != nullptr)
{
g_on12->AcquireWrappedResources(&wrapped, 1);
ID3D11Texture2D* wtex = nullptr;
if (SUCCEEDED(wrapped->QueryInterface(__uuidof(ID3D11Texture2D),
reinterpret_cast<void**>(&wtex))) &&
wtex != nullptr)
{
D3D11_TEXTURE2D_DESC d{};
wtex->GetDesc(&d);
w = d.Width;
h = d.Height;
fmt = d.Format;
if (d.SampleDesc.Count == 1 && ensure_shared_texture_locked(g_on12_d3d11, w, h, fmt) &&
g_shared_mutex != nullptr)
{
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
{
g_on12_ctx->CopyResource(g_shared_tex, wtex);
g_shared_mutex->ReleaseSync(kVideoMutexKey);
shared = true;
}
}
wtex->Release();
}
g_on12->ReleaseWrappedResources(&wrapped, 1);
g_on12_ctx->Flush();
wrapped->Release();
}
else if (!g_unsupported_logged)
{
logf("present(d3d12): CreateWrappedResource failed hr=0x%08lX", static_cast<unsigned long>(hr));
g_unsupported_logged = true;
}
}
}
if (shared)
{
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->publish_video_frame(w, h, static_cast<std::uint32_t>(fmt));
}
}
if (dev != nullptr)
{
dev->Release();
}
bb->Release();
}
// Copy the swapchain's backbuffer into the shared texture and publish it.
void capture_backbuffer(IDXGISwapChain* sc)
{
@@ -154,11 +334,7 @@ void capture_backbuffer(IDXGISwapChain* sc)
if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(&backbuf))) ||
backbuf == nullptr)
{
if (!g_unsupported_logged)
{
logf("present: backbuffer is not an ID3D11Texture2D (D3D12/D3D9?); video hook idle");
g_unsupported_logged = true;
}
capture_backbuffer_d3d12(sc); // D3D12 game: bridge via D3D11On12 (or idle if neither)
return;
}
@@ -370,6 +546,7 @@ void remove_present_hooks()
{
std::scoped_lock lock(g_tex_mutex);
release_shared_locked();
release_on12_locked();
}
g_present_calls.store(0, std::memory_order_relaxed);
g_frames_shared.store(0, std::memory_order_relaxed);