M2(DX9): D3D9 capture via Present read-back (covers plain D3D9 + D3D9Ex)
New d3d9_hook.cpp (own file like opengl_hook): inline-hooks IDirect3DDevice9::Present (vtable index 17, discovered from a throwaway device; clean prologue so inline is safe, stdcall() on x86) and read-backs the backbuffer with GetRenderTargetData into a D3DPOOL_SYSTEMMEM surface, swizzles BGRA->RGBA, and uploads it into the shared keyed-mutex texture on a hook-owned D3D11 device (a D3D9 surface isn't D3D11-shareable). Both plain D3D9 and D3D9Ex call this same Present, so one read-back path serves both -- the GPU shared-surface fast path the plan sketched for D3D9Ex wasn't worth it. Wired into the video subsystem (install/remove in dllmain); hook links d3d9. mock_game_test decodes DX9 + DX9Ex frames through the hook; 15/15 ctest (incl. the x86 sub-build). Roadmap: DX9 milestone done and removed (remaining renumbered); architecture + lessons updated. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
395
hook/src/d3d9_hook.cpp
Normal file
395
hook/src/d3d9_hook.cpp
Normal file
@@ -0,0 +1,395 @@
|
||||
#include "d3d9_hook.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <vector>
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#include <d3d9.h>
|
||||
#include <d3d11.h>
|
||||
#include <dxgi1_2.h>
|
||||
|
||||
#include <safetyhook.hpp>
|
||||
|
||||
#include "coop/protocol.hpp"
|
||||
#include "coop/shared_memory.hpp"
|
||||
#include "debug_log.hpp"
|
||||
#include "hook_registry.hpp"
|
||||
|
||||
namespace coop::hook
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
// IDirect3DDevice9 vtable: IUnknown 0-2, then the device methods. Present is index 17
|
||||
// (TestCooperativeLevel 3, GetAvailableTextureMem 4, EvictManagedResources 5, GetDirect3D 6,
|
||||
// GetDeviceCaps 7, GetDisplayMode 8, GetCreationParameters 9, SetCursorProperties 10,
|
||||
// SetCursorPosition 11, ShowCursor 12, CreateAdditionalSwapChain 13, GetSwapChain 14,
|
||||
// GetNumberOfSwapChains 15, Reset 16, Present 17). IDirect3DDevice9Ex games that call the
|
||||
// inherited Present hit this too (PresentEx is a separate, higher slot -- not needed for the
|
||||
// common case / the mock).
|
||||
constexpr unsigned kIdx_IDirect3DDevice9_Present = 17;
|
||||
|
||||
IpcClient* g_ipc = nullptr;
|
||||
unsigned long g_pid = 0;
|
||||
|
||||
safetyhook::InlineHook g_hk_present9;
|
||||
int g_id_present9 = -1;
|
||||
|
||||
std::atomic<std::uint64_t> g_presents{0};
|
||||
std::atomic<std::uint64_t> g_frames_shared{0};
|
||||
bool g_unsupported_logged = false;
|
||||
|
||||
// Our own D3D11 device hosting the shared texture (the D3D9 game has no D3D11 device).
|
||||
ID3D11Device* g_device = nullptr;
|
||||
ID3D11DeviceContext* g_ctx = nullptr;
|
||||
ID3D11Texture2D* g_shared_tex = nullptr;
|
||||
IDXGIKeyedMutex* g_shared_mutex = nullptr;
|
||||
HANDLE g_shared_handle = nullptr;
|
||||
UINT g_share_w = 0;
|
||||
UINT g_share_h = 0;
|
||||
|
||||
// System-memory read-back surface on the game's D3D9 device (GetRenderTargetData target).
|
||||
IDirect3DSurface9* g_sysmem = nullptr;
|
||||
IDirect3DDevice9* g_sysmem_dev = nullptr;
|
||||
UINT g_sysmem_w = 0;
|
||||
UINT g_sysmem_h = 0;
|
||||
D3DFORMAT g_sysmem_fmt = D3DFMT_UNKNOWN;
|
||||
|
||||
std::vector<unsigned char> g_rgba; // swizzled RGBA, uploaded to D3D11
|
||||
|
||||
// Present may be issued re-entrantly by some engines; capture only on the outermost call.
|
||||
thread_local bool t_in_present = false;
|
||||
|
||||
void* vtable_method(void* obj, unsigned index)
|
||||
{
|
||||
return (*reinterpret_cast<void***>(obj))[index];
|
||||
}
|
||||
|
||||
bool ensure_device()
|
||||
{
|
||||
if (g_device != nullptr)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
const HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0,
|
||||
D3D11_SDK_VERSION, &g_device, nullptr, &g_ctx);
|
||||
if (FAILED(hr) || g_device == nullptr)
|
||||
{
|
||||
logf("d3d9: D3D11CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void release_shared()
|
||||
{
|
||||
if (g_shared_mutex != nullptr)
|
||||
{
|
||||
g_shared_mutex->Release();
|
||||
g_shared_mutex = nullptr;
|
||||
}
|
||||
if (g_shared_tex != nullptr)
|
||||
{
|
||||
g_shared_tex->Release();
|
||||
g_shared_tex = nullptr;
|
||||
}
|
||||
if (g_shared_handle != nullptr)
|
||||
{
|
||||
CloseHandle(g_shared_handle);
|
||||
g_shared_handle = nullptr;
|
||||
}
|
||||
g_share_w = g_share_h = 0;
|
||||
}
|
||||
|
||||
void release_sysmem()
|
||||
{
|
||||
if (g_sysmem != nullptr)
|
||||
{
|
||||
g_sysmem->Release();
|
||||
g_sysmem = nullptr;
|
||||
}
|
||||
if (g_sysmem_dev != nullptr)
|
||||
{
|
||||
g_sysmem_dev->Release();
|
||||
g_sysmem_dev = nullptr;
|
||||
}
|
||||
g_sysmem_w = g_sysmem_h = 0;
|
||||
g_sysmem_fmt = D3DFMT_UNKNOWN;
|
||||
}
|
||||
|
||||
bool ensure_shared_texture(UINT w, UINT h)
|
||||
{
|
||||
if (g_shared_tex != nullptr && g_share_w == w && g_share_h == h)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
release_shared();
|
||||
|
||||
D3D11_TEXTURE2D_DESC desc{};
|
||||
desc.Width = w;
|
||||
desc.Height = h;
|
||||
desc.MipLevels = 1;
|
||||
desc.ArraySize = 1;
|
||||
desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; // we swizzle the D3D9 BGRA backbuffer to RGBA
|
||||
desc.SampleDesc.Count = 1;
|
||||
desc.Usage = D3D11_USAGE_DEFAULT;
|
||||
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
|
||||
|
||||
if (FAILED(g_device->CreateTexture2D(&desc, nullptr, &g_shared_tex)) || g_shared_tex == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
IDXGIResource1* res = nullptr;
|
||||
if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast<void**>(&res))) ||
|
||||
res == nullptr)
|
||||
{
|
||||
release_shared();
|
||||
return false;
|
||||
}
|
||||
const std::wstring name = video_share_name(g_pid);
|
||||
const HRESULT hr = res->CreateSharedHandle(
|
||||
nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, name.c_str(), &g_shared_handle);
|
||||
res->Release();
|
||||
if (FAILED(hr) || g_shared_handle == nullptr)
|
||||
{
|
||||
release_shared();
|
||||
return false;
|
||||
}
|
||||
if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast<void**>(&g_shared_mutex))))
|
||||
{
|
||||
release_shared();
|
||||
return false;
|
||||
}
|
||||
g_share_w = w;
|
||||
g_share_h = h;
|
||||
logf("d3d9: shared texture ready %ux%u name=%ls", w, h, name.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
// Read the game's D3D9 backbuffer back to system memory, swizzle BGRA->RGBA, and upload it.
|
||||
void capture_d3d9(IDirect3DDevice9* dev)
|
||||
{
|
||||
IDirect3DSurface9* back = nullptr;
|
||||
if (FAILED(dev->GetBackBuffer(0, 0, D3DBACKBUFFER_TYPE_MONO, &back)) || back == nullptr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
D3DSURFACE_DESC d{};
|
||||
back->GetDesc(&d);
|
||||
const UINT w = d.Width;
|
||||
const UINT h = d.Height;
|
||||
// We only handle the standard 32-bit BGRX/BGRA back buffers (the common D3D9 case).
|
||||
if ((d.Format != D3DFMT_X8R8G8B8 && d.Format != D3DFMT_A8R8G8B8) || w == 0 || h == 0)
|
||||
{
|
||||
if (!g_unsupported_logged)
|
||||
{
|
||||
logf("d3d9: unsupported backbuffer format=%d (only X8R8G8B8 / A8R8G8B8); idle", static_cast<int>(d.Format));
|
||||
g_unsupported_logged = true;
|
||||
}
|
||||
back->Release();
|
||||
return;
|
||||
}
|
||||
|
||||
// (Re)create the system-memory read-back surface on the game's device.
|
||||
if (!(g_sysmem != nullptr && g_sysmem_dev == dev && g_sysmem_w == w && g_sysmem_h == h && g_sysmem_fmt == d.Format))
|
||||
{
|
||||
release_sysmem();
|
||||
if (SUCCEEDED(dev->CreateOffscreenPlainSurface(w, h, d.Format, D3DPOOL_SYSTEMMEM, &g_sysmem, nullptr)) &&
|
||||
g_sysmem != nullptr)
|
||||
{
|
||||
g_sysmem_dev = dev;
|
||||
dev->AddRef();
|
||||
g_sysmem_w = w;
|
||||
g_sysmem_h = h;
|
||||
g_sysmem_fmt = d.Format;
|
||||
}
|
||||
}
|
||||
|
||||
bool shared = false;
|
||||
if (g_sysmem != nullptr && SUCCEEDED(dev->GetRenderTargetData(back, g_sysmem))) // GPU->sysmem, blocks
|
||||
{
|
||||
D3DLOCKED_RECT lr{};
|
||||
if (SUCCEEDED(g_sysmem->LockRect(&lr, nullptr, D3DLOCK_READONLY)) && lr.pBits != nullptr)
|
||||
{
|
||||
const size_t dst_row = static_cast<size_t>(w) * 4;
|
||||
if (g_rgba.size() != dst_row * h)
|
||||
{
|
||||
g_rgba.resize(dst_row * h);
|
||||
}
|
||||
// X8R8G8B8 / A8R8G8B8 store as little-endian 0xAARRGGBB -> bytes B,G,R,A. Swizzle to
|
||||
// R,G,B,A and force opaque alpha so the host's RGBA decode matches the other backends.
|
||||
for (UINT y = 0; y < h; ++y)
|
||||
{
|
||||
const unsigned char* src = static_cast<const unsigned char*>(lr.pBits) + static_cast<size_t>(y) * lr.Pitch;
|
||||
unsigned char* out = g_rgba.data() + static_cast<size_t>(y) * dst_row;
|
||||
for (UINT x = 0; x < w; ++x)
|
||||
{
|
||||
out[x * 4 + 0] = src[x * 4 + 2]; // R
|
||||
out[x * 4 + 1] = src[x * 4 + 1]; // G
|
||||
out[x * 4 + 2] = src[x * 4 + 0]; // B
|
||||
out[x * 4 + 3] = 255; // A
|
||||
}
|
||||
}
|
||||
g_sysmem->UnlockRect();
|
||||
|
||||
if (ensure_device() && ensure_shared_texture(w, h) && g_shared_mutex != nullptr &&
|
||||
g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
|
||||
{
|
||||
g_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, g_rgba.data(), static_cast<UINT>(dst_row), 0);
|
||||
g_ctx->Flush();
|
||||
g_shared_mutex->ReleaseSync(kVideoMutexKey);
|
||||
shared = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (shared)
|
||||
{
|
||||
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->publish_video_frame(w, h, static_cast<std::uint32_t>(DXGI_FORMAT_R8G8B8A8_UNORM));
|
||||
}
|
||||
}
|
||||
back->Release();
|
||||
}
|
||||
|
||||
HRESULT STDMETHODCALLTYPE hk_Present9(IDirect3DDevice9* dev, const RECT* src, const RECT* dst, HWND wnd,
|
||||
const RGNDATA* dirty)
|
||||
{
|
||||
hook_note_call(g_id_present9);
|
||||
g_presents.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->note_present();
|
||||
}
|
||||
if (!t_in_present)
|
||||
{
|
||||
t_in_present = true;
|
||||
capture_d3d9(dev);
|
||||
t_in_present = false;
|
||||
}
|
||||
// stdcall(): IDirect3DDevice9::Present is __stdcall; call() would invoke the trampoline as
|
||||
// __cdecl on x86 -> ESP imbalance -> crash. No-op on x64. (Present has a clean prologue, so
|
||||
// an inline hook is safe -- unlike the WASAPI COM methods, which need a vtable swap.)
|
||||
return g_hk_present9.stdcall<HRESULT>(dev, src, dst, wnd, dirty);
|
||||
}
|
||||
|
||||
// Create a throwaway D3D9 device to read IDirect3DDevice9::Present's address, so we can
|
||||
// inline-hook it (catching the game's existing device regardless of when we injected). Returns
|
||||
// null when d3d9.dll isn't loaded (not a D3D9 game) or a probe device can't be created.
|
||||
void* grab_present9_address()
|
||||
{
|
||||
HMODULE d3d9 = GetModuleHandleW(L"d3d9.dll");
|
||||
if (d3d9 == nullptr)
|
||||
{
|
||||
return nullptr; // not a D3D9 game
|
||||
}
|
||||
using PFN_Direct3DCreate9 = IDirect3D9*(WINAPI*)(UINT);
|
||||
auto create = reinterpret_cast<PFN_Direct3DCreate9>(GetProcAddress(d3d9, "Direct3DCreate9"));
|
||||
if (create == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
IDirect3D9* d3d = create(D3D_SDK_VERSION);
|
||||
if (d3d == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
WNDCLASSEXW wc{};
|
||||
wc.cbSize = sizeof(wc);
|
||||
wc.lpfnWndProc = DefWindowProcW;
|
||||
wc.hInstance = GetModuleHandleW(nullptr);
|
||||
wc.lpszClassName = L"coop_d3d9_probe";
|
||||
RegisterClassExW(&wc);
|
||||
HWND hwnd = CreateWindowExW(0, wc.lpszClassName, L"", WS_OVERLAPPEDWINDOW, 0, 0, 8, 8, nullptr, nullptr,
|
||||
wc.hInstance, nullptr);
|
||||
|
||||
void* present = nullptr;
|
||||
if (hwnd != nullptr)
|
||||
{
|
||||
D3DPRESENT_PARAMETERS pp{};
|
||||
pp.BackBufferWidth = 8;
|
||||
pp.BackBufferHeight = 8;
|
||||
pp.BackBufferFormat = D3DFMT_X8R8G8B8;
|
||||
pp.BackBufferCount = 1;
|
||||
pp.SwapEffect = D3DSWAPEFFECT_DISCARD;
|
||||
pp.hDeviceWindow = hwnd;
|
||||
pp.Windowed = TRUE;
|
||||
IDirect3DDevice9* dev = nullptr;
|
||||
if (SUCCEEDED(d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, hwnd,
|
||||
D3DCREATE_HARDWARE_VERTEXPROCESSING | D3DCREATE_MULTITHREADED, &pp, &dev)) &&
|
||||
dev != nullptr)
|
||||
{
|
||||
present = vtable_method(dev, kIdx_IDirect3DDevice9_Present);
|
||||
dev->Release();
|
||||
}
|
||||
DestroyWindow(hwnd);
|
||||
}
|
||||
UnregisterClassW(wc.lpszClassName, wc.hInstance);
|
||||
d3d->Release();
|
||||
return present;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool install_d3d9_hooks(IpcClient& ipc)
|
||||
{
|
||||
g_ipc = &ipc;
|
||||
g_pid = GetCurrentProcessId();
|
||||
if (g_hk_present9)
|
||||
{
|
||||
return true; // already installed
|
||||
}
|
||||
g_id_present9 = hook_register("IDirect3DDevice9::Present", HookSubsys_Video);
|
||||
g_unsupported_logged = false;
|
||||
|
||||
void* present = grab_present9_address();
|
||||
if (present == nullptr)
|
||||
{
|
||||
hook_set_installed(g_id_present9, false); // not a D3D9 game (or no probe device)
|
||||
return false;
|
||||
}
|
||||
g_hk_present9 = safetyhook::create_inline(present, reinterpret_cast<void*>(&hk_Present9));
|
||||
hook_set_installed(g_id_present9, static_cast<bool>(g_hk_present9));
|
||||
logf("install_d3d9_hooks: Present=%p hooked=%d", present, static_cast<bool>(g_hk_present9) ? 1 : 0);
|
||||
return static_cast<bool>(g_hk_present9);
|
||||
}
|
||||
|
||||
void remove_d3d9_hooks()
|
||||
{
|
||||
g_hk_present9 = {};
|
||||
hook_set_installed(g_id_present9, false);
|
||||
release_shared();
|
||||
release_sysmem();
|
||||
if (g_ctx != nullptr)
|
||||
{
|
||||
g_ctx->Release();
|
||||
g_ctx = nullptr;
|
||||
}
|
||||
if (g_device != nullptr)
|
||||
{
|
||||
g_device->Release();
|
||||
g_device = nullptr;
|
||||
}
|
||||
g_rgba.clear();
|
||||
g_presents.store(0, std::memory_order_relaxed);
|
||||
g_frames_shared.store(0, std::memory_order_relaxed);
|
||||
g_ipc = nullptr;
|
||||
}
|
||||
|
||||
std::uint64_t d3d9_presents()
|
||||
{
|
||||
return g_presents.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
std::uint64_t d3d9_frames_shared()
|
||||
{
|
||||
return g_frames_shared.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
} // namespace coop::hook
|
||||
30
hook/src/d3d9_hook.hpp
Normal file
30
hook/src/d3d9_hook.hpp
Normal file
@@ -0,0 +1,30 @@
|
||||
// Injected D3D9 capture path: inline-hooks IDirect3DDevice9::Present (discovered from a
|
||||
// throwaway device's vtable, so it catches the game's existing device regardless of when we
|
||||
// injected) in Direct3D 9 / 9Ex games, which don't present via IDXGISwapChain. It reads the
|
||||
// backbuffer back with GetRenderTargetData, swizzles BGRA->RGBA, and uploads it into the same
|
||||
// shared keyed-mutex texture (coop_video_<pid>) the host samples. Part of the video subsystem,
|
||||
// alongside present_hook and opengl_hook.
|
||||
//
|
||||
// A D3D9 backbuffer isn't directly shareable with D3D11, so this is a CPU read-back
|
||||
// (GetRenderTargetData -> system memory -> upload via a hook-owned D3D11 device), like the
|
||||
// OpenGL path. Works for both plain D3D9 and D3D9Ex (both call Present here).
|
||||
#pragma once
|
||||
|
||||
#include "ipc_client.hpp"
|
||||
|
||||
namespace coop::hook
|
||||
{
|
||||
|
||||
// Installs the D3D9 Present hook. `ipc` must outlive the hook. Returns true if Present was
|
||||
// hooked (i.e. d3d9.dll is present and a probe device came up). Safe to call repeatedly.
|
||||
bool install_d3d9_hooks(IpcClient& ipc);
|
||||
|
||||
// Removes the D3D9 hook and releases the shared texture (best effort).
|
||||
void remove_d3d9_hooks();
|
||||
|
||||
// --- Diagnostics -----------------------------------------------------------
|
||||
|
||||
std::uint64_t d3d9_presents(); // cumulative IDirect3DDevice9::Present detours
|
||||
std::uint64_t d3d9_frames_shared(); // frames uploaded into the shared texture
|
||||
|
||||
} // namespace coop::hook
|
||||
@@ -21,6 +21,7 @@
|
||||
#include "hook_registry.hpp"
|
||||
#include "ipc_client.hpp"
|
||||
#include "mkb_hook.hpp"
|
||||
#include "d3d9_hook.hpp"
|
||||
#include "opengl_hook.hpp"
|
||||
#include "present_hook.hpp"
|
||||
#include "xinput_hook.hpp"
|
||||
@@ -134,17 +135,19 @@ DWORD WINAPI worker_thread(LPVOID)
|
||||
{
|
||||
const bool present_ok = coop::hook::install_present_hooks(g_ipc);
|
||||
const bool gl_ok = coop::hook::install_opengl_hooks(g_ipc);
|
||||
video_installed = present_ok || gl_ok;
|
||||
const bool d3d9_ok = coop::hook::install_d3d9_hooks(g_ipc);
|
||||
video_installed = present_ok || gl_ok || d3d9_ok;
|
||||
if (video_installed)
|
||||
{
|
||||
coop::hook::logf("worker_thread: video hooks installed (present=%d opengl=%d)",
|
||||
present_ok ? 1 : 0, gl_ok ? 1 : 0);
|
||||
coop::hook::logf("worker_thread: video hooks installed (present=%d opengl=%d d3d9=%d)",
|
||||
present_ok ? 1 : 0, gl_ok ? 1 : 0, d3d9_ok ? 1 : 0);
|
||||
}
|
||||
}
|
||||
else if (!want_video && video_installed)
|
||||
{
|
||||
coop::hook::remove_present_hooks();
|
||||
coop::hook::remove_opengl_hooks();
|
||||
coop::hook::remove_d3d9_hooks();
|
||||
video_installed = false;
|
||||
coop::hook::logf("worker_thread: video hooks removed (host request)");
|
||||
}
|
||||
@@ -255,6 +258,7 @@ BOOL APIENTRY DllMain(HMODULE module, DWORD reason, LPVOID reserved)
|
||||
coop::hook::shutdown_audio_hooks(); // detach: remove detours + release the kept probe
|
||||
coop::hook::remove_present_hooks();
|
||||
coop::hook::remove_opengl_hooks();
|
||||
coop::hook::remove_d3d9_hooks();
|
||||
coop::hook::remove_mkb_hooks();
|
||||
coop::hook::hook_registry_reset();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user