From 68b9aabc8c830b9e86c0129a3ac76d73444f6f79 Mon Sep 17 00:00:00 2001 From: BlackMark Date: Mon, 22 Jun 2026 11:43:25 +0200 Subject: [PATCH] M2: DX10 capture via D3D10 read-back; mock_game_test covers DX10 A pure-D3D10 game's backbuffer QIs to ID3D11Texture2D but that view reads back empty (content lives on the game's D3D10 device), and a D3D11 backbuffer also QIs to ID3D10Texture2D -- so GetBuffer can't discriminate. The reliable signal is that a feature-level-10 device rejects CreateTexture2D with the NT-handle keyed-mutex share flags (E_INVALIDARG). The present hook now tries the D3D11 fast path and, on that failure, switches (sticky) to reading the backbuffer through the game's own D3D10 device into a staging texture and uploading it into the shared texture on a hook-owned D3D11 device (Map blocks until the GPU copy completes -> no cross-device race). The host reader is unchanged. mock_game_test now decodes DX10 frames through the hook (monotonic/advancing) alongside DX11/DX12; 15/15 ctest. Roadmap: DX10 milestone done and removed (remaining renumbered); architecture + lessons updated. Co-Authored-By: Claude Opus 4.8 --- README.md | 41 +++--- hook/src/present_hook.cpp | 263 ++++++++++++++++++++++++++++++++------ tests/mock_game_test.cpp | 8 +- 3 files changed, 254 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 49cab39..19f6000 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,12 @@ and forwards guest controllers back into it. The hooked video path has two producers: **Direct3D (DXGI)** hooks `IDXGISwapChain::Present` / `Present1` and copies the backbuffer — directly for -D3D10/11 games (the backbuffer is an `ID3D11Texture2D`), and via a **D3D11On12 +D3D11 games (the backbuffer is an `ID3D11Texture2D`), via a **D3D11On12 bridge** for D3D12 games (wrap the `ID3D12Resource` backbuffer, `CopyResource` into -the shared texture); **OpenGL** hooks `SwapBuffers` / `wglSwapBuffers` and reads the +the shared texture), and via a **D3D10 read-back** for D3D10 games (their backbuffer's +D3D11 view is empty and a feature-level-10 device can't host the shared texture, so +read it through the game's own D3D10 device and upload it via a hook-owned D3D11 +device); **OpenGL** hooks `SwapBuffers` / `wglSwapBuffers` and reads the backbuffer with `glReadPixels` (for games that never touch DXGI, e.g. Phantom Brave). The host samples the copy as plain UNORM (`srgb_to_unorm`) so `*_SRGB`-backbuffer games mirror at correct brightness. **WGC remains the default** @@ -131,22 +134,10 @@ Conventions for every milestone below: panel so it fits (content can be moved between columns / rows — the most detailed case should still fit). Land an automated harness check (drive-to-max → screenshot → assert no overflow) so later milestones that add UI keep it green. *Independent of the backend work; - every milestone below must preserve this test* (M5's red banner and Vulkan-layer checkbox + every milestone below must preserve this test* (M4's red banner and Vulkan-layer checkbox in particular). -- **M2 — DX10 (mock → capture).** - 1. **Mock backend** (`render_dx10.cpp`). `ID3D10Device` + DXGI swap chain; Windows SDK only - (`d3d10`, `dxgi`), **no new deps**. Sub-region fills (bar, counter block) via - `CopySubresourceRegion` of small solid-colour textures (DX10 has no clear-rect). Top-left - origin, so the counter block maps straight to the capture's sample point. - 2. **Capture.** The DXGI `Present`/`Present1` hook already catches D3D10 swap chains, but the - copy QIs the backbuffer to `ID3D11Texture2D`, which a *pure* D3D10 device fails. Share the - D3D10 backbuffer into the hook's **own D3D11 device** (legacy shared handle) and copy it - into the standard keyed-mutex texture; add a `dx10_present_hook_test` that decodes the - mock's frames. Validates the "D3D10/11" claim the Architecture section currently makes - untested. - -- **M3 — DX9 (mock → D3D9Ex capture → plain-D3D9 capture).** +- **M2 — DX9 (mock → D3D9Ex capture → plain-D3D9 capture).** 1. **Mock backend** (`render_dx09.cpp`). `IDirect3DDevice9` / `IDirect3DDevice9Ex` + present; Windows SDK only (`d3d9`), **no new deps**. `Clear` for the background and `ColorFill` for the bar + block (D3D9's built-in rect fill — exactly the primitive DX10/11 lack). Top-left @@ -170,7 +161,7 @@ Conventions for every milestone below: texture* differs (CPU copy, not GPU). `GetRenderTargetData` is a GPU→sysmem stall, so **drop / throttle** mirror frames rather than back-pressure the game. Same test, plain mode. -- **M4 — OpenGL (mock → capture coverage).** +- **M3 — OpenGL (mock → capture coverage).** 1. **Mock backend** (`render_gl.cpp`). Raw WGL context (`wglCreateContextAttribsARB`) with the **glad** loader (new submodule, `Dav1dde/glad`). Background via `glClearColor`/`glClear`; bar + block via `glScissor` + clear (shader-free GL 1.x). GL's framebuffer is @@ -181,7 +172,7 @@ Conventions for every milestone below: ships; add a mock-backed regression that decodes the mock's frames through it (upgrading the synthetic `opengl_hook_test` to a real animated game). Small. -- **M5 — Vulkan (mock → capture).** The largest. +- **M4 — Vulkan (mock → capture).** The largest. 1. **Mock backend** (`render_vk.cpp`). New submodules **Vulkan-Headers** (`KhronosGroup/Vulkan-Headers`, official) + **volk** (`zeux/volk`); raw `vkCreateWin32SurfaceKHR`, swap chain, per-frame acquire → clear → present. Background via @@ -326,10 +317,10 @@ ctest --test-dir build -C Debug --output-on-failure endpoint format). Skips cleanly if the machine has no audio endpoint. - **`mock_game_test`** — comprehensive capture/audio/hook stress test against **`coop_mock_game`** (an animated, frame-numbered A/V test game under - [`tools/mock_game`](tools/mock_game) with selectable **DX11 / DX12** backends and a + [`tools/mock_game`](tools/mock_game) with selectable **DX10 / DX11 / DX12** backends and a configurable WASAPI tone). It launches the game, injects `coop_hook.dll`, opens the hook's shared video texture, and **decodes the frame number out of the captured pixels** - to assert the mirror sees a *monotonic, advancing* sequence for both backends (the bar + to assert the mirror sees a *monotonic, advancing* sequence for each backend (the bar for no dropped / stale / out-of-order frames — what the DX12 rotating-backbuffer bug broke). It launches the game at several **audio formats** (44100/48000/96000, PCM + float) and asserts the hook measures each one's rate through the full inject path, then @@ -467,6 +458,16 @@ Non-obvious things that cost time and constrain the design: the producer-side `AcquireSync` non-blocking (`timeout 0`) so a busy mutex drops a *mirror* frame instead of stalling the game; the Video panel's "Frames lost" line surfaces both capture- and display-stage drops. +- **A D3D10 game's backbuffer lies about being D3D11.** A pure-D3D10 swapchain's backbuffer + QIs to `ID3D11Texture2D` *successfully*, but that D3D11 view reads back **empty** — the + rendered content only exists on the game's own D3D10 device. (And a D3D11 backbuffer QIs to + `ID3D10Texture2D` too, so `GetBuffer` alone can't tell them apart.) The reliable signal is + that a feature-level-10 device **rejects** `CreateTexture2D` with the NT-handle keyed-mutex + share flags (`E_INVALIDARG`): so try the D3D11 fast path, and on that failure switch (sticky) + to reading the backbuffer through the game's **D3D10** device into a staging texture and + `UpdateSubresource`-ing it into the shared texture on a **hook-owned D3D11 device** (the game + has no usable D3D11 device of its own). The staging `Map` blocks until the GPU copy completes, + so there's no cross-device race. - **A render client that predates our injection has no knowable format — measure it.** We inject into already-running games, so we usually never see the game's `IAudioClient::Initialize`; the render-hook then assumes the device mix format for that diff --git a/hook/src/present_hook.cpp b/hook/src/present_hook.cpp index af0fa19..c053440 100644 --- a/hook/src/present_hook.cpp +++ b/hook/src/present_hook.cpp @@ -5,6 +5,7 @@ #include +#include // before d3d11on12.h (which also pulls it in); gives the ID3D10* types #include #include #include @@ -71,6 +72,22 @@ ID3D12Device* g_on12_d3d12 = nullptr; ID3D12Fence* g_copy_fence = nullptr; UINT64 g_copy_fence_val = 0; +// D3D10 path state. A pure D3D10 game's backbuffer can be QI'd to ID3D11Texture2D, but that +// D3D11 view does NOT carry the content the D3D10 device rendered, so it must be read through +// the game's own D3D10 device: copy it into a D3D10 staging texture, Map it (which blocks until +// the GPU copy completes -> no race), and UpdateSubresource it into the shared keyed-mutex +// texture, which lives on a hook-owned D3D11 device (g_aux_d3d11) since the game has no usable +// D3D11 device of its own. The aux device is reusable by later GL/Vulkan paths. Guarded by +// g_tex_mutex. +ID3D11Device* g_aux_d3d11 = nullptr; +ID3D11DeviceContext* g_aux_ctx = nullptr; +ID3D10Texture2D* g_d3d10_staging = nullptr; // staging on the game's D3D10 device +ID3D10Device* g_d3d10_dev = nullptr; // the game device the staging belongs to +UINT g_d3d10_w = 0; +UINT g_d3d10_h = 0; +DXGI_FORMAT g_d3d10_fmt = DXGI_FORMAT_UNKNOWN; +bool g_force_d3d10 = false; // sticky: the game device rejected the shared texture -> D3D10 read-back + // ExecuteCommandLists hook: recovers the game's D3D12 *present* command queue so we can // signal the ordering fence (above) on it. We hook the per-frame method rather than // swapchain/queue creation, so it works for late injection (the queue already exists). @@ -480,62 +497,135 @@ void capture_backbuffer_d3d12(IDXGISwapChain* sc) bb->Release(); } -// Copy the swapchain's backbuffer into the shared texture and publish it. -void capture_backbuffer(IDXGISwapChain* sc) +// Drop the hook-owned D3D11 device and the D3D10 staging texture. Caller holds g_tex_mutex. +void release_aux_locked() { - ID3D11Texture2D* backbuf = nullptr; - if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast(&backbuf))) || - backbuf == nullptr) + if (g_d3d10_staging != nullptr) { - capture_backbuffer_d3d12(sc); // D3D12 game: bridge via D3D11On12 (or idle if neither) - return; + g_d3d10_staging->Release(); + g_d3d10_staging = nullptr; } + if (g_d3d10_dev != nullptr) + { + g_d3d10_dev->Release(); + g_d3d10_dev = nullptr; + } + g_d3d10_w = g_d3d10_h = 0; + g_d3d10_fmt = DXGI_FORMAT_UNKNOWN; + g_force_d3d10 = false; + if (g_aux_ctx != nullptr) + { + g_aux_ctx->Release(); + g_aux_ctx = nullptr; + } + if (g_aux_d3d11 != nullptr) + { + g_aux_d3d11->Release(); + g_aux_d3d11 = nullptr; + } +} - D3D11_TEXTURE2D_DESC bd{}; +// Create the hook-owned D3D11 device that backs the shared texture for non-D3D11 games +// (the game has no D3D11 device of its own). Caller holds g_tex_mutex. +bool ensure_aux_d3d11_locked() +{ + if (g_aux_d3d11 != nullptr) + { + return true; + } + const D3D_FEATURE_LEVEL levels[] = {D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_10_0}; + HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, levels, + static_cast(std::size(levels)), D3D11_SDK_VERSION, &g_aux_d3d11, nullptr, + &g_aux_ctx); + if (FAILED(hr) || g_aux_d3d11 == nullptr) + { + logf("present(d3d10): aux D3D11CreateDevice failed hr=0x%08lX", static_cast(hr)); + g_aux_d3d11 = nullptr; + g_aux_ctx = nullptr; + return false; + } + return true; +} + +// D3D10 backbuffer path: read it back through a D3D10 staging texture (Map blocks until the +// game's GPU copy completes -> no race) and UpdateSubresource it into the shared keyed-mutex +// texture on the hook-owned D3D11 device. Reached by trying ID3D10Texture2D *before* +// ID3D11Texture2D, because a D3D10 backbuffer's ID3D11 view doesn't carry the rendered content. +void capture_backbuffer_d3d10(IDXGISwapChain* sc, ID3D10Texture2D* backbuf) +{ + D3D10_TEXTURE2D_DESC bd{}; backbuf->GetDesc(&bd); - - // Log each distinct swapchain feeding the capture once (see the D3D12 path). if (first_capture_from(sc)) { - logf("present: swapchain=%p capturing D3D11 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width, bd.Height, + logf("present: swapchain=%p capturing D3D10 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width, bd.Height, static_cast(bd.Format), bd.SampleDesc.Count); } - - // Multisampled backbuffers would need ResolveSubresource; flip-model swapchains - // are single-sampled. Skip the rare MSAA case rather than mis-copy. if (bd.SampleDesc.Count != 1) { - backbuf->Release(); - return; + return; // MSAA: would need ResolveSubresource; skip rather than mis-copy } - ID3D11Device* device = nullptr; - backbuf->GetDevice(&device); - ID3D11DeviceContext* ctx = nullptr; - if (device != nullptr) + ID3D10Device* gdev = nullptr; + backbuf->GetDevice(&gdev); + if (gdev == nullptr) { - device->GetImmediateContext(&ctx); + return; } bool shared = false; bool dropped = false; - if (device != nullptr && ctx != nullptr) { std::scoped_lock lock(g_tex_mutex); - if (ensure_shared_texture_locked(device, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr) + if (!(g_d3d10_staging != nullptr && g_d3d10_dev == gdev && g_d3d10_w == bd.Width && + g_d3d10_h == bd.Height && g_d3d10_fmt == bd.Format)) { - // Key 0 on both sides: a plain cross-process mutex on the texture (the - // keyed mutex is created released at key 0). Bounded wait so a stalled - // host consumer can never hang the game's render thread. - if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK) + if (g_d3d10_staging != nullptr) { - ctx->CopyResource(g_shared_tex, backbuf); - g_shared_mutex->ReleaseSync(kVideoMutexKey); - shared = true; + g_d3d10_staging->Release(); + g_d3d10_staging = nullptr; } - else + if (g_d3d10_dev != nullptr) { - dropped = true; // host held the mutex past the wait -> frame lost (rare on D3D11) + g_d3d10_dev->Release(); + g_d3d10_dev = nullptr; + } + D3D10_TEXTURE2D_DESC sd{}; + sd.Width = bd.Width; + sd.Height = bd.Height; + sd.MipLevels = 1; + sd.ArraySize = 1; + sd.Format = bd.Format; + sd.SampleDesc.Count = 1; + sd.Usage = D3D10_USAGE_STAGING; + sd.CPUAccessFlags = D3D10_CPU_ACCESS_READ; + if (SUCCEEDED(gdev->CreateTexture2D(&sd, nullptr, &g_d3d10_staging)) && g_d3d10_staging != nullptr) + { + g_d3d10_dev = gdev; + gdev->AddRef(); + g_d3d10_w = bd.Width; + g_d3d10_h = bd.Height; + g_d3d10_fmt = bd.Format; + } + } + + if (g_d3d10_staging != nullptr && ensure_aux_d3d11_locked() && + ensure_shared_texture_locked(g_aux_d3d11, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr) + { + gdev->CopyResource(g_d3d10_staging, backbuf); + D3D10_MAPPED_TEXTURE2D m{}; + if (SUCCEEDED(g_d3d10_staging->Map(0, D3D10_MAP_READ, 0, &m)) && m.pData != nullptr) + { + if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK) + { + g_aux_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, m.pData, m.RowPitch, 0); + g_shared_mutex->ReleaseSync(kVideoMutexKey); + shared = true; + } + else + { + dropped = true; + } + g_d3d10_staging->Unmap(0); } } } @@ -552,16 +642,114 @@ void capture_backbuffer(IDXGISwapChain* sc) { g_ipc->note_video_dropped(); } + gdev->Release(); +} - if (ctx != nullptr) +// Copy the swapchain's backbuffer into the shared texture and publish it. +void capture_backbuffer(IDXGISwapChain* sc) +{ + // Fast path: a normal D3D11 (feature level 11.1+) game, whose device can host the shared + // texture, so a single CopyResource publishes the frame. Note a D3D10 backbuffer ALSO QIs to + // ID3D11Texture2D (so we can't discriminate by GetBuffer), but its feature-level-10 device + // rejects the share flags -- so a failed shared-texture creation is the signal to switch + // (sticky) to the D3D10 read-back path, which reads through the game's own D3D10 device. + if (!g_force_d3d10) { - ctx->Release(); + ID3D11Texture2D* backbuf = nullptr; + if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast(&backbuf))) || + backbuf == nullptr) + { + capture_backbuffer_d3d12(sc); // D3D12 game: bridge via D3D11On12 (or idle if neither) + return; + } + + D3D11_TEXTURE2D_DESC bd{}; + backbuf->GetDesc(&bd); + bool shared = false; + bool dropped = false; + bool cant_host = false; + if (bd.SampleDesc.Count != 1) + { + backbuf->Release(); // MSAA would need ResolveSubresource; skip rather than mis-copy + return; + } + + ID3D11Device* device = nullptr; + backbuf->GetDevice(&device); + ID3D11DeviceContext* ctx = nullptr; + if (device != nullptr) + { + device->GetImmediateContext(&ctx); + } + if (device != nullptr && ctx != nullptr) + { + std::scoped_lock lock(g_tex_mutex); + if (ensure_shared_texture_locked(device, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr) + { + if (first_capture_from(sc)) + { + logf("present: swapchain=%p capturing D3D11 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width, + bd.Height, static_cast(bd.Format), bd.SampleDesc.Count); + } + // Key 0 on both sides: a plain cross-process mutex on the texture (created + // released at key 0). Bounded wait so a stalled host consumer can never hang + // the game's render thread. + if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK) + { + ctx->CopyResource(g_shared_tex, backbuf); + g_shared_mutex->ReleaseSync(kVideoMutexKey); + shared = true; + } + else + { + dropped = true; // host held the mutex past the wait -> frame lost (rare on D3D11) + } + } + else + { + cant_host = true; // device can't host the shared texture -> try the D3D10 path + release_shared_locked(); + } + } + if (ctx != nullptr) + { + ctx->Release(); + } + if (device != nullptr) + { + device->Release(); + } + backbuf->Release(); + + if (shared) + { + g_frames_shared.fetch_add(1, std::memory_order_relaxed); + if (g_ipc != nullptr) + { + g_ipc->publish_video_frame(bd.Width, bd.Height, static_cast(bd.Format)); + } + return; + } + if (!cant_host) + { + if (dropped && g_ipc != nullptr) + { + g_ipc->note_video_dropped(); + } + return; // captured-or-dropped on the D3D11 path; nothing else to try this frame + } + g_force_d3d10 = true; // switch (sticky) to the D3D10 read-back path + logf("present: game device can't host the shared texture -> D3D10 read-back path"); + // fall through to the D3D10 path below } - if (device != nullptr) + + // D3D10 game: its backbuffer must be read through its own D3D10 device. + ID3D10Texture2D* bb10 = nullptr; + if (SUCCEEDED(sc->GetBuffer(0, __uuidof(ID3D10Texture2D), reinterpret_cast(&bb10))) && bb10 != nullptr) { - device->Release(); + capture_backbuffer_d3d10(sc, bb10); + bb10->Release(); } - backbuf->Release(); } void STDMETHODCALLTYPE hk_ExecuteCommandLists(ID3D12CommandQueue* queue, UINT num_lists, @@ -803,6 +991,7 @@ void remove_present_hooks() std::scoped_lock lock(g_tex_mutex); release_shared_locked(); release_on12_locked(); + release_aux_locked(); } g_present_calls.store(0, std::memory_order_relaxed); g_frames_shared.store(0, std::memory_order_relaxed); diff --git a/tests/mock_game_test.cpp b/tests/mock_game_test.cpp index 13bb349..7c3fecb 100644 --- a/tests/mock_game_test.cpp +++ b/tests/mock_game_test.cpp @@ -223,7 +223,12 @@ VideoShareView read_video_share(const SharedBlock* block) void test_video_capture(const char* backend, ID3D11Device* device) { std::printf("== video capture: %s ==\n", backend); - const std::wstring args = (std::string(backend) == "dx12" ? L"dx12 " : L"dx11 ") + std::wstring(L"30"); + std::wstring args; + for (const char* p = backend; *p != '\0'; ++p) // backend names are ASCII (dx10/dx11/dx12/...) + { + args.push_back(static_cast(*p)); + } + args += L" 30"; MockGame game = MockGame::launch(args); if (!game.ok) { @@ -517,6 +522,7 @@ int main() return 0; } + test_video_capture("dx10", device); test_video_capture("dx11", device); test_video_capture("dx12", device);