M2: DX10 capture via D3D10 read-back; mock_game_test covers DX10

A pure-D3D10 game's backbuffer QIs to ID3D11Texture2D but that view reads back empty (content lives on the game's D3D10 device), and a D3D11 backbuffer also QIs to ID3D10Texture2D -- so GetBuffer can't discriminate. The reliable signal is that a feature-level-10 device rejects CreateTexture2D with the NT-handle keyed-mutex share flags (E_INVALIDARG). The present hook now tries the D3D11 fast path and, on that failure, switches (sticky) to reading the backbuffer through the game's own D3D10 device into a staging texture and uploading it into the shared texture on a hook-owned D3D11 device (Map blocks until the GPU copy completes -> no cross-device race). The host reader is unchanged. mock_game_test now decodes DX10 frames through the hook (monotonic/advancing) alongside DX11/DX12; 15/15 ctest. Roadmap: DX10 milestone done and removed (remaining renumbered); architecture + lessons updated. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-22 11:43:25 +02:00
parent a8de75b2f6
commit 68b9aabc8c
3 changed files with 254 additions and 58 deletions
--- a/hook/src/present_hook.cpp
+++ b/hook/src/present_hook.cpp
@@ -5,6 +5,7 @@

 #include <windows.h>

+#include <d3d10_1.h> // before d3d11on12.h (which also pulls it in); gives the ID3D10* types
 #include <d3d11.h>
 #include <d3d11on12.h>
 #include <d3d12.h>
@@ -71,6 +72,22 @@ ID3D12Device* g_on12_d3d12 = nullptr;
 ID3D12Fence* g_copy_fence = nullptr;
 UINT64 g_copy_fence_val = 0;

+// D3D10 path state. A pure D3D10 game's backbuffer can be QI'd to ID3D11Texture2D, but that
+// D3D11 view does NOT carry the content the D3D10 device rendered, so it must be read through
+// the game's own D3D10 device: copy it into a D3D10 staging texture, Map it (which blocks until
+// the GPU copy completes -> no race), and UpdateSubresource it into the shared keyed-mutex
+// texture, which lives on a hook-owned D3D11 device (g_aux_d3d11) since the game has no usable
+// D3D11 device of its own. The aux device is reusable by later GL/Vulkan paths. Guarded by
+// g_tex_mutex.
+ID3D11Device* g_aux_d3d11 = nullptr;
+ID3D11DeviceContext* g_aux_ctx = nullptr;
+ID3D10Texture2D* g_d3d10_staging = nullptr; // staging on the game's D3D10 device
+ID3D10Device* g_d3d10_dev = nullptr;        // the game device the staging belongs to
+UINT g_d3d10_w = 0;
+UINT g_d3d10_h = 0;
+DXGI_FORMAT g_d3d10_fmt = DXGI_FORMAT_UNKNOWN;
+bool g_force_d3d10 = false; // sticky: the game device rejected the shared texture -> D3D10 read-back
+
 // ExecuteCommandLists hook: recovers the game's D3D12 *present* command queue so we can
 // signal the ordering fence (above) on it. We hook the per-frame method rather than
 // swapchain/queue creation, so it works for late injection (the queue already exists).
@@ -480,62 +497,135 @@ void capture_backbuffer_d3d12(IDXGISwapChain* sc)
 	bb->Release();
 }

-// Copy the swapchain's backbuffer into the shared texture and publish it.
-void capture_backbuffer(IDXGISwapChain* sc)
+// Drop the hook-owned D3D11 device and the D3D10 staging texture. Caller holds g_tex_mutex.
+void release_aux_locked()
 {
-	ID3D11Texture2D* backbuf = nullptr;
-	if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(&backbuf))) ||
-	    backbuf == nullptr)
+	if (g_d3d10_staging != nullptr)
 	{
-		capture_backbuffer_d3d12(sc); // D3D12 game: bridge via D3D11On12 (or idle if neither)
-		return;
+		g_d3d10_staging->Release();
+		g_d3d10_staging = nullptr;
 	}
+	if (g_d3d10_dev != nullptr)
+	{
+		g_d3d10_dev->Release();
+		g_d3d10_dev = nullptr;
+	}
+	g_d3d10_w = g_d3d10_h = 0;
+	g_d3d10_fmt = DXGI_FORMAT_UNKNOWN;
+	g_force_d3d10 = false;
+	if (g_aux_ctx != nullptr)
+	{
+		g_aux_ctx->Release();
+		g_aux_ctx = nullptr;
+	}
+	if (g_aux_d3d11 != nullptr)
+	{
+		g_aux_d3d11->Release();
+		g_aux_d3d11 = nullptr;
+	}
+}

-	D3D11_TEXTURE2D_DESC bd{};
+// Create the hook-owned D3D11 device that backs the shared texture for non-D3D11 games
+// (the game has no D3D11 device of its own). Caller holds g_tex_mutex.
+bool ensure_aux_d3d11_locked()
+{
+	if (g_aux_d3d11 != nullptr)
+	{
+		return true;
+	}
+	const D3D_FEATURE_LEVEL levels[] = {D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_10_0};
+	HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, levels,
+	                               static_cast<UINT>(std::size(levels)), D3D11_SDK_VERSION, &g_aux_d3d11, nullptr,
+	                               &g_aux_ctx);
+	if (FAILED(hr) || g_aux_d3d11 == nullptr)
+	{
+		logf("present(d3d10): aux D3D11CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
+		g_aux_d3d11 = nullptr;
+		g_aux_ctx = nullptr;
+		return false;
+	}
+	return true;
+}
+
+// D3D10 backbuffer path: read it back through a D3D10 staging texture (Map blocks until the
+// game's GPU copy completes -> no race) and UpdateSubresource it into the shared keyed-mutex
+// texture on the hook-owned D3D11 device. Reached by trying ID3D10Texture2D *before*
+// ID3D11Texture2D, because a D3D10 backbuffer's ID3D11 view doesn't carry the rendered content.
+void capture_backbuffer_d3d10(IDXGISwapChain* sc, ID3D10Texture2D* backbuf)
+{
+	D3D10_TEXTURE2D_DESC bd{};
 	backbuf->GetDesc(&bd);
-
-	// Log each distinct swapchain feeding the capture once (see the D3D12 path).
 	if (first_capture_from(sc))
 	{
-		logf("present: swapchain=%p capturing D3D11 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width, bd.Height,
+		logf("present: swapchain=%p capturing D3D10 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width, bd.Height,
 		     static_cast<int>(bd.Format), bd.SampleDesc.Count);
 	}
-
-	// Multisampled backbuffers would need ResolveSubresource; flip-model swapchains
-	// are single-sampled. Skip the rare MSAA case rather than mis-copy.
 	if (bd.SampleDesc.Count != 1)
 	{
-		backbuf->Release();
-		return;
+		return; // MSAA: would need ResolveSubresource; skip rather than mis-copy
 	}

-	ID3D11Device* device = nullptr;
-	backbuf->GetDevice(&device);
-	ID3D11DeviceContext* ctx = nullptr;
-	if (device != nullptr)
+	ID3D10Device* gdev = nullptr;
+	backbuf->GetDevice(&gdev);
+	if (gdev == nullptr)
 	{
-		device->GetImmediateContext(&ctx);
+		return;
 	}

 	bool shared = false;
 	bool dropped = false;
-	if (device != nullptr && ctx != nullptr)
 	{
 		std::scoped_lock lock(g_tex_mutex);
-		if (ensure_shared_texture_locked(device, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr)
+		if (!(g_d3d10_staging != nullptr && g_d3d10_dev == gdev && g_d3d10_w == bd.Width &&
+		      g_d3d10_h == bd.Height && g_d3d10_fmt == bd.Format))
 		{
-			// Key 0 on both sides: a plain cross-process mutex on the texture (the
-			// keyed mutex is created released at key 0). Bounded wait so a stalled
-			// host consumer can never hang the game's render thread.
-			if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
+			if (g_d3d10_staging != nullptr)
 			{
-				ctx->CopyResource(g_shared_tex, backbuf);
-				g_shared_mutex->ReleaseSync(kVideoMutexKey);
-				shared = true;
+				g_d3d10_staging->Release();
+				g_d3d10_staging = nullptr;
 			}
-			else
+			if (g_d3d10_dev != nullptr)
 			{
-				dropped = true; // host held the mutex past the wait -> frame lost (rare on D3D11)
+				g_d3d10_dev->Release();
+				g_d3d10_dev = nullptr;
+			}
+			D3D10_TEXTURE2D_DESC sd{};
+			sd.Width = bd.Width;
+			sd.Height = bd.Height;
+			sd.MipLevels = 1;
+			sd.ArraySize = 1;
+			sd.Format = bd.Format;
+			sd.SampleDesc.Count = 1;
+			sd.Usage = D3D10_USAGE_STAGING;
+			sd.CPUAccessFlags = D3D10_CPU_ACCESS_READ;
+			if (SUCCEEDED(gdev->CreateTexture2D(&sd, nullptr, &g_d3d10_staging)) && g_d3d10_staging != nullptr)
+			{
+				g_d3d10_dev = gdev;
+				gdev->AddRef();
+				g_d3d10_w = bd.Width;
+				g_d3d10_h = bd.Height;
+				g_d3d10_fmt = bd.Format;
+			}
+		}
+
+		if (g_d3d10_staging != nullptr && ensure_aux_d3d11_locked() &&
+		    ensure_shared_texture_locked(g_aux_d3d11, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr)
+		{
+			gdev->CopyResource(g_d3d10_staging, backbuf);
+			D3D10_MAPPED_TEXTURE2D m{};
+			if (SUCCEEDED(g_d3d10_staging->Map(0, D3D10_MAP_READ, 0, &m)) && m.pData != nullptr)
+			{
+				if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
+				{
+					g_aux_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, m.pData, m.RowPitch, 0);
+					g_shared_mutex->ReleaseSync(kVideoMutexKey);
+					shared = true;
+				}
+				else
+				{
+					dropped = true;
+				}
+				g_d3d10_staging->Unmap(0);
 			}
 		}
 	}
@@ -552,16 +642,114 @@ void capture_backbuffer(IDXGISwapChain* sc)
 	{
 		g_ipc->note_video_dropped();
 	}
+	gdev->Release();
+}

-	if (ctx != nullptr)
+// Copy the swapchain's backbuffer into the shared texture and publish it.
+void capture_backbuffer(IDXGISwapChain* sc)
+{
+	// Fast path: a normal D3D11 (feature level 11.1+) game, whose device can host the shared
+	// texture, so a single CopyResource publishes the frame. Note a D3D10 backbuffer ALSO QIs to
+	// ID3D11Texture2D (so we can't discriminate by GetBuffer), but its feature-level-10 device
+	// rejects the share flags -- so a failed shared-texture creation is the signal to switch
+	// (sticky) to the D3D10 read-back path, which reads through the game's own D3D10 device.
+	if (!g_force_d3d10)
 	{
-		ctx->Release();
+		ID3D11Texture2D* backbuf = nullptr;
+		if (FAILED(sc->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void**>(&backbuf))) ||
+		    backbuf == nullptr)
+		{
+			capture_backbuffer_d3d12(sc); // D3D12 game: bridge via D3D11On12 (or idle if neither)
+			return;
+		}
+
+		D3D11_TEXTURE2D_DESC bd{};
+		backbuf->GetDesc(&bd);
+		bool shared = false;
+		bool dropped = false;
+		bool cant_host = false;
+		if (bd.SampleDesc.Count != 1)
+		{
+			backbuf->Release(); // MSAA would need ResolveSubresource; skip rather than mis-copy
+			return;
+		}
+
+		ID3D11Device* device = nullptr;
+		backbuf->GetDevice(&device);
+		ID3D11DeviceContext* ctx = nullptr;
+		if (device != nullptr)
+		{
+			device->GetImmediateContext(&ctx);
+		}
+		if (device != nullptr && ctx != nullptr)
+		{
+			std::scoped_lock lock(g_tex_mutex);
+			if (ensure_shared_texture_locked(device, bd.Width, bd.Height, bd.Format) && g_shared_mutex != nullptr)
+			{
+				if (first_capture_from(sc))
+				{
+					logf("present: swapchain=%p capturing D3D11 backbuffer %ux%u fmt=%d samples=%u", sc, bd.Width,
+					     bd.Height, static_cast<int>(bd.Format), bd.SampleDesc.Count);
+				}
+				// Key 0 on both sides: a plain cross-process mutex on the texture (created
+				// released at key 0). Bounded wait so a stalled host consumer can never hang
+				// the game's render thread.
+				if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
+				{
+					ctx->CopyResource(g_shared_tex, backbuf);
+					g_shared_mutex->ReleaseSync(kVideoMutexKey);
+					shared = true;
+				}
+				else
+				{
+					dropped = true; // host held the mutex past the wait -> frame lost (rare on D3D11)
+				}
+			}
+			else
+			{
+				cant_host = true; // device can't host the shared texture -> try the D3D10 path
+				release_shared_locked();
+			}
+		}
+		if (ctx != nullptr)
+		{
+			ctx->Release();
+		}
+		if (device != nullptr)
+		{
+			device->Release();
+		}
+		backbuf->Release();
+
+		if (shared)
+		{
+			g_frames_shared.fetch_add(1, std::memory_order_relaxed);
+			if (g_ipc != nullptr)
+			{
+				g_ipc->publish_video_frame(bd.Width, bd.Height, static_cast<std::uint32_t>(bd.Format));
+			}
+			return;
+		}
+		if (!cant_host)
+		{
+			if (dropped && g_ipc != nullptr)
+			{
+				g_ipc->note_video_dropped();
+			}
+			return; // captured-or-dropped on the D3D11 path; nothing else to try this frame
+		}
+		g_force_d3d10 = true; // switch (sticky) to the D3D10 read-back path
+		logf("present: game device can't host the shared texture -> D3D10 read-back path");
+		// fall through to the D3D10 path below
 	}
-	if (device != nullptr)
+
+	// D3D10 game: its backbuffer must be read through its own D3D10 device.
+	ID3D10Texture2D* bb10 = nullptr;
+	if (SUCCEEDED(sc->GetBuffer(0, __uuidof(ID3D10Texture2D), reinterpret_cast<void**>(&bb10))) && bb10 != nullptr)
 	{
-		device->Release();
+		capture_backbuffer_d3d10(sc, bb10);
+		bb10->Release();
 	}
-	backbuf->Release();
 }

 void STDMETHODCALLTYPE hk_ExecuteCommandLists(ID3D12CommandQueue* queue, UINT num_lists,
@@ -803,6 +991,7 @@ void remove_present_hooks()
 		std::scoped_lock lock(g_tex_mutex);
 		release_shared_locked();
 		release_on12_locked();
+		release_aux_locked();
 	}
 	g_present_calls.store(0, std::memory_order_relaxed);
 	g_frames_shared.store(0, std::memory_order_relaxed);