CoopAllTheThings/tests/d3d9_hook_test.cpp

// In-process self-test for the D3D9 capture path (hook/src/d3d9_hook.cpp). Like present_hook_test but
// for Direct3D 9: install the Present hook (discovered from a probe device's vtable), create our own
// D3D9 device, Clear the backbuffer to a known color and Present -- which must fire the detour,
// GetRenderTargetData the backbuffer, swizzle BGRA->RGBA, and upload it into the shared keyed-mutex
// texture (coop_video_<pid>). A D3D11 device then opens that texture by name and verifies the color.
// Also a present-thread overhead guard at a realistic resolution (the D3D9 half of the perf section:
// GetRenderTargetData reads CACHED system memory, so it must stay well under one frame -- it is NOT
// the Vulkan write-combined-memory stall). Reports SKIP and exits 0 without a D3D9/D3D11 device.
#include <cstdint>
#include <cstdio>

#include <windows.h>

#include <d3d9.h>
#include <d3d11_1.h>
#include <dxgi1_2.h>

#include "coop/protocol.hpp"
#include "coop/shared_memory.hpp"
#include "ipc_client.hpp"
#include "d3d9_hook.hpp"
#include "present_overhead.hpp"

using namespace coop;

namespace {
int g_failures = 0;
void check(bool ok, const char* what)
{
	if (!ok) {
		std::printf("  FAIL: %s\n", what);
		++g_failures;
	}
}
template <typename T>
void release(T*& p)
{
	if (p) {
		p->Release();
		p = nullptr;
	}
}
bool near_byte(std::uint8_t got, int expected)
{
	return std::abs(static_cast<int>(got) - expected) <= 2;
}

constexpr UINT kW = 1280; // realistic resolution so the overhead guard is meaningful (not a 64x64 toy)
constexpr UINT kH = 720;
} // namespace

int main()
{
	SharedMemory shm;
	if (!shm.create(shared_memory_name(GetCurrentProcessId()), sizeof(SharedBlock))) {
		std::printf("FAIL: create shared memory\n");
		return 1;
	}
	auto* block = shm.as<SharedBlock>();
	block->version = kProtocolVersion;
	block->sequence.store(0, std::memory_order_relaxed);
	block->magic = kProtocolMagic;

	hook::IpcClient ipc;
	check(ipc.connect(10, 5), "IPC client connect");

	if (!hook::install_d3d9_hooks(ipc)) {
		std::printf("SKIP: could not install the D3D9 Present hook (no d3d9.dll / device?)\n");
		return 0;
	}

	IDirect3D9* d3d = Direct3DCreate9(D3D_SDK_VERSION);
	if (d3d == nullptr) {
		std::printf("SKIP: Direct3DCreate9 failed\n");
		hook::remove_d3d9_hooks();
		return 0;
	}
	WNDCLASSEXW wc{};
	wc.cbSize = sizeof(wc);
	wc.lpfnWndProc = DefWindowProcW;
	wc.hInstance = GetModuleHandleW(nullptr);
	wc.lpszClassName = L"coop_d3d9_hooktest";
	RegisterClassExW(&wc);
	HWND hwnd = CreateWindowExW(0, wc.lpszClassName, L"", WS_OVERLAPPEDWINDOW, 0, 0, kW, kH, nullptr, nullptr,
	                            wc.hInstance, nullptr);

	D3DPRESENT_PARAMETERS pp{};
	pp.BackBufferWidth = kW;
	pp.BackBufferHeight = kH;
	pp.BackBufferFormat = D3DFMT_X8R8G8B8;
	pp.BackBufferCount = 1;
	pp.SwapEffect = D3DSWAPEFFECT_DISCARD;
	pp.hDeviceWindow = hwnd;
	pp.Windowed = TRUE;
	IDirect3DDevice9* dev = nullptr;
	HRESULT hr = d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, hwnd,
	                               D3DCREATE_HARDWARE_VERTEXPROCESSING | D3DCREATE_MULTITHREADED, &pp, &dev);
	if (FAILED(hr) || dev == nullptr) {
		std::printf("SKIP: CreateDevice failed (hr=0x%08lX)\n", static_cast<unsigned long>(hr));
		release(d3d);
		hook::remove_d3d9_hooks();
		return 0;
	}

	// Clear to a known color (R=51,G=102,B=153) and Present -> fires the detour.
	const D3DCOLOR color = D3DCOLOR_XRGB(51, 102, 153);
	for (int frame = 0; frame < 3; ++frame) {
		dev->Clear(0, nullptr, D3DCLEAR_TARGET, color, 1.0f, 0);
		dev->Present(nullptr, nullptr, nullptr, nullptr);
	}

	std::printf("d3d9_presents=%llu frames_shared=%llu video{gen=%u %ux%u fmt=%u}\n",
	            static_cast<unsigned long long>(hook::d3d9_presents()),
	            static_cast<unsigned long long>(hook::d3d9_frames_shared()), block->video.generation.load(),
	            block->video.width, block->video.height, block->video.format);

	check(hook::d3d9_presents() >= 3, "D3D9 Present detour fired");
	check(hook::d3d9_frames_shared() > 0, "backbuffer copied into the shared texture");
	check(block->video.generation.load() > 0, "video generation published to IPC");
	check(block->video.width == kW && block->video.height == kH, "shared dimensions published");

	// Consumer side: open the shared texture (D3D11) and verify the swizzled color.
	{
		ID3D11Device* devB = nullptr;
		ID3D11DeviceContext* ctxB = nullptr;
		if (SUCCEEDED(D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION,
		                                &devB, nullptr, &ctxB))) {
			ID3D11Device1* dev1 = nullptr;
			devB->QueryInterface(IID_PPV_ARGS(&dev1));
			ID3D11Texture2D* sharedB = nullptr;
			IDXGIKeyedMutex* km = nullptr;
			const std::wstring name = video_share_name(GetCurrentProcessId());
			if (dev1 != nullptr
			    && SUCCEEDED(dev1->OpenSharedResourceByName(
			        name.c_str(), DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, IID_PPV_ARGS(&sharedB)))) {
				sharedB->QueryInterface(IID_PPV_ARGS(&km));
				D3D11_TEXTURE2D_DESC sd{};
				sharedB->GetDesc(&sd);
				sd.Usage = D3D11_USAGE_STAGING;
				sd.BindFlags = 0;
				sd.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
				sd.MiscFlags = 0;
				ID3D11Texture2D* staging = nullptr;
				check(SUCCEEDED(devB->CreateTexture2D(&sd, nullptr, &staging)), "create staging texture");
				if (km != nullptr && staging != nullptr && km->AcquireSync(kVideoMutexKey, 1000) == S_OK) {
					ctxB->CopyResource(staging, sharedB);
					km->ReleaseSync(kVideoMutexKey);
					D3D11_MAPPED_SUBRESOURCE mapped{};
					if (SUCCEEDED(ctxB->Map(staging, 0, D3D11_MAP_READ, 0, &mapped))) {
						const auto* px = static_cast<const std::uint8_t*>(mapped.pData);
						std::printf("readback pixel0 = {%u,%u,%u,%u}\n", px[0], px[1], px[2], px[3]);
						check(near_byte(px[0], 51) && near_byte(px[1], 102) && near_byte(px[2], 153),
						      "shared texture carries the rendered color (BGRA->RGBA swizzle)");
						ctxB->Unmap(staging, 0);
					} else {
						check(false, "map staging texture");
					}
				} else {
					check(false, "acquire keyed mutex + copy shared texture");
				}
				release(staging);
			} else {
				check(false, "open shared texture by name");
			}
			release(km);
			release(sharedB);
			release(dev1);
		}
		release(ctxB);
		release(devB);
	}

	// Present-thread overhead guard (the D3D9 perf section): the cached-memory read-back must stay
	// well under one frame -- a regression that put a write-combined-class stall here would trip it.
	{
		auto render = [&] { dev->Clear(0, nullptr, D3DCLEAR_TARGET, color, 1.0f, 0); };
		auto present = [&] { dev->Present(nullptr, nullptr, nullptr, nullptr); };
		const double hooked = cooptest::avg_present_ms(120, render, present);
		hook::remove_d3d9_hooks();
		const double base = cooptest::avg_present_ms(120, render, present);
		std::printf("present-thread: hooked %.3f ms, unhooked %.3f ms, capture overhead %.3f ms\n", hooked, base,
		            hooked - base);
		check(hooked - base < cooptest::kPresentOverheadBudgetMs,
		      "D3D9 capture stays off the present thread's critical path (overhead < one 60 Hz frame)");
	}

	release(dev);
	release(d3d);
	DestroyWindow(hwnd);
	UnregisterClassW(wc.lpszClassName, wc.hInstance);

	std::printf(g_failures == 0 ? "D3D9 HOOK TEST PASS\n" : "D3D9 HOOK TEST FAILED (%d)\n", g_failures);
	return g_failures == 0 ? 0 : 1;
}