From dd976979a103f09ee727cfe21b4fdd05ddc83f51 Mon Sep 17 00:00:00 2001 From: BlackMark Date: Mon, 22 Jun 2026 12:39:07 +0200 Subject: [PATCH] M2(Vulkan): capture hook via GPA interception + vkCmdCopyImageToBuffer read-back New vk_hook.cpp inline-hooks the vulkan-1.dll vkGetInstanceProcAddr export and hands back our wrappers for vkCreateInstance / vkCreateDevice / vkGetDeviceProcAddr / vkCreateSwapchainKHR / vkQueuePresentKHR, so a volk-using (loader-bypass) app resolves our hooks. On present it reads the swap-chain image back with vkCmdCopyImageToBuffer into a host-visible buffer (same read-back model as D3D10/D3D9/OpenGL), swizzles BGRA->RGBA, and uploads it into the shared keyed-mutex texture on a hook-owned D3D11 device. The read-back submit re-chains the present's wait semaphores (consume the originals, signal our own that the real present waits on) so capture orders after rendering without double-waiting. Wired into the video subsystem with a lazy retry (vulkan-1.dll loads late). Hook links the official Vulkan-Headers (headers only, VK_NO_PROTOTYPES) via the include dir so the x86 sub-build builds too. Because Vulkan caches its present pointer at init, late injection can't hook it: mock_game_test launches the mock **suspended**, injects, resumes, and under COOP_MOCK_VK_EARLY the mock loads Vulkan and waits so the hook arms first -- then decodes frames through the hook like the other backends. 15/15 ctest (x64 + x86); skips cleanly without a Vulkan driver. Co-Authored-By: Claude Opus 4.8 --- hook/CMakeLists.txt | 7 + hook/src/dllmain.cpp | 16 + hook/src/vk_hook.cpp | 712 +++++++++++++++++++++++++++++++++++++++ hook/src/vk_hook.hpp | 34 ++ tests/mock_game_test.cpp | 123 +++++-- tools/mock_game/main.cpp | 9 + 6 files changed, 875 insertions(+), 26 deletions(-) create mode 100644 hook/src/vk_hook.cpp create mode 100644 hook/src/vk_hook.hpp diff --git a/hook/CMakeLists.txt b/hook/CMakeLists.txt index 738edb6..4579e86 100644 --- a/hook/CMakeLists.txt +++ b/hook/CMakeLists.txt @@ -6,6 +6,7 @@ add_library(coop_hook SHARED src/present_hook.cpp src/opengl_hook.cpp src/d3d9_hook.cpp + src/vk_hook.cpp src/mkb_hook.cpp src/debug_log.cpp src/hook_registry.cpp) @@ -16,6 +17,12 @@ target_include_directories(coop_hook PRIVATE src) # the Windows 10 20H1 (NTDDI_WIN10_CO) headers. target_compile_definitions(coop_hook PRIVATE NTDDI_VERSION=0x0A00000B) +# Vulkan headers only (the hook resolves entry points itself, VK_NO_PROTOTYPES). Use the include +# dir directly rather than the third_party `vulkan_headers` target, so the x86 sub-build -- which +# doesn't add third_party/ -- still builds the hook. +coop_require_submodule("Vulkan-Headers" "third_party/Vulkan-Headers/include/vulkan/vulkan.h") +target_include_directories(coop_hook PRIVATE ${CMAKE_SOURCE_DIR}/third_party/Vulkan-Headers/include) + target_link_libraries(coop_hook PRIVATE coop_common safetyhook::safetyhook diff --git a/hook/src/dllmain.cpp b/hook/src/dllmain.cpp index e79d726..138a228 100644 --- a/hook/src/dllmain.cpp +++ b/hook/src/dllmain.cpp @@ -24,6 +24,7 @@ #include "d3d9_hook.hpp" #include "opengl_hook.hpp" #include "present_hook.hpp" +#include "vk_hook.hpp" #include "xinput_hook.hpp" namespace @@ -73,6 +74,7 @@ DWORD WINAPI worker_thread(LPVOID) bool audio_installed = false; bool audio_ring_open = false; bool video_installed = false; + bool vk_installed = false; // tracked separately: vulkan-1.dll loads lazily, so retry until present bool mkb_installed = false; // Each tick, reconcile each subsystem with the host's requested state: install @@ -143,11 +145,24 @@ DWORD WINAPI worker_thread(LPVOID) present_ok ? 1 : 0, gl_ok ? 1 : 0, d3d9_ok ? 1 : 0); } } + // Vulkan separately: vulkan-1.dll loads lazily (volk dlopens it after start), so the DXGI/ + // GL/D3D9 hooks above may install before it exists. Keep trying each tick until it appears. + if (want_video && !vk_installed) + { + vk_installed = coop::hook::install_vk_hooks(g_ipc); + if (vk_installed) + { + video_installed = true; // a Vulkan-only game otherwise has no video hook installed + coop::hook::logf("worker_thread: vulkan video hook installed"); + } + } else if (!want_video && video_installed) { coop::hook::remove_present_hooks(); coop::hook::remove_opengl_hooks(); coop::hook::remove_d3d9_hooks(); + coop::hook::remove_vk_hooks(); + vk_installed = false; video_installed = false; coop::hook::logf("worker_thread: video hooks removed (host request)"); } @@ -259,6 +274,7 @@ BOOL APIENTRY DllMain(HMODULE module, DWORD reason, LPVOID reserved) coop::hook::remove_present_hooks(); coop::hook::remove_opengl_hooks(); coop::hook::remove_d3d9_hooks(); + coop::hook::remove_vk_hooks(); coop::hook::remove_mkb_hooks(); coop::hook::hook_registry_reset(); } diff --git a/hook/src/vk_hook.cpp b/hook/src/vk_hook.cpp new file mode 100644 index 0000000..652c268 --- /dev/null +++ b/hook/src/vk_hook.cpp @@ -0,0 +1,712 @@ +#include "vk_hook.hpp" + +#include +#include +#include + +#include + +#include +#include + +#define VK_NO_PROTOTYPES +#define VK_USE_PLATFORM_WIN32_KHR +#include + +#include + +#include "coop/protocol.hpp" +#include "coop/shared_memory.hpp" +#include "debug_log.hpp" +#include "hook_registry.hpp" + +namespace coop::hook +{ + +namespace +{ + +IpcClient* g_ipc = nullptr; +unsigned long g_pid = 0; + +safetyhook::InlineHook g_hk_gipa; // vulkan-1.dll!vkGetInstanceProcAddr +int g_id_present = -1; + +std::atomic g_presents{0}; +std::atomic g_frames_shared{0}; +std::atomic g_present_captured{false}; // set once we successfully read a present back +bool g_unsupported_logged = false; + +// Real entry points. g_real_gdpa and below are unhooked exports/results, so they're plain PFNs; +// the real vkGetInstanceProcAddr is reached through the inline hook's trampoline (real_gipa()). +PFN_vkGetDeviceProcAddr g_real_gdpa = nullptr; +PFN_vkCreateDevice g_real_create_device = nullptr; +PFN_vkCreateSwapchainKHR g_real_create_swapchain = nullptr; +PFN_vkQueuePresentKHR g_real_present = nullptr; + +// The single tracked device's real functions (one device is the overwhelmingly common case). +struct VkFns +{ + PFN_vkGetDeviceQueue GetDeviceQueue; + PFN_vkCreateCommandPool CreateCommandPool; + PFN_vkDestroyCommandPool DestroyCommandPool; + PFN_vkAllocateCommandBuffers AllocateCommandBuffers; + PFN_vkBeginCommandBuffer BeginCommandBuffer; + PFN_vkEndCommandBuffer EndCommandBuffer; + PFN_vkResetCommandBuffer ResetCommandBuffer; + PFN_vkCmdPipelineBarrier CmdPipelineBarrier; + PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer; + PFN_vkQueueSubmit QueueSubmit; + PFN_vkCreateFence CreateFence; + PFN_vkDestroyFence DestroyFence; + PFN_vkWaitForFences WaitForFences; + PFN_vkResetFences ResetFences; + PFN_vkCreateSemaphore CreateSemaphore; + PFN_vkDestroySemaphore DestroySemaphore; + PFN_vkCreateBuffer CreateBuffer; + PFN_vkDestroyBuffer DestroyBuffer; + PFN_vkGetBufferMemoryRequirements GetBufferMemoryRequirements; + PFN_vkAllocateMemory AllocateMemory; + PFN_vkFreeMemory FreeMemory; + PFN_vkBindBufferMemory BindBufferMemory; + PFN_vkMapMemory MapMemory; + PFN_vkUnmapMemory UnmapMemory; + PFN_vkGetSwapchainImagesKHR GetSwapchainImagesKHR; + PFN_vkDeviceWaitIdle DeviceWaitIdle; + PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties; // instance-level +}; +VkFns g_fns{}; + +VkInstance g_instance = VK_NULL_HANDLE; +VkPhysicalDevice g_phys = VK_NULL_HANDLE; +VkDevice g_device = VK_NULL_HANDLE; +std::uint32_t g_qfam = 0; +VkQueue g_queue = VK_NULL_HANDLE; + +// Read-back resources on the game's device (created once, staging re-created on size change). +VkCommandPool g_pool = VK_NULL_HANDLE; +VkCommandBuffer g_cmd = VK_NULL_HANDLE; +VkFence g_fence = VK_NULL_HANDLE; +VkSemaphore g_present_sem = VK_NULL_HANDLE; // re-chains the present's wait (see hk_vkQueuePresentKHR) +VkBuffer g_staging = VK_NULL_HANDLE; +VkDeviceMemory g_staging_mem = VK_NULL_HANDLE; +VkDeviceSize g_staging_size = 0; +void* g_staging_mapped = nullptr; + +// Tracked swap chains (small; engines have one or two). +struct SwapInfo +{ + VkSwapchainKHR sc; + VkFormat fmt; + std::uint32_t w; + std::uint32_t h; + std::vector images; +}; +std::vector g_swaps; + +// Hook-owned D3D11 device + shared keyed-mutex texture (the Vulkan path has no D3D device). +ID3D11Device* g_d3d = nullptr; +ID3D11DeviceContext* g_d3d_ctx = nullptr; +ID3D11Texture2D* g_shared_tex = nullptr; +IDXGIKeyedMutex* g_shared_mutex = nullptr; +HANDLE g_shared_handle = nullptr; +UINT g_share_w = 0; +UINT g_share_h = 0; +std::vector g_rgba; + +// --- the real vkGetInstanceProcAddr, via the inline hook's trampoline -------- +PFN_vkVoidFunction real_gipa(VkInstance inst, const char* name) +{ + return g_hk_gipa.stdcall(inst, name); +} + +// --- aux D3D11 shared texture (same contract as opengl_hook / d3d9_hook) ------ +bool ensure_d3d() +{ + if (g_d3d != nullptr) + { + return true; + } + const HRESULT hr = + D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION, &g_d3d, + nullptr, &g_d3d_ctx); + if (FAILED(hr) || g_d3d == nullptr) + { + logf("vk: D3D11CreateDevice failed hr=0x%08lX", static_cast(hr)); + return false; + } + return true; +} + +void release_shared() +{ + if (g_shared_mutex != nullptr) + { + g_shared_mutex->Release(); + g_shared_mutex = nullptr; + } + if (g_shared_tex != nullptr) + { + g_shared_tex->Release(); + g_shared_tex = nullptr; + } + if (g_shared_handle != nullptr) + { + CloseHandle(g_shared_handle); + g_shared_handle = nullptr; + } + g_share_w = g_share_h = 0; +} + +bool ensure_shared_texture(UINT w, UINT h) +{ + if (g_shared_tex != nullptr && g_share_w == w && g_share_h == h) + { + return true; + } + release_shared(); + D3D11_TEXTURE2D_DESC desc{}; + desc.Width = w; + desc.Height = h; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; // we swizzle the Vulkan image to RGBA + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX; + if (FAILED(g_d3d->CreateTexture2D(&desc, nullptr, &g_shared_tex)) || g_shared_tex == nullptr) + { + return false; + } + IDXGIResource1* res = nullptr; + if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast(&res))) || + res == nullptr) + { + release_shared(); + return false; + } + const std::wstring name = video_share_name(g_pid); + const HRESULT hr = res->CreateSharedHandle( + nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, name.c_str(), &g_shared_handle); + res->Release(); + if (FAILED(hr) || g_shared_handle == nullptr) + { + release_shared(); + return false; + } + if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast(&g_shared_mutex)))) + { + release_shared(); + return false; + } + g_share_w = w; + g_share_h = h; + logf("vk: shared texture ready %ux%u name=%ls", w, h, name.c_str()); + return true; +} + +// --- Vulkan read-back resources ---------------------------------------------- +bool find_host_visible_memory(std::uint32_t type_bits, std::uint32_t& out_index) +{ + VkPhysicalDeviceMemoryProperties mp{}; + g_fns.GetPhysicalDeviceMemoryProperties(g_phys, &mp); + for (std::uint32_t i = 0; i < mp.memoryTypeCount; ++i) + { + const bool usable = (type_bits & (1u << i)) != 0; + const bool host = (mp.memoryTypes[i].propertyFlags & + (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == + (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (usable && host) + { + out_index = i; + return true; + } + } + return false; +} + +void release_staging() +{ + if (g_staging_mapped != nullptr && g_staging_mem != VK_NULL_HANDLE) + { + g_fns.UnmapMemory(g_device, g_staging_mem); + g_staging_mapped = nullptr; + } + if (g_staging != VK_NULL_HANDLE) + { + g_fns.DestroyBuffer(g_device, g_staging, nullptr); + g_staging = VK_NULL_HANDLE; + } + if (g_staging_mem != VK_NULL_HANDLE) + { + g_fns.FreeMemory(g_device, g_staging_mem, nullptr); + g_staging_mem = VK_NULL_HANDLE; + } + g_staging_size = 0; +} + +bool ensure_vk_resources(std::uint32_t w, std::uint32_t h) +{ + if (g_queue == VK_NULL_HANDLE) + { + g_fns.GetDeviceQueue(g_device, g_qfam, 0, &g_queue); + } + if (g_pool == VK_NULL_HANDLE) + { + VkCommandPoolCreateInfo pci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pci.queueFamilyIndex = g_qfam; + if (g_fns.CreateCommandPool(g_device, &pci, nullptr, &g_pool) != VK_SUCCESS) + { + return false; + } + VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ai.commandPool = g_pool; + ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + ai.commandBufferCount = 1; + if (g_fns.AllocateCommandBuffers(g_device, &ai, &g_cmd) != VK_SUCCESS) + { + return false; + } + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + if (g_fns.CreateFence(g_device, &fi, nullptr, &g_fence) != VK_SUCCESS) + { + return false; + } + VkSemaphoreCreateInfo si{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; + if (g_fns.CreateSemaphore(g_device, &si, nullptr, &g_present_sem) != VK_SUCCESS) + { + return false; + } + } + const VkDeviceSize need = static_cast(w) * h * 4; + if (g_staging != VK_NULL_HANDLE && g_staging_size == need) + { + return true; + } + release_staging(); + VkBufferCreateInfo bci{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bci.size = need; + bci.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (g_fns.CreateBuffer(g_device, &bci, nullptr, &g_staging) != VK_SUCCESS) + { + return false; + } + VkMemoryRequirements mr{}; + g_fns.GetBufferMemoryRequirements(g_device, g_staging, &mr); + std::uint32_t mem_type = 0; + if (!find_host_visible_memory(mr.memoryTypeBits, mem_type)) + { + release_staging(); + return false; + } + VkMemoryAllocateInfo mai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = mr.size; + mai.memoryTypeIndex = mem_type; + if (g_fns.AllocateMemory(g_device, &mai, nullptr, &g_staging_mem) != VK_SUCCESS || + g_fns.BindBufferMemory(g_device, g_staging, g_staging_mem, 0) != VK_SUCCESS || + g_fns.MapMemory(g_device, g_staging_mem, 0, VK_WHOLE_SIZE, 0, &g_staging_mapped) != VK_SUCCESS) + { + release_staging(); + return false; + } + g_staging_size = need; + return true; +} + +void image_barrier(VkCommandBuffer cb, VkImage img, VkImageLayout from, VkImageLayout to, VkAccessFlags src, + VkAccessFlags dst) +{ + VkImageMemoryBarrier b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + b.srcAccessMask = src; + b.dstAccessMask = dst; + b.oldLayout = from; + b.newLayout = to; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.image = img; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + g_fns.CmdPipelineBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, + nullptr, 0, nullptr, 1, &b); +} + +// Read `image` (in PRESENT_SRC layout) back into the shared texture. `wait`/`wait_count` are the +// present's wait semaphores, which our submit consumes and replaces with g_present_sem (returned +// via *out_sem) so the real present orders correctly after our copy without double-waiting. +bool capture_present(VkImage image, VkFormat fmt, std::uint32_t w, std::uint32_t h, const VkSemaphore* wait, + std::uint32_t wait_count, VkSemaphore& out_sem) +{ + const bool bgra = fmt == VK_FORMAT_B8G8R8A8_UNORM || fmt == VK_FORMAT_B8G8R8A8_SRGB; + const bool rgba = fmt == VK_FORMAT_R8G8B8A8_UNORM || fmt == VK_FORMAT_R8G8B8A8_SRGB; + if (!bgra && !rgba) + { + if (!g_unsupported_logged) + { + logf("vk: unsupported swapchain format=%d (only 8888 BGRA/RGBA); idle", static_cast(fmt)); + g_unsupported_logged = true; + } + return false; + } + if (!ensure_d3d() || !ensure_shared_texture(w, h) || !ensure_vk_resources(w, h)) + { + return false; + } + + g_fns.ResetCommandBuffer(g_cmd, 0); + VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + g_fns.BeginCommandBuffer(g_cmd, &bi); + image_barrier(g_cmd, image, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {w, h, 1}; + g_fns.CmdCopyImageToBuffer(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, g_staging, 1, ®ion); + image_barrier(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_MEMORY_READ_BIT); + g_fns.EndCommandBuffer(g_cmd); + + std::vector wait_stages(wait_count, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.waitSemaphoreCount = wait_count; + si.pWaitSemaphores = wait; + si.pWaitDstStageMask = wait_count != 0 ? wait_stages.data() : nullptr; + si.commandBufferCount = 1; + si.pCommandBuffers = &g_cmd; + si.signalSemaphoreCount = 1; + si.pSignalSemaphores = &g_present_sem; + g_fns.ResetFences(g_device, 1, &g_fence); + if (g_fns.QueueSubmit(g_queue, 1, &si, g_fence) != VK_SUCCESS) + { + return false; + } + g_fns.WaitForFences(g_device, 1, &g_fence, VK_TRUE, UINT64_MAX); + + // Swizzle the mapped staging (tightly packed w*4) into RGBA and upload. + const size_t row = static_cast(w) * 4; + if (g_rgba.size() != row * h) + { + g_rgba.resize(row * h); + } + const auto* src = static_cast(g_staging_mapped); + for (std::uint32_t y = 0; y < h; ++y) + { + const unsigned char* s = src + static_cast(y) * row; + unsigned char* o = g_rgba.data() + static_cast(y) * row; + if (bgra) + { + for (std::uint32_t x = 0; x < w; ++x) + { + o[x * 4 + 0] = s[x * 4 + 2]; + o[x * 4 + 1] = s[x * 4 + 1]; + o[x * 4 + 2] = s[x * 4 + 0]; + o[x * 4 + 3] = 255; + } + } + else + { + std::memcpy(o, s, row); + } + } + + bool shared = false; + if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK) + { + g_d3d_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, g_rgba.data(), static_cast(row), 0); + g_d3d_ctx->Flush(); + g_shared_mutex->ReleaseSync(kVideoMutexKey); + shared = true; + } + out_sem = g_present_sem; + if (shared) + { + g_present_captured.store(true, std::memory_order_relaxed); + g_frames_shared.fetch_add(1, std::memory_order_relaxed); + if (g_ipc != nullptr) + { + g_ipc->publish_video_frame(w, h, static_cast(DXGI_FORMAT_R8G8B8A8_UNORM)); + } + } + return true; +} + +const SwapInfo* find_swap(VkSwapchainKHR sc) +{ + for (const SwapInfo& s : g_swaps) + { + if (s.sc == sc) + { + return &s; + } + } + return nullptr; +} + +VKAPI_ATTR VkResult VKAPI_CALL hk_vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* pPresentInfo) +{ + hook_note_call(g_id_present); + g_presents.fetch_add(1, std::memory_order_relaxed); + if (g_ipc != nullptr) + { + g_ipc->note_present(); + } + + // Capture only the simple, common single-swapchain present; pass anything else through. + if (g_device != VK_NULL_HANDLE && pPresentInfo != nullptr && pPresentInfo->swapchainCount == 1) + { + const SwapInfo* s = find_swap(pPresentInfo->pSwapchains[0]); + if (s != nullptr && pPresentInfo->pImageIndices[0] < s->images.size()) + { + VkSemaphore chained = VK_NULL_HANDLE; + if (capture_present(s->images[pPresentInfo->pImageIndices[0]], s->fmt, s->w, s->h, + pPresentInfo->pWaitSemaphores, pPresentInfo->waitSemaphoreCount, chained)) + { + // Replace the present's wait with our chained semaphore (our submit consumed the + // originals and signals this one), so the present still orders after rendering. + VkPresentInfoKHR pi = *pPresentInfo; + pi.waitSemaphoreCount = 1; + pi.pWaitSemaphores = &chained; + return g_real_present(queue, &pi); + } + } + } + return g_real_present(queue, pPresentInfo); +} + +VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR* ci, + const VkAllocationCallbacks* alloc, VkSwapchainKHR* out) +{ + const VkResult r = g_real_create_swapchain(device, ci, alloc, out); + if (r == VK_SUCCESS && out != nullptr && g_fns.GetSwapchainImagesKHR != nullptr) + { + SwapInfo info{}; + info.sc = *out; + info.fmt = ci->imageFormat; + info.w = ci->imageExtent.width; + info.h = ci->imageExtent.height; + std::uint32_t n = 0; + g_fns.GetSwapchainImagesKHR(device, *out, &n, nullptr); + info.images.resize(n); + g_fns.GetSwapchainImagesKHR(device, *out, &n, info.images.data()); + g_swaps.push_back(std::move(info)); + // (Don't log the VkSwapchainKHR handle: it's a uint64_t on x86, not a pointer.) + logf("vk: swapchain %ux%u fmt=%d images=%u", ci->imageExtent.width, ci->imageExtent.height, + static_cast(ci->imageFormat), n); + } + return r; +} + +// Resolve every device function we need for the read-back via the real vkGetDeviceProcAddr. +void load_device_fns(VkDevice device) +{ +#define LOAD(field, vkname) g_fns.field = reinterpret_cast(g_real_gdpa(device, #vkname)) + LOAD(GetDeviceQueue, vkGetDeviceQueue); + LOAD(CreateCommandPool, vkCreateCommandPool); + LOAD(DestroyCommandPool, vkDestroyCommandPool); + LOAD(AllocateCommandBuffers, vkAllocateCommandBuffers); + LOAD(BeginCommandBuffer, vkBeginCommandBuffer); + LOAD(EndCommandBuffer, vkEndCommandBuffer); + LOAD(ResetCommandBuffer, vkResetCommandBuffer); + LOAD(CmdPipelineBarrier, vkCmdPipelineBarrier); + LOAD(CmdCopyImageToBuffer, vkCmdCopyImageToBuffer); + LOAD(QueueSubmit, vkQueueSubmit); + LOAD(CreateFence, vkCreateFence); + LOAD(DestroyFence, vkDestroyFence); + LOAD(WaitForFences, vkWaitForFences); + LOAD(ResetFences, vkResetFences); + LOAD(CreateSemaphore, vkCreateSemaphore); + LOAD(DestroySemaphore, vkDestroySemaphore); + LOAD(CreateBuffer, vkCreateBuffer); + LOAD(DestroyBuffer, vkDestroyBuffer); + LOAD(GetBufferMemoryRequirements, vkGetBufferMemoryRequirements); + LOAD(AllocateMemory, vkAllocateMemory); + LOAD(FreeMemory, vkFreeMemory); + LOAD(BindBufferMemory, vkBindBufferMemory); + LOAD(MapMemory, vkMapMemory); + LOAD(UnmapMemory, vkUnmapMemory); + LOAD(GetSwapchainImagesKHR, vkGetSwapchainImagesKHR); + LOAD(DeviceWaitIdle, vkDeviceWaitIdle); +#undef LOAD + g_fns.GetPhysicalDeviceMemoryProperties = + reinterpret_cast(real_gipa(g_instance, "vkGetPhysicalDeviceMemoryProperties")); +} + +VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateDevice(VkPhysicalDevice phys, const VkDeviceCreateInfo* ci, + const VkAllocationCallbacks* alloc, VkDevice* out) +{ + const VkResult r = g_real_create_device(phys, ci, alloc, out); + if (r == VK_SUCCESS && out != nullptr && g_device == VK_NULL_HANDLE) // track the first device + { + g_phys = phys; + g_device = *out; + g_qfam = ci->queueCreateInfoCount > 0 ? ci->pQueueCreateInfos[0].queueFamilyIndex : 0; + g_real_gdpa = reinterpret_cast(real_gipa(g_instance, "vkGetDeviceProcAddr")); + g_real_create_swapchain = + reinterpret_cast(g_real_gdpa(*out, "vkCreateSwapchainKHR")); + g_real_present = reinterpret_cast(g_real_gdpa(*out, "vkQueuePresentKHR")); + load_device_fns(*out); + logf("vk: device created (qfam=%u) -- present capture armed", g_qfam); + } + return r; +} + +VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateInstance(const VkInstanceCreateInfo* ci, + const VkAllocationCallbacks* alloc, VkInstance* out) +{ + auto real_create = reinterpret_cast(real_gipa(nullptr, "vkCreateInstance")); + const VkResult r = real_create(ci, alloc, out); + if (r == VK_SUCCESS && out != nullptr) + { + g_instance = *out; + g_real_create_device = reinterpret_cast(real_gipa(*out, "vkCreateDevice")); + logf("vk: instance created -- intercepting device/swapchain/present"); + } + return r; +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL hk_vkGetDeviceProcAddr(VkDevice device, const char* name) +{ + if (name != nullptr) + { + if (std::strcmp(name, "vkQueuePresentKHR") == 0) + { + return reinterpret_cast(&hk_vkQueuePresentKHR); + } + if (std::strcmp(name, "vkCreateSwapchainKHR") == 0) + { + return reinterpret_cast(&hk_vkCreateSwapchainKHR); + } + } + return g_real_gdpa != nullptr ? g_real_gdpa(device, name) : nullptr; +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL hk_vkGetInstanceProcAddr(VkInstance instance, const char* name) +{ + if (name != nullptr) + { + if (std::strcmp(name, "vkGetInstanceProcAddr") == 0) + { + return reinterpret_cast(&hk_vkGetInstanceProcAddr); + } + if (std::strcmp(name, "vkCreateInstance") == 0) + { + return reinterpret_cast(&hk_vkCreateInstance); + } + if (std::strcmp(name, "vkCreateDevice") == 0) + { + return reinterpret_cast(&hk_vkCreateDevice); + } + if (std::strcmp(name, "vkGetDeviceProcAddr") == 0) + { + return reinterpret_cast(&hk_vkGetDeviceProcAddr); + } + } + return real_gipa(instance, name); +} + +void* gipa_export_address() +{ + HMODULE vk = GetModuleHandleW(L"vulkan-1.dll"); + if (vk == nullptr) + { + return nullptr; // not a Vulkan process (yet) + } + return reinterpret_cast(GetProcAddress(vk, "vkGetInstanceProcAddr")); +} + +} // namespace + +bool install_vk_hooks(IpcClient& ipc) +{ + g_ipc = &ipc; + g_pid = GetCurrentProcessId(); + if (g_hk_gipa) + { + return true; // already installed + } + if (g_id_present < 0) + { + g_id_present = hook_register("vkQueuePresentKHR", HookSubsys_Video); + } + void* gipa = gipa_export_address(); + if (gipa == nullptr) + { + hook_set_installed(g_id_present, false); + return false; // vulkan-1.dll not loaded; caller can retry once the game loads it + } + g_unsupported_logged = false; + g_hk_gipa = safetyhook::create_inline(gipa, reinterpret_cast(&hk_vkGetInstanceProcAddr)); + hook_set_installed(g_id_present, static_cast(g_hk_gipa)); + logf("install_vk_hooks: vkGetInstanceProcAddr=%p hooked=%d", gipa, static_cast(g_hk_gipa) ? 1 : 0); + return static_cast(g_hk_gipa); +} + +void remove_vk_hooks() +{ + if (g_device != VK_NULL_HANDLE && g_fns.DeviceWaitIdle != nullptr) + { + g_fns.DeviceWaitIdle(g_device); + release_staging(); + if (g_present_sem != VK_NULL_HANDLE) + { + g_fns.DestroySemaphore(g_device, g_present_sem, nullptr); + g_present_sem = VK_NULL_HANDLE; + } + if (g_fence != VK_NULL_HANDLE) + { + g_fns.DestroyFence(g_device, g_fence, nullptr); + g_fence = VK_NULL_HANDLE; + } + if (g_pool != VK_NULL_HANDLE) + { + g_fns.DestroyCommandPool(g_device, g_pool, nullptr); + g_pool = VK_NULL_HANDLE; + } + } + g_hk_gipa = {}; + hook_set_installed(g_id_present, false); + release_shared(); + if (g_d3d_ctx != nullptr) + { + g_d3d_ctx->Release(); + g_d3d_ctx = nullptr; + } + if (g_d3d != nullptr) + { + g_d3d->Release(); + g_d3d = nullptr; + } + g_swaps.clear(); + g_rgba.clear(); + g_cmd = VK_NULL_HANDLE; + g_queue = VK_NULL_HANDLE; + g_device = VK_NULL_HANDLE; + g_instance = VK_NULL_HANDLE; + g_real_gdpa = nullptr; + g_present_captured.store(false, std::memory_order_relaxed); + g_presents.store(0, std::memory_order_relaxed); + g_frames_shared.store(0, std::memory_order_relaxed); + g_ipc = nullptr; +} + +std::uint64_t vk_presents() +{ + return g_presents.load(std::memory_order_relaxed); +} + +std::uint64_t vk_frames_shared() +{ + return g_frames_shared.load(std::memory_order_relaxed); +} + +bool vk_injected_too_late() +{ + // vulkan-1.dll is loaded but we never captured a present -> the app resolved its present + // pointer before we hooked (or doesn't go through our chain). The host shows the banner. + return GetModuleHandleW(L"vulkan-1.dll") != nullptr && g_presents.load(std::memory_order_relaxed) == 0 && + !g_present_captured.load(std::memory_order_relaxed); +} + +} // namespace coop::hook diff --git a/hook/src/vk_hook.hpp b/hook/src/vk_hook.hpp new file mode 100644 index 0000000..ae6ba6c --- /dev/null +++ b/hook/src/vk_hook.hpp @@ -0,0 +1,34 @@ +// Injected Vulkan capture path. Vulkan games present via vkQueuePresentKHR and (with volk or +// their own loader) cache that pointer at init, so they can't be hooked by *late* injection -- +// the hook must be present before vkCreateInstance. We catch the resolution chain instead: +// inline-hook the vulkan-1.dll export vkGetInstanceProcAddr and hand back our own wrappers for +// vkCreateInstance / vkCreateDevice / vkGetDeviceProcAddr / vkCreateSwapchainKHR / +// vkQueuePresentKHR. On present we read the swap-chain image back with vkCmdCopyImageToBuffer +// (the same read-back pattern as the D3D10/D3D9/OpenGL paths) and upload it into the shared +// keyed-mutex texture via a hook-owned D3D11 device. Part of the video subsystem. +#pragma once + +#include "ipc_client.hpp" + +namespace coop::hook +{ + +// Installs the Vulkan capture hook (inline-hooks vkGetInstanceProcAddr). `ipc` must outlive the +// hook. Returns true if vulkan-1.dll is loaded and the export was hooked; false otherwise, so +// the caller can retry once the game loads Vulkan. Safe to call repeatedly. +bool install_vk_hooks(IpcClient& ipc); + +// Removes the Vulkan hook and releases the shared texture + Vulkan read-back resources. +void remove_vk_hooks(); + +// --- Diagnostics ----------------------------------------------------------- + +std::uint64_t vk_presents(); // cumulative vkQueuePresentKHR detours +std::uint64_t vk_frames_shared(); // frames uploaded into the shared texture + +// True when vulkan-1.dll is loaded in this process but we never captured a present -- i.e. the +// app resolved its present pointer before we hooked (injected too late). Drives the host's +// "Vulkan, injected too late" banner. +bool vk_injected_too_late(); + +} // namespace coop::hook diff --git a/tests/mock_game_test.cpp b/tests/mock_game_test.cpp index 3161b79..af95f60 100644 --- a/tests/mock_game_test.cpp +++ b/tests/mock_game_test.cpp @@ -300,35 +300,107 @@ void test_video_capture(const char* backend, ID3D11Device* device) game.kill(); } -// Liveness smoke check for a backend whose capture can't be exercised by late injection -// (Vulkan caches its present pointer at init): launch it for a couple of seconds and assert it -// comes up and exits cleanly. Exit code 2 = backend unavailable on this machine (e.g. no Vulkan -// driver) -> skip without failing. -void test_liveness(const char* backend) +// Vulkan capture: Vulkan caches its present pointer at init, so late injection can't hook it. +// We launch the mock **suspended**, inject the hook, and resume; under COOP_MOCK_VK_EARLY the mock +// loads vulkan-1.dll and waits, giving the hook's worker time to hook vkGetInstanceProcAddr before +// the mock calls vkCreateInstance. Then we decode frames out of the captured pixels like the other +// backends. Exit code 2 = no Vulkan driver -> skip without failing. +void test_vk_capture(ID3D11Device* device) { - std::printf("== liveness: %s ==\n", backend); - std::wstring args; - for (const char* p = backend; *p != '\0'; ++p) + std::printf("== video capture: vk (early-load) ==\n"); + SetEnvironmentVariableW(L"COOP_MOCK_VK_EARLY", L"1"); + PROCESS_INFORMATION pi{}; + STARTUPINFOW si{}; + si.cb = sizeof(si); + const std::wstring exe = tool_path(L"coop_mock_game.exe"); + std::wstring cmd = L"\"" + exe + L"\" vk 30"; + const BOOL ok = + CreateProcessW(exe.c_str(), cmd.data(), nullptr, nullptr, FALSE, CREATE_SUSPENDED, nullptr, nullptr, &si, &pi); + SetEnvironmentVariableW(L"COOP_MOCK_VK_EARLY", nullptr); + if (!ok) { - args.push_back(static_cast(*p)); - } - args += L" 2"; // run 2s then exit on its own - MockGame game = MockGame::launch(args); - if (!game.ok) - { - check(false, "launch coop_mock_game (liveness)"); + check(false, "launch suspended vk mock"); return; } - WaitForSingleObject(game.pi.hProcess, 6000); // let the 2s run finish - if (!game.alive() && game.exit_code() == 2) + + SharedMemory shm; + const std::uint32_t disabled = (1u << HookSubsys_Input) | (1u << HookSubsys_Focus) | + (1u << HookSubsys_Audio) | (1u << HookSubsys_Mkb); + SharedBlock* block = make_ipc(shm, pi.dwProcessId, disabled); + const bool injected = block != nullptr && inject_retry(pi.dwProcessId); + ResumeThread(pi.hThread); // the mock loads vulkan + waits, then renders + auto cleanup = [&] { + TerminateProcess(pi.hProcess, 0); + WaitForSingleObject(pi.hProcess, 2000); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + }; + if (!injected) { - std::printf(" backend '%s' unavailable on this machine -- skipping\n", backend); - game.kill(); + check(false, "inject suspended vk mock"); + cleanup(); return; } - check(!game.alive(), "backend ran and exited within the time limit (no hang)"); - check(game.exit_code() == 0, "backend came up and exited cleanly"); - game.kill(); + + auto alive = [&] { return WaitForSingleObject(pi.hProcess, 0) == WAIT_TIMEOUT; }; + auto exit_code = [&] { + DWORD c = 0; + GetExitCodeProcess(pi.hProcess, &c); + return c; + }; + + SharedTextureSource src; + src.init(device); + std::vector seq; + std::uint64_t backward = 0; + std::uint32_t last = 0; + bool have_last = false; + for (int i = 0; i < 200 && alive(); ++i) // up to ~10 s (the mock waits 1.5 s at start) + { + Sleep(50); + const VideoShareView share = read_video_share(block); + if (!src.update(share, pi.dwProcessId)) + { + continue; + } + std::uint8_t px[4] = {}; + if (!src.read_pixel(coop::mock::kFrameBlock / 2, coop::mock::kFrameBlock / 2, px)) + { + continue; + } + const std::uint32_t f = coop::mock::rgb_to_frame(px[0], px[1], px[2]); + if (have_last && f < last) + { + ++backward; + } + last = f; + have_last = true; + seq.push_back(f); + if (seq.size() >= 40) + { + break; + } + } + + if (!alive() && exit_code() == 2 && seq.empty()) + { + std::printf(" Vulkan unavailable on this machine -- skipping vk capture\n"); + cleanup(); + return; + } + const std::uint32_t present = static_cast(block->video.present_calls); + std::printf(" present_calls=%u copied=%llu samples=%zu backward=%llu\n", present, + static_cast(src.frames_copied()), seq.size(), + static_cast(backward)); + check(present > 0, "hook captured Vulkan present calls"); + check(src.frames_copied() >= 5, "host copied multiple shared frames (vk)"); + check(seq.size() >= 5, "decoded multiple frame numbers from the captured pixels (vk)"); + check(backward == 0, "captured vk frame numbers never go backwards"); + if (!seq.empty()) + { + check(seq.back() - seq.front() >= 10, "captured vk frame numbers advance"); + } + cleanup(); } // Launch the mock game rendering audio at `rate`/`channels`/`bits`/`fmt`, inject the audio @@ -553,10 +625,9 @@ int main() return 0; } - // Vulkan: its present pointer is cached at init, so late injection can't hook it -- a - // liveness check (the mock comes up + presents + exits) is the automated coverage here; the - // capture path is exercised via the early-load path documented in the README. - test_liveness("vk"); + // Vulkan: present pointer cached at init -> can't be late-hooked, so we capture via the + // early-load path (suspended launch + inject + resume; the mock loads Vulkan and waits). + test_vk_capture(device); test_video_capture("gl", device); test_video_capture("dx9ex", device); test_video_capture("dx9", device); diff --git a/tools/mock_game/main.cpp b/tools/mock_game/main.cpp index 685ec41..64955c1 100644 --- a/tools/mock_game/main.cpp +++ b/tools/mock_game/main.cpp @@ -96,6 +96,15 @@ int main(int argc, char** argv) return 1; } + // Test hook (early-load): Vulkan caches its present pointer at init, so the capture hook must + // be in place before vkCreateInstance. Under COOP_MOCK_VK_EARLY the mock loads vulkan-1.dll + // now and waits, giving an already-injected hook time to hook vkGetInstanceProcAddr first. + if (backend_name == "vk" && GetEnvironmentVariableW(L"COOP_MOCK_VK_EARLY", nullptr, 0) != 0) + { + LoadLibraryW(L"vulkan-1.dll"); + Sleep(1500); + } + auto backend = coop::mock::RenderBackend::create(backend_name); if (!backend || !backend->init(hwnd, kW, kH)) {