M2(Vulkan): capture hook via GPA interception + vkCmdCopyImageToBuffer read-back
New vk_hook.cpp inline-hooks the vulkan-1.dll vkGetInstanceProcAddr export and hands back our wrappers for vkCreateInstance / vkCreateDevice / vkGetDeviceProcAddr / vkCreateSwapchainKHR / vkQueuePresentKHR, so a volk-using (loader-bypass) app resolves our hooks. On present it reads the swap-chain image back with vkCmdCopyImageToBuffer into a host-visible buffer (same read-back model as D3D10/D3D9/OpenGL), swizzles BGRA->RGBA, and uploads it into the shared keyed-mutex texture on a hook-owned D3D11 device. The read-back submit re-chains the present's wait semaphores (consume the originals, signal our own that the real present waits on) so capture orders after rendering without double-waiting. Wired into the video subsystem with a lazy retry (vulkan-1.dll loads late). Hook links the official Vulkan-Headers (headers only, VK_NO_PROTOTYPES) via the include dir so the x86 sub-build builds too. Because Vulkan caches its present pointer at init, late injection can't hook it: mock_game_test launches the mock **suspended**, injects, resumes, and under COOP_MOCK_VK_EARLY the mock loads Vulkan and waits so the hook arms first -- then decodes frames through the hook like the other backends. 15/15 ctest (x64 + x86); skips cleanly without a Vulkan driver. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ add_library(coop_hook SHARED
|
||||
src/present_hook.cpp
|
||||
src/opengl_hook.cpp
|
||||
src/d3d9_hook.cpp
|
||||
src/vk_hook.cpp
|
||||
src/mkb_hook.cpp
|
||||
src/debug_log.cpp
|
||||
src/hook_registry.cpp)
|
||||
@@ -16,6 +17,12 @@ target_include_directories(coop_hook PRIVATE src)
|
||||
# the Windows 10 20H1 (NTDDI_WIN10_CO) headers.
|
||||
target_compile_definitions(coop_hook PRIVATE NTDDI_VERSION=0x0A00000B)
|
||||
|
||||
# Vulkan headers only (the hook resolves entry points itself, VK_NO_PROTOTYPES). Use the include
|
||||
# dir directly rather than the third_party `vulkan_headers` target, so the x86 sub-build -- which
|
||||
# doesn't add third_party/ -- still builds the hook.
|
||||
coop_require_submodule("Vulkan-Headers" "third_party/Vulkan-Headers/include/vulkan/vulkan.h")
|
||||
target_include_directories(coop_hook PRIVATE ${CMAKE_SOURCE_DIR}/third_party/Vulkan-Headers/include)
|
||||
|
||||
target_link_libraries(coop_hook PRIVATE
|
||||
coop_common
|
||||
safetyhook::safetyhook
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
#include "d3d9_hook.hpp"
|
||||
#include "opengl_hook.hpp"
|
||||
#include "present_hook.hpp"
|
||||
#include "vk_hook.hpp"
|
||||
#include "xinput_hook.hpp"
|
||||
|
||||
namespace
|
||||
@@ -73,6 +74,7 @@ DWORD WINAPI worker_thread(LPVOID)
|
||||
bool audio_installed = false;
|
||||
bool audio_ring_open = false;
|
||||
bool video_installed = false;
|
||||
bool vk_installed = false; // tracked separately: vulkan-1.dll loads lazily, so retry until present
|
||||
bool mkb_installed = false;
|
||||
|
||||
// Each tick, reconcile each subsystem with the host's requested state: install
|
||||
@@ -143,11 +145,24 @@ DWORD WINAPI worker_thread(LPVOID)
|
||||
present_ok ? 1 : 0, gl_ok ? 1 : 0, d3d9_ok ? 1 : 0);
|
||||
}
|
||||
}
|
||||
// Vulkan separately: vulkan-1.dll loads lazily (volk dlopens it after start), so the DXGI/
|
||||
// GL/D3D9 hooks above may install before it exists. Keep trying each tick until it appears.
|
||||
if (want_video && !vk_installed)
|
||||
{
|
||||
vk_installed = coop::hook::install_vk_hooks(g_ipc);
|
||||
if (vk_installed)
|
||||
{
|
||||
video_installed = true; // a Vulkan-only game otherwise has no video hook installed
|
||||
coop::hook::logf("worker_thread: vulkan video hook installed");
|
||||
}
|
||||
}
|
||||
else if (!want_video && video_installed)
|
||||
{
|
||||
coop::hook::remove_present_hooks();
|
||||
coop::hook::remove_opengl_hooks();
|
||||
coop::hook::remove_d3d9_hooks();
|
||||
coop::hook::remove_vk_hooks();
|
||||
vk_installed = false;
|
||||
video_installed = false;
|
||||
coop::hook::logf("worker_thread: video hooks removed (host request)");
|
||||
}
|
||||
@@ -259,6 +274,7 @@ BOOL APIENTRY DllMain(HMODULE module, DWORD reason, LPVOID reserved)
|
||||
coop::hook::remove_present_hooks();
|
||||
coop::hook::remove_opengl_hooks();
|
||||
coop::hook::remove_d3d9_hooks();
|
||||
coop::hook::remove_vk_hooks();
|
||||
coop::hook::remove_mkb_hooks();
|
||||
coop::hook::hook_registry_reset();
|
||||
}
|
||||
|
||||
712
hook/src/vk_hook.cpp
Normal file
712
hook/src/vk_hook.cpp
Normal file
@@ -0,0 +1,712 @@
|
||||
#include "vk_hook.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#include <d3d11.h>
|
||||
#include <dxgi1_2.h>
|
||||
|
||||
#define VK_NO_PROTOTYPES
|
||||
#define VK_USE_PLATFORM_WIN32_KHR
|
||||
#include <vulkan/vulkan.h>
|
||||
|
||||
#include <safetyhook.hpp>
|
||||
|
||||
#include "coop/protocol.hpp"
|
||||
#include "coop/shared_memory.hpp"
|
||||
#include "debug_log.hpp"
|
||||
#include "hook_registry.hpp"
|
||||
|
||||
namespace coop::hook
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
IpcClient* g_ipc = nullptr;
|
||||
unsigned long g_pid = 0;
|
||||
|
||||
safetyhook::InlineHook g_hk_gipa; // vulkan-1.dll!vkGetInstanceProcAddr
|
||||
int g_id_present = -1;
|
||||
|
||||
std::atomic<std::uint64_t> g_presents{0};
|
||||
std::atomic<std::uint64_t> g_frames_shared{0};
|
||||
std::atomic<bool> g_present_captured{false}; // set once we successfully read a present back
|
||||
bool g_unsupported_logged = false;
|
||||
|
||||
// Real entry points. g_real_gdpa and below are unhooked exports/results, so they're plain PFNs;
|
||||
// the real vkGetInstanceProcAddr is reached through the inline hook's trampoline (real_gipa()).
|
||||
PFN_vkGetDeviceProcAddr g_real_gdpa = nullptr;
|
||||
PFN_vkCreateDevice g_real_create_device = nullptr;
|
||||
PFN_vkCreateSwapchainKHR g_real_create_swapchain = nullptr;
|
||||
PFN_vkQueuePresentKHR g_real_present = nullptr;
|
||||
|
||||
// The single tracked device's real functions (one device is the overwhelmingly common case).
|
||||
struct VkFns
|
||||
{
|
||||
PFN_vkGetDeviceQueue GetDeviceQueue;
|
||||
PFN_vkCreateCommandPool CreateCommandPool;
|
||||
PFN_vkDestroyCommandPool DestroyCommandPool;
|
||||
PFN_vkAllocateCommandBuffers AllocateCommandBuffers;
|
||||
PFN_vkBeginCommandBuffer BeginCommandBuffer;
|
||||
PFN_vkEndCommandBuffer EndCommandBuffer;
|
||||
PFN_vkResetCommandBuffer ResetCommandBuffer;
|
||||
PFN_vkCmdPipelineBarrier CmdPipelineBarrier;
|
||||
PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer;
|
||||
PFN_vkQueueSubmit QueueSubmit;
|
||||
PFN_vkCreateFence CreateFence;
|
||||
PFN_vkDestroyFence DestroyFence;
|
||||
PFN_vkWaitForFences WaitForFences;
|
||||
PFN_vkResetFences ResetFences;
|
||||
PFN_vkCreateSemaphore CreateSemaphore;
|
||||
PFN_vkDestroySemaphore DestroySemaphore;
|
||||
PFN_vkCreateBuffer CreateBuffer;
|
||||
PFN_vkDestroyBuffer DestroyBuffer;
|
||||
PFN_vkGetBufferMemoryRequirements GetBufferMemoryRequirements;
|
||||
PFN_vkAllocateMemory AllocateMemory;
|
||||
PFN_vkFreeMemory FreeMemory;
|
||||
PFN_vkBindBufferMemory BindBufferMemory;
|
||||
PFN_vkMapMemory MapMemory;
|
||||
PFN_vkUnmapMemory UnmapMemory;
|
||||
PFN_vkGetSwapchainImagesKHR GetSwapchainImagesKHR;
|
||||
PFN_vkDeviceWaitIdle DeviceWaitIdle;
|
||||
PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties; // instance-level
|
||||
};
|
||||
VkFns g_fns{};
|
||||
|
||||
VkInstance g_instance = VK_NULL_HANDLE;
|
||||
VkPhysicalDevice g_phys = VK_NULL_HANDLE;
|
||||
VkDevice g_device = VK_NULL_HANDLE;
|
||||
std::uint32_t g_qfam = 0;
|
||||
VkQueue g_queue = VK_NULL_HANDLE;
|
||||
|
||||
// Read-back resources on the game's device (created once, staging re-created on size change).
|
||||
VkCommandPool g_pool = VK_NULL_HANDLE;
|
||||
VkCommandBuffer g_cmd = VK_NULL_HANDLE;
|
||||
VkFence g_fence = VK_NULL_HANDLE;
|
||||
VkSemaphore g_present_sem = VK_NULL_HANDLE; // re-chains the present's wait (see hk_vkQueuePresentKHR)
|
||||
VkBuffer g_staging = VK_NULL_HANDLE;
|
||||
VkDeviceMemory g_staging_mem = VK_NULL_HANDLE;
|
||||
VkDeviceSize g_staging_size = 0;
|
||||
void* g_staging_mapped = nullptr;
|
||||
|
||||
// Tracked swap chains (small; engines have one or two).
|
||||
struct SwapInfo
|
||||
{
|
||||
VkSwapchainKHR sc;
|
||||
VkFormat fmt;
|
||||
std::uint32_t w;
|
||||
std::uint32_t h;
|
||||
std::vector<VkImage> images;
|
||||
};
|
||||
std::vector<SwapInfo> g_swaps;
|
||||
|
||||
// Hook-owned D3D11 device + shared keyed-mutex texture (the Vulkan path has no D3D device).
|
||||
ID3D11Device* g_d3d = nullptr;
|
||||
ID3D11DeviceContext* g_d3d_ctx = nullptr;
|
||||
ID3D11Texture2D* g_shared_tex = nullptr;
|
||||
IDXGIKeyedMutex* g_shared_mutex = nullptr;
|
||||
HANDLE g_shared_handle = nullptr;
|
||||
UINT g_share_w = 0;
|
||||
UINT g_share_h = 0;
|
||||
std::vector<unsigned char> g_rgba;
|
||||
|
||||
// --- the real vkGetInstanceProcAddr, via the inline hook's trampoline --------
|
||||
PFN_vkVoidFunction real_gipa(VkInstance inst, const char* name)
|
||||
{
|
||||
return g_hk_gipa.stdcall<PFN_vkVoidFunction>(inst, name);
|
||||
}
|
||||
|
||||
// --- aux D3D11 shared texture (same contract as opengl_hook / d3d9_hook) ------
|
||||
bool ensure_d3d()
|
||||
{
|
||||
if (g_d3d != nullptr)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
const HRESULT hr =
|
||||
D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION, &g_d3d,
|
||||
nullptr, &g_d3d_ctx);
|
||||
if (FAILED(hr) || g_d3d == nullptr)
|
||||
{
|
||||
logf("vk: D3D11CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void release_shared()
|
||||
{
|
||||
if (g_shared_mutex != nullptr)
|
||||
{
|
||||
g_shared_mutex->Release();
|
||||
g_shared_mutex = nullptr;
|
||||
}
|
||||
if (g_shared_tex != nullptr)
|
||||
{
|
||||
g_shared_tex->Release();
|
||||
g_shared_tex = nullptr;
|
||||
}
|
||||
if (g_shared_handle != nullptr)
|
||||
{
|
||||
CloseHandle(g_shared_handle);
|
||||
g_shared_handle = nullptr;
|
||||
}
|
||||
g_share_w = g_share_h = 0;
|
||||
}
|
||||
|
||||
bool ensure_shared_texture(UINT w, UINT h)
|
||||
{
|
||||
if (g_shared_tex != nullptr && g_share_w == w && g_share_h == h)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
release_shared();
|
||||
D3D11_TEXTURE2D_DESC desc{};
|
||||
desc.Width = w;
|
||||
desc.Height = h;
|
||||
desc.MipLevels = 1;
|
||||
desc.ArraySize = 1;
|
||||
desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; // we swizzle the Vulkan image to RGBA
|
||||
desc.SampleDesc.Count = 1;
|
||||
desc.Usage = D3D11_USAGE_DEFAULT;
|
||||
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
|
||||
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
|
||||
if (FAILED(g_d3d->CreateTexture2D(&desc, nullptr, &g_shared_tex)) || g_shared_tex == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
IDXGIResource1* res = nullptr;
|
||||
if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast<void**>(&res))) ||
|
||||
res == nullptr)
|
||||
{
|
||||
release_shared();
|
||||
return false;
|
||||
}
|
||||
const std::wstring name = video_share_name(g_pid);
|
||||
const HRESULT hr = res->CreateSharedHandle(
|
||||
nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, name.c_str(), &g_shared_handle);
|
||||
res->Release();
|
||||
if (FAILED(hr) || g_shared_handle == nullptr)
|
||||
{
|
||||
release_shared();
|
||||
return false;
|
||||
}
|
||||
if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast<void**>(&g_shared_mutex))))
|
||||
{
|
||||
release_shared();
|
||||
return false;
|
||||
}
|
||||
g_share_w = w;
|
||||
g_share_h = h;
|
||||
logf("vk: shared texture ready %ux%u name=%ls", w, h, name.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
// --- Vulkan read-back resources ----------------------------------------------
|
||||
bool find_host_visible_memory(std::uint32_t type_bits, std::uint32_t& out_index)
|
||||
{
|
||||
VkPhysicalDeviceMemoryProperties mp{};
|
||||
g_fns.GetPhysicalDeviceMemoryProperties(g_phys, &mp);
|
||||
for (std::uint32_t i = 0; i < mp.memoryTypeCount; ++i)
|
||||
{
|
||||
const bool usable = (type_bits & (1u << i)) != 0;
|
||||
const bool host = (mp.memoryTypes[i].propertyFlags &
|
||||
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
|
||||
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
|
||||
if (usable && host)
|
||||
{
|
||||
out_index = i;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void release_staging()
|
||||
{
|
||||
if (g_staging_mapped != nullptr && g_staging_mem != VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.UnmapMemory(g_device, g_staging_mem);
|
||||
g_staging_mapped = nullptr;
|
||||
}
|
||||
if (g_staging != VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.DestroyBuffer(g_device, g_staging, nullptr);
|
||||
g_staging = VK_NULL_HANDLE;
|
||||
}
|
||||
if (g_staging_mem != VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.FreeMemory(g_device, g_staging_mem, nullptr);
|
||||
g_staging_mem = VK_NULL_HANDLE;
|
||||
}
|
||||
g_staging_size = 0;
|
||||
}
|
||||
|
||||
bool ensure_vk_resources(std::uint32_t w, std::uint32_t h)
|
||||
{
|
||||
if (g_queue == VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.GetDeviceQueue(g_device, g_qfam, 0, &g_queue);
|
||||
}
|
||||
if (g_pool == VK_NULL_HANDLE)
|
||||
{
|
||||
VkCommandPoolCreateInfo pci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
|
||||
pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
|
||||
pci.queueFamilyIndex = g_qfam;
|
||||
if (g_fns.CreateCommandPool(g_device, &pci, nullptr, &g_pool) != VK_SUCCESS)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
|
||||
ai.commandPool = g_pool;
|
||||
ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
|
||||
ai.commandBufferCount = 1;
|
||||
if (g_fns.AllocateCommandBuffers(g_device, &ai, &g_cmd) != VK_SUCCESS)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
||||
if (g_fns.CreateFence(g_device, &fi, nullptr, &g_fence) != VK_SUCCESS)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
VkSemaphoreCreateInfo si{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
|
||||
if (g_fns.CreateSemaphore(g_device, &si, nullptr, &g_present_sem) != VK_SUCCESS)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const VkDeviceSize need = static_cast<VkDeviceSize>(w) * h * 4;
|
||||
if (g_staging != VK_NULL_HANDLE && g_staging_size == need)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
release_staging();
|
||||
VkBufferCreateInfo bci{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
||||
bci.size = need;
|
||||
bci.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
|
||||
if (g_fns.CreateBuffer(g_device, &bci, nullptr, &g_staging) != VK_SUCCESS)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
VkMemoryRequirements mr{};
|
||||
g_fns.GetBufferMemoryRequirements(g_device, g_staging, &mr);
|
||||
std::uint32_t mem_type = 0;
|
||||
if (!find_host_visible_memory(mr.memoryTypeBits, mem_type))
|
||||
{
|
||||
release_staging();
|
||||
return false;
|
||||
}
|
||||
VkMemoryAllocateInfo mai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
||||
mai.allocationSize = mr.size;
|
||||
mai.memoryTypeIndex = mem_type;
|
||||
if (g_fns.AllocateMemory(g_device, &mai, nullptr, &g_staging_mem) != VK_SUCCESS ||
|
||||
g_fns.BindBufferMemory(g_device, g_staging, g_staging_mem, 0) != VK_SUCCESS ||
|
||||
g_fns.MapMemory(g_device, g_staging_mem, 0, VK_WHOLE_SIZE, 0, &g_staging_mapped) != VK_SUCCESS)
|
||||
{
|
||||
release_staging();
|
||||
return false;
|
||||
}
|
||||
g_staging_size = need;
|
||||
return true;
|
||||
}
|
||||
|
||||
void image_barrier(VkCommandBuffer cb, VkImage img, VkImageLayout from, VkImageLayout to, VkAccessFlags src,
|
||||
VkAccessFlags dst)
|
||||
{
|
||||
VkImageMemoryBarrier b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
|
||||
b.srcAccessMask = src;
|
||||
b.dstAccessMask = dst;
|
||||
b.oldLayout = from;
|
||||
b.newLayout = to;
|
||||
b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
||||
b.image = img;
|
||||
b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
g_fns.CmdPipelineBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0,
|
||||
nullptr, 0, nullptr, 1, &b);
|
||||
}
|
||||
|
||||
// Read `image` (in PRESENT_SRC layout) back into the shared texture. `wait`/`wait_count` are the
|
||||
// present's wait semaphores, which our submit consumes and replaces with g_present_sem (returned
|
||||
// via *out_sem) so the real present orders correctly after our copy without double-waiting.
|
||||
bool capture_present(VkImage image, VkFormat fmt, std::uint32_t w, std::uint32_t h, const VkSemaphore* wait,
|
||||
std::uint32_t wait_count, VkSemaphore& out_sem)
|
||||
{
|
||||
const bool bgra = fmt == VK_FORMAT_B8G8R8A8_UNORM || fmt == VK_FORMAT_B8G8R8A8_SRGB;
|
||||
const bool rgba = fmt == VK_FORMAT_R8G8B8A8_UNORM || fmt == VK_FORMAT_R8G8B8A8_SRGB;
|
||||
if (!bgra && !rgba)
|
||||
{
|
||||
if (!g_unsupported_logged)
|
||||
{
|
||||
logf("vk: unsupported swapchain format=%d (only 8888 BGRA/RGBA); idle", static_cast<int>(fmt));
|
||||
g_unsupported_logged = true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (!ensure_d3d() || !ensure_shared_texture(w, h) || !ensure_vk_resources(w, h))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
g_fns.ResetCommandBuffer(g_cmd, 0);
|
||||
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
|
||||
bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
|
||||
g_fns.BeginCommandBuffer(g_cmd, &bi);
|
||||
image_barrier(g_cmd, image, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
|
||||
VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_READ_BIT);
|
||||
VkBufferImageCopy region{};
|
||||
region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
|
||||
region.imageExtent = {w, h, 1};
|
||||
g_fns.CmdCopyImageToBuffer(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, g_staging, 1, ®ion);
|
||||
image_barrier(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
|
||||
VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_MEMORY_READ_BIT);
|
||||
g_fns.EndCommandBuffer(g_cmd);
|
||||
|
||||
std::vector<VkPipelineStageFlags> wait_stages(wait_count, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
|
||||
VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO};
|
||||
si.waitSemaphoreCount = wait_count;
|
||||
si.pWaitSemaphores = wait;
|
||||
si.pWaitDstStageMask = wait_count != 0 ? wait_stages.data() : nullptr;
|
||||
si.commandBufferCount = 1;
|
||||
si.pCommandBuffers = &g_cmd;
|
||||
si.signalSemaphoreCount = 1;
|
||||
si.pSignalSemaphores = &g_present_sem;
|
||||
g_fns.ResetFences(g_device, 1, &g_fence);
|
||||
if (g_fns.QueueSubmit(g_queue, 1, &si, g_fence) != VK_SUCCESS)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
g_fns.WaitForFences(g_device, 1, &g_fence, VK_TRUE, UINT64_MAX);
|
||||
|
||||
// Swizzle the mapped staging (tightly packed w*4) into RGBA and upload.
|
||||
const size_t row = static_cast<size_t>(w) * 4;
|
||||
if (g_rgba.size() != row * h)
|
||||
{
|
||||
g_rgba.resize(row * h);
|
||||
}
|
||||
const auto* src = static_cast<const unsigned char*>(g_staging_mapped);
|
||||
for (std::uint32_t y = 0; y < h; ++y)
|
||||
{
|
||||
const unsigned char* s = src + static_cast<size_t>(y) * row;
|
||||
unsigned char* o = g_rgba.data() + static_cast<size_t>(y) * row;
|
||||
if (bgra)
|
||||
{
|
||||
for (std::uint32_t x = 0; x < w; ++x)
|
||||
{
|
||||
o[x * 4 + 0] = s[x * 4 + 2];
|
||||
o[x * 4 + 1] = s[x * 4 + 1];
|
||||
o[x * 4 + 2] = s[x * 4 + 0];
|
||||
o[x * 4 + 3] = 255;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
std::memcpy(o, s, row);
|
||||
}
|
||||
}
|
||||
|
||||
bool shared = false;
|
||||
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
|
||||
{
|
||||
g_d3d_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, g_rgba.data(), static_cast<UINT>(row), 0);
|
||||
g_d3d_ctx->Flush();
|
||||
g_shared_mutex->ReleaseSync(kVideoMutexKey);
|
||||
shared = true;
|
||||
}
|
||||
out_sem = g_present_sem;
|
||||
if (shared)
|
||||
{
|
||||
g_present_captured.store(true, std::memory_order_relaxed);
|
||||
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->publish_video_frame(w, h, static_cast<std::uint32_t>(DXGI_FORMAT_R8G8B8A8_UNORM));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const SwapInfo* find_swap(VkSwapchainKHR sc)
|
||||
{
|
||||
for (const SwapInfo& s : g_swaps)
|
||||
{
|
||||
if (s.sc == sc)
|
||||
{
|
||||
return &s;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL hk_vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* pPresentInfo)
|
||||
{
|
||||
hook_note_call(g_id_present);
|
||||
g_presents.fetch_add(1, std::memory_order_relaxed);
|
||||
if (g_ipc != nullptr)
|
||||
{
|
||||
g_ipc->note_present();
|
||||
}
|
||||
|
||||
// Capture only the simple, common single-swapchain present; pass anything else through.
|
||||
if (g_device != VK_NULL_HANDLE && pPresentInfo != nullptr && pPresentInfo->swapchainCount == 1)
|
||||
{
|
||||
const SwapInfo* s = find_swap(pPresentInfo->pSwapchains[0]);
|
||||
if (s != nullptr && pPresentInfo->pImageIndices[0] < s->images.size())
|
||||
{
|
||||
VkSemaphore chained = VK_NULL_HANDLE;
|
||||
if (capture_present(s->images[pPresentInfo->pImageIndices[0]], s->fmt, s->w, s->h,
|
||||
pPresentInfo->pWaitSemaphores, pPresentInfo->waitSemaphoreCount, chained))
|
||||
{
|
||||
// Replace the present's wait with our chained semaphore (our submit consumed the
|
||||
// originals and signals this one), so the present still orders after rendering.
|
||||
VkPresentInfoKHR pi = *pPresentInfo;
|
||||
pi.waitSemaphoreCount = 1;
|
||||
pi.pWaitSemaphores = &chained;
|
||||
return g_real_present(queue, &pi);
|
||||
}
|
||||
}
|
||||
}
|
||||
return g_real_present(queue, pPresentInfo);
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR* ci,
|
||||
const VkAllocationCallbacks* alloc, VkSwapchainKHR* out)
|
||||
{
|
||||
const VkResult r = g_real_create_swapchain(device, ci, alloc, out);
|
||||
if (r == VK_SUCCESS && out != nullptr && g_fns.GetSwapchainImagesKHR != nullptr)
|
||||
{
|
||||
SwapInfo info{};
|
||||
info.sc = *out;
|
||||
info.fmt = ci->imageFormat;
|
||||
info.w = ci->imageExtent.width;
|
||||
info.h = ci->imageExtent.height;
|
||||
std::uint32_t n = 0;
|
||||
g_fns.GetSwapchainImagesKHR(device, *out, &n, nullptr);
|
||||
info.images.resize(n);
|
||||
g_fns.GetSwapchainImagesKHR(device, *out, &n, info.images.data());
|
||||
g_swaps.push_back(std::move(info));
|
||||
// (Don't log the VkSwapchainKHR handle: it's a uint64_t on x86, not a pointer.)
|
||||
logf("vk: swapchain %ux%u fmt=%d images=%u", ci->imageExtent.width, ci->imageExtent.height,
|
||||
static_cast<int>(ci->imageFormat), n);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// Resolve every device function we need for the read-back via the real vkGetDeviceProcAddr.
|
||||
void load_device_fns(VkDevice device)
|
||||
{
|
||||
#define LOAD(field, vkname) g_fns.field = reinterpret_cast<PFN_##vkname>(g_real_gdpa(device, #vkname))
|
||||
LOAD(GetDeviceQueue, vkGetDeviceQueue);
|
||||
LOAD(CreateCommandPool, vkCreateCommandPool);
|
||||
LOAD(DestroyCommandPool, vkDestroyCommandPool);
|
||||
LOAD(AllocateCommandBuffers, vkAllocateCommandBuffers);
|
||||
LOAD(BeginCommandBuffer, vkBeginCommandBuffer);
|
||||
LOAD(EndCommandBuffer, vkEndCommandBuffer);
|
||||
LOAD(ResetCommandBuffer, vkResetCommandBuffer);
|
||||
LOAD(CmdPipelineBarrier, vkCmdPipelineBarrier);
|
||||
LOAD(CmdCopyImageToBuffer, vkCmdCopyImageToBuffer);
|
||||
LOAD(QueueSubmit, vkQueueSubmit);
|
||||
LOAD(CreateFence, vkCreateFence);
|
||||
LOAD(DestroyFence, vkDestroyFence);
|
||||
LOAD(WaitForFences, vkWaitForFences);
|
||||
LOAD(ResetFences, vkResetFences);
|
||||
LOAD(CreateSemaphore, vkCreateSemaphore);
|
||||
LOAD(DestroySemaphore, vkDestroySemaphore);
|
||||
LOAD(CreateBuffer, vkCreateBuffer);
|
||||
LOAD(DestroyBuffer, vkDestroyBuffer);
|
||||
LOAD(GetBufferMemoryRequirements, vkGetBufferMemoryRequirements);
|
||||
LOAD(AllocateMemory, vkAllocateMemory);
|
||||
LOAD(FreeMemory, vkFreeMemory);
|
||||
LOAD(BindBufferMemory, vkBindBufferMemory);
|
||||
LOAD(MapMemory, vkMapMemory);
|
||||
LOAD(UnmapMemory, vkUnmapMemory);
|
||||
LOAD(GetSwapchainImagesKHR, vkGetSwapchainImagesKHR);
|
||||
LOAD(DeviceWaitIdle, vkDeviceWaitIdle);
|
||||
#undef LOAD
|
||||
g_fns.GetPhysicalDeviceMemoryProperties =
|
||||
reinterpret_cast<PFN_vkGetPhysicalDeviceMemoryProperties>(real_gipa(g_instance, "vkGetPhysicalDeviceMemoryProperties"));
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateDevice(VkPhysicalDevice phys, const VkDeviceCreateInfo* ci,
|
||||
const VkAllocationCallbacks* alloc, VkDevice* out)
|
||||
{
|
||||
const VkResult r = g_real_create_device(phys, ci, alloc, out);
|
||||
if (r == VK_SUCCESS && out != nullptr && g_device == VK_NULL_HANDLE) // track the first device
|
||||
{
|
||||
g_phys = phys;
|
||||
g_device = *out;
|
||||
g_qfam = ci->queueCreateInfoCount > 0 ? ci->pQueueCreateInfos[0].queueFamilyIndex : 0;
|
||||
g_real_gdpa = reinterpret_cast<PFN_vkGetDeviceProcAddr>(real_gipa(g_instance, "vkGetDeviceProcAddr"));
|
||||
g_real_create_swapchain =
|
||||
reinterpret_cast<PFN_vkCreateSwapchainKHR>(g_real_gdpa(*out, "vkCreateSwapchainKHR"));
|
||||
g_real_present = reinterpret_cast<PFN_vkQueuePresentKHR>(g_real_gdpa(*out, "vkQueuePresentKHR"));
|
||||
load_device_fns(*out);
|
||||
logf("vk: device created (qfam=%u) -- present capture armed", g_qfam);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateInstance(const VkInstanceCreateInfo* ci,
|
||||
const VkAllocationCallbacks* alloc, VkInstance* out)
|
||||
{
|
||||
auto real_create = reinterpret_cast<PFN_vkCreateInstance>(real_gipa(nullptr, "vkCreateInstance"));
|
||||
const VkResult r = real_create(ci, alloc, out);
|
||||
if (r == VK_SUCCESS && out != nullptr)
|
||||
{
|
||||
g_instance = *out;
|
||||
g_real_create_device = reinterpret_cast<PFN_vkCreateDevice>(real_gipa(*out, "vkCreateDevice"));
|
||||
logf("vk: instance created -- intercepting device/swapchain/present");
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL hk_vkGetDeviceProcAddr(VkDevice device, const char* name)
|
||||
{
|
||||
if (name != nullptr)
|
||||
{
|
||||
if (std::strcmp(name, "vkQueuePresentKHR") == 0)
|
||||
{
|
||||
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkQueuePresentKHR);
|
||||
}
|
||||
if (std::strcmp(name, "vkCreateSwapchainKHR") == 0)
|
||||
{
|
||||
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkCreateSwapchainKHR);
|
||||
}
|
||||
}
|
||||
return g_real_gdpa != nullptr ? g_real_gdpa(device, name) : nullptr;
|
||||
}
|
||||
|
||||
VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL hk_vkGetInstanceProcAddr(VkInstance instance, const char* name)
|
||||
{
|
||||
if (name != nullptr)
|
||||
{
|
||||
if (std::strcmp(name, "vkGetInstanceProcAddr") == 0)
|
||||
{
|
||||
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkGetInstanceProcAddr);
|
||||
}
|
||||
if (std::strcmp(name, "vkCreateInstance") == 0)
|
||||
{
|
||||
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkCreateInstance);
|
||||
}
|
||||
if (std::strcmp(name, "vkCreateDevice") == 0)
|
||||
{
|
||||
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkCreateDevice);
|
||||
}
|
||||
if (std::strcmp(name, "vkGetDeviceProcAddr") == 0)
|
||||
{
|
||||
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkGetDeviceProcAddr);
|
||||
}
|
||||
}
|
||||
return real_gipa(instance, name);
|
||||
}
|
||||
|
||||
void* gipa_export_address()
|
||||
{
|
||||
HMODULE vk = GetModuleHandleW(L"vulkan-1.dll");
|
||||
if (vk == nullptr)
|
||||
{
|
||||
return nullptr; // not a Vulkan process (yet)
|
||||
}
|
||||
return reinterpret_cast<void*>(GetProcAddress(vk, "vkGetInstanceProcAddr"));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool install_vk_hooks(IpcClient& ipc)
|
||||
{
|
||||
g_ipc = &ipc;
|
||||
g_pid = GetCurrentProcessId();
|
||||
if (g_hk_gipa)
|
||||
{
|
||||
return true; // already installed
|
||||
}
|
||||
if (g_id_present < 0)
|
||||
{
|
||||
g_id_present = hook_register("vkQueuePresentKHR", HookSubsys_Video);
|
||||
}
|
||||
void* gipa = gipa_export_address();
|
||||
if (gipa == nullptr)
|
||||
{
|
||||
hook_set_installed(g_id_present, false);
|
||||
return false; // vulkan-1.dll not loaded; caller can retry once the game loads it
|
||||
}
|
||||
g_unsupported_logged = false;
|
||||
g_hk_gipa = safetyhook::create_inline(gipa, reinterpret_cast<void*>(&hk_vkGetInstanceProcAddr));
|
||||
hook_set_installed(g_id_present, static_cast<bool>(g_hk_gipa));
|
||||
logf("install_vk_hooks: vkGetInstanceProcAddr=%p hooked=%d", gipa, static_cast<bool>(g_hk_gipa) ? 1 : 0);
|
||||
return static_cast<bool>(g_hk_gipa);
|
||||
}
|
||||
|
||||
void remove_vk_hooks()
|
||||
{
|
||||
if (g_device != VK_NULL_HANDLE && g_fns.DeviceWaitIdle != nullptr)
|
||||
{
|
||||
g_fns.DeviceWaitIdle(g_device);
|
||||
release_staging();
|
||||
if (g_present_sem != VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.DestroySemaphore(g_device, g_present_sem, nullptr);
|
||||
g_present_sem = VK_NULL_HANDLE;
|
||||
}
|
||||
if (g_fence != VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.DestroyFence(g_device, g_fence, nullptr);
|
||||
g_fence = VK_NULL_HANDLE;
|
||||
}
|
||||
if (g_pool != VK_NULL_HANDLE)
|
||||
{
|
||||
g_fns.DestroyCommandPool(g_device, g_pool, nullptr);
|
||||
g_pool = VK_NULL_HANDLE;
|
||||
}
|
||||
}
|
||||
g_hk_gipa = {};
|
||||
hook_set_installed(g_id_present, false);
|
||||
release_shared();
|
||||
if (g_d3d_ctx != nullptr)
|
||||
{
|
||||
g_d3d_ctx->Release();
|
||||
g_d3d_ctx = nullptr;
|
||||
}
|
||||
if (g_d3d != nullptr)
|
||||
{
|
||||
g_d3d->Release();
|
||||
g_d3d = nullptr;
|
||||
}
|
||||
g_swaps.clear();
|
||||
g_rgba.clear();
|
||||
g_cmd = VK_NULL_HANDLE;
|
||||
g_queue = VK_NULL_HANDLE;
|
||||
g_device = VK_NULL_HANDLE;
|
||||
g_instance = VK_NULL_HANDLE;
|
||||
g_real_gdpa = nullptr;
|
||||
g_present_captured.store(false, std::memory_order_relaxed);
|
||||
g_presents.store(0, std::memory_order_relaxed);
|
||||
g_frames_shared.store(0, std::memory_order_relaxed);
|
||||
g_ipc = nullptr;
|
||||
}
|
||||
|
||||
std::uint64_t vk_presents()
|
||||
{
|
||||
return g_presents.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
std::uint64_t vk_frames_shared()
|
||||
{
|
||||
return g_frames_shared.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
bool vk_injected_too_late()
|
||||
{
|
||||
// vulkan-1.dll is loaded but we never captured a present -> the app resolved its present
|
||||
// pointer before we hooked (or doesn't go through our chain). The host shows the banner.
|
||||
return GetModuleHandleW(L"vulkan-1.dll") != nullptr && g_presents.load(std::memory_order_relaxed) == 0 &&
|
||||
!g_present_captured.load(std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
} // namespace coop::hook
|
||||
34
hook/src/vk_hook.hpp
Normal file
34
hook/src/vk_hook.hpp
Normal file
@@ -0,0 +1,34 @@
|
||||
// Injected Vulkan capture path. Vulkan games present via vkQueuePresentKHR and (with volk or
|
||||
// their own loader) cache that pointer at init, so they can't be hooked by *late* injection --
|
||||
// the hook must be present before vkCreateInstance. We catch the resolution chain instead:
|
||||
// inline-hook the vulkan-1.dll export vkGetInstanceProcAddr and hand back our own wrappers for
|
||||
// vkCreateInstance / vkCreateDevice / vkGetDeviceProcAddr / vkCreateSwapchainKHR /
|
||||
// vkQueuePresentKHR. On present we read the swap-chain image back with vkCmdCopyImageToBuffer
|
||||
// (the same read-back pattern as the D3D10/D3D9/OpenGL paths) and upload it into the shared
|
||||
// keyed-mutex texture via a hook-owned D3D11 device. Part of the video subsystem.
|
||||
#pragma once
|
||||
|
||||
#include "ipc_client.hpp"
|
||||
|
||||
namespace coop::hook
|
||||
{
|
||||
|
||||
// Installs the Vulkan capture hook (inline-hooks vkGetInstanceProcAddr). `ipc` must outlive the
|
||||
// hook. Returns true if vulkan-1.dll is loaded and the export was hooked; false otherwise, so
|
||||
// the caller can retry once the game loads Vulkan. Safe to call repeatedly.
|
||||
bool install_vk_hooks(IpcClient& ipc);
|
||||
|
||||
// Removes the Vulkan hook and releases the shared texture + Vulkan read-back resources.
|
||||
void remove_vk_hooks();
|
||||
|
||||
// --- Diagnostics -----------------------------------------------------------
|
||||
|
||||
std::uint64_t vk_presents(); // cumulative vkQueuePresentKHR detours
|
||||
std::uint64_t vk_frames_shared(); // frames uploaded into the shared texture
|
||||
|
||||
// True when vulkan-1.dll is loaded in this process but we never captured a present -- i.e. the
|
||||
// app resolved its present pointer before we hooked (injected too late). Drives the host's
|
||||
// "Vulkan, injected too late" banner.
|
||||
bool vk_injected_too_late();
|
||||
|
||||
} // namespace coop::hook
|
||||
Reference in New Issue
Block a user