M2(Vulkan): capture hook via GPA interception + vkCmdCopyImageToBuffer read-back

New vk_hook.cpp inline-hooks the vulkan-1.dll vkGetInstanceProcAddr export and
hands back our wrappers for vkCreateInstance / vkCreateDevice / vkGetDeviceProcAddr
/ vkCreateSwapchainKHR / vkQueuePresentKHR, so a volk-using (loader-bypass) app
resolves our hooks. On present it reads the swap-chain image back with
vkCmdCopyImageToBuffer into a host-visible buffer (same read-back model as
D3D10/D3D9/OpenGL), swizzles BGRA->RGBA, and uploads it into the shared
keyed-mutex texture on a hook-owned D3D11 device. The read-back submit re-chains
the present's wait semaphores (consume the originals, signal our own that the
real present waits on) so capture orders after rendering without double-waiting.

Wired into the video subsystem with a lazy retry (vulkan-1.dll loads late). Hook
links the official Vulkan-Headers (headers only, VK_NO_PROTOTYPES) via the
include dir so the x86 sub-build builds too.

Because Vulkan caches its present pointer at init, late injection can't hook it:
mock_game_test launches the mock **suspended**, injects, resumes, and under
COOP_MOCK_VK_EARLY the mock loads Vulkan and waits so the hook arms first -- then
decodes frames through the hook like the other backends. 15/15 ctest (x64 +
x86); skips cleanly without a Vulkan driver.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-22 12:39:07 +02:00
parent d4a83e75bb
commit dd976979a1
6 changed files with 875 additions and 26 deletions

View File

@@ -24,6 +24,7 @@
#include "d3d9_hook.hpp"
#include "opengl_hook.hpp"
#include "present_hook.hpp"
#include "vk_hook.hpp"
#include "xinput_hook.hpp"
namespace
@@ -73,6 +74,7 @@ DWORD WINAPI worker_thread(LPVOID)
bool audio_installed = false;
bool audio_ring_open = false;
bool video_installed = false;
bool vk_installed = false; // tracked separately: vulkan-1.dll loads lazily, so retry until present
bool mkb_installed = false;
// Each tick, reconcile each subsystem with the host's requested state: install
@@ -143,11 +145,24 @@ DWORD WINAPI worker_thread(LPVOID)
present_ok ? 1 : 0, gl_ok ? 1 : 0, d3d9_ok ? 1 : 0);
}
}
// Vulkan separately: vulkan-1.dll loads lazily (volk dlopens it after start), so the DXGI/
// GL/D3D9 hooks above may install before it exists. Keep trying each tick until it appears.
if (want_video && !vk_installed)
{
vk_installed = coop::hook::install_vk_hooks(g_ipc);
if (vk_installed)
{
video_installed = true; // a Vulkan-only game otherwise has no video hook installed
coop::hook::logf("worker_thread: vulkan video hook installed");
}
}
else if (!want_video && video_installed)
{
coop::hook::remove_present_hooks();
coop::hook::remove_opengl_hooks();
coop::hook::remove_d3d9_hooks();
coop::hook::remove_vk_hooks();
vk_installed = false;
video_installed = false;
coop::hook::logf("worker_thread: video hooks removed (host request)");
}
@@ -259,6 +274,7 @@ BOOL APIENTRY DllMain(HMODULE module, DWORD reason, LPVOID reserved)
coop::hook::remove_present_hooks();
coop::hook::remove_opengl_hooks();
coop::hook::remove_d3d9_hooks();
coop::hook::remove_vk_hooks();
coop::hook::remove_mkb_hooks();
coop::hook::hook_registry_reset();
}

712
hook/src/vk_hook.cpp Normal file
View File

@@ -0,0 +1,712 @@
#include "vk_hook.hpp"
#include <atomic>
#include <cstring>
#include <vector>
#include <windows.h>
#include <d3d11.h>
#include <dxgi1_2.h>
#define VK_NO_PROTOTYPES
#define VK_USE_PLATFORM_WIN32_KHR
#include <vulkan/vulkan.h>
#include <safetyhook.hpp>
#include "coop/protocol.hpp"
#include "coop/shared_memory.hpp"
#include "debug_log.hpp"
#include "hook_registry.hpp"
namespace coop::hook
{
namespace
{
IpcClient* g_ipc = nullptr;
unsigned long g_pid = 0;
safetyhook::InlineHook g_hk_gipa; // vulkan-1.dll!vkGetInstanceProcAddr
int g_id_present = -1;
std::atomic<std::uint64_t> g_presents{0};
std::atomic<std::uint64_t> g_frames_shared{0};
std::atomic<bool> g_present_captured{false}; // set once we successfully read a present back
bool g_unsupported_logged = false;
// Real entry points. g_real_gdpa and below are unhooked exports/results, so they're plain PFNs;
// the real vkGetInstanceProcAddr is reached through the inline hook's trampoline (real_gipa()).
PFN_vkGetDeviceProcAddr g_real_gdpa = nullptr;
PFN_vkCreateDevice g_real_create_device = nullptr;
PFN_vkCreateSwapchainKHR g_real_create_swapchain = nullptr;
PFN_vkQueuePresentKHR g_real_present = nullptr;
// The single tracked device's real functions (one device is the overwhelmingly common case).
struct VkFns
{
PFN_vkGetDeviceQueue GetDeviceQueue;
PFN_vkCreateCommandPool CreateCommandPool;
PFN_vkDestroyCommandPool DestroyCommandPool;
PFN_vkAllocateCommandBuffers AllocateCommandBuffers;
PFN_vkBeginCommandBuffer BeginCommandBuffer;
PFN_vkEndCommandBuffer EndCommandBuffer;
PFN_vkResetCommandBuffer ResetCommandBuffer;
PFN_vkCmdPipelineBarrier CmdPipelineBarrier;
PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer;
PFN_vkQueueSubmit QueueSubmit;
PFN_vkCreateFence CreateFence;
PFN_vkDestroyFence DestroyFence;
PFN_vkWaitForFences WaitForFences;
PFN_vkResetFences ResetFences;
PFN_vkCreateSemaphore CreateSemaphore;
PFN_vkDestroySemaphore DestroySemaphore;
PFN_vkCreateBuffer CreateBuffer;
PFN_vkDestroyBuffer DestroyBuffer;
PFN_vkGetBufferMemoryRequirements GetBufferMemoryRequirements;
PFN_vkAllocateMemory AllocateMemory;
PFN_vkFreeMemory FreeMemory;
PFN_vkBindBufferMemory BindBufferMemory;
PFN_vkMapMemory MapMemory;
PFN_vkUnmapMemory UnmapMemory;
PFN_vkGetSwapchainImagesKHR GetSwapchainImagesKHR;
PFN_vkDeviceWaitIdle DeviceWaitIdle;
PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties; // instance-level
};
VkFns g_fns{};
VkInstance g_instance = VK_NULL_HANDLE;
VkPhysicalDevice g_phys = VK_NULL_HANDLE;
VkDevice g_device = VK_NULL_HANDLE;
std::uint32_t g_qfam = 0;
VkQueue g_queue = VK_NULL_HANDLE;
// Read-back resources on the game's device (created once, staging re-created on size change).
VkCommandPool g_pool = VK_NULL_HANDLE;
VkCommandBuffer g_cmd = VK_NULL_HANDLE;
VkFence g_fence = VK_NULL_HANDLE;
VkSemaphore g_present_sem = VK_NULL_HANDLE; // re-chains the present's wait (see hk_vkQueuePresentKHR)
VkBuffer g_staging = VK_NULL_HANDLE;
VkDeviceMemory g_staging_mem = VK_NULL_HANDLE;
VkDeviceSize g_staging_size = 0;
void* g_staging_mapped = nullptr;
// Tracked swap chains (small; engines have one or two).
struct SwapInfo
{
VkSwapchainKHR sc;
VkFormat fmt;
std::uint32_t w;
std::uint32_t h;
std::vector<VkImage> images;
};
std::vector<SwapInfo> g_swaps;
// Hook-owned D3D11 device + shared keyed-mutex texture (the Vulkan path has no D3D device).
ID3D11Device* g_d3d = nullptr;
ID3D11DeviceContext* g_d3d_ctx = nullptr;
ID3D11Texture2D* g_shared_tex = nullptr;
IDXGIKeyedMutex* g_shared_mutex = nullptr;
HANDLE g_shared_handle = nullptr;
UINT g_share_w = 0;
UINT g_share_h = 0;
std::vector<unsigned char> g_rgba;
// --- the real vkGetInstanceProcAddr, via the inline hook's trampoline --------
PFN_vkVoidFunction real_gipa(VkInstance inst, const char* name)
{
return g_hk_gipa.stdcall<PFN_vkVoidFunction>(inst, name);
}
// --- aux D3D11 shared texture (same contract as opengl_hook / d3d9_hook) ------
bool ensure_d3d()
{
if (g_d3d != nullptr)
{
return true;
}
const HRESULT hr =
D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, D3D11_SDK_VERSION, &g_d3d,
nullptr, &g_d3d_ctx);
if (FAILED(hr) || g_d3d == nullptr)
{
logf("vk: D3D11CreateDevice failed hr=0x%08lX", static_cast<unsigned long>(hr));
return false;
}
return true;
}
void release_shared()
{
if (g_shared_mutex != nullptr)
{
g_shared_mutex->Release();
g_shared_mutex = nullptr;
}
if (g_shared_tex != nullptr)
{
g_shared_tex->Release();
g_shared_tex = nullptr;
}
if (g_shared_handle != nullptr)
{
CloseHandle(g_shared_handle);
g_shared_handle = nullptr;
}
g_share_w = g_share_h = 0;
}
bool ensure_shared_texture(UINT w, UINT h)
{
if (g_shared_tex != nullptr && g_share_w == w && g_share_h == h)
{
return true;
}
release_shared();
D3D11_TEXTURE2D_DESC desc{};
desc.Width = w;
desc.Height = h;
desc.MipLevels = 1;
desc.ArraySize = 1;
desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; // we swizzle the Vulkan image to RGBA
desc.SampleDesc.Count = 1;
desc.Usage = D3D11_USAGE_DEFAULT;
desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
if (FAILED(g_d3d->CreateTexture2D(&desc, nullptr, &g_shared_tex)) || g_shared_tex == nullptr)
{
return false;
}
IDXGIResource1* res = nullptr;
if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast<void**>(&res))) ||
res == nullptr)
{
release_shared();
return false;
}
const std::wstring name = video_share_name(g_pid);
const HRESULT hr = res->CreateSharedHandle(
nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, name.c_str(), &g_shared_handle);
res->Release();
if (FAILED(hr) || g_shared_handle == nullptr)
{
release_shared();
return false;
}
if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast<void**>(&g_shared_mutex))))
{
release_shared();
return false;
}
g_share_w = w;
g_share_h = h;
logf("vk: shared texture ready %ux%u name=%ls", w, h, name.c_str());
return true;
}
// --- Vulkan read-back resources ----------------------------------------------
bool find_host_visible_memory(std::uint32_t type_bits, std::uint32_t& out_index)
{
VkPhysicalDeviceMemoryProperties mp{};
g_fns.GetPhysicalDeviceMemoryProperties(g_phys, &mp);
for (std::uint32_t i = 0; i < mp.memoryTypeCount; ++i)
{
const bool usable = (type_bits & (1u << i)) != 0;
const bool host = (mp.memoryTypes[i].propertyFlags &
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (usable && host)
{
out_index = i;
return true;
}
}
return false;
}
void release_staging()
{
if (g_staging_mapped != nullptr && g_staging_mem != VK_NULL_HANDLE)
{
g_fns.UnmapMemory(g_device, g_staging_mem);
g_staging_mapped = nullptr;
}
if (g_staging != VK_NULL_HANDLE)
{
g_fns.DestroyBuffer(g_device, g_staging, nullptr);
g_staging = VK_NULL_HANDLE;
}
if (g_staging_mem != VK_NULL_HANDLE)
{
g_fns.FreeMemory(g_device, g_staging_mem, nullptr);
g_staging_mem = VK_NULL_HANDLE;
}
g_staging_size = 0;
}
bool ensure_vk_resources(std::uint32_t w, std::uint32_t h)
{
if (g_queue == VK_NULL_HANDLE)
{
g_fns.GetDeviceQueue(g_device, g_qfam, 0, &g_queue);
}
if (g_pool == VK_NULL_HANDLE)
{
VkCommandPoolCreateInfo pci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
pci.queueFamilyIndex = g_qfam;
if (g_fns.CreateCommandPool(g_device, &pci, nullptr, &g_pool) != VK_SUCCESS)
{
return false;
}
VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
ai.commandPool = g_pool;
ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
ai.commandBufferCount = 1;
if (g_fns.AllocateCommandBuffers(g_device, &ai, &g_cmd) != VK_SUCCESS)
{
return false;
}
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
if (g_fns.CreateFence(g_device, &fi, nullptr, &g_fence) != VK_SUCCESS)
{
return false;
}
VkSemaphoreCreateInfo si{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO};
if (g_fns.CreateSemaphore(g_device, &si, nullptr, &g_present_sem) != VK_SUCCESS)
{
return false;
}
}
const VkDeviceSize need = static_cast<VkDeviceSize>(w) * h * 4;
if (g_staging != VK_NULL_HANDLE && g_staging_size == need)
{
return true;
}
release_staging();
VkBufferCreateInfo bci{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
bci.size = need;
bci.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
if (g_fns.CreateBuffer(g_device, &bci, nullptr, &g_staging) != VK_SUCCESS)
{
return false;
}
VkMemoryRequirements mr{};
g_fns.GetBufferMemoryRequirements(g_device, g_staging, &mr);
std::uint32_t mem_type = 0;
if (!find_host_visible_memory(mr.memoryTypeBits, mem_type))
{
release_staging();
return false;
}
VkMemoryAllocateInfo mai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
mai.allocationSize = mr.size;
mai.memoryTypeIndex = mem_type;
if (g_fns.AllocateMemory(g_device, &mai, nullptr, &g_staging_mem) != VK_SUCCESS ||
g_fns.BindBufferMemory(g_device, g_staging, g_staging_mem, 0) != VK_SUCCESS ||
g_fns.MapMemory(g_device, g_staging_mem, 0, VK_WHOLE_SIZE, 0, &g_staging_mapped) != VK_SUCCESS)
{
release_staging();
return false;
}
g_staging_size = need;
return true;
}
void image_barrier(VkCommandBuffer cb, VkImage img, VkImageLayout from, VkImageLayout to, VkAccessFlags src,
VkAccessFlags dst)
{
VkImageMemoryBarrier b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
b.srcAccessMask = src;
b.dstAccessMask = dst;
b.oldLayout = from;
b.newLayout = to;
b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
b.image = img;
b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
g_fns.CmdPipelineBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0,
nullptr, 0, nullptr, 1, &b);
}
// Read `image` (in PRESENT_SRC layout) back into the shared texture. `wait`/`wait_count` are the
// present's wait semaphores, which our submit consumes and replaces with g_present_sem (returned
// via *out_sem) so the real present orders correctly after our copy without double-waiting.
bool capture_present(VkImage image, VkFormat fmt, std::uint32_t w, std::uint32_t h, const VkSemaphore* wait,
std::uint32_t wait_count, VkSemaphore& out_sem)
{
const bool bgra = fmt == VK_FORMAT_B8G8R8A8_UNORM || fmt == VK_FORMAT_B8G8R8A8_SRGB;
const bool rgba = fmt == VK_FORMAT_R8G8B8A8_UNORM || fmt == VK_FORMAT_R8G8B8A8_SRGB;
if (!bgra && !rgba)
{
if (!g_unsupported_logged)
{
logf("vk: unsupported swapchain format=%d (only 8888 BGRA/RGBA); idle", static_cast<int>(fmt));
g_unsupported_logged = true;
}
return false;
}
if (!ensure_d3d() || !ensure_shared_texture(w, h) || !ensure_vk_resources(w, h))
{
return false;
}
g_fns.ResetCommandBuffer(g_cmd, 0);
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
g_fns.BeginCommandBuffer(g_cmd, &bi);
image_barrier(g_cmd, image, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_READ_BIT);
VkBufferImageCopy region{};
region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
region.imageExtent = {w, h, 1};
g_fns.CmdCopyImageToBuffer(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, g_staging, 1, &region);
image_barrier(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_MEMORY_READ_BIT);
g_fns.EndCommandBuffer(g_cmd);
std::vector<VkPipelineStageFlags> wait_stages(wait_count, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO};
si.waitSemaphoreCount = wait_count;
si.pWaitSemaphores = wait;
si.pWaitDstStageMask = wait_count != 0 ? wait_stages.data() : nullptr;
si.commandBufferCount = 1;
si.pCommandBuffers = &g_cmd;
si.signalSemaphoreCount = 1;
si.pSignalSemaphores = &g_present_sem;
g_fns.ResetFences(g_device, 1, &g_fence);
if (g_fns.QueueSubmit(g_queue, 1, &si, g_fence) != VK_SUCCESS)
{
return false;
}
g_fns.WaitForFences(g_device, 1, &g_fence, VK_TRUE, UINT64_MAX);
// Swizzle the mapped staging (tightly packed w*4) into RGBA and upload.
const size_t row = static_cast<size_t>(w) * 4;
if (g_rgba.size() != row * h)
{
g_rgba.resize(row * h);
}
const auto* src = static_cast<const unsigned char*>(g_staging_mapped);
for (std::uint32_t y = 0; y < h; ++y)
{
const unsigned char* s = src + static_cast<size_t>(y) * row;
unsigned char* o = g_rgba.data() + static_cast<size_t>(y) * row;
if (bgra)
{
for (std::uint32_t x = 0; x < w; ++x)
{
o[x * 4 + 0] = s[x * 4 + 2];
o[x * 4 + 1] = s[x * 4 + 1];
o[x * 4 + 2] = s[x * 4 + 0];
o[x * 4 + 3] = 255;
}
}
else
{
std::memcpy(o, s, row);
}
}
bool shared = false;
if (g_shared_mutex->AcquireSync(kVideoMutexKey, 8) == S_OK)
{
g_d3d_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, g_rgba.data(), static_cast<UINT>(row), 0);
g_d3d_ctx->Flush();
g_shared_mutex->ReleaseSync(kVideoMutexKey);
shared = true;
}
out_sem = g_present_sem;
if (shared)
{
g_present_captured.store(true, std::memory_order_relaxed);
g_frames_shared.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->publish_video_frame(w, h, static_cast<std::uint32_t>(DXGI_FORMAT_R8G8B8A8_UNORM));
}
}
return true;
}
const SwapInfo* find_swap(VkSwapchainKHR sc)
{
for (const SwapInfo& s : g_swaps)
{
if (s.sc == sc)
{
return &s;
}
}
return nullptr;
}
VKAPI_ATTR VkResult VKAPI_CALL hk_vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* pPresentInfo)
{
hook_note_call(g_id_present);
g_presents.fetch_add(1, std::memory_order_relaxed);
if (g_ipc != nullptr)
{
g_ipc->note_present();
}
// Capture only the simple, common single-swapchain present; pass anything else through.
if (g_device != VK_NULL_HANDLE && pPresentInfo != nullptr && pPresentInfo->swapchainCount == 1)
{
const SwapInfo* s = find_swap(pPresentInfo->pSwapchains[0]);
if (s != nullptr && pPresentInfo->pImageIndices[0] < s->images.size())
{
VkSemaphore chained = VK_NULL_HANDLE;
if (capture_present(s->images[pPresentInfo->pImageIndices[0]], s->fmt, s->w, s->h,
pPresentInfo->pWaitSemaphores, pPresentInfo->waitSemaphoreCount, chained))
{
// Replace the present's wait with our chained semaphore (our submit consumed the
// originals and signals this one), so the present still orders after rendering.
VkPresentInfoKHR pi = *pPresentInfo;
pi.waitSemaphoreCount = 1;
pi.pWaitSemaphores = &chained;
return g_real_present(queue, &pi);
}
}
}
return g_real_present(queue, pPresentInfo);
}
VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR* ci,
const VkAllocationCallbacks* alloc, VkSwapchainKHR* out)
{
const VkResult r = g_real_create_swapchain(device, ci, alloc, out);
if (r == VK_SUCCESS && out != nullptr && g_fns.GetSwapchainImagesKHR != nullptr)
{
SwapInfo info{};
info.sc = *out;
info.fmt = ci->imageFormat;
info.w = ci->imageExtent.width;
info.h = ci->imageExtent.height;
std::uint32_t n = 0;
g_fns.GetSwapchainImagesKHR(device, *out, &n, nullptr);
info.images.resize(n);
g_fns.GetSwapchainImagesKHR(device, *out, &n, info.images.data());
g_swaps.push_back(std::move(info));
// (Don't log the VkSwapchainKHR handle: it's a uint64_t on x86, not a pointer.)
logf("vk: swapchain %ux%u fmt=%d images=%u", ci->imageExtent.width, ci->imageExtent.height,
static_cast<int>(ci->imageFormat), n);
}
return r;
}
// Resolve every device function we need for the read-back via the real vkGetDeviceProcAddr.
void load_device_fns(VkDevice device)
{
#define LOAD(field, vkname) g_fns.field = reinterpret_cast<PFN_##vkname>(g_real_gdpa(device, #vkname))
LOAD(GetDeviceQueue, vkGetDeviceQueue);
LOAD(CreateCommandPool, vkCreateCommandPool);
LOAD(DestroyCommandPool, vkDestroyCommandPool);
LOAD(AllocateCommandBuffers, vkAllocateCommandBuffers);
LOAD(BeginCommandBuffer, vkBeginCommandBuffer);
LOAD(EndCommandBuffer, vkEndCommandBuffer);
LOAD(ResetCommandBuffer, vkResetCommandBuffer);
LOAD(CmdPipelineBarrier, vkCmdPipelineBarrier);
LOAD(CmdCopyImageToBuffer, vkCmdCopyImageToBuffer);
LOAD(QueueSubmit, vkQueueSubmit);
LOAD(CreateFence, vkCreateFence);
LOAD(DestroyFence, vkDestroyFence);
LOAD(WaitForFences, vkWaitForFences);
LOAD(ResetFences, vkResetFences);
LOAD(CreateSemaphore, vkCreateSemaphore);
LOAD(DestroySemaphore, vkDestroySemaphore);
LOAD(CreateBuffer, vkCreateBuffer);
LOAD(DestroyBuffer, vkDestroyBuffer);
LOAD(GetBufferMemoryRequirements, vkGetBufferMemoryRequirements);
LOAD(AllocateMemory, vkAllocateMemory);
LOAD(FreeMemory, vkFreeMemory);
LOAD(BindBufferMemory, vkBindBufferMemory);
LOAD(MapMemory, vkMapMemory);
LOAD(UnmapMemory, vkUnmapMemory);
LOAD(GetSwapchainImagesKHR, vkGetSwapchainImagesKHR);
LOAD(DeviceWaitIdle, vkDeviceWaitIdle);
#undef LOAD
g_fns.GetPhysicalDeviceMemoryProperties =
reinterpret_cast<PFN_vkGetPhysicalDeviceMemoryProperties>(real_gipa(g_instance, "vkGetPhysicalDeviceMemoryProperties"));
}
VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateDevice(VkPhysicalDevice phys, const VkDeviceCreateInfo* ci,
const VkAllocationCallbacks* alloc, VkDevice* out)
{
const VkResult r = g_real_create_device(phys, ci, alloc, out);
if (r == VK_SUCCESS && out != nullptr && g_device == VK_NULL_HANDLE) // track the first device
{
g_phys = phys;
g_device = *out;
g_qfam = ci->queueCreateInfoCount > 0 ? ci->pQueueCreateInfos[0].queueFamilyIndex : 0;
g_real_gdpa = reinterpret_cast<PFN_vkGetDeviceProcAddr>(real_gipa(g_instance, "vkGetDeviceProcAddr"));
g_real_create_swapchain =
reinterpret_cast<PFN_vkCreateSwapchainKHR>(g_real_gdpa(*out, "vkCreateSwapchainKHR"));
g_real_present = reinterpret_cast<PFN_vkQueuePresentKHR>(g_real_gdpa(*out, "vkQueuePresentKHR"));
load_device_fns(*out);
logf("vk: device created (qfam=%u) -- present capture armed", g_qfam);
}
return r;
}
VKAPI_ATTR VkResult VKAPI_CALL hk_vkCreateInstance(const VkInstanceCreateInfo* ci,
const VkAllocationCallbacks* alloc, VkInstance* out)
{
auto real_create = reinterpret_cast<PFN_vkCreateInstance>(real_gipa(nullptr, "vkCreateInstance"));
const VkResult r = real_create(ci, alloc, out);
if (r == VK_SUCCESS && out != nullptr)
{
g_instance = *out;
g_real_create_device = reinterpret_cast<PFN_vkCreateDevice>(real_gipa(*out, "vkCreateDevice"));
logf("vk: instance created -- intercepting device/swapchain/present");
}
return r;
}
VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL hk_vkGetDeviceProcAddr(VkDevice device, const char* name)
{
if (name != nullptr)
{
if (std::strcmp(name, "vkQueuePresentKHR") == 0)
{
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkQueuePresentKHR);
}
if (std::strcmp(name, "vkCreateSwapchainKHR") == 0)
{
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkCreateSwapchainKHR);
}
}
return g_real_gdpa != nullptr ? g_real_gdpa(device, name) : nullptr;
}
VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL hk_vkGetInstanceProcAddr(VkInstance instance, const char* name)
{
if (name != nullptr)
{
if (std::strcmp(name, "vkGetInstanceProcAddr") == 0)
{
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkGetInstanceProcAddr);
}
if (std::strcmp(name, "vkCreateInstance") == 0)
{
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkCreateInstance);
}
if (std::strcmp(name, "vkCreateDevice") == 0)
{
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkCreateDevice);
}
if (std::strcmp(name, "vkGetDeviceProcAddr") == 0)
{
return reinterpret_cast<PFN_vkVoidFunction>(&hk_vkGetDeviceProcAddr);
}
}
return real_gipa(instance, name);
}
void* gipa_export_address()
{
HMODULE vk = GetModuleHandleW(L"vulkan-1.dll");
if (vk == nullptr)
{
return nullptr; // not a Vulkan process (yet)
}
return reinterpret_cast<void*>(GetProcAddress(vk, "vkGetInstanceProcAddr"));
}
} // namespace
bool install_vk_hooks(IpcClient& ipc)
{
g_ipc = &ipc;
g_pid = GetCurrentProcessId();
if (g_hk_gipa)
{
return true; // already installed
}
if (g_id_present < 0)
{
g_id_present = hook_register("vkQueuePresentKHR", HookSubsys_Video);
}
void* gipa = gipa_export_address();
if (gipa == nullptr)
{
hook_set_installed(g_id_present, false);
return false; // vulkan-1.dll not loaded; caller can retry once the game loads it
}
g_unsupported_logged = false;
g_hk_gipa = safetyhook::create_inline(gipa, reinterpret_cast<void*>(&hk_vkGetInstanceProcAddr));
hook_set_installed(g_id_present, static_cast<bool>(g_hk_gipa));
logf("install_vk_hooks: vkGetInstanceProcAddr=%p hooked=%d", gipa, static_cast<bool>(g_hk_gipa) ? 1 : 0);
return static_cast<bool>(g_hk_gipa);
}
void remove_vk_hooks()
{
if (g_device != VK_NULL_HANDLE && g_fns.DeviceWaitIdle != nullptr)
{
g_fns.DeviceWaitIdle(g_device);
release_staging();
if (g_present_sem != VK_NULL_HANDLE)
{
g_fns.DestroySemaphore(g_device, g_present_sem, nullptr);
g_present_sem = VK_NULL_HANDLE;
}
if (g_fence != VK_NULL_HANDLE)
{
g_fns.DestroyFence(g_device, g_fence, nullptr);
g_fence = VK_NULL_HANDLE;
}
if (g_pool != VK_NULL_HANDLE)
{
g_fns.DestroyCommandPool(g_device, g_pool, nullptr);
g_pool = VK_NULL_HANDLE;
}
}
g_hk_gipa = {};
hook_set_installed(g_id_present, false);
release_shared();
if (g_d3d_ctx != nullptr)
{
g_d3d_ctx->Release();
g_d3d_ctx = nullptr;
}
if (g_d3d != nullptr)
{
g_d3d->Release();
g_d3d = nullptr;
}
g_swaps.clear();
g_rgba.clear();
g_cmd = VK_NULL_HANDLE;
g_queue = VK_NULL_HANDLE;
g_device = VK_NULL_HANDLE;
g_instance = VK_NULL_HANDLE;
g_real_gdpa = nullptr;
g_present_captured.store(false, std::memory_order_relaxed);
g_presents.store(0, std::memory_order_relaxed);
g_frames_shared.store(0, std::memory_order_relaxed);
g_ipc = nullptr;
}
std::uint64_t vk_presents()
{
return g_presents.load(std::memory_order_relaxed);
}
std::uint64_t vk_frames_shared()
{
return g_frames_shared.load(std::memory_order_relaxed);
}
bool vk_injected_too_late()
{
// vulkan-1.dll is loaded but we never captured a present -> the app resolved its present
// pointer before we hooked (or doesn't go through our chain). The host shows the banner.
return GetModuleHandleW(L"vulkan-1.dll") != nullptr && g_presents.load(std::memory_order_relaxed) == 0 &&
!g_present_captured.load(std::memory_order_relaxed);
}
} // namespace coop::hook

34
hook/src/vk_hook.hpp Normal file
View File

@@ -0,0 +1,34 @@
// Injected Vulkan capture path. Vulkan games present via vkQueuePresentKHR and (with volk or
// their own loader) cache that pointer at init, so they can't be hooked by *late* injection --
// the hook must be present before vkCreateInstance. We catch the resolution chain instead:
// inline-hook the vulkan-1.dll export vkGetInstanceProcAddr and hand back our own wrappers for
// vkCreateInstance / vkCreateDevice / vkGetDeviceProcAddr / vkCreateSwapchainKHR /
// vkQueuePresentKHR. On present we read the swap-chain image back with vkCmdCopyImageToBuffer
// (the same read-back pattern as the D3D10/D3D9/OpenGL paths) and upload it into the shared
// keyed-mutex texture via a hook-owned D3D11 device. Part of the video subsystem.
#pragma once
#include "ipc_client.hpp"
namespace coop::hook
{
// Installs the Vulkan capture hook (inline-hooks vkGetInstanceProcAddr). `ipc` must outlive the
// hook. Returns true if vulkan-1.dll is loaded and the export was hooked; false otherwise, so
// the caller can retry once the game loads Vulkan. Safe to call repeatedly.
bool install_vk_hooks(IpcClient& ipc);
// Removes the Vulkan hook and releases the shared texture + Vulkan read-back resources.
void remove_vk_hooks();
// --- Diagnostics -----------------------------------------------------------
std::uint64_t vk_presents(); // cumulative vkQueuePresentKHR detours
std::uint64_t vk_frames_shared(); // frames uploaded into the shared texture
// True when vulkan-1.dll is loaded in this process but we never captured a present -- i.e. the
// app resolved its present pointer before we hooked (injected too late). Drives the host's
// "Vulkan, injected too late" banner.
bool vk_injected_too_late();
} // namespace coop::hook