From 2532ffed561a1bff678224de13ca4c61b8ba2daf Mon Sep 17 00:00:00 2001 From: BlackMark Date: Mon, 22 Jun 2026 13:24:37 +0200 Subject: [PATCH] M2(Vulkan): implicit capture layer (coop_vk_layer) + chain dispatch + env test A real chain-aware Vulkan implicit layer the loader inserts at vkCreateInstance -- the reliable early-presence path for games that init Vulkan immediately, which the inline-hook vk_hook can't catch. It intercepts vkCreateInstance / Device / CreateSwapchainKHR / QueuePresentKHR via proper layer-chain dispatch and does the same read-back capture (vkCmdCopyImageToBuffer -> swizzle -> hook-owned D3D11 shared texture, with present-semaphore re-chaining) as vk_hook. The loader/layer link structs (VkLayer*CreateInfo, VkNegotiateLayerInterface) aren't in Vulkan-Headers, so they're hand-declared to interface version 2. Key gotcha found via tracing: the loader tags those link structs with small internal sType values (LOADER_INSTANCE_CREATE_INFO=47, _DEVICE=48), not the 1000000000 range -- matching the wrong value made the device-chain walk fail. Scoping: an implicit layer loads into every Vulkan app, so it only *captures* when COOP_VK_LAYER_FORCE is set (tests) or this process's image matches %TEMP%\coop_vk_target.txt (the host writes it); otherwise pure pass-through. mock_game_test registers it via VK_LAYER_PATH/VK_INSTANCE_LAYERS and decodes frames through it. Ships at the bin root with its JSON manifest. Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 1 + tests/mock_game_test.cpp | 93 +++++ vk_layer/CMakeLists.txt | 19 + vk_layer/coop_vk_layer.cpp | 797 ++++++++++++++++++++++++++++++++++++ vk_layer/coop_vk_layer.json | 14 + 5 files changed, 924 insertions(+) create mode 100644 vk_layer/CMakeLists.txt create mode 100644 vk_layer/coop_vk_layer.cpp create mode 100644 vk_layer/coop_vk_layer.json diff --git a/CMakeLists.txt b/CMakeLists.txt index f89f294..5e9ab88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,6 +131,7 @@ if(COOP_BUILD_HOOK) set(SAFETYHOOK_BUILD_DOCS OFF CACHE BOOL "" FORCE) add_subdirectory(third_party/safetyhook) add_subdirectory(hook) + add_subdirectory(vk_layer) # coop_vk_layer: implicit Vulkan capture layer (early-presence path) enable_testing() add_subdirectory(tools/audio_tone) # coop_tone: audio source for the loopback test add_subdirectory(tools/mock_game) # coop_mock_game: A/V test game for the capture/hook tests diff --git a/tests/mock_game_test.cpp b/tests/mock_game_test.cpp index a3fa5c9..a04cb00 100644 --- a/tests/mock_game_test.cpp +++ b/tests/mock_game_test.cpp @@ -403,6 +403,98 @@ void test_vk_capture(ID3D11Device* device) cleanup(); } +// Vulkan capture via the implicit layer: register coop_vk_layer through the loader (VK_LAYER_PATH +// + VK_INSTANCE_LAYERS, with COOP_VK_LAYER_FORCE so it captures this process), launch the vk mock +// normally (the layer is in the chain from the first frame -- no early-load games), and decode +// frames out of the captured pixels. This is the productized form of the early-load path. +void test_vk_layer_capture(ID3D11Device* device) +{ + std::printf("== video capture: vk implicit layer ==\n"); + const std::wstring manifest = deployed_artifact_path(L"coop_vk_layer.json"); + if (GetFileAttributesW(manifest.c_str()) == INVALID_FILE_ATTRIBUTES) + { + check(false, "coop_vk_layer.json staged"); + return; + } + const std::wstring layer_dir = manifest.substr(0, manifest.find_last_of(L"\\/")); + SetEnvironmentVariableW(L"VK_LAYER_PATH", layer_dir.c_str()); + SetEnvironmentVariableW(L"VK_INSTANCE_LAYERS", L"VK_LAYER_coop_capture"); + SetEnvironmentVariableW(L"COOP_VK_LAYER_FORCE", L"1"); + + MockGame game = MockGame::launch(L"vk 30"); // normal launch; the layer is already in the chain + auto unset_env = [] { + SetEnvironmentVariableW(L"VK_LAYER_PATH", nullptr); + SetEnvironmentVariableW(L"VK_INSTANCE_LAYERS", nullptr); + SetEnvironmentVariableW(L"COOP_VK_LAYER_FORCE", nullptr); + }; + unset_env(); + if (!game.ok) + { + check(false, "launch vk mock (layer)"); + return; + } + + SharedMemory shm; + const std::uint32_t disabled = (1u << HookSubsys_Input) | (1u << HookSubsys_Focus) | + (1u << HookSubsys_Audio) | (1u << HookSubsys_Mkb); + SharedBlock* block = make_ipc(shm, game.pid(), disabled); // the layer connects to this + publishes + if (block == nullptr) + { + check(false, "ipc block (layer)"); + game.kill(); + return; + } + + SharedTextureSource src; + src.init(device); + std::vector seq; + std::uint64_t backward = 0; + std::uint32_t last = 0; + bool have_last = false; + for (int i = 0; i < 160 && game.alive(); ++i) // ~8 s + { + Sleep(50); + const VideoShareView share = read_video_share(block); + if (!src.update(share, game.pid())) + { + continue; + } + std::uint8_t px[4] = {}; + if (!src.read_pixel(coop::mock::kFrameBlock / 2, coop::mock::kFrameBlock / 2, px)) + { + continue; + } + const std::uint32_t f = coop::mock::rgb_to_frame(px[0], px[1], px[2]); + if (have_last && f < last) + { + ++backward; + } + last = f; + have_last = true; + seq.push_back(f); + if (seq.size() >= 40) + { + break; + } + } + if (!game.alive() && game.exit_code() == 2 && seq.empty()) + { + std::printf(" Vulkan unavailable on this machine -- skipping layer capture\n"); + game.kill(); + return; + } + std::printf(" copied=%llu samples=%zu backward=%llu\n", static_cast(src.frames_copied()), + seq.size(), static_cast(backward)); + check(src.frames_copied() >= 5, "layer copied multiple shared frames"); + check(seq.size() >= 5, "decoded multiple frame numbers via the layer"); + check(backward == 0, "layer-captured frame numbers never go backwards"); + if (!seq.empty()) + { + check(seq.back() - seq.front() >= 10, "layer-captured frame numbers advance"); + } + game.kill(); +} + // Vulkan too-late detection: launch the vk mock normally (it inits Vulkan immediately), inject // *late* (the realistic case), and assert the hook reports vk_too_late -- it sees vulkan-1.dll // loaded but never caught the device, because the app resolved its present pointer first. This is @@ -679,6 +771,7 @@ int main() // Vulkan: present pointer cached at init -> can't be late-hooked, so we capture via the // early-load path (suspended launch + inject + resume; the mock loads Vulkan and waits). test_vk_capture(device); + test_vk_layer_capture(device); test_vk_too_late(); test_video_capture("gl", device); test_video_capture("dx9ex", device); diff --git a/vk_layer/CMakeLists.txt b/vk_layer/CMakeLists.txt new file mode 100644 index 0000000..3aadd46 --- /dev/null +++ b/vk_layer/CMakeLists.txt @@ -0,0 +1,19 @@ +# CoopAllTheThings Vulkan capture layer: a real implicit Vulkan layer the loader inserts at +# vkCreateInstance, so it's in the chain before a game (that caches its present pointer at init) +# can resolve vkQueuePresentKHR -- the reliable early-presence path the inline-hook vk_hook can't +# get for immediate-init games. Ships at the bin root (registered for real games), with its JSON +# manifest copied alongside (the manifest's library_path is relative). +coop_require_submodule("Vulkan-Headers" "third_party/Vulkan-Headers/include/vulkan/vulkan.h") + +add_library(coop_vk_layer SHARED coop_vk_layer.cpp) + +target_include_directories(coop_vk_layer PRIVATE + ${CMAKE_SOURCE_DIR}/hook/src # ipc_client.hpp (reused IPC client) + ${CMAKE_SOURCE_DIR}/third_party/Vulkan-Headers/include) + +target_link_libraries(coop_vk_layer PRIVATE coop_common d3d11 dxgi) + +# Drop the manifest next to the built DLL so VK_LAYER_PATH / the registry can find it. +add_custom_command(TARGET coop_vk_layer POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_SOURCE_DIR}/coop_vk_layer.json $/coop_vk_layer.json) diff --git a/vk_layer/coop_vk_layer.cpp b/vk_layer/coop_vk_layer.cpp new file mode 100644 index 0000000..9fc02f0 --- /dev/null +++ b/vk_layer/coop_vk_layer.cpp @@ -0,0 +1,797 @@ +// CoopAllTheThings Vulkan capture layer. +// +// A real Vulkan *implicit layer* the loader inserts at vkCreateInstance -- guaranteed to be in +// the chain before the game resolves vkQueuePresentKHR. This is the reliable early-presence path +// for games that initialize Vulkan immediately (which inject-after-launch + the inline-hook +// vk_hook can't catch). It does the same capture as vk_hook -- read the presented image back with +// vkCmdCopyImageToBuffer, swizzle BGRA->RGBA, upload into the shared keyed-mutex texture on a +// hook-owned D3D11 device, and re-chain the present's wait semaphores -- but via proper +// layer-chain dispatch instead of an inline hook. +// +// Scoping: an implicit layer loads into *every* Vulkan app, so the layer only *captures* when it +// recognises the process as the host's target -- env COOP_VK_LAYER_FORCE=1 (tests), or this +// process's image basename matches %TEMP%\coop_vk_target.txt (written by the host's "Set up Vulkan +// layer" checkbox). Otherwise it's a pure pass-through. +// +// The loader/layer interface structs (VkLayer*CreateInfo, VkNegotiateLayerInterface) live in +// vk_layer.h, which Vulkan-Headers doesn't ship, so they're declared here to the stable +// loader-interface-version-2 ABI. +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#define VK_NO_PROTOTYPES +#define VK_USE_PLATFORM_WIN32_KHR +#include + +#include "coop/shared_memory.hpp" +#include "ipc_client.hpp" + +// --- Loader/layer interface (interface version 2) --------------------------- +extern "C" +{ + typedef enum VkLayerFunction_ + { + COOP_VK_LAYER_LINK_INFO = 0, + COOP_VK_LOADER_DATA_CALLBACK = 1, + COOP_VK_LOADER_LAYER_CREATE_DEVICE_CALLBACK = 2, + COOP_VK_LOADER_FEATURES = 3, + } CoopVkLayerFunction; + + typedef PFN_vkVoidFunction(VKAPI_PTR* PFN_GetPhysicalDeviceProcAddr)(VkInstance, const char*); + + typedef struct VkLayerInstanceLink_ + { + struct VkLayerInstanceLink_* pNext; + PFN_vkGetInstanceProcAddr pfnNextGetInstanceProcAddr; + PFN_GetPhysicalDeviceProcAddr pfnNextGetPhysicalDeviceProcAddr; + } VkLayerInstanceLink; + + typedef struct VkLayerInstanceCreateInfo + { + VkStructureType sType; // 1000000000 = VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO + const void* pNext; + CoopVkLayerFunction function; + union { + VkLayerInstanceLink* pLayerInfo; + void* pfnCallback; // other callbacks (unused here); keeps the union pointer-sized + } u; + } VkLayerInstanceCreateInfo; + + typedef struct VkLayerDeviceLink_ + { + struct VkLayerDeviceLink_* pNext; + PFN_vkGetInstanceProcAddr pfnNextGetInstanceProcAddr; + PFN_vkGetDeviceProcAddr pfnNextGetDeviceProcAddr; + } VkLayerDeviceLink; + + typedef struct VkLayerDeviceCreateInfo + { + VkStructureType sType; // 1000000001 = VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO + const void* pNext; + CoopVkLayerFunction function; + union { + VkLayerDeviceLink* pLayerInfo; + void* pfnCallback; + } u; + } VkLayerDeviceCreateInfo; + + typedef struct VkNegotiateLayerInterface + { + uint32_t sType; // 1 = LAYER_NEGOTIATE_INTERFACE_STRUCT + void* pNext; + uint32_t loaderLayerInterfaceVersion; + PFN_vkGetInstanceProcAddr pfnGetInstanceProcAddr; + PFN_vkGetDeviceProcAddr pfnGetDeviceProcAddr; + PFN_GetPhysicalDeviceProcAddr pfnGetPhysicalDeviceProcAddr; + } VkNegotiateLayerInterface; +} + +namespace +{ +// The loader tags its chain-link structs with small, loader-internal sType values (not the +// 1000000000-range): VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO = 47, _DEVICE = 48. These are +// from the (unvendored) vk_layer.h and are stable across loader versions. +constexpr VkStructureType kLoaderInstanceCreateInfo = static_cast(47); +constexpr VkStructureType kLoaderDeviceCreateInfo = static_cast(48); + +using coop::hook::IpcClient; + +IpcClient g_ipc; +bool g_ipc_tried = false; +bool g_active = false; // do we capture in this process? (scoping) + +// Chain dispatch. +PFN_vkGetInstanceProcAddr g_next_gipa = nullptr; +PFN_vkGetDeviceProcAddr g_next_gdpa = nullptr; +PFN_vkQueuePresentKHR g_real_present = nullptr; +PFN_vkCreateSwapchainKHR g_real_create_swapchain = nullptr; + +VkInstance g_instance = VK_NULL_HANDLE; +VkPhysicalDevice g_phys = VK_NULL_HANDLE; +VkDevice g_device = VK_NULL_HANDLE; +std::uint32_t g_qfam = 0; +VkQueue g_queue = VK_NULL_HANDLE; + +// Device functions for the read-back (same set as vk_hook). +struct VkFns +{ + PFN_vkGetDeviceQueue GetDeviceQueue; + PFN_vkCreateCommandPool CreateCommandPool; + PFN_vkDestroyCommandPool DestroyCommandPool; + PFN_vkAllocateCommandBuffers AllocateCommandBuffers; + PFN_vkBeginCommandBuffer BeginCommandBuffer; + PFN_vkEndCommandBuffer EndCommandBuffer; + PFN_vkResetCommandBuffer ResetCommandBuffer; + PFN_vkCmdPipelineBarrier CmdPipelineBarrier; + PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer; + PFN_vkQueueSubmit QueueSubmit; + PFN_vkCreateFence CreateFence; + PFN_vkDestroyFence DestroyFence; + PFN_vkWaitForFences WaitForFences; + PFN_vkResetFences ResetFences; + PFN_vkCreateSemaphore CreateSemaphore; + PFN_vkDestroySemaphore DestroySemaphore; + PFN_vkCreateBuffer CreateBuffer; + PFN_vkDestroyBuffer DestroyBuffer; + PFN_vkGetBufferMemoryRequirements GetBufferMemoryRequirements; + PFN_vkAllocateMemory AllocateMemory; + PFN_vkFreeMemory FreeMemory; + PFN_vkBindBufferMemory BindBufferMemory; + PFN_vkMapMemory MapMemory; + PFN_vkUnmapMemory UnmapMemory; + PFN_vkGetSwapchainImagesKHR GetSwapchainImagesKHR; + PFN_vkDeviceWaitIdle DeviceWaitIdle; + PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties; +}; +VkFns g_fns{}; + +VkCommandPool g_pool = VK_NULL_HANDLE; +VkCommandBuffer g_cmd = VK_NULL_HANDLE; +VkFence g_fence = VK_NULL_HANDLE; +VkSemaphore g_present_sem = VK_NULL_HANDLE; +VkBuffer g_staging = VK_NULL_HANDLE; +VkDeviceMemory g_staging_mem = VK_NULL_HANDLE; +VkDeviceSize g_staging_size = 0; +void* g_staging_mapped = nullptr; + +struct SwapInfo +{ + VkSwapchainKHR sc; + VkFormat fmt; + std::uint32_t w, h; + std::vector images; +}; +std::vector g_swaps; + +ID3D11Device* g_d3d = nullptr; +ID3D11DeviceContext* g_d3d_ctx = nullptr; +ID3D11Texture2D* g_shared_tex = nullptr; +IDXGIKeyedMutex* g_shared_mutex = nullptr; +HANDLE g_shared_handle = nullptr; +UINT g_share_w = 0, g_share_h = 0; +std::vector g_rgba; + +bool eq(const char* a, const char* b) +{ + return std::strcmp(a, b) == 0; +} + +// Optional file trace for debugging the chain dispatch (enable with COOP_VK_LAYER_LOG). +void logvk(const char* fmt, ...) +{ + static int enabled = -1; + if (enabled < 0) + { + enabled = GetEnvironmentVariableW(L"COOP_VK_LAYER_LOG", nullptr, 0) != 0 ? 1 : 0; + } + if (enabled == 0) + { + return; + } + wchar_t dir[MAX_PATH] = {}; + if (GetTempPathW(MAX_PATH, dir) == 0) + { + return; + } + FILE* f = _wfopen((std::wstring(dir) + L"coop_vk_layer.log").c_str(), L"a"); + if (f == nullptr) + { + return; + } + va_list ap; + va_start(ap, fmt); + std::vfprintf(f, fmt, ap); + va_end(ap); + std::fputc('\n', f); + std::fclose(f); +} + +// Decide whether this process is the host's capture target (see file header). +bool decide_active() +{ + if (GetEnvironmentVariableW(L"COOP_VK_LAYER_FORCE", nullptr, 0) != 0) + { + return true; + } + wchar_t dir[MAX_PATH] = {}; + const DWORD n = GetTempPathW(MAX_PATH, dir); + if (n == 0 || n >= MAX_PATH) + { + return false; + } + HANDLE f = CreateFileW((std::wstring(dir) + L"coop_vk_target.txt").c_str(), GENERIC_READ, FILE_SHARE_READ, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) + { + return false; + } + char want[MAX_PATH] = {}; + DWORD got = 0; + ReadFile(f, want, sizeof(want) - 1, &got, nullptr); + CloseHandle(f); + // Trim trailing whitespace/newline. + while (got > 0 && (want[got - 1] == '\n' || want[got - 1] == '\r' || want[got - 1] == ' ')) + { + want[--got] = '\0'; + } + if (got == 0) + { + return false; + } + wchar_t self[MAX_PATH] = {}; + GetModuleFileNameW(nullptr, self, MAX_PATH); + const wchar_t* base = wcsrchr(self, L'\\'); + base = base != nullptr ? base + 1 : self; + char self8[MAX_PATH] = {}; + WideCharToMultiByte(CP_UTF8, 0, base, -1, self8, sizeof(self8), nullptr, nullptr); + return _stricmp(self8, want) == 0; +} + +bool ensure_d3d() +{ + if (g_d3d != nullptr) + { + return true; + } + return SUCCEEDED(D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, nullptr, 0, + D3D11_SDK_VERSION, &g_d3d, nullptr, &g_d3d_ctx)) && + g_d3d != nullptr; +} + +void release_shared() +{ + if (g_shared_mutex) + { + g_shared_mutex->Release(); + g_shared_mutex = nullptr; + } + if (g_shared_tex) + { + g_shared_tex->Release(); + g_shared_tex = nullptr; + } + if (g_shared_handle) + { + CloseHandle(g_shared_handle); + g_shared_handle = nullptr; + } + g_share_w = g_share_h = 0; +} + +bool ensure_shared_texture(UINT w, UINT h) +{ + if (g_shared_tex && g_share_w == w && g_share_h == h) + { + return true; + } + release_shared(); + D3D11_TEXTURE2D_DESC d{}; + d.Width = w; + d.Height = h; + d.MipLevels = 1; + d.ArraySize = 1; + d.Format = DXGI_FORMAT_R8G8B8A8_UNORM; + d.SampleDesc.Count = 1; + d.Usage = D3D11_USAGE_DEFAULT; + d.BindFlags = D3D11_BIND_SHADER_RESOURCE; + d.MiscFlags = D3D11_RESOURCE_MISC_SHARED_NTHANDLE | D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX; + if (FAILED(g_d3d->CreateTexture2D(&d, nullptr, &g_shared_tex)) || !g_shared_tex) + { + return false; + } + IDXGIResource1* res = nullptr; + if (FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIResource1), reinterpret_cast(&res))) || !res) + { + release_shared(); + return false; + } + const std::wstring name = coop::video_share_name(GetCurrentProcessId()); + const HRESULT hr = res->CreateSharedHandle( + nullptr, DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE, name.c_str(), &g_shared_handle); + res->Release(); + if (FAILED(hr) || !g_shared_handle || + FAILED(g_shared_tex->QueryInterface(__uuidof(IDXGIKeyedMutex), reinterpret_cast(&g_shared_mutex)))) + { + release_shared(); + return false; + } + g_share_w = w; + g_share_h = h; + return true; +} + +bool find_host_visible_memory(std::uint32_t bits, std::uint32_t& out) +{ + VkPhysicalDeviceMemoryProperties mp{}; + g_fns.GetPhysicalDeviceMemoryProperties(g_phys, &mp); + const VkMemoryPropertyFlags want = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + for (std::uint32_t i = 0; i < mp.memoryTypeCount; ++i) + { + if ((bits & (1u << i)) && (mp.memoryTypes[i].propertyFlags & want) == want) + { + out = i; + return true; + } + } + return false; +} + +void release_staging() +{ + if (g_staging_mapped && g_staging_mem) + { + g_fns.UnmapMemory(g_device, g_staging_mem); + g_staging_mapped = nullptr; + } + if (g_staging) + { + g_fns.DestroyBuffer(g_device, g_staging, nullptr); + g_staging = VK_NULL_HANDLE; + } + if (g_staging_mem) + { + g_fns.FreeMemory(g_device, g_staging_mem, nullptr); + g_staging_mem = VK_NULL_HANDLE; + } + g_staging_size = 0; +} + +bool ensure_vk_resources(std::uint32_t w, std::uint32_t h) +{ + if (g_queue == VK_NULL_HANDLE) + { + g_fns.GetDeviceQueue(g_device, g_qfam, 0, &g_queue); + } + if (g_pool == VK_NULL_HANDLE) + { + VkCommandPoolCreateInfo pci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pci.queueFamilyIndex = g_qfam; + if (g_fns.CreateCommandPool(g_device, &pci, nullptr, &g_pool) != VK_SUCCESS) + { + return false; + } + VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ai.commandPool = g_pool; + ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + ai.commandBufferCount = 1; + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkSemaphoreCreateInfo si{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; + if (g_fns.AllocateCommandBuffers(g_device, &ai, &g_cmd) != VK_SUCCESS || + g_fns.CreateFence(g_device, &fi, nullptr, &g_fence) != VK_SUCCESS || + g_fns.CreateSemaphore(g_device, &si, nullptr, &g_present_sem) != VK_SUCCESS) + { + return false; + } + } + const VkDeviceSize need = static_cast(w) * h * 4; + if (g_staging != VK_NULL_HANDLE && g_staging_size == need) + { + return true; + } + release_staging(); + VkBufferCreateInfo bci{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bci.size = need; + bci.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (g_fns.CreateBuffer(g_device, &bci, nullptr, &g_staging) != VK_SUCCESS) + { + return false; + } + VkMemoryRequirements mr{}; + g_fns.GetBufferMemoryRequirements(g_device, g_staging, &mr); + std::uint32_t mt = 0; + if (!find_host_visible_memory(mr.memoryTypeBits, mt)) + { + release_staging(); + return false; + } + VkMemoryAllocateInfo mai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = mr.size; + mai.memoryTypeIndex = mt; + if (g_fns.AllocateMemory(g_device, &mai, nullptr, &g_staging_mem) != VK_SUCCESS || + g_fns.BindBufferMemory(g_device, g_staging, g_staging_mem, 0) != VK_SUCCESS || + g_fns.MapMemory(g_device, g_staging_mem, 0, VK_WHOLE_SIZE, 0, &g_staging_mapped) != VK_SUCCESS) + { + release_staging(); + return false; + } + g_staging_size = need; + return true; +} + +void barrier(VkCommandBuffer cb, VkImage img, VkImageLayout from, VkImageLayout to, VkAccessFlags s, + VkAccessFlags d) +{ + VkImageMemoryBarrier b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + b.srcAccessMask = s; + b.dstAccessMask = d; + b.oldLayout = from; + b.newLayout = to; + b.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.image = img; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + g_fns.CmdPipelineBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, + nullptr, 0, nullptr, 1, &b); +} + +bool capture(VkImage image, VkFormat fmt, std::uint32_t w, std::uint32_t h, const VkSemaphore* wait, + std::uint32_t wait_count, VkSemaphore& out_sem) +{ + const bool bgra = fmt == VK_FORMAT_B8G8R8A8_UNORM || fmt == VK_FORMAT_B8G8R8A8_SRGB; + const bool rgba = fmt == VK_FORMAT_R8G8B8A8_UNORM || fmt == VK_FORMAT_R8G8B8A8_SRGB; + if ((!bgra && !rgba) || !ensure_d3d() || !ensure_shared_texture(w, h) || !ensure_vk_resources(w, h)) + { + return false; + } + g_fns.ResetCommandBuffer(g_cmd, 0); + VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + g_fns.BeginCommandBuffer(g_cmd, &bi); + barrier(g_cmd, image, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {w, h, 1}; + g_fns.CmdCopyImageToBuffer(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, g_staging, 1, ®ion); + barrier(g_cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, + VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_MEMORY_READ_BIT); + g_fns.EndCommandBuffer(g_cmd); + + std::vector stages(wait_count, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.waitSemaphoreCount = wait_count; + si.pWaitSemaphores = wait; + si.pWaitDstStageMask = wait_count ? stages.data() : nullptr; + si.commandBufferCount = 1; + si.pCommandBuffers = &g_cmd; + si.signalSemaphoreCount = 1; + si.pSignalSemaphores = &g_present_sem; + g_fns.ResetFences(g_device, 1, &g_fence); + if (g_fns.QueueSubmit(g_queue, 1, &si, g_fence) != VK_SUCCESS) + { + return false; + } + g_fns.WaitForFences(g_device, 1, &g_fence, VK_TRUE, UINT64_MAX); + + const size_t row = static_cast(w) * 4; + if (g_rgba.size() != row * h) + { + g_rgba.resize(row * h); + } + const auto* src = static_cast(g_staging_mapped); + for (std::uint32_t y = 0; y < h; ++y) + { + const unsigned char* s = src + static_cast(y) * row; + unsigned char* o = g_rgba.data() + static_cast(y) * row; + if (bgra) + { + for (std::uint32_t x = 0; x < w; ++x) + { + o[x * 4 + 0] = s[x * 4 + 2]; + o[x * 4 + 1] = s[x * 4 + 1]; + o[x * 4 + 2] = s[x * 4 + 0]; + o[x * 4 + 3] = 255; + } + } + else + { + std::memcpy(o, s, row); + } + } + out_sem = g_present_sem; + if (g_shared_mutex->AcquireSync(coop::kVideoMutexKey, 8) == S_OK) + { + g_d3d_ctx->UpdateSubresource(g_shared_tex, 0, nullptr, g_rgba.data(), static_cast(row), 0); + g_d3d_ctx->Flush(); + g_shared_mutex->ReleaseSync(coop::kVideoMutexKey); + if (!g_ipc_tried) + { + g_ipc_tried = true; + g_ipc.connect(/*attempts=*/40, /*delay_ms=*/25); + } + if (g_ipc.connected()) + { + g_ipc.publish_video_frame(w, h, static_cast(DXGI_FORMAT_R8G8B8A8_UNORM)); + } + } + return true; +} + +const SwapInfo* find_swap(VkSwapchainKHR sc) +{ + for (const SwapInfo& s : g_swaps) + { + if (s.sc == sc) + { + return &s; + } + } + return nullptr; +} + +VKAPI_ATTR VkResult VKAPI_CALL layer_QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* pi) +{ + if (g_active && g_ipc.connected()) + { + g_ipc.note_present(); + } + if (g_active && pi != nullptr && pi->swapchainCount == 1) + { + const SwapInfo* s = find_swap(pi->pSwapchains[0]); + if (s != nullptr && pi->pImageIndices[0] < s->images.size()) + { + VkSemaphore chained = VK_NULL_HANDLE; + if (capture(s->images[pi->pImageIndices[0]], s->fmt, s->w, s->h, pi->pWaitSemaphores, + pi->waitSemaphoreCount, chained)) + { + VkPresentInfoKHR p = *pi; + p.waitSemaphoreCount = 1; + p.pWaitSemaphores = &chained; + return g_real_present(queue, &p); + } + } + } + return g_real_present(queue, pi); +} + +VKAPI_ATTR VkResult VKAPI_CALL layer_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR* ci, + const VkAllocationCallbacks* a, VkSwapchainKHR* out) +{ + const VkResult r = g_real_create_swapchain(device, ci, a, out); + if (g_active && r == VK_SUCCESS && out && g_fns.GetSwapchainImagesKHR) + { + SwapInfo info{}; + info.sc = *out; + info.fmt = ci->imageFormat; + info.w = ci->imageExtent.width; + info.h = ci->imageExtent.height; + std::uint32_t n = 0; + g_fns.GetSwapchainImagesKHR(device, *out, &n, nullptr); + info.images.resize(n); + g_fns.GetSwapchainImagesKHR(device, *out, &n, info.images.data()); + g_swaps.push_back(std::move(info)); + } + return r; +} + +void load_device_fns(VkDevice dev) +{ +#define LOAD(field, vkname) g_fns.field = reinterpret_cast(g_next_gdpa(dev, #vkname)) + LOAD(GetDeviceQueue, vkGetDeviceQueue); + LOAD(CreateCommandPool, vkCreateCommandPool); + LOAD(DestroyCommandPool, vkDestroyCommandPool); + LOAD(AllocateCommandBuffers, vkAllocateCommandBuffers); + LOAD(BeginCommandBuffer, vkBeginCommandBuffer); + LOAD(EndCommandBuffer, vkEndCommandBuffer); + LOAD(ResetCommandBuffer, vkResetCommandBuffer); + LOAD(CmdPipelineBarrier, vkCmdPipelineBarrier); + LOAD(CmdCopyImageToBuffer, vkCmdCopyImageToBuffer); + LOAD(QueueSubmit, vkQueueSubmit); + LOAD(CreateFence, vkCreateFence); + LOAD(DestroyFence, vkDestroyFence); + LOAD(WaitForFences, vkWaitForFences); + LOAD(ResetFences, vkResetFences); + LOAD(CreateSemaphore, vkCreateSemaphore); + LOAD(DestroySemaphore, vkDestroySemaphore); + LOAD(CreateBuffer, vkCreateBuffer); + LOAD(DestroyBuffer, vkDestroyBuffer); + LOAD(GetBufferMemoryRequirements, vkGetBufferMemoryRequirements); + LOAD(AllocateMemory, vkAllocateMemory); + LOAD(FreeMemory, vkFreeMemory); + LOAD(BindBufferMemory, vkBindBufferMemory); + LOAD(MapMemory, vkMapMemory); + LOAD(UnmapMemory, vkUnmapMemory); + LOAD(GetSwapchainImagesKHR, vkGetSwapchainImagesKHR); + LOAD(DeviceWaitIdle, vkDeviceWaitIdle); +#undef LOAD + g_fns.GetPhysicalDeviceMemoryProperties = reinterpret_cast( + g_next_gipa(g_instance, "vkGetPhysicalDeviceMemoryProperties")); +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL layer_gdpa(VkDevice device, const char* name); + +VKAPI_ATTR VkResult VKAPI_CALL layer_CreateDevice(VkPhysicalDevice phys, const VkDeviceCreateInfo* ci, + const VkAllocationCallbacks* a, VkDevice* out) +{ + auto* link = reinterpret_cast(const_cast(ci->pNext)); + while (link != nullptr && + !(link->sType == kLoaderDeviceCreateInfo && link->function == COOP_VK_LAYER_LINK_INFO)) + { + link = reinterpret_cast(const_cast(link->pNext)); + } + if (link == nullptr) + { + logvk("CreateDevice: LINK_INFO not found"); + return VK_ERROR_INITIALIZATION_FAILED; + } + PFN_vkGetInstanceProcAddr next_gipa = link->u.pLayerInfo->pfnNextGetInstanceProcAddr; + PFN_vkGetDeviceProcAddr next_gdpa = link->u.pLayerInfo->pfnNextGetDeviceProcAddr; + link->u.pLayerInfo = link->u.pLayerInfo->pNext; // advance the chain for the next layer + auto create = reinterpret_cast(next_gipa(g_instance, "vkCreateDevice")); + const VkResult r = create(phys, ci, a, out); + logvk("CreateDevice: result=%d active=%d", (int)r, g_active ? 1 : 0); + if (r == VK_SUCCESS && out != nullptr && g_device == VK_NULL_HANDLE) + { + g_phys = phys; + g_device = *out; + g_next_gdpa = next_gdpa; + g_qfam = ci->queueCreateInfoCount > 0 ? ci->pQueueCreateInfos[0].queueFamilyIndex : 0; + g_real_present = reinterpret_cast(next_gdpa(*out, "vkQueuePresentKHR")); + g_real_create_swapchain = reinterpret_cast(next_gdpa(*out, "vkCreateSwapchainKHR")); + if (g_active) + { + load_device_fns(*out); + } + } + return r; +} + +VKAPI_ATTR VkResult VKAPI_CALL layer_CreateInstance(const VkInstanceCreateInfo* ci, + const VkAllocationCallbacks* a, VkInstance* out) +{ + auto* link = reinterpret_cast(const_cast(ci->pNext)); + while (link != nullptr && + !(link->sType == kLoaderInstanceCreateInfo && link->function == COOP_VK_LAYER_LINK_INFO)) + { + link = reinterpret_cast(const_cast(link->pNext)); + } + if (link == nullptr) + { + logvk("CreateInstance: LINK_INFO not found"); + return VK_ERROR_INITIALIZATION_FAILED; + } + PFN_vkGetInstanceProcAddr next_gipa = link->u.pLayerInfo->pfnNextGetInstanceProcAddr; + link->u.pLayerInfo = link->u.pLayerInfo->pNext; // advance the chain + auto create = reinterpret_cast(next_gipa(nullptr, "vkCreateInstance")); + const VkResult r = create(ci, a, out); + if (r == VK_SUCCESS && out != nullptr) + { + g_instance = *out; + g_next_gipa = next_gipa; + g_active = decide_active(); + } + logvk("CreateInstance: result=%d active=%d", (int)r, g_active ? 1 : 0); + return r; +} + +VKAPI_ATTR void VKAPI_CALL layer_DestroyDevice(VkDevice device, const VkAllocationCallbacks* a) +{ + auto destroy = reinterpret_cast(g_next_gdpa(device, "vkDestroyDevice")); + if (g_active && device == g_device && g_fns.DeviceWaitIdle != nullptr) + { + g_fns.DeviceWaitIdle(device); + release_staging(); + if (g_present_sem) + { + g_fns.DestroySemaphore(device, g_present_sem, nullptr); + g_present_sem = VK_NULL_HANDLE; + } + if (g_fence) + { + g_fns.DestroyFence(device, g_fence, nullptr); + g_fence = VK_NULL_HANDLE; + } + if (g_pool) + { + g_fns.DestroyCommandPool(device, g_pool, nullptr); + g_pool = VK_NULL_HANDLE; + } + release_shared(); + if (g_d3d_ctx) + { + g_d3d_ctx->Release(); + g_d3d_ctx = nullptr; + } + if (g_d3d) + { + g_d3d->Release(); + g_d3d = nullptr; + } + g_swaps.clear(); + g_device = VK_NULL_HANDLE; + g_queue = VK_NULL_HANDLE; + g_cmd = VK_NULL_HANDLE; + } + destroy(device, a); +} + +VKAPI_ATTR void VKAPI_CALL layer_DestroyInstance(VkInstance instance, const VkAllocationCallbacks* a) +{ + auto destroy = reinterpret_cast(g_next_gipa(instance, "vkDestroyInstance")); + g_instance = VK_NULL_HANDLE; + destroy(instance, a); +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL layer_gdpa(VkDevice device, const char* name) +{ + if (name == nullptr) + { + return nullptr; + } + if (eq(name, "vkGetDeviceProcAddr")) + return reinterpret_cast(&layer_gdpa); + if (eq(name, "vkQueuePresentKHR")) + return reinterpret_cast(&layer_QueuePresentKHR); + if (eq(name, "vkCreateSwapchainKHR")) + return reinterpret_cast(&layer_CreateSwapchainKHR); + if (eq(name, "vkDestroyDevice")) + return reinterpret_cast(&layer_DestroyDevice); + return g_next_gdpa != nullptr ? g_next_gdpa(device, name) : nullptr; +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL layer_gipa(VkInstance instance, const char* name) +{ + if (name == nullptr) + { + return nullptr; + } + if (eq(name, "vkGetInstanceProcAddr")) + return reinterpret_cast(&layer_gipa); + if (eq(name, "vkCreateInstance")) + return reinterpret_cast(&layer_CreateInstance); + if (eq(name, "vkCreateDevice")) + return reinterpret_cast(&layer_CreateDevice); + if (eq(name, "vkDestroyInstance")) + return reinterpret_cast(&layer_DestroyInstance); + if (eq(name, "vkGetDeviceProcAddr")) + return reinterpret_cast(&layer_gdpa); + return g_next_gipa != nullptr ? g_next_gipa(instance, name) : nullptr; +} + +} // namespace + +extern "C" __declspec(dllexport) VkResult VKAPI_CALL + vkNegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface* pVersionStruct) +{ + logvk("negotiate: requestedVersion=%u", pVersionStruct->loaderLayerInterfaceVersion); + if (pVersionStruct->loaderLayerInterfaceVersion > 2) + { + pVersionStruct->loaderLayerInterfaceVersion = 2; + } + pVersionStruct->pfnGetInstanceProcAddr = layer_gipa; + pVersionStruct->pfnGetDeviceProcAddr = layer_gdpa; + pVersionStruct->pfnGetPhysicalDeviceProcAddr = nullptr; + return VK_SUCCESS; +} + +// Also export the entry points directly, for loaders that probe them by name. +extern "C" __declspec(dllexport) PFN_vkVoidFunction VKAPI_CALL coop_vkGetInstanceProcAddr(VkInstance i, + const char* n) +{ + return layer_gipa(i, n); +} +extern "C" __declspec(dllexport) PFN_vkVoidFunction VKAPI_CALL coop_vkGetDeviceProcAddr(VkDevice d, const char* n) +{ + return layer_gdpa(d, n); +} diff --git a/vk_layer/coop_vk_layer.json b/vk_layer/coop_vk_layer.json new file mode 100644 index 0000000..5c0fde9 --- /dev/null +++ b/vk_layer/coop_vk_layer.json @@ -0,0 +1,14 @@ +{ + "file_format_version": "1.2.0", + "layer": { + "name": "VK_LAYER_coop_capture", + "type": "GLOBAL", + "library_path": ".\\coop_vk_layer.dll", + "api_version": "1.1.0", + "implementation_version": "1", + "description": "CoopAllTheThings Vulkan capture layer (mirrors the swap-chain image into a shared texture; captures only the host's target process)", + "disable_environment": { + "COOP_VK_LAYER_DISABLE": "1" + } + } +}