M2(Vulkan): implicit capture layer (coop_vk_layer) + chain dispatch + env test

A real chain-aware Vulkan implicit layer the loader inserts at vkCreateInstance
-- the reliable early-presence path for games that init Vulkan immediately,
which the inline-hook vk_hook can't catch. It intercepts vkCreateInstance /
Device / CreateSwapchainKHR / QueuePresentKHR via proper layer-chain dispatch
and does the same read-back capture (vkCmdCopyImageToBuffer -> swizzle ->
hook-owned D3D11 shared texture, with present-semaphore re-chaining) as vk_hook.

The loader/layer link structs (VkLayer*CreateInfo, VkNegotiateLayerInterface)
aren't in Vulkan-Headers, so they're hand-declared to interface version 2. Key
gotcha found via tracing: the loader tags those link structs with small internal
sType values (LOADER_INSTANCE_CREATE_INFO=47, _DEVICE=48), not the 1000000000
range -- matching the wrong value made the device-chain walk fail.

Scoping: an implicit layer loads into every Vulkan app, so it only *captures*
when COOP_VK_LAYER_FORCE is set (tests) or this process's image matches
%TEMP%\coop_vk_target.txt (the host writes it); otherwise pure pass-through.
mock_game_test registers it via VK_LAYER_PATH/VK_INSTANCE_LAYERS and decodes
frames through it. Ships at the bin root with its JSON manifest.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-22 13:24:37 +02:00
parent 945d7fbf78
commit 2532ffed56
5 changed files with 924 additions and 0 deletions

View File

@@ -403,6 +403,98 @@ void test_vk_capture(ID3D11Device* device)
cleanup();
}
// Vulkan capture via the implicit layer: register coop_vk_layer through the loader (VK_LAYER_PATH
// + VK_INSTANCE_LAYERS, with COOP_VK_LAYER_FORCE so it captures this process), launch the vk mock
// normally (the layer is in the chain from the first frame -- no early-load games), and decode
// frames out of the captured pixels. This is the productized form of the early-load path.
void test_vk_layer_capture(ID3D11Device* device)
{
std::printf("== video capture: vk implicit layer ==\n");
const std::wstring manifest = deployed_artifact_path(L"coop_vk_layer.json");
if (GetFileAttributesW(manifest.c_str()) == INVALID_FILE_ATTRIBUTES)
{
check(false, "coop_vk_layer.json staged");
return;
}
const std::wstring layer_dir = manifest.substr(0, manifest.find_last_of(L"\\/"));
SetEnvironmentVariableW(L"VK_LAYER_PATH", layer_dir.c_str());
SetEnvironmentVariableW(L"VK_INSTANCE_LAYERS", L"VK_LAYER_coop_capture");
SetEnvironmentVariableW(L"COOP_VK_LAYER_FORCE", L"1");
MockGame game = MockGame::launch(L"vk 30"); // normal launch; the layer is already in the chain
auto unset_env = [] {
SetEnvironmentVariableW(L"VK_LAYER_PATH", nullptr);
SetEnvironmentVariableW(L"VK_INSTANCE_LAYERS", nullptr);
SetEnvironmentVariableW(L"COOP_VK_LAYER_FORCE", nullptr);
};
unset_env();
if (!game.ok)
{
check(false, "launch vk mock (layer)");
return;
}
SharedMemory shm;
const std::uint32_t disabled = (1u << HookSubsys_Input) | (1u << HookSubsys_Focus) |
(1u << HookSubsys_Audio) | (1u << HookSubsys_Mkb);
SharedBlock* block = make_ipc(shm, game.pid(), disabled); // the layer connects to this + publishes
if (block == nullptr)
{
check(false, "ipc block (layer)");
game.kill();
return;
}
SharedTextureSource src;
src.init(device);
std::vector<std::uint32_t> seq;
std::uint64_t backward = 0;
std::uint32_t last = 0;
bool have_last = false;
for (int i = 0; i < 160 && game.alive(); ++i) // ~8 s
{
Sleep(50);
const VideoShareView share = read_video_share(block);
if (!src.update(share, game.pid()))
{
continue;
}
std::uint8_t px[4] = {};
if (!src.read_pixel(coop::mock::kFrameBlock / 2, coop::mock::kFrameBlock / 2, px))
{
continue;
}
const std::uint32_t f = coop::mock::rgb_to_frame(px[0], px[1], px[2]);
if (have_last && f < last)
{
++backward;
}
last = f;
have_last = true;
seq.push_back(f);
if (seq.size() >= 40)
{
break;
}
}
if (!game.alive() && game.exit_code() == 2 && seq.empty())
{
std::printf(" Vulkan unavailable on this machine -- skipping layer capture\n");
game.kill();
return;
}
std::printf(" copied=%llu samples=%zu backward=%llu\n", static_cast<unsigned long long>(src.frames_copied()),
seq.size(), static_cast<unsigned long long>(backward));
check(src.frames_copied() >= 5, "layer copied multiple shared frames");
check(seq.size() >= 5, "decoded multiple frame numbers via the layer");
check(backward == 0, "layer-captured frame numbers never go backwards");
if (!seq.empty())
{
check(seq.back() - seq.front() >= 10, "layer-captured frame numbers advance");
}
game.kill();
}
// Vulkan too-late detection: launch the vk mock normally (it inits Vulkan immediately), inject
// *late* (the realistic case), and assert the hook reports vk_too_late -- it sees vulkan-1.dll
// loaded but never caught the device, because the app resolved its present pointer first. This is
@@ -679,6 +771,7 @@ int main()
// Vulkan: present pointer cached at init -> can't be late-hooked, so we capture via the
// early-load path (suspended launch + inject + resume; the mock loads Vulkan and waits).
test_vk_capture(device);
test_vk_layer_capture(device);
test_vk_too_late();
test_video_capture("gl", device);
test_video_capture("dx9ex", device);