diff --git a/README.md b/README.md index cb8091a..a66dfff 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,12 @@ the focused window. A hook→host status back-channel shows whether the hook is attached and how fast the game is polling it. The in-process `hook_selftest` validates the IPC + hook core without needing a game. -Still ahead (scoped in the plan): video mirror (WGC, then a `Present` hook), -audio (WASAPI process loopback), and x86 support. +**Phase 1b — video mirror (current).** The host captures the injected game's +window with Windows Graphics Capture and draws it letterboxed as its background, +so RPT streams a live mirror of the game. See the test steps below. + +Still ahead (scoped in the plan): a `Present`-hook capture path if WGC latency +disappoints, audio (WASAPI process loopback), and x86 support. ## Building @@ -149,3 +153,21 @@ forwarded — while the game is unfocused it won't receive OS keyboard/mouse. For a quick sanity check of the forwarding core without a game, run `bin\Debug\hook_selftest.exe` — it should print `SELFTEST PASS`. + +## Phase 1b: video mirror (Windows Graphics Capture) + +The **Video mirror** panel captures the injected game's window (via Windows +Graphics Capture — no injection needed for video) and draws it, letterboxed, as +the host window's background. This is what Steam RPT streams to guests. + +1. Inject into a game as in Phase 1a (the hook reports the game's window, which + the mirror uses as its source). +2. In the **Video mirror** panel, tick **Mirror game window**. The host window + should now show a live copy of the game; the panel reports the capture + resolution and render FPS. +3. The game can be behind the host window — WGC still captures occluded (but not + minimized) windows, and focus spoofing keeps it rendering. + +Requires Windows 10 1903+ for WGC; hiding the capture border needs build 20348+. +This is the place to judge capture latency/stutter; if it's not good enough, the +fallback is a `Present`-hook capture path in the same injected DLL. diff --git a/host/CMakeLists.txt b/host/CMakeLists.txt index e6b4fed..5ce82db 100644 --- a/host/CMakeLists.txt +++ b/host/CMakeLists.txt @@ -4,10 +4,13 @@ add_executable(coop_host WIN32 src/imgui_layer.cpp src/debug_overlay.cpp src/injection_panel.cpp + src/capture_panel.cpp src/input/xinput_source.cpp src/inject/process_list.cpp src/inject/injector.cpp - src/ipc/ipc_server.cpp) + src/ipc/ipc_server.cpp + src/capture/frame_renderer.cpp + src/capture/window_capture.cpp) target_include_directories(coop_host PRIVATE src) @@ -17,6 +20,8 @@ target_link_libraries(coop_host PRIVATE d3d11 dxgi dwmapi + d3dcompiler + windowsapp xinput) set_target_properties(coop_host PROPERTIES OUTPUT_NAME "coop_host") diff --git a/host/src/capture/frame_renderer.cpp b/host/src/capture/frame_renderer.cpp new file mode 100644 index 0000000..6d1c069 --- /dev/null +++ b/host/src/capture/frame_renderer.cpp @@ -0,0 +1,119 @@ +#include "capture/frame_renderer.hpp" + +#include + +#include + +using Microsoft::WRL::ComPtr; + +namespace coop +{ + +namespace +{ + +// Fullscreen triangle generated from SV_VertexID -- no vertex/index buffers +// needed. Samples the source texture across the [0,1] UV range. +constexpr char kShaderSource[] = R"( +Texture2D g_tex : register(t0); +SamplerState g_smp : register(s0); + +struct VSOut { float4 pos : SV_Position; float2 uv : TEXCOORD0; }; + +VSOut vs_main(uint id : SV_VertexID) +{ + VSOut o; + o.uv = float2((id << 1) & 2, id & 2); + o.pos = float4(o.uv * float2(2, -2) + float2(-1, 1), 0, 1); + return o; +} + +float4 ps_main(VSOut i) : SV_Target +{ + return g_tex.Sample(g_smp, i.uv); +} +)"; + +ComPtr compile(const char* entry, const char* target) +{ + ComPtr blob; + ComPtr errors; + const HRESULT hr = D3DCompile(kShaderSource, sizeof(kShaderSource) - 1, "frame_renderer", nullptr, nullptr, entry, + target, D3DCOMPILE_OPTIMIZATION_LEVEL3, 0, blob.GetAddressOf(), errors.GetAddressOf()); + if (FAILED(hr)) + { + return nullptr; + } + return blob; +} + +} // namespace + +bool FrameRenderer::init(ID3D11Device* device) +{ + ComPtr vs_blob = compile("vs_main", "vs_5_0"); + ComPtr ps_blob = compile("ps_main", "ps_5_0"); + if (vs_blob == nullptr || ps_blob == nullptr) + { + return false; + } + if (FAILED(device->CreateVertexShader(vs_blob->GetBufferPointer(), vs_blob->GetBufferSize(), nullptr, + vs_.GetAddressOf()))) + { + return false; + } + if (FAILED(device->CreatePixelShader(ps_blob->GetBufferPointer(), ps_blob->GetBufferSize(), nullptr, + ps_.GetAddressOf()))) + { + return false; + } + + D3D11_SAMPLER_DESC sd = {}; + sd.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR; + sd.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; + sd.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP; + sd.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + sd.ComparisonFunc = D3D11_COMPARISON_NEVER; + if (FAILED(device->CreateSamplerState(&sd, sampler_.GetAddressOf()))) + { + return false; + } + return true; +} + +void FrameRenderer::draw(ID3D11DeviceContext* ctx, ID3D11ShaderResourceView* srv, std::uint32_t src_w, + std::uint32_t src_h, std::uint32_t dst_w, std::uint32_t dst_h) +{ + if (srv == nullptr || src_w == 0 || src_h == 0 || dst_w == 0 || dst_h == 0) + { + return; + } + + // Letterbox: fit the source rect inside the destination, preserving aspect. + const float scale = std::min(static_cast(dst_w) / src_w, static_cast(dst_h) / src_h); + const float vp_w = src_w * scale; + const float vp_h = src_h * scale; + + D3D11_VIEWPORT vp = {}; + vp.TopLeftX = (static_cast(dst_w) - vp_w) * 0.5f; + vp.TopLeftY = (static_cast(dst_h) - vp_h) * 0.5f; + vp.Width = vp_w; + vp.Height = vp_h; + vp.MinDepth = 0.0f; + vp.MaxDepth = 1.0f; + ctx->RSSetViewports(1, &vp); + + ctx->IASetInputLayout(nullptr); + ctx->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + ctx->VSSetShader(vs_.Get(), nullptr, 0); + ctx->PSSetShader(ps_.Get(), nullptr, 0); + ctx->PSSetShaderResources(0, 1, &srv); + ctx->PSSetSamplers(0, 1, sampler_.GetAddressOf()); + ctx->Draw(3, 0); + + // Unbind the SRV so the same texture can be a copy destination next frame. + ID3D11ShaderResourceView* null_srv = nullptr; + ctx->PSSetShaderResources(0, 1, &null_srv); +} + +} // namespace coop diff --git a/host/src/capture/frame_renderer.hpp b/host/src/capture/frame_renderer.hpp new file mode 100644 index 0000000..91da5da --- /dev/null +++ b/host/src/capture/frame_renderer.hpp @@ -0,0 +1,30 @@ +// Draws a captured frame texture into the current render target as a letterboxed +// full-screen image (aspect-preserved; the cleared background fills the bars). +#pragma once + +#include + +#include +#include + +namespace coop +{ + +class FrameRenderer +{ +public: + bool init(ID3D11Device* device); + + // Draws `srv` (a srcW x srcH image) centered and scaled to fit within a + // dstW x dstH target. Sets its own viewport; callers that draw afterwards + // (e.g. ImGui) should restore theirs. + void draw(ID3D11DeviceContext* ctx, ID3D11ShaderResourceView* srv, std::uint32_t src_w, std::uint32_t src_h, + std::uint32_t dst_w, std::uint32_t dst_h); + +private: + Microsoft::WRL::ComPtr vs_; + Microsoft::WRL::ComPtr ps_; + Microsoft::WRL::ComPtr sampler_; +}; + +} // namespace coop diff --git a/host/src/capture/window_capture.cpp b/host/src/capture/window_capture.cpp new file mode 100644 index 0000000..c5ff203 --- /dev/null +++ b/host/src/capture/window_capture.cpp @@ -0,0 +1,218 @@ +#include "capture/window_capture.hpp" + +#include "capture/frame_renderer.hpp" + +// Interop headers bridge classic D3D11/HWND types and the WinRT capture API. +#include +#include + +// Foundation must be included for IClosable::Close() definitions used below. +#include +#include + +using Microsoft::WRL::ComPtr; + +namespace winrt +{ +using namespace Windows::Graphics; +using namespace Windows::Graphics::Capture; +using namespace Windows::Graphics::DirectX; +using namespace Windows::Graphics::DirectX::Direct3D11; +} // namespace winrt + +namespace coop +{ + +namespace +{ + +constexpr auto kPixelFormat = winrt::DirectXPixelFormat::B8G8R8A8UIntNormalized; + +// Pulls the underlying ID3D11Texture2D out of a captured frame's surface. +ComPtr texture_from_surface(winrt::IDirect3DSurface const& surface) +{ + auto access = surface.as<::Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>(); + ComPtr texture; + if (access) + { + access->GetInterface(__uuidof(ID3D11Texture2D), reinterpret_cast(texture.GetAddressOf())); + } + return texture; +} + +} // namespace + +WindowCapture::~WindowCapture() +{ + stop(); +} + +bool WindowCapture::start(HWND target, ID3D11Device* device) +{ + stop(); + if (target == nullptr || device == nullptr || !IsWindow(target)) + { + return false; + } + + try + { + device_ = device; + + // Wrap our D3D11 device as the WinRT device the frame pool renders on. + ComPtr dxgi_device; + if (FAILED(device->QueryInterface(IID_PPV_ARGS(dxgi_device.GetAddressOf())))) + { + return false; + } + winrt::com_ptr<::IInspectable> inspectable; + if (FAILED(CreateDirect3D11DeviceFromDXGIDevice(dxgi_device.Get(), inspectable.put()))) + { + return false; + } + winrt_device_ = inspectable.as(); + + // Create a capture item for the target window via the interop factory. + auto interop = winrt::get_activation_factory(); + if (FAILED(interop->CreateForWindow(target, winrt::guid_of(), + winrt::put_abi(item_)))) + { + return false; + } + + pool_size_ = item_.Size(); + frame_pool_ = + winrt::Direct3D11CaptureFramePool::CreateFreeThreaded(winrt_device_, kPixelFormat, 2, pool_size_); + session_ = frame_pool_.CreateCaptureSession(item_); + frame_token_ = frame_pool_.FrameArrived({this, &WindowCapture::on_frame_arrived}); + + // Best-effort: hide the cursor and the yellow capture border (the border + // API requires a recent Windows build, hence the guard). + try + { + session_.IsCursorCaptureEnabled(false); + } + catch (...) + { + } + try + { + session_.IsBorderRequired(false); + } + catch (...) + { + } + + session_.StartCapture(); + target_ = target; + return true; + } + catch (...) + { + stop(); + return false; + } +} + +void WindowCapture::stop() +{ + if (frame_pool_ != nullptr && frame_token_) + { + frame_pool_.FrameArrived(frame_token_); + frame_token_ = {}; + } + if (session_ != nullptr) + { + session_.Close(); + session_ = nullptr; + } + if (frame_pool_ != nullptr) + { + frame_pool_.Close(); + frame_pool_ = nullptr; + } + { + std::lock_guard lock(mutex_); + pending_ = nullptr; + } + item_ = nullptr; + winrt_device_ = nullptr; + latest_srv_.Reset(); + latest_.Reset(); + device_.Reset(); + target_ = nullptr; + width_ = 0; + height_ = 0; +} + +void WindowCapture::on_frame_arrived(winrt::Direct3D11CaptureFramePool const& pool, + winrt::Windows::Foundation::IInspectable const&) +{ + // Just grab the newest frame and stash it; the render thread does the copy. + auto frame = pool.TryGetNextFrame(); + std::lock_guard lock(mutex_); + if (pending_ != nullptr) + { + pending_.Close(); // drop the un-consumed previous frame back to the pool + } + pending_ = frame; +} + +void WindowCapture::draw_latest(FrameRenderer& renderer, ID3D11DeviceContext* ctx, std::uint32_t dst_w, + std::uint32_t dst_h) +{ + winrt::Direct3D11CaptureFrame frame{nullptr}; + { + std::lock_guard lock(mutex_); + frame = pending_; + pending_ = nullptr; + } + + if (frame != nullptr) + { + if (ComPtr src = texture_from_surface(frame.Surface())) + { + D3D11_TEXTURE2D_DESC desc = {}; + src->GetDesc(&desc); + + if (latest_ == nullptr || desc.Width != width_ || desc.Height != height_) + { + latest_srv_.Reset(); + latest_.Reset(); + + D3D11_TEXTURE2D_DESC dst = desc; + dst.Usage = D3D11_USAGE_DEFAULT; + dst.BindFlags = D3D11_BIND_SHADER_RESOURCE; + dst.CPUAccessFlags = 0; + dst.MiscFlags = 0; + if (SUCCEEDED(device_->CreateTexture2D(&dst, nullptr, latest_.GetAddressOf()))) + { + device_->CreateShaderResourceView(latest_.Get(), nullptr, latest_srv_.GetAddressOf()); + width_ = desc.Width; + height_ = desc.Height; + } + } + + if (latest_ != nullptr) + { + ctx->CopyResource(latest_.Get(), src.Get()); + } + } + frame.Close(); + + // If the window resized, the capture item changes size; re-fit the pool. + const winrt::SizeInt32 size = item_.Size(); + if (size.Width != pool_size_.Width || size.Height != pool_size_.Height) + { + pool_size_ = size; + frame_pool_.Recreate(winrt_device_, kPixelFormat, 2, size); + } + } + + if (latest_srv_ != nullptr) + { + renderer.draw(ctx, latest_srv_.Get(), width_, height_, dst_w, dst_h); + } +} + +} // namespace coop diff --git a/host/src/capture/window_capture.hpp b/host/src/capture/window_capture.hpp new file mode 100644 index 0000000..4424843 --- /dev/null +++ b/host/src/capture/window_capture.hpp @@ -0,0 +1,75 @@ +// Mirrors a target window into a D3D11 texture using Windows.Graphics.Capture +// (no injection). Frames arrive on a WGC thread; the newest is handed to the +// render thread, which copies and draws it -- so all D3D11 context use stays on +// one thread. +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +namespace coop +{ + +class FrameRenderer; + +class WindowCapture +{ +public: + ~WindowCapture(); + + // Begins capturing `target`. Returns false if WGC is unavailable or the + // window can't be captured. + bool start(HWND target, ID3D11Device* device); + void stop(); + + [[nodiscard]] bool running() const + { + return session_ != nullptr; + } + [[nodiscard]] HWND target() const + { + return target_; + } + [[nodiscard]] std::uint32_t frame_width() const + { + return width_; + } + [[nodiscard]] std::uint32_t frame_height() const + { + return height_; + } + + // Render thread: consume the newest frame (if any) and draw it letterboxed + // into a dst_w x dst_h target via `renderer`. + void draw_latest(FrameRenderer& renderer, ID3D11DeviceContext* ctx, std::uint32_t dst_w, std::uint32_t dst_h); + +private: + void on_frame_arrived(winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool const& pool, + winrt::Windows::Foundation::IInspectable const&); + + HWND target_ = nullptr; + Microsoft::WRL::ComPtr device_; + Microsoft::WRL::ComPtr latest_; + Microsoft::WRL::ComPtr latest_srv_; + std::uint32_t width_ = 0; + std::uint32_t height_ = 0; + + std::mutex mutex_; // guards pending_ handoff only (no context work under it) + winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame pending_{nullptr}; + + winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice winrt_device_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureItem item_{nullptr}; + winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool frame_pool_{nullptr}; + winrt::Windows::Graphics::Capture::GraphicsCaptureSession session_{nullptr}; + winrt::event_token frame_token_{}; + winrt::Windows::Graphics::SizeInt32 pool_size_{0, 0}; +}; + +} // namespace coop diff --git a/host/src/capture_panel.cpp b/host/src/capture_panel.cpp new file mode 100644 index 0000000..b2480ef --- /dev/null +++ b/host/src/capture_panel.cpp @@ -0,0 +1,63 @@ +#include "capture_panel.hpp" + +#include "imgui.h" + +namespace coop +{ + +bool CapturePanel::init(ID3D11Device* device) +{ + device_ = device; + return renderer_.init(device); +} + +void CapturePanel::draw_ui() +{ + ImGui::SetNextWindowPos(ImVec2(460, 24), ImGuiCond_FirstUseEver); + ImGui::SetNextWindowSize(ImVec2(360, 0), ImGuiCond_FirstUseEver); + ImGui::Begin("Video mirror"); + + const bool have_target = target_ != nullptr && IsWindow(target_); + ImGui::BeginDisabled(!have_target); + if (ImGui::Checkbox("Mirror game window", &enabled_)) + { + if (!enabled_) + { + capture_.stop(); + } + } + ImGui::EndDisabled(); + if (!have_target) + { + ImGui::TextDisabled("Inject into a game first (its window is the source)."); + } + + // Start/restart capture when enabled and the target window changes. + if (enabled_ && have_target && capture_.target() != target_) + { + if (!capture_.start(target_, device_)) + { + enabled_ = false; + ImGui::TextColored(ImVec4(1.0f, 0.45f, 0.4f, 1.0f), "Failed to start capture."); + } + } + + if (capture_.running()) + { + ImGui::TextColored(ImVec4(0.4f, 1.0f, 0.4f, 1.0f), "Capturing %ux%u", capture_.frame_width(), + capture_.frame_height()); + ImGui::Text("Render: %.1f FPS", ImGui::GetIO().Framerate); + } + + ImGui::End(); +} + +void CapturePanel::render(ID3D11DeviceContext* ctx, std::uint32_t dst_w, std::uint32_t dst_h) +{ + if (capture_.running()) + { + capture_.draw_latest(renderer_, ctx, dst_w, dst_h); + } +} + +} // namespace coop diff --git a/host/src/capture_panel.hpp b/host/src/capture_panel.hpp new file mode 100644 index 0000000..4d855eb --- /dev/null +++ b/host/src/capture_panel.hpp @@ -0,0 +1,40 @@ +// Owns the video-mirror pipeline (WGC capture + frame renderer) and its UI. +// The target window comes from the injected hook's reported game HWND. +#pragma once + +#include + +#include +#include + +#include "capture/frame_renderer.hpp" +#include "capture/window_capture.hpp" + +namespace coop +{ + +class CapturePanel +{ +public: + bool init(ID3D11Device* device); + + // The window to mirror (0 if none yet); typically the injected game's HWND. + void set_target(HWND target) + { + target_ = target; + } + + void draw_ui(); + + // Render thread: draw the mirrored frame as the window background. + void render(ID3D11DeviceContext* ctx, std::uint32_t dst_w, std::uint32_t dst_h); + +private: + ID3D11Device* device_ = nullptr; + FrameRenderer renderer_; + WindowCapture capture_; + HWND target_ = nullptr; + bool enabled_ = false; +}; + +} // namespace coop diff --git a/host/src/injection_panel.hpp b/host/src/injection_panel.hpp index cae5d13..5c4ad51 100644 --- a/host/src/injection_panel.hpp +++ b/host/src/injection_panel.hpp @@ -6,6 +6,8 @@ #include #include +#include + #include "coop/protocol.hpp" #include "imgui.h" #include "inject/process_list.hpp" @@ -25,6 +27,12 @@ public: // test-input mode is on, a synthetic pattern is sent instead of `pads`. void publish(const std::array& pads); + // The injected game's main window, as reported by the hook (null if none). + [[nodiscard]] HWND game_hwnd() const + { + return reinterpret_cast(server_.hook_status().game_hwnd); + } + private: void refresh_processes(); void inject_selected(); diff --git a/host/src/main.cpp b/host/src/main.cpp index 720a5b9..89aa36f 100644 --- a/host/src/main.cpp +++ b/host/src/main.cpp @@ -1,16 +1,17 @@ -// CoopAllTheThings host -- Phase 0 spike. +// CoopAllTheThings host. // -// Brings up the borderless window Steam Remote Play Together will capture and an -// ImGui overlay that lists every controller it can see. The goal of this build -// is to validate the riskiest assumption end-to-end: launch this exe under a -// donor appid (steam -applaunch ), invite a -// friend, and confirm (a) the window streams and (b) the guest's gamepad shows -// up in the overlay. Capture/injection are built on top of this once it holds. +// Borderless window that Steam Remote Play Together captures, plus the overlay +// that drives the tool: it forwards controller input into an injected game +// (Injection panel), spoofs the game's focus, and mirrors the game's window into +// this window via Windows Graphics Capture (Video mirror panel). #include #include +#include + +#include "capture_panel.hpp" #include "d3d11_window.hpp" #include "debug_overlay.hpp" #include "imgui_layer.hpp" @@ -38,17 +39,34 @@ int run() auto input = std::make_unique(); coop::InjectionPanel injection; + coop::CapturePanel capture; + if (!capture.init(window.device())) + { + MessageBoxW(nullptr, L"Failed to initialize the video mirror.", L"CoopAllTheThings", MB_ICONERROR); + return 1; + } while (window.pump_messages()) { input->poll(); injection.publish(input->pads()); + capture.set_target(injection.game_hwnd()); imgui.begin_frame(); coop::draw_debug_overlay(*input); injection.draw(); + capture.draw_ui(); - window.render_frame([&imgui]() { imgui.end_frame(); }); + RECT client = {}; + GetClientRect(window.hwnd(), &client); + const auto dst_w = static_cast(client.right - client.left); + const auto dst_h = static_cast(client.bottom - client.top); + + // Mirrored frame first (the window background), ImGui overlay on top. + window.render_frame([&]() { + capture.render(window.context(), dst_w, dst_h); + imgui.end_frame(); + }); } return 0; @@ -58,5 +76,10 @@ int run() int WINAPI wWinMain(HINSTANCE, HINSTANCE, LPWSTR, int) { - return run(); + // WGC requires an initialized apartment; multi-threaded suits the + // free-threaded frame pool. + winrt::init_apartment(winrt::apartment_type::multi_threaded); + const int result = run(); + winrt::uninit_apartment(); + return result; }