D3D12Hook: Fix swapchain creation causing crashes

src/D3D12Hook.cpp

@@ -57,6 +57,35 @@ void* D3D12Hook::Streamline::link_swapchain_to_cmd_queue(void* rcx, void* rdx, v
     return result;
 }
 
+HRESULT WINAPI D3D12Hook::create_swapchain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain) {
+    auto create_swap_chain_fn = s_create_swapchain_hook->get_original<decltype(D3D12Hook::create_swapchain)*>();
+
+    if (g_inside_d3d12_hook) {
+        spdlog::info("create_swapchain (inside D3D12 hook)");
+        return create_swap_chain_fn(factory, device, hwnd, desc, p_fullscreen_desc, p_restrict_to_output, swap_chain);
+    }
+
+    spdlog::info("create_swapchain called");
+
+    std::scoped_lock _{g_framework->get_hook_monitor_mutex()};
+
+    bool hook_was_nullptr = g_d3d12_hook == nullptr;
+
+    if (g_d3d12_hook != nullptr && g_framework->get_d3d12_hook() != nullptr) {
+        g_framework->on_reset(); // Needed to prevent a crash due to resources hanging around
+        g_d3d12_hook->unhook(); // Removes all vtable hooks
+    }
+
+    const auto result = create_swap_chain_fn(factory, device, hwnd, desc, p_fullscreen_desc, p_restrict_to_output, swap_chain);
+
+    // rather than waiting on the hook monitor to notice the hook isn't working
+    if (!hook_was_nullptr) {
+        g_framework->hook_d3d12();
+    }
+
+    return result;
+}
+
 void D3D12Hook::hook_streamline(HMODULE dlssg_module) try {
     if (D3D12Hook::s_streamline.setup) {
         return;
@@ -346,6 +375,8 @@ bool D3D12Hook::hook() {
 
     spdlog::info("Finding command queue offset");
 
+    m_command_queue_offset = 0;
+
     // Find the command queue offset in the swapchain
     for (auto i = 0; i < 512 * sizeof(void*); i += sizeof(void*)) {
         const auto base = (uintptr_t)swap_chain1 + i;
@@ -442,6 +473,12 @@ bool D3D12Hook::hook() {
 
         auto& present_fn = (*(void***)target_swapchain)[8]; // Present
         m_present_hook = std::make_unique<PointerHook>(&present_fn, &D3D12Hook::present);
+
+        if (s_create_swapchain_hook == nullptr) {
+            auto& create_swapchain_fn = (*(void***)factory)[15]; // CreateSwapChainForHwnd
+            s_create_swapchain_hook = std::make_unique<PointerHook>(&create_swapchain_fn, &D3D12Hook::create_swapchain);
+        }
+
         m_hooked = true;
     } catch (const std::exception& e) {
         spdlog::error("Failed to initialize hooks: {}", e.what());
@@ -786,21 +823,3 @@ HRESULT WINAPI D3D12Hook::resize_target(IDXGISwapChain3* swap_chain, const DXGI_
 
     return result;
 }
-
-/*HRESULT WINAPI D3D12Hook::create_swap_chain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain)
-{
-    spdlog::info("D3D12 create swapchain called");
-
-    auto d3d12 = g_d3d12_hook;
-
-    d3d12->m_command_queue = (ID3D12CommandQueue*)device;
-    
-    if (d3d12->m_on_create_swap_chain) {
-        d3d12->m_on_create_swap_chain(*d3d12);
-    }
-
-    auto create_swap_chain_fn = d3d12->m_create_swap_chain_hook->get_original<decltype(D3D12Hook::create_swap_chain)>();
-
-    return create_swap_chain_fn(factory, device, hwnd, desc, p_fullscreen_desc, p_restrict_to_output, swap_chain);
-}*/
-

src/D3D12Hook.hpp

@@ -122,9 +122,7 @@ class D3D12Hook
     bool m_ignore_next_present{false};
 
     std::unique_ptr<PointerHook> m_present_hook{};
-    //std::unique_ptr<PointerHook> m_release_hook{};
     std::unique_ptr<VtableHook> m_swapchain_hook{};
-    //std::unique_ptr<FunctionHook> m_create_swap_chain_hook{};
 
     struct Streamline {
         static void* link_swapchain_to_cmd_queue(void* rcx, void* rdx, void* r8, void* r9);
@@ -135,6 +133,9 @@ class D3D12Hook
     };
 
     static inline Streamline s_streamline{};
+
+    // This is static because unhooking it seems to cause a crash sometimes
+    static inline std::unique_ptr<PointerHook> s_create_swapchain_hook{};
 
     OnPresentFn m_on_present{ nullptr };
     OnPresentFn m_on_post_present{ nullptr };
@@ -145,6 +146,6 @@ class D3D12Hook
     static HRESULT WINAPI present(IDXGISwapChain3* swap_chain, uint64_t sync_interval, uint64_t flags, void* r9);
     static HRESULT WINAPI resize_buffers(IDXGISwapChain3* swap_chain, UINT buffer_count, UINT width, UINT height, DXGI_FORMAT new_format, UINT swap_chain_flags);
     static HRESULT WINAPI resize_target(IDXGISwapChain3* swap_chain, const DXGI_MODE_DESC* new_target_parameters);
-    //static HRESULT WINAPI create_swap_chain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain);
+    static HRESULT WINAPI create_swapchain(IDXGIFactory4* factory, IUnknown* device, HWND hwnd, const DXGI_SWAP_CHAIN_DESC* desc, const DXGI_SWAP_CHAIN_FULLSCREEN_DESC* p_fullscreen_desc, IDXGIOutput* p_restrict_to_output, IDXGISwapChain** swap_chain);
 };