feat: move rendering to an imgui window

destrum/include/destrum/App.h

@@ -51,6 +51,8 @@ public:
     virtual void customFixedUpdate(float dt) = 0;
 
     virtual void onWindowResize(int, int) {};
+    virtual void onViewportResize(int, int) {}
+    virtual void drawViewportOverlay() {}
 
     virtual void onPlayModeChanged(bool) {}
 
@@ -61,6 +63,15 @@ public:
 
     virtual void customWindowsMenu() {}
 
+    struct ViewportRect {
+        glm::vec2 min{0.0f};
+        glm::vec2 size{0.0f};
+    };
+
+    [[nodiscard]] bool usesEditorViewport() const { return m_params.showDebugUi; }
+    [[nodiscard]] const ViewportRect& getViewportRect() const { return viewportRect; }
+    void claimViewportMouse() { viewportMouseClaimed = true; }
+
 protected:
     Window window;
     AppParams m_params{};
@@ -105,6 +116,16 @@ protected:
     void drawPhysicsPanel(float dt, float fixedDt, float accumulator);
 
 private:
+    void drawDockSpace();
+    void drawViewportWindow();
+    void applyViewportResize();
+
+    ViewportRect viewportRect{};
+    glm::ivec2 viewportDesiredSize{0};
+    bool viewportHovered{false};
+    bool viewportFocused{false};
+    bool viewportMouseClaimed{false};
+
     void drawPlayControls();
     void applyPlayModeToggle();
     bool enterPlayMode();

destrum/include/destrum/Graphics/GfxDevice.h

@@ -52,6 +52,7 @@ public:
         VkClearColorValue clearColor{{0.f, 0.f, 0.f, 1.f}};
         glm::ivec4 drawImageBlitRect{};
         ImguiPass* imguiPass{nullptr};
+        bool copyDrawImageToSwapchain{true};
     };
 
 public:

destrum/include/destrum/Util/RectGizmo.h

@@ -9,16 +9,17 @@ class RectGizmo final {
 public:
     [[nodiscard]] static bool CanManipulate(GameObject& object);
 
-    // drawImageSize is what the canvas renders into, it is stretched over the main ImGui viewport
+    bool Manipulate(GameObject& object, const glm::vec2& drawImageSize, const glm::vec2& viewMin, const glm::vec2& viewSize);
-    bool Manipulate(GameObject& object, const glm::vec2& drawImageSize);
 
     [[nodiscard]] bool IsUsing() const { return m_ActiveHandle >= 0; }
+    [[nodiscard]] bool IsInteracting() const { return m_ActiveHandle >= 0 || m_Hovered; }
 
     bool snap{false};
     float snapStep{5.0f};
 
 private:
     int m_ActiveHandle{-1};
+    bool m_Hovered{false};
     glm::vec2 m_DragStartMouse{0.0f};
     glm::vec2 m_DragStartPosition{0.0f};
     glm::vec2 m_DragStartSize{0.0f};

destrum/include/destrum/Util/TransformGizmo.h

@@ -1,6 +1,8 @@
 #ifndef DESTRUM_TRANSFORMGIZMO_H
 #define DESTRUM_TRANSFORMGIZMO_H
 
+#include <glm/glm.hpp>
+
 class Camera;
 class GameObject;
 
@@ -19,7 +21,7 @@ class TransformGizmo final {
 public:
     void HandleShortcuts();
     void DrawToolbar();
-    bool Manipulate(GameObject& object, const Camera& camera);
+    bool Manipulate(GameObject& object, const Camera& camera, const glm::vec2& viewMin, const glm::vec2& viewSize);
 
     [[nodiscard]] bool IsUsing() const;
     [[nodiscard]] bool IsOver() const;

destrum/src/App.cpp

@@ -14,6 +14,7 @@
 #include <destrum/Util/DeltaTime.h>
 
 #include "imgui.h"
+#include "imgui_internal.h"
 #include "glm/gtx/transform.hpp"
 #include "spdlog/spdlog.h"
 
@@ -158,9 +159,15 @@ void App::run()
                     event.type == SDL_KEYUP ||
                     event.type == SDL_TEXTINPUT;
 
+                // ImGui wants every event while a window is hovered or focused, the editor viewport
+                // is one, so the game gets them there instead
+                const bool mouseCaptured = InputManager::GetInstance().IsMouseCaptured();
+                const bool viewportGetsMouse = usesEditorViewport() && viewportHovered && !viewportMouseClaimed;
+                const bool viewportGetsKeyboard = usesEditorViewport() && (viewportFocused || mouseCaptured) &&
+                                                  !ImGui::GetIO().WantTextInput;
                 const bool capturedByImgui =
-                    (mouseEvent && !InputManager::GetInstance().IsMouseCaptured() && imguiPass.wantsMouse()) ||
+                    (mouseEvent && !mouseCaptured && !viewportGetsMouse && imguiPass.wantsMouse()) ||
-                    (keyboardEvent && imguiPass.wantsKeyboard());
+                    (keyboardEvent && !viewportGetsKeyboard && imguiPass.wantsKeyboard());
 
                 if (!capturedByImgui)
                 {
@@ -184,24 +191,39 @@ if (!isRunning) break;
             continue;
         }
 
-        if (InputManager::GetInstance().WasKeyPressed(SDL_SCANCODE_F5))
-        {
-            playModeToggleRequested = true;
-        }
-        if (playing && InputManager::GetInstance().WasKeyPressed(SDL_SCANCODE_F7))
-        {
-            setPaused(!paused);
-        }
-
         // Consume SDL events before updating the base camera so held and
         // newly pressed inputs are applied in the same frame.
         camera.Update(dt);
 
+        if (usesEditorViewport())
+        {
+            applyViewportResize();
+        }
+
         {
             ZoneScopedN("ImGui BeginFrame");
             imguiPass.beginFrame();
         }
 
+        {
+            // Global shortcuts, also when an editor window has keyboard focus
+            const auto& input = InputManager::GetInstance();
+            const bool typing = ImGui::GetIO().WantTextInput;
+            if (input.WasKeyPressed(SDL_SCANCODE_F5) || (!typing && ImGui::IsKeyPressed(ImGuiKey_F5, false)))
+            {
+                playModeToggleRequested = true;
+            }
+            if (playing && (input.WasKeyPressed(SDL_SCANCODE_F7) || (!typing && ImGui::IsKeyPressed(ImGuiKey_F7, false))))
+            {
+                setPaused(!paused);
+            }
+        }
+
+        if (usesEditorViewport())
+        {
+            drawDockSpace();
+        }
+
         {
             ZoneScopedN("customUpdate");
             customUpdate(dt);
@@ -211,6 +233,7 @@ if (!isRunning) break;
         {
             ZoneScopedN("Debug ImGui");
             drawWindowsMenu();
+            drawViewportWindow();
 
             if (m_DebugWindows.inspector)
             {
@@ -336,6 +359,11 @@ if (!isRunning) break;
                     }
                     spdlog::info("Recreated swapchain at {}x{}", w, h);
                     imguiPass.onSwapchainRecreated();
+                    if (!usesEditorViewport())
+                    {
+                        renderer.resize(gfxDevice, drawableSize);
+                        onViewportResize(w, h);
+                    }
                     onWindowResize(w, h);
                 }
 
@@ -433,6 +461,106 @@ void App::drawWindowsMenu()
     ImGui::EndMainMenuBar();
 }
 
+void App::drawDockSpace()
+{
+    const ImGuiID dockspaceId = ImGui::GetID("EditorDockSpace");
+    const bool hasSavedLayout = ImGui::DockBuilderGetNode(dockspaceId) != nullptr;
+    const ImGuiViewport* mainViewport = ImGui::GetMainViewport();
+    ImGui::DockSpaceOverViewport(dockspaceId, mainViewport);
+    if (hasSavedLayout)
+    {
+        return;
+    }
+
+    // First run without an imgui.ini: scene in the middle, inspector right, tools left
+    ImGui::DockBuilderRemoveNode(dockspaceId);
+    ImGui::DockBuilderAddNode(dockspaceId, ImGuiDockNodeFlags_DockSpace);
+    ImGui::DockBuilderSetNodeSize(dockspaceId, mainViewport->WorkSize);
+
+    ImGuiID center = dockspaceId;
+    const ImGuiID right = ImGui::DockBuilderSplitNode(center, ImGuiDir_Right, 0.25f, nullptr, &center);
+    const ImGuiID left = ImGui::DockBuilderSplitNode(center, ImGuiDir_Left, 0.22f, nullptr, &center);
+
+    ImGui::DockBuilderDockWindow("Viewport", center);
+    ImGui::DockBuilderDockWindow("Inspector", right);
+    ImGui::DockBuilderDockWindow("Debug", left);
+    ImGui::DockBuilderDockWindow("Physics", left);
+    ImGui::DockBuilderFinish(dockspaceId);
+}
+
+void App::drawViewportWindow()
+{
+    viewportMouseClaimed = false;
+
+    ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2{0.0f, 0.0f});
+    const bool visible = ImGui::Begin("Viewport", nullptr, ImGuiWindowFlags_NoScrollbar | ImGuiWindowFlags_NoScrollWithMouse);
+    ImGui::PopStyleVar();
+
+    if (!visible)
+    {
+        viewportHovered = false;
+        viewportFocused = false;
+        ImGui::End();
+        return;
+    }
+
+    const ImVec2 origin = ImGui::GetCursorScreenPos();
+    const ImVec2 available = ImGui::GetContentRegionAvail();
+    const ImVec2 framebufferScale = ImGui::GetIO().DisplayFramebufferScale;
+    viewportRect = {
+        .min = {origin.x, origin.y},
+        .size = {std::max(available.x, 1.0f), std::max(available.y, 1.0f)},
+    };
+    viewportDesiredSize = {
+        std::max(static_cast<int>(available.x * framebufferScale.x), 1),
+        std::max(static_cast<int>(available.y * framebufferScale.y), 1),
+    };
+
+    const GPUImage& drawImage = renderer.getDrawImage();
+    if (const ImTextureID texture = imguiPass.getFrameTexture(drawImage.imageView); texture != ImTextureID{})
+    {
+        ImGui::Image(texture, ImVec2{viewportRect.size.x, viewportRect.size.y});
+    }
+
+    viewportHovered = ImGui::IsWindowHovered();
+    // ImGui only focuses windows on a left click, the free camera looks around with the right button
+    if (viewportHovered && (ImGui::IsMouseClicked(ImGuiMouseButton_Right) || ImGui::IsMouseClicked(ImGuiMouseButton_Middle)))
+    {
+        ImGui::SetWindowFocus();
+    }
+    viewportFocused = ImGui::IsWindowFocused();
+
+    ImGui::SetCursorScreenPos(origin);
+    drawViewportOverlay();
+
+    ImGui::End();
+}
+
+void App::applyViewportResize()
+{
+    if (viewportDesiredSize.x <= 0 || viewportDesiredSize.y <= 0)
+    {
+        return;
+    }
+
+    const VkExtent3D current = renderer.getDrawImage().extent;
+    if (static_cast<int>(current.width) == viewportDesiredSize.x &&
+        static_cast<int>(current.height) == viewportDesiredSize.y)
+    {
+        return;
+    }
+
+    // While a dock splitter is dragged the old image is stretched, recreating it every frame is wasteful
+    if (ImGui::GetIO().MouseDown[ImGuiMouseButton_Left])
+    {
+        return;
+    }
+
+    ZoneScopedN("Resize Viewport");
+    renderer.resize(gfxDevice, viewportDesiredSize);
+    onViewportResize(viewportDesiredSize.x, viewportDesiredSize.y);
+}
+
 void App::drawPlayControls()
 {
     const char* state = !playing ? "EDIT" : paused ? "PAUSED" : "PLAYING";

destrum/src/Graphics/GfxDevice.cpp

@@ -178,7 +178,17 @@ void GfxDevice::endFrame(VkCommandBuffer cmd, const GPUImage& drawImage, const E
         vkCmdClearColorImage(cmd, swapchainImage, VK_IMAGE_LAYOUT_GENERAL, &clearValue, 1, &clearRange);
     }
 
-    if (true) {
+    if (!props.copyDrawImageToSwapchain) {
+        ZoneScopedN("Transition DrawImage For ImGui");
+        vkutil::ScopedCmdLabel label(cmd, "Transition DrawImage For ImGui", vkutil::labelColors::transfer);
+
+        vkutil::transitionImage(
+            cmd,
+            drawImage.image,
+            drawImage.layout,
+            VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+        drawImage.setLayout(VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+    } else {
         ZoneScopedN("Copy DrawImage To Swapchain");
         vkutil::ScopedCmdLabel label(cmd, "Copy DrawImage To Swapchain", vkutil::labelColors::transfer);
 

destrum/src/Graphics/Pipelines/ImguiPass.cpp

@@ -28,6 +28,9 @@ void ImguiPass::init(SDL_Window* window, GfxDevice& gfxDevice) {
     ImGuiIO& io = ImGui::GetIO();
     io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard;
     io.ConfigFlags |= ImGuiConfigFlags_NavEnableGamepad;
+    io.ConfigFlags |= ImGuiConfigFlags_DockingEnable;
+    // Dragging inside the editor viewport is for the scene, not for moving the window
+    io.ConfigWindowsMoveFromTitleBarOnly = true;
 
     ImGui::StyleColorsDark();
 

destrum/src/Util/RectGizmo.cpp

@@ -56,8 +56,9 @@ bool RectGizmo::CanManipulate(GameObject& object)
            Canvas::FindFor(object) != nullptr;
 }
 
-bool RectGizmo::Manipulate(GameObject& object, const glm::vec2& drawImageSize)
+bool RectGizmo::Manipulate(GameObject& object, const glm::vec2& drawImageSize, const glm::vec2& viewMin, const glm::vec2& viewSize)
 {
+    m_Hovered = false;
     auto* rectTransform = object.GetComponent<RectTransform>();
     const Canvas* canvas = Canvas::FindFor(object);
     if (!CanManipulate(object) || drawImageSize.x <= 0.0f || drawImageSize.y <= 0.0f) {
@@ -65,20 +66,19 @@ bool RectGizmo::Manipulate(GameObject& object, const glm::vec2& drawImageSize)
         return false;
     }
 
-    const ImGuiViewport* viewport = ImGui::GetMainViewport();
+    const glm::vec2 viewScale = viewSize / drawImageSize;
-    const glm::vec2 viewOrigin = ToVec(viewport->Pos);
+    const auto toView = [&](const glm::vec2& point) { return viewMin + point * viewScale; };
-    const glm::vec2 viewScale = ToVec(viewport->Size) / drawImageSize;
-    const auto toView = [&](const glm::vec2& point) { return viewOrigin + point * viewScale; };
 
     const UIRect& rect = rectTransform->GetScreenRect();
-    const glm::vec2 viewMin = toView(rect.min);
+    const glm::vec2 rectMin = toView(rect.min);
-    const glm::vec2 viewMax = toView(rect.Max());
+    const glm::vec2 rectMax = toView(rect.Max());
-    const glm::vec2 viewCenter = (viewMin + viewMax) * 0.5f;
+    const glm::vec2 viewCenter = (rectMin + rectMax) * 0.5f;
-    const glm::vec2 viewHalfSize = (viewMax - viewMin) * 0.5f;
+    const glm::vec2 viewHalfSize = (rectMax - rectMin) * 0.5f;
 
     const glm::vec2 mouse = ToVec(ImGui::GetIO().MousePos);
     int hovered = -1;
-    const bool overImGui = ImGui::IsWindowHovered(ImGuiHoveredFlags_AnyWindow) || ImGui::IsAnyItemActive();
+    // Called inside the window that shows the scene, anything on top of it blocks the handles
+    const bool overImGui = !ImGui::IsWindowHovered() || ImGui::IsAnyItemActive();
     if (m_ActiveHandle < 0 && !overImGui) {
         for (std::size_t i = 0; i < HandleDirections.size(); ++i) {
             const glm::vec2 handle = viewCenter + HandleDirections[i] * viewHalfSize;
@@ -87,7 +87,7 @@ bool RectGizmo::Manipulate(GameObject& object, const glm::vec2& drawImageSize)
                 break;
             }
         }
-        if (hovered < 0 && mouse.x >= viewMin.x && mouse.y >= viewMin.y && mouse.x <= viewMax.x && mouse.y <= viewMax.y) {
+        if (hovered < 0 && mouse.x >= rectMin.x && mouse.y >= rectMin.y && mouse.x <= rectMax.x && mouse.y <= rectMax.y) {
             hovered = MoveHandle;
         }
         if (hovered >= 0 && ImGui::IsMouseClicked(ImGuiMouseButton_Left)) {
@@ -135,13 +135,12 @@ bool RectGizmo::Manipulate(GameObject& object, const glm::vec2& drawImageSize)
     }
 
     const int highlighted = m_ActiveHandle >= 0 ? m_ActiveHandle : hovered;
+    m_Hovered = highlighted >= 0;
     if (highlighted >= 0) {
         ImGui::SetMouseCursor(CursorFor(highlighted));
-        // Keeps the click from reaching the game, same as the 3D gizmo
-        ImGui::SetNextFrameWantCaptureMouse(true);
     }
 
-    ImDrawList& drawList = *ImGui::GetBackgroundDrawList();
+    ImDrawList& drawList = *ImGui::GetWindowDrawList();
     const ImU32 outlineColor = object.IsActiveInHierarchy() ? OutlineColor : HiddenColor;
 
     const UIRect& parentRect = rectTransform->GetParentRect();
@@ -152,9 +151,9 @@ bool RectGizmo::Manipulate(GameObject& object, const glm::vec2& drawImageSize)
     DrawAnchor(drawList, {anchorMax.x, anchorMax.y}, {1.0f, 1.0f});
     DrawAnchor(drawList, {anchorMin.x, anchorMax.y}, {-1.0f, 1.0f});
 
-    drawList.AddRect(ToImVec(viewMin), ToImVec(viewMax), highlighted == MoveHandle ? ActiveColor : outlineColor, 0.0f, 0, 1.5f);
+    drawList.AddRect(ToImVec(rectMin), ToImVec(rectMax), highlighted == MoveHandle ? ActiveColor : outlineColor, 0.0f, 0, 1.5f);
 
-    const glm::vec2 pivot = viewMin + (viewMax - viewMin) * rectTransform->GetPivot();
+    const glm::vec2 pivot = rectMin + (rectMax - rectMin) * rectTransform->GetPivot();
     drawList.AddCircle(ToImVec(pivot), 4.0f, outlineColor, 12, 1.5f);
 
     for (std::size_t i = 0; i < HandleDirections.size(); ++i) {

destrum/src/Util/SceneInspector.cpp

@@ -338,10 +338,13 @@ void SceneInspector::DrawNode(Scene& scene, GameObject& object, ObjectId& select
 
     DrawNodeDragDrop(object);
 
+    // Leaf nodes push no ID, without this every sibling shares one popup and draws it again
+    ImGui::PushID(reinterpret_cast<void*>(static_cast<std::uintptr_t>(object.GetId())));
     if (ImGui::BeginPopupContextItem("ObjectMenu")) {
         DrawObjectMenu(scene, &object, selectedObjectId);
         ImGui::EndPopup();
     }
+    ImGui::PopID();
 
     if (renaming) {
         ImGui::SameLine();

destrum/src/Util/TransformGizmo.cpp

@@ -89,13 +89,14 @@ void TransformGizmo::DrawToolbar()
     ImGui::End();
 }
 
-bool TransformGizmo::Manipulate(GameObject& object, const Camera& camera)
+bool TransformGizmo::Manipulate(GameObject& object, const Camera& camera, const glm::vec2& viewMin, const glm::vec2& viewSize)
 {
     ImGuizmo::BeginFrame();
 
-    const ImGuiViewport* viewport = ImGui::GetMainViewport();
+    // Drawn into the window that shows the scene, so it stays on top of it and hovers with it
+    ImGuizmo::SetDrawlist();
     ImGuizmo::SetOrthographic(false);
-    ImGuizmo::SetRect(viewport->Pos.x, viewport->Pos.y, viewport->Size.x, viewport->Size.y);
+    ImGuizmo::SetRect(viewMin.x, viewMin.y, viewSize.x, viewSize.y);
 
     const glm::mat4 view = camera.GetViewMatrix();
     glm::mat4 projection = camera.GetProjectionMatrix();
@@ -121,11 +122,6 @@ bool TransformGizmo::Manipulate(GameObject& object, const Camera& camera)
         nullptr,
         snap ? snapValues : nullptr);
 
-    // Stops the click on a handle from reaching the game, e.g. grabbing the mouse for first person
-    if (ImGuizmo::IsOver() || ImGuizmo::IsUsing()) {
-        ImGui::SetNextFrameWantCaptureMouse(true);
-    }
-
     if (!changed) {
         return false;
     }

lightkeeper/include/Lightkeeper.h

@@ -27,7 +27,8 @@ public:
     void customCleanup() override;
     void customFixedUpdate(float dt) override;
 
-    void onWindowResize(int newWidth, int newHeight) override;
+    void onViewportResize(int newWidth, int newHeight) override;
+    void drawViewportOverlay() override;
     void customWindowsMenu() override;
     void onPlayModeChanged(bool nowPlaying) override;
 
@@ -42,7 +43,6 @@ private:
     void updateDebugUi();
     void drawDebugMenuBar();
     void drawWindowMenu();
-    void drawSelectionGizmo();
     void drawTestWindow();
     void spawnBall();
     void saveCurrentScene();

lightkeeper/src/Lightkeeper.cpp

@@ -166,6 +166,7 @@ void LightKeeper::customDraw()
             .clearColor = {{0.f, 0.f, 0.5f, 1.f}},
             .drawImageBlitRect = glm::ivec4{},
             .imguiPass = &imguiPass,
+            .copyDrawImageToSwapchain = !usesEditorViewport(),
         });
 }
 
@@ -204,9 +205,8 @@ void LightKeeper::onPlayModeChanged(bool nowPlaying)
     }
 }
 
-void LightKeeper::onWindowResize(int newWidth, int newHeight)
+void LightKeeper::onViewportResize(int newWidth, int newHeight)
 {
-    renderer.resize(gfxDevice, glm::ivec2{newWidth, newHeight});
     const float aspectRatio = static_cast<float>(newWidth) / static_cast<float>(newHeight);
     camera.setAspectRatio(aspectRatio);
 }

lightkeeper/src/LightkeeperDebugUi.cpp

@@ -54,7 +54,11 @@ void LightKeeper::updateDebugUi()
 
     drawDebugMenuBar();
     if (showTransformGizmo && !isPlayerViewActive()) {
-        drawSelectionGizmo();
+        GameObject* selected = SceneManager::GetInstance().GetCurrentScene().GetSelectedObject();
+        if (selected != nullptr && !RectGizmo::CanManipulate(*selected)) {
+            transformGizmo.HandleShortcuts();
+            transformGizmo.DrawToolbar();
+        }
     }
     if (showTestWindow) {
         drawTestWindow();
@@ -183,22 +187,31 @@ void LightKeeper::drawDebugMenuBar()
     ImGui::EndMainMenuBar();
 }
 
-void LightKeeper::drawSelectionGizmo()
+void LightKeeper::drawViewportOverlay()
 {
+    if (!showTransformGizmo || isPlayerViewActive()) {
+        return;
+    }
     GameObject* selected = SceneManager::GetInstance().GetCurrentScene().GetSelectedObject();
     if (selected == nullptr) {
         return;
     }
 
+    const ViewportRect& view = getViewportRect();
     if (RectGizmo::CanManipulate(*selected)) {
         const VkExtent3D extent = renderer.getDrawImage().extent;
-        rectGizmo.Manipulate(*selected, glm::vec2{static_cast<float>(extent.width), static_cast<float>(extent.height)});
+        rectGizmo.Manipulate(*selected, glm::vec2{static_cast<float>(extent.width), static_cast<float>(extent.height)},
+                             view.min, view.size);
+        if (rectGizmo.IsInteracting()) {
+            claimViewportMouse();
+        }
         return;
     }
 
-    transformGizmo.HandleShortcuts();
+    transformGizmo.Manipulate(*selected, getViewCamera(), view.min, view.size);
-    transformGizmo.DrawToolbar();
+    if (transformGizmo.IsOver() || transformGizmo.IsUsing()) {
-    transformGizmo.Manipulate(*selected, getViewCamera());
+        claimViewportMouse();
+    }
 }
 
 void LightKeeper::drawWindowMenu()