feat: make image loading async

destrum/include/destrum/Graphics/Caches/ImageCache.h

@@ -4,7 +4,9 @@
 #pragma once
 
 #include <filesystem>
+#include <future>
 #include <memory>
+#include <optional>
 #include <unordered_map>
 #include <vector>
 
@@ -12,6 +14,7 @@
 #include <destrum/Graphics/GPUImage.h>
 #include <destrum/Graphics/ids.h>
 #include <destrum/Graphics/TextureIntent.h>
+#include <destrum/Graphics/imageLoader.h>
 
 class GfxDevice;
 
@@ -33,6 +36,14 @@ public:
         VkImageUsageFlags usage,
         bool mipMap, TextureIntent intent = TextureIntent::ColorSrgb);
 
+    // Reserves the ID now and decodes on a job worker, the slot is filled in by finishPendingLoads
+    ImageID requestImage(
+        const std::filesystem::path& path,
+        VkImageUsageFlags usage,
+        bool mipMap, TextureIntent intent = TextureIntent::ColorSrgb);
+    void finishPendingLoads();
+    [[nodiscard]] std::size_t getPendingLoadCount() const { return pendingImages.size(); }
+
     ImageID addImage(GPUImage image);
     ImageID addImage(ImageID id, GPUImage image);
     [[nodiscard]] const GPUImage& getImage(ImageID id) const;
@@ -61,7 +72,22 @@ public:
     }
 
 private:
+    struct PendingImage {
+        ImageID id{NULL_IMAGE_ID};
+        std::future<ImageData> data;
+    };
+
+    [[nodiscard]] std::optional<ImageID> findLoadedImage(
+        const std::filesystem::path& path,
+        VkImageUsageFlags usage,
+        bool mipMap,
+        TextureIntent intent) const;
+    ImageID reserveImageId();
+    void finishPendingLoad(ImageID id);
+    void completePendingLoad(PendingImage pending);
+
     std::vector<GPUImage> images;
+    std::vector<PendingImage> pendingImages;
     std::shared_ptr<std::vector<ImageID>> freeIds = std::make_shared<std::vector<ImageID>>();
     GfxDevice& gfxDevice;
 

destrum/include/destrum/Graphics/GfxDevice.h

@@ -141,6 +141,15 @@ public:
         return imageManager.loadImageFromFile(path, usage, mipMap, intent);
     }
 
+    [[nodiscard]] std::optional<GPUImage> createImageFromData(
+        const ImageData& data,
+        VkImageUsageFlags usage,
+        bool mipMap,
+        const std::string& debugName) const
+    {
+        return imageManager.createImageFromData(data, usage, mipMap, debugName);
+    }
+
     void uploadImageDataSized(
         const GPUImage& image,
         const void* pixelData,

destrum/include/destrum/Graphics/Managers/ImageManager.h

@@ -14,6 +14,7 @@
 
 class VulkanImmediateExecutor;
 class MemoryManager;
+struct ImageData;
 
 class ImageManager {
 public:
@@ -41,6 +42,12 @@ public:
         bool mipMap,
         TextureIntent intent) const;
 
+    [[nodiscard]] std::optional<GPUImage> createImageFromData(
+        const ImageData& data,
+        VkImageUsageFlags usage,
+        bool mipMap,
+        const std::string& debugName) const;
+
     void uploadImageData(
         const GPUImage& image,
         const void* pixelData,

destrum/include/destrum/Graphics/RenderResources.h

@@ -48,6 +48,10 @@ public:
 
     ImageID loadMaterialTexture(GfxDevice& gfxDevice, const std::filesystem::path& path, TextureIntent intent);
 
+    // While a scope is open material textures decode on job workers, closing the last one uploads them
+    void beginDeferredTextureLoads();
+    void endDeferredTextureLoads();
+
     struct ReleaseStats {
         std::size_t meshes{0};
         std::size_t materials{0};
@@ -105,6 +109,20 @@ private:
     ImageID whiteImageId{NULL_IMAGE_ID};
     ImageID errorImageId{NULL_IMAGE_ID};
     ImageID defaultNormalImageId{NULL_IMAGE_ID};
+
+    int deferredTextureLoads{0};
+};
+
+class DeferredTextureLoadScope {
+public:
+    explicit DeferredTextureLoadScope(RenderResources* resources);
+    ~DeferredTextureLoadScope();
+
+    DeferredTextureLoadScope(const DeferredTextureLoadScope&) = delete;
+    DeferredTextureLoadScope& operator=(const DeferredTextureLoadScope&) = delete;
+
+private:
+    RenderResources* resources;
 };
 
 #endif

destrum/include/destrum/Threading/JobSystem.h

@@ -58,12 +58,12 @@ public:
     void Wait(JobCounter& counter);
 
     template <typename F>
-    auto Async(F&& fn) -> std::future<std::invoke_result_t<std::decay_t<F>>>
+    auto Async(F&& fn, JobPriority priority = JobPriority::Low) -> std::future<std::invoke_result_t<std::decay_t<F>>>
     {
         using Result = std::invoke_result_t<std::decay_t<F>>;
         auto task = std::make_shared<std::packaged_task<Result()>>(std::forward<F>(fn));
         auto future = task->get_future();
-        Submit([task] { (*task)(); }, JobPriority::Low);
+        Submit([task] { (*task)(); }, priority);
         return future;
     }
 

destrum/src/Graphics/Caches/ImageCache.cpp

@@ -1,9 +1,12 @@
 #include <destrum/Graphics/Caches/ImageCache.h>
 
 #include <destrum/Graphics/GfxDevice.h>
+#include <destrum/Threading/JobSystem.h>
 
 #include "spdlog/spdlog.h"
+#include <tracy/Tracy.hpp>
 #include <algorithm>
+#include <chrono>
 #include <limits>
 #include <stdexcept>
 
@@ -16,14 +19,9 @@ ImageID ImageCache::loadImageFromFile(
     bool mipMap,
     TextureIntent intent)
 {
-    for (const auto& [id, info] : loadedImagesInfo) {
+    if (const auto existing = findLoadedImage(path, usage, mipMap, intent)) {
-        if (info.path == path &&
+        finishPendingLoad(*existing);
-            info.intent == intent &&
+        return *existing;
-            info.usage == usage &&
-            info.mipMap == mipMap)
-        {
-            return id;
-        }
     }
 
     auto imageOpt = gfxDevice.loadImageFromFileRaw(path, usage, mipMap, intent);
@@ -63,6 +61,120 @@ ImageID ImageCache::loadImageFromFile(
     }
 }
 
+ImageID ImageCache::requestImage(
+    const std::filesystem::path& path,
+    VkImageUsageFlags usage,
+    bool mipMap,
+    TextureIntent intent)
+{
+    if (const auto existing = findLoadedImage(path, usage, mipMap, intent)) {
+        return *existing;
+    }
+
+    const ImageID id = reserveImageId();
+    try {
+        loadedImagesInfo.emplace(
+            id,
+            LoadedImageInfo{
+                .path = path,
+                .intent = intent,
+                .usage = usage,
+                .mipMap = mipMap,
+            });
+        pendingImages.push_back(PendingImage{
+            .id = id,
+            .data = JobSystem::GetInstance().Async(
+                [path, intent] { return util::loadImage(path, intent); },
+                JobPriority::High),
+        });
+        return id;
+    } catch (...) {
+        loadedImagesInfo.erase(id);
+        freeIds->push_back(id);
+        throw;
+    }
+}
+
+void ImageCache::finishPendingLoads()
+{
+    ZoneScopedN("ImageCache::finishPendingLoads");
+    // Upload in completion order so decoded pixels don't pile up in memory
+    while (!pendingImages.empty()) {
+        const auto ready = std::ranges::find_if(pendingImages, [](const PendingImage& pending) {
+            return pending.data.wait_for(std::chrono::seconds{0}) == std::future_status::ready;
+        });
+        if (ready == pendingImages.end()) {
+            pendingImages.front().data.wait_for(std::chrono::milliseconds{1});
+            continue;
+        }
+        PendingImage pending = std::move(*ready);
+        pendingImages.erase(ready);
+        completePendingLoad(std::move(pending));
+    }
+}
+
+std::optional<ImageID> ImageCache::findLoadedImage(
+    const std::filesystem::path& path,
+    VkImageUsageFlags usage,
+    bool mipMap,
+    TextureIntent intent) const
+{
+    for (const auto& [id, info] : loadedImagesInfo) {
+        if (info.path == path &&
+            info.intent == intent &&
+            info.usage == usage &&
+            info.mipMap == mipMap)
+        {
+            return id;
+        }
+    }
+    return std::nullopt;
+}
+
+ImageID ImageCache::reserveImageId()
+{
+    const ImageID id = getFreeImageId();
+    if (const auto it = std::find(freeIds->begin(), freeIds->end(), id); it != freeIds->end()) {
+        freeIds->erase(it);
+    }
+    if (id == images.size()) {
+        images.emplace_back();
+    }
+    return id;
+}
+
+void ImageCache::finishPendingLoad(ImageID id)
+{
+    const auto it = std::ranges::find_if(pendingImages, [id](const PendingImage& pending) { return pending.id == id; });
+    if (it == pendingImages.end()) {
+        return;
+    }
+    PendingImage pending = std::move(*it);
+    pendingImages.erase(it);
+    completePendingLoad(std::move(pending));
+}
+
+void ImageCache::completePendingLoad(PendingImage pending)
+{
+    const LoadedImageInfo& info = loadedImagesInfo.at(pending.id);
+    std::optional<GPUImage> image;
+    try {
+        image = gfxDevice.createImageFromData(pending.data.get(), info.usage, info.mipMap, info.path.string());
+    } catch (const std::exception& exception) {
+        spdlog::error("Failed to load image '{}': {}", info.path.string(), exception.what());
+    }
+
+    if (!image) {
+        // Materials already hold this ID, so the slot shows the error texture instead
+        spdlog::warn("Using error texture for failed image load: '{}'", info.path.string());
+        if (errorImageId != NULL_IMAGE_ID) {
+            bindlessSetManager.addImage(gfxDevice.getDevice(), pending.id, images.at(errorImageId).imageView);
+        }
+        return;
+    }
+    addImage(pending.id, std::move(*image));
+}
+
 ImageID ImageCache::addImage(GPUImage image) {
     try {
         return addImage(getFreeImageId(), std::move(image));
@@ -113,11 +225,16 @@ const ImageCache::LoadedImageInfo* ImageCache::findLoadedImageInfo(ImageID id) c
 }
 
 void ImageCache::removeImage(ImageID id) {
-    if (id >= images.size() || images[id].image == VK_NULL_HANDLE) {
+    if (id >= images.size()) {
+        return;
+    }
+    // Pending and failed loads own a slot without an image
+    const bool tracked = loadedImagesInfo.erase(id) > 0;
+    std::erase_if(pendingImages, [id](const PendingImage& pending) { return pending.id == id; });
+    if (images[id].image == VK_NULL_HANDLE && !tracked) {
         return;
     }
 
-    loadedImagesInfo.erase(id);
     gfxDevice.destroyImage(images[id]);
 
     // In-flight frames may still sample this bindless slot, so it only becomes
@@ -146,6 +263,7 @@ ImageID ImageCache::getFreeImageId() const {
 }
 
 void ImageCache::destroyImages() {
+    pendingImages.clear();
     for (auto& image: images) {
         gfxDevice.destroyImage(image);
     }

destrum/src/Graphics/Managers/ImageManager.cpp

@@ -130,13 +130,20 @@ std::optional<GPUImage> ImageManager::loadImageFromFile(
     bool mipMap,
     TextureIntent intent) const
 {
-    const auto data = util::loadImage(path, intent);
+    return createImageFromData(util::loadImage(path, intent), usage, mipMap, path.string());
+}
 
+std::optional<GPUImage> ImageManager::createImageFromData(
+    const ImageData& data,
+    VkImageUsageFlags usage,
+    bool mipMap,
+    const std::string& debugName) const
+{
     if (data.vkFormat == VK_FORMAT_UNDEFINED ||
         data.byteSize == 0 ||
         (data.hdr ? data.hdrPixels == nullptr : data.pixels == nullptr))
     {
-        spdlog::error("Failed to load image '{}'", path.string());
+        spdlog::error("Failed to load image '{}'", debugName);
         return std::nullopt;
     }
 
@@ -151,7 +158,7 @@ std::optional<GPUImage> ImageManager::loadImageFromFile(
             spdlog::warn(
                 "Format {} does not support linear mip generation, loading '{}' without mips",
                 static_cast<int>(data.vkFormat),
-                path.string());
+                debugName);
             mipMap = false;
         }
     }
@@ -178,7 +185,7 @@ std::optional<GPUImage> ImageManager::loadImageFromFile(
         throw;
     }
 
-    image.debugName = path.string();
+    image.debugName = debugName;
     vkutil::addDebugLabel(device, image.image, image.debugName.c_str());
     const std::string viewName = image.debugName + " (view)";
     vkutil::addDebugLabel(device, image.imageView, viewName.c_str());

destrum/src/Graphics/RenderResources.cpp

@@ -204,13 +204,49 @@ ImageID RenderResources::loadMaterialTexture(
     TextureIntent intent)
 {
     (void)gfxDevice;
-    const ImageID id = imageCache->loadImageFromFile(path, VK_IMAGE_USAGE_SAMPLED_BIT, true, intent);
+    const ImageID id = deferredTextureLoads > 0
+        ? imageCache->requestImage(path, VK_IMAGE_USAGE_SAMPLED_BIT, true, intent)
+        : imageCache->loadImageFromFile(path, VK_IMAGE_USAGE_SAMPLED_BIT, true, intent);
     if (id != errorImageId) {
         materialTextures.insert(id);
     }
     return id;
 }
 
+void RenderResources::beginDeferredTextureLoads()
+{
+    ++deferredTextureLoads;
+}
+
+void RenderResources::endDeferredTextureLoads()
+{
+    if (deferredTextureLoads > 0 && --deferredTextureLoads == 0 && imageCache) {
+        imageCache->finishPendingLoads();
+    }
+}
+
+DeferredTextureLoadScope::DeferredTextureLoadScope(RenderResources* resources)
+    : resources(resources)
+{
+    if (resources != nullptr) {
+        resources->beginDeferredTextureLoads();
+    }
+}
+
+DeferredTextureLoadScope::~DeferredTextureLoadScope()
+{
+    if (resources == nullptr) {
+        return;
+    }
+    try {
+        resources->endDeferredTextureLoads();
+    } catch (const std::exception& exception) {
+        spdlog::error("Finishing deferred texture loads failed: {}", exception.what());
+    } catch (...) {
+        spdlog::error("Finishing deferred texture loads failed");
+    }
+}
+
 RenderResources::ReleaseStats RenderResources::releaseUnused(
     GfxDevice& gfxDevice,
     const std::unordered_set<MeshID>& usedMeshes,

destrum/src/Serialization/SceneSerializer.cpp

@@ -16,6 +16,8 @@
 #include <destrum/Serialization/MaterialSerialization.h>
 #include <destrum/Serialization/ObjectSerialization.h>
 #include <destrum/Threading/JobSystem.h>
+#include <destrum/Graphics/Renderer.h>
+#include <destrum/Util/GameState.h>
 
 using json = nlohmann::json;
 
@@ -207,6 +209,10 @@ bool SceneSerializer::ConstructScene(Scene& scene, LoadResult& result,
     std::vector<GameObject*> registeredObjects;
     std::string context = "scene";
 
+    auto& gameState = GameState::GetInstance();
+    const DeferredTextureLoadScope textureLoads{
+        gameState.HasRenderer() && gameState.HasGfxDevice() ? gameState.Renderer().getResources() : nullptr};
+
     try {
         json objects = root.at("objects");
         if (root.contains("materials")) {

tests/jobs_tests.cpp

@@ -5,7 +5,9 @@
 #include <stdexcept>
 #include <thread>
 #include <vector>
+#include <cstring>
 
+#include <destrum/Graphics/imageLoader.h>
 #include <destrum/Serialization/SceneSerializer.h>
 #include <destrum/Threading/JobSystem.h>
 
@@ -161,6 +163,47 @@ namespace {
         Jobs().Shutdown();
     }
 
+    void testAsyncHighPriority()
+    {
+        Jobs().Init(2);
+        auto future = Jobs().Async([] { return std::this_thread::get_id(); }, JobPriority::High);
+        check(future.get() != std::this_thread::get_id(), "high priority Async should run on a worker");
+        Jobs().Shutdown();
+    }
+
+    void testImageDecodeOnWorkersMatchesSerial()
+    {
+        const std::filesystem::path root = DESTRUM_ENGINE_ASSETS_SRC_DIR;
+        const std::vector<std::filesystem::path> paths{
+            root / "238882.png",
+            root / "char.jpg",
+            root / "flight-helmet" / "FlightHelmet_Materials_GlassPlasticMat_Normal.png",
+            root / "does_not_exist.png",
+        };
+
+        Jobs().Init(4);
+        std::vector<std::future<ImageData>> decoded;
+        for (const auto& path : paths) {
+            decoded.push_back(Jobs().Async([path] { return util::loadImage(path, TextureIntent::ColorSrgb); },
+                                           JobPriority::High));
+        }
+
+        for (std::size_t i = 0; i < paths.size(); ++i) {
+            const ImageData parallel = decoded[i].get();
+            const ImageData serial = util::loadImage(paths[i], TextureIntent::ColorSrgb);
+            check(parallel.vkFormat == serial.vkFormat && parallel.width == serial.width &&
+                  parallel.height == serial.height && parallel.byteSize == serial.byteSize,
+                  "decoding on a worker should give the same image layout");
+            if (serial.pixels != nullptr) {
+                check(parallel.pixels != nullptr &&
+                      std::memcmp(parallel.pixels, serial.pixels, serial.byteSize) == 0,
+                      "decoding on a worker should give the same pixels");
+            }
+        }
+        check(decoded.size() == 4, "every image should be decoded");
+        Jobs().Shutdown();
+    }
+
     void testRunsInlineWhenStopped()
     {
         check(!Jobs().IsRunning(), "job system should start stopped");
@@ -183,4 +226,6 @@ void RunJobsTests()
     testMainThreadQueue();
     testShutdownDrainsPendingJobs();
     testLoadSceneFileAsync();
+    testAsyncHighPriority();
+    testImageDecodeOnWorkersMatchesSerial();
 }