From 0fbc023018475a0c8fecbc8f410eea229d130e93 Mon Sep 17 00:00:00 2001 From: Bram Verhulst Date: Fri, 30 Oct 2026 15:42:08 +0100 Subject: [PATCH] feat: add export / file packing --- CMakeLists.txt | 3 + Readme.md | 10 + TheChef | 2 +- destrum/CMakeLists.txt | 36 +++ destrum/include/destrum/Assets/PackFile.h | 46 ++++ destrum/include/destrum/FS/AssetFS.h | 13 +- destrum/include/destrum/FS/Manifest.h | 3 +- destrum/include/destrum/Util/ModelDoc.h | 27 ++- destrum/src/Assets/AssetDatabase.cpp | 23 +- destrum/src/Assets/AssetFSIOSystem.h | 45 ++++ destrum/src/Assets/AssetManager.cpp | 18 +- destrum/src/Assets/PackFile.cpp | 126 +++++++++++ destrum/src/FS/AssetFS.cpp | 213 ++++++++++++++---- destrum/src/FS/Manifest.cpp | 11 +- destrum/src/Graphics/ComputePipeline.cpp | 18 +- destrum/src/Graphics/ImageLoader.cpp | 30 ++- destrum/src/Graphics/Pipeline.cpp | 19 +- destrum/src/Graphics/Resources/Font.cpp | 9 +- destrum/src/Graphics/Util.cpp | 21 +- .../src/Serialization/ObjectSerialization.cpp | 2 +- destrum/src/Serialization/PrefabCache.cpp | 7 +- .../src/Serialization/PrefabSerializer.cpp | 14 +- destrum/src/Serialization/SceneSerializer.cpp | 13 +- export.bat | 18 ++ lightkeeper/CMakeLists.txt | 3 + lightkeeper/src/main.cpp | 44 +++- tests/assets_tests.cpp | 113 ++++++++++ tests/cooker_tests.cpp | 52 ++++- 28 files changed, 782 insertions(+), 157 deletions(-) create mode 100644 destrum/include/destrum/Assets/PackFile.h create mode 100644 destrum/src/Assets/AssetFSIOSystem.h create mode 100644 destrum/src/Assets/PackFile.cpp create mode 100644 export.bat diff --git a/CMakeLists.txt b/CMakeLists.txt index 44fef56..723641f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,9 @@ add_dependencies(CookAssets _internal_cook_engine_assets ) +add_custom_target(Export) +add_dependencies(Export _internal_export_lightkeeper) + add_custom_target(ValidateAssets) add_dependencies(ValidateAssets _internal_validate_game_assets diff --git a/Readme.md b/Readme.md index 0b2a1df..8b17d46 100644 --- a/Readme.md +++ b/Readme.md @@ -118,6 +118,16 @@ cmake --build build --target ValidateShaders `CookAssets` processes both engine and game assets. `CleanupAssets` removes generated runtime asset output. `ValidateShaders` runs `spirv-val` over the cooked engine shaders when the Vulkan SDK provides it. +Export a folder other people can run (the game, its DLLs and every cooked asset packed into one `assets.dpak`): + +```bash +cmake --build build --config Release --target Export +``` + +It ends up in `build/export/Release/lightkeeper/`, plus a `lightkeeper.zip` next to it. On Windows, `export.bat` does +the same and opens the folder. `TheChef --list build/export/Release/lightkeeper/assets.dpak` shows what takes up the +space. See `docs/CookedAssets.md`. + Install the application, engine library, public headers, and runtime assets: ```bash diff --git a/TheChef b/TheChef index 3d4b505..abcec12 160000 --- a/TheChef +++ b/TheChef @@ -1 +1 @@ -Subproject commit 3d4b50529e4318cd81605b2cb979f645d3a8fb66 +Subproject commit abcec12c90d59d5bab24e7d96fef91774d81024d diff --git a/destrum/CMakeLists.txt b/destrum/CMakeLists.txt index e8ff40f..538fdb6 100644 --- a/destrum/CMakeLists.txt +++ b/destrum/CMakeLists.txt @@ -4,6 +4,7 @@ add_subdirectory(third_party) add_library(destrum_assets "src/Assets/ModelFile.cpp" "src/Assets/TextureFile.cpp" + "src/Assets/PackFile.cpp" ) add_library(destrum::assets ALIAS destrum_assets) @@ -297,6 +298,41 @@ function(destrum_cook_engine_assets GAME_TARGET GAME_OUTPUT_DIR) install(DIRECTORY "${_engine_runtime}/" DESTINATION "bin/assets/engine") endfunction() +# A folder (and zip) other people can run: the game exe, its DLLs and every cooked asset packed into assets.dpak +function(destrum_add_export GAME_TARGET GAME_RUNTIME_DIR) + set(_engine_runtime "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/assets_runtime") + set(_export_root "${CMAKE_BINARY_DIR}/export/$") + set(_export_dir "${_export_root}/${GAME_TARGET}") + + # The release VC++ runtime ships app-local, debug builds only run where Visual Studio is installed + set(_system_libs) + if (MSVC) + set(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP TRUE) + include(InstallRequiredSystemLibraries) + set(_system_libs "$<$>:${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS}>") + endif() + + add_custom_target(_internal_export_${GAME_TARGET} + COMMAND ${CMAKE_COMMAND} -E rm -rf "${_export_dir}" "${_export_dir}.zip" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_export_dir}" + COMMAND TheChef + --pack "${_export_dir}/assets.dpak" + --mount "engine=${_engine_runtime}" + --mount "game=${GAME_RUNTIME_DIR}" + COMMAND TheChef --list "${_export_dir}/assets.dpak" --top 10 + COMMAND ${CMAKE_COMMAND} -E copy -t "${_export_dir}" + "$" + "$" + "${_system_libs}" + COMMAND ${CMAKE_COMMAND} -E chdir "${_export_root}" + ${CMAKE_COMMAND} -E tar cf "${GAME_TARGET}.zip" --format=zip "${GAME_TARGET}" + COMMAND ${CMAKE_COMMAND} -E echo "Exported ${GAME_TARGET} to ${_export_dir}" + COMMAND_EXPAND_LISTS + VERBATIM + ) + add_dependencies(_internal_export_${GAME_TARGET} ${GAME_TARGET}) +endfunction() + # OpenAL is a DLL, every exe that links the engine needs a copy next to it function(destrum_copy_runtime_dlls TARGET) if (WIN32) diff --git a/destrum/include/destrum/Assets/PackFile.h b/destrum/include/destrum/Assets/PackFile.h new file mode 100644 index 0000000..28b2971 --- /dev/null +++ b/destrum/include/destrum/Assets/PackFile.h @@ -0,0 +1,46 @@ +#ifndef DESTRUM_PACKFILE_H +#define DESTRUM_PACKFILE_H + +#include +#include +#include +#include +#include +#include +#include + +namespace PackFile { + inline constexpr std::uint32_t Version = 1; + inline constexpr std::string_view Extension = ".dpak"; + + struct Entry { + std::string path; + std::uint64_t offset{0}; + std::uint64_t size{0}; + }; + + struct Source { + std::string path; + std::filesystem::path file; + }; + + // Paths are "/", e.g. "engine/shaders/mesh.vert.spv" + void Write(const std::vector& files, const std::filesystem::path& path); + + class Reader { + public: + explicit Reader(std::filesystem::path path); + + [[nodiscard]] const Entry* Find(std::string_view path) const; + [[nodiscard]] std::vector Read(const Entry& entry) const; + [[nodiscard]] const std::vector& Entries() const { return m_Entries; } + [[nodiscard]] const std::filesystem::path& Path() const { return m_Path; } + + private: + std::filesystem::path m_Path; + std::vector m_Entries; + std::unordered_map m_ByPath; + }; +} + +#endif // DESTRUM_PACKFILE_H diff --git a/destrum/include/destrum/FS/AssetFS.h b/destrum/include/destrum/FS/AssetFS.h index abdfe50..041a986 100644 --- a/destrum/include/destrum/FS/AssetFS.h +++ b/destrum/include/destrum/FS/AssetFS.h @@ -3,12 +3,14 @@ #include #include +#include #include #include #include #include #include +#include #include @@ -18,17 +20,26 @@ struct FSMount { std::filesystem::path root; std::optional manifest; std::optional sourceRoot; + std::shared_ptr pack; }; class AssetFS final: public Singleton { public: + static constexpr std::string_view PackName = "assets.dpak"; + void Init(std::filesystem::path exeDir); void Reset(); void Mount(std::string scheme, std::filesystem::path root); + void MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root); void ReloadManifest(std::string_view scheme); std::vector ReadBytes(std::string_view vpath); + [[nodiscard]] std::vector ReadFile(const std::filesystem::path& fullPath) const; + [[nodiscard]] bool Exists(const std::filesystem::path& fullPath) const; + [[nodiscard]] std::vector ListFiles(const FSMount& mount) const; + [[nodiscard]] bool IsPacked() const; + [[nodiscard]] std::filesystem::path GetFullPath(std::string_view vpath) const; [[nodiscard]] std::filesystem::path GetCookedPathForFile(std::string_view vpath) const; @@ -43,7 +54,7 @@ public: bool WriteFile(std::string_view vpath, std::string_view contents); private: - static std::vector ReadFile(const std::filesystem::path& fullPath); + void AddMount(std::string scheme, std::filesystem::path root, std::shared_ptr pack); std::vector mounts; diff --git a/destrum/include/destrum/FS/Manifest.h b/destrum/include/destrum/FS/Manifest.h index c6b6d8f..d6b46d3 100644 --- a/destrum/include/destrum/FS/Manifest.h +++ b/destrum/include/destrum/FS/Manifest.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,7 @@ struct AssetManifest { }; namespace FS { - AssetManifest LoadAssetManifest(const std::filesystem::path& manifestPath); + AssetManifest ParseAssetManifest(std::span bytes); } #endif //MANIFEST_H diff --git a/destrum/include/destrum/Util/ModelDoc.h b/destrum/include/destrum/Util/ModelDoc.h index 25dbf92..5932040 100644 --- a/destrum/include/destrum/Util/ModelDoc.h +++ b/destrum/include/destrum/Util/ModelDoc.h @@ -20,6 +20,7 @@ // // Link against: assimp +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -998,8 +1000,12 @@ static void CollectMergedPerNodeMeshes(const aiScene* scene, // ─── Public API ─────────────────────────────────────────────────────────────── -static ModelAsset LoadModel(const std::string& path, const LoadOptions& options = {}) { +// The importer owns io, without one it reads from disk +static ModelAsset LoadModel(const std::string& path, const LoadOptions& options = {}, + std::unique_ptr io = nullptr) { Assimp::Importer importer; + if (io) + importer.SetIOHandler(io.release()); const aiScene* scene = LoadScene(importer, path, options); ModelAsset model{}; @@ -1084,16 +1090,18 @@ static const aiScene* LoadAnimationScene( Assimp::Importer& importer, const std::string& path ) { - if (!std::filesystem::exists(path)) { + if (!importer.GetIOHandler()->Exists(path.c_str())) { throw std::runtime_error("Animation file does not exist: " + path); } - if (!std::filesystem::is_regular_file(path)) { - throw std::runtime_error("Animation path is not a regular file: " + path); - } + if (importer.IsDefaultIOHandler()) { + if (!std::filesystem::is_regular_file(path)) { + throw std::runtime_error("Animation path is not a regular file: " + path); + } - if (std::filesystem::file_size(path) == 0) { - throw std::runtime_error("Animation file is empty: " + path); + if (std::filesystem::file_size(path) == 0) { + throw std::runtime_error("Animation file is empty: " + path); + } } // Important: do not use mesh post-process flags for animation-only files. @@ -1141,9 +1149,12 @@ static const aiScene* LoadAnimationScene( static std::vector LoadAnimationClips( const std::string& path, - const Skeleton& targetSkeleton + const Skeleton& targetSkeleton, + std::unique_ptr io = nullptr ) { Assimp::Importer importer; + if (io) + importer.SetIOHandler(io.release()); const aiScene* scene = LoadAnimationScene(importer, path); auto clips = LoadAnimations(scene, targetSkeleton); diff --git a/destrum/src/Assets/AssetDatabase.cpp b/destrum/src/Assets/AssetDatabase.cpp index bd59c3a..6d0a71a 100644 --- a/destrum/src/Assets/AssetDatabase.cpp +++ b/destrum/src/Assets/AssetDatabase.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -19,12 +18,9 @@ namespace { [[nodiscard]] std::optional ReadMetaFile(const std::filesystem::path& path) { - std::ifstream file(path); - if (!file) { - return std::nullopt; - } try { - const auto meta = nlohmann::json::parse(file); + const std::vector bytes = AssetFS::GetInstance().ReadFile(path); + const auto meta = nlohmann::json::parse(bytes.begin(), bytes.end()); return AssetGuid::Parse(meta.at("guid").get()); } catch (const std::exception& exception) { spdlog::warn("Ignoring broken asset meta file {}: {}", path.string(), exception.what()); @@ -83,14 +79,14 @@ void AssetDatabase::Clear() void AssetDatabase::Refresh() { Clear(); - for (const FSMount& mount : AssetFS::GetInstance().GetMounts()) { - std::error_code error; - for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) { - if (!entry.is_regular_file(error) || !AssetTypes::IsMeta(entry.path().string())) { + const auto& assetFs = AssetFS::GetInstance(); + for (const FSMount& mount : assetFs.GetMounts()) { + for (const std::filesystem::path& file : assetFs.ListFiles(mount)) { + if (!AssetTypes::IsMeta(file.string())) { continue; } - if (const auto guid = ReadMetaFile(entry.path())) { - std::filesystem::path asset = entry.path(); + if (const auto guid = ReadMetaFile(file)) { + std::filesystem::path asset = file; asset.replace_extension(); Register(mount.scheme + "://" + asset.lexically_relative(mount.root).generic_string(), *guid); } @@ -131,8 +127,7 @@ std::optional AssetDatabase::GuidForSave(std::string_view path) } auto& assetFs = AssetFS::GetInstance(); - std::error_code error; - if (!std::filesystem::is_regular_file(assetFs.GetFullPath(path), error)) { + if (!assetFs.Exists(assetFs.GetFullPath(path))) { return std::nullopt; } diff --git a/destrum/src/Assets/AssetFSIOSystem.h b/destrum/src/Assets/AssetFSIOSystem.h new file mode 100644 index 0000000..e3be155 --- /dev/null +++ b/destrum/src/Assets/AssetFSIOSystem.h @@ -0,0 +1,45 @@ +#ifndef DESTRUM_ASSETFSIOSYSTEM_H +#define DESTRUM_ASSETFSIOSYSTEM_H + +#include +#include +#include +#include + +#include +#include + +#include + +class AssetFSIOSystem final : public Assimp::IOSystem { +public: + bool Exists(const char* file) const override { + return AssetFS::GetInstance().Exists(file); + } + + char getOsSeparator() const override { + return '/'; + } + + Assimp::IOStream* Open(const char* file, const char* mode) override { + if (std::string_view{mode}.find_first_of("wa+") != std::string_view::npos) { + return nullptr; + } + + std::vector bytes; + try { + bytes = AssetFS::GetInstance().ReadFile(file); + } catch (const std::exception&) { + return nullptr; + } + auto* buffer = new std::uint8_t[bytes.size()]; + std::memcpy(buffer, bytes.data(), bytes.size()); + return new Assimp::MemoryIOStream(buffer, bytes.size(), true); + } + + void Close(Assimp::IOStream* stream) override { + delete stream; + } +}; + +#endif // DESTRUM_ASSETFSIOSYSTEM_H diff --git a/destrum/src/Assets/AssetManager.cpp b/destrum/src/Assets/AssetManager.cpp index 2814288..44e37c3 100644 --- a/destrum/src/Assets/AssetManager.cpp +++ b/destrum/src/Assets/AssetManager.cpp @@ -1,11 +1,14 @@ #include +#include #include #include #include #include +#include "AssetFSIOSystem.h" + namespace { std::string ToReference(std::string_view reference) { if (reference.empty()) { @@ -30,6 +33,14 @@ namespace { return std::nullopt; } + // Assimp opens files itself, so with packed assets it has to go through AssetFS + std::unique_ptr MakeIOSystem() { + if (!AssetFS::GetInstance().IsPacked()) { + return nullptr; + } + return std::make_unique(); + } + // Cooked texture paths are relative to the model file, imported ones are absolute void ResolveTexturePaths(ModelDoc::ModelAsset& model, const std::filesystem::path& modelDir) { for (ModelDoc::MaterialInfo& material : model.materials) { @@ -95,7 +106,7 @@ ModelDoc::ModelAsset AssetManager::LoadCookedOrImport( const ModelDoc::LoadOptions& options) const { if (const auto cookedPath = FindCookedModel(reference)) { - ModelDoc::ModelAsset model = ModelFile::ReadFromFile(*cookedPath); + ModelDoc::ModelAsset model = ModelFile::Read(AssetFS::GetInstance().ReadFile(*cookedPath)); if (ModelFile::SupportsLoadOptions(model, options)) { ModelFile::ApplyLoadOptions(model, options); ResolveTexturePaths(model, cookedPath->parent_path()); @@ -105,7 +116,7 @@ ModelDoc::ModelAsset AssetManager::LoadCookedOrImport( spdlog::debug("Importing {}, its cooked model doesn't match the requested load options", reference); } - return ModelDoc::LoadModel(Resolve(reference).generic_string(), options); + return ModelDoc::LoadModel(Resolve(reference).generic_string(), options, MakeIOSystem()); } std::vector AssetManager::LoadAnimationClips( @@ -115,7 +126,8 @@ std::vector AssetManager::LoadAnimationClips( const std::string sourceReference = ToReference(reference); auto clips = ModelDoc::LoadAnimationClips( Resolve(sourceReference).generic_string(), - targetSkeleton); + targetSkeleton, + MakeIOSystem()); for (std::size_t index = 0; index < clips.size(); ++index) { auto& clip = clips[index]; clip.assetReference.path = sourceReference; diff --git a/destrum/src/Assets/PackFile.cpp b/destrum/src/Assets/PackFile.cpp new file mode 100644 index 0000000..97a0a3c --- /dev/null +++ b/destrum/src/Assets/PackFile.cpp @@ -0,0 +1,126 @@ +#include + +#include + +#include "BinaryStream.h" + +namespace { + using AssetBinary::BinaryReader; + using AssetBinary::BinaryWriter; + + constexpr AssetBinary::Magic Magic{'D', 'P', 'A', 'K'}; + constexpr std::string_view Kind = "pack"; + constexpr std::size_t HeaderSize = sizeof(AssetBinary::Magic) + sizeof(std::uint32_t) + sizeof(std::uint64_t); +} + +void PackFile::Write(const std::vector& files, const std::filesystem::path& path) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + if (!out) { + throw std::runtime_error("Failed to open file for writing: " + path.string()); + } + + BinaryWriter header; + header.WriteHeader(Magic, Version); + header.Write(std::uint64_t{0}); + const std::vector headerBytes = header.TakeBytes(); + out.write(reinterpret_cast(headerBytes.data()), static_cast(headerBytes.size())); + + std::set seen; + BinaryWriter index; + index.WriteCount(files.size()); + for (const Source& file : files) { + if (!seen.insert(file.path).second) { + throw std::runtime_error("Pack has two files at " + file.path); + } + + std::ifstream in(file.file, std::ios::binary); + if (!in) { + throw std::runtime_error("Failed to open file: " + file.file.string()); + } + const auto offset = static_cast(out.tellp()); + // Inserting an empty stream buffer sets failbit + if (std::filesystem::file_size(file.file) > 0) { + out << in.rdbuf(); + } + const auto size = static_cast(out.tellp()) - offset; + + index.WriteString(file.path); + index.Write(offset); + index.Write(size); + } + + const auto indexOffset = static_cast(out.tellp()); + const std::vector indexBytes = index.TakeBytes(); + out.write(reinterpret_cast(indexBytes.data()), static_cast(indexBytes.size())); + out.seekp(static_cast(HeaderSize - sizeof(std::uint64_t))); + out.write(reinterpret_cast(&indexOffset), sizeof(indexOffset)); + if (!out) { + throw std::runtime_error("Failed to write file: " + path.string()); + } +} + +PackFile::Reader::Reader(std::filesystem::path path) + : m_Path(std::move(path)) { + std::ifstream file(m_Path, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error("Failed to open file: " + m_Path.string()); + } + const auto fileSize = static_cast(file.tellg()); + + std::array headerBytes{}; + file.seekg(0); + file.read(reinterpret_cast(headerBytes.data()), static_cast(headerBytes.size())); + if (!file) { + throw std::runtime_error("Not a " + std::string{Kind} + " file: " + m_Path.string()); + } + BinaryReader header{headerBytes}; + header.ReadHeader(Magic, Version, Kind); + const auto indexOffset = header.Read(); + if (indexOffset < HeaderSize || indexOffset > fileSize) { + throw std::runtime_error("Pack file is corrupt: index is out of range"); + } + + std::vector indexBytes(static_cast(fileSize - indexOffset)); + file.seekg(static_cast(indexOffset)); + file.read(reinterpret_cast(indexBytes.data()), static_cast(indexBytes.size())); + if (!file) { + throw std::runtime_error("Failed to read file: " + m_Path.string()); + } + + BinaryReader index{indexBytes}; + m_Entries.resize(index.ReadCount(3 * sizeof(std::uint64_t))); + for (std::size_t i = 0; i < m_Entries.size(); ++i) { + Entry& entry = m_Entries[i]; + entry.path = index.ReadString(); + entry.offset = index.Read(); + entry.size = index.Read(); + if (entry.offset < HeaderSize || entry.offset > indexOffset || entry.size > indexOffset - entry.offset) { + throw std::runtime_error("Pack file is corrupt: " + entry.path + " is out of range"); + } + if (!m_ByPath.emplace(entry.path, i).second) { + throw std::runtime_error("Pack file is corrupt: " + entry.path + " is listed twice"); + } + } + index.ExpectEnd(Kind); +} + +const PackFile::Entry* PackFile::Reader::Find(std::string_view path) const { + const auto it = m_ByPath.find(std::string{path}); + return it != m_ByPath.end() ? &m_Entries[it->second] : nullptr; +} + +// Opens its own stream so loads on worker threads don't share a file position +std::vector PackFile::Reader::Read(const Entry& entry) const { + std::ifstream file(m_Path, std::ios::binary); + if (!file) { + throw std::runtime_error("Failed to open file: " + m_Path.string()); + } + + std::vector bytes(static_cast(entry.size)); + file.seekg(static_cast(entry.offset)); + file.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (!file) { + throw std::runtime_error("Failed to read " + entry.path + " from " + m_Path.string()); + } + return bytes; +} diff --git a/destrum/src/FS/AssetFS.cpp b/destrum/src/FS/AssetFS.cpp index 49c015b..39a5b66 100644 --- a/destrum/src/FS/AssetFS.cpp +++ b/destrum/src/FS/AssetFS.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -97,6 +98,79 @@ namespace { return nullptr; } + + [[nodiscard]] std::optional RelativeToMount(const FSMount& mount, + const std::filesystem::path& fullPath) { + const std::filesystem::path relative = fullPath.lexically_normal().lexically_relative(mount.root); + if (relative.empty() || relative.is_absolute() || relative == "." || + relative == ".." || relative.generic_string().starts_with("../")) { + return std::nullopt; + } + return relative; + } + + struct PackLocation { + const FSMount* mount; + std::string key; + }; + + // Everything under a packed mount comes from the pack, even files that also exist on disk + [[nodiscard]] std::optional FindInPack(const std::vector& mounts, + const std::filesystem::path& fullPath) { + for (const FSMount& mount : mounts) { + if (!mount.pack) { + continue; + } + if (const auto relative = RelativeToMount(mount, fullPath)) { + return PackLocation{&mount, mount.scheme + "/" + relative->generic_string()}; + } + } + return std::nullopt; + } + + [[nodiscard]] std::vector ReadDiskFile(const std::filesystem::path& fullPath) { + std::ifstream file(fullPath, std::ios::binary); + if (!file) { + throw std::runtime_error("failed to open file: " + fullPath.string()); + } + + file.seekg(0, std::ios::end); + const std::streamsize size = file.tellg(); + if (size < 0) { + throw std::runtime_error("failed to determine file size: " + fullPath.string()); + } + file.seekg(0, std::ios::beg); + + std::vector buffer(static_cast(size)); + if (size > 0 && !file.read(reinterpret_cast(buffer.data()), size)) { + throw std::runtime_error("failed to read file: " + fullPath.string()); + } + return buffer; + } + + [[nodiscard]] std::optional LoadManifest(const FSMount& mount) { + const std::filesystem::path path = mount.root / "manifest.json"; + std::vector bytes; + if (mount.pack) { + const PackFile::Entry* entry = mount.pack->Find(mount.scheme + "/manifest.json"); + if (entry == nullptr) { + return std::nullopt; + } + bytes = mount.pack->Read(*entry); + } else { + std::error_code error; + if (!std::filesystem::is_regular_file(path, error)) { + return std::nullopt; + } + bytes = ReadDiskFile(path); + } + + try { + return FS::ParseAssetManifest(bytes); + } catch (const std::exception& exception) { + throw std::runtime_error("invalid manifest " + path.string() + ": " + exception.what()); + } + } } void AssetFS::Init(std::filesystem::path exeDir) { @@ -106,8 +180,14 @@ void AssetFS::Init(std::filesystem::path exeDir) { const std::size_t originalMountCount = mounts.size(); try { - Mount("engine", exeDir / "assets/engine"); - Mount("game", exeDir / "assets/game"); + const std::filesystem::path packPath = exeDir / PackName; + std::error_code error; + if (std::filesystem::is_regular_file(packPath, error)) { + MountPack(packPath, exeDir / "assets"); + } else { + Mount("engine", exeDir / "assets/engine"); + Mount("game", exeDir / "assets/game"); + } initialized = true; } catch (...) { mounts.resize(originalMountCount); @@ -121,6 +201,40 @@ void AssetFS::Reset() { } void AssetFS::Mount(std::string scheme, std::filesystem::path root) { + AddMount(std::move(scheme), std::move(root), nullptr); +} + +// Each scheme in the pack mounts at root/, so full paths look the same as with the unpacked folders +void AssetFS::MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root) { + auto pack = std::make_shared(packPath); + + std::vector schemes; + for (const PackFile::Entry& entry : pack->Entries()) { + const std::size_t separator = entry.path.find('/'); + if (separator == std::string::npos || separator == 0) { + throw std::runtime_error("asset pack entry has no scheme: " + entry.path); + } + std::string scheme = entry.path.substr(0, separator); + if (std::ranges::find(schemes, scheme) == schemes.end()) { + schemes.push_back(std::move(scheme)); + } + } + + const std::size_t originalMountCount = mounts.size(); + try { + for (std::string& scheme : schemes) { + std::filesystem::path schemeRoot = root / scheme; + AddMount(std::move(scheme), std::move(schemeRoot), pack); + } + } catch (...) { + mounts.resize(originalMountCount); + throw; + } + spdlog::info("Mounted asset pack {} ({} files)", packPath.string(), pack->Entries().size()); +} + +void AssetFS::AddMount(std::string scheme, std::filesystem::path root, + std::shared_ptr pack) { if (scheme.empty() || scheme.find_first_of("/\\:") != std::string::npos) { throw std::runtime_error("asset mount scheme is invalid"); } @@ -131,12 +245,8 @@ void AssetFS::Mount(std::string scheme, std::filesystem::path root) { FSMount mount; mount.scheme = std::move(scheme); mount.root = std::filesystem::absolute(std::move(root)).lexically_normal(); - - const std::filesystem::path manifestPath = mount.root / "manifest.json"; - std::error_code error; - if (std::filesystem::is_regular_file(manifestPath, error)) { - mount.manifest = FS::LoadAssetManifest(manifestPath); - } + mount.pack = std::move(pack); + mount.manifest = LoadManifest(mount); spdlog::debug("Mounting assetfs scheme '{}' to root '{}'", mount.scheme, mount.root.string()); mounts.push_back(std::move(mount)); @@ -147,12 +257,8 @@ void AssetFS::ReloadManifest(std::string_view scheme) { if (mount.scheme != scheme) { continue; } - const std::filesystem::path manifestPath = mount.root / "manifest.json"; - std::error_code error; mount.manifest.reset(); - if (std::filesystem::is_regular_file(manifestPath, error)) { - mount.manifest = FS::LoadAssetManifest(manifestPath); - } + mount.manifest = LoadManifest(mount); return; } throw std::runtime_error("asset mount not found: " + std::string{scheme}); @@ -162,6 +268,51 @@ std::vector AssetFS::ReadBytes(std::string_view vpath) { return ReadFile(GetFullPath(vpath)); } +std::vector AssetFS::ReadFile(const std::filesystem::path& fullPath) const { + if (const auto location = FindInPack(mounts, fullPath)) { + const PackFile::Entry* entry = location->mount->pack->Find(location->key); + if (entry == nullptr) { + throw std::runtime_error("failed to open file: " + fullPath.string() + " is not in " + + location->mount->pack->Path().string()); + } + return location->mount->pack->Read(*entry); + } + return ReadDiskFile(fullPath); +} + +bool AssetFS::Exists(const std::filesystem::path& fullPath) const { + if (const auto location = FindInPack(mounts, fullPath)) { + return location->mount->pack->Find(location->key) != nullptr; + } + std::error_code error; + return std::filesystem::is_regular_file(fullPath, error); +} + +std::vector AssetFS::ListFiles(const FSMount& mount) const { + std::vector files; + if (mount.pack) { + const std::string prefix = mount.scheme + "/"; + for (const PackFile::Entry& entry : mount.pack->Entries()) { + if (entry.path.starts_with(prefix)) { + files.push_back((mount.root / entry.path.substr(prefix.size())).lexically_normal()); + } + } + return files; + } + + std::error_code error; + for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) { + if (entry.is_regular_file(error)) { + files.push_back(entry.path()); + } + } + return files; +} + +bool AssetFS::IsPacked() const { + return std::ranges::any_of(mounts, [](const FSMount& mount) { return mount.pack != nullptr; }); +} + std::filesystem::path AssetFS::GetFullPath(std::string_view vpath) const { if (!initialized) { throw std::runtime_error("AssetFS is not initialized"); @@ -194,6 +345,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons const std::filesystem::path cookedRelative = ValidateRelativePath(*asset->out); const std::filesystem::path cookedPath = ResolvePath(*mount, cookedRelative); + if (mount->pack && Exists(cookedPath)) { + return cookedPath; + } + std::error_code error; if (std::filesystem::is_regular_file(cookedPath, error)) { error.clear(); @@ -218,14 +373,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons } std::optional AssetFS::ToVirtualPath(const std::filesystem::path& fullPath) const { - const std::filesystem::path normalized = fullPath.lexically_normal(); for (const FSMount& mount : mounts) { - const std::filesystem::path relative = normalized.lexically_relative(mount.root); - if (relative.empty() || relative.is_absolute() || relative == "." || - relative == ".." || relative.generic_string().starts_with("../")) { - continue; + if (const auto relative = RelativeToMount(mount, fullPath)) { + return mount.scheme + "://" + relative->generic_string(); } - return mount.scheme + "://" + relative.generic_string(); } return std::nullopt; } @@ -251,6 +402,10 @@ std::optional AssetFS::GetSourcePath(std::string_view vpa bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) { std::vector targets{GetFullPath(vpath)}; + if (FindInPack(mounts, targets.front())) { + spdlog::error("Can't write {}, it's in a read only asset pack", vpath); + return false; + } if (auto source = GetSourcePath(vpath)) { targets.push_back(std::move(*source)); } @@ -276,23 +431,3 @@ bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) { } return true; } - -std::vector AssetFS::ReadFile(const std::filesystem::path& fullPath) { - std::ifstream file(fullPath, std::ios::binary); - if (!file) { - throw std::runtime_error("failed to open file: " + fullPath.string()); - } - - file.seekg(0, std::ios::end); - const std::streamsize size = file.tellg(); - if (size < 0) { - throw std::runtime_error("failed to determine file size: " + fullPath.string()); - } - file.seekg(0, std::ios::beg); - - std::vector buffer(static_cast(size)); - if (size > 0 && !file.read(reinterpret_cast(buffer.data()), size)) { - throw std::runtime_error("failed to read file: " + fullPath.string()); - } - return buffer; -} diff --git a/destrum/src/FS/Manifest.cpp b/destrum/src/FS/Manifest.cpp index 9bd29b2..8b883ef 100644 --- a/destrum/src/FS/Manifest.cpp +++ b/destrum/src/FS/Manifest.cpp @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -27,14 +26,8 @@ namespace { } } -AssetManifest FS::LoadAssetManifest(const std::filesystem::path& manifestPath) { - std::ifstream f(manifestPath); - if (!f) { - throw std::runtime_error("Failed to open manifest: " + manifestPath.string()); - } - - nlohmann::json j; - f >> j; +AssetManifest FS::ParseAssetManifest(std::span bytes) { + const nlohmann::json j = nlohmann::json::parse(bytes.begin(), bytes.end()); AssetManifest manifest; manifest.version = j.value("version", 0); diff --git a/destrum/src/Graphics/ComputePipeline.cpp b/destrum/src/Graphics/ComputePipeline.cpp index 719ecd3..c912e89 100644 --- a/destrum/src/Graphics/ComputePipeline.cpp +++ b/destrum/src/Graphics/ComputePipeline.cpp @@ -2,10 +2,10 @@ #include #include -#include #include #include "volk.h" +#include "destrum/FS/AssetFS.h" #include "destrum/Util/GameState.h" #include "spdlog/spdlog.h" @@ -46,20 +46,8 @@ void ComputePipeline::DefaultPipelineConfigInfo(ComputePipelineConfigInfo& confi } std::vector ComputePipeline::readFile(const std::string& filename) { - std::ifstream file(filename, std::ios::ate | std::ios::binary); - - if (!file.is_open()) { - throw std::runtime_error("Failed to open file: " + filename); - } - - const size_t fileSize = static_cast(file.tellg()); - std::vector buffer(fileSize); - - file.seekg(0); - file.read(buffer.data(), static_cast(fileSize)); - file.close(); - - return buffer; + const std::vector bytes = AssetFS::GetInstance().ReadFile(filename); + return {bytes.begin(), bytes.end()}; } void ComputePipeline::CreateComputePipeline(const std::string& compPath, diff --git a/destrum/src/Graphics/ImageLoader.cpp b/destrum/src/Graphics/ImageLoader.cpp index 3f957da..b642657 100644 --- a/destrum/src/Graphics/ImageLoader.cpp +++ b/destrum/src/Graphics/ImageLoader.cpp @@ -12,6 +12,7 @@ #include "spdlog/spdlog.h" #include +#include ImageData::~ImageData() { @@ -69,10 +70,10 @@ namespace util std::filesystem::path cooked = p; cooked += TextureFile::Extension; - std::error_code error; - if (!std::filesystem::is_regular_file(cooked, error)) { + if (!AssetFS::GetInstance().Exists(cooked)) { return {}; } + std::error_code error; const auto sourceTime = std::filesystem::last_write_time(p, error); if (!error && sourceTime > std::filesystem::last_write_time(cooked, error)) { return {}; @@ -94,7 +95,7 @@ namespace util if (isCookedTextureExt(p)) { try { - TextureFile::Texture texture = TextureFile::ReadFromFile(p); + TextureFile::Texture texture = TextureFile::Read(AssetFS::GetInstance().ReadFile(p)); data.cookedPixels = std::move(texture.pixels); data.pixels = data.cookedPixels.data(); @@ -111,12 +112,21 @@ namespace util return data; } + std::vector bytes; + try { + bytes = AssetFS::GetInstance().ReadFile(p); + } catch (const std::exception& exception) { + spdlog::error("Failed to read image '{}': {}", p.string(), exception.what()); + return {}; + } + const int byteCount = static_cast(bytes.size()); + if (isExrExt(p)) { float* out = nullptr; int w = 0, h = 0; const char* err = nullptr; - const int ret = LoadEXR(&out, &w, &h, p.string().c_str(), &err); + const int ret = LoadEXRFromMemory(&out, &w, &h, bytes.data(), bytes.size(), &err); if (ret != TINYEXR_SUCCESS || !out || w <= 0 || h <= 0) { if (err) { spdlog::error("TinyEXR error: {}", err); @@ -140,10 +150,11 @@ namespace util // ---------- STB HDR (.hdr etc) ---------- data.shouldSTBFree = true; - if (stbi_is_hdr(p.string().c_str())) { + if (stbi_is_hdr_from_memory(bytes.data(), byteCount)) { data.hdr = true; - data.hdrPixels = stbi_loadf( - p.string().c_str(), + data.hdrPixels = stbi_loadf_from_memory( + bytes.data(), + byteCount, &data.width, &data.height, &data.comp, @@ -161,8 +172,9 @@ namespace util } // ---------- STB LDR (png/jpg/...) ---------- - data.pixels = stbi_load( - p.string().c_str(), + data.pixels = stbi_load_from_memory( + bytes.data(), + byteCount, &data.width, &data.height, &data.channels, diff --git a/destrum/src/Graphics/Pipeline.cpp b/destrum/src/Graphics/Pipeline.cpp index 98c5d42..4cf3cac 100644 --- a/destrum/src/Graphics/Pipeline.cpp +++ b/destrum/src/Graphics/Pipeline.cpp @@ -2,10 +2,10 @@ #include #include -#include #include #include "volk.h" +#include "destrum/FS/AssetFS.h" #include "destrum/Graphics/Util.h" #include "spdlog/spdlog.h" @@ -115,21 +115,8 @@ void Pipeline::DefaultPipelineConfigInfo(PipelineConfigInfo& configInfo) { } std::vector Pipeline::readFile(const std::string& filename) { - std::ifstream file(filename, std::ios::ate | std::ios::binary); - - if (!file.is_open()) { - throw std::runtime_error("Failed to open file: " + filename); - } - - const size_t fileSize = (size_t)file.tellg(); - std::vector buffer(fileSize); - - file.seekg(0); - file.read(buffer.data(), static_cast(fileSize)); - - file.close(); - - return buffer; + const std::vector bytes = AssetFS::GetInstance().ReadFile(filename); + return {bytes.begin(), bytes.end()}; } void Pipeline::CreateGraphicsPipeline(const std::string& vertPath, const std::string& fragPath, diff --git a/destrum/src/Graphics/Resources/Font.cpp b/destrum/src/Graphics/Resources/Font.cpp index 3e5a48f..1457c9b 100644 --- a/destrum/src/Graphics/Resources/Font.cpp +++ b/destrum/src/Graphics/Resources/Font.cpp @@ -2,8 +2,6 @@ #include #include -#include -#include #include #include @@ -13,6 +11,7 @@ #include +#include #include #include @@ -57,11 +56,7 @@ namespace { std::vector readFile(const std::filesystem::path& path) { - std::ifstream file(path, std::ios::binary); - if (!file) { - throw std::runtime_error("Font: failed to open " + path.string()); - } - return {std::istreambuf_iterator(file), std::istreambuf_iterator()}; + return AssetFS::GetInstance().ReadFile(path); } void check(FT_Error error, const char* what, const std::filesystem::path& path) diff --git a/destrum/src/Graphics/Util.cpp b/destrum/src/Graphics/Util.cpp index d957fd7..d726055 100644 --- a/destrum/src/Graphics/Util.cpp +++ b/destrum/src/Graphics/Util.cpp @@ -1,6 +1,7 @@ -#include +#include #include #include +#include #include @@ -401,22 +402,12 @@ vkutil::RenderInfo vkutil::createRenderingInfo(const RenderingInfoParams& params } VkShaderModule vkutil::loadShaderModule(const std::filesystem::path& path, VkDevice device) { - std::ifstream file(path, std::ios::ate | std::ios::binary); - if (!file.is_open()) { - throw std::runtime_error("Failed to open shader: " + path.string()); - } - - const auto fileSize = file.tellg(); - if (fileSize < 0 || fileSize % static_cast(sizeof(std::uint32_t)) != 0) { + const std::vector bytes = AssetFS::GetInstance().ReadFile(path); + if (bytes.size() % sizeof(std::uint32_t) != 0) { throw std::runtime_error("Shader file has an invalid size: " + path.string()); } - std::vector buffer(fileSize / sizeof(std::uint32_t)); - - file.seekg(0); - if (!file.read(reinterpret_cast(buffer.data()), fileSize)) { - throw std::runtime_error("Failed to read shader: " + path.string()); - } - file.close(); + std::vector buffer(bytes.size() / sizeof(std::uint32_t)); + std::memcpy(buffer.data(), bytes.data(), bytes.size()); auto info = VkShaderModuleCreateInfo{ .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, diff --git a/destrum/src/Serialization/ObjectSerialization.cpp b/destrum/src/Serialization/ObjectSerialization.cpp index 81227ad..dd8d924 100644 --- a/destrum/src/Serialization/ObjectSerialization.cpp +++ b/destrum/src/Serialization/ObjectSerialization.cpp @@ -151,7 +151,7 @@ namespace { fullPath = path; } - return std::filesystem::exists(fullPath); + return AssetFS::GetInstance().Exists(fullPath); } void CollectHierarchyRecursive(const GameObject& object, std::vector& out, diff --git a/destrum/src/Serialization/PrefabCache.cpp b/destrum/src/Serialization/PrefabCache.cpp index 8012d55..0a8ff52 100644 --- a/destrum/src/Serialization/PrefabCache.cpp +++ b/destrum/src/Serialization/PrefabCache.cpp @@ -14,12 +14,13 @@ std::shared_ptr PrefabCache::Get(std::string_view path) { return nullptr; } - std::error_code error; - const auto writeTime = std::filesystem::last_write_time(fullPath, error); - if (error) { + if (!AssetFS::GetInstance().Exists(fullPath)) { spdlog::error("Prefab file not found: {}", fullPath.string()); return nullptr; } + // Packed files have no write time, they can't change anyway + std::error_code error; + const auto writeTime = std::filesystem::last_write_time(fullPath, error); const std::string key = MakeKey(fullPath); if (const auto it = m_Prefabs.find(key); it != m_Prefabs.end() && it->second.writeTime == writeTime) { diff --git a/destrum/src/Serialization/PrefabSerializer.cpp b/destrum/src/Serialization/PrefabSerializer.cpp index 4ef013a..4622a9c 100644 --- a/destrum/src/Serialization/PrefabSerializer.cpp +++ b/destrum/src/Serialization/PrefabSerializer.cpp @@ -1,9 +1,11 @@ #include -#include +#include +#include #include +#include #include #include #include @@ -100,15 +102,17 @@ bool PrefabSerializer::Save(const GameObject& root, const std::filesystem::path& } std::optional PrefabSerializer::LoadFile(const std::filesystem::path& path) { - std::ifstream file(path); - if (!file.is_open()) { - spdlog::error("Failed to open prefab file for reading: {}", path.string()); + std::vector bytes; + try { + bytes = AssetFS::GetInstance().ReadFile(path); + } catch (const std::exception& exception) { + spdlog::error("Failed to open prefab file for reading: {} ({})", path.string(), exception.what()); return std::nullopt; } json prefab; try { - file >> prefab; + prefab = json::parse(bytes.begin(), bytes.end()); } catch (const std::exception& exception) { spdlog::error("Failed to parse prefab file {}: {}", path.string(), exception.what()); return std::nullopt; diff --git a/destrum/src/Serialization/SceneSerializer.cpp b/destrum/src/Serialization/SceneSerializer.cpp index 211adee..d604ed0 100644 --- a/destrum/src/Serialization/SceneSerializer.cpp +++ b/destrum/src/Serialization/SceneSerializer.cpp @@ -1,11 +1,12 @@ #include -#include +#include #include #include #include +#include #include #include #include @@ -152,15 +153,17 @@ SceneSerializer::LoadResult SceneSerializer::LoadSceneFile(const std::filesystem LoadResult result; RegisterEngineComponents(); - std::ifstream file(path); - if (!file.is_open()) { - result.errorMessage = "Failed to open scene file for reading: " + path.string(); + std::vector bytes; + try { + bytes = AssetFS::GetInstance().ReadFile(path); + } catch (const std::exception& exception) { + result.errorMessage = "Failed to open scene file for reading: " + path.string() + " (" + exception.what() + ")"; spdlog::error("{}", result.errorMessage); return result; } try { - file >> result.root; + result.root = json::parse(bytes.begin(), bytes.end()); } catch (const std::exception& exception) { result.errorMessage = "Failed to parse scene file " + path.string() + ": " + exception.what(); spdlog::error("{}", result.errorMessage); diff --git a/export.bat b/export.bat new file mode 100644 index 0000000..e1c15c3 --- /dev/null +++ b/export.bat @@ -0,0 +1,18 @@ +@echo off +rem Builds a Release export of Lightkeeper (build\export\Release) and opens the folder +setlocal +cd /d "%~dp0" + +if not exist build\CMakeCache.txt ( + cmake -S . -B build || goto :failed +) +cmake --build build --config Release --target Export --parallel || goto :failed + +explorer build\export\Release +exit /b 0 + +:failed +echo. +echo Export failed, see the output above. +pause +exit /b 1 diff --git a/lightkeeper/CMakeLists.txt b/lightkeeper/CMakeLists.txt index e747363..32195ef 100644 --- a/lightkeeper/CMakeLists.txt +++ b/lightkeeper/CMakeLists.txt @@ -28,9 +28,11 @@ set(GAME_SRC add_executable(lightkeeper ${GAME_SRC}) +# Release is what gets exported, players shouldn't get a console next to the game (it logs to a file too) set_target_properties(lightkeeper PROPERTIES CXX_STANDARD 20 CXX_EXTENSIONS OFF + WIN32_EXECUTABLE $ ) #target_add_extra_warnings(lightkeeper) @@ -108,6 +110,7 @@ add_dependencies(lightkeeper _internal_cook_game_assets) destrum_cook_engine_assets(lightkeeper "${CMAKE_CURRENT_BINARY_DIR}") destrum_copy_runtime_dlls(lightkeeper) +destrum_add_export(lightkeeper "${ASSETS_RUNTIME_DIR}") install(TARGETS lightkeeper RUNTIME DESTINATION bin) install(DIRECTORY "${ASSETS_RUNTIME_DIR}/" DESTINATION "bin/assets/game") diff --git a/lightkeeper/src/main.cpp b/lightkeeper/src/main.cpp index 950fe18..fd0572b 100644 --- a/lightkeeper/src/main.cpp +++ b/lightkeeper/src/main.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,15 +7,52 @@ #include #include +#include #include +#include #include #include "Lightkeeper.h" +namespace { + // Release builds have no console, so players only have this file to send when something goes wrong + std::filesystem::path addFileLog() { + char* prefPath = SDL_GetPrefPath("Destrum", "Lightkeeper"); + if (prefPath == nullptr) { + spdlog::warn("No folder to write the log to: {}", SDL_GetError()); + return {}; + } + const std::filesystem::path logPath = std::filesystem::path{prefPath} / "lightkeeper.log"; + SDL_free(prefPath); + + try { + spdlog::default_logger()->sinks().push_back( + std::make_shared(logPath.string(), true)); + } catch (const spdlog::spdlog_ex& exception) { + spdlog::warn("Can't write the log to {}: {}", logPath.string(), exception.what()); + return {}; + } + // Every line, so the log survives a hard crash + spdlog::flush_on(spdlog::level::trace); + return logPath; + } + + void showError(const std::string& message, const std::filesystem::path& logPath) { + std::string text = message; + if (!logPath.empty()) { + text += "\n\nThe log is at " + logPath.string(); + } + SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "Lightkeeper", text.c_str(), nullptr); + } +} + int main(int argc, char* argv[]) { + const std::filesystem::path logPath = addFileLog(); + if (SDL_Init(SDL_INIT_VIDEO) != 0) { - SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "Error", SDL_GetError(), nullptr); + spdlog::critical("SDL_Init failed: {}", SDL_GetError()); + showError(std::string{"Couldn't start SDL: "} + SDL_GetError(), logPath); return 1; } @@ -39,7 +77,8 @@ int main(int argc, char* argv[]) { .windowTitle = "Lightkeeper", .exeDir = exeDir, }); - if (!runGameOnly) { + // Exported builds only ship the cooked asset pack, the editor needs the source folders + if (!runGameOnly && !AssetFS::GetInstance().IsPacked()) { app.attachLayer(std::make_unique(app, EditorSettings{ .gameAssetsSource = LIGHTKEEPER_ASSETS_SRC_DIR, .scenePath = std::string{LightKeeper::MainScenePath}, @@ -49,6 +88,7 @@ int main(int argc, char* argv[]) { app.cleanup(); } catch (const std::exception& exception) { spdlog::critical("Lightkeeper terminated with an error: {}", exception.what()); + showError(std::string{"Lightkeeper ran into an error and has to close.\n\n"} + exception.what(), logPath); SDL_Quit(); return 1; } diff --git a/tests/assets_tests.cpp b/tests/assets_tests.cpp index da48ecf..201d301 100644 --- a/tests/assets_tests.cpp +++ b/tests/assets_tests.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -458,6 +459,116 @@ namespace { "loading a source image must use its cooked sibling"); } + bool packOpenThrows(const std::filesystem::path& path) + { + try { + (void)PackFile::Reader{path}; + } catch (const std::exception&) { + return true; + } + return false; + } + + void testPackFileRoundTrip() + { + const auto dir = std::filesystem::temp_directory_path() / "destrum_pack_file_test"; + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir); + std::ofstream{dir / "a.txt", std::ios::binary} << "hello"; + std::ofstream{dir / "empty.txt", std::ios::binary}; + + const auto pack = dir / "test.dpak"; + PackFile::Write({{"engine/a.txt", dir / "a.txt"}, {"game/empty.txt", dir / "empty.txt"}}, pack); + + const PackFile::Reader reader{pack}; + const PackFile::Entry* text = reader.Find("engine/a.txt"); + const PackFile::Entry* empty = reader.Find("game/empty.txt"); + check(reader.Entries().size() == 2 && text != nullptr && empty != nullptr && !reader.Find("a.txt"), + "packs must list every file by its full path"); + const auto bytes = reader.Read(*text); + check(std::string(bytes.begin(), bytes.end()) == "hello" && reader.Read(*empty).empty(), + "packed files must read back as written"); + + bool duplicateRejected = false; + try { + PackFile::Write({{"engine/a.txt", dir / "a.txt"}, {"engine/a.txt", dir / "a.txt"}}, dir / "dupe.dpak"); + } catch (const std::exception&) { + duplicateRejected = true; + } + check(duplicateRejected, "packs can't hold two files at one path"); + + std::ofstream{dir / "bad.dpak", std::ios::binary} << "DTEX not a pack"; + check(packOpenThrows(dir / "bad.dpak"), "files that aren't packs must be rejected"); + + const auto size = std::filesystem::file_size(pack); + std::filesystem::resize_file(pack, size - 4); + check(packOpenThrows(pack), "truncated packs must be rejected"); + + std::filesystem::remove_all(dir); + } + + void testAssetFSMountsPacks() + { + const auto root = std::filesystem::temp_directory_path() / "destrum_packed_assets_test"; + const auto staging = root / "staging"; + std::filesystem::remove_all(root); + std::filesystem::create_directories(staging); + + std::ofstream{staging / "notes.txt", std::ios::binary} << "packed"; + std::ofstream{staging / "notes.txt.meta", std::ios::binary} << R"({"guid":"0123456789abcdef0123456789abcdef"})"; + std::ofstream{staging / "model.fbx", std::ios::binary} << "not a model"; + ModelFile::WriteToFile(makeTestModelAsset(), staging / "model.fbx.dmodel"); + const std::vector rgba{255, 0, 0, 255}; + TextureFile::WriteToFile(TextureFile::FromRGBA8(1, 1, rgba), staging / "tex.png.dtex"); + std::ofstream{staging / "manifest.json", std::ios::binary} + << R"({"version":2,"assets":[{"src":"model.fbx","out":"model.fbx.dmodel","type":"model"}]})"; + + PackFile::Write({ + {"engine/manifest.json", staging / "manifest.json"}, + {"engine/model.fbx", staging / "model.fbx"}, + {"engine/model.fbx.dmodel", staging / "model.fbx.dmodel"}, + {"engine/tex.png.dtex", staging / "tex.png.dtex"}, + {"game/notes.txt", staging / "notes.txt"}, + {"game/notes.txt.meta", staging / "notes.txt.meta"}, + }, root / AssetFS::PackName); + + // A loose copy next to the pack must not be picked up + std::filesystem::create_directories(root / "assets" / "game"); + std::ofstream{root / "assets" / "game" / "notes.txt", std::ios::binary} << "loose"; + std::ofstream{root / "assets" / "game" / "loose.txt", std::ios::binary} << "loose"; + + auto& assetFs = AssetFS::GetInstance(); + assetFs.Init(root); + check(assetFs.IsPacked() && assetFs.GetMounts().size() == 2, "a pack next to the exe must mount every scheme in it"); + + const auto notes = assetFs.ReadBytes("game://notes.txt"); + check(std::string(notes.begin(), notes.end()) == "packed", "packed mounts must read from the pack"); + check(assetFs.GetFullPath("game://notes.txt") == (root / "assets" / "game" / "notes.txt").lexically_normal(), + "packed files must keep the full path they have unpacked"); + check(assetFs.Exists(assetFs.GetFullPath("game://notes.txt")) && + !assetFs.Exists(assetFs.GetFullPath("game://loose.txt")), + "only packed files exist under a packed mount"); + check(assetFs.ToVirtualPath(assetFs.GetFullPath("engine://tex.png")) == "engine://tex.png", + "packed full paths must map back to virtual paths"); + check(!assetFs.WriteFile("game://notes.txt", "changed"), "packed mounts are read only"); + + const auto model = AssetManager::GetInstance().LoadModel("engine://model.fbx"); + check(model.primitives.size() == 1, "cooked models must load from the pack"); + + const auto image = util::loadImage(assetFs.GetFullPath("engine://tex.png"), TextureIntent::ColorSrgb); + check(image.width == 1 && image.pixels != nullptr && std::memcmp(image.pixels, rgba.data(), rgba.size()) == 0, + "images must find their cooked sibling in the pack"); + + auto& database = AssetDatabase::GetInstance(); + database.Refresh(); + check(database.FindPath(*AssetGuid::Parse("0123456789abcdef0123456789abcdef")) == "game://notes.txt", + "asset ids must be read from packed meta files"); + database.Clear(); + + assetFs.Reset(); + std::filesystem::remove_all(root); + } + void testAssetReferenceCacheKeys() { const AssetReference reference{ @@ -555,6 +666,8 @@ void RunAssetsTests() testTextureFileRoundTrip(); testImageLoaderReadsCookedTextures(); testLdrTextureFile(); + testPackFileRoundTrip(); + testAssetFSMountsPacks(); testAssetReferenceCacheKeys(); testAssetGuids(); testAssetDatabaseFollowsMovedAssets(); diff --git a/tests/cooker_tests.cpp b/tests/cooker_tests.cpp index 4140428..7a518e3 100644 --- a/tests/cooker_tests.cpp +++ b/tests/cooker_tests.cpp @@ -9,6 +9,8 @@ #include +#include + #include "TestUtils.h" using namespace test; @@ -51,11 +53,10 @@ namespace { } } - int runChef(const CookDirs& dirs, const std::string& flags = {}) + int runChefArgs(const CookDirs& dirs, const std::string& args) { const fs::path log = dirs.root / "chef.log"; - std::string command = "\"" + std::string(DESTRUM_THECHEF_PATH) + "\" --input \"" + dirs.input.string() + - "\" --output \"" + dirs.output.string() + "\" " + flags + " > \"" + log.string() + + std::string command = "\"" + std::string(DESTRUM_THECHEF_PATH) + "\" " + args + " > \"" + log.string() + "\" 2>&1"; #ifdef _WIN32 // cmd.exe strips the outer quotes, the inner ones keep paths with spaces intact @@ -64,6 +65,11 @@ namespace { return std::system(command.c_str()); } + int runChef(const CookDirs& dirs, const std::string& flags = {}) + { + return runChefArgs(dirs, "--input \"" + dirs.input.string() + "\" --output \"" + dirs.output.string() + "\" " + flags); + } + void expectChef(const CookDirs& dirs, const std::string& flags, bool succeeds, const char* message) { if ((runChef(dirs, flags) == 0) != succeeds) { @@ -307,6 +313,45 @@ namespace { } } +namespace { + void testPackBundlesCookedOutputs() + { + const CookDirs dirs = makeDirs("destrum_cooker_pack"); + writeShaders(dirs); + writeText(dirs.input / "notes.txt", "hello"); + expectChef(dirs, "", true, "the first cook must succeed"); + writeText(dirs.output / "stray.txt", "not cooked"); + + const fs::path pack = dirs.root / "assets.dpak"; + const std::string packArgs = "--pack \"" + pack.string() + "\" --mount \"game=" + dirs.output.string() + "\""; + if (runChefArgs(dirs, packArgs) != 0) { + std::cerr << readFile(dirs.root / "chef.log") << '\n'; + check(false, "packing a cooked folder must succeed"); + return; + } + + const PackFile::Reader reader{pack}; + check(reader.Find("game/manifest.json") && reader.Find("game/shaders/plain.frag.spv") && + reader.Find("game/notes.txt"), + "packs must hold the manifest and every cooked output"); + check(!reader.Find("game/stray.txt") && !reader.Find("game/shaders/include/common.glsl"), + "packs must only hold what the manifest lists as outputs"); + const auto notes = reader.Read(*reader.Find("game/notes.txt")); + check(std::string(notes.begin(), notes.end()) == "hello", "packed files must keep their contents"); + + check(runChefArgs(dirs, "--list \"" + pack.string() + "\" --top 0") == 0 && + readFile(dirs.root / "chef.log").find("game/notes.txt") != std::string::npos, + "listing a pack must show its files"); + check(runChefArgs(dirs, "--list \"" + dirs.output.string() + "\"") == 0 && + readFile(dirs.root / "chef.log").find("Not in the manifest, won't be packed: 1 files") != std::string::npos, + "listing a cooked folder must point out files that won't be packed"); + + fs::remove(dirs.output / "notes.txt"); + check(runChefArgs(dirs, packArgs) != 0, "packing must fail when a listed output is missing"); + check(!fs::exists(pack.string() + ".tmp"), "a failed pack must not leave a temp file"); + } +} + void RunCookerTests() { testCookRecordsEveryOutput(); @@ -317,4 +362,5 @@ void RunCookerTests() testFailedCookKeepsLastGoodOutput(); testFallbacksAndStrictMode(); testValidateFindsMissingOutputs(); + testPackBundlesCookedOutputs(); }