brammie15/Destrum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add export / file packing

Browse Source
This commit is contained in:
brammie15
2026-10-30 15:42:08 +01:00
parent de7e489cc0
commit 0fbc023018
28 changed files with 782 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+3
View File
@@ -61,6 +61,9 @@ add_dependencies(CookAssets
_internal_cook_engine_assets
)
add_custom_target(Export)
add_dependencies(Export _internal_export_lightkeeper)
add_custom_target(ValidateAssets)
add_dependencies(ValidateAssets
_internal_validate_game_assets
Readme.md
+10
View File
@@ -118,6 +118,16 @@ cmake --build build --target ValidateShaders
`CookAssets` processes both engine and game assets. `CleanupAssets` removes generated runtime asset output.
`ValidateShaders` runs `spirv-val` over the cooked engine shaders when the Vulkan SDK provides it.
Export a folder other people can run (the game, its DLLs and every cooked asset packed into one `assets.dpak`):
```bash
cmake --build build --config Release --target Export
```
It ends up in `build/export/Release/lightkeeper/`, plus a `lightkeeper.zip` next to it. On Windows, `export.bat` does
the same and opens the folder. `TheChef --list build/export/Release/lightkeeper/assets.dpak` shows what takes up the
space. See `docs/CookedAssets.md`.
Install the application, engine library, public headers, and runtime assets:
```bash
TheChef
+1 -1
Submodule TheChef updated: 3d4b50529e...abcec12c90
destrum/CMakeLists.txt
+36
View File
@@ -4,6 +4,7 @@ add_subdirectory(third_party)
add_library(destrum_assets
"src/Assets/ModelFile.cpp"
"src/Assets/TextureFile.cpp"
"src/Assets/PackFile.cpp"
)
add_library(destrum::assets ALIAS destrum_assets)
@@ -297,6 +298,41 @@ function(destrum_cook_engine_assets GAME_TARGET GAME_OUTPUT_DIR)
install(DIRECTORY "${_engine_runtime}/" DESTINATION "bin/assets/engine")
endfunction()
# A folder (and zip) other people can run: the game exe, its DLLs and every cooked asset packed into assets.dpak
function(destrum_add_export GAME_TARGET GAME_RUNTIME_DIR)
set(_engine_runtime "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/assets_runtime")
set(_export_root "${CMAKE_BINARY_DIR}/export/$<CONFIG>")
set(_export_dir "${_export_root}/${GAME_TARGET}")
# The release VC++ runtime ships app-local, debug builds only run where Visual Studio is installed
set(_system_libs)
if (MSVC)
set(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP TRUE)
include(InstallRequiredSystemLibraries)
set(_system_libs "$<$<NOT:$<CONFIG:Debug>>:${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS}>")
endif()
add_custom_target(_internal_export_${GAME_TARGET}
COMMAND ${CMAKE_COMMAND} -E rm -rf "${_export_dir}" "${_export_dir}.zip"
COMMAND ${CMAKE_COMMAND} -E make_directory "${_export_dir}"
COMMAND TheChef
--pack "${_export_dir}/assets.dpak"
--mount "engine=${_engine_runtime}"
--mount "game=${GAME_RUNTIME_DIR}"
COMMAND TheChef --list "${_export_dir}/assets.dpak" --top 10
COMMAND ${CMAKE_COMMAND} -E copy -t "${_export_dir}"
"$<TARGET_FILE:${GAME_TARGET}>"
"$<TARGET_RUNTIME_DLLS:${GAME_TARGET}>"
"${_system_libs}"
COMMAND ${CMAKE_COMMAND} -E chdir "${_export_root}"
${CMAKE_COMMAND} -E tar cf "${GAME_TARGET}.zip" --format=zip "${GAME_TARGET}"
COMMAND ${CMAKE_COMMAND} -E echo "Exported ${GAME_TARGET} to ${_export_dir}"
COMMAND_EXPAND_LISTS
VERBATIM
)
add_dependencies(_internal_export_${GAME_TARGET} ${GAME_TARGET})
endfunction()
# OpenAL is a DLL, every exe that links the engine needs a copy next to it
function(destrum_copy_runtime_dlls TARGET)
if (WIN32)
destrum/include/destrum/Assets/PackFile.h
+46
View File
@@ -0,0 +1,46 @@
#ifndef DESTRUM_PACKFILE_H
#define DESTRUM_PACKFILE_H
#include <cstdint>
#include <filesystem>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
namespace PackFile {
inline constexpr std::uint32_t Version = 1;
inline constexpr std::string_view Extension = ".dpak";
struct Entry {
std::string path;
std::uint64_t offset{0};
std::uint64_t size{0};
};
struct Source {
std::string path;
std::filesystem::path file;
};
// Paths are "<scheme>/<relative path>", e.g. "engine/shaders/mesh.vert.spv"
void Write(const std::vector<Source>& files, const std::filesystem::path& path);
class Reader {
public:
explicit Reader(std::filesystem::path path);
[[nodiscard]] const Entry* Find(std::string_view path) const;
[[nodiscard]] std::vector<std::uint8_t> Read(const Entry& entry) const;
[[nodiscard]] const std::vector<Entry>& Entries() const { return m_Entries; }
[[nodiscard]] const std::filesystem::path& Path() const { return m_Path; }
private:
std::filesystem::path m_Path;
std::vector<Entry> m_Entries;
std::unordered_map<std::string, std::size_t> m_ByPath;
};
}
#endif // DESTRUM_PACKFILE_H
destrum/include/destrum/FS/AssetFS.h
+12 -1
View File
@@ -3,12 +3,14 @@
#include <filesystem>
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include <destrum/Singleton.h>
#include <destrum/Assets/PackFile.h>
#include <destrum/FS/Manifest.h>
@@ -18,17 +20,26 @@ struct FSMount {
std::filesystem::path root;
std::optional<AssetManifest> manifest;
std::optional<std::filesystem::path> sourceRoot;
std::shared_ptr<const PackFile::Reader> pack;
};
class AssetFS final: public Singleton<AssetFS> {
public:
static constexpr std::string_view PackName = "assets.dpak";
void Init(std::filesystem::path exeDir);
void Reset();
void Mount(std::string scheme, std::filesystem::path root);
void MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root);
void ReloadManifest(std::string_view scheme);
std::vector<uint8_t> ReadBytes(std::string_view vpath);
[[nodiscard]] std::vector<uint8_t> ReadFile(const std::filesystem::path& fullPath) const;
[[nodiscard]] bool Exists(const std::filesystem::path& fullPath) const;
[[nodiscard]] std::vector<std::filesystem::path> ListFiles(const FSMount& mount) const;
[[nodiscard]] bool IsPacked() const;
[[nodiscard]] std::filesystem::path GetFullPath(std::string_view vpath) const;
[[nodiscard]] std::filesystem::path GetCookedPathForFile(std::string_view vpath) const;
@@ -43,7 +54,7 @@ public:
bool WriteFile(std::string_view vpath, std::string_view contents);
private:
static std::vector<uint8_t> ReadFile(const std::filesystem::path& fullPath);
void AddMount(std::string scheme, std::filesystem::path root, std::shared_ptr<const PackFile::Reader> pack);
std::vector<FSMount> mounts;
destrum/include/destrum/FS/Manifest.h
+2 -1
View File
@@ -4,6 +4,7 @@
#include <cstdint>
#include <filesystem>
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <unordered_map>
@@ -29,7 +30,7 @@ struct AssetManifest {
};
namespace FS {
AssetManifest LoadAssetManifest(const std::filesystem::path& manifestPath);
AssetManifest ParseAssetManifest(std::span<const std::uint8_t> bytes);
}
#endif //MANIFEST_H
destrum/include/destrum/Util/ModelDoc.h
+14 -3
View File
@@ -20,6 +20,7 @@
//
// Link against: assimp
#include <assimp/IOSystem.hpp>
#include <assimp/Importer.hpp>
#include <assimp/material.h>
#include <assimp/postprocess.h>
@@ -43,6 +44,7 @@
#include <filesystem>
#include <iostream>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
#include <unordered_map>
@@ -998,8 +1000,12 @@ static void CollectMergedPerNodeMeshes(const aiScene* scene,
// ─── Public API ───────────────────────────────────────────────────────────────
static ModelAsset LoadModel(const std::string& path, const LoadOptions& options = {}) {
// The importer owns io, without one it reads from disk
static ModelAsset LoadModel(const std::string& path, const LoadOptions& options = {},
std::unique_ptr<Assimp::IOSystem> io = nullptr) {
Assimp::Importer importer;
if (io)
importer.SetIOHandler(io.release());
const aiScene* scene = LoadScene(importer, path, options);
ModelAsset model{};
@@ -1084,10 +1090,11 @@ static const aiScene* LoadAnimationScene(
Assimp::Importer& importer,
const std::string& path
) {
if (!std::filesystem::exists(path)) {
if (!importer.GetIOHandler()->Exists(path.c_str())) {
throw std::runtime_error("Animation file does not exist: " + path);
}
if (importer.IsDefaultIOHandler()) {
if (!std::filesystem::is_regular_file(path)) {
throw std::runtime_error("Animation path is not a regular file: " + path);
}
@@ -1095,6 +1102,7 @@ static const aiScene* LoadAnimationScene(
if (std::filesystem::file_size(path) == 0) {
throw std::runtime_error("Animation file is empty: " + path);
}
}
// Important: do not use mesh post-process flags for animation-only files.
const unsigned int flags = 0;
@@ -1141,9 +1149,12 @@ static const aiScene* LoadAnimationScene(
static std::vector<SkeletalAnimation> LoadAnimationClips(
const std::string& path,
const Skeleton& targetSkeleton
const Skeleton& targetSkeleton,
std::unique_ptr<Assimp::IOSystem> io = nullptr
) {
Assimp::Importer importer;
if (io)
importer.SetIOHandler(io.release());
const aiScene* scene = LoadAnimationScene(importer, path);
auto clips = LoadAnimations(scene, targetSkeleton);
destrum/src/Assets/AssetDatabase.cpp
+9 -14
View File
@@ -3,7 +3,6 @@
#include <charconv>
#include <filesystem>
#include <format>
#include <fstream>
#include <random>
#include <spdlog/spdlog.h>
@@ -19,12 +18,9 @@ namespace {
[[nodiscard]] std::optional<AssetGuid> ReadMetaFile(const std::filesystem::path& path)
{
std::ifstream file(path);
if (!file) {
return std::nullopt;
}
try {
const auto meta = nlohmann::json::parse(file);
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(path);
const auto meta = nlohmann::json::parse(bytes.begin(), bytes.end());
return AssetGuid::Parse(meta.at("guid").get<std::string>());
} catch (const std::exception& exception) {
spdlog::warn("Ignoring broken asset meta file {}: {}", path.string(), exception.what());
@@ -83,14 +79,14 @@ void AssetDatabase::Clear()
void AssetDatabase::Refresh()
{
Clear();
for (const FSMount& mount : AssetFS::GetInstance().GetMounts()) {
std::error_code error;
for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) {
if (!entry.is_regular_file(error) || !AssetTypes::IsMeta(entry.path().string())) {
const auto& assetFs = AssetFS::GetInstance();
for (const FSMount& mount : assetFs.GetMounts()) {
for (const std::filesystem::path& file : assetFs.ListFiles(mount)) {
if (!AssetTypes::IsMeta(file.string())) {
continue;
}
if (const auto guid = ReadMetaFile(entry.path())) {
std::filesystem::path asset = entry.path();
if (const auto guid = ReadMetaFile(file)) {
std::filesystem::path asset = file;
asset.replace_extension();
Register(mount.scheme + "://" + asset.lexically_relative(mount.root).generic_string(), *guid);
}
@@ -131,8 +127,7 @@ std::optional<AssetGuid> AssetDatabase::GuidForSave(std::string_view path)
}
auto& assetFs = AssetFS::GetInstance();
std::error_code error;
if (!std::filesystem::is_regular_file(assetFs.GetFullPath(path), error)) {
if (!assetFs.Exists(assetFs.GetFullPath(path))) {
return std::nullopt;
}
destrum/src/Assets/AssetFSIOSystem.h
+45
View File
@@ -0,0 +1,45 @@
#ifndef DESTRUM_ASSETFSIOSYSTEM_H
#define DESTRUM_ASSETFSIOSYSTEM_H
#include <cstdint>
#include <cstring>
#include <string_view>
#include <vector>
#include <assimp/IOSystem.hpp>
#include <assimp/MemoryIOWrapper.h>
#include <destrum/FS/AssetFS.h>
class AssetFSIOSystem final : public Assimp::IOSystem {
public:
bool Exists(const char* file) const override {
return AssetFS::GetInstance().Exists(file);
}
char getOsSeparator() const override {
return '/';
}
Assimp::IOStream* Open(const char* file, const char* mode) override {
if (std::string_view{mode}.find_first_of("wa+") != std::string_view::npos) {
return nullptr;
}
std::vector<std::uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(file);
} catch (const std::exception&) {
return nullptr;
}
auto* buffer = new std::uint8_t[bytes.size()];
std::memcpy(buffer, bytes.data(), bytes.size());
return new Assimp::MemoryIOStream(buffer, bytes.size(), true);
}
void Close(Assimp::IOStream* stream) override {
delete stream;
}
};
#endif // DESTRUM_ASSETFSIOSYSTEM_H
destrum/src/Assets/AssetManager.cpp
+15 -3
View File
@@ -1,11 +1,14 @@
#include <destrum/Assets/AssetManager.h>
#include <memory>
#include <optional>
#include <stdexcept>
#include <destrum/Assets/ModelFile.h>
#include <destrum/FS/AssetFS.h>
#include "AssetFSIOSystem.h"
namespace {
std::string ToReference(std::string_view reference) {
if (reference.empty()) {
@@ -30,6 +33,14 @@ namespace {
return std::nullopt;
}
// Assimp opens files itself, so with packed assets it has to go through AssetFS
std::unique_ptr<Assimp::IOSystem> MakeIOSystem() {
if (!AssetFS::GetInstance().IsPacked()) {
return nullptr;
}
return std::make_unique<AssetFSIOSystem>();
}
// Cooked texture paths are relative to the model file, imported ones are absolute
void ResolveTexturePaths(ModelDoc::ModelAsset& model, const std::filesystem::path& modelDir) {
for (ModelDoc::MaterialInfo& material : model.materials) {
@@ -95,7 +106,7 @@ ModelDoc::ModelAsset AssetManager::LoadCookedOrImport(
const ModelDoc::LoadOptions& options) const
{
if (const auto cookedPath = FindCookedModel(reference)) {
ModelDoc::ModelAsset model = ModelFile::ReadFromFile(*cookedPath);
ModelDoc::ModelAsset model = ModelFile::Read(AssetFS::GetInstance().ReadFile(*cookedPath));
if (ModelFile::SupportsLoadOptions(model, options)) {
ModelFile::ApplyLoadOptions(model, options);
ResolveTexturePaths(model, cookedPath->parent_path());
@@ -105,7 +116,7 @@ ModelDoc::ModelAsset AssetManager::LoadCookedOrImport(
spdlog::debug("Importing {}, its cooked model doesn't match the requested load options", reference);
}
return ModelDoc::LoadModel(Resolve(reference).generic_string(), options);
return ModelDoc::LoadModel(Resolve(reference).generic_string(), options, MakeIOSystem());
}
std::vector<SkeletalAnimation> AssetManager::LoadAnimationClips(
@@ -115,7 +126,8 @@ std::vector<SkeletalAnimation> AssetManager::LoadAnimationClips(
const std::string sourceReference = ToReference(reference);
auto clips = ModelDoc::LoadAnimationClips(
Resolve(sourceReference).generic_string(),
targetSkeleton);
targetSkeleton,
MakeIOSystem());
for (std::size_t index = 0; index < clips.size(); ++index) {
auto& clip = clips[index];
clip.assetReference.path = sourceReference;
destrum/src/Assets/PackFile.cpp
+126
View File
@@ -0,0 +1,126 @@
#include <destrum/Assets/PackFile.h>
#include <set>
#include "BinaryStream.h"
namespace {
using AssetBinary::BinaryReader;
using AssetBinary::BinaryWriter;
constexpr AssetBinary::Magic Magic{'D', 'P', 'A', 'K'};
constexpr std::string_view Kind = "pack";
constexpr std::size_t HeaderSize = sizeof(AssetBinary::Magic) + sizeof(std::uint32_t) + sizeof(std::uint64_t);
}
void PackFile::Write(const std::vector<Source>& files, const std::filesystem::path& path) {
std::ofstream out(path, std::ios::binary | std::ios::trunc);
if (!out) {
throw std::runtime_error("Failed to open file for writing: " + path.string());
}
BinaryWriter header;
header.WriteHeader(Magic, Version);
header.Write(std::uint64_t{0});
const std::vector<std::uint8_t> headerBytes = header.TakeBytes();
out.write(reinterpret_cast<const char*>(headerBytes.data()), static_cast<std::streamsize>(headerBytes.size()));
std::set<std::string> seen;
BinaryWriter index;
index.WriteCount(files.size());
for (const Source& file : files) {
if (!seen.insert(file.path).second) {
throw std::runtime_error("Pack has two files at " + file.path);
}
std::ifstream in(file.file, std::ios::binary);
if (!in) {
throw std::runtime_error("Failed to open file: " + file.file.string());
}
const auto offset = static_cast<std::uint64_t>(out.tellp());
// Inserting an empty stream buffer sets failbit
if (std::filesystem::file_size(file.file) > 0) {
out << in.rdbuf();
}
const auto size = static_cast<std::uint64_t>(out.tellp()) - offset;
index.WriteString(file.path);
index.Write(offset);
index.Write(size);
}
const auto indexOffset = static_cast<std::uint64_t>(out.tellp());
const std::vector<std::uint8_t> indexBytes = index.TakeBytes();
out.write(reinterpret_cast<const char*>(indexBytes.data()), static_cast<std::streamsize>(indexBytes.size()));
out.seekp(static_cast<std::streamoff>(HeaderSize - sizeof(std::uint64_t)));
out.write(reinterpret_cast<const char*>(&indexOffset), sizeof(indexOffset));
if (!out) {
throw std::runtime_error("Failed to write file: " + path.string());
}
}
PackFile::Reader::Reader(std::filesystem::path path)
: m_Path(std::move(path)) {
std::ifstream file(m_Path, std::ios::binary | std::ios::ate);
if (!file) {
throw std::runtime_error("Failed to open file: " + m_Path.string());
}
const auto fileSize = static_cast<std::uint64_t>(file.tellg());
std::array<std::uint8_t, HeaderSize> headerBytes{};
file.seekg(0);
file.read(reinterpret_cast<char*>(headerBytes.data()), static_cast<std::streamsize>(headerBytes.size()));
if (!file) {
throw std::runtime_error("Not a " + std::string{Kind} + " file: " + m_Path.string());
}
BinaryReader header{headerBytes};
header.ReadHeader(Magic, Version, Kind);
const auto indexOffset = header.Read<std::uint64_t>();
if (indexOffset < HeaderSize || indexOffset > fileSize) {
throw std::runtime_error("Pack file is corrupt: index is out of range");
}
std::vector<std::uint8_t> indexBytes(static_cast<std::size_t>(fileSize - indexOffset));
file.seekg(static_cast<std::streamoff>(indexOffset));
file.read(reinterpret_cast<char*>(indexBytes.data()), static_cast<std::streamsize>(indexBytes.size()));
if (!file) {
throw std::runtime_error("Failed to read file: " + m_Path.string());
}
BinaryReader index{indexBytes};
m_Entries.resize(index.ReadCount(3 * sizeof(std::uint64_t)));
for (std::size_t i = 0; i < m_Entries.size(); ++i) {
Entry& entry = m_Entries[i];
entry.path = index.ReadString();
entry.offset = index.Read<std::uint64_t>();
entry.size = index.Read<std::uint64_t>();
if (entry.offset < HeaderSize || entry.offset > indexOffset || entry.size > indexOffset - entry.offset) {
throw std::runtime_error("Pack file is corrupt: " + entry.path + " is out of range");
}
if (!m_ByPath.emplace(entry.path, i).second) {
throw std::runtime_error("Pack file is corrupt: " + entry.path + " is listed twice");
}
}
index.ExpectEnd(Kind);
}
const PackFile::Entry* PackFile::Reader::Find(std::string_view path) const {
const auto it = m_ByPath.find(std::string{path});
return it != m_ByPath.end() ? &m_Entries[it->second] : nullptr;
}
// Opens its own stream so loads on worker threads don't share a file position
std::vector<std::uint8_t> PackFile::Reader::Read(const Entry& entry) const {
std::ifstream file(m_Path, std::ios::binary);
if (!file) {
throw std::runtime_error("Failed to open file: " + m_Path.string());
}
std::vector<std::uint8_t> bytes(static_cast<std::size_t>(entry.size));
file.seekg(static_cast<std::streamoff>(entry.offset));
file.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
if (!file) {
throw std::runtime_error("Failed to read " + entry.path + " from " + m_Path.string());
}
return bytes;
}
destrum/src/FS/AssetFS.cpp
+172 -37
View File
@@ -1,5 +1,6 @@
#include <destrum/FS/AssetFS.h>
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <stdexcept>
@@ -97,6 +98,79 @@ namespace {
return nullptr;
}
[[nodiscard]] std::optional<std::filesystem::path> RelativeToMount(const FSMount& mount,
const std::filesystem::path& fullPath) {
const std::filesystem::path relative = fullPath.lexically_normal().lexically_relative(mount.root);
if (relative.empty() || relative.is_absolute() || relative == "." ||
relative == ".." || relative.generic_string().starts_with("../")) {
return std::nullopt;
}
return relative;
}
struct PackLocation {
const FSMount* mount;
std::string key;
};
// Everything under a packed mount comes from the pack, even files that also exist on disk
[[nodiscard]] std::optional<PackLocation> FindInPack(const std::vector<FSMount>& mounts,
const std::filesystem::path& fullPath) {
for (const FSMount& mount : mounts) {
if (!mount.pack) {
continue;
}
if (const auto relative = RelativeToMount(mount, fullPath)) {
return PackLocation{&mount, mount.scheme + "/" + relative->generic_string()};
}
}
return std::nullopt;
}
[[nodiscard]] std::vector<uint8_t> ReadDiskFile(const std::filesystem::path& fullPath) {
std::ifstream file(fullPath, std::ios::binary);
if (!file) {
throw std::runtime_error("failed to open file: " + fullPath.string());
}
file.seekg(0, std::ios::end);
const std::streamsize size = file.tellg();
if (size < 0) {
throw std::runtime_error("failed to determine file size: " + fullPath.string());
}
file.seekg(0, std::ios::beg);
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
if (size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("failed to read file: " + fullPath.string());
}
return buffer;
}
[[nodiscard]] std::optional<AssetManifest> LoadManifest(const FSMount& mount) {
const std::filesystem::path path = mount.root / "manifest.json";
std::vector<uint8_t> bytes;
if (mount.pack) {
const PackFile::Entry* entry = mount.pack->Find(mount.scheme + "/manifest.json");
if (entry == nullptr) {
return std::nullopt;
}
bytes = mount.pack->Read(*entry);
} else {
std::error_code error;
if (!std::filesystem::is_regular_file(path, error)) {
return std::nullopt;
}
bytes = ReadDiskFile(path);
}
try {
return FS::ParseAssetManifest(bytes);
} catch (const std::exception& exception) {
throw std::runtime_error("invalid manifest " + path.string() + ": " + exception.what());
}
}
}
void AssetFS::Init(std::filesystem::path exeDir) {
@@ -106,8 +180,14 @@ void AssetFS::Init(std::filesystem::path exeDir) {
const std::size_t originalMountCount = mounts.size();
try {
const std::filesystem::path packPath = exeDir / PackName;
std::error_code error;
if (std::filesystem::is_regular_file(packPath, error)) {
MountPack(packPath, exeDir / "assets");
} else {
Mount("engine", exeDir / "assets/engine");
Mount("game", exeDir / "assets/game");
}
initialized = true;
} catch (...) {
mounts.resize(originalMountCount);
@@ -121,6 +201,40 @@ void AssetFS::Reset() {
}
void AssetFS::Mount(std::string scheme, std::filesystem::path root) {
AddMount(std::move(scheme), std::move(root), nullptr);
}
// Each scheme in the pack mounts at root/<scheme>, so full paths look the same as with the unpacked folders
void AssetFS::MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root) {
auto pack = std::make_shared<const PackFile::Reader>(packPath);
std::vector<std::string> schemes;
for (const PackFile::Entry& entry : pack->Entries()) {
const std::size_t separator = entry.path.find('/');
if (separator == std::string::npos || separator == 0) {
throw std::runtime_error("asset pack entry has no scheme: " + entry.path);
}
std::string scheme = entry.path.substr(0, separator);
if (std::ranges::find(schemes, scheme) == schemes.end()) {
schemes.push_back(std::move(scheme));
}
}
const std::size_t originalMountCount = mounts.size();
try {
for (std::string& scheme : schemes) {
std::filesystem::path schemeRoot = root / scheme;
AddMount(std::move(scheme), std::move(schemeRoot), pack);
}
} catch (...) {
mounts.resize(originalMountCount);
throw;
}
spdlog::info("Mounted asset pack {} ({} files)", packPath.string(), pack->Entries().size());
}
void AssetFS::AddMount(std::string scheme, std::filesystem::path root,
std::shared_ptr<const PackFile::Reader> pack) {
if (scheme.empty() || scheme.find_first_of("/\\:") != std::string::npos) {
throw std::runtime_error("asset mount scheme is invalid");
}
@@ -131,12 +245,8 @@ void AssetFS::Mount(std::string scheme, std::filesystem::path root) {
FSMount mount;
mount.scheme = std::move(scheme);
mount.root = std::filesystem::absolute(std::move(root)).lexically_normal();
const std::filesystem::path manifestPath = mount.root / "manifest.json";
std::error_code error;
if (std::filesystem::is_regular_file(manifestPath, error)) {
mount.manifest = FS::LoadAssetManifest(manifestPath);
}
mount.pack = std::move(pack);
mount.manifest = LoadManifest(mount);
spdlog::debug("Mounting assetfs scheme '{}' to root '{}'", mount.scheme, mount.root.string());
mounts.push_back(std::move(mount));
@@ -147,12 +257,8 @@ void AssetFS::ReloadManifest(std::string_view scheme) {
if (mount.scheme != scheme) {
continue;
}
const std::filesystem::path manifestPath = mount.root / "manifest.json";
std::error_code error;
mount.manifest.reset();
if (std::filesystem::is_regular_file(manifestPath, error)) {
mount.manifest = FS::LoadAssetManifest(manifestPath);
}
mount.manifest = LoadManifest(mount);
return;
}
throw std::runtime_error("asset mount not found: " + std::string{scheme});
@@ -162,6 +268,51 @@ std::vector<uint8_t> AssetFS::ReadBytes(std::string_view vpath) {
return ReadFile(GetFullPath(vpath));
}
std::vector<uint8_t> AssetFS::ReadFile(const std::filesystem::path& fullPath) const {
if (const auto location = FindInPack(mounts, fullPath)) {
const PackFile::Entry* entry = location->mount->pack->Find(location->key);
if (entry == nullptr) {
throw std::runtime_error("failed to open file: " + fullPath.string() + " is not in " +
location->mount->pack->Path().string());
}
return location->mount->pack->Read(*entry);
}
return ReadDiskFile(fullPath);
}
bool AssetFS::Exists(const std::filesystem::path& fullPath) const {
if (const auto location = FindInPack(mounts, fullPath)) {
return location->mount->pack->Find(location->key) != nullptr;
}
std::error_code error;
return std::filesystem::is_regular_file(fullPath, error);
}
std::vector<std::filesystem::path> AssetFS::ListFiles(const FSMount& mount) const {
std::vector<std::filesystem::path> files;
if (mount.pack) {
const std::string prefix = mount.scheme + "/";
for (const PackFile::Entry& entry : mount.pack->Entries()) {
if (entry.path.starts_with(prefix)) {
files.push_back((mount.root / entry.path.substr(prefix.size())).lexically_normal());
}
}
return files;
}
std::error_code error;
for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) {
if (entry.is_regular_file(error)) {
files.push_back(entry.path());
}
}
return files;
}
bool AssetFS::IsPacked() const {
return std::ranges::any_of(mounts, [](const FSMount& mount) { return mount.pack != nullptr; });
}
std::filesystem::path AssetFS::GetFullPath(std::string_view vpath) const {
if (!initialized) {
throw std::runtime_error("AssetFS is not initialized");
@@ -194,6 +345,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons
const std::filesystem::path cookedRelative = ValidateRelativePath(*asset->out);
const std::filesystem::path cookedPath = ResolvePath(*mount, cookedRelative);
if (mount->pack && Exists(cookedPath)) {
return cookedPath;
}
std::error_code error;
if (std::filesystem::is_regular_file(cookedPath, error)) {
error.clear();
@@ -218,14 +373,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons
}
std::optional<std::string> AssetFS::ToVirtualPath(const std::filesystem::path& fullPath) const {
const std::filesystem::path normalized = fullPath.lexically_normal();
for (const FSMount& mount : mounts) {
const std::filesystem::path relative = normalized.lexically_relative(mount.root);
if (relative.empty() || relative.is_absolute() || relative == "." ||
relative == ".." || relative.generic_string().starts_with("../")) {
continue;
if (const auto relative = RelativeToMount(mount, fullPath)) {
return mount.scheme + "://" + relative->generic_string();
}
return mount.scheme + "://" + relative.generic_string();
}
return std::nullopt;
}
@@ -251,6 +402,10 @@ std::optional<std::filesystem::path> AssetFS::GetSourcePath(std::string_view vpa
bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) {
std::vector<std::filesystem::path> targets{GetFullPath(vpath)};
if (FindInPack(mounts, targets.front())) {
spdlog::error("Can't write {}, it's in a read only asset pack", vpath);
return false;
}
if (auto source = GetSourcePath(vpath)) {
targets.push_back(std::move(*source));
}
@@ -276,23 +431,3 @@ bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) {
}
return true;
}
std::vector<uint8_t> AssetFS::ReadFile(const std::filesystem::path& fullPath) {
std::ifstream file(fullPath, std::ios::binary);
if (!file) {
throw std::runtime_error("failed to open file: " + fullPath.string());
}
file.seekg(0, std::ios::end);
const std::streamsize size = file.tellg();
if (size < 0) {
throw std::runtime_error("failed to determine file size: " + fullPath.string());
}
file.seekg(0, std::ios::beg);
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
if (size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("failed to read file: " + fullPath.string());
}
return buffer;
}
destrum/src/FS/Manifest.cpp
+2 -9
View File
@@ -1,7 +1,6 @@
#include <filesystem>
#include <destrum/FS/Manifest.h>
#include <fstream>
#include <stdexcept>
#include <nlohmann/json.hpp>
@@ -27,14 +26,8 @@ namespace {
}
}
AssetManifest FS::LoadAssetManifest(const std::filesystem::path& manifestPath) {
std::ifstream f(manifestPath);
if (!f) {
throw std::runtime_error("Failed to open manifest: " + manifestPath.string());
}
nlohmann::json j;
f >> j;
AssetManifest FS::ParseAssetManifest(std::span<const std::uint8_t> bytes) {
const nlohmann::json j = nlohmann::json::parse(bytes.begin(), bytes.end());
AssetManifest manifest;
manifest.version = j.value("version", 0);
destrum/src/Graphics/ComputePipeline.cpp
+3 -15
View File
@@ -2,10 +2,10 @@
#include <cassert>
#include <filesystem>
#include <fstream>
#include <iostream>
#include "volk.h"
#include "destrum/FS/AssetFS.h"
#include "destrum/Util/GameState.h"
#include "spdlog/spdlog.h"
@@ -46,20 +46,8 @@ void ComputePipeline::DefaultPipelineConfigInfo(ComputePipelineConfigInfo& confi
}
std::vector<char> ComputePipeline::readFile(const std::string& filename) {
std::ifstream file(filename, std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("Failed to open file: " + filename);
}
const size_t fileSize = static_cast<size_t>(file.tellg());
std::vector<char> buffer(fileSize);
file.seekg(0);
file.read(buffer.data(), static_cast<std::streamsize>(fileSize));
file.close();
return buffer;
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(filename);
return {bytes.begin(), bytes.end()};
}
void ComputePipeline::CreateComputePipeline(const std::string& compPath,
destrum/src/Graphics/ImageLoader.cpp
+21 -9
View File
@@ -12,6 +12,7 @@
#include "spdlog/spdlog.h"
#include <destrum/Assets/TextureFile.h>
#include <destrum/FS/AssetFS.h>
ImageData::~ImageData()
{
@@ -69,10 +70,10 @@ namespace util
std::filesystem::path cooked = p;
cooked += TextureFile::Extension;
std::error_code error;
if (!std::filesystem::is_regular_file(cooked, error)) {
if (!AssetFS::GetInstance().Exists(cooked)) {
return {};
}
std::error_code error;
const auto sourceTime = std::filesystem::last_write_time(p, error);
if (!error && sourceTime > std::filesystem::last_write_time(cooked, error)) {
return {};
@@ -94,7 +95,7 @@ namespace util
if (isCookedTextureExt(p)) {
try {
TextureFile::Texture texture = TextureFile::ReadFromFile(p);
TextureFile::Texture texture = TextureFile::Read(AssetFS::GetInstance().ReadFile(p));
data.cookedPixels = std::move(texture.pixels);
data.pixels = data.cookedPixels.data();
@@ -111,12 +112,21 @@ namespace util
return data;
}
std::vector<uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(p);
} catch (const std::exception& exception) {
spdlog::error("Failed to read image '{}': {}", p.string(), exception.what());
return {};
}
const int byteCount = static_cast<int>(bytes.size());
if (isExrExt(p)) {
float* out = nullptr;
int w = 0, h = 0;
const char* err = nullptr;
const int ret = LoadEXR(&out, &w, &h, p.string().c_str(), &err);
const int ret = LoadEXRFromMemory(&out, &w, &h, bytes.data(), bytes.size(), &err);
if (ret != TINYEXR_SUCCESS || !out || w <= 0 || h <= 0) {
if (err) {
spdlog::error("TinyEXR error: {}", err);
@@ -140,10 +150,11 @@ namespace util
// ---------- STB HDR (.hdr etc) ----------
data.shouldSTBFree = true;
if (stbi_is_hdr(p.string().c_str())) {
if (stbi_is_hdr_from_memory(bytes.data(), byteCount)) {
data.hdr = true;
data.hdrPixels = stbi_loadf(
p.string().c_str(),
data.hdrPixels = stbi_loadf_from_memory(
bytes.data(),
byteCount,
&data.width,
&data.height,
&data.comp,
@@ -161,8 +172,9 @@ namespace util
}
// ---------- STB LDR (png/jpg/...) ----------
data.pixels = stbi_load(
p.string().c_str(),
data.pixels = stbi_load_from_memory(
bytes.data(),
byteCount,
&data.width,
&data.height,
&data.channels,
destrum/src/Graphics/Pipeline.cpp
+3 -16
View File
@@ -2,10 +2,10 @@
#include <cassert>
#include <filesystem>
#include <fstream>
#include <iostream>
#include "volk.h"
#include "destrum/FS/AssetFS.h"
#include "destrum/Graphics/Util.h"
#include "spdlog/spdlog.h"
@@ -115,21 +115,8 @@ void Pipeline::DefaultPipelineConfigInfo(PipelineConfigInfo& configInfo) {
}
std::vector<char> Pipeline::readFile(const std::string& filename) {
std::ifstream file(filename, std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("Failed to open file: " + filename);
}
const size_t fileSize = (size_t)file.tellg();
std::vector<char> buffer(fileSize);
file.seekg(0);
file.read(buffer.data(), static_cast<std::streamsize>(fileSize));
file.close();
return buffer;
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(filename);
return {bytes.begin(), bytes.end()};
}
void Pipeline::CreateGraphicsPipeline(const std::string& vertPath, const std::string& fragPath,
destrum/src/Graphics/Resources/Font.cpp
+2 -7
View File
@@ -2,8 +2,6 @@
#include <algorithm>
#include <cstring>
#include <fstream>
#include <iterator>
#include <stdexcept>
#include <string>
@@ -13,6 +11,7 @@
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/Graphics/GfxDevice.h>
#include <destrum/Graphics/RenderResources.h>
@@ -57,11 +56,7 @@ namespace {
std::vector<FT_Byte> readFile(const std::filesystem::path& path)
{
std::ifstream file(path, std::ios::binary);
if (!file) {
throw std::runtime_error("Font: failed to open " + path.string());
}
return {std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()};
return AssetFS::GetInstance().ReadFile(path);
}
void check(FT_Error error, const char* what, const std::filesystem::path& path)
destrum/src/Graphics/Util.cpp
+6 -15
View File
@@ -1,6 +1,7 @@
#include <fstream>
#include <cstring>
#include <destrum/Graphics/Init.h>
#include <destrum/Graphics/Util.h>
#include <destrum/FS/AssetFS.h>
#include <volk.h>
@@ -401,22 +402,12 @@ vkutil::RenderInfo vkutil::createRenderingInfo(const RenderingInfoParams& params
}
VkShaderModule vkutil::loadShaderModule(const std::filesystem::path& path, VkDevice device) {
std::ifstream file(path, std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("Failed to open shader: " + path.string());
}
const auto fileSize = file.tellg();
if (fileSize < 0 || fileSize % static_cast<std::streamoff>(sizeof(std::uint32_t)) != 0) {
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(path);
if (bytes.size() % sizeof(std::uint32_t) != 0) {
throw std::runtime_error("Shader file has an invalid size: " + path.string());
}
std::vector<std::uint32_t> buffer(fileSize / sizeof(std::uint32_t));
file.seekg(0);
if (!file.read(reinterpret_cast<char*>(buffer.data()), fileSize)) {
throw std::runtime_error("Failed to read shader: " + path.string());
}
file.close();
std::vector<std::uint32_t> buffer(bytes.size() / sizeof(std::uint32_t));
std::memcpy(buffer.data(), bytes.data(), bytes.size());
auto info = VkShaderModuleCreateInfo{
.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
destrum/src/Serialization/ObjectSerialization.cpp
+1 -1
View File
@@ -151,7 +151,7 @@ namespace {
fullPath = path;
}
return std::filesystem::exists(fullPath);
return AssetFS::GetInstance().Exists(fullPath);
}
void CollectHierarchyRecursive(const GameObject& object, std::vector<const GameObject*>& out,
destrum/src/Serialization/PrefabCache.cpp
+4 -3
View File
@@ -14,12 +14,13 @@ std::shared_ptr<const nlohmann::json> PrefabCache::Get(std::string_view path) {
return nullptr;
}
std::error_code error;
const auto writeTime = std::filesystem::last_write_time(fullPath, error);
if (error) {
if (!AssetFS::GetInstance().Exists(fullPath)) {
spdlog::error("Prefab file not found: {}", fullPath.string());
return nullptr;
}
// Packed files have no write time, they can't change anyway
std::error_code error;
const auto writeTime = std::filesystem::last_write_time(fullPath, error);
const std::string key = MakeKey(fullPath);
if (const auto it = m_Prefabs.find(key); it != m_Prefabs.end() && it->second.writeTime == writeTime) {
destrum/src/Serialization/PrefabSerializer.cpp
+9 -5
View File
@@ -1,9 +1,11 @@
#include <destrum/Serialization/PrefabSerializer.h>
#include <fstream>
#include <cstdint>
#include <vector>
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/ObjectModel/GameObject.h>
#include <destrum/ObjectModel/Transform.h>
#include <destrum/Scene/Scene.h>
@@ -100,15 +102,17 @@ bool PrefabSerializer::Save(const GameObject& root, const std::filesystem::path&
}
std::optional<json> PrefabSerializer::LoadFile(const std::filesystem::path& path) {
std::ifstream file(path);
if (!file.is_open()) {
spdlog::error("Failed to open prefab file for reading: {}", path.string());
std::vector<std::uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(path);
} catch (const std::exception& exception) {
spdlog::error("Failed to open prefab file for reading: {} ({})", path.string(), exception.what());
return std::nullopt;
}
json prefab;
try {
file >> prefab;
prefab = json::parse(bytes.begin(), bytes.end());
} catch (const std::exception& exception) {
spdlog::error("Failed to parse prefab file {}: {}", path.string(), exception.what());
return std::nullopt;
destrum/src/Serialization/SceneSerializer.cpp
+8 -5
View File
@@ -1,11 +1,12 @@
#include <destrum/Serialization/SceneSerializer.h>
#include <fstream>
#include <cstdint>
#include <vector>
#include <nlohmann/json.hpp>
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/Scene/Scene.h>
#include <destrum/ObjectModel/GameObject.h>
#include <destrum/ObjectModel/ObjectId.h>
@@ -152,15 +153,17 @@ SceneSerializer::LoadResult SceneSerializer::LoadSceneFile(const std::filesystem
LoadResult result;
RegisterEngineComponents();
std::ifstream file(path);
if (!file.is_open()) {
result.errorMessage = "Failed to open scene file for reading: " + path.string();
std::vector<std::uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(path);
} catch (const std::exception& exception) {
result.errorMessage = "Failed to open scene file for reading: " + path.string() + " (" + exception.what() + ")";
spdlog::error("{}", result.errorMessage);
return result;
}
try {
file >> result.root;
result.root = json::parse(bytes.begin(), bytes.end());
} catch (const std::exception& exception) {
result.errorMessage = "Failed to parse scene file " + path.string() + ": " + exception.what();
spdlog::error("{}", result.errorMessage);
export.bat
+18
View File
@@ -0,0 +1,18 @@
@echo off
rem Builds a Release export of Lightkeeper (build\export\Release) and opens the folder
setlocal
cd /d "%~dp0"
if not exist build\CMakeCache.txt (
cmake -S . -B build || goto :failed
)
cmake --build build --config Release --target Export --parallel || goto :failed
explorer build\export\Release
exit /b 0
:failed
echo.
echo Export failed, see the output above.
pause
exit /b 1
lightkeeper/CMakeLists.txt
+3
View File
@@ -28,9 +28,11 @@ set(GAME_SRC
add_executable(lightkeeper ${GAME_SRC})
# Release is what gets exported, players shouldn't get a console next to the game (it logs to a file too)
set_target_properties(lightkeeper PROPERTIES
CXX_STANDARD 20
CXX_EXTENSIONS OFF
WIN32_EXECUTABLE $<CONFIG:Release,MinSizeRel>
)
#target_add_extra_warnings(lightkeeper)
@@ -108,6 +110,7 @@ add_dependencies(lightkeeper _internal_cook_game_assets)
destrum_cook_engine_assets(lightkeeper "${CMAKE_CURRENT_BINARY_DIR}")
destrum_copy_runtime_dlls(lightkeeper)
destrum_add_export(lightkeeper "${ASSETS_RUNTIME_DIR}")
install(TARGETS lightkeeper RUNTIME DESTINATION bin)
install(DIRECTORY "${ASSETS_RUNTIME_DIR}/" DESTINATION "bin/assets/game")
lightkeeper/src/main.cpp
+42 -2
View File
@@ -1,4 +1,5 @@
#include <algorithm>
#include <filesystem>
#include <memory>
#include <span>
#include <stdexcept>
@@ -6,15 +7,52 @@
#include <string_view>
#include <SDL.h>
#include <spdlog/sinks/basic_file_sink.h>
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <editor/Editor.h>
#include "Lightkeeper.h"
namespace {
// Release builds have no console, so players only have this file to send when something goes wrong
std::filesystem::path addFileLog() {
char* prefPath = SDL_GetPrefPath("Destrum", "Lightkeeper");
if (prefPath == nullptr) {
spdlog::warn("No folder to write the log to: {}", SDL_GetError());
return {};
}
const std::filesystem::path logPath = std::filesystem::path{prefPath} / "lightkeeper.log";
SDL_free(prefPath);
try {
spdlog::default_logger()->sinks().push_back(
std::make_shared<spdlog::sinks::basic_file_sink_mt>(logPath.string(), true));
} catch (const spdlog::spdlog_ex& exception) {
spdlog::warn("Can't write the log to {}: {}", logPath.string(), exception.what());
return {};
}
// Every line, so the log survives a hard crash
spdlog::flush_on(spdlog::level::trace);
return logPath;
}
void showError(const std::string& message, const std::filesystem::path& logPath) {
std::string text = message;
if (!logPath.empty()) {
text += "\n\nThe log is at " + logPath.string();
}
SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "Lightkeeper", text.c_str(), nullptr);
}
}
int main(int argc, char* argv[]) {
const std::filesystem::path logPath = addFileLog();
if (SDL_Init(SDL_INIT_VIDEO) != 0) {
SDL_ShowSimpleMessageBox(SDL_MESSAGEBOX_ERROR, "Error", SDL_GetError(), nullptr);
spdlog::critical("SDL_Init failed: {}", SDL_GetError());
showError(std::string{"Couldn't start SDL: "} + SDL_GetError(), logPath);
return 1;
}
@@ -39,7 +77,8 @@ int main(int argc, char* argv[]) {
.windowTitle = "Lightkeeper",
.exeDir = exeDir,
});
if (!runGameOnly) {
// Exported builds only ship the cooked asset pack, the editor needs the source folders
if (!runGameOnly && !AssetFS::GetInstance().IsPacked()) {
app.attachLayer(std::make_unique<Editor>(app, EditorSettings{
.gameAssetsSource = LIGHTKEEPER_ASSETS_SRC_DIR,
.scenePath = std::string{LightKeeper::MainScenePath},
@@ -49,6 +88,7 @@ int main(int argc, char* argv[]) {
app.cleanup();
} catch (const std::exception& exception) {
spdlog::critical("Lightkeeper terminated with an error: {}", exception.what());
showError(std::string{"Lightkeeper ran into an error and has to close.\n\n"} + exception.what(), logPath);
SDL_Quit();
return 1;
}
tests/assets_tests.cpp
+113
View File
@@ -10,6 +10,7 @@
#include <destrum/Assets/AssetTypes.h>
#include <destrum/Assets/AssetReference.h>
#include <destrum/Assets/ModelFile.h>
#include <destrum/Assets/PackFile.h>
#include <destrum/Assets/TextureFile.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/ObjectModel/Component.h>
@@ -458,6 +459,116 @@ namespace {
"loading a source image must use its cooked sibling");
}
bool packOpenThrows(const std::filesystem::path& path)
{
try {
(void)PackFile::Reader{path};
} catch (const std::exception&) {
return true;
}
return false;
}
void testPackFileRoundTrip()
{
const auto dir = std::filesystem::temp_directory_path() / "destrum_pack_file_test";
std::filesystem::remove_all(dir);
std::filesystem::create_directories(dir);
std::ofstream{dir / "a.txt", std::ios::binary} << "hello";
std::ofstream{dir / "empty.txt", std::ios::binary};
const auto pack = dir / "test.dpak";
PackFile::Write({{"engine/a.txt", dir / "a.txt"}, {"game/empty.txt", dir / "empty.txt"}}, pack);
const PackFile::Reader reader{pack};
const PackFile::Entry* text = reader.Find("engine/a.txt");
const PackFile::Entry* empty = reader.Find("game/empty.txt");
check(reader.Entries().size() == 2 && text != nullptr && empty != nullptr && !reader.Find("a.txt"),
"packs must list every file by its full path");
const auto bytes = reader.Read(*text);
check(std::string(bytes.begin(), bytes.end()) == "hello" && reader.Read(*empty).empty(),
"packed files must read back as written");
bool duplicateRejected = false;
try {
PackFile::Write({{"engine/a.txt", dir / "a.txt"}, {"engine/a.txt", dir / "a.txt"}}, dir / "dupe.dpak");
} catch (const std::exception&) {
duplicateRejected = true;
}
check(duplicateRejected, "packs can't hold two files at one path");
std::ofstream{dir / "bad.dpak", std::ios::binary} << "DTEX not a pack";
check(packOpenThrows(dir / "bad.dpak"), "files that aren't packs must be rejected");
const auto size = std::filesystem::file_size(pack);
std::filesystem::resize_file(pack, size - 4);
check(packOpenThrows(pack), "truncated packs must be rejected");
std::filesystem::remove_all(dir);
}
void testAssetFSMountsPacks()
{
const auto root = std::filesystem::temp_directory_path() / "destrum_packed_assets_test";
const auto staging = root / "staging";
std::filesystem::remove_all(root);
std::filesystem::create_directories(staging);
std::ofstream{staging / "notes.txt", std::ios::binary} << "packed";
std::ofstream{staging / "notes.txt.meta", std::ios::binary} << R"({"guid":"0123456789abcdef0123456789abcdef"})";
std::ofstream{staging / "model.fbx", std::ios::binary} << "not a model";
ModelFile::WriteToFile(makeTestModelAsset(), staging / "model.fbx.dmodel");
const std::vector<std::uint8_t> rgba{255, 0, 0, 255};
TextureFile::WriteToFile(TextureFile::FromRGBA8(1, 1, rgba), staging / "tex.png.dtex");
std::ofstream{staging / "manifest.json", std::ios::binary}
<< R"({"version":2,"assets":[{"src":"model.fbx","out":"model.fbx.dmodel","type":"model"}]})";
PackFile::Write({
{"engine/manifest.json", staging / "manifest.json"},
{"engine/model.fbx", staging / "model.fbx"},
{"engine/model.fbx.dmodel", staging / "model.fbx.dmodel"},
{"engine/tex.png.dtex", staging / "tex.png.dtex"},
{"game/notes.txt", staging / "notes.txt"},
{"game/notes.txt.meta", staging / "notes.txt.meta"},
}, root / AssetFS::PackName);
// A loose copy next to the pack must not be picked up
std::filesystem::create_directories(root / "assets" / "game");
std::ofstream{root / "assets" / "game" / "notes.txt", std::ios::binary} << "loose";
std::ofstream{root / "assets" / "game" / "loose.txt", std::ios::binary} << "loose";
auto& assetFs = AssetFS::GetInstance();
assetFs.Init(root);
check(assetFs.IsPacked() && assetFs.GetMounts().size() == 2, "a pack next to the exe must mount every scheme in it");
const auto notes = assetFs.ReadBytes("game://notes.txt");
check(std::string(notes.begin(), notes.end()) == "packed", "packed mounts must read from the pack");
check(assetFs.GetFullPath("game://notes.txt") == (root / "assets" / "game" / "notes.txt").lexically_normal(),
"packed files must keep the full path they have unpacked");
check(assetFs.Exists(assetFs.GetFullPath("game://notes.txt")) &&
!assetFs.Exists(assetFs.GetFullPath("game://loose.txt")),
"only packed files exist under a packed mount");
check(assetFs.ToVirtualPath(assetFs.GetFullPath("engine://tex.png")) == "engine://tex.png",
"packed full paths must map back to virtual paths");
check(!assetFs.WriteFile("game://notes.txt", "changed"), "packed mounts are read only");
const auto model = AssetManager::GetInstance().LoadModel("engine://model.fbx");
check(model.primitives.size() == 1, "cooked models must load from the pack");
const auto image = util::loadImage(assetFs.GetFullPath("engine://tex.png"), TextureIntent::ColorSrgb);
check(image.width == 1 && image.pixels != nullptr && std::memcmp(image.pixels, rgba.data(), rgba.size()) == 0,
"images must find their cooked sibling in the pack");
auto& database = AssetDatabase::GetInstance();
database.Refresh();
check(database.FindPath(*AssetGuid::Parse("0123456789abcdef0123456789abcdef")) == "game://notes.txt",
"asset ids must be read from packed meta files");
database.Clear();
assetFs.Reset();
std::filesystem::remove_all(root);
}
void testAssetReferenceCacheKeys()
{
const AssetReference reference{
@@ -555,6 +666,8 @@ void RunAssetsTests()
testTextureFileRoundTrip();
testImageLoaderReadsCookedTextures();
testLdrTextureFile();
testPackFileRoundTrip();
testAssetFSMountsPacks();
testAssetReferenceCacheKeys();
testAssetGuids();
testAssetDatabaseFollowsMovedAssets();
tests/cooker_tests.cpp
+49 -3
View File
@@ -9,6 +9,8 @@
#include <nlohmann/json.hpp>
#include <destrum/Assets/PackFile.h>
#include "TestUtils.h"
using namespace test;
@@ -51,11 +53,10 @@ namespace {
}
}
int runChef(const CookDirs& dirs, const std::string& flags = {})
int runChefArgs(const CookDirs& dirs, const std::string& args)
{
const fs::path log = dirs.root / "chef.log";
std::string command = "\"" + std::string(DESTRUM_THECHEF_PATH) + "\" --input \"" + dirs.input.string() +
"\" --output \"" + dirs.output.string() + "\" " + flags + " > \"" + log.string() +
std::string command = "\"" + std::string(DESTRUM_THECHEF_PATH) + "\" " + args + " > \"" + log.string() +
"\" 2>&1";
#ifdef _WIN32
// cmd.exe strips the outer quotes, the inner ones keep paths with spaces intact
@@ -64,6 +65,11 @@ namespace {
return std::system(command.c_str());
}
int runChef(const CookDirs& dirs, const std::string& flags = {})
{
return runChefArgs(dirs, "--input \"" + dirs.input.string() + "\" --output \"" + dirs.output.string() + "\" " + flags);
}
void expectChef(const CookDirs& dirs, const std::string& flags, bool succeeds, const char* message)
{
if ((runChef(dirs, flags) == 0) != succeeds) {
@@ -307,6 +313,45 @@ namespace {
}
}
namespace {
void testPackBundlesCookedOutputs()
{
const CookDirs dirs = makeDirs("destrum_cooker_pack");
writeShaders(dirs);
writeText(dirs.input / "notes.txt", "hello");
expectChef(dirs, "", true, "the first cook must succeed");
writeText(dirs.output / "stray.txt", "not cooked");
const fs::path pack = dirs.root / "assets.dpak";
const std::string packArgs = "--pack \"" + pack.string() + "\" --mount \"game=" + dirs.output.string() + "\"";
if (runChefArgs(dirs, packArgs) != 0) {
std::cerr << readFile(dirs.root / "chef.log") << '\n';
check(false, "packing a cooked folder must succeed");
return;
}
const PackFile::Reader reader{pack};
check(reader.Find("game/manifest.json") && reader.Find("game/shaders/plain.frag.spv") &&
reader.Find("game/notes.txt"),
"packs must hold the manifest and every cooked output");
check(!reader.Find("game/stray.txt") && !reader.Find("game/shaders/include/common.glsl"),
"packs must only hold what the manifest lists as outputs");
const auto notes = reader.Read(*reader.Find("game/notes.txt"));
check(std::string(notes.begin(), notes.end()) == "hello", "packed files must keep their contents");
check(runChefArgs(dirs, "--list \"" + pack.string() + "\" --top 0") == 0 &&
readFile(dirs.root / "chef.log").find("game/notes.txt") != std::string::npos,
"listing a pack must show its files");
check(runChefArgs(dirs, "--list \"" + dirs.output.string() + "\"") == 0 &&
readFile(dirs.root / "chef.log").find("Not in the manifest, won't be packed: 1 files") != std::string::npos,
"listing a cooked folder must point out files that won't be packed");
fs::remove(dirs.output / "notes.txt");
check(runChefArgs(dirs, packArgs) != 0, "packing must fail when a listed output is missing");
check(!fs::exists(pack.string() + ".tmp"), "a failed pack must not leave a temp file");
}
}
void RunCookerTests()
{
testCookRecordsEveryOutput();
@@ -317,4 +362,5 @@ void RunCookerTests()
testFailedCookKeepsLastGoodOutput();
testFallbacksAndStrictMode();
testValidateFindsMissingOutputs();
testPackBundlesCookedOutputs();
}