brammie15/Destrum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add export / file packing

Browse Source
This commit is contained in:
brammie15
2026-10-30 15:42:08 +01:00
parent de7e489cc0
commit 0fbc023018
28 changed files with 782 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
destrum/CMakeLists.txt
+36
View File
@@ -4,6 +4,7 @@ add_subdirectory(third_party)
add_library(destrum_assets
"src/Assets/ModelFile.cpp"
"src/Assets/TextureFile.cpp"
"src/Assets/PackFile.cpp"
)
add_library(destrum::assets ALIAS destrum_assets)
@@ -297,6 +298,41 @@ function(destrum_cook_engine_assets GAME_TARGET GAME_OUTPUT_DIR)
install(DIRECTORY "${_engine_runtime}/" DESTINATION "bin/assets/engine")
endfunction()
# A folder (and zip) other people can run: the game exe, its DLLs and every cooked asset packed into assets.dpak
function(destrum_add_export GAME_TARGET GAME_RUNTIME_DIR)
set(_engine_runtime "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/assets_runtime")
set(_export_root "${CMAKE_BINARY_DIR}/export/$<CONFIG>")
set(_export_dir "${_export_root}/${GAME_TARGET}")
# The release VC++ runtime ships app-local, debug builds only run where Visual Studio is installed
set(_system_libs)
if (MSVC)
set(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP TRUE)
include(InstallRequiredSystemLibraries)
set(_system_libs "$<$<NOT:$<CONFIG:Debug>>:${CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS}>")
endif()
add_custom_target(_internal_export_${GAME_TARGET}
COMMAND ${CMAKE_COMMAND} -E rm -rf "${_export_dir}" "${_export_dir}.zip"
COMMAND ${CMAKE_COMMAND} -E make_directory "${_export_dir}"
COMMAND TheChef
--pack "${_export_dir}/assets.dpak"
--mount "engine=${_engine_runtime}"
--mount "game=${GAME_RUNTIME_DIR}"
COMMAND TheChef --list "${_export_dir}/assets.dpak" --top 10
COMMAND ${CMAKE_COMMAND} -E copy -t "${_export_dir}"
"$<TARGET_FILE:${GAME_TARGET}>"
"$<TARGET_RUNTIME_DLLS:${GAME_TARGET}>"
"${_system_libs}"
COMMAND ${CMAKE_COMMAND} -E chdir "${_export_root}"
${CMAKE_COMMAND} -E tar cf "${GAME_TARGET}.zip" --format=zip "${GAME_TARGET}"
COMMAND ${CMAKE_COMMAND} -E echo "Exported ${GAME_TARGET} to ${_export_dir}"
COMMAND_EXPAND_LISTS
VERBATIM
)
add_dependencies(_internal_export_${GAME_TARGET} ${GAME_TARGET})
endfunction()
# OpenAL is a DLL, every exe that links the engine needs a copy next to it
function(destrum_copy_runtime_dlls TARGET)
if (WIN32)
destrum/include/destrum/Assets/PackFile.h
+46
View File
@@ -0,0 +1,46 @@
#ifndef DESTRUM_PACKFILE_H
#define DESTRUM_PACKFILE_H
#include <cstdint>
#include <filesystem>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>
namespace PackFile {
inline constexpr std::uint32_t Version = 1;
inline constexpr std::string_view Extension = ".dpak";
struct Entry {
std::string path;
std::uint64_t offset{0};
std::uint64_t size{0};
};
struct Source {
std::string path;
std::filesystem::path file;
};
// Paths are "<scheme>/<relative path>", e.g. "engine/shaders/mesh.vert.spv"
void Write(const std::vector<Source>& files, const std::filesystem::path& path);
class Reader {
public:
explicit Reader(std::filesystem::path path);
[[nodiscard]] const Entry* Find(std::string_view path) const;
[[nodiscard]] std::vector<std::uint8_t> Read(const Entry& entry) const;
[[nodiscard]] const std::vector<Entry>& Entries() const { return m_Entries; }
[[nodiscard]] const std::filesystem::path& Path() const { return m_Path; }
private:
std::filesystem::path m_Path;
std::vector<Entry> m_Entries;
std::unordered_map<std::string, std::size_t> m_ByPath;
};
}
#endif // DESTRUM_PACKFILE_H
destrum/include/destrum/FS/AssetFS.h
+12 -1
View File
@@ -3,12 +3,14 @@
#include <filesystem>
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
#include <destrum/Singleton.h>
#include <destrum/Assets/PackFile.h>
#include <destrum/FS/Manifest.h>
@@ -18,17 +20,26 @@ struct FSMount {
std::filesystem::path root;
std::optional<AssetManifest> manifest;
std::optional<std::filesystem::path> sourceRoot;
std::shared_ptr<const PackFile::Reader> pack;
};
class AssetFS final: public Singleton<AssetFS> {
public:
static constexpr std::string_view PackName = "assets.dpak";
void Init(std::filesystem::path exeDir);
void Reset();
void Mount(std::string scheme, std::filesystem::path root);
void MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root);
void ReloadManifest(std::string_view scheme);
std::vector<uint8_t> ReadBytes(std::string_view vpath);
[[nodiscard]] std::vector<uint8_t> ReadFile(const std::filesystem::path& fullPath) const;
[[nodiscard]] bool Exists(const std::filesystem::path& fullPath) const;
[[nodiscard]] std::vector<std::filesystem::path> ListFiles(const FSMount& mount) const;
[[nodiscard]] bool IsPacked() const;
[[nodiscard]] std::filesystem::path GetFullPath(std::string_view vpath) const;
[[nodiscard]] std::filesystem::path GetCookedPathForFile(std::string_view vpath) const;
@@ -43,7 +54,7 @@ public:
bool WriteFile(std::string_view vpath, std::string_view contents);
private:
static std::vector<uint8_t> ReadFile(const std::filesystem::path& fullPath);
void AddMount(std::string scheme, std::filesystem::path root, std::shared_ptr<const PackFile::Reader> pack);
std::vector<FSMount> mounts;
destrum/include/destrum/FS/Manifest.h
+2 -1
View File
@@ -4,6 +4,7 @@
#include <cstdint>
#include <filesystem>
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <unordered_map>
@@ -29,7 +30,7 @@ struct AssetManifest {
};
namespace FS {
AssetManifest LoadAssetManifest(const std::filesystem::path& manifestPath);
AssetManifest ParseAssetManifest(std::span<const std::uint8_t> bytes);
}
#endif //MANIFEST_H
destrum/include/destrum/Util/ModelDoc.h
+19 -8
View File
@@ -20,6 +20,7 @@
//
// Link against: assimp
#include <assimp/IOSystem.hpp>
#include <assimp/Importer.hpp>
#include <assimp/material.h>
#include <assimp/postprocess.h>
@@ -43,6 +44,7 @@
#include <filesystem>
#include <iostream>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
#include <unordered_map>
@@ -998,8 +1000,12 @@ static void CollectMergedPerNodeMeshes(const aiScene* scene,
// ─── Public API ───────────────────────────────────────────────────────────────
static ModelAsset LoadModel(const std::string& path, const LoadOptions& options = {}) {
// The importer owns io, without one it reads from disk
static ModelAsset LoadModel(const std::string& path, const LoadOptions& options = {},
std::unique_ptr<Assimp::IOSystem> io = nullptr) {
Assimp::Importer importer;
if (io)
importer.SetIOHandler(io.release());
const aiScene* scene = LoadScene(importer, path, options);
ModelAsset model{};
@@ -1084,16 +1090,18 @@ static const aiScene* LoadAnimationScene(
Assimp::Importer& importer,
const std::string& path
) {
if (!std::filesystem::exists(path)) {
if (!importer.GetIOHandler()->Exists(path.c_str())) {
throw std::runtime_error("Animation file does not exist: " + path);
}
if (!std::filesystem::is_regular_file(path)) {
throw std::runtime_error("Animation path is not a regular file: " + path);
}
if (importer.IsDefaultIOHandler()) {
if (!std::filesystem::is_regular_file(path)) {
throw std::runtime_error("Animation path is not a regular file: " + path);
}
if (std::filesystem::file_size(path) == 0) {
throw std::runtime_error("Animation file is empty: " + path);
if (std::filesystem::file_size(path) == 0) {
throw std::runtime_error("Animation file is empty: " + path);
}
}
// Important: do not use mesh post-process flags for animation-only files.
@@ -1141,9 +1149,12 @@ static const aiScene* LoadAnimationScene(
static std::vector<SkeletalAnimation> LoadAnimationClips(
const std::string& path,
const Skeleton& targetSkeleton
const Skeleton& targetSkeleton,
std::unique_ptr<Assimp::IOSystem> io = nullptr
) {
Assimp::Importer importer;
if (io)
importer.SetIOHandler(io.release());
const aiScene* scene = LoadAnimationScene(importer, path);
auto clips = LoadAnimations(scene, targetSkeleton);
destrum/src/Assets/AssetDatabase.cpp
+9 -14
View File
@@ -3,7 +3,6 @@
#include <charconv>
#include <filesystem>
#include <format>
#include <fstream>
#include <random>
#include <spdlog/spdlog.h>
@@ -19,12 +18,9 @@ namespace {
[[nodiscard]] std::optional<AssetGuid> ReadMetaFile(const std::filesystem::path& path)
{
std::ifstream file(path);
if (!file) {
return std::nullopt;
}
try {
const auto meta = nlohmann::json::parse(file);
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(path);
const auto meta = nlohmann::json::parse(bytes.begin(), bytes.end());
return AssetGuid::Parse(meta.at("guid").get<std::string>());
} catch (const std::exception& exception) {
spdlog::warn("Ignoring broken asset meta file {}: {}", path.string(), exception.what());
@@ -83,14 +79,14 @@ void AssetDatabase::Clear()
void AssetDatabase::Refresh()
{
Clear();
for (const FSMount& mount : AssetFS::GetInstance().GetMounts()) {
std::error_code error;
for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) {
if (!entry.is_regular_file(error) || !AssetTypes::IsMeta(entry.path().string())) {
const auto& assetFs = AssetFS::GetInstance();
for (const FSMount& mount : assetFs.GetMounts()) {
for (const std::filesystem::path& file : assetFs.ListFiles(mount)) {
if (!AssetTypes::IsMeta(file.string())) {
continue;
}
if (const auto guid = ReadMetaFile(entry.path())) {
std::filesystem::path asset = entry.path();
if (const auto guid = ReadMetaFile(file)) {
std::filesystem::path asset = file;
asset.replace_extension();
Register(mount.scheme + "://" + asset.lexically_relative(mount.root).generic_string(), *guid);
}
@@ -131,8 +127,7 @@ std::optional<AssetGuid> AssetDatabase::GuidForSave(std::string_view path)
}
auto& assetFs = AssetFS::GetInstance();
std::error_code error;
if (!std::filesystem::is_regular_file(assetFs.GetFullPath(path), error)) {
if (!assetFs.Exists(assetFs.GetFullPath(path))) {
return std::nullopt;
}
destrum/src/Assets/AssetFSIOSystem.h
+45
View File
@@ -0,0 +1,45 @@
#ifndef DESTRUM_ASSETFSIOSYSTEM_H
#define DESTRUM_ASSETFSIOSYSTEM_H
#include <cstdint>
#include <cstring>
#include <string_view>
#include <vector>
#include <assimp/IOSystem.hpp>
#include <assimp/MemoryIOWrapper.h>
#include <destrum/FS/AssetFS.h>
class AssetFSIOSystem final : public Assimp::IOSystem {
public:
bool Exists(const char* file) const override {
return AssetFS::GetInstance().Exists(file);
}
char getOsSeparator() const override {
return '/';
}
Assimp::IOStream* Open(const char* file, const char* mode) override {
if (std::string_view{mode}.find_first_of("wa+") != std::string_view::npos) {
return nullptr;
}
std::vector<std::uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(file);
} catch (const std::exception&) {
return nullptr;
}
auto* buffer = new std::uint8_t[bytes.size()];
std::memcpy(buffer, bytes.data(), bytes.size());
return new Assimp::MemoryIOStream(buffer, bytes.size(), true);
}
void Close(Assimp::IOStream* stream) override {
delete stream;
}
};
#endif // DESTRUM_ASSETFSIOSYSTEM_H
destrum/src/Assets/AssetManager.cpp
+15 -3
View File
@@ -1,11 +1,14 @@
#include <destrum/Assets/AssetManager.h>
#include <memory>
#include <optional>
#include <stdexcept>
#include <destrum/Assets/ModelFile.h>
#include <destrum/FS/AssetFS.h>
#include "AssetFSIOSystem.h"
namespace {
std::string ToReference(std::string_view reference) {
if (reference.empty()) {
@@ -30,6 +33,14 @@ namespace {
return std::nullopt;
}
// Assimp opens files itself, so with packed assets it has to go through AssetFS
std::unique_ptr<Assimp::IOSystem> MakeIOSystem() {
if (!AssetFS::GetInstance().IsPacked()) {
return nullptr;
}
return std::make_unique<AssetFSIOSystem>();
}
// Cooked texture paths are relative to the model file, imported ones are absolute
void ResolveTexturePaths(ModelDoc::ModelAsset& model, const std::filesystem::path& modelDir) {
for (ModelDoc::MaterialInfo& material : model.materials) {
@@ -95,7 +106,7 @@ ModelDoc::ModelAsset AssetManager::LoadCookedOrImport(
const ModelDoc::LoadOptions& options) const
{
if (const auto cookedPath = FindCookedModel(reference)) {
ModelDoc::ModelAsset model = ModelFile::ReadFromFile(*cookedPath);
ModelDoc::ModelAsset model = ModelFile::Read(AssetFS::GetInstance().ReadFile(*cookedPath));
if (ModelFile::SupportsLoadOptions(model, options)) {
ModelFile::ApplyLoadOptions(model, options);
ResolveTexturePaths(model, cookedPath->parent_path());
@@ -105,7 +116,7 @@ ModelDoc::ModelAsset AssetManager::LoadCookedOrImport(
spdlog::debug("Importing {}, its cooked model doesn't match the requested load options", reference);
}
return ModelDoc::LoadModel(Resolve(reference).generic_string(), options);
return ModelDoc::LoadModel(Resolve(reference).generic_string(), options, MakeIOSystem());
}
std::vector<SkeletalAnimation> AssetManager::LoadAnimationClips(
@@ -115,7 +126,8 @@ std::vector<SkeletalAnimation> AssetManager::LoadAnimationClips(
const std::string sourceReference = ToReference(reference);
auto clips = ModelDoc::LoadAnimationClips(
Resolve(sourceReference).generic_string(),
targetSkeleton);
targetSkeleton,
MakeIOSystem());
for (std::size_t index = 0; index < clips.size(); ++index) {
auto& clip = clips[index];
clip.assetReference.path = sourceReference;
destrum/src/Assets/PackFile.cpp
+126
View File
@@ -0,0 +1,126 @@
#include <destrum/Assets/PackFile.h>
#include <set>
#include "BinaryStream.h"
namespace {
using AssetBinary::BinaryReader;
using AssetBinary::BinaryWriter;
constexpr AssetBinary::Magic Magic{'D', 'P', 'A', 'K'};
constexpr std::string_view Kind = "pack";
constexpr std::size_t HeaderSize = sizeof(AssetBinary::Magic) + sizeof(std::uint32_t) + sizeof(std::uint64_t);
}
void PackFile::Write(const std::vector<Source>& files, const std::filesystem::path& path) {
std::ofstream out(path, std::ios::binary | std::ios::trunc);
if (!out) {
throw std::runtime_error("Failed to open file for writing: " + path.string());
}
BinaryWriter header;
header.WriteHeader(Magic, Version);
header.Write(std::uint64_t{0});
const std::vector<std::uint8_t> headerBytes = header.TakeBytes();
out.write(reinterpret_cast<const char*>(headerBytes.data()), static_cast<std::streamsize>(headerBytes.size()));
std::set<std::string> seen;
BinaryWriter index;
index.WriteCount(files.size());
for (const Source& file : files) {
if (!seen.insert(file.path).second) {
throw std::runtime_error("Pack has two files at " + file.path);
}
std::ifstream in(file.file, std::ios::binary);
if (!in) {
throw std::runtime_error("Failed to open file: " + file.file.string());
}
const auto offset = static_cast<std::uint64_t>(out.tellp());
// Inserting an empty stream buffer sets failbit
if (std::filesystem::file_size(file.file) > 0) {
out << in.rdbuf();
}
const auto size = static_cast<std::uint64_t>(out.tellp()) - offset;
index.WriteString(file.path);
index.Write(offset);
index.Write(size);
}
const auto indexOffset = static_cast<std::uint64_t>(out.tellp());
const std::vector<std::uint8_t> indexBytes = index.TakeBytes();
out.write(reinterpret_cast<const char*>(indexBytes.data()), static_cast<std::streamsize>(indexBytes.size()));
out.seekp(static_cast<std::streamoff>(HeaderSize - sizeof(std::uint64_t)));
out.write(reinterpret_cast<const char*>(&indexOffset), sizeof(indexOffset));
if (!out) {
throw std::runtime_error("Failed to write file: " + path.string());
}
}
PackFile::Reader::Reader(std::filesystem::path path)
: m_Path(std::move(path)) {
std::ifstream file(m_Path, std::ios::binary | std::ios::ate);
if (!file) {
throw std::runtime_error("Failed to open file: " + m_Path.string());
}
const auto fileSize = static_cast<std::uint64_t>(file.tellg());
std::array<std::uint8_t, HeaderSize> headerBytes{};
file.seekg(0);
file.read(reinterpret_cast<char*>(headerBytes.data()), static_cast<std::streamsize>(headerBytes.size()));
if (!file) {
throw std::runtime_error("Not a " + std::string{Kind} + " file: " + m_Path.string());
}
BinaryReader header{headerBytes};
header.ReadHeader(Magic, Version, Kind);
const auto indexOffset = header.Read<std::uint64_t>();
if (indexOffset < HeaderSize || indexOffset > fileSize) {
throw std::runtime_error("Pack file is corrupt: index is out of range");
}
std::vector<std::uint8_t> indexBytes(static_cast<std::size_t>(fileSize - indexOffset));
file.seekg(static_cast<std::streamoff>(indexOffset));
file.read(reinterpret_cast<char*>(indexBytes.data()), static_cast<std::streamsize>(indexBytes.size()));
if (!file) {
throw std::runtime_error("Failed to read file: " + m_Path.string());
}
BinaryReader index{indexBytes};
m_Entries.resize(index.ReadCount(3 * sizeof(std::uint64_t)));
for (std::size_t i = 0; i < m_Entries.size(); ++i) {
Entry& entry = m_Entries[i];
entry.path = index.ReadString();
entry.offset = index.Read<std::uint64_t>();
entry.size = index.Read<std::uint64_t>();
if (entry.offset < HeaderSize || entry.offset > indexOffset || entry.size > indexOffset - entry.offset) {
throw std::runtime_error("Pack file is corrupt: " + entry.path + " is out of range");
}
if (!m_ByPath.emplace(entry.path, i).second) {
throw std::runtime_error("Pack file is corrupt: " + entry.path + " is listed twice");
}
}
index.ExpectEnd(Kind);
}
const PackFile::Entry* PackFile::Reader::Find(std::string_view path) const {
const auto it = m_ByPath.find(std::string{path});
return it != m_ByPath.end() ? &m_Entries[it->second] : nullptr;
}
// Opens its own stream so loads on worker threads don't share a file position
std::vector<std::uint8_t> PackFile::Reader::Read(const Entry& entry) const {
std::ifstream file(m_Path, std::ios::binary);
if (!file) {
throw std::runtime_error("Failed to open file: " + m_Path.string());
}
std::vector<std::uint8_t> bytes(static_cast<std::size_t>(entry.size));
file.seekg(static_cast<std::streamoff>(entry.offset));
file.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
if (!file) {
throw std::runtime_error("Failed to read " + entry.path + " from " + m_Path.string());
}
return bytes;
}
destrum/src/FS/AssetFS.cpp
+174 -39
View File
@@ -1,5 +1,6 @@
#include <destrum/FS/AssetFS.h>
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <stdexcept>
@@ -97,6 +98,79 @@ namespace {
return nullptr;
}
[[nodiscard]] std::optional<std::filesystem::path> RelativeToMount(const FSMount& mount,
const std::filesystem::path& fullPath) {
const std::filesystem::path relative = fullPath.lexically_normal().lexically_relative(mount.root);
if (relative.empty() || relative.is_absolute() || relative == "." ||
relative == ".." || relative.generic_string().starts_with("../")) {
return std::nullopt;
}
return relative;
}
struct PackLocation {
const FSMount* mount;
std::string key;
};
// Everything under a packed mount comes from the pack, even files that also exist on disk
[[nodiscard]] std::optional<PackLocation> FindInPack(const std::vector<FSMount>& mounts,
const std::filesystem::path& fullPath) {
for (const FSMount& mount : mounts) {
if (!mount.pack) {
continue;
}
if (const auto relative = RelativeToMount(mount, fullPath)) {
return PackLocation{&mount, mount.scheme + "/" + relative->generic_string()};
}
}
return std::nullopt;
}
[[nodiscard]] std::vector<uint8_t> ReadDiskFile(const std::filesystem::path& fullPath) {
std::ifstream file(fullPath, std::ios::binary);
if (!file) {
throw std::runtime_error("failed to open file: " + fullPath.string());
}
file.seekg(0, std::ios::end);
const std::streamsize size = file.tellg();
if (size < 0) {
throw std::runtime_error("failed to determine file size: " + fullPath.string());
}
file.seekg(0, std::ios::beg);
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
if (size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("failed to read file: " + fullPath.string());
}
return buffer;
}
[[nodiscard]] std::optional<AssetManifest> LoadManifest(const FSMount& mount) {
const std::filesystem::path path = mount.root / "manifest.json";
std::vector<uint8_t> bytes;
if (mount.pack) {
const PackFile::Entry* entry = mount.pack->Find(mount.scheme + "/manifest.json");
if (entry == nullptr) {
return std::nullopt;
}
bytes = mount.pack->Read(*entry);
} else {
std::error_code error;
if (!std::filesystem::is_regular_file(path, error)) {
return std::nullopt;
}
bytes = ReadDiskFile(path);
}
try {
return FS::ParseAssetManifest(bytes);
} catch (const std::exception& exception) {
throw std::runtime_error("invalid manifest " + path.string() + ": " + exception.what());
}
}
}
void AssetFS::Init(std::filesystem::path exeDir) {
@@ -106,8 +180,14 @@ void AssetFS::Init(std::filesystem::path exeDir) {
const std::size_t originalMountCount = mounts.size();
try {
Mount("engine", exeDir / "assets/engine");
Mount("game", exeDir / "assets/game");
const std::filesystem::path packPath = exeDir / PackName;
std::error_code error;
if (std::filesystem::is_regular_file(packPath, error)) {
MountPack(packPath, exeDir / "assets");
} else {
Mount("engine", exeDir / "assets/engine");
Mount("game", exeDir / "assets/game");
}
initialized = true;
} catch (...) {
mounts.resize(originalMountCount);
@@ -121,6 +201,40 @@ void AssetFS::Reset() {
}
void AssetFS::Mount(std::string scheme, std::filesystem::path root) {
AddMount(std::move(scheme), std::move(root), nullptr);
}
// Each scheme in the pack mounts at root/<scheme>, so full paths look the same as with the unpacked folders
void AssetFS::MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root) {
auto pack = std::make_shared<const PackFile::Reader>(packPath);
std::vector<std::string> schemes;
for (const PackFile::Entry& entry : pack->Entries()) {
const std::size_t separator = entry.path.find('/');
if (separator == std::string::npos || separator == 0) {
throw std::runtime_error("asset pack entry has no scheme: " + entry.path);
}
std::string scheme = entry.path.substr(0, separator);
if (std::ranges::find(schemes, scheme) == schemes.end()) {
schemes.push_back(std::move(scheme));
}
}
const std::size_t originalMountCount = mounts.size();
try {
for (std::string& scheme : schemes) {
std::filesystem::path schemeRoot = root / scheme;
AddMount(std::move(scheme), std::move(schemeRoot), pack);
}
} catch (...) {
mounts.resize(originalMountCount);
throw;
}
spdlog::info("Mounted asset pack {} ({} files)", packPath.string(), pack->Entries().size());
}
void AssetFS::AddMount(std::string scheme, std::filesystem::path root,
std::shared_ptr<const PackFile::Reader> pack) {
if (scheme.empty() || scheme.find_first_of("/\\:") != std::string::npos) {
throw std::runtime_error("asset mount scheme is invalid");
}
@@ -131,12 +245,8 @@ void AssetFS::Mount(std::string scheme, std::filesystem::path root) {
FSMount mount;
mount.scheme = std::move(scheme);
mount.root = std::filesystem::absolute(std::move(root)).lexically_normal();
const std::filesystem::path manifestPath = mount.root / "manifest.json";
std::error_code error;
if (std::filesystem::is_regular_file(manifestPath, error)) {
mount.manifest = FS::LoadAssetManifest(manifestPath);
}
mount.pack = std::move(pack);
mount.manifest = LoadManifest(mount);
spdlog::debug("Mounting assetfs scheme '{}' to root '{}'", mount.scheme, mount.root.string());
mounts.push_back(std::move(mount));
@@ -147,12 +257,8 @@ void AssetFS::ReloadManifest(std::string_view scheme) {
if (mount.scheme != scheme) {
continue;
}
const std::filesystem::path manifestPath = mount.root / "manifest.json";
std::error_code error;
mount.manifest.reset();
if (std::filesystem::is_regular_file(manifestPath, error)) {
mount.manifest = FS::LoadAssetManifest(manifestPath);
}
mount.manifest = LoadManifest(mount);
return;
}
throw std::runtime_error("asset mount not found: " + std::string{scheme});
@@ -162,6 +268,51 @@ std::vector<uint8_t> AssetFS::ReadBytes(std::string_view vpath) {
return ReadFile(GetFullPath(vpath));
}
std::vector<uint8_t> AssetFS::ReadFile(const std::filesystem::path& fullPath) const {
if (const auto location = FindInPack(mounts, fullPath)) {
const PackFile::Entry* entry = location->mount->pack->Find(location->key);
if (entry == nullptr) {
throw std::runtime_error("failed to open file: " + fullPath.string() + " is not in " +
location->mount->pack->Path().string());
}
return location->mount->pack->Read(*entry);
}
return ReadDiskFile(fullPath);
}
bool AssetFS::Exists(const std::filesystem::path& fullPath) const {
if (const auto location = FindInPack(mounts, fullPath)) {
return location->mount->pack->Find(location->key) != nullptr;
}
std::error_code error;
return std::filesystem::is_regular_file(fullPath, error);
}
std::vector<std::filesystem::path> AssetFS::ListFiles(const FSMount& mount) const {
std::vector<std::filesystem::path> files;
if (mount.pack) {
const std::string prefix = mount.scheme + "/";
for (const PackFile::Entry& entry : mount.pack->Entries()) {
if (entry.path.starts_with(prefix)) {
files.push_back((mount.root / entry.path.substr(prefix.size())).lexically_normal());
}
}
return files;
}
std::error_code error;
for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) {
if (entry.is_regular_file(error)) {
files.push_back(entry.path());
}
}
return files;
}
bool AssetFS::IsPacked() const {
return std::ranges::any_of(mounts, [](const FSMount& mount) { return mount.pack != nullptr; });
}
std::filesystem::path AssetFS::GetFullPath(std::string_view vpath) const {
if (!initialized) {
throw std::runtime_error("AssetFS is not initialized");
@@ -194,6 +345,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons
const std::filesystem::path cookedRelative = ValidateRelativePath(*asset->out);
const std::filesystem::path cookedPath = ResolvePath(*mount, cookedRelative);
if (mount->pack && Exists(cookedPath)) {
return cookedPath;
}
std::error_code error;
if (std::filesystem::is_regular_file(cookedPath, error)) {
error.clear();
@@ -218,14 +373,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons
}
std::optional<std::string> AssetFS::ToVirtualPath(const std::filesystem::path& fullPath) const {
const std::filesystem::path normalized = fullPath.lexically_normal();
for (const FSMount& mount : mounts) {
const std::filesystem::path relative = normalized.lexically_relative(mount.root);
if (relative.empty() || relative.is_absolute() || relative == "." ||
relative == ".." || relative.generic_string().starts_with("../")) {
continue;
if (const auto relative = RelativeToMount(mount, fullPath)) {
return mount.scheme + "://" + relative->generic_string();
}
return mount.scheme + "://" + relative.generic_string();
}
return std::nullopt;
}
@@ -251,6 +402,10 @@ std::optional<std::filesystem::path> AssetFS::GetSourcePath(std::string_view vpa
bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) {
std::vector<std::filesystem::path> targets{GetFullPath(vpath)};
if (FindInPack(mounts, targets.front())) {
spdlog::error("Can't write {}, it's in a read only asset pack", vpath);
return false;
}
if (auto source = GetSourcePath(vpath)) {
targets.push_back(std::move(*source));
}
@@ -276,23 +431,3 @@ bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) {
}
return true;
}
std::vector<uint8_t> AssetFS::ReadFile(const std::filesystem::path& fullPath) {
std::ifstream file(fullPath, std::ios::binary);
if (!file) {
throw std::runtime_error("failed to open file: " + fullPath.string());
}
file.seekg(0, std::ios::end);
const std::streamsize size = file.tellg();
if (size < 0) {
throw std::runtime_error("failed to determine file size: " + fullPath.string());
}
file.seekg(0, std::ios::beg);
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
if (size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("failed to read file: " + fullPath.string());
}
return buffer;
}
destrum/src/FS/Manifest.cpp
+2 -9
View File
@@ -1,7 +1,6 @@
#include <filesystem>
#include <destrum/FS/Manifest.h>
#include <fstream>
#include <stdexcept>
#include <nlohmann/json.hpp>
@@ -27,14 +26,8 @@ namespace {
}
}
AssetManifest FS::LoadAssetManifest(const std::filesystem::path& manifestPath) {
std::ifstream f(manifestPath);
if (!f) {
throw std::runtime_error("Failed to open manifest: " + manifestPath.string());
}
nlohmann::json j;
f >> j;
AssetManifest FS::ParseAssetManifest(std::span<const std::uint8_t> bytes) {
const nlohmann::json j = nlohmann::json::parse(bytes.begin(), bytes.end());
AssetManifest manifest;
manifest.version = j.value("version", 0);
destrum/src/Graphics/ComputePipeline.cpp
+3 -15
View File
@@ -2,10 +2,10 @@
#include <cassert>
#include <filesystem>
#include <fstream>
#include <iostream>
#include "volk.h"
#include "destrum/FS/AssetFS.h"
#include "destrum/Util/GameState.h"
#include "spdlog/spdlog.h"
@@ -46,20 +46,8 @@ void ComputePipeline::DefaultPipelineConfigInfo(ComputePipelineConfigInfo& confi
}
std::vector<char> ComputePipeline::readFile(const std::string& filename) {
std::ifstream file(filename, std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("Failed to open file: " + filename);
}
const size_t fileSize = static_cast<size_t>(file.tellg());
std::vector<char> buffer(fileSize);
file.seekg(0);
file.read(buffer.data(), static_cast<std::streamsize>(fileSize));
file.close();
return buffer;
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(filename);
return {bytes.begin(), bytes.end()};
}
void ComputePipeline::CreateComputePipeline(const std::string& compPath,
destrum/src/Graphics/ImageLoader.cpp
+21 -9
View File
@@ -12,6 +12,7 @@
#include "spdlog/spdlog.h"
#include <destrum/Assets/TextureFile.h>
#include <destrum/FS/AssetFS.h>
ImageData::~ImageData()
{
@@ -69,10 +70,10 @@ namespace util
std::filesystem::path cooked = p;
cooked += TextureFile::Extension;
std::error_code error;
if (!std::filesystem::is_regular_file(cooked, error)) {
if (!AssetFS::GetInstance().Exists(cooked)) {
return {};
}
std::error_code error;
const auto sourceTime = std::filesystem::last_write_time(p, error);
if (!error && sourceTime > std::filesystem::last_write_time(cooked, error)) {
return {};
@@ -94,7 +95,7 @@ namespace util
if (isCookedTextureExt(p)) {
try {
TextureFile::Texture texture = TextureFile::ReadFromFile(p);
TextureFile::Texture texture = TextureFile::Read(AssetFS::GetInstance().ReadFile(p));
data.cookedPixels = std::move(texture.pixels);
data.pixels = data.cookedPixels.data();
@@ -111,12 +112,21 @@ namespace util
return data;
}
std::vector<uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(p);
} catch (const std::exception& exception) {
spdlog::error("Failed to read image '{}': {}", p.string(), exception.what());
return {};
}
const int byteCount = static_cast<int>(bytes.size());
if (isExrExt(p)) {
float* out = nullptr;
int w = 0, h = 0;
const char* err = nullptr;
const int ret = LoadEXR(&out, &w, &h, p.string().c_str(), &err);
const int ret = LoadEXRFromMemory(&out, &w, &h, bytes.data(), bytes.size(), &err);
if (ret != TINYEXR_SUCCESS || !out || w <= 0 || h <= 0) {
if (err) {
spdlog::error("TinyEXR error: {}", err);
@@ -140,10 +150,11 @@ namespace util
// ---------- STB HDR (.hdr etc) ----------
data.shouldSTBFree = true;
if (stbi_is_hdr(p.string().c_str())) {
if (stbi_is_hdr_from_memory(bytes.data(), byteCount)) {
data.hdr = true;
data.hdrPixels = stbi_loadf(
p.string().c_str(),
data.hdrPixels = stbi_loadf_from_memory(
bytes.data(),
byteCount,
&data.width,
&data.height,
&data.comp,
@@ -161,8 +172,9 @@ namespace util
}
// ---------- STB LDR (png/jpg/...) ----------
data.pixels = stbi_load(
p.string().c_str(),
data.pixels = stbi_load_from_memory(
bytes.data(),
byteCount,
&data.width,
&data.height,
&data.channels,
destrum/src/Graphics/Pipeline.cpp
+3 -16
View File
@@ -2,10 +2,10 @@
#include <cassert>
#include <filesystem>
#include <fstream>
#include <iostream>
#include "volk.h"
#include "destrum/FS/AssetFS.h"
#include "destrum/Graphics/Util.h"
#include "spdlog/spdlog.h"
@@ -115,21 +115,8 @@ void Pipeline::DefaultPipelineConfigInfo(PipelineConfigInfo& configInfo) {
}
std::vector<char> Pipeline::readFile(const std::string& filename) {
std::ifstream file(filename, std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("Failed to open file: " + filename);
}
const size_t fileSize = (size_t)file.tellg();
std::vector<char> buffer(fileSize);
file.seekg(0);
file.read(buffer.data(), static_cast<std::streamsize>(fileSize));
file.close();
return buffer;
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(filename);
return {bytes.begin(), bytes.end()};
}
void Pipeline::CreateGraphicsPipeline(const std::string& vertPath, const std::string& fragPath,
destrum/src/Graphics/Resources/Font.cpp
+2 -7
View File
@@ -2,8 +2,6 @@
#include <algorithm>
#include <cstring>
#include <fstream>
#include <iterator>
#include <stdexcept>
#include <string>
@@ -13,6 +11,7 @@
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/Graphics/GfxDevice.h>
#include <destrum/Graphics/RenderResources.h>
@@ -57,11 +56,7 @@ namespace {
std::vector<FT_Byte> readFile(const std::filesystem::path& path)
{
std::ifstream file(path, std::ios::binary);
if (!file) {
throw std::runtime_error("Font: failed to open " + path.string());
}
return {std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()};
return AssetFS::GetInstance().ReadFile(path);
}
void check(FT_Error error, const char* what, const std::filesystem::path& path)
destrum/src/Graphics/Util.cpp
+6 -15
View File
@@ -1,6 +1,7 @@
#include <fstream>
#include <cstring>
#include <destrum/Graphics/Init.h>
#include <destrum/Graphics/Util.h>
#include <destrum/FS/AssetFS.h>
#include <volk.h>
@@ -401,22 +402,12 @@ vkutil::RenderInfo vkutil::createRenderingInfo(const RenderingInfoParams& params
}
VkShaderModule vkutil::loadShaderModule(const std::filesystem::path& path, VkDevice device) {
std::ifstream file(path, std::ios::ate | std::ios::binary);
if (!file.is_open()) {
throw std::runtime_error("Failed to open shader: " + path.string());
}
const auto fileSize = file.tellg();
if (fileSize < 0 || fileSize % static_cast<std::streamoff>(sizeof(std::uint32_t)) != 0) {
const std::vector<std::uint8_t> bytes = AssetFS::GetInstance().ReadFile(path);
if (bytes.size() % sizeof(std::uint32_t) != 0) {
throw std::runtime_error("Shader file has an invalid size: " + path.string());
}
std::vector<std::uint32_t> buffer(fileSize / sizeof(std::uint32_t));
file.seekg(0);
if (!file.read(reinterpret_cast<char*>(buffer.data()), fileSize)) {
throw std::runtime_error("Failed to read shader: " + path.string());
}
file.close();
std::vector<std::uint32_t> buffer(bytes.size() / sizeof(std::uint32_t));
std::memcpy(buffer.data(), bytes.data(), bytes.size());
auto info = VkShaderModuleCreateInfo{
.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
destrum/src/Serialization/ObjectSerialization.cpp
+1 -1
View File
@@ -151,7 +151,7 @@ namespace {
fullPath = path;
}
return std::filesystem::exists(fullPath);
return AssetFS::GetInstance().Exists(fullPath);
}
void CollectHierarchyRecursive(const GameObject& object, std::vector<const GameObject*>& out,
destrum/src/Serialization/PrefabCache.cpp
+4 -3
View File
@@ -14,12 +14,13 @@ std::shared_ptr<const nlohmann::json> PrefabCache::Get(std::string_view path) {
return nullptr;
}
std::error_code error;
const auto writeTime = std::filesystem::last_write_time(fullPath, error);
if (error) {
if (!AssetFS::GetInstance().Exists(fullPath)) {
spdlog::error("Prefab file not found: {}", fullPath.string());
return nullptr;
}
// Packed files have no write time, they can't change anyway
std::error_code error;
const auto writeTime = std::filesystem::last_write_time(fullPath, error);
const std::string key = MakeKey(fullPath);
if (const auto it = m_Prefabs.find(key); it != m_Prefabs.end() && it->second.writeTime == writeTime) {
destrum/src/Serialization/PrefabSerializer.cpp
+9 -5
View File
@@ -1,9 +1,11 @@
#include <destrum/Serialization/PrefabSerializer.h>
#include <fstream>
#include <cstdint>
#include <vector>
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/ObjectModel/GameObject.h>
#include <destrum/ObjectModel/Transform.h>
#include <destrum/Scene/Scene.h>
@@ -100,15 +102,17 @@ bool PrefabSerializer::Save(const GameObject& root, const std::filesystem::path&
}
std::optional<json> PrefabSerializer::LoadFile(const std::filesystem::path& path) {
std::ifstream file(path);
if (!file.is_open()) {
spdlog::error("Failed to open prefab file for reading: {}", path.string());
std::vector<std::uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(path);
} catch (const std::exception& exception) {
spdlog::error("Failed to open prefab file for reading: {} ({})", path.string(), exception.what());
return std::nullopt;
}
json prefab;
try {
file >> prefab;
prefab = json::parse(bytes.begin(), bytes.end());
} catch (const std::exception& exception) {
spdlog::error("Failed to parse prefab file {}: {}", path.string(), exception.what());
return std::nullopt;
destrum/src/Serialization/SceneSerializer.cpp
+8 -5
View File
@@ -1,11 +1,12 @@
#include <destrum/Serialization/SceneSerializer.h>
#include <fstream>
#include <cstdint>
#include <vector>
#include <nlohmann/json.hpp>
#include <spdlog/spdlog.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/Scene/Scene.h>
#include <destrum/ObjectModel/GameObject.h>
#include <destrum/ObjectModel/ObjectId.h>
@@ -152,15 +153,17 @@ SceneSerializer::LoadResult SceneSerializer::LoadSceneFile(const std::filesystem
LoadResult result;
RegisterEngineComponents();
std::ifstream file(path);
if (!file.is_open()) {
result.errorMessage = "Failed to open scene file for reading: " + path.string();
std::vector<std::uint8_t> bytes;
try {
bytes = AssetFS::GetInstance().ReadFile(path);
} catch (const std::exception& exception) {
result.errorMessage = "Failed to open scene file for reading: " + path.string() + " (" + exception.what() + ")";
spdlog::error("{}", result.errorMessage);
return result;
}
try {
file >> result.root;
result.root = json::parse(bytes.begin(), bytes.end());
} catch (const std::exception& exception) {
result.errorMessage = "Failed to parse scene file " + path.string() + ": " + exception.what();
spdlog::error("{}", result.errorMessage);