brammie15/Destrum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add export / file packing

Browse Source
This commit is contained in:
brammie15
2026-10-30 15:42:08 +01:00
parent de7e489cc0
commit 0fbc023018
28 changed files with 782 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
destrum/src/FS/AssetFS.cpp
+174 -39
View File
@@ -1,5 +1,6 @@
#include <destrum/FS/AssetFS.h>
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <stdexcept>
@@ -97,6 +98,79 @@ namespace {
return nullptr;
}
[[nodiscard]] std::optional<std::filesystem::path> RelativeToMount(const FSMount& mount,
const std::filesystem::path& fullPath) {
const std::filesystem::path relative = fullPath.lexically_normal().lexically_relative(mount.root);
if (relative.empty() || relative.is_absolute() || relative == "." ||
relative == ".." || relative.generic_string().starts_with("../")) {
return std::nullopt;
}
return relative;
}
struct PackLocation {
const FSMount* mount;
std::string key;
};
// Everything under a packed mount comes from the pack, even files that also exist on disk
[[nodiscard]] std::optional<PackLocation> FindInPack(const std::vector<FSMount>& mounts,
const std::filesystem::path& fullPath) {
for (const FSMount& mount : mounts) {
if (!mount.pack) {
continue;
}
if (const auto relative = RelativeToMount(mount, fullPath)) {
return PackLocation{&mount, mount.scheme + "/" + relative->generic_string()};
}
}
return std::nullopt;
}
[[nodiscard]] std::vector<uint8_t> ReadDiskFile(const std::filesystem::path& fullPath) {
std::ifstream file(fullPath, std::ios::binary);
if (!file) {
throw std::runtime_error("failed to open file: " + fullPath.string());
}
file.seekg(0, std::ios::end);
const std::streamsize size = file.tellg();
if (size < 0) {
throw std::runtime_error("failed to determine file size: " + fullPath.string());
}
file.seekg(0, std::ios::beg);
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
if (size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("failed to read file: " + fullPath.string());
}
return buffer;
}
[[nodiscard]] std::optional<AssetManifest> LoadManifest(const FSMount& mount) {
const std::filesystem::path path = mount.root / "manifest.json";
std::vector<uint8_t> bytes;
if (mount.pack) {
const PackFile::Entry* entry = mount.pack->Find(mount.scheme + "/manifest.json");
if (entry == nullptr) {
return std::nullopt;
}
bytes = mount.pack->Read(*entry);
} else {
std::error_code error;
if (!std::filesystem::is_regular_file(path, error)) {
return std::nullopt;
}
bytes = ReadDiskFile(path);
}
try {
return FS::ParseAssetManifest(bytes);
} catch (const std::exception& exception) {
throw std::runtime_error("invalid manifest " + path.string() + ": " + exception.what());
}
}
}
void AssetFS::Init(std::filesystem::path exeDir) {
@@ -106,8 +180,14 @@ void AssetFS::Init(std::filesystem::path exeDir) {
const std::size_t originalMountCount = mounts.size();
try {
Mount("engine", exeDir / "assets/engine");
Mount("game", exeDir / "assets/game");
const std::filesystem::path packPath = exeDir / PackName;
std::error_code error;
if (std::filesystem::is_regular_file(packPath, error)) {
MountPack(packPath, exeDir / "assets");
} else {
Mount("engine", exeDir / "assets/engine");
Mount("game", exeDir / "assets/game");
}
initialized = true;
} catch (...) {
mounts.resize(originalMountCount);
@@ -121,6 +201,40 @@ void AssetFS::Reset() {
}
void AssetFS::Mount(std::string scheme, std::filesystem::path root) {
AddMount(std::move(scheme), std::move(root), nullptr);
}
// Each scheme in the pack mounts at root/<scheme>, so full paths look the same as with the unpacked folders
void AssetFS::MountPack(const std::filesystem::path& packPath, const std::filesystem::path& root) {
auto pack = std::make_shared<const PackFile::Reader>(packPath);
std::vector<std::string> schemes;
for (const PackFile::Entry& entry : pack->Entries()) {
const std::size_t separator = entry.path.find('/');
if (separator == std::string::npos || separator == 0) {
throw std::runtime_error("asset pack entry has no scheme: " + entry.path);
}
std::string scheme = entry.path.substr(0, separator);
if (std::ranges::find(schemes, scheme) == schemes.end()) {
schemes.push_back(std::move(scheme));
}
}
const std::size_t originalMountCount = mounts.size();
try {
for (std::string& scheme : schemes) {
std::filesystem::path schemeRoot = root / scheme;
AddMount(std::move(scheme), std::move(schemeRoot), pack);
}
} catch (...) {
mounts.resize(originalMountCount);
throw;
}
spdlog::info("Mounted asset pack {} ({} files)", packPath.string(), pack->Entries().size());
}
void AssetFS::AddMount(std::string scheme, std::filesystem::path root,
std::shared_ptr<const PackFile::Reader> pack) {
if (scheme.empty() || scheme.find_first_of("/\\:") != std::string::npos) {
throw std::runtime_error("asset mount scheme is invalid");
}
@@ -131,12 +245,8 @@ void AssetFS::Mount(std::string scheme, std::filesystem::path root) {
FSMount mount;
mount.scheme = std::move(scheme);
mount.root = std::filesystem::absolute(std::move(root)).lexically_normal();
const std::filesystem::path manifestPath = mount.root / "manifest.json";
std::error_code error;
if (std::filesystem::is_regular_file(manifestPath, error)) {
mount.manifest = FS::LoadAssetManifest(manifestPath);
}
mount.pack = std::move(pack);
mount.manifest = LoadManifest(mount);
spdlog::debug("Mounting assetfs scheme '{}' to root '{}'", mount.scheme, mount.root.string());
mounts.push_back(std::move(mount));
@@ -147,12 +257,8 @@ void AssetFS::ReloadManifest(std::string_view scheme) {
if (mount.scheme != scheme) {
continue;
}
const std::filesystem::path manifestPath = mount.root / "manifest.json";
std::error_code error;
mount.manifest.reset();
if (std::filesystem::is_regular_file(manifestPath, error)) {
mount.manifest = FS::LoadAssetManifest(manifestPath);
}
mount.manifest = LoadManifest(mount);
return;
}
throw std::runtime_error("asset mount not found: " + std::string{scheme});
@@ -162,6 +268,51 @@ std::vector<uint8_t> AssetFS::ReadBytes(std::string_view vpath) {
return ReadFile(GetFullPath(vpath));
}
std::vector<uint8_t> AssetFS::ReadFile(const std::filesystem::path& fullPath) const {
if (const auto location = FindInPack(mounts, fullPath)) {
const PackFile::Entry* entry = location->mount->pack->Find(location->key);
if (entry == nullptr) {
throw std::runtime_error("failed to open file: " + fullPath.string() + " is not in " +
location->mount->pack->Path().string());
}
return location->mount->pack->Read(*entry);
}
return ReadDiskFile(fullPath);
}
bool AssetFS::Exists(const std::filesystem::path& fullPath) const {
if (const auto location = FindInPack(mounts, fullPath)) {
return location->mount->pack->Find(location->key) != nullptr;
}
std::error_code error;
return std::filesystem::is_regular_file(fullPath, error);
}
std::vector<std::filesystem::path> AssetFS::ListFiles(const FSMount& mount) const {
std::vector<std::filesystem::path> files;
if (mount.pack) {
const std::string prefix = mount.scheme + "/";
for (const PackFile::Entry& entry : mount.pack->Entries()) {
if (entry.path.starts_with(prefix)) {
files.push_back((mount.root / entry.path.substr(prefix.size())).lexically_normal());
}
}
return files;
}
std::error_code error;
for (const auto& entry : std::filesystem::recursive_directory_iterator(mount.root, error)) {
if (entry.is_regular_file(error)) {
files.push_back(entry.path());
}
}
return files;
}
bool AssetFS::IsPacked() const {
return std::ranges::any_of(mounts, [](const FSMount& mount) { return mount.pack != nullptr; });
}
std::filesystem::path AssetFS::GetFullPath(std::string_view vpath) const {
if (!initialized) {
throw std::runtime_error("AssetFS is not initialized");
@@ -194,6 +345,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons
const std::filesystem::path cookedRelative = ValidateRelativePath(*asset->out);
const std::filesystem::path cookedPath = ResolvePath(*mount, cookedRelative);
if (mount->pack && Exists(cookedPath)) {
return cookedPath;
}
std::error_code error;
if (std::filesystem::is_regular_file(cookedPath, error)) {
error.clear();
@@ -218,14 +373,10 @@ std::filesystem::path AssetFS::GetCookedPathForFile(std::string_view vpath) cons
}
std::optional<std::string> AssetFS::ToVirtualPath(const std::filesystem::path& fullPath) const {
const std::filesystem::path normalized = fullPath.lexically_normal();
for (const FSMount& mount : mounts) {
const std::filesystem::path relative = normalized.lexically_relative(mount.root);
if (relative.empty() || relative.is_absolute() || relative == "." ||
relative == ".." || relative.generic_string().starts_with("../")) {
continue;
if (const auto relative = RelativeToMount(mount, fullPath)) {
return mount.scheme + "://" + relative->generic_string();
}
return mount.scheme + "://" + relative.generic_string();
}
return std::nullopt;
}
@@ -251,6 +402,10 @@ std::optional<std::filesystem::path> AssetFS::GetSourcePath(std::string_view vpa
bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) {
std::vector<std::filesystem::path> targets{GetFullPath(vpath)};
if (FindInPack(mounts, targets.front())) {
spdlog::error("Can't write {}, it's in a read only asset pack", vpath);
return false;
}
if (auto source = GetSourcePath(vpath)) {
targets.push_back(std::move(*source));
}
@@ -276,23 +431,3 @@ bool AssetFS::WriteFile(std::string_view vpath, std::string_view contents) {
}
return true;
}
std::vector<uint8_t> AssetFS::ReadFile(const std::filesystem::path& fullPath) {
std::ifstream file(fullPath, std::ios::binary);
if (!file) {
throw std::runtime_error("failed to open file: " + fullPath.string());
}
file.seekg(0, std::ios::end);
const std::streamsize size = file.tellg();
if (size < 0) {
throw std::runtime_error("failed to determine file size: " + fullPath.string());
}
file.seekg(0, std::ios::beg);
std::vector<uint8_t> buffer(static_cast<std::size_t>(size));
if (size > 0 && !file.read(reinterpret_cast<char*>(buffer.data()), size)) {
throw std::runtime_error("failed to read file: " + fullPath.string());
}
return buffer;
}