brammie15/Destrum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add export / file packing

Browse Source
This commit is contained in:
brammie15
2026-10-30 15:42:08 +01:00
parent de7e489cc0
commit 0fbc023018
28 changed files with 782 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/assets_tests.cpp
+113
View File
@@ -10,6 +10,7 @@
#include <destrum/Assets/AssetTypes.h>
#include <destrum/Assets/AssetReference.h>
#include <destrum/Assets/ModelFile.h>
#include <destrum/Assets/PackFile.h>
#include <destrum/Assets/TextureFile.h>
#include <destrum/FS/AssetFS.h>
#include <destrum/ObjectModel/Component.h>
@@ -458,6 +459,116 @@ namespace {
"loading a source image must use its cooked sibling");
}
bool packOpenThrows(const std::filesystem::path& path)
{
try {
(void)PackFile::Reader{path};
} catch (const std::exception&) {
return true;
}
return false;
}
void testPackFileRoundTrip()
{
const auto dir = std::filesystem::temp_directory_path() / "destrum_pack_file_test";
std::filesystem::remove_all(dir);
std::filesystem::create_directories(dir);
std::ofstream{dir / "a.txt", std::ios::binary} << "hello";
std::ofstream{dir / "empty.txt", std::ios::binary};
const auto pack = dir / "test.dpak";
PackFile::Write({{"engine/a.txt", dir / "a.txt"}, {"game/empty.txt", dir / "empty.txt"}}, pack);
const PackFile::Reader reader{pack};
const PackFile::Entry* text = reader.Find("engine/a.txt");
const PackFile::Entry* empty = reader.Find("game/empty.txt");
check(reader.Entries().size() == 2 && text != nullptr && empty != nullptr && !reader.Find("a.txt"),
"packs must list every file by its full path");
const auto bytes = reader.Read(*text);
check(std::string(bytes.begin(), bytes.end()) == "hello" && reader.Read(*empty).empty(),
"packed files must read back as written");
bool duplicateRejected = false;
try {
PackFile::Write({{"engine/a.txt", dir / "a.txt"}, {"engine/a.txt", dir / "a.txt"}}, dir / "dupe.dpak");
} catch (const std::exception&) {
duplicateRejected = true;
}
check(duplicateRejected, "packs can't hold two files at one path");
std::ofstream{dir / "bad.dpak", std::ios::binary} << "DTEX not a pack";
check(packOpenThrows(dir / "bad.dpak"), "files that aren't packs must be rejected");
const auto size = std::filesystem::file_size(pack);
std::filesystem::resize_file(pack, size - 4);
check(packOpenThrows(pack), "truncated packs must be rejected");
std::filesystem::remove_all(dir);
}
void testAssetFSMountsPacks()
{
const auto root = std::filesystem::temp_directory_path() / "destrum_packed_assets_test";
const auto staging = root / "staging";
std::filesystem::remove_all(root);
std::filesystem::create_directories(staging);
std::ofstream{staging / "notes.txt", std::ios::binary} << "packed";
std::ofstream{staging / "notes.txt.meta", std::ios::binary} << R"({"guid":"0123456789abcdef0123456789abcdef"})";
std::ofstream{staging / "model.fbx", std::ios::binary} << "not a model";
ModelFile::WriteToFile(makeTestModelAsset(), staging / "model.fbx.dmodel");
const std::vector<std::uint8_t> rgba{255, 0, 0, 255};
TextureFile::WriteToFile(TextureFile::FromRGBA8(1, 1, rgba), staging / "tex.png.dtex");
std::ofstream{staging / "manifest.json", std::ios::binary}
<< R"({"version":2,"assets":[{"src":"model.fbx","out":"model.fbx.dmodel","type":"model"}]})";
PackFile::Write({
{"engine/manifest.json", staging / "manifest.json"},
{"engine/model.fbx", staging / "model.fbx"},
{"engine/model.fbx.dmodel", staging / "model.fbx.dmodel"},
{"engine/tex.png.dtex", staging / "tex.png.dtex"},
{"game/notes.txt", staging / "notes.txt"},
{"game/notes.txt.meta", staging / "notes.txt.meta"},
}, root / AssetFS::PackName);
// A loose copy next to the pack must not be picked up
std::filesystem::create_directories(root / "assets" / "game");
std::ofstream{root / "assets" / "game" / "notes.txt", std::ios::binary} << "loose";
std::ofstream{root / "assets" / "game" / "loose.txt", std::ios::binary} << "loose";
auto& assetFs = AssetFS::GetInstance();
assetFs.Init(root);
check(assetFs.IsPacked() && assetFs.GetMounts().size() == 2, "a pack next to the exe must mount every scheme in it");
const auto notes = assetFs.ReadBytes("game://notes.txt");
check(std::string(notes.begin(), notes.end()) == "packed", "packed mounts must read from the pack");
check(assetFs.GetFullPath("game://notes.txt") == (root / "assets" / "game" / "notes.txt").lexically_normal(),
"packed files must keep the full path they have unpacked");
check(assetFs.Exists(assetFs.GetFullPath("game://notes.txt")) &&
!assetFs.Exists(assetFs.GetFullPath("game://loose.txt")),
"only packed files exist under a packed mount");
check(assetFs.ToVirtualPath(assetFs.GetFullPath("engine://tex.png")) == "engine://tex.png",
"packed full paths must map back to virtual paths");
check(!assetFs.WriteFile("game://notes.txt", "changed"), "packed mounts are read only");
const auto model = AssetManager::GetInstance().LoadModel("engine://model.fbx");
check(model.primitives.size() == 1, "cooked models must load from the pack");
const auto image = util::loadImage(assetFs.GetFullPath("engine://tex.png"), TextureIntent::ColorSrgb);
check(image.width == 1 && image.pixels != nullptr && std::memcmp(image.pixels, rgba.data(), rgba.size()) == 0,
"images must find their cooked sibling in the pack");
auto& database = AssetDatabase::GetInstance();
database.Refresh();
check(database.FindPath(*AssetGuid::Parse("0123456789abcdef0123456789abcdef")) == "game://notes.txt",
"asset ids must be read from packed meta files");
database.Clear();
assetFs.Reset();
std::filesystem::remove_all(root);
}
void testAssetReferenceCacheKeys()
{
const AssetReference reference{
@@ -555,6 +666,8 @@ void RunAssetsTests()
testTextureFileRoundTrip();
testImageLoaderReadsCookedTextures();
testLdrTextureFile();
testPackFileRoundTrip();
testAssetFSMountsPacks();
testAssetReferenceCacheKeys();
testAssetGuids();
testAssetDatabaseFollowsMovedAssets();
tests/cooker_tests.cpp
+49 -3
View File
@@ -9,6 +9,8 @@
#include <nlohmann/json.hpp>
#include <destrum/Assets/PackFile.h>
#include "TestUtils.h"
using namespace test;
@@ -51,11 +53,10 @@ namespace {
}
}
int runChef(const CookDirs& dirs, const std::string& flags = {})
int runChefArgs(const CookDirs& dirs, const std::string& args)
{
const fs::path log = dirs.root / "chef.log";
std::string command = "\"" + std::string(DESTRUM_THECHEF_PATH) + "\" --input \"" + dirs.input.string() +
"\" --output \"" + dirs.output.string() + "\" " + flags + " > \"" + log.string() +
std::string command = "\"" + std::string(DESTRUM_THECHEF_PATH) + "\" " + args + " > \"" + log.string() +
"\" 2>&1";
#ifdef _WIN32
// cmd.exe strips the outer quotes, the inner ones keep paths with spaces intact
@@ -64,6 +65,11 @@ namespace {
return std::system(command.c_str());
}
int runChef(const CookDirs& dirs, const std::string& flags = {})
{
return runChefArgs(dirs, "--input \"" + dirs.input.string() + "\" --output \"" + dirs.output.string() + "\" " + flags);
}
void expectChef(const CookDirs& dirs, const std::string& flags, bool succeeds, const char* message)
{
if ((runChef(dirs, flags) == 0) != succeeds) {
@@ -307,6 +313,45 @@ namespace {
}
}
namespace {
void testPackBundlesCookedOutputs()
{
const CookDirs dirs = makeDirs("destrum_cooker_pack");
writeShaders(dirs);
writeText(dirs.input / "notes.txt", "hello");
expectChef(dirs, "", true, "the first cook must succeed");
writeText(dirs.output / "stray.txt", "not cooked");
const fs::path pack = dirs.root / "assets.dpak";
const std::string packArgs = "--pack \"" + pack.string() + "\" --mount \"game=" + dirs.output.string() + "\"";
if (runChefArgs(dirs, packArgs) != 0) {
std::cerr << readFile(dirs.root / "chef.log") << '\n';
check(false, "packing a cooked folder must succeed");
return;
}
const PackFile::Reader reader{pack};
check(reader.Find("game/manifest.json") && reader.Find("game/shaders/plain.frag.spv") &&
reader.Find("game/notes.txt"),
"packs must hold the manifest and every cooked output");
check(!reader.Find("game/stray.txt") && !reader.Find("game/shaders/include/common.glsl"),
"packs must only hold what the manifest lists as outputs");
const auto notes = reader.Read(*reader.Find("game/notes.txt"));
check(std::string(notes.begin(), notes.end()) == "hello", "packed files must keep their contents");
check(runChefArgs(dirs, "--list \"" + pack.string() + "\" --top 0") == 0 &&
readFile(dirs.root / "chef.log").find("game/notes.txt") != std::string::npos,
"listing a pack must show its files");
check(runChefArgs(dirs, "--list \"" + dirs.output.string() + "\"") == 0 &&
readFile(dirs.root / "chef.log").find("Not in the manifest, won't be packed: 1 files") != std::string::npos,
"listing a cooked folder must point out files that won't be packed");
fs::remove(dirs.output / "notes.txt");
check(runChefArgs(dirs, packArgs) != 0, "packing must fail when a listed output is missing");
check(!fs::exists(pack.string() + ".tmp"), "a failed pack must not leave a temp file");
}
}
void RunCookerTests()
{
testCookRecordsEveryOutput();
@@ -317,4 +362,5 @@ void RunCookerTests()
testFailedCookKeepsLastGoodOutput();
testFallbacksAndStrictMode();
testValidateFindsMissingOutputs();
testPackBundlesCookedOutputs();
}