diff options
| author | Lywx <kiritodev01@gmail.com> | 2023-11-15 15:11:15 -0600 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-11-15 15:11:15 -0600 |
| commit | eaddecd0b402c12a780831e3139d7a35e78322e6 (patch) | |
| tree | 0c961e1ba2339f52892b54f44b8ff0018b77c382 /src/Companion.cpp | |
| parent | a66b259d7c10c3367a04819c6b55d47fb9a99f5e (diff) | |
Torch v0.1 (#12)
* Splitted version yamls and implemented wip geo and dlist extraction
* Fixed linux compilation
* Added PoC WSL 2 detection
* Add undefined behaviour
* Added gfxdis as submodule
* update
* Removed gfxdis as a submodule
* Code works
* Replace ByteSwap with BSWAP
* Cleanup vertice factory
* readability
* readability 2
* Cleanup gfxfactory
* last one
* Rename name to symbol
* More implementation
* Implemented first PoC of C file generation
* Added output formatting on factories
* changes
* Moved generated dlists onto their respective folder
* Migrated more factories onto the new extractor system
* Fully ported all factories and added header generation
* Some updates
* Added Lights factory
* fix
* Fixed vtx extraction
* Lights work now
* delete old startup_logo yaml
* Generate all sections in single file
* Added support to generate header and code for textures
* Added O3 for release builds
* Added WIP dlist extraction and moved to use f3dold microcode
* Added segment support, autogeneration and fixed C generation
* Added symbol names on display list extraction
* Removed warnings
* Moved back to transform and replace
* Removed unnecesary macros
* Added extraction order and reversed vtx order
* Implemented true backwards sort
* [WIP] Implemented runtime gbi, more headers and a few config features
* Fix CLI11 on WSL and variety of fixes
* Fixes
* Update DisplayListFactory.h
* Update TextureFactory.cpp
* wip lights
* fix
* Correct callback
* Added waypoints
* Update ceremony_data.yml
* Changes
* Update TextureFactory.cpp
* Update commandline args
* Update comment
* Remove printf
* Update couple commands
* Renamed cubeos to torch
* Removed unused stuff and added an example config file
---------
Co-authored-by: KiritoDv <kiritodev01@gmail.com>
Co-authored-by: MegaMech <MegaMech@users.noreply.github.com>
Diffstat (limited to 'src/Companion.cpp')
| -rw-r--r-- | src/Companion.cpp | 472 |
1 files changed, 357 insertions, 115 deletions
diff --git a/src/Companion.cpp b/src/Companion.cpp index 1b98d2d..761f2eb 100644 --- a/src/Companion.cpp +++ b/src/Companion.cpp @@ -2,208 +2,450 @@ #include "storm/SWrapper.h" #include "utils/MIODecoder.h" -#include "factories/RawFactory.h" -#include "factories/BlobFactory.h" -#include "factories/debug/MIO0Factory.h" +#include "factories/sm64/AnimationFactory.h" +#include "factories/sm64/DialogFactory.h" +#include "factories/sm64/DictionaryFactory.h" +#include "factories/sm64/TextFactory.h" +#include "factories/BankFactory.h" #include "factories/AudioHeaderFactory.h" -#include "factories/SequenceFactory.h" #include "factories/SampleFactory.h" -#include "factories/BankFactory.h" +#include "factories/SequenceFactory.h" +#include "factories/VtxFactory.h" #include "factories/TextureFactory.h" -#include "factories/sm64/SAnimFactory.h" -#include "factories/sm64/STextFactory.h" -#include "factories/sm64/SDialogFactory.h" -#include "factories/sm64/SDictionaryFactory.h" +#include "factories/DisplayListFactory.h" +#include "factories/BlobFactory.h" +#include "factories/LightsFactory.h" +#include "factories/mk64/WaypointFactory.h" #include "spdlog/spdlog.h" -#include <chrono> #include <fstream> #include <iostream> #include <filesystem> -namespace fs = std::filesystem; using namespace std::chrono; +namespace fs = std::filesystem; + +static const std::string regular = "[%Y-%m-%d %H:%M:%S.%e] [%l] %v"; +static const std::string line = "[%Y-%m-%d %H:%M:%S.%e] [%l] > %v"; -void Companion::Init() { +void Companion::Init(const ExportType type) { spdlog::set_level(spdlog::level::debug); spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); - this->RegisterFactory("AUDIO:HEADER", new AudioHeaderFactory()); - this->RegisterFactory("SEQUENCE", new SequenceFactory()); - this->RegisterFactory("SAMPLE", new SampleFactory()); - this->RegisterFactory("BANK", new BankFactory()); + this->gExporterType = type; + this->RegisterFactory("BLOB", std::make_shared<BlobFactory>()); + this->RegisterFactory("TEXTURE", std::make_shared<TextureFactory>()); + this->RegisterFactory("VTX", std::make_shared<VtxFactory>()); + this->RegisterFactory("LIGHTS", std::make_shared<LightsFactory>()); + this->RegisterFactory("GFX", std::make_shared<DListFactory>()); + this->RegisterFactory("AUDIO:HEADER", std::make_shared<AudioHeaderFactory>()); + this->RegisterFactory("SEQUENCE", std::make_shared<SequenceFactory>()); + this->RegisterFactory("SAMPLE", std::make_shared<SampleFactory>()); + this->RegisterFactory("BANK", std::make_shared<BankFactory>()); + + // SM64 specific + this->RegisterFactory("SM64:DIALOG", std::make_shared<SM64::DialogFactory>()); + this->RegisterFactory("SM64:TEXT", std::make_shared<SM64::TextFactory>()); + this->RegisterFactory("SM64:DICTIONARY", std::make_shared<SM64::DictionaryFactory>()); + this->RegisterFactory("SM64:ANIM", std::make_shared<SM64::AnimationFactory>()); + // this->RegisterFactory("GEO_LAYOUT", new SGeoFactory()); + + // MK64 specific + this->RegisterFactory("MK64:TRACKWAYPOINTS", std::make_shared<MK64::WaypointFactory>()); - this->RegisterFactory("TEXTURE", new TextureFactory()); - this->RegisterFactory("BLOB", new BlobFactory()); + this->Process(); +} - // SM64 Specific - this->RegisterFactory("SM64:DIALOG", new SDialogFactory()); - this->RegisterFactory("SM64:ANIM", new SAnimFactory()); - this->RegisterFactory("SM64:TEXT", new STextFactory()); - this->RegisterFactory("SM64:DICTIONARY", new SDictionaryFactory()); +void Companion::ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary) { + std::ostringstream stream; - // Debug - this->RegisterFactory("MIO0", new MIO0Factory()); + auto type = node["type"].as<std::string>(); + auto offset = node["offset"].as<uint32_t>(); + std::transform(type.begin(), type.end(), type.begin(), ::toupper); - this->Process(); + SPDLOG_INFO("[{}] Processing {} at 0x{:X}", type, name, offset); + auto factory = this->GetFactory(type); + if(!factory.has_value()){ + SPDLOG_ERROR("No factory found for {}", name); + return; + } + + auto result = factory->get()->parse(this->gRomData, node); + if(!result.has_value()){ + SPDLOG_ERROR("Failed to process {}", name); + return; + } + + auto exporter = factory->get()->GetExporter(this->gExporterType); + if(!exporter.has_value()){ + SPDLOG_ERROR("No exporter found for {}", name); + return; + } + + for (auto [fst, snd] : this->gAssetDependencies[this->gCurrentFile]) { + if(snd.second) continue; + std::string doutput = (this->gCurrentDirectory / fst).string(); + std::replace(doutput.begin(), doutput.end(), '\\', '/'); + this->gAssetDependencies[this->gCurrentFile][fst].second = true; + this->ExtractNode(snd.first, doutput, binary); + } + + switch (this->gExporterType) { + case ExportType::Binary: { + if(binary == nullptr) break; + stream.str(""); + stream.clear(); + exporter->get()->Export(stream, result.value(), name, node, &name); + auto data = stream.str(); + binary->CreateFile(name, std::vector(data.begin(), data.end())); + break; + } + default: { + exporter->get()->Export(stream, result.value(), name, node, &name); + break; + } + } + + SPDLOG_INFO("Processed {}", name); + this->gWriteMap[this->gCurrentFile][type].emplace_back(node["offset"].as<uint32_t>(), stream.str()); } void Companion::Process() { - const std::string regular = "[%Y-%m-%d %H:%M:%S.%e] [%l] %v"; - const std::string line = "[%Y-%m-%d %H:%M:%S.%e] [%l] > %v"; + if(!fs::exists("config.yml")) { + SPDLOG_ERROR("No config file found"); + return; + } auto start = duration_cast<milliseconds>(system_clock::now().time_since_epoch()); std::ifstream input( this->gRomPath, std::ios::binary ); - this->gRomData = std::vector<uint8_t>( std::istreambuf_iterator<char>( input ), {} ); + this->gRomData = std::vector<uint8_t>( std::istreambuf_iterator( input ), {} ); input.close(); - this->cartridge = new N64::Cartridge(this->gRomData); - cartridge->Initialize(); + this->gCartridge = new N64::Cartridge(this->gRomData); + gCartridge->Initialize(); YAML::Node config = YAML::LoadFile("config.yml"); - if(!config[cartridge->GetHash()]){ - SPDLOG_ERROR("No config found for {}", cartridge->GetHash()); + if(!config[gCartridge->GetHash()]){ + SPDLOG_ERROR("No config found for {}", gCartridge->GetHash()); return; } - auto cfg = config[cartridge->GetHash()]; - auto path = cfg["path"].as<std::string>(); - auto otr = cfg["output"].as<std::string>(); + auto rom = config[gCartridge->GetHash()]; + auto cfg = rom["config"]; + + if(!cfg) { + SPDLOG_ERROR("No config found for {}", gCartridge->GetHash()); + return; + } + + if(rom["segments"]) { + this->gSegments = rom["segments"].as<std::vector<uint32_t>>(); + } + auto path = rom["path"].as<std::string>(); + auto opath = cfg["output"]; + auto gbi = cfg["gbi"]; + + switch (gExporterType) { + case ExportType::Binary: { + this->gOutputPath = opath && opath["binary"] ? opath["binary"].as<std::string>() : "generic.otr"; + break; + } + case ExportType::Header: { + this->gOutputPath = opath && opath["headers"] ? opath["headers"].as<std::string>() : "headers"; + break; + } + case ExportType::Code: { + this->gOutputPath = opath && opath["code"] ? opath["code"].as<std::string>() : "code"; + break; + } + } + + if(gbi) { + auto key = gbi.as<std::string>(); + + if(key == "F3D") { + this->gGBIVersion = GBIVersion::f3d; + } else if(key == "F3DEX") { + this->gGBIVersion = GBIVersion::f3dex; + } else if(key == "F3DB") { + this->gGBIVersion = GBIVersion::f3db; + } else if(key == "F3DEX2") { + this->gGBIVersion = GBIVersion::f3dex2; + } else if(key == "F3DEXB") { + this->gGBIVersion = GBIVersion::f3dexb; + } else if (key == "F3DEX_MK64") { + this->gGBIVersion = GBIVersion::f3dex; + this->gGBIMinorVersion = GBIMinorVersion::Mk64; + } else { + SPDLOG_ERROR("Invalid GBI version"); + return; + } + } + + if(auto sort = cfg["sort"]) { + if(sort.IsSequence()) { + this->gWriteOrder = sort.as<std::vector<std::string>>(); + } else { + this->gWriteOrder = sort.as<std::string>(); + } + } else { + this->gWriteOrder = std::vector<std::string> { + "LIGHTS", "TEXTURE", "VTX", "GFX" + }; + } + + if(std::holds_alternative<std::vector<std::string>>(this->gWriteOrder)) { + for (auto& [key, _] : this->gFactories) { + auto entries = std::get<std::vector<std::string>>(this->gWriteOrder); + + if(std::find(entries.begin(), entries.end(), key) != entries.end()) { + continue; + } + + entries.push_back(key); + } + } SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); - SPDLOG_INFO("Starting CubeOS..."); - SPDLOG_INFO("Game: {}", cartridge->GetGameTitle()); - SPDLOG_INFO("CRC: {}", cartridge->GetCRC()); - SPDLOG_INFO("Version: {}", cartridge->GetVersion()); - SPDLOG_INFO("Country: [{}]", cartridge->GetCountryCode()); - SPDLOG_INFO("Hash: {}", cartridge->GetHash()); + SPDLOG_INFO("Starting Torch..."); + SPDLOG_INFO("Game: {}", gCartridge->GetGameTitle()); + SPDLOG_INFO("CRC: {}", gCartridge->GetCRC()); + SPDLOG_INFO("Version: {}", gCartridge->GetVersion()); + SPDLOG_INFO("Country: [{}]", gCartridge->GetCountryCode()); + SPDLOG_INFO("Hash: {}", gCartridge->GetHash()); SPDLOG_INFO("Assets: {}", path); - auto wrapper = SWrapper(otr); + auto wrapper = this->gExporterType == ExportType::Binary ? new SWrapper(this->gOutputPath) : nullptr; auto vWriter = LUS::BinaryWriter(); vWriter.SetEndianness(LUS::Endianness::Big); - vWriter.Write((uint8_t) LUS::Endianness::Big); - vWriter.Write(cartridge->GetCRC()); + vWriter.Write(static_cast<uint8_t>(LUS::Endianness::Big)); + vWriter.Write(gCartridge->GetCRC()); - for (const auto & entry : fs::directory_iterator(path)){ + for (const auto & entry : fs::recursive_directory_iterator(path)){ + if(entry.is_directory()) continue; YAML::Node root = YAML::LoadFile(entry.path().string()); + this->gCurrentDirectory = relative(entry.path(), path).replace_extension(""); + this->gCurrentFile = entry.path().string(); + for(auto asset = root.begin(); asset != root.end(); ++asset){ - auto type = asset->second["type"].as<std::string>(); + auto node = asset->second; + if(!asset->second["offset"]) continue; + + auto output = (this->gCurrentDirectory / asset->first.as<std::string>()).string(); + std::replace(output.begin(), output.end(), '\\', '/'); + + this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = std::make_tuple(output, node); + } - LUS::BinaryWriter write = LUS::BinaryWriter(); - RawFactory* factory = this->GetFactory(type); - factory->SetCartridge(cartridge); + // Stupid hack because the iteration broke the assets + root = YAML::LoadFile(entry.path().string()); + + for(auto asset = root.begin(); asset != root.end(); ++asset){ spdlog::set_pattern(regular); SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); - SPDLOG_INFO("Processing {} [{}]", asset->first.as<std::string>(), type); - SPDLOG_INFO("Root: {}", entry.path().string()); - if(!factory->process(&write, asset->second, this->gRomData)){ - SPDLOG_ERROR("Failed to process {}", asset->first.as<std::string>()); - continue; - } + auto entryName = asset->first.as<std::string>(); + std::string output = (this->gCurrentDirectory / entryName).string(); + std::replace(output.begin(), output.end(), '\\', '/'); - auto buffer = write.ToVector(); - auto output = (entry.path().stem() / asset->first.as<std::string>()).string(); + this->ExtractNode(asset->second, output, wrapper); + } - std::replace(output.begin(), output.end(), '\\', '/'); + if(gExporterType != ExportType::Binary){ + std::string output = (this->gOutputPath / this->gCurrentDirectory).string(); - wrapper.CreateFile(output, buffer); + switch (gExporterType) { + case ExportType::Header: { + output += "/definition.h"; + break; + } + case ExportType::Code: { + output += "/root.inc.c"; + break; + } + default: break; + } + + std::ostringstream stream; + + if(std::holds_alternative<std::string>(this->gWriteOrder)) { + auto sort = std::get<std::string>(this->gWriteOrder); + std::vector<std::pair<uint32_t, std::string>> outbuf; + for (const auto& [type, buffer] : this->gWriteMap[this->gCurrentFile]) { + outbuf.insert(outbuf.end(), buffer.begin(), buffer.end()); + } + this->gWriteMap.clear(); + + if(sort == "OFFSET") { + std::sort(outbuf.begin(), outbuf.end(), [](const auto& a, const auto& b) { + return std::get<uint32_t>(a) < std::get<uint32_t>(b); + }); + } else if(sort == "ROFFSET") { + std::sort(outbuf.begin(), outbuf.end(), [](const auto& a, const auto& b) { + return std::get<uint32_t>(a) > std::get<uint32_t>(b); + }); + } else if(sort != "LINEAR") { + throw std::runtime_error("Invalid write order"); + } - if(type != "TEXTURE") { - SPDLOG_DEBUG("Writing debug file"); - std::string dpath = "debug/" + output; - if(!fs::exists(fs::path(dpath).parent_path())){ - fs::create_directories(fs::path(dpath).parent_path()); + for (auto& [symbol, buffer] : outbuf) { + stream << buffer; } - std::ofstream stream(dpath, std::ios::binary); - stream.write((char*)buffer.data(), buffer.size()); - stream.close(); + outbuf.clear(); + } else { + for (const auto& type : std::get<std::vector<std::string>>(this->gWriteOrder)) { + std::vector<std::pair<uint32_t, std::string>> outbuf = this->gWriteMap[this->gCurrentFile][type]; + + std::sort(outbuf.begin(), outbuf.end(), [](const auto& a, const auto& b) { + return std::get<uint32_t>(a) > std::get<uint32_t>(b); + }); + + for (auto& [symbol, buffer] : outbuf) { + stream << buffer; + } + } + } + + std::string buffer = stream.str(); + + if(buffer.empty()) { + continue; + } + + std::replace(output.begin(), output.end(), '\\', '/'); + if(!exists(fs::path(output).parent_path())){ + create_directories(fs::path(output).parent_path()); } - SPDLOG_INFO("Processed {}", output); + std::ofstream file(output, std::ios::binary); + + if(gExporterType == ExportType::Header) { + std::string symbol = entry.path().stem(); + std::transform(symbol.begin(), symbol.end(), symbol.begin(), toupper); + file << "#ifndef " << symbol << "_H" << std::endl; + file << "#define " << symbol << "_H" << std::endl << std::endl; + file << buffer; + file << std::endl << "#endif" << std::endl; + } else { + file << buffer; + } - write.Close(); + file.close(); } } spdlog::set_pattern(regular); SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); - SPDLOG_INFO("Writing version file"); - wrapper.CreateFile("version", vWriter.ToVector()); - vWriter.Close(); + if(wrapper != nullptr) { + SPDLOG_INFO("Writing version file"); + wrapper->CreateFile("version", vWriter.ToVector()); + vWriter.Close(); + wrapper->Close(); + } auto end = duration_cast<milliseconds>(system_clock::now().time_since_epoch()); SPDLOG_INFO("Done! Took {}ms", end.count() - start.count()); - SPDLOG_INFO("Exported to {}", otr); spdlog::set_pattern(regular); SPDLOG_INFO("------------------------------------------------"); MIO0Decoder::ClearCache(); - wrapper.Close(); - this->cartridge = nullptr; - Companion::Instance = nullptr; + this->gCartridge = nullptr; + Instance = nullptr; } -void Companion::RegisterFactory(const std::string &extension, RawFactory *factory) { - this->gFactories[extension] = factory; - SPDLOG_DEBUG("Registered factory for {}", extension); -} +void Companion::Pack(const std::string& folder, const std::string& output) { + + spdlog::set_level(spdlog::level::debug); + spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); + + SPDLOG_INFO("------------------------------------------------"); + + SPDLOG_INFO("Starting Torch..."); + SPDLOG_INFO("Scanning {}", folder); + + auto start = duration_cast<milliseconds>(system_clock::now().time_since_epoch()); + std::unordered_map<std::string, std::vector<char>> files; + + for (const auto & entry : fs::recursive_directory_iterator(folder)){ + if(entry.is_directory()) continue; + std::ifstream input( entry.path(), std::ios::binary ); + auto data = std::vector( std::istreambuf_iterator( input ), {} ); + input.close(); + files[entry.path().string()] = data; + } + + auto wrapper = SWrapper(output); -RawFactory *Companion::GetFactory(const std::string &extension) { - if(!this->gFactories.contains(extension)){ - throw std::runtime_error("No factory found for " + extension); + for(auto& [path, data] : files){ + std::string normalized = path; + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + // Remove parent folder + normalized = normalized.substr(folder.length() + 1); + wrapper.CreateFile(normalized, data); + SPDLOG_INFO("> Added {}", normalized); } - return this->gFactories[extension]; + auto end = duration_cast<milliseconds>(system_clock::now().time_since_epoch()); + SPDLOG_INFO("Done! Took {}ms", end.count() - start.count()); + SPDLOG_INFO("Exported to {}", output); + spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); + SPDLOG_INFO("------------------------------------------------"); + + wrapper.Close(); } -void Companion::Pack(const std::string& folder, const std::string& output) { +void Companion::RegisterAsset(const std::string& name, YAML::Node& node) { + if(!node["offset"]) return; + this->gAssetDependencies[this->gCurrentFile][name] = std::make_pair(node, false); - spdlog::set_level(spdlog::level::debug); - spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); + auto output = (this->gCurrentDirectory / name).string(); + std::replace(output.begin(), output.end(), '\\', '/'); + this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = std::make_tuple(output, node); +} - SPDLOG_INFO("------------------------------------------------"); +void Companion::RegisterFactory(const std::string& type, const std::shared_ptr<BaseFactory>& factory) { + this->gFactories[type] = factory; + SPDLOG_DEBUG("Registered factory for {}", type); +} - SPDLOG_INFO("Starting CubeOS..."); - SPDLOG_INFO("Scanning {}", folder); +std::optional<std::shared_ptr<BaseFactory>> Companion::GetFactory(const std::string &type) { + if(!this->gFactories.contains(type)){ + return std::nullopt; + } - auto start = duration_cast<milliseconds>(system_clock::now().time_since_epoch()); - std::unordered_map<std::string, std::vector<char>> files; + return this->gFactories[type]; +} - for (const auto & entry : fs::recursive_directory_iterator(folder)){ - if(entry.is_directory()) continue; - std::ifstream input( entry.path(), std::ios::binary ); - auto data = std::vector<char>( std::istreambuf_iterator<char>( input ), {} ); - input.close(); - files[entry.path().string()] = data; - } +std::optional<std::uint32_t> Companion::GetSegmentedAddr(const uint8_t segment) { + if(segment >= this->gSegments.size()) { + return std::nullopt; + } - auto wrapper = SWrapper(output); + return this->gSegments[segment]; +} - for(auto& [path, data] : files){ - std::string normalized = path; - std::replace(normalized.begin(), normalized.end(), '\\', '/'); - // Remove parent folder - normalized = normalized.substr(folder.length() + 1); - wrapper.CreateFile(normalized, data); - SPDLOG_INFO("> Added {}", normalized); - } +std::optional<std::tuple<std::string, YAML::Node>> Companion::GetNodeByAddr(const uint32_t addr){ + if(!this->gAddrMap.contains(this->gCurrentFile)){ + return std::nullopt; + } - auto end = duration_cast<milliseconds>(system_clock::now().time_since_epoch()); - SPDLOG_INFO("Done! Took {}ms", end.count() - start.count()); - SPDLOG_INFO("Exported to {}", output); - spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); - SPDLOG_INFO("------------------------------------------------"); + if(!this->gAddrMap[this->gCurrentFile].contains(addr)){ + return std::nullopt; + } + + return this->gAddrMap[this->gCurrentFile][addr]; +} - wrapper.Close(); +std::string Companion::NormalizeAsset(const std::string& name) const { + auto path = fs::path(this->gCurrentFile).stem().string() + "_" + name; + return path; }
\ No newline at end of file |
