#include "Companion.h" #include "utils/Decompressor.h" #include "utils/StringHelper.h" #include "utils/TorchUtils.h" #include "archive/SWrapper.h" #include "archive/ZWrapper.h" #include "spdlog/spdlog.h" #include "TinySHA1.hpp" #include #include #include #include #include "factories/GenericArrayFactory.h" #include "factories/VtxFactory.h" #include "factories/MtxFactory.h" #include "factories/FloatFactory.h" #include "factories/IncludeFactory.h" #include "factories/TextureFactory.h" #include "factories/DisplayListFactory.h" #include "factories/DisplayListOverrides.h" #include "factories/BlobFactory.h" #include "factories/LightsFactory.h" #include "factories/Vec3fFactory.h" #include "factories/Vec3sFactory.h" #include "factories/AssetArrayFactory.h" #include "factories/ViewportFactory.h" #include "factories/CompressedTextureFactory.h" #ifdef SM64_SUPPORT #include "factories/sm64/AnimationFactory.h" #include "factories/sm64/BehaviorScriptFactory.h" #include "factories/sm64/CollisionFactory.h" #include "factories/sm64/DialogFactory.h" #include "factories/sm64/DictionaryFactory.h" #include "factories/sm64/TextFactory.h" #include "factories/sm64/GeoLayoutFactory.h" #include "factories/sm64/LevelScriptFactory.h" #include "factories/sm64/MacroFactory.h" #include "factories/sm64/MovtexFactory.h" #include "factories/sm64/MovtexQuadFactory.h" #include "factories/sm64/PaintingFactory.h" #include "factories/sm64/PaintingMapFactory.h" #include "factories/sm64/TrajectoryFactory.h" #include "factories/sm64/WaterDropletFactory.h" #endif #ifdef MK64_SUPPORT #include "factories/mk64/CourseVtx.h" #include "factories/mk64/Paths.h" #include "factories/mk64/TrackSections.h" #include "factories/mk64/SpawnData.h" #include "factories/mk64/UnkSpawnData.h" #include "factories/mk64/DrivingBehaviour.h" #include "factories/mk64/ItemCurve.h" #include "factories/mk64/CourseMetadata.h" #include "factories/mk64/PackedDisplayListFactory.h" #endif #ifdef SF64_SUPPORT #include "factories/sf64/ColPolyFactory.h" #include "factories/sf64/MessageFactory.h" #include "factories/sf64/MessageLookupFactory.h" #include "factories/sf64/SkeletonFactory.h" #include "factories/sf64/AnimFactory.h" #include "factories/sf64/ScriptFactory.h" #include "factories/sf64/HitboxFactory.h" #include "factories/sf64/EnvironmentFactory.h" #include "factories/sf64/ObjInitFactory.h" #include "factories/sf64/TriangleFactory.h" #endif #ifdef PM64_SUPPORT #include "factories/pm64/SpriteFactory.h" #include "factories/pm64/ShapeFactory.h" #include "factories/pm64/BackgroundFactory.h" #include "factories/pm64/CollisionFactory.h" #include "factories/pm64/MapTextureFactory.h" #include "factories/pm64/AudioFactory.h" #include "factories/pm64/StoryImageFactory.h" #include "factories/pm64/ImgFXAnimFactory.h" #include "factories/pm64/TitleDataFactory.h" #include "factories/pm64/EntityGfxFactory.h" #include "factories/pm64/EffectDListFactory.h" #endif #ifdef FZERO_SUPPORT #include "factories/fzerox/EADAnimationFactory.h" #include "factories/fzerox/CourseFactory.h" #include "factories/fzerox/GhostRecordFactory.h" #include "factories/fzerox/EADLimbFactory.h" #include "factories/fzerox/SequenceFactory.h" #include "factories/fzerox/SoundFontFactory.h" #endif #ifdef MARIO_ARTIST_SUPPORT #include "factories/mario_artist/MA2D1Factory.h" #endif #ifdef NAUDIO_SUPPORT #include "factories/naudio/v0/AudioHeaderFactory.h" #include "factories/naudio/v0/BankFactory.h" #include "factories/naudio/v0/SampleFactory.h" #include "factories/naudio/v0/SequenceFactory.h" #include "factories/naudio/v1/AudioContext.h" #include "factories/naudio/v1/SoundFontFactory.h" #include "factories/naudio/v1/AudioTableFactory.h" #include "factories/naudio/v1/InstrumentFactory.h" #include "factories/naudio/v1/SampleFactory.h" #include "factories/naudio/v1/DrumFactory.h" #include "factories/naudio/v1/EnvelopeFactory.h" #include "factories/naudio/v1/LoopFactory.h" #include "factories/naudio/v1/BookFactory.h" #include "factories/naudio/v1/SequenceFactory.h" #endif #include "preprocess/CompTool.h" using namespace std::chrono; namespace fs = std::filesystem; static const std::string regular = "[%Y-%m-%d %H:%M:%S.%e] [%l] %v"; static const std::string line = "[%Y-%m-%d %H:%M:%S.%e] [%l] > %v"; static std::string ConvertType(std::string type) { int index = type.find(':'); if(index != std::string::npos) { type = type.substr(index + 1); } std::transform(type.begin(), type.end(), type.begin(), tolower); return type; } static std::string GetTypeNode(YAML::Node& node) { auto type = GetSafeNode(node, "type"); std::transform(type.begin(), type.end(), type.begin(), toupper); return type; } Companion* Companion::Instance; void Companion::Init(const ExportType type) { std::atomic assetCount{0}; Init(type, assetCount); } void Companion::Init(const ExportType type, std::atomic& assetCount) { spdlog::set_level(spdlog::level::debug); spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); this->gConfig.exporterType = type; this->RegisterFactory("BLOB", std::make_shared()); this->RegisterFactory("TEXTURE", std::make_shared()); this->RegisterFactory("VTX", std::make_shared()); this->RegisterFactory("MTX", std::make_shared()); this->RegisterFactory("F32", std::make_shared()); this->RegisterFactory("INC", std::make_shared()); this->RegisterFactory("LIGHTS", std::make_shared()); this->RegisterFactory("GFX", std::make_shared()); this->RegisterFactory("VEC3F", std::make_shared()); this->RegisterFactory("VEC3S", std::make_shared()); this->RegisterFactory("ARRAY", std::make_shared()); this->RegisterFactory("ASSET_ARRAY", std::make_shared()); this->RegisterFactory("VP", std::make_shared()); this->RegisterFactory("COMPRESSED_TEXTURE", std::make_shared()); #ifdef SM64_SUPPORT this->RegisterFactory("SM64:DIALOG", std::make_shared()); this->RegisterFactory("SM64:TEXT", std::make_shared()); this->RegisterFactory("SM64:DICTIONARY", std::make_shared()); this->RegisterFactory("SM64:ANIM", std::make_shared()); this->RegisterFactory("SM64:BEHAVIOR_SCRIPT", std::make_shared()); this->RegisterFactory("SM64:COLLISION", std::make_shared()); this->RegisterFactory("SM64:GEO_LAYOUT", std::make_shared()); this->RegisterFactory("SM64:LEVEL_SCRIPT", std::make_shared()); this->RegisterFactory("SM64:MACRO", std::make_shared()); this->RegisterFactory("SM64:MOVTEX_QUAD", std::make_shared()); this->RegisterFactory("SM64:MOVTEX", std::make_shared()); this->RegisterFactory("SM64:PAINTING", std::make_shared()); this->RegisterFactory("SM64:PAINTING_MAP", std::make_shared()); this->RegisterFactory("SM64:TRAJECTORY", std::make_shared()); this->RegisterFactory("SM64:WATER_DROPLET", std::make_shared()); #endif #ifdef MK64_SUPPORT this->RegisterFactory("MK64:COURSE_VTX", std::make_shared()); this->RegisterFactory("MK64:TRACK_PATH", std::make_shared()); this->RegisterFactory("MK64:TRACK_SECTIONS", std::make_shared()); this->RegisterFactory("MK64:SPAWN_DATA", std::make_shared()); this->RegisterFactory("MK64:UNK_SPAWN_DATA", std::make_shared()); this->RegisterFactory("MK64:DRIVING_BEHAVIOUR", std::make_shared()); this->RegisterFactory("MK64:ITEM_CURVE", std::make_shared()); // Item curve for decomp only this->RegisterFactory("MK64:METADATA", std::make_shared()); this->RegisterFactory("MK64:PACKED_GFX", std::make_shared()); #endif #ifdef PM64_SUPPORT this->RegisterFactory("PM64:SPRITE", std::make_shared()); this->RegisterFactory("PM64:SHAPE", std::make_shared()); this->RegisterFactory("PM64:BACKGROUND", std::make_shared()); this->RegisterFactory("PM64:COLLISION", std::make_shared()); this->RegisterFactory("PM64:MAP_TEXTURE", std::make_shared()); this->RegisterFactory("PM64:AUDIO", std::make_shared()); this->RegisterFactory("PM64:STORY_IMAGE", std::make_shared()); this->RegisterFactory("PM64:IMGFX_ANIM", std::make_shared()); this->RegisterFactory("PM64:TITLE_DATA", std::make_shared()); this->RegisterFactory("PM64:ENTITY_GFX", std::make_shared()); this->RegisterFactory("PM64:EFFECT_DL", std::make_shared()); #endif #ifdef SF64_SUPPORT this->RegisterFactory("SF64:ANIM", std::make_shared()); this->RegisterFactory("SF64:SKELETON", std::make_shared()); this->RegisterFactory("SF64:MESSAGE", std::make_shared()); this->RegisterFactory("SF64:MSG_TABLE", std::make_shared()); this->RegisterFactory("SF64:SCRIPT", std::make_shared()); this->RegisterFactory("SF64:HITBOX", std::make_shared()); this->RegisterFactory("SF64:ENVIRONMENT", std::make_shared()); this->RegisterFactory("SF64:OBJECT_INIT", std::make_shared()); this->RegisterFactory("SF64:COLPOLY", std::make_shared()); this->RegisterFactory("SF64:TRIANGLE", std::make_shared()); #endif #ifdef FZERO_SUPPORT this->RegisterFactory("FZX:ANIM", std::make_shared()); this->RegisterFactory("FZX:COURSE", std::make_shared()); this->RegisterFactory("FZX:GHOST", std::make_shared()); this->RegisterFactory("FZX:LIMB", std::make_shared()); this->RegisterFactory("FZX:SEQUENCE", std::make_shared()); this->RegisterFactory("FZX:SOUNDFONT", std::make_shared()); #endif #ifdef MARIO_ARTIST_SUPPORT this->RegisterFactory("MA:MA2D1", std::make_shared()); #endif #ifdef NAUDIO_SUPPORT this->RegisterFactory("NAUDIO:V0:AUDIO_HEADER", std::make_shared()); this->RegisterFactory("NAUDIO:V0:SEQUENCE", std::make_shared()); this->RegisterFactory("NAUDIO:V0:SAMPLE", std::make_shared()); this->RegisterFactory("NAUDIO:V0:BANK", std::make_shared()); this->RegisterFactory("NAUDIO:V1:AUDIO_SETUP", std::make_shared()); this->RegisterFactory("NAUDIO:V1:AUDIO_TABLE", std::make_shared()); this->RegisterFactory("NAUDIO:V1:SOUND_FONT", std::make_shared()); this->RegisterFactory("NAUDIO:V1:INSTRUMENT", std::make_shared()); this->RegisterFactory("NAUDIO:V1:DRUM", std::make_shared()); this->RegisterFactory("NAUDIO:V1:SAMPLE", std::make_shared()); this->RegisterFactory("NAUDIO:V1:ENVELOPE", std::make_shared()); this->RegisterFactory("NAUDIO:V1:ADPCM_LOOP", std::make_shared()); this->RegisterFactory("NAUDIO:V1:ADPCM_BOOK", std::make_shared()); this->RegisterFactory("NAUDIO:V1:SEQUENCE", std::make_shared()); #endif #ifndef __EMSCRIPTEN__ // We call this manually this->Process(assetCount); #endif } void Companion::ParseEnums(std::string& header) { std::ifstream file(header); if (!file.is_open()) { throw std::runtime_error("Failed to open header files for enums node in config"); } std::regex enumRegex(R"(enum\s+(\w+)\s*(?:\s*:\s*(\w+))?[\s\n\r]*\{)"); std::string line; std::smatch match; std::string enumName; bool inEnum = false; int enumIndex; while (std::getline(file, line)) { if (!inEnum && std::regex_search(line, match, enumRegex) && match.size() > 1) { enumName = match.str(1); inEnum = true; enumIndex = -1; continue; } if(!inEnum) { continue; } if(line.find('}') != std::string::npos) { inEnum = false; continue; } // Remove any comments and non-alphanumeric characters line = std::regex_replace(line, std::regex(R"((/\*.*?\*/)|(//.*$)|([^a-zA-Z0-9=_\-\.]))"), ""); if(line.find('=') != std::string::npos) { auto value = line.substr(line.find('=') + 1); auto name = line.substr(0, line.find('=')); enumIndex = static_cast(std::stoll(value, nullptr, 0)); this->gEnums[enumName][enumIndex] = name; } else { enumIndex++; this->gEnums[enumName][enumIndex] = line; } } } std::optional Companion::ParseNode(YAML::Node& node, std::string& name) { auto type = GetTypeNode(node); spdlog::set_pattern(regular); if(node["offset"]) { auto offset = node["offset"].as(); SPDLOG_INFO("- [{}] Processing {} at 0x{:X}", type, name, offset); } else { SPDLOG_INFO("- [{}] Processing {}", type, name); } spdlog::set_pattern(line); node["vpath"] = name; auto factory = this->GetFactory(type); if(!factory.has_value()){ throw std::runtime_error("No factory by the name '"+type+"' found for '"+name+"'"); } auto impl = factory->get(); auto exporter = impl->GetExporter(this->gConfig.exporterType); if(!exporter.has_value() && !impl->HasModdedDependencies()){ SPDLOG_WARN("No exporter found for {}", name); return std::nullopt; } bool executeDef = true; std::optional> result; if(this->gConfig.modding && impl->SupportModdedAssets() && Torch::contains(this->gModdedAssetPaths, name)) { auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name]; if(!exists(path)) { SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]); } else { std::ifstream input(path, std::ios::binary); std::vector data = std::vector( std::istreambuf_iterator( input ), {}); input.close(); result = impl->parse_modding(data, node); executeDef = !result.has_value(); } } if(executeDef && this->gConfig.parseMode == ParseMode::Default) { result = impl->parse(this->gRomData, node); } if(executeDef && this->gConfig.parseMode == ParseMode::Directory) { auto path = GetSafeNode(node, "path"); std::ifstream input( path, std::ios::binary ); auto data = std::vector( std::istreambuf_iterator( input ), {} ); result = impl->parse(data, node); input.close(); } if(!result.has_value()){ SPDLOG_ERROR("Failed to process {}", name); return std::nullopt; } SPDLOG_INFO("Processed {}", name); return ParseResultData { name, type, node, result }; } void Companion::ParseModdingConfig() { auto path = fs::path(this->gConfig.moddingPath) / "modding.yml"; if(!fs::exists(path)) { throw std::runtime_error("No modding config found, please run in export mode first"); } auto modding = YAML::LoadFile(path.string()); for(auto assets = modding["assets"].begin(); assets != modding["assets"].end(); ++assets) { auto name = assets->first.as(); auto asset = assets->second.as(); this->gModdedAssetPaths[name] = asset; } } void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic& assetCount) { if (node["external_files"]) { auto externalFiles = node["external_files"]; if (externalFiles.IsSequence() && externalFiles.size()) { for(size_t i = 0; i < externalFiles.size(); i++) { auto externalFile = externalFiles[i]; if (externalFile.size() == 0) { this->gCurrentExternalFiles.push_back((this->gSourceDirectory / externalFile.as()).string()); } else { SPDLOG_INFO("External File size {}", externalFile.size()); throw std::runtime_error("Incorrect yaml syntax for external files.\n\nThe yaml expects:\n:config:\n external_files:\n - \n\ne.g.:\nexternal_files:\n - actors/actor1.yaml"); } std::string externalFileName = (this->gSourceDirectory / externalFile.as()).string(); if (StringHelper::StartsWith(std::filesystem::relative(externalFileName, this->gAssetPath).string(), "../")) { throw std::runtime_error("External File " + externalFileName + " Not In Asset Directory " + this->gAssetPath); } else if (std::filesystem::relative(externalFileName, this->gAssetPath).string() == "") { throw std::runtime_error("External File " + externalFileName + " Not In Asset Directory " + this->gAssetPath); } if (!Torch::contains(this->gAddrMap, externalFileName)) { SPDLOG_INFO("Dependency on external file {}. Now processing {}", externalFileName, externalFileName); auto currentFile = this->gCurrentFile; auto currentDirectory = this->gCurrentDirectory; auto currentExternalFiles = this->gCurrentExternalFiles; this->gCurrentFile = externalFileName; this->gCurrentDirectory = std::filesystem::relative(externalFileName, this->gAssetPath).replace_extension(""); YAML::Node root = YAML::LoadFile(externalFileName); if (!Torch::contains(this->gProcessedFiles, this->gCurrentFile)) { ProcessFile(root, assetCount); if (process) { this->gProcessedFiles.insert(this->gCurrentFile); } } SPDLOG_INFO("Finishing processing of file: {}", currentFile); this->gCurrentFile = currentFile; this->gCurrentDirectory = currentDirectory; this->gCurrentExternalFiles = currentExternalFiles; this->gFileHeader.clear(); } else { SPDLOG_INFO("Skipping external file {} as it has already been processed", externalFileName); } } } } if(node["manual_segments"]) { auto manualSegments = node["manual_segments"]; if (manualSegments.IsSequence() && manualSegments.size()) { for(size_t i = 0; i < manualSegments.size(); i++) { auto segment = manualSegments[i]; if (segment.IsSequence() && segment.size() == 2) { const auto id = segment[0].as(); const auto replacement = segment[1].as(); this->gManualSegments[id] = replacement; SPDLOG_DEBUG("Manual Segment {} replaced with {}", id, replacement); } else { throw std::runtime_error("Incorrect yaml syntax for manual segments.\n\nThe yaml expects:\n:config:\n manual_segments:\n - [, ]\n\nLike so:\nmanual_segments:\n - [0x05000000, \"textures/other_textures/texture_6447C4\"]"); } } } } if(node["segments"]) { auto segments = node["segments"]; // Set global variables for segmented data if (segments.IsSequence() && segments.size()) { if (segments[0].IsSequence() && segments[0].size() == 2) { gCurrentSegmentNumber = segments[0][0].as(); gCurrentFileOffset = segments[0][1].as(); gCurrentCompressionType = Decompressor::GetCompressionType(this->gRomData, gCurrentFileOffset); if(node["no_compression"]) { gCurrentCompressionType = CompressionType::None; } } else { throw std::runtime_error("Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n segments:\n - [, ]\n\nLike so:\nsegments:\n - [0x06, 0x821D10]"); } } // Set file offset for later use. for(size_t i = 0; i < segments.size(); i++) { auto segment = segments[i]; if (segment.IsSequence() && segment.size() == 2) { const auto id = segment[0].as(); const auto replacement = segment[1].as(); this->gConfig.segment.local[id] = replacement; SPDLOG_DEBUG("Segment {} replaced with 0x{:X}", id, replacement); } else { throw std::runtime_error("Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n segments:\n - [, ]\n\nLike so:\nsegments:\n - [0x06, 0x821D10]"); } } } if (node["virtual"]) { auto virtualAddrMap = node["virtual"]; gVirtualAddrMap[gCurrentFile] = std::make_tuple(virtualAddrMap[0].as(), virtualAddrMap[1].as()); } if(node["header"]) { auto header = node["header"]; switch (this->gConfig.exporterType) { case ExportType::Header: { if(header["header"].IsSequence()) { for(auto line = header["header"].begin(); line != header["header"].end(); ++line) { this->gFileHeader += line->as() + "\n"; } } break; } case ExportType::Code: { if(header["code"].IsSequence()) { for(auto line = header["code"].begin(); line != header["code"].end(); ++line) { this->gFileHeader += line->as() + "\n"; } } break; } default: break; } } if(node["tables"]){ for(auto table = node["tables"].begin(); table != node["tables"].end(); ++table){ auto name = table->first.as(); auto range = table->second["range"].as>(); auto start = gCurrentSegmentNumber ? gCurrentSegmentNumber << 24 | range[0] : range[0]; auto end = gCurrentSegmentNumber ? gCurrentSegmentNumber << 24 | range[1] : range[1]; auto mode = GetSafeNode(table->second, "mode", "APPEND"); TableMode tMode = mode == "REFERENCE" ? TableMode::Reference : TableMode::Append; auto index_size = GetSafeNode(table->second, "index_size", -1); this->gTables.push_back({name, start, end, tMode, index_size}); } } if(node["vram"]){ auto vram = node["vram"]; const auto addr = GetSafeNode(vram, "addr"); const auto offset = GetSafeNode(vram, "offset"); this->gCurrentVram = { addr, offset }; } this->gEnablePadGen = GetSafeNode(node, "autopads", true); this->gNodeForceProcessing = GetSafeNode(node, "force", false); this->gIndividualIncludes = GetSafeNode(node, "individual_data_incs", false); this->gCurrentVirtualPath = GetSafeNode(node, "path", ""); } void Companion::ParseHash() { const auto out = this->gDestinationDirectory / "torch.hash.yml"; if(fs::exists(out)) { this->gHashNode = YAML::LoadFile(out.string()); } else { this->gHashNode = YAML::Node(); } } std::string ExportTypeToString(ExportType type) { switch (type) { case ExportType::Binary: return "Binary"; case ExportType::Header: return "Header"; case ExportType::Code: return "Code"; case ExportType::Modding: return "Modding"; case ExportType::XML: return "XML"; default: throw std::runtime_error("Invalid ExportType"); } } bool Companion::NodeHasChanges(const std::string& path) { if(this->gConfig.modding) { return true; } std::ifstream yaml(path); const std::vector data = std::vector(std::istreambuf_iterator( yaml ), {}); this->gCurrentHash = CalculateHash(data); bool needsInit = true; auto srcRelativePath = RelativePathToSrcDir(path); if(this->gHashNode[srcRelativePath]) { auto entry = GetSafeNode(this->gHashNode, srcRelativePath); const auto hash = GetSafeNode(entry, "hash", "no-hash"); auto modes = GetSafeNode(entry, "extracted"); auto extracted = GetSafeNode(modes, ExportTypeToString(this->gConfig.exporterType)); if(hash == this->gCurrentHash) { needsInit = false; if(extracted) { SPDLOG_INFO("Skipping {} as it has not changed", srcRelativePath); return false; } } } if(needsInit) { this->gHashNode[srcRelativePath] = YAML::Node(); this->gHashNode[srcRelativePath]["hash"] = this->gCurrentHash; this->gHashNode[srcRelativePath]["extracted"] = YAML::Node(); for(size_t m = 0; m <= static_cast(ExportType::Modding); m++) { this->gHashNode[srcRelativePath]["extracted"][ExportTypeToString(static_cast(m))] = false; } } return true; } void Companion::LoadYAMLRecursively(const std::string &dirPath, std::vector &result, bool skipRoot) { for (const auto &entry : std::filesystem::directory_iterator(dirPath)) { if (entry.is_directory()) { // Skip the root directory if specified if (skipRoot && entry.path() == dirPath) { continue; } // Recursive call for subdirectories LoadYAMLRecursively(entry.path().generic_string(), result, false); } else if (entry.path().extension() == ".yaml" || entry.path().extension() == ".yml") { // Load YAML file and add it to the result vector result.push_back(YAML::LoadFile(entry.path().generic_string())); } } } /** * Config yaml requires tables: [assets/courses] * Activate the factory using a normal asset yaml with type, input_directory, and output_directory nodes. */ void Companion::ProcessTables(YAML::Node& rom) { auto dirs = rom["metadata"].as>(); for (const auto &dir : dirs) { std::vector configNodes; LoadYAMLRecursively(dir, configNodes, true); gCourseMetadata[dir] = configNodes; } // Write yaml data to console if (this->IsDebug()) { SPDLOG_INFO("------ Metadata ouptut ------"); for (auto &node : gCourseMetadata[dirs[0]]) { std::cout << node << std::endl; SPDLOG_INFO("------------"); } SPDLOG_INFO("------ Metadata end ------"); } } void Companion::ProcessFile(YAML::Node root) { std::atomic assetCount {0}; ProcessFile(root, assetCount); } void Companion::ProcessFile(YAML::Node root, std::atomic& assetCount) { assetCount++; // Set compressed file offsets and compression type if (auto segments = root[":config"]["segments"]) { if (segments.IsSequence() && segments.size() > 0) { if (segments[0].IsSequence() && segments[0].size() == 2) { gCurrentSegmentNumber = segments[0][0].as(); gCurrentFileOffset = segments[0][1].as(); gCurrentCompressionType = Decompressor::GetCompressionType(this->gRomData, gCurrentFileOffset); if(root[":config"]["no_compression"]) { gCurrentCompressionType = CompressionType::None; } } else { throw std::runtime_error("Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n segments:\n - [, ]\n\nLike so:\nsegments:\n - [0x06, 0x821D10]"); } } } for(auto asset = root.begin(); asset != root.end(); ++asset){ auto node = asset->second; auto entryName = asset->first.as(); auto output = (this->gCurrentDirectory / entryName).string(); std::replace(output.begin(), output.end(), '\\', '/'); if(node["type"]){ const auto type = GetTypeNode(node); if(type == "NAUDIO:V0:SAMPLE"){ AudioManager::Instance->bind_sample(node, output); } } if(!node["offset"]) { continue; } if(gCurrentSegmentNumber) { if (IS_SEGMENTED(node["offset"].as()) == false) { node["offset"] = (gCurrentSegmentNumber << 24) | node["offset"].as(); } } if(!gCurrentVirtualPath.empty()) { node["path"] = gCurrentVirtualPath; } this->gAddrMap[this->gCurrentFile][node["offset"].as()] = std::make_tuple(output, node); } // Stupid hack because the iteration broke the assets root = YAML::LoadFile(this->gCurrentFile); this->gConfig.segment.local.clear(); this->gFileHeader.clear(); this->gCurrentPad = 0; this->gCurrentVram = std::nullopt; this->gCurrentVirtualPath = ""; this->gCurrentSegmentNumber = 0; this->gCurrentCompressionType = CompressionType::None; this->gCurrentFileOffset = 0; this->gTables.clear(); this->gCurrentExternalFiles.clear(); this->gManualSegments.clear(); GFXDOverride::ClearVtx(); if(root[":config"]) { this->ParseCurrentFileConfig(root[":config"], assetCount); } if(!process || (!this->NodeHasChanges(this->gCurrentFile) && !this->gNodeForceProcessing)) { return; } spdlog::set_pattern(regular); SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); for(auto asset = root.begin(); asset != root.end(); ++asset){ auto entryName = asset->first.as(); auto assetNode = asset->second; if(entryName.find(":config") != std::string::npos) { continue; } if(gCurrentFileOffset && assetNode["offset"]) { const auto offset = assetNode["offset"].as(); if (!IS_SEGMENTED(offset)) { assetNode["offset"] = (gCurrentSegmentNumber << 24) | offset; } } if(!gCurrentVirtualPath.empty()) { assetNode["path"] = gCurrentVirtualPath; } std::string output = (this->gCurrentDirectory / entryName).string(); std::replace(output.begin(), output.end(), '\\', '/'); this->gConfig.segment.temporal.clear(); auto result = this->ParseNode(assetNode, output); if(result.has_value()) { this->gParseResults[this->gCurrentFile].push_back(result.value()); } spdlog::set_pattern(regular); SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); } for(auto& result : this->gParseResults[this->gCurrentFile]){ std::ostringstream stream; ExportResult endptr = std::nullopt; WriteEntry wEntry; auto data = result.data.value(); const auto impl = this->GetFactory(result.type)->get(); const auto exporter = impl->GetExporter(this->gConfig.exporterType); if(!exporter.has_value()) { continue; } switch (this->gConfig.exporterType) { case ExportType::Binary: { stream.str(""); stream.clear(); exporter->get()->Export(stream, data, result.name, result.node, &result.name); auto data = stream.str(); this->gCurrentWrapper->AddFile(result.name, std::vector(data.begin(), data.end())); for(auto& entry : this->gCompanionFiles){ auto output = (this->gCurrentDirectory / entry.first).string(); std::replace(output.begin(), output.end(), '\\', '/'); this->gCurrentWrapper->AddFile(output, entry.second); } break; } case ExportType::XML: case ExportType::Modding: { stream.str(""); stream.clear(); std::string ogname = result.name; exporter->get()->Export(stream, data, result.name, result.node, &result.name); auto data = stream.str(); if(data.empty()) { break; } std::string dpath = Instance->GetOutputPath() + "/" + result.name; if(!exists(fs::path(dpath).parent_path())){ create_directories(fs::path(dpath).parent_path()); } this->gModdedAssetPaths[ogname] = result.name; std::ofstream file(dpath, std::ios::binary); file.write(data.c_str(), data.size()); file.close(); for(auto& entry : this->gCompanionFiles){ auto cpath = (Instance->GetOutputPath() / this->gCurrentDirectory / entry.first).string(); std::replace(cpath.begin(), cpath.end(), '\\', '/'); if(!exists(fs::path(cpath).parent_path())){ create_directories(fs::path(cpath).parent_path()); } std::ofstream cfile(cpath, std::ios::binary); cfile.write(entry.second.data(), entry.second.size()); cfile.close(); } break; } default: { endptr = exporter->get()->Export(stream, data, result.name, result.node, &result.name); break; } } this->gCompanionFiles.clear(); if(result.node["offset"]) { auto alignment = GetSafeNode(result.node, "alignment", impl->GetAlignment()); if(!endptr.has_value()) { wEntry = { result.name, result.node["offset"].as(), alignment, stream.str(), GetNode(result.node, "comment"), std::nullopt }; } else { switch (endptr->index()) { case 0: wEntry = { result.name, result.node["offset"].as(), alignment, stream.str(), GetNode(result.node, "comment"), std::get(endptr.value()) }; break; case 1: { const auto oentry = std::get(endptr.value()); wEntry = { result.name, oentry.start, alignment, stream.str(), GetNode(result.node, "comment"), oentry.end }; break; } default: SPDLOG_ERROR("Invalid endptr index {}", endptr->index()); SPDLOG_ERROR("Type of endptr: {}", typeid(endptr).name()); throw std::runtime_error("We should never reach this point"); } } } this->gWriteMap[this->gCurrentFile][result.type].push_back(wEntry); } auto fsout = fs::path(this->gConfig.outputPath); if(this->gConfig.exporterType == ExportType::Modding || this->gConfig.exporterType == ExportType::XML) { fsout /= "modding.yml"; YAML::Node modding; for (const auto& [key, value] : this->gModdedAssetPaths) { modding["assets"][key] = value; } std::ofstream file(fsout.string(), std::ios::binary); file << modding; file.close(); } else if(this->gConfig.exporterType != ExportType::Binary){ std::string filename = this->gCurrentDirectory.filename().string(); switch (this->gConfig.exporterType) { case ExportType::Header: { fsout /= this->gCurrentDirectory.parent_path() / (filename + ".h"); break; } case ExportType::Code: { fsout /= this->gCurrentDirectory / (filename + ".c"); break; } default: break; } std::ostringstream stream; std::vector entries; if(std::holds_alternative(this->gWriteOrder)) { auto sort = std::get(this->gWriteOrder); for (const auto& [type, raw] : this->gWriteMap[this->gCurrentFile]) { entries.insert(entries.end(), raw.begin(), raw.end()); } if(sort == "OFFSET") { std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return a.addr < b.addr; }); } else if(sort == "ROFFSET") { std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return a.addr > b.addr; }); } else if(sort != "LINEAR") { throw std::runtime_error("Invalid write order"); } } else { for (const auto& type : std::get>(this->gWriteOrder)) { entries = this->gWriteMap[this->gCurrentFile][type]; std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return a.addr > b.addr; }); } } for (size_t i = 0; i < entries.size(); i++) { const auto result = entries[i]; const auto hasSize = result.endptr.has_value(); if(result.comment.has_value()){ stream << "// " << result.comment.value() << "\n"; } if (hasSize && this->IsDebug()) { stream << "// 0x" << std::hex << std::uppercase << ASSET_PTR(result.addr) << "\n"; } stream << result.buffer; if (hasSize && this->IsDebug()) { stream << "// 0x" << std::hex << std::uppercase << ASSET_PTR(result.endptr.value()) << "\n\n"; } if(hasSize && i < entries.size() - 1 && this->gConfig.exporterType == ExportType::Code && !this->gIndividualIncludes){ int32_t startptr = ASSET_PTR(result.endptr.value()); int32_t end = ASSET_PTR(entries[i + 1].addr); uint32_t alignment = entries[i + 1].alignment; int32_t gap = end - startptr; if(gap < 0) { stream << "// WARNING: Overlap detected between 0x" << std::hex << startptr << " and 0x" << end << " with size 0x" << std::abs(gap) << "\n"; SPDLOG_WARN("Overlap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end, gap, this->gCurrentFile); } else if(gap < 0x10 && gap >= alignment && end % alignment == 0 && this->gEnablePadGen) { SPDLOG_WARN("Gap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end, gap, this->gCurrentFile); SPDLOG_WARN("Creating pad of 0x{:X} bytes", gap); const auto padfile = this->gCurrentDirectory.filename().string(); if(this->IsDebug()){ stream << "// 0x" << std::hex << std::uppercase << startptr << "\n"; } stream << "char pad_" << padfile << "_" << std::to_string(gCurrentPad++) << "[] = {\n" << tab_t; auto gapSize = gap & ~3; for(size_t j = 0; j < gapSize; j++){ stream << "0x00, "; } stream << "\n};\n"; if(this->IsDebug()){ stream << "// 0x" << std::hex << std::uppercase << end << "\n\n"; } else { stream << "\n"; } } else if(gap >= 0x10) { stream << "// WARNING: Gap detected between 0x" << std::hex << startptr << " and 0x" << end << " with size 0x" << gap << "\n"; } } if (this->gConfig.exporterType == ExportType::Code && this->gIndividualIncludes) { fs::path outinc = fs::path(this->gConfig.outputPath) / this->gCurrentDirectory.parent_path() / fs::relative(fs::path(result.name + ".inc.c"), this->gCurrentDirectory.parent_path()); if(!exists(outinc.parent_path())){ create_directories(outinc.parent_path()); } std::ofstream file(outinc, std::ios::binary); if(!this->gFileHeader.empty()) { file << this->gFileHeader << std::endl; } file << stream.str(); stream.str(""); stream.seekp(0); file.close(); } } this->gWriteMap.clear(); if (this->gConfig.exporterType != ExportType::Code || !this->gIndividualIncludes) { std::string buffer = stream.str(); if(buffer.empty()) { SPDLOG_WARN("No data to write for {}", this->gCurrentFile); return; } std::string output = fsout.string(); std::replace(output.begin(), output.end(), '\\', '/'); if(!exists(fs::path(output).parent_path())){ create_directories(fs::path(output).parent_path()); } std::ofstream file(output, std::ios::binary); SPDLOG_INFO("Writing {} to {}", this->gCurrentFile, output); if(this->gConfig.exporterType == ExportType::Header) { fs::path entryPath = this->gCurrentFile; std::string symbol = entryPath.stem().string(); std::transform(symbol.begin(), symbol.end(), symbol.begin(), toupper); if(this->IsOTRMode()){ file << "#pragma once\n\n"; } else { file << "#ifndef " << symbol << "_H" << std::endl; file << "#define " << symbol << "_H" << std::endl << std::endl; } if(!this->gFileHeader.empty()) { file << this->gFileHeader << std::endl; } file << buffer; if(!this->IsOTRMode()){ file << std::endl << "#endif" << std::endl; } } else { if(!this->gFileHeader.empty()) { file << this->gFileHeader << std::endl; } file << buffer; } file.close(); } } if(this->gConfig.exporterType != ExportType::Binary) { this->gHashNode[RelativePathToSrcDir(this->gCurrentFile)]["extracted"][ExportTypeToString(this->gConfig.exporterType)] = true; } } void Companion::Process(std::atomic& assetCount) { auto configPath = this->gSourceDirectory / "config.yml"; if(!fs::exists(configPath)) { SPDLOG_ERROR("No config file found"); return; } auto start = duration_cast(system_clock::now().time_since_epoch()); YAML::Node config = YAML::LoadFile(configPath.string()); bool isDirectoryMode = config["mode"] && config["mode"].as() == "directory"; if(!isDirectoryMode) { if(this->gRomPath.has_value()){ std::ifstream input( this->gRomPath.value(), std::ios::binary ); this->gRomData = std::vector( std::istreambuf_iterator( input ), {} ); input.close(); } this->gCartridge = std::make_shared(this->gRomData); this->gCartridge->Initialize(); if(!config[this->gCartridge->GetHash()]){ SPDLOG_ERROR("No config found for {}", this->gCartridge->GetHash()); return; } this->gConfig.parseMode = ParseMode::Default; } else { this->gConfig.parseMode = ParseMode::Directory; } auto rom = !isDirectoryMode ? config[this->gCartridge->GetHash()] : config; if(rom["preprocess"]) { auto preprocess = rom["preprocess"]; for(auto job = preprocess.begin(); job != preprocess.end(); job++) { auto name = job->first.as(); auto item = job->second; auto method = GetSafeNode(item, "method"); if (method == "mio0-comptool") { auto type = GetTypeNode(item); auto target = GetSafeNode(item, "target"); auto restart = GetSafeNode(item, "restart"); if (type == "DECOMPRESS") { this->gRomData = CompTool::Decompress(this->gRomData); this->gCartridge = std::make_shared(this->gRomData); this->gCartridge->Initialize(); auto hash = this->gCartridge->GetHash(); SPDLOG_CRITICAL("ROM decompressed to {}", hash); if (hash != target) { throw std::runtime_error("Hash mismatch"); } if(restart){ rom = config[this->gCartridge->GetHash()]; } } else { throw std::runtime_error("Only decompression is supported"); } } else { throw std::runtime_error("Invalid preprocess method"); } } } auto cfg = rom["config"]; if(!cfg) { SPDLOG_ERROR("No config found for {}", !isDirectoryMode ? this->gCartridge->GetHash() : GetSafeNode(config, "folder")); return; } if (rom["metadata"]) { ProcessTables(rom); } if(rom["segments"]) { auto segments = rom["segments"].as>(); for (int i = 0; i < segments.size(); i++) { this->gConfig.segment.global[i + 1] = segments[i]; } } this->gAssetPath = (this->gSourceDirectory / rom["path"].as()).string(); auto opath = cfg["output"]; auto gbi = cfg["gbi"]; auto gbi_floats = cfg["gbi_floats"]; auto modding_path = opath && opath["modding"] ? opath["modding"].as() : "modding"; if (!this->gDestinationDirectory.empty() && !fs::exists(this->gDestinationDirectory)) { create_directories(this->gDestinationDirectory); } auto output_path = this->gDestinationDirectory; this->gConfig.moddingPath = (this->gDestinationDirectory / modding_path).string(); switch (this->gConfig.exporterType) { case ExportType::Binary: { std::string extension = ""; switch (this->gConfig.otrMode) { case ArchiveType::OTR: extension = ".otr"; break; case ArchiveType::O2R: extension = ".o2r"; break; default: throw std::runtime_error("Invalid archive type for export type Binary"); } output_path /= opath && opath["binary"] ? opath["binary"].as() : ("generic" + extension); break; } case ExportType::Header: { output_path /= opath && opath["headers"] ? opath["headers"].as() : "headers"; break; } case ExportType::Code: { output_path /= opath && opath["code"] ? opath["code"].as() : "code"; break; } case ExportType::XML: case ExportType::Modding: { output_path /= modding_path; break; } } this->gConfig.outputPath = output_path.string(); if(gbi) { auto key = gbi.as(); if(key == "F3D") { this->gConfig.gbi.version = GBIVersion::f3d; } else if(key == "F3DEX") { this->gConfig.gbi.version = GBIVersion::f3dex; } else if(key == "F3DB") { this->gConfig.gbi.version = GBIVersion::f3db; } else if(key == "F3DEX2") { this->gConfig.gbi.version = GBIVersion::f3dex2; } else if(key == "F3DEXB") { this->gConfig.gbi.version = GBIVersion::f3dexb; } else if (key == "F3DEX_MK64") { this->gConfig.gbi.version = GBIVersion::f3dex; this->gConfig.gbi.subversion = GBIMinorVersion::Mk64; } else { SPDLOG_ERROR("Invalid GBI version"); return; } if(gbi_floats) { this->gConfig.gbi.useFloats = gbi_floats.as(); } } if(auto sort = cfg["sort"]) { if(sort.IsSequence()) { this->gWriteOrder = sort.as>(); } else { this->gWriteOrder = sort.as(); } } else { this->gWriteOrder = std::vector { "LIGHTS", "TEXTURE", "VTX", "GFX" }; } if((this->gConfig.exporterType == ExportType::Code || this->gConfig.exporterType == ExportType::Binary) && this->gConfig.modding) { this->ParseModdingConfig(); } if(std::holds_alternative>(this->gWriteOrder)) { for (auto& [key, _] : this->gFactories) { auto entries = std::get>(this->gWriteOrder); if(std::find(entries.begin(), entries.end(), key) != entries.end()) { continue; } entries.push_back(key); } } #ifdef STANDALONE if(cfg["enums"]) { auto enums = GetSafeNode>(cfg, "enums"); for (auto& file : enums) { file = (this->gSourceDirectory / file).string(); this->ParseEnums(file); } } #endif if(cfg["logging"]){ auto level = cfg["logging"].as(); if(level == "TRACE") { spdlog::set_level(spdlog::level::trace); } else if(level == "DEBUG") { spdlog::set_level(spdlog::level::debug); } else if(level == "INFO") { spdlog::set_level(spdlog::level::info); } else if(level == "WARN") { spdlog::set_level(spdlog::level::warn); } else if(level == "ERROR") { spdlog::set_level(spdlog::level::err); } else if(level == "CRITICAL") { spdlog::set_level(spdlog::level::critical); } else if(level == "OFF") { spdlog::set_level(spdlog::level::off); } else { throw std::runtime_error("Invalid logging level, please use TRACE, DEBUG, INFO, WARN, ERROR, CRITICAL or OFF"); } } this->gConfig.textureDefines = cfg["textures"] && (cfg["textures"].as() == "ADDITIONAL_DEFINES"); this->ParseHash(); SPDLOG_CRITICAL("------------------------------------------------"); spdlog::set_pattern(line); SPDLOG_CRITICAL("Starting Torch..."); if(this->gConfig.parseMode == ParseMode::Default) { SPDLOG_CRITICAL("Game: {}", this->gCartridge->GetGameTitle()); SPDLOG_CRITICAL("CRC: {}", this->gCartridge->GetCRC()); SPDLOG_CRITICAL("Version: {}", this->gCartridge->GetVersion()); SPDLOG_CRITICAL("Country: [{}]", this->gCartridge->GetCountryCode()); SPDLOG_CRITICAL("Hash: {}", this->gCartridge->GetHash()); SPDLOG_CRITICAL("Assets: {}", this->gAssetPath); } else { SPDLOG_CRITICAL("Mode: Directory"); SPDLOG_CRITICAL("Directory: {}", rom["folder"].as()); } SPDLOG_CRITICAL("------------------------------------------------"); AudioManager::Instance = new AudioManager(); BinaryWrapper* wrapper = nullptr; if (this->gConfig.exporterType == ExportType::Binary) { switch (this->gConfig.otrMode) { case ArchiveType::OTR: wrapper = new SWrapper(this->gConfig.outputPath); break; case ArchiveType::O2R: wrapper = new ZWrapper(this->gConfig.outputPath); break; default: throw std::runtime_error("Invalid archive type for export type Binary"); } } if (wrapper) { wrapper->CreateArchive(); } this->gCurrentWrapper = wrapper; for(auto& entry : this->gCompanionFiles) { auto output = entry.first; std::replace(output.begin(), output.end(), '\\', '/'); this->gCurrentWrapper->AddFile(output, entry.second); } auto vWriter = LUS::BinaryWriter(); vWriter.SetEndianness(Torch::Endianness::Big); vWriter.Write(static_cast(Torch::Endianness::Big)); if(this->gConfig.parseMode == ParseMode::Default) { vWriter.Write(this->gCartridge->GetCRC()); } else { vWriter.Write((uint32_t) 0); } for (const auto & entry : Torch::getRecursiveEntries(this->gAssetPath)){ if(entry.is_directory()) { continue; } const auto yamlPath = entry.path().generic_string(); if(yamlPath.find(".yaml") == std::string::npos && yamlPath.find(".yml") == std::string::npos) { continue; } if(yamlPath.find("config.yml") != std::string::npos) { continue; } YAML::Node root = YAML::LoadFile(yamlPath); this->gCurrentDirectory = relative(entry.path(), this->gAssetPath).replace_extension(""); this->gCurrentFile = yamlPath; if (!Torch::contains(this->gProcessedFiles, this->gCurrentFile)) { ProcessFile(root, assetCount); if (process) { this->gProcessedFiles.insert(this->gCurrentFile); } } } if(wrapper != nullptr) { // Add additional files specified by the user for (const auto& filePath : this->gAdditionalFiles) { std::ifstream input(this->gSourceDirectory / filePath, std::ios::binary); if (input.is_open()) { std::vector data((std::istreambuf_iterator(input)), std::istreambuf_iterator()); input.close(); std::string filename = fs::path(filePath).filename().string(); wrapper->AddFile(filename, data); SPDLOG_INFO("Added additional file: {}", filename); } else { SPDLOG_WARN("Could not open additional file: {}", filePath); } } if(!this->gVersion.empty()) { auto data = ParseVersionString(this->gVersion); wrapper->AddFile("portVersion", data); } SPDLOG_CRITICAL("Writing version file"); wrapper->AddFile("version", vWriter.ToVector()); vWriter.Close(); wrapper->Close(); } // Write entries hash std::ofstream file(this->gDestinationDirectory / "torch.hash.yml", std::ios::binary); file << this->gHashNode; file.close(); auto end = duration_cast(system_clock::now().time_since_epoch()); auto level = spdlog::get_level(); spdlog::set_level(spdlog::level::info); SPDLOG_CRITICAL("Done! Took {}ms", end.count() - start.count()); SPDLOG_CRITICAL("------------------------------------------------"); spdlog::set_level(level); spdlog::set_pattern(regular); Decompressor::ClearCache(); this->gCartridge = nullptr; Instance = nullptr; } void Companion::Pack(const std::string& folder, const std::string& output, const ArchiveType otrMode, const std::string& version) { spdlog::set_level(spdlog::level::debug); spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); SPDLOG_CRITICAL("------------------------------------------------"); SPDLOG_CRITICAL("Starting Torch..."); SPDLOG_CRITICAL("Scanning {}", folder); auto start = duration_cast(system_clock::now().time_since_epoch()); std::unordered_map> files; for (const auto & entry : Torch::getRecursiveEntries(folder)){ if(entry.is_directory()) { continue; } std::ifstream input( entry.path(), std::ios::binary ); auto data = std::vector( std::istreambuf_iterator( input ), {} ); input.close(); files[entry.path().generic_string()] = data; } std::unique_ptr wrapper; switch (otrMode) { case ArchiveType::OTR: wrapper.reset(new SWrapper(output)); break; case ArchiveType::O2R: wrapper.reset(new ZWrapper(output)); break; default: throw std::runtime_error("Invalid archive type for export type Binary"); } wrapper->CreateArchive(); for(auto& [path, data] : files){ std::string normalized = path; std::replace(normalized.begin(), normalized.end(), '\\', '/'); // Remove parent folder normalized = normalized.substr(folder.length() + 1); wrapper->AddFile(normalized, data); SPDLOG_CRITICAL("> Added {}", normalized); } if(!version.empty()) { SPDLOG_CRITICAL("Adding version file"); auto data = ParseVersionString(version); wrapper->AddFile("portVersion", data); } auto end = duration_cast(system_clock::now().time_since_epoch()); SPDLOG_CRITICAL("Done! Took {}ms", end.count() - start.count()); SPDLOG_CRITICAL("Exported to {}", output); spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v"); SPDLOG_CRITICAL("------------------------------------------------"); wrapper->Close(); } std::optional> Companion::RegisterAsset(const std::string& name, YAML::Node& node) { if(!node["offset"]) { return std::nullopt; } auto output = (this->gCurrentDirectory / name).string(); std::replace(output.begin(), output.end(), '\\', '/'); auto entry = std::make_tuple(output, node); this->gAddrMap[this->gCurrentFile][node["offset"].as()] = entry; auto dResult = this->ParseNode(node, output); if(dResult.has_value()) { this->gParseResults[this->gCurrentFile].push_back(dResult.value()); } spdlog::set_pattern(regular); SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); return entry; } void Companion::RegisterFactory(const std::string& type, const std::shared_ptr& factory) { this->gFactories[type] = factory; SPDLOG_INFO("Registered factory for {}", type); } std::optional> Companion::GetFactory(const std::string &type) { if(!Torch::contains(this->gFactories, type)){ return std::nullopt; } return this->gFactories[type]; } std::optional Companion::SearchTable(uint32_t addr){ for(auto& table : this->gTables){ if(addr >= table.start && addr <= table.end){ return table; } } return std::nullopt; } std::optional Companion::GetEnumFromValue(const std::string& key, int32_t id) { if(!Torch::contains(this->gEnums, key)){ return std::nullopt; } if(!Torch::contains(this->gEnums[key], id)){ return std::nullopt; } return this->gEnums[key][id]; } std::optional Companion::GetFileOffsetFromSegmentedAddr(const uint8_t segment) const { auto segments = this->gConfig.segment; if(Torch::contains(segments.temporal, segment)) { return segments.temporal[segment]; } if(Torch::contains(segments.local, segment)) { return segments.local[segment]; } if(Torch::contains(segments.global, segment)) { return segments.global[segment]; } return std::nullopt; } uint32_t Companion::PatchVirtualAddr(uint32_t addr) { if (addr & 0x80000000) { if (Torch::contains(gVirtualAddrMap, gCurrentFile)) { addr -= std::get<0>(gVirtualAddrMap[gCurrentFile]); addr += std::get<1>(gVirtualAddrMap[gCurrentFile]); } } return addr; } std::optional> Companion::GetNodeByAddr(uint32_t addr){ if(!Torch::contains(this->gAddrMap, this->gCurrentFile)){ return std::nullopt; } // HACK: Adjust address to rom address if virtual address addr = PatchVirtualAddr(addr); if(!Torch::contains(this->gAddrMap[this->gCurrentFile], addr)){ for (auto &file : this->gCurrentExternalFiles) { if (!Torch::contains(this->gAddrMap, file)) { SPDLOG_WARN("GetNodeByAddr: External File {} Not Found.", file); continue; } if (!Torch::contains(this->gAddrMap[file], addr)) { continue; } return this->gAddrMap[file][addr]; } return std::nullopt; } return this->gAddrMap[this->gCurrentFile][addr]; } std::optional Companion::GetStringByAddr(const uint32_t addr) { if(Torch::contains(this->gManualSegments, addr)) { return this->gManualSegments[addr]; } auto node = this->GetNodeByAddr(addr); if(!node.has_value()) { return std::nullopt; } return std::get<0>(node.value()); } std::optional> Companion::GetSafeNodeByAddr(const uint32_t addr, std::string type) { auto node = this->GetNodeByAddr(addr); if(!node.has_value()) { return std::nullopt; } auto [name, n] = node.value(); auto n_type = GetTypeNode(n); if(n_type != type) { throw std::runtime_error("Requested node type does not match with the target node type at " + Torch::to_hex(addr, false) + " Found: " + n_type + " Expected: " + type); } return node; } std::optional Companion::GetSafeStringByAddr(const uint32_t addr, std::string type) { if(Torch::contains(this->gManualSegments, addr)) { return this->gManualSegments[addr]; } auto node = this->GetNodeByAddr(addr); if(!node.has_value()) { return std::nullopt; } auto [name, n] = node.value(); auto n_type = GetTypeNode(n); if(n_type != type) { throw std::runtime_error("Requested node type does not match with the target node type at " + Torch::to_hex(addr, false) + " Found: " + n_type + " Expected: " + type); } return std::get<0>(node.value()); } std::string Companion::GetSymbolFromAddr(uint32_t address, bool validZero) { auto dec = Companion::Instance->GetNodeByAddr(address); std::ostringstream outSymbol; if(address == 0 && !validZero) { outSymbol << "NULL"; } else if (dec.has_value()) { auto node = std::get<1>(dec.value()); auto symbol = GetSafeNode(node, "symbol"); outSymbol << "&" << symbol; } else { outSymbol << "0x" << std::uppercase << std::hex << address; } return outSymbol.str(); } std::optional Companion::GetParseDataByAddr(uint32_t addr) { if (CONTAINS(this->gParseResults, this->gCurrentFile)) { for (auto& result : this->gParseResults[this->gCurrentFile]) { if (result.data.has_value() && result.GetOffset() == addr) { return result; } } } for (auto &file : this->gCurrentExternalFiles) { if (!CONTAINS(this->gParseResults, this->gCurrentFile)) { SPDLOG_INFO("GetParseDataByAddr: External File {} Not Found.", file); continue; } for (auto& result : this->gParseResults[file]) { if (result.data.has_value() && result.GetOffset() == addr) { return result; } } } return std::nullopt; } std::optional Companion::GetParseDataBySymbol(const std::string& symbol) { if (CONTAINS(this->gParseResults, this->gCurrentFile)) { for (auto& result : this->gParseResults[this->gCurrentFile]) { auto sym = GetNode(result.node, "symbol"); if (result.data.has_value() && sym.has_value() && sym.value() == symbol) { return result; } } } for (auto &file : this->gCurrentExternalFiles) { if (!CONTAINS(this->gParseResults, this->gCurrentFile)) { SPDLOG_INFO("GetParseDataBySymbol: External File {} Not Found.", file); continue; } for (auto& result : this->gParseResults[file]) { auto sym = GetNode(result.node, "symbol"); if (result.data.has_value() && sym.has_value() && sym.value() == symbol) { return result; } } } return std::nullopt; } std::optional>> Companion::GetNodesByType(const std::string& type){ std::vector> nodes; if(!Torch::contains(this->gAddrMap, this->gCurrentFile)){ return nodes; } for(auto& [addr, tpl] : this->gAddrMap[this->gCurrentFile]){ auto [name, node] = tpl; const auto n_type = GetTypeNode(node); if(node["autogen"]){ SPDLOG_DEBUG("Skipping autogenerated asset {}", name); continue; } if(n_type == type){ nodes.push_back(tpl); } } return nodes; } void Companion::SetProcess(bool shouldProcess) { this->process = shouldProcess; } void Companion::RegisterCompanionFile(const std::string path, std::vector data) { this->gCompanionFiles[path] = data; SPDLOG_TRACE("Registered companion file {}", path); } std::string Companion::NormalizeAsset(const std::string& name) const { auto path = fs::path(this->gCurrentFile).stem().string() + "_" + name; return path; } static std::string& ConvertWinToUnixSlash(std::string& path) { std::replace(path.begin(), path.end(), '\\', '/'); return path; } std::string Companion::RelativePath(const std::string& path) const { std::string doutput = (this->gCurrentDirectory / path).string(); ConvertWinToUnixSlash(doutput); return doutput; } std::string Companion::RelativePathToDestDir(const std::string& path) const { std::string doutput = fs::path(path).lexically_relative(this->gDestinationDirectory).string(); ConvertWinToUnixSlash(doutput); return doutput; } std::string Companion::RelativePathToSrcDir(const std::string& path) const { std::string doutput = fs::path(path).lexically_relative(this->gSourceDirectory).string(); ConvertWinToUnixSlash(doutput); return doutput; } std::string Companion::CalculateHash(const std::vector& data) { sha1::SHA1 s; s.processBytes(data.data(), data.size()); uint32_t hash[5]; s.getDigest(hash); char buf[41]; std::snprintf(buf, sizeof(buf), "%08x%08x%08x%08x%08x", hash[0], hash[1], hash[2], hash[3], hash[4]); return std::string(buf); } std::vector Companion::ParseVersionString(const std::string& version) { uint16_t major = 0; uint16_t minor = 0; uint16_t patch = 0; std::sscanf(version.c_str(), "%hu.%hu.%hu", &major, &minor, &patch); auto wv = LUS::BinaryWriter(); wv.SetEndianness(Torch::Endianness::Big); wv.Write(major); wv.Write(minor); wv.Write(patch); wv.Close(); return wv.ToVector(); } std::optional Companion::AddAsset(YAML::Node asset) { if(!asset["offset"] || !asset["type"]) { return std::nullopt; } asset["offset"] = PatchVirtualAddr(GetSafeNode(asset, "offset")); const auto type = GetTypeNode(asset); const auto offset = GetSafeNode(asset, "offset"); const auto symbol = GetSafeNode(asset, "symbol", ""); const auto decl = this->GetNodeByAddr(offset); if(decl.has_value()) { auto found = std::get<1>(decl.value()); if(GetTypeNode(found) != type) { SPDLOG_ERROR("Asset clash detected {} vs {} at 0x{:X}", type, GetTypeNode(found), offset); } else { return found; } } auto rom = this->GetRomData(); auto factory = this->GetFactory(type); if(!factory.has_value()) { return std::nullopt; } std::string output; std::string typeId = ConvertType(type); if(!symbol.empty()) { output = symbol; } else if(Decompressor::IsSegmented(offset)){ output = this->NormalizeAsset("seg" + std::to_string(SEGMENT_NUMBER(offset)) +"_" + typeId + "_" + Torch::to_hex(SEGMENT_OFFSET(offset), false)); } else { output = this->NormalizeAsset(typeId + "_" + Torch::to_hex(offset, false)); } std::replace(output.begin(), output.end(), ':', '_'); asset["autogen"] = true; asset["symbol"] = output; auto result = this->RegisterAsset(output, asset); if(!gCurrentVirtualPath.empty()) { asset["path"] = gCurrentVirtualPath; } if(result.has_value()){ asset["vpath"] = std::get<0>(result.value()); return std::get<1>(result.value()); } return std::nullopt; }