Fixed dependency system and modding reimport

7 files changed, 207 insertions, 35 deletions

diff --git a/src/Companion.cpp b/src/Companion.cpp
index 99ca2a7..ce915cb 100644
--- a/src/Companion.cpp
+++ b/src/Companion.cpp
@@ -163,7 +163,7 @@ void Companion::ParseEnums(std::string& header) {
             continue;
         }
 
-        if(line.find("}") != std::string::npos) {
+        if(line.find('}') != std::string::npos) {
             inEnum = false;
             continue;
         }
@@ -171,10 +171,10 @@ void Companion::ParseEnums(std::string& header) {
         // Remove any comments and non-alphanumeric characters
         line = std::regex_replace(line, std::regex(R"((/\*.*?\*/)|(//.*$)|([^a-zA-Z0-9=_\-\.]))"), "");
 
-        if(line.find("=") != std::string::npos) {
-            auto value = line.substr(line.find("=") + 1);
-            auto name = line.substr(0, line.find("="));
-            enumIndex = static_cast<int32_t>(std::stoll(value, 0, 0));
+        if(line.find('=') != std::string::npos) {
+            auto value = line.substr(line.find('=') + 1);
+            auto name = line.substr(0, line.find('='));
+            enumIndex = static_cast<int32_t>(std::stoll(value, nullptr, 0));
             this->gEnums[enumName][enumIndex] = name;
         } else {
             enumIndex++;
@@ -196,6 +196,7 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
         SPDLOG_INFO("- [{}] Processing {}", type, name);
     }
     spdlog::set_pattern(line);
+    node["vpath"] = name;
 
     auto factory = this->GetFactory(type);
     if(!factory.has_value()){
@@ -205,27 +206,32 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
     auto impl = factory->get();
 
     auto exporter = impl->GetExporter(this->gConfig.exporterType);
-    if(!exporter.has_value()){
+    if(!exporter.has_value() && !impl->HasModdedDependencies()){
         SPDLOG_WARN("No exporter found for {}", name);
         return std::nullopt;
     }
 
+    bool executeDef = true;
     std::optional<std::shared_ptr<IParsedData>> result;
     if(this->gConfig.modding && impl->SupportModdedAssets() && this->gModdedAssetPaths.contains(name)) {
         auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name];
-        if(!fs::exists(path)) {
+        if(!exists(path)) {
             SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]);
-            return std::nullopt;
-        }
+        } else {
+            std::ifstream input(path, std::ios::binary);
+            std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
+            input.close();
 
-        std::ifstream input(path, std::ios::binary);
-        std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
-        input.close();
+            result = impl->parse_modding(data, node);
+            executeDef = !result.has_value();
+        }
+    }
 
-        result = impl->parse_modding(data, node);
-    } else if(this->gConfig.parseMode == ParseMode::Default) {
+    if(executeDef && this->gConfig.parseMode == ParseMode::Default) {
         result = impl->parse(this->gRomData, node);
-    } else if(this->gConfig.parseMode == ParseMode::Directory) {
+    }
+
+    if(executeDef && this->gConfig.parseMode == ParseMode::Directory) {
         auto path = GetSafeNode<std::string>(node, "path");
         std::ifstream input( path, std::ios::binary );
         auto data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {} );
@@ -238,16 +244,16 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
         return std::nullopt;
     }
 
-    for (auto [fst, snd] : this->gAssetDependencies[this->gCurrentFile]) {
-        if(snd.second) {
+    for (auto& [name, node, processed] : this->gDependencies) {
+        if(processed) {
             continue;
         }
-        std::string doutput = (this->gCurrentDirectory / fst).string();
+        std::string doutput = (this->gCurrentDirectory / name).string();
         std::replace(doutput.begin(), doutput.end(), '\\', '/');
-        this->gAssetDependencies[this->gCurrentFile][fst].second = true;
-        auto result = this->ParseNode(snd.first, doutput);
-        if(result.has_value()) {
-            this->gParseResults[this->gCurrentFile].push_back(result.value());
+        processed = true;
+        auto dResult = this->ParseNode(node, doutput);
+        if(dResult.has_value()) {
+            this->gParseResults[this->gCurrentFile].push_back(dResult.value());
         }
         spdlog::set_pattern(regular);
         SPDLOG_INFO("------------------------------------------------");
@@ -256,6 +262,7 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
 
 
     SPDLOG_INFO("Processed {}", name);
+    this->gDependencies.clear();
 
     return ParseResultData {
         name, type, node, result
@@ -397,6 +404,7 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) {
     this->gEnablePadGen = GetSafeNode<bool>(node, "autopads", true);
     this->gNodeForceProcessing = GetSafeNode<bool>(node, "force", false);
     this->gIndividualIncludes = GetSafeNode<bool>(node, "individual_data_incs", false);
+    this->gCurrentVirtualPath = GetSafeNode<std::string>(node, "path", "");
 }
 
 void Companion::ParseHash() {
@@ -545,6 +553,10 @@ void Companion::ProcessFile(YAML::Node root) {
                     assetNode["offset"] = (segment << 24) | assetNode["offset"].as<uint32_t>();
                 }
 
+                if(!gCurrentVirtualPath.empty()) {
+                    node["path"] = gCurrentVirtualPath;
+                }
+
                 this->gAddrMap[this->gCurrentFile][assetNode["offset"].as<uint32_t>()] = std::make_tuple(output, assetNode);
             }
         } else {
@@ -570,6 +582,10 @@ void Companion::ProcessFile(YAML::Node root) {
                 }
             }
 
+            if(!gCurrentVirtualPath.empty()) {
+                node["path"] = gCurrentVirtualPath;
+            }
+
             this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = std::make_tuple(output, node);
         }
     }
@@ -580,6 +596,7 @@ void Companion::ProcessFile(YAML::Node root) {
     this->gFileHeader.clear();
     this->gCurrentPad = 0;
     this->gCurrentVram = std::nullopt;
+    this->gCurrentVirtualPath = "";
     this->gCurrentSegmentNumber = 0;
     this->gCurrentCompressionType = CompressionType::None;
     this->gCurrentFileOffset = 0;
@@ -624,6 +641,10 @@ void Companion::ProcessFile(YAML::Node root) {
                     node["offset"] = (segment << 24) | node["offset"].as<uint32_t>();
                 }
 
+                if(!gCurrentVirtualPath.empty()) {
+                    node["path"] = gCurrentVirtualPath;
+                }
+
                 auto output = (this->gCurrentDirectory / entryName / childName).string();
                 std::replace(output.begin(), output.end(), '\\', '/');
                 this->gConfig.segment.temporal.clear();
@@ -639,6 +660,11 @@ void Companion::ProcessFile(YAML::Node root) {
                     assetNode["offset"] = (gCurrentSegmentNumber << 24) | offset;
                 }
             }
+
+            if(!gCurrentVirtualPath.empty()) {
+                assetNode["path"] = gCurrentVirtualPath;
+            }
+
             std::string output = (this->gCurrentDirectory / entryName).string();
             std::replace(output.begin(), output.end(), '\\', '/');
             this->gConfig.segment.temporal.clear();
@@ -662,7 +688,7 @@ void Companion::ProcessFile(YAML::Node root) {
         const auto impl = this->GetFactory(result.type)->get();
         const auto exporter = impl->GetExporter(this->gConfig.exporterType);
 
-        if(exporter == nullptr) {
+        if(!exporter.has_value()) {
             continue;
         }
 
@@ -691,6 +717,7 @@ void Companion::ProcessFile(YAML::Node root) {
                     create_directories(fs::path(dpath).parent_path());
                 }
 
+                SPDLOG_INFO(ogname);
                 this->gModdedAssetPaths[ogname] = result.name;
 
                 std::ofstream file(dpath, std::ios::binary);
@@ -1199,7 +1226,9 @@ std::optional<std::tuple<std::string, YAML::Node>> Companion::RegisterAsset(cons
         return std::nullopt;
     }
 
-    this->gAssetDependencies[this->gCurrentFile][name] = std::make_pair(node, false);
+    this->gDependencies.push_back(Dependency {
+        name, node, false
+    });
 
     auto output = (this->gCurrentDirectory / name).string();
     std::replace(output.begin(), output.end(), '\\', '/');
@@ -1394,13 +1423,12 @@ std::string Companion::CalculateHash(const std::vector<uint8_t>& data) {
 }
 
 static std::string ConvertType(std::string type) {
-    int index;
+    int index = type.find(':');
 
-    if((index = type.find(':')) != std::string::npos) {
+    if(index != std::string::npos) {
         type = type.substr(index + 1);
     }
-    type = std::regex_replace(type, std::regex(R"([^_A-Za-z0-9]*)"), "");
-    std::transform(type.begin(), type.end(), type.begin(), ::tolower);
+    std::transform(type.begin(), type.end(), type.begin(), tolower);
     return type;
 }
 
@@ -1428,7 +1456,7 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
     std::string output;
     std::string typeId = ConvertType(type);
 
-    if(symbol != "") {
+    if(!symbol.empty()) {
         output = symbol;
     } else if(Decompressor::IsSegmented(offset)){
         output = this->NormalizeAsset("seg" + std::to_string(SEGMENT_NUMBER(offset)) +"_" + typeId + "_" + Torch::to_hex(SEGMENT_OFFSET(offset), false));
@@ -1441,7 +1469,7 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
     auto result = this->RegisterAsset(output, asset);
 
     if(result.has_value()){
-        asset["path"] = std::get<0>(result.value());
+        asset["vpath"] = std::get<0>(result.value());
 
         return std::get<1>(result.value());
     }
diff --git a/src/Companion.h b/src/Companion.h
index f8990b5..15a607c 100644
--- a/src/Companion.h
+++ b/src/Companion.h
@@ -99,6 +99,12 @@ struct ParseResultData {
     }
 };
 
+struct Dependency {
+    std::string name;
+    YAML::Node node;
+    bool processed;
+};
+
 class Companion {
 public:
     static Companion* Instance;
@@ -171,6 +177,7 @@ private:
 
     // Temporal Variables
     std::string gCurrentFile;
+    std::string gCurrentVirtualPath;
     std::string gFileHeader;
     bool gEnablePadGen = false;
     uint32_t gCurrentPad = 0;
@@ -183,12 +190,12 @@ private:
     std::unordered_set<std::string> gProcessedFiles;
 
     std::unordered_map<std::string, std::vector<ParseResultData>> gParseResults;
+    std::vector<Dependency> gDependencies;
 
     std::unordered_map<std::string, std::string> gModdedAssetPaths;
     std::variant<std::vector<std::string>, std::string> gWriteOrder;
     std::unordered_map<std::string, std::shared_ptr<BaseFactory>> gFactories;
     std::unordered_map<std::string, std::map<std::string, std::vector<WriteEntry>>> gWriteMap;
-    std::unordered_map<std::string, std::map<std::string, std::pair<YAML::Node, bool>>> gAssetDependencies;
     std::unordered_map<std::string, std::unordered_map<uint32_t, std::tuple<std::string, YAML::Node>>> gAddrMap;
 
     void ProcessFile(YAML::Node root);
diff --git a/src/factories/BaseFactory.h b/src/factories/BaseFactory.h
index bc1fb8c..ff2297f 100644
--- a/src/factories/BaseFactory.h
+++ b/src/factories/BaseFactory.h
@@ -101,6 +101,9 @@ public:
     virtual bool SupportModdedAssets() {
         return false;
     }
+    virtual bool HasModdedDependencies() {
+        return false;
+    }
     virtual uint32_t GetAlignment() {
         return 4;
     }
diff --git a/src/factories/TextureFactory.cpp b/src/factories/TextureFactory.cpp
index 0299ce0..3762ad8 100644
--- a/src/factories/TextureFactory.cpp
+++ b/src/factories/TextureFactory.cpp
@@ -245,7 +245,7 @@ ExportResult TextureModdingExporter::Export(std::ostream&write, std::shared_ptr<
     auto ext = GetSafeNode<std::string>(node, "format");
 
     std::transform(ext.begin(), ext.end(), ext.begin(), tolower);
-    (*replacement) += "." + ext + ".png";
+    *replacement += "." + ext + ".png";
 
     switch (format.type) {
         case TextureType::TLUT:
@@ -269,7 +269,6 @@ ExportResult TextureModdingExporter::Export(std::ostream&write, std::shared_ptr<
         }
         case TextureType::Palette8bpp:
         case TextureType::Palette4bpp: {
-            // This check needed until sf64 has tluts fixed.
             if (node["tlut_symbol"]) {
                 auto tlut = GetSafeNode<std::string>(node,"tlut_symbol");
                 auto palette = Companion::Instance->GetParseDataBySymbol(tlut);
@@ -474,7 +473,8 @@ std::optional<std::shared_ptr<IParsedData>> TextureFactory::parse_modding(std::v
             // } else {
 
             // }
-            break;
+            SPDLOG_ERROR("Unsupported texture format for modding: {}", format);
+            return std::nullopt;
         }
         case TextureType::Grayscale8bpp:
         case TextureType::Grayscale4bpp: {
diff --git a/src/factories/sf64/MessageFactory.cpp b/src/factories/sf64/MessageFactory.cpp
index ee1beb0..81b167f 100644
--- a/src/factories/sf64/MessageFactory.cpp
+++ b/src/factories/sf64/MessageFactory.cpp
@@ -4,6 +4,7 @@
 #include "spdlog/spdlog.h"
 #include "Companion.h"
 #include <regex>
+#include <sstream>
 
 #define END_CODE 0
 #define NEWLINE_CODE 1
@@ -50,6 +51,21 @@ std::unordered_map<std::string, std::string> ASCIITable = {
     { "CLN", ":" }, { "PIP", "| " }
 };
 
+std::vector<std::string> gASCIIFullTable = {
+    "\0", "\n", "{NP:2}", "{NP:3}", "{NP:4}", "{NP:5}", "{NP:6}", "{NP:7}",
+    "{PRI:0}", "{PRI:1}", "{PRI:2}", "{PRI:3}", " ", "{HSP}", "{QSP}", "{NXT}",
+    "{C:<}", "{C:^}", "{C:>}", "{C:v}", "^", "<", "v", ">",
+    "A", "B", "C", "D", "E", "F", "G", "H",
+    "I", "J", "K", "L", "M", "N", "O", "P",
+    "Q", "R", "S", "T", "U", "V", "W", "X",
+    "Y", "Z", "a", "b", "c", "d", "e", "f",
+    "g", "h", "i", "j", "k", "l", "m", "n",
+    "o", "p", "q", "r", "s", "t", "u", "v",
+    "w", "x", "y", "z", "!", "?", "-", ",",
+    ".", "0", "1", "2", "3", "4", "5", "6",
+    "7", "8", "9", "'", "(", ")", ":", "|",
+};
+
 ExportResult SF64::MessageHeaderExporter::Export(std::ostream &write, std::shared_ptr<IParsedData> raw, std::string& entryName, YAML::Node &node, std::string* replacement) {
     const auto symbol = GetSafeNode(node, "symbol", entryName);
 
@@ -105,6 +121,34 @@ ExportResult SF64::MessageBinaryExporter::Export(std::ostream &write, std::share
     return std::nullopt;
 }
 
+ExportResult SF64::MessageModdingExporter::Export(std::ostream&write, std::shared_ptr<IParsedData> raw, std::string&entryName, YAML::Node&node, std::string* replacement) {
+    const auto data = std::static_pointer_cast<MessageData>(raw);
+    const auto symbol = GetSafeNode(node, "symbol", entryName);
+    *replacement += ".yml";
+
+    auto count = data->mMessage.size();
+    std::stringstream stream;
+    YAML::Emitter out;
+    out << YAML::BeginMap;
+    out << YAML::Key << symbol;
+    out << YAML::Value << YAML::BeginSeq;
+
+    for(size_t i = 0; i < count; i++) {
+        if(data->mMessage[i] == NEWLINE_CODE){
+            stream << "\0";
+            out << stream.str();
+            stream.str("");
+        } else {
+            stream << gASCIIFullTable[data->mMessage[i]];
+        }
+    }
+
+    out << YAML::EndSeq << YAML::EndMap;
+    write.write(out.c_str(), out.size());
+
+    return std::nullopt;
+}
+
 std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse(std::vector<uint8_t>& buffer, YAML::Node& node) {
     std::vector<uint16_t> message;
     std::ostringstream mesgStr;
@@ -113,7 +157,6 @@ std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse(std::vec
     reader.SetEndianness(LUS::Endianness::Big);
 
     uint16_t c;
-    bool lastWasSpace = false;
     std::string whitespace = "";
     do {
         c = reader.ReadUInt16();
@@ -139,4 +182,87 @@ std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse(std::vec
     } while(c != END_CODE);
 
     return std::make_shared<MessageData>(message, mesgStr.str());
+}
+
+std::optional<uint16_t> getCharByCode(const std::string& code) {
+    auto it = std::find(gASCIIFullTable.begin(), gASCIIFullTable.end(), code) - gASCIIFullTable.begin();
+    if(it < gASCIIFullTable.size()){
+        return it;
+    }
+    return std::nullopt;
+}
+
+std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse_modding(std::vector<uint8_t>& buffer, YAML::Node& data) {
+    std::vector<uint16_t> message;
+    std::ostringstream mesgStr;
+    std::string whitespace = "";
+
+    YAML::Node node;
+
+    try {
+        std::string text((char*) buffer.data(), buffer.size());
+        node = YAML::Load(text.c_str());
+    } catch (YAML::ParserException& e) {
+        SPDLOG_ERROR("Failed to parse message data: {}", e.what());
+        SPDLOG_ERROR("{}", (char*) buffer.data());
+        return std::nullopt;
+    }
+
+    std::regex fmt("\\(([^)]+)\\)");
+
+    std::vector<std::string> lines = node.begin()->second.as<std::vector<std::string>>();
+    for(auto& line : lines){
+        for(size_t i = 0; i < line.size(); i++){
+            char c = line[i];
+            std::string enumCode;
+            if(c == '{' && line.substr(i).find('}') != std::string::npos){
+                auto code = line.substr(i, line.substr(i).find('}') + 1);
+                auto opcode = getCharByCode(code);
+                if(opcode.has_value()){
+                    auto x = opcode.value();
+                    message.push_back(x);
+                    i += line.substr(i).find('}');
+                    enumCode = gCharCodeEnums[x];
+                } else {
+                    SPDLOG_WARN("Unknown character: {}", code);
+                }
+            } else {
+                auto code = getCharByCode(std::string(1, c));
+                if(code.has_value()){
+                    auto x = code.value();
+                    message.push_back(x);
+                    enumCode = gCharCodeEnums[x];
+                } else {
+                    SPDLOG_WARN("Unknown character: {}", c);
+                }
+            }
+
+            if(!enumCode.empty()) {
+                if(enumCode.find("SP") != std::string::npos && whitespace.empty()) {
+                    whitespace = " ";
+                }
+                if(c == NEWLINE_CODE) {
+                    whitespace += "\n";
+                }
+                if (c >= CLF_CODE) {
+                    mesgStr << whitespace;
+                    whitespace = "";
+                }
+                if(enumCode.starts_with("_")){
+                    mesgStr << enumCode.substr(1);
+                } else if(ASCIITable.contains(enumCode)){
+                    mesgStr << ASCIITable[enumCode];
+                }
+            }
+        }
+
+        if(line != lines.back()){
+            message.push_back(NEWLINE_CODE);
+            mesgStr << "\n";
+        }
+    }
+
+    message.push_back(END_CODE);
+
+    return std::make_shared<MessageData>(message, mesgStr.str());
 }
\ No newline at end of file
diff --git a/src/factories/sf64/MessageFactory.h b/src/factories/sf64/MessageFactory.h
index 9b119f6..c000c36 100644
--- a/src/factories/sf64/MessageFactory.h
+++ b/src/factories/sf64/MessageFactory.h
@@ -24,15 +24,22 @@ class MessageBinaryExporter : public BaseExporter {
     ExportResult Export(std::ostream& write, std::shared_ptr<IParsedData> data, std::string& entryName, YAML::Node& node, std::string* replacement) override;
 };
 
+class MessageModdingExporter : public BaseExporter {
+    ExportResult Export(std::ostream& write, std::shared_ptr<IParsedData> data, std::string& entryName, YAML::Node& node, std::string* replacement) override;
+};
+
 class MessageFactory : public BaseFactory {
 public:
     std::optional<std::shared_ptr<IParsedData>> parse(std::vector<uint8_t>& buffer, YAML::Node& data) override;
+    std::optional<std::shared_ptr<IParsedData>> parse_modding(std::vector<uint8_t>& buffer, YAML::Node& data) override;
     inline std::unordered_map<ExportType, std::shared_ptr<BaseExporter>> GetExporters() override {
         return {
             REGISTER(Code, MessageCodeExporter)
             REGISTER(Header, MessageHeaderExporter)
             REGISTER(Binary, MessageBinaryExporter)
+            REGISTER(Modding, MessageModdingExporter)
         };
     }
+    bool SupportModdedAssets() override { return true; }
 };
 }
\ No newline at end of file
diff --git a/src/factories/sf64/MessageLookupFactory.h b/src/factories/sf64/MessageLookupFactory.h
index 1c2c884..f647121 100644
--- a/src/factories/sf64/MessageLookupFactory.h
+++ b/src/factories/sf64/MessageLookupFactory.h
@@ -38,5 +38,6 @@ public:
             REGISTER(Binary, MessageLookupBinaryExporter)
         };
     }
+    bool HasModdedDependencies() override { return true; }
 };
 }
\ No newline at end of file