Implemented full file change detection

4 files changed, 54 insertions, 127 deletions

diff --git a/src/Companion.cpp b/src/Companion.cpp
index 8d96b3c..40870c2 100644
--- a/src/Companion.cpp
+++ b/src/Companion.cpp
@@ -199,19 +199,6 @@ void Companion::ExtractNode(YAML::Node& node, std::string& name, SWrapper* binar
         input.close();
 
         result = impl->parse_modding(data, node);
-    } else if(this->gNodeIsCacheable && impl->IsCacheable()) {
-        if(this->gCacheData.contains(this->gCurrentCacheHash)) {
-            result = impl->CreateDataPointer();
-            result->get()->FromCacheBuffer(this->GetCache(name));
-        } else {
-            result = impl->parse(this->gRomData, node);
-            if(result.has_value()) {
-                auto cache = result.value()->ToCacheBuffer();
-                if(cache.has_value()) {
-                    this->StoreCache(name, cache.value());
-                }
-            }
-        }
     } else {
         result = impl->parse(this->gRomData, node);
     }
@@ -405,92 +392,60 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) {
     }
 
     this->gEnablePadGen = GetSafeNode<bool>(node, "autopads", true);
-    this->gNodeIsCacheable = GetSafeNode<bool>(node, "cacheable", true);
+    this->gNodeForceProcessing = GetSafeNode<bool>(node, "force", false);
 }
 
-void Companion::ParseCache() {
-    const std::string out = "torch.cache.yml";
+void Companion::ParseHash() {
+    const std::string out = "torch.hash.yml";
 
     if(fs::exists(out)) {
-        this->gCacheNode = YAML::LoadFile(out);
+        this->gHashNode = YAML::LoadFile(out);
     } else {
-        this->gCacheNode = YAML::Node();
+        this->gHashNode = YAML::Node();
     }
 }
 
-void Companion::PrepareCache(const std::string& path) {
-    std::ifstream yaml(path);
-    bool hasChanges = false;
-    const std::vector<uint8_t> data = std::vector<uint8_t>(std::istreambuf_iterator( yaml ), {});
-    this->gCurrentCacheHash = CalculateHash(data);
-
-    if(this->gCacheNode[path]) {
-        const auto cache = GetSafeNode<std::string>(this->gCacheNode, path);
-        if(cache == this->gCurrentCacheHash) {
-            const auto tdir = "tcache/" + cache;
-            if(!fs::exists(tdir)) {
-                fs::create_directories(tdir);
-            } else {
-                for(auto& entry : fs::directory_iterator("tcache/" + cache)) {
-                    const auto ext = entry.path().extension().string();
-
-                    if(ext == ".data") {
-                        std::ifstream file(entry.path(), std::ios::binary);
-                        std::vector<uint8_t> buffer = std::vector<uint8_t>(std::istreambuf_iterator(file), {});
-                        file.close();
-                        std::string cpath = entry.path().filename().string();
-                        std::replace(cpath.begin(), cpath.end(), '+', '/');
-                        this->gCacheData[cache][cpath] = buffer;
-                    } else if(ext == ".yaml") {
-                        std::ifstream file(entry.path(), std::ios::binary);
-                        YAML::Node dependencies = YAML::LoadFile(entry.path().string());
-                        for(auto dep = dependencies.begin(); dep != dependencies.end(); ++dep) {
-                            this->gAssetDependencies[this->gCurrentFile][dep->first.as<std::string>()] = std::make_pair(dep->second.as<YAML::Node>(), false);
-                        }
-                    }
-                }
-            }
-        } else {
-            hasChanges = true;
-            this->gCacheNode[path] = this->gCurrentCacheHash;
-            fs::remove("cache/" + cache);
-        }
-    } else {
-        this->gCacheNode[path] = this->gCurrentCacheHash;
-    }
-
-    if(hasChanges) {
-        std::ofstream file("torch.cache.yml", std::ios::binary);
-        file << this->gCacheNode;
-        file.close();
+std::string ExportTypeToString(ExportType type) {
+    switch (type) {
+        case ExportType::Binary: return "Binary";
+        case ExportType::Header: return "Header";
+        case ExportType::Code: return "Code";
+        case ExportType::Modding: return "Modding";
+        default:
+            throw std::runtime_error("Invalid ExportType");
     }
 }
 
-void Companion::StoreCache(const std::string path, const std::vector<uint8_t>& value) {
-    std::string outpath = path;
-    std::replace(outpath.begin(), outpath.end(), '/', '+');
+bool Companion::NodeHasChanges(const std::string& path) {
 
+    if(this->gConfig.exporterType == ExportType::Modding) {
+        return true;
+    }
 
-    YAML::Node dependencies = YAML::Node();
+    std::ifstream yaml(path);
+    const std::vector<uint8_t> data = std::vector<uint8_t>(std::istreambuf_iterator( yaml ), {});
+    this->gCurrentHash = CalculateHash(data);
 
-    // Dump asset dependencies
-    for (auto [fst, snd] : this->gAssetDependencies[this->gCurrentFile]) {
-        dependencies[fst] = snd.first;
-    }
+    if(this->gHashNode[path]) {
+        auto entry = GetSafeNode<YAML::Node>(this->gHashNode, path);
+        const auto hash = GetSafeNode<std::string>(entry, "hash");
+        auto modes = GetSafeNode<YAML::Node>(entry, "extracted");
+        auto extracted = GetSafeNode<bool>(modes, ExportTypeToString(this->gConfig.exporterType));
 
-    std::ofstream deps("tcache/" + this->gCurrentCacheHash + "/" + outpath + "-deps.yaml", std::ios::binary);
-    deps << dependencies;
-    deps.close();
+        if(hash == this->gCurrentHash && extracted) {
+            SPDLOG_INFO("Skipping {} as it has not changed", path);
+            return false;
+        }
+    }
 
-    std::ofstream file("tcache/" + this->gCurrentCacheHash + "/" + outpath + ".data", std::ios::binary);
-    file.write(reinterpret_cast<const char*>(value.data()), value.size());
-    file.close();
-}
+    this->gHashNode[path] = YAML::Node();
+    this->gHashNode[path]["hash"] = this->gCurrentHash;
+    this->gHashNode[path]["extracted"] = YAML::Node();
+    for(size_t m = 0; m < static_cast<size_t>(ExportType::Modding); m++) {
+        this->gHashNode[path]["extracted"][ExportTypeToString(static_cast<ExportType>(m))] = false;
+    }
 
-std::vector<uint8_t>& Companion::GetCache(const std::string path) {
-    std::string outpath = path;
-    std::replace(outpath.begin(), outpath.end(), '+', '/');
-    return this->gCacheData[this->gCurrentCacheHash][outpath];
+    return true;
 }
 
 void Companion::Process() {
@@ -633,7 +588,7 @@ void Companion::Process() {
         }
     }
 
-    this->ParseCache();
+    this->ParseHash();
 
     SPDLOG_INFO("------------------------------------------------");
     spdlog::set_pattern(line);
@@ -669,8 +624,6 @@ void Companion::Process() {
         this->gCurrentDirectory = relative(entry.path(), path).replace_extension("");
         this->gCurrentFile = yamlPath;
 
-        this->PrepareCache(yamlPath);
-
         // Set compressed file offsets and compression type
         if (auto segments = root[":config"]["segments"]) {
             if (segments.IsSequence() && segments.size() > 0) {
@@ -757,8 +710,8 @@ void Companion::Process() {
             this->ParseCurrentFileConfig(root[":config"]);
         }
 
-        if(this->gNodeIsCacheable) {
-            this->PrepareCache(yamlPath);
+        if(!this->NodeHasChanges(yamlPath) && !this->gNodeForceProcessing) {
+            continue;
         }
 
         spdlog::set_pattern(regular);
@@ -953,6 +906,7 @@ void Companion::Process() {
             }
 
             file.close();
+            this->gHashNode[this->gCurrentFile]["extracted"][ExportTypeToString(this->gConfig.exporterType)] = true;
         }
     }
 
@@ -962,6 +916,12 @@ void Companion::Process() {
         vWriter.Close();
         wrapper->Close();
     }
+
+    // Write entries hash
+    std::ofstream file("torch.hash.yml", std::ios::binary);
+    file << this->gHashNode;
+    file.close();
+
     auto end = duration_cast<milliseconds>(system_clock::now().time_since_epoch());
     auto level = spdlog::get_level();
     spdlog::set_level(spdlog::level::info);
diff --git a/src/Companion.h b/src/Companion.h
index 07d9f20..6f687c2 100644
--- a/src/Companion.h
+++ b/src/Companion.h
@@ -86,7 +86,7 @@ public:
 
     void Init(ExportType type);
 
-    void PrepareCache(const std::string& string);
+    bool NodeHasChanges(const std::string& string);
 
     void Process();
 
@@ -125,11 +125,11 @@ private:
     TorchConfig gConfig;
     YAML::Node gModdingConfig;
     fs::path gCurrentDirectory;
-    std::string gCurrentCacheHash;
+    std::string gCurrentHash;
     std::vector<uint8_t> gRomData;
     std::filesystem::path gRomPath;
-    bool gNodeIsCacheable;
-    YAML::Node gCacheNode;
+    bool gNodeForceProcessing = false;
+    YAML::Node gHashNode;
     std::shared_ptr<N64::Cartridge> gCartridge;
     std::unordered_map<std::string, std::unordered_map<int32_t, std::string>> gEnums;
 
@@ -146,19 +146,14 @@ private:
     std::variant<std::vector<std::string>, std::string> gWriteOrder;
     std::unordered_map<std::string, std::shared_ptr<BaseFactory>> gFactories;
     std::unordered_map<std::string, std::string> gModdedAssetPaths;
-    std::unordered_map<std::string, std::unordered_map<std::string, std::vector<uint8_t>>> gCacheData;
     std::unordered_map<std::string, std::map<std::string, std::vector<WriteEntry>>> gWriteMap;
     std::unordered_map<std::string, std::map<std::string, std::pair<YAML::Node, bool>>> gAssetDependencies;
     std::unordered_map<std::string, std::unordered_map<uint32_t, std::tuple<std::string, YAML::Node>>> gAddrMap;
 
     void ParseEnums(std::string& file);
-    void ParseCache();
+    void ParseHash();
     void ParseModdingConfig();
     void ParseCurrentFileConfig(YAML::Node node);
     void RegisterFactory(const std::string& type, const std::shared_ptr<BaseFactory>& factory);
-
-    void StoreCache(std::string path, const std::vector<uint8_t>& value);
-    std::vector<uint8_t>& GetCache(std::string path);
-
     void ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary);
 };
diff --git a/src/factories/BaseFactory.h b/src/factories/BaseFactory.h
index 5ea4a8d..048c91e 100644
--- a/src/factories/BaseFactory.h
+++ b/src/factories/BaseFactory.h
@@ -37,7 +37,7 @@ enum class ExportType {
     Header,
     Code,
     Binary,
-    Modding,
+    Modding
 };
 
 template<typename T>
@@ -71,16 +71,7 @@ T GetSafeNode(YAML::Node& node, const std::string& key, const T& def) {
     return node[key].as<T>();
 }
 
-class IParsedData {
-public:
-    virtual std::optional<std::vector<uint8_t>> ToCacheBuffer() {
-        return std::nullopt;
-    }
-
-    virtual void FromCacheBuffer(std::vector<uint8_t>& buffer) {
-
-    }
-};
+class IParsedData{};
 
 template<typename T>
 class GenericData : public IParsedData {
@@ -113,9 +104,6 @@ public:
     virtual uint32_t GetAlignment() {
         return 4;
     }
-    virtual bool IsCacheable() {
-        return false;
-    }
     virtual std::optional<std::shared_ptr<IParsedData>> CreateDataPointer() {
         return std::nullopt;
     }
diff --git a/src/factories/DisplayListFactory.h b/src/factories/DisplayListFactory.h
index fb69e01..27efae4 100644
--- a/src/factories/DisplayListFactory.h
+++ b/src/factories/DisplayListFactory.h
@@ -10,16 +10,6 @@ public:
 
     DListData() {}
     DListData(std::vector<uint32_t> gfxs) : mGfxs(gfxs) {}
-
-    std::optional<std::vector<uint8_t>> ToCacheBuffer() override {
-        std::vector<uint8_t> buffer;
-        buffer.insert(buffer.end(), reinterpret_cast<uint8_t *>(mGfxs.data()), reinterpret_cast<uint8_t *>(mGfxs.data()) + mGfxs.size() * sizeof(uint32_t));
-        return buffer;
-    }
-
-    void FromCacheBuffer(std::vector<uint8_t>& buffer) override {
-        mGfxs = std::vector(reinterpret_cast<uint32_t *>(buffer.data()), reinterpret_cast<uint32_t *>(buffer.data()) + buffer.size() / sizeof(uint32_t));
-    }
 };
 
 class DListHeaderExporter : public BaseExporter {
@@ -47,10 +37,4 @@ public:
     uint32_t GetAlignment() override {
         return 8;
     }
-    bool IsCacheable() override {
-        return true;
-    }
-    virtual std::optional<std::shared_ptr<IParsedData>> CreateDataPointer() {
-        return std::make_shared<DListData>();
-    }
 };