From 4b08b2b4c58534e6b434b6b6253c54505a0af1c6 Mon Sep 17 00:00:00 2001 From: KiritoDv Date: Mon, 25 Mar 2024 19:00:25 -0600 Subject: Fixed parse from cache --- src/Companion.cpp | 77 +++++++++++++++++++++++++------------- src/Companion.h | 7 +++- src/factories/BaseFactory.h | 3 ++ src/factories/DisplayListFactory.h | 21 ++++++++++- 4 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/Companion.cpp b/src/Companion.cpp index 3228eb4..34e2a62 100644 --- a/src/Companion.cpp +++ b/src/Companion.cpp @@ -198,15 +198,15 @@ void Companion::ExtractNode(YAML::Node& node, std::string& name, SWrapper* binar result = impl->parse_modding(data, node); } else if(this->gNodeIsCacheable && impl->IsCacheable()) { - if(this->gCacheData.contains(name)) { - result = std::make_shared(); - result->get()->FromCacheBuffer(this->gCacheData[name]); + if(this->gCacheData.contains(this->gCurrentCacheHash)) { + result = impl->CreateDataPointer(); + result->get()->FromCacheBuffer(this->GetCache(name)); } else { result = impl->parse(this->gRomData, node); if(result.has_value()) { auto cache = result.value()->ToCacheBuffer(); if(cache.has_value()) { - this->StoreCache(cache.value()); + this->StoreCache(name, cache.value()); } } } @@ -406,45 +406,68 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) { this->gNodeIsCacheable = GetSafeNode(node, "cacheable", true); } -void Companion::PrepareCache(const std::string& path) { - std::ifstream yaml(path); - const std::vector data = std::vector(std::istreambuf_iterator( yaml ), {}); - this->gCurrentCacheHash = CalculateHash(data); - +void Companion::ParseCache() { const std::string out = "torch.cache.yml"; - YAML::Node root; if(fs::exists(out)) { - root = YAML::LoadFile(out); + this->gCacheNode = YAML::LoadFile(out); } else { - root = YAML::Node(); + this->gCacheNode = YAML::Node(); } +} + +void Companion::PrepareCache(const std::string& path) { + std::ifstream yaml(path); + bool hasChanges = false; + const std::vector data = std::vector(std::istreambuf_iterator( yaml ), {}); + this->gCurrentCacheHash = CalculateHash(data); - if(root[this->gCurrentCacheHash]) { - const auto cache = GetSafeNode(root, this->gCurrentCacheHash); + if(this->gCacheNode[path]) { + const auto cache = GetSafeNode(this->gCacheNode, path); if(cache == this->gCurrentCacheHash) { - std::ifstream file("cache/" + cache, std::ios::binary); - this->gCacheData[cache] = std::vector(std::istreambuf_iterator( file ), {}); - file.close(); + const auto tdir = "tcache/" + cache; + if(!fs::exists(tdir)) { + fs::create_directories(tdir); + } else { + for(auto& entry : fs::directory_iterator("tcache/" + cache)) { + std::ifstream file(entry.path(), std::ios::binary); + std::vector buffer = std::vector(std::istreambuf_iterator(file), {}); + file.close(); + std::string cpath = entry.path().filename().string(); + std::replace(cpath.begin(), cpath.end(), '+', '/'); + this->gCacheData[cache][cpath] = buffer; + } + } } else { - root[path] = this->gCurrentCacheHash; + hasChanges = true; + this->gCacheNode[path] = this->gCurrentCacheHash; fs::remove("cache/" + cache); } } else { - root[path] = this->gCurrentCacheHash; + this->gCacheNode[path] = this->gCurrentCacheHash; } - std::ofstream file(out, std::ios::binary); - file << root; - file.close(); + if(hasChanges) { + std::ofstream file("torch.cache.yml", std::ios::binary); + file << this->gCacheNode; + file.close(); + } } -void Companion::StoreCache(const std::vector& value) { - std::ofstream file("cache/" + this->gCurrentCacheHash, std::ios::binary); +void Companion::StoreCache(const std::string path, const std::vector& value) { + std::string outpath = path; + std::replace(outpath.begin(), outpath.end(), '/', '+'); + std::ofstream file("tcache/" + this->gCurrentCacheHash + "/" + outpath, std::ios::binary); file.write(reinterpret_cast(value.data()), value.size()); file.close(); } +std::vector& Companion::GetCache(const std::string path) { + std::string outpath = path; + std::replace(outpath.begin(), outpath.end(), '+', '/'); + return this->gCacheData[this->gCurrentCacheHash][outpath]; +} + void Companion::Process() { if(!fs::exists("config.yml")) { @@ -585,6 +608,8 @@ void Companion::Process() { } } + this->ParseCache(); + SPDLOG_INFO("------------------------------------------------"); spdlog::set_pattern(line); @@ -843,8 +868,8 @@ void Companion::Process() { int32_t gap = end - startptr; if(gap < 0) { - stream << "\n// WARNING: Overlap detected between 0x" << std::hex << startptr << " and 0x" << end << "\n"; - SPDLOG_WARN("Overlap detected between 0x{:X} and 0x{:X}", startptr, end); + stream << "\n// WARNING: Overlap detected between 0x" << std::hex << startptr << " and 0x" << end << " with size 0x" << gap << "\n"; + SPDLOG_ERROR("Overlap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end, gap, this->gCurrentFile); } else if(gap < 0x10 && gap >= alignment && end % 0x10 == 0 && this->gEnablePadGen) { SPDLOG_WARN("Gap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end, gap, this->gCurrentFile); SPDLOG_WARN("Creating pad of 0x{:X} bytes", gap); diff --git a/src/Companion.h b/src/Companion.h index 8fcde4a..07d9f20 100644 --- a/src/Companion.h +++ b/src/Companion.h @@ -129,6 +129,7 @@ private: std::vector gRomData; std::filesystem::path gRomPath; bool gNodeIsCacheable; + YAML::Node gCacheNode; std::shared_ptr gCartridge; std::unordered_map> gEnums; @@ -145,17 +146,19 @@ private: std::variant, std::string> gWriteOrder; std::unordered_map> gFactories; std::unordered_map gModdedAssetPaths; - std::unordered_map> gCacheData; + std::unordered_map>> gCacheData; std::unordered_map>> gWriteMap; std::unordered_map>> gAssetDependencies; std::unordered_map>> gAddrMap; void ParseEnums(std::string& file); + void ParseCache(); void ParseModdingConfig(); void ParseCurrentFileConfig(YAML::Node node); void RegisterFactory(const std::string& type, const std::shared_ptr& factory); - void StoreCache(const std::vector& value); + void StoreCache(std::string path, const std::vector& value); + std::vector& GetCache(std::string path); void ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary); }; diff --git a/src/factories/BaseFactory.h b/src/factories/BaseFactory.h index 5dc8b81..5ea4a8d 100644 --- a/src/factories/BaseFactory.h +++ b/src/factories/BaseFactory.h @@ -116,6 +116,9 @@ public: virtual bool IsCacheable() { return false; } + virtual std::optional> CreateDataPointer() { + return std::nullopt; + } private: virtual std::unordered_map> GetExporters() = 0; }; \ No newline at end of file diff --git a/src/factories/DisplayListFactory.h b/src/factories/DisplayListFactory.h index 71a7e42..fb69e01 100644 --- a/src/factories/DisplayListFactory.h +++ b/src/factories/DisplayListFactory.h @@ -8,7 +8,18 @@ class DListData : public IParsedData { public: std::vector mGfxs; - explicit DListData(std::vector gfxs) : mGfxs(gfxs) {} + DListData() {} + DListData(std::vector gfxs) : mGfxs(gfxs) {} + + std::optional> ToCacheBuffer() override { + std::vector buffer; + buffer.insert(buffer.end(), reinterpret_cast(mGfxs.data()), reinterpret_cast(mGfxs.data()) + mGfxs.size() * sizeof(uint32_t)); + return buffer; + } + + void FromCacheBuffer(std::vector& buffer) override { + mGfxs = std::vector(reinterpret_cast(buffer.data()), reinterpret_cast(buffer.data()) + buffer.size() / sizeof(uint32_t)); + } }; class DListHeaderExporter : public BaseExporter { @@ -35,5 +46,11 @@ public: } uint32_t GetAlignment() override { return 8; - }; + } + bool IsCacheable() override { + return true; + } + virtual std::optional> CreateDataPointer() { + return std::make_shared(); + } }; -- cgit v1.2.3