Split extraction into parse/export phases with progress reporting

2 files changed, 423 insertions, 325 deletions

diff --git a/src/Companion.cpp b/src/Companion.cpp
index 93aefbc..e449f8b 100644
--- a/src/Companion.cpp
+++ b/src/Companion.cpp
@@ -139,16 +139,14 @@ static std::string GetTypeNode(YAML::Node& node) {
     return type;
 }
 
-Companion* Companion::Instance;
-
 void Companion::Init(const ExportType type) {
     std::atomic<size_t> assetCount{ 0 };
 
-    Init(type, assetCount);
+    Init(type, assetCount, true);
 }
 
-void Companion::Init(const ExportType type, std::atomic<size_t>& assetCount) {
-
+void Companion::Init(const ExportType type, std::atomic<size_t>& assetCount, bool shouldProcess) {
+    this->mShouldProcess = shouldProcess;
     spdlog::set_level(spdlog::level::debug);
     spdlog::set_pattern("[%Y-%m-%d %H:%M:%S.%e] [%l] %v");
 
@@ -347,16 +345,21 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
         }
     }
 
-    if (executeDef && this->gConfig.parseMode == ParseMode::Default) {
-        result = impl->parse(this->gRomData, node);
-    }
+    try {
+        if (executeDef && this->gConfig.parseMode == ParseMode::Default) {
+            result = impl->parse(this->gRomData, node);
+        }
 
-    if (executeDef && this->gConfig.parseMode == ParseMode::Directory) {
-        auto path = GetSafeNode<std::string>(node, "path");
-        std::ifstream input(path, std::ios::binary);
-        auto data = std::vector<uint8_t>(std::istreambuf_iterator(input), {});
-        result = impl->parse(data, node);
-        input.close();
+        if (executeDef && this->gConfig.parseMode == ParseMode::Directory) {
+            auto path = GetSafeNode<std::string>(node, "path");
+            std::ifstream input(path, std::ios::binary);
+            auto data = std::vector<uint8_t>(std::istreambuf_iterator(input), {});
+            result = impl->parse(data, node);
+            input.close();
+        }
+    } catch (const std::exception& e) {
+        SPDLOG_ERROR("Exception parsing {}: {}", name, e.what());
+        throw std::runtime_error("Error parsing " + name + ": " + e.what());
     }
 
     if (!result.has_value()) {
@@ -414,6 +417,7 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
                     auto currentFile = this->gCurrentFile;
                     auto currentDirectory = this->gCurrentDirectory;
                     auto currentExternalFiles = this->gCurrentExternalFiles;
+                    auto currentVirtualPath = this->gCurrentVirtualPath;
 
                     this->gCurrentFile = externalFileName;
                     this->gCurrentDirectory =
@@ -423,7 +427,7 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
 
                     if (!Torch::contains(this->gProcessedFiles, this->gCurrentFile)) {
                         ProcessFile(root, assetCount);
-                        if (process) {
+                        if (mShouldProcess) {
                             this->gProcessedFiles.insert(this->gCurrentFile);
                         }
                     }
@@ -433,6 +437,7 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
                     this->gCurrentFile = currentFile;
                     this->gCurrentDirectory = currentDirectory;
                     this->gCurrentExternalFiles = currentExternalFiles;
+                    this->gCurrentVirtualPath = currentVirtualPath;
                     this->gFileHeader.clear();
                 } else {
                     SPDLOG_INFO("Skipping external file {} as it has already been processed", externalFileName);
@@ -664,89 +669,7 @@ void Companion::ProcessFile(YAML::Node root) {
     ProcessFile(root, assetCount);
 }
 
-void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
-    assetCount++;
-    // Reset per-file state so segment/offset settings from a previous file don't
-    // bleed into this file's Phase 1 gAddrMap registration.
-    gCurrentSegmentNumber = 0;
-    gCurrentFileOffset = 0;
-    gCurrentCompressionType = CompressionType::None;
-    // Set compressed file offsets and compression type
-    if (auto segments = root[":config"]["segments"]) {
-        if (segments.IsSequence() && segments.size() > 0) {
-            if (segments[0].IsSequence() && segments[0].size() == 2) {
-                gCurrentSegmentNumber = segments[0][0].as<uint32_t>();
-                gCurrentFileOffset = segments[0][1].as<uint32_t>();
-                gCurrentCompressionType = Decompressor::GetCompressionType(this->gRomData, gCurrentFileOffset);
-                if (root[":config"]["no_compression"]) {
-                    gCurrentCompressionType = CompressionType::None;
-                }
-            } else {
-                throw std::runtime_error(
-                    "Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n  segments:\n  - [<segment>, "
-                    "<file_offset>]\n\nLike so:\nsegments:\n  - [0x06, 0x821D10]");
-            }
-        }
-    }
-
-    for (auto asset = root.begin(); asset != root.end(); ++asset) {
-        auto node = asset->second;
-        auto entryName = asset->first.as<std::string>();
-        auto output = (this->gCurrentDirectory / entryName).string();
-        std::replace(output.begin(), output.end(), '\\', '/');
-
-        if (node["type"]) {
-            const auto type = GetTypeNode(node);
-            if (type == "NAUDIO:V0:SAMPLE") {
-                AudioManager::Instance->bind_sample(node, output);
-            }
-        }
-
-        if (!node["offset"]) {
-            continue;
-        }
-
-        if (gCurrentSegmentNumber) {
-            if (IS_SEGMENTED(node["offset"].as<uint32_t>()) == false) {
-                node["offset"] = (gCurrentSegmentNumber << 24) | node["offset"].as<uint32_t>();
-            }
-        }
-
-        if (!gCurrentVirtualPath.empty()) {
-            node["path"] = gCurrentVirtualPath;
-        }
-
-        auto off = node["offset"].as<uint32_t>();
-        this->gAddrMap[this->gCurrentFile][off] = std::make_tuple(output, node);
-    }
-
-    // Stupid hack because the iteration broke the assets
-    root = YAML::LoadFile(this->gCurrentFile);
-    this->gConfig.segment.local.clear();
-    this->gFileHeader.clear();
-    this->gCurrentPad = 0;
-    this->gCurrentVram = std::nullopt;
-    this->gCurrentVirtualPath = "";
-    this->gCurrentSegmentNumber = 0;
-    this->gCurrentCompressionType = CompressionType::None;
-    this->gCurrentFileOffset = 0;
-    this->gTables.clear();
-    this->gCurrentExternalFiles.clear();
-    this->gManualSegments.clear();
-    GFXDOverride::ClearVtx();
-
-    if (root[":config"]) {
-        this->ParseCurrentFileConfig(root[":config"], assetCount);
-    }
-
-    if (!process || (!this->NodeHasChanges(this->gCurrentFile) && !this->gNodeForceProcessing)) {
-        return;
-    }
-
-    spdlog::set_pattern(regular);
-    SPDLOG_INFO("------------------------------------------------");
-    spdlog::set_pattern(line);
-
+void Companion::ProcessParseFile(YAML::Node root, std::atomic<size_t>& assetCount) {
     for (auto asset = root.begin(); asset != root.end(); ++asset) {
 
         auto entryName = asset->first.as<std::string>();
@@ -756,6 +679,8 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
             continue;
         }
 
+        this->gCurrentAssetName = "Parsing: " + entryName;
+
         if (gCurrentFileOffset && assetNode["offset"]) {
             const auto offset = assetNode["offset"].as<uint32_t>();
             if (!IS_SEGMENTED(offset)) {
@@ -779,12 +704,19 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
         SPDLOG_INFO("------------------------------------------------");
         spdlog::set_pattern(line);
     }
+}
 
+void Companion::ProcessExportFile() {
     for (auto& result : this->gParseResults[this->gCurrentFile]) {
         std::ostringstream stream;
         ExportResult endptr = std::nullopt;
         WriteEntry wEntry;
 
+        this->gCurrentAssetName = "Exporting: " + result.name;
+        if (this->gAssetCounter) {
+            (*this->gAssetCounter)++;
+        }
+
         auto data = result.data.value();
         const auto impl = this->GetFactory(result.type)->get();
         const auto exporter = impl->GetExporter(this->gConfig.exporterType);
@@ -793,62 +725,68 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
             continue;
         }
 
-        switch (this->gConfig.exporterType) {
-            case ExportType::Binary: {
-                stream.str("");
-                stream.clear();
-                exporter->get()->Export(stream, data, result.name, result.node, &result.name);
-                auto data = stream.str();
-                this->gCurrentWrapper->AddFile(result.name, std::vector(data.begin(), data.end()));
-
-                for (auto& entry : this->gCompanionFiles) {
-                    auto output = (this->gCurrentDirectory / entry.first).string();
-                    std::replace(output.begin(), output.end(), '\\', '/');
-                    this->gCurrentWrapper->AddFile(output, entry.second);
-                }
+        if (mShouldProcess) {
+            try {
+                switch (this->gConfig.exporterType) {
+                    case ExportType::Binary: {
+                        stream.str("");
+                        stream.clear();
+                        exporter->get()->Export(stream, data, result.name, result.node, &result.name);
+                        auto data = stream.str();
+                        this->gCurrentWrapper->AddFile(result.name, std::vector(data.begin(), data.end()));
+
+                        for (auto& entry : this->gCompanionFiles) {
+                            auto output = (this->gCurrentDirectory / entry.first).string();
+                            std::replace(output.begin(), output.end(), '\\', '/');
+                            this->gCurrentWrapper->AddFile(output, entry.second);
+                        }
 
-                break;
-            }
-            case ExportType::XML:
-            case ExportType::Modding: {
-                stream.str("");
-                stream.clear();
-                std::string ogname = result.name;
-                exporter->get()->Export(stream, data, result.name, result.node, &result.name);
-
-                auto data = stream.str();
-                if (data.empty()) {
-                    break;
-                }
+                        break;
+                    }
+                    case ExportType::XML:
+                    case ExportType::Modding: {
+                        stream.str("");
+                        stream.clear();
+                        std::string ogname = result.name;
+                        exporter->get()->Export(stream, data, result.name, result.node, &result.name);
+
+                        auto data = stream.str();
+                        if (data.empty()) {
+                            break;
+                        }
 
-                std::string dpath = Instance->GetOutputPath() + "/" + result.name;
-                if (!exists(fs::path(dpath).parent_path())) {
-                    create_directories(fs::path(dpath).parent_path());
-                }
+                        std::string dpath = Instance->GetOutputPath() + "/" + result.name;
+                        if (!exists(fs::path(dpath).parent_path())) {
+                            create_directories(fs::path(dpath).parent_path());
+                        }
 
-                this->gModdedAssetPaths[ogname] = result.name;
+                        this->gModdedAssetPaths[ogname] = result.name;
 
-                std::ofstream file(dpath, std::ios::binary);
-                file.write(data.c_str(), data.size());
-                file.close();
+                        std::ofstream file(dpath, std::ios::binary);
+                        file.write(data.c_str(), data.size());
+                        file.close();
 
-                for (auto& entry : this->gCompanionFiles) {
-                    auto cpath = (Instance->GetOutputPath() / this->gCurrentDirectory / entry.first).string();
-                    std::replace(cpath.begin(), cpath.end(), '\\', '/');
-                    if (!exists(fs::path(cpath).parent_path())) {
-                        create_directories(fs::path(cpath).parent_path());
-                    }
+                        for (auto& entry : this->gCompanionFiles) {
+                            auto cpath = (Instance->GetOutputPath() / this->gCurrentDirectory / entry.first).string();
+                            std::replace(cpath.begin(), cpath.end(), '\\', '/');
+                            if (!exists(fs::path(cpath).parent_path())) {
+                                create_directories(fs::path(cpath).parent_path());
+                            }
 
-                    std::ofstream cfile(cpath, std::ios::binary);
-                    cfile.write(entry.second.data(), entry.second.size());
-                    cfile.close();
-                }
+                            std::ofstream cfile(cpath, std::ios::binary);
+                            cfile.write(entry.second.data(), entry.second.size());
+                            cfile.close();
+                        }
 
-                break;
-            }
-            default: {
-                endptr = exporter->get()->Export(stream, data, result.name, result.node, &result.name);
-                break;
+                        break;
+                    }
+                    default: {
+                        endptr = exporter->get()->Export(stream, data, result.name, result.node, &result.name);
+                        break;
+                    }
+                }
+            } catch (const std::exception& e) {
+                throw std::runtime_error("Error exporting " + result.name + ": " + e.what());
             }
         }
 
@@ -890,188 +828,299 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
 
     auto fsout = fs::path(this->gConfig.outputPath);
 
-    if (this->gConfig.exporterType == ExportType::Modding || this->gConfig.exporterType == ExportType::XML) {
-        fsout /= "modding.yml";
-        YAML::Node modding;
+    if (mShouldProcess) {
+        if (this->gConfig.exporterType == ExportType::Modding || this->gConfig.exporterType == ExportType::XML) {
+            fsout /= "modding.yml";
+            YAML::Node modding;
 
-        for (const auto& [key, value] : this->gModdedAssetPaths) {
-            modding["assets"][key] = value;
-        }
+            for (const auto& [key, value] : this->gModdedAssetPaths) {
+                modding["assets"][key] = value;
+            }
 
-        std::ofstream file(fsout.string(), std::ios::binary);
-        file << modding;
-        file.close();
-    } else if (this->gConfig.exporterType != ExportType::Binary) {
-        std::string filename = this->gCurrentDirectory.filename().string();
+            std::ofstream file(fsout.string(), std::ios::binary);
+            file << modding;
+            file.close();
+        } else if (this->gConfig.exporterType != ExportType::Binary) {
+            std::string filename = this->gCurrentDirectory.filename().string();
 
-        switch (this->gConfig.exporterType) {
-            case ExportType::Header: {
-                fsout /= this->gCurrentDirectory.parent_path() / (filename + ".h");
-                break;
-            }
-            case ExportType::Code: {
-                fsout /= this->gCurrentDirectory / (filename + ".c");
-                break;
+            switch (this->gConfig.exporterType) {
+                case ExportType::Header: {
+                    fsout /= this->gCurrentDirectory.parent_path() / (filename + ".h");
+                    break;
+                }
+                case ExportType::Code: {
+                    fsout /= this->gCurrentDirectory / (filename + ".c");
+                    break;
+                }
+                default:
+                    break;
             }
-            default:
-                break;
-        }
 
-        std::ostringstream stream;
+            std::ostringstream stream;
 
-        std::vector<WriteEntry> entries;
+            std::vector<WriteEntry> entries;
 
-        if (std::holds_alternative<std::string>(this->gWriteOrder)) {
-            auto sort = std::get<std::string>(this->gWriteOrder);
-            for (const auto& [type, raw] : this->gWriteMap[this->gCurrentFile]) {
-                entries.insert(entries.end(), raw.begin(), raw.end());
-            }
+            if (std::holds_alternative<std::string>(this->gWriteOrder)) {
+                auto sort = std::get<std::string>(this->gWriteOrder);
+                for (const auto& [type, raw] : this->gWriteMap[this->gCurrentFile]) {
+                    entries.insert(entries.end(), raw.begin(), raw.end());
+                }
 
-            if (sort == "OFFSET") {
-                std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return a.addr < b.addr; });
-            } else if (sort == "ROFFSET") {
-                std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return a.addr > b.addr; });
-            } else if (sort != "LINEAR") {
-                throw std::runtime_error("Invalid write order");
-            }
-        } else {
-            for (const auto& type : std::get<std::vector<std::string>>(this->gWriteOrder)) {
-                entries = this->gWriteMap[this->gCurrentFile][type];
+                if (sort == "OFFSET") {
+                    std::sort(entries.begin(), entries.end(),
+                              [](const auto& a, const auto& b) { return a.addr < b.addr; });
+                } else if (sort == "ROFFSET") {
+                    std::sort(entries.begin(), entries.end(),
+                              [](const auto& a, const auto& b) { return a.addr > b.addr; });
+                } else if (sort != "LINEAR") {
+                    throw std::runtime_error("Invalid write order");
+                }
+            } else {
+                for (const auto& type : std::get<std::vector<std::string>>(this->gWriteOrder)) {
+                    entries = this->gWriteMap[this->gCurrentFile][type];
 
-                std::sort(entries.begin(), entries.end(), [](const auto& a, const auto& b) { return a.addr > b.addr; });
+                    std::sort(entries.begin(), entries.end(),
+                              [](const auto& a, const auto& b) { return a.addr > b.addr; });
+                }
             }
-        }
 
-        for (size_t i = 0; i < entries.size(); i++) {
-            const auto result = entries[i];
-            const auto hasSize = result.endptr.has_value();
+            for (size_t i = 0; i < entries.size(); i++) {
+                const auto result = entries[i];
+                const auto hasSize = result.endptr.has_value();
 
-            if (result.comment.has_value()) {
-                stream << "// " << result.comment.value() << "\n";
-            }
+                if (result.comment.has_value()) {
+                    stream << "// " << result.comment.value() << "\n";
+                }
 
-            if (hasSize && this->IsDebug()) {
-                stream << "// 0x" << std::hex << std::uppercase << ASSET_PTR(result.addr) << "\n";
-            }
+                if (hasSize && this->IsDebug()) {
+                    stream << "// 0x" << std::hex << std::uppercase << ASSET_PTR(result.addr) << "\n";
+                }
 
-            stream << result.buffer;
+                stream << result.buffer;
 
-            if (hasSize && this->IsDebug()) {
-                stream << "// 0x" << std::hex << std::uppercase << ASSET_PTR(result.endptr.value()) << "\n\n";
-            }
+                if (hasSize && this->IsDebug()) {
+                    stream << "// 0x" << std::hex << std::uppercase << ASSET_PTR(result.endptr.value()) << "\n\n";
+                }
 
-            if (hasSize && i < entries.size() - 1 && this->gConfig.exporterType == ExportType::Code &&
-                !this->gIndividualIncludes) {
-                int32_t startptr = ASSET_PTR(result.endptr.value());
-                int32_t end = ASSET_PTR(entries[i + 1].addr);
-
-                uint32_t alignment = entries[i + 1].alignment;
-                int32_t gap = end - startptr;
-
-                if (gap < 0) {
-                    stream << "// WARNING: Overlap detected between 0x" << std::hex << startptr << " and 0x" << end
-                           << " with size 0x" << std::abs(gap) << "\n";
-                    SPDLOG_WARN("Overlap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end,
-                                gap, this->gCurrentFile);
-                } else if (gap < 0x10 && gap >= alignment && end % alignment == 0 && this->gEnablePadGen) {
-                    SPDLOG_WARN("Gap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end,
-                                gap, this->gCurrentFile);
-                    SPDLOG_WARN("Creating pad of 0x{:X} bytes", gap);
-                    const auto padfile = this->gCurrentDirectory.filename().string();
-                    if (this->IsDebug()) {
-                        stream << "// 0x" << std::hex << std::uppercase << startptr << "\n";
+                if (hasSize && i < entries.size() - 1 && this->gConfig.exporterType == ExportType::Code &&
+                    !this->gIndividualIncludes) {
+                    int32_t startptr = ASSET_PTR(result.endptr.value());
+                    int32_t end = ASSET_PTR(entries[i + 1].addr);
+
+                    uint32_t alignment = entries[i + 1].alignment;
+                    int32_t gap = end - startptr;
+
+                    if (gap < 0) {
+                        stream << "// WARNING: Overlap detected between 0x" << std::hex << startptr << " and 0x" << end
+                               << " with size 0x" << std::abs(gap) << "\n";
+                        SPDLOG_WARN("Overlap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr,
+                                    end, gap, this->gCurrentFile);
+                    } else if (gap < 0x10 && gap >= alignment && end % alignment == 0 && this->gEnablePadGen) {
+                        SPDLOG_WARN("Gap detected between 0x{:X} and 0x{:X} with size 0x{:X} on file {}", startptr, end,
+                                    gap, this->gCurrentFile);
+                        SPDLOG_WARN("Creating pad of 0x{:X} bytes", gap);
+                        const auto padfile = this->gCurrentDirectory.filename().string();
+                        if (this->IsDebug()) {
+                            stream << "// 0x" << std::hex << std::uppercase << startptr << "\n";
+                        }
+                        stream << "char pad_" << padfile << "_" << std::to_string(gCurrentPad++) << "[] = {\n" << tab_t;
+                        auto gapSize = gap & ~3;
+                        for (size_t j = 0; j < gapSize; j++) {
+                            stream << "0x00, ";
+                        }
+                        stream << "\n};\n";
+                        if (this->IsDebug()) {
+                            stream << "// 0x" << std::hex << std::uppercase << end << "\n\n";
+                        } else {
+                            stream << "\n";
+                        }
+                    } else if (gap >= 0x10) {
+                        stream << "// WARNING: Gap detected between 0x" << std::hex << startptr << " and 0x" << end
+                               << " with size 0x" << gap << "\n";
                     }
-                    stream << "char pad_" << padfile << "_" << std::to_string(gCurrentPad++) << "[] = {\n" << tab_t;
-                    auto gapSize = gap & ~3;
-                    for (size_t j = 0; j < gapSize; j++) {
-                        stream << "0x00, ";
+                }
+
+                if (this->gConfig.exporterType == ExportType::Code && this->gIndividualIncludes) {
+                    fs::path outinc =
+                        fs::path(this->gConfig.outputPath) / this->gCurrentDirectory.parent_path() /
+                        fs::relative(fs::path(result.name + ".inc.c"), this->gCurrentDirectory.parent_path());
+
+                    if (!exists(outinc.parent_path())) {
+                        create_directories(outinc.parent_path());
                     }
-                    stream << "\n};\n";
-                    if (this->IsDebug()) {
-                        stream << "// 0x" << std::hex << std::uppercase << end << "\n\n";
-                    } else {
-                        stream << "\n";
+
+                    std::ofstream file(outinc, std::ios::binary);
+
+                    if (!this->gFileHeader.empty()) {
+                        file << this->gFileHeader << std::endl;
                     }
-                } else if (gap >= 0x10) {
-                    stream << "// WARNING: Gap detected between 0x" << std::hex << startptr << " and 0x" << end
-                           << " with size 0x" << gap << "\n";
+                    file << stream.str();
+                    stream.str("");
+                    stream.seekp(0);
+                    file.close();
                 }
             }
 
-            if (this->gConfig.exporterType == ExportType::Code && this->gIndividualIncludes) {
-                fs::path outinc = fs::path(this->gConfig.outputPath) / this->gCurrentDirectory.parent_path() /
-                                  fs::relative(fs::path(result.name + ".inc.c"), this->gCurrentDirectory.parent_path());
+            this->gWriteMap.clear();
+
+            if (this->gConfig.exporterType != ExportType::Code || !this->gIndividualIncludes) {
+                std::string buffer = stream.str();
 
-                if (!exists(outinc.parent_path())) {
-                    create_directories(outinc.parent_path());
+                if (buffer.empty()) {
+                    SPDLOG_WARN("No data to write for {}", this->gCurrentFile);
+                    return;
                 }
 
-                std::ofstream file(outinc, std::ios::binary);
+                std::string output = fsout.string();
+                std::replace(output.begin(), output.end(), '\\', '/');
+                if (!exists(fs::path(output).parent_path())) {
+                    create_directories(fs::path(output).parent_path());
+                }
+
+                std::ofstream file(output, std::ios::binary);
+                SPDLOG_INFO("Writing {} to {}", this->gCurrentFile, output);
 
-                if (!this->gFileHeader.empty()) {
-                    file << this->gFileHeader << std::endl;
+                if (this->gConfig.exporterType == ExportType::Header) {
+                    fs::path entryPath = this->gCurrentFile;
+                    std::string symbol = entryPath.stem().string();
+                    std::transform(symbol.begin(), symbol.end(), symbol.begin(), toupper);
+                    if (this->IsOTRMode()) {
+                        file << "#pragma once\n\n";
+                    } else {
+                        file << "#ifndef " << symbol << "_H" << std::endl;
+                        file << "#define " << symbol << "_H" << std::endl << std::endl;
+                    }
+                    if (!this->gFileHeader.empty()) {
+                        file << this->gFileHeader << std::endl;
+                    }
+                    file << buffer;
+                    if (!this->IsOTRMode()) {
+                        file << std::endl << "#endif" << std::endl;
+                    }
+                } else {
+                    if (!this->gFileHeader.empty()) {
+                        file << this->gFileHeader << std::endl;
+                    }
+                    file << buffer;
                 }
-                file << stream.str();
-                stream.str("");
-                stream.seekp(0);
+
                 file.close();
             }
         }
+    }
 
-        this->gWriteMap.clear();
-
-        if (this->gConfig.exporterType != ExportType::Code || !this->gIndividualIncludes) {
-            std::string buffer = stream.str();
+    if (this->gConfig.exporterType != ExportType::Binary) {
+        this->gHashNode[RelativePathToSrcDir(this->gCurrentFile)]["extracted"]
+                       [ExportTypeToString(this->gConfig.exporterType)] = true;
+    }
+}
 
-            if (buffer.empty()) {
-                SPDLOG_WARN("No data to write for {}", this->gCurrentFile);
-                return;
+void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
+    // Reset per-file state so segment/offset settings from a previous file don't
+    // bleed into this file's Phase 1 gAddrMap registration.
+    gCurrentSegmentNumber = 0;
+    gCurrentFileOffset = 0;
+    gCurrentCompressionType = CompressionType::None;
+    // Set compressed file offsets and compression type
+    if (auto segments = root[":config"]["segments"]) {
+        if (segments.IsSequence() && segments.size() > 0) {
+            if (segments[0].IsSequence() && segments[0].size() == 2) {
+                gCurrentSegmentNumber = segments[0][0].as<uint32_t>();
+                gCurrentFileOffset = segments[0][1].as<uint32_t>();
+                gCurrentCompressionType = Decompressor::GetCompressionType(this->gRomData, gCurrentFileOffset);
+                if (root[":config"]["no_compression"]) {
+                    gCurrentCompressionType = CompressionType::None;
+                }
+            } else {
+                throw std::runtime_error(
+                    "Incorrect yaml syntax for segments.\n\nThe yaml expects:\n:config:\n  segments:\n  - [<segment>, "
+                    "<file_offset>]\n\nLike so:\nsegments:\n  - [0x06, 0x821D10]");
             }
+        }
+    }
 
-            std::string output = fsout.string();
-            std::replace(output.begin(), output.end(), '\\', '/');
-            if (!exists(fs::path(output).parent_path())) {
-                create_directories(fs::path(output).parent_path());
+    for (auto asset = root.begin(); asset != root.end(); ++asset) {
+        auto node = asset->second;
+        auto entryName = asset->first.as<std::string>();
+        auto output = (this->gCurrentDirectory / entryName).string();
+        std::replace(output.begin(), output.end(), '\\', '/');
+
+        if (node["type"]) {
+            const auto type = GetTypeNode(node);
+#ifdef NAUDIO_SUPPORT
+            if (type == "NAUDIO:V0:SAMPLE") {
+                AudioManager::Instance->bind_sample(node, output);
             }
+#endif
+        }
 
-            std::ofstream file(output, std::ios::binary);
-            SPDLOG_INFO("Writing {} to {}", this->gCurrentFile, output);
+        if (!node["offset"]) {
+            continue;
+        }
 
-            if (this->gConfig.exporterType == ExportType::Header) {
-                fs::path entryPath = this->gCurrentFile;
-                std::string symbol = entryPath.stem().string();
-                std::transform(symbol.begin(), symbol.end(), symbol.begin(), toupper);
-                if (this->IsOTRMode()) {
-                    file << "#pragma once\n\n";
-                } else {
-                    file << "#ifndef " << symbol << "_H" << std::endl;
-                    file << "#define " << symbol << "_H" << std::endl << std::endl;
-                }
-                if (!this->gFileHeader.empty()) {
-                    file << this->gFileHeader << std::endl;
-                }
-                file << buffer;
-                if (!this->IsOTRMode()) {
-                    file << std::endl << "#endif" << std::endl;
-                }
-            } else {
-                if (!this->gFileHeader.empty()) {
-                    file << this->gFileHeader << std::endl;
-                }
-                file << buffer;
+        if (gCurrentSegmentNumber) {
+            if (IS_SEGMENTED(node["offset"].as<uint32_t>()) == false) {
+                node["offset"] = (gCurrentSegmentNumber << 24) | node["offset"].as<uint32_t>();
             }
+        }
 
-            file.close();
+        if (!gCurrentVirtualPath.empty()) {
+            node["path"] = gCurrentVirtualPath;
         }
+
+        this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = std::make_tuple(output, node);
     }
 
-    if (this->gConfig.exporterType != ExportType::Binary) {
-        this->gHashNode[RelativePathToSrcDir(this->gCurrentFile)]["extracted"]
-                       [ExportTypeToString(this->gConfig.exporterType)] = true;
+    // Stupid hack because the iteration broke the assets
+    root = YAML::LoadFile(this->gCurrentFile);
+    this->gConfig.segment.local.clear();
+    this->gFileHeader.clear();
+    this->gCurrentPad = 0;
+    this->gCurrentVram = std::nullopt;
+    this->gCurrentVirtualPath = "";
+    this->gCurrentSegmentNumber = 0;
+    this->gCurrentCompressionType = CompressionType::None;
+    this->gCurrentFileOffset = 0;
+    this->gTables.clear();
+    this->gCurrentExternalFiles.clear();
+    this->gManualSegments.clear();
+    GFXDOverride::ClearVtx();
+
+    if (root[":config"]) {
+        this->ParseCurrentFileConfig(root[":config"], assetCount);
+    }
+
+    if (!this->NodeHasChanges(this->gCurrentFile) && !this->gNodeForceProcessing) {
+        return;
+    }
+
+    spdlog::set_pattern(regular);
+    SPDLOG_INFO("------------------------------------------------");
+    spdlog::set_pattern(line);
+
+    ProcessParseFile(root, assetCount);
+
+    // Transition to export phase: reset counter and set total from actual parse results
+    if (this->gPhaseCallback) {
+        this->gPhaseCallback(2);
     }
+    if (this->gAssetCounter) {
+        *this->gAssetCounter = 0;
+    }
+    if (this->gAssetTotal) {
+        size_t total = 0;
+        for (const auto& [file, results] : this->gParseResults) {
+            total += results.size();
+        }
+        *this->gAssetTotal = total;
+    }
+
+    ProcessExportFile();
 }
 
 void Companion::Process(std::atomic<size_t>& assetCount) {
+    this->gAssetCounter = &assetCount;
+    // gAssetTotal is set by the caller (port-side) if progress reporting is needed
     auto configPath = this->gSourceDirectory / "config.yml";
 
     if (!fs::exists(configPath)) {
@@ -1313,31 +1362,36 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
 
     SPDLOG_CRITICAL("------------------------------------------------");
 
+#ifdef NAUDIO_SUPPORT
     AudioManager::Instance = new AudioManager();
+#endif
     BinaryWrapper* wrapper = nullptr;
 
-    if (this->gConfig.exporterType == ExportType::Binary) {
-        switch (this->gConfig.otrMode) {
-            case ArchiveType::OTR:
-                wrapper = new SWrapper(this->gConfig.outputPath);
-                break;
-            case ArchiveType::O2R:
-                wrapper = new ZWrapper(this->gConfig.outputPath);
-                break;
-            default:
-                throw std::runtime_error("Invalid archive type for export type Binary");
+    if (mShouldProcess) {
+        if (this->gConfig.exporterType == ExportType::Binary) {
+            switch (this->gConfig.otrMode) {
+                case ArchiveType::OTR:
+                    wrapper = new SWrapper(this->gConfig.outputPath);
+                    break;
+                case ArchiveType::O2R:
+                    wrapper = new ZWrapper(this->gConfig.outputPath);
+                    break;
+                default:
+                    throw std::runtime_error("Invalid archive type for export type Binary");
+            }
         }
-    }
 
-    if (wrapper) {
-        wrapper->CreateArchive();
-    }
-    this->gCurrentWrapper = wrapper;
+        if (wrapper) {
+            wrapper->CreateArchive();
+        }
 
-    for (auto& entry : this->gCompanionFiles) {
-        auto output = entry.first;
-        std::replace(output.begin(), output.end(), '\\', '/');
-        this->gCurrentWrapper->AddFile(output, entry.second);
+        this->gCurrentWrapper = wrapper;
+
+        for (auto& entry : this->gCompanionFiles) {
+            auto output = entry.first;
+            std::replace(output.begin(), output.end(), '\\', '/');
+            this->gCurrentWrapper->AddFile(output, entry.second);
+        }
     }
 
     auto vWriter = LUS::BinaryWriter();
@@ -1365,19 +1419,25 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
             continue;
         }
 
+        if (yamlPath.find("hashes.yaml") != std::string::npos) {
+            continue;
+        }
+
         YAML::Node root = YAML::LoadFile(yamlPath);
         this->gCurrentDirectory = relative(entry.path(), this->gAssetPath).replace_extension("");
         this->gCurrentFile = yamlPath;
 
         if (!Torch::contains(this->gProcessedFiles, this->gCurrentFile)) {
             ProcessFile(root, assetCount);
-            if (process) {
+            if (mShouldProcess) {
                 this->gProcessedFiles.insert(this->gCurrentFile);
             }
         }
     }
 
-    if (wrapper != nullptr) {
+    this->gCurrentAssetName.clear();
+
+    if (wrapper != nullptr && mShouldProcess) {
         // Add additional files specified by the user
         for (const auto& filePath : this->gAdditionalFiles) {
             std::ifstream input(this->gSourceDirectory / filePath, std::ios::binary);
@@ -1400,6 +1460,8 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
         SPDLOG_CRITICAL("Writing version file");
         wrapper->AddFile("version", vWriter.ToVector());
         vWriter.Close();
+
+        this->gCurrentAssetName = "Writing archive to disk...";
         wrapper->Close();
     }
 
@@ -1418,7 +1480,6 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
 
     Decompressor::ClearCache();
     this->gCartridge = nullptr;
-    Instance = nullptr;
 }
 
 void Companion::Pack(const std::string& folder, const std::string& output, const ArchiveType otrMode,
@@ -1492,6 +1553,26 @@ std::optional<std::tuple<std::string, YAML::Node>> Companion::RegisterAsset(cons
 
     auto entry = std::make_tuple(output, node);
     this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = entry;
+    // Append to the GetNodesByType cache for this (file, type) if it's already
+    // built. Avoids a full rebuild on the hot DListFactory path where many new
+    // VTX nodes are registered while parsing a single display list.
+    {
+        const auto type = GetTypeNode(node);
+        auto vIt = this->gNodesByTypeCacheValid.find(this->gCurrentFile);
+        if (vIt != this->gNodesByTypeCacheValid.end()) {
+            auto bIt = vIt->second.find(type);
+            if (bIt != vIt->second.end() && bIt->second != 0) {
+                auto& cacheBuckets = this->gNodesByTypeCache[this->gCurrentFile][type];
+                // Index 1 always includes the new entry; index 0 includes it only if not autogen.
+                if (bIt->second & 0x2) {
+                    cacheBuckets[1].push_back(entry);
+                }
+                if ((bIt->second & 0x1) && !node["autogen"]) {
+                    cacheBuckets[0].push_back(entry);
+                }
+            }
+        }
+    }
     auto dResult = this->ParseNode(node, output);
     if (dResult.has_value()) {
         this->gParseResults[this->gCurrentFile].push_back(dResult.value());
@@ -1719,30 +1800,45 @@ std::optional<ParseResultData> Companion::GetParseDataBySymbol(const std::string
     return std::nullopt;
 }
 
-std::optional<std::vector<std::tuple<std::string, YAML::Node>>> Companion::GetNodesByType(const std::string& type) {
-    std::vector<std::tuple<std::string, YAML::Node>> nodes;
-
+const std::vector<std::tuple<std::string, YAML::Node>>* Companion::GetNodesByTypeRef(const std::string& type,
+                                                                                     bool includeAutogen) {
     if (!Torch::contains(this->gAddrMap, this->gCurrentFile)) {
-        return nodes;
-    }
-
-    for (auto& [addr, tpl] : this->gAddrMap[this->gCurrentFile]) {
-        auto [name, node] = tpl;
-        const auto n_type = GetTypeNode(node);
-        if (node["autogen"]) {
-            SPDLOG_DEBUG("Skipping autogenerated asset {}", name);
-            continue;
-        }
-        if (n_type == type) {
-            nodes.push_back(tpl);
+        return nullptr;
+    }
+
+    // Cache hit path so we don't rescan every node in gAddrMap on every call
+    const size_t agIdx = includeAutogen ? 1 : 0;
+    const uint8_t agBit = static_cast<uint8_t>(1u << agIdx);
+    auto& validBits = this->gNodesByTypeCacheValid[this->gCurrentFile][type];
+    if (!(validBits & agBit)) {
+        auto& cached = this->gNodesByTypeCache[this->gCurrentFile][type][agIdx];
+        cached.clear();
+        for (auto& [addr, tpl] : this->gAddrMap[this->gCurrentFile]) {
+            const auto& [name, node] = tpl;
+            const auto n_type = GetTypeNode(const_cast<YAML::Node&>(node));
+            if (!includeAutogen && node["autogen"]) {
+                continue;
+            }
+            if (n_type == type) {
+                cached.push_back(tpl);
+            }
         }
+        validBits |= agBit;
     }
+    return &this->gNodesByTypeCache[this->gCurrentFile][type][agIdx];
+}
 
-    return nodes;
+std::optional<std::vector<std::tuple<std::string, YAML::Node>>> Companion::GetNodesByType(const std::string& type,
+                                                                                          bool includeAutogen) {
+    auto* ref = GetNodesByTypeRef(type, includeAutogen);
+    if (ref == nullptr) {
+        return std::vector<std::tuple<std::string, YAML::Node>>{};
+    }
+    return *ref;
 }
 
 void Companion::SetProcess(bool shouldProcess) {
-    this->process = shouldProcess;
+    this->mShouldProcess = shouldProcess;
 }
 
 void Companion::RegisterCompanionFile(const std::string path, std::vector<char> data) {
@@ -1851,12 +1947,12 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
     asset["autogen"] = true;
     asset["symbol"] = output;
 
-    auto result = this->RegisterAsset(output, asset);
-
     if (!gCurrentVirtualPath.empty()) {
         asset["path"] = gCurrentVirtualPath;
     }
 
+    auto result = this->RegisterAsset(output, asset);
+
     if (result.has_value()) {
         asset["vpath"] = std::get<0>(result.value());
 
@@ -1864,4 +1960,4 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
     }
 
     return std::nullopt;
-}
\ No newline at end of file
+}
diff --git a/src/main.cpp b/src/main.cpp
index 55b49d6..030eb42 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -2,6 +2,8 @@
 #include "CLI11.hpp"
 #include "Companion.h"
 
+Companion* Companion::Instance;
+
 #if defined(STANDALONE) && !defined(__EMSCRIPTEN__)
 
 int main(int argc, char* argv[]) {