Implemented zip streaming (#271)

* Implemented zip streaming

* Handled Jeod comments

3 files changed, 92 insertions, 28 deletions

diff --git a/src/Companion.cpp b/src/Companion.cpp
index 2bf3ec1..1f52ac5 100644
--- a/src/Companion.cpp
+++ b/src/Companion.cpp
@@ -184,6 +184,14 @@ static std::string GetTypeNode(YAML::Node& node) {
     return type;
 }
 
+static fs::path RelativeTo(const fs::path& path, const fs::path& base) {
+#ifdef __WINRT__
+    return path.lexically_normal().lexically_relative(base.lexically_normal());
+#else
+    return fs::relative(path, base);
+#endif
+}
+
 void Companion::Init(const ExportType type) {
     std::atomic<size_t> assetCount{ 0 };
 
@@ -553,8 +561,8 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
 
                 std::string externalFileName =
                     (this->gSourceDirectory / externalFile.as<std::string>()).generic_string();
-                const auto relPath = std::filesystem::relative(externalFileName, this->gAssetPath).string();
-                const auto relCommonPath = std::filesystem::relative(externalFileName, this->gCommonAssetPath).string();
+                const auto relPath = RelativeTo(externalFileName, this->gAssetPath).string();
+                const auto relCommonPath = RelativeTo(externalFileName, this->gCommonAssetPath).string();
                 if (StringHelper::StartsWith(relPath, "../")) {
                     if (StringHelper::StartsWith(relCommonPath, "../"))
                         throw std::runtime_error("External File " + externalFileName + " Not In Asset Directory " +
@@ -575,7 +583,7 @@ void Companion::ParseCurrentFileConfig(YAML::Node node, std::atomic<size_t>& ass
 
                     this->gCurrentFile = externalFileName;
                     this->gCurrentDirectory =
-                        std::filesystem::relative(externalFileName, this->gAssetPath).replace_extension("");
+                        RelativeTo(externalFileName, this->gAssetPath).replace_extension("");
 
                     YAML::Node root = YAML::LoadFile(externalFileName);
 
@@ -1225,9 +1233,9 @@ void Companion::ProcessFile(YAML::Node root, std::atomic<size_t>& assetCount) {
     // directory must be resolved before the loop. Default to the file's own
     // path, then honor a :config directory override (used to register a room's
     // assets under its scene's directory).
-    auto relPath = relative(fs::path(this->gCurrentFile), this->gAssetPath).replace_extension("");
+    auto relPath = RelativeTo(this->gCurrentFile, this->gAssetPath).replace_extension("");
     if (StringHelper::StartsWith(relPath.string(), "../"))
-        relPath = relative(fs::path(this->gCurrentFile), this->gCommonAssetPath).replace_extension("");
+        relPath = RelativeTo(this->gCurrentFile, this->gCommonAssetPath).replace_extension("");
 
     this->gCurrentDirectory = relPath;
 
@@ -1518,9 +1526,10 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
             }
         }
         this->gConfig.outputPath = output_path.string();
-    }
-    if (auto outParent = output_path.parent_path(); !outParent.empty() && !exists(outParent)) {
-        create_directories(outParent);
+
+        if (auto outParent = output_path.parent_path(); !outParent.empty() && !exists(outParent)) {
+            create_directories(outParent);
+        }
     }
 
     if (gbi) {
@@ -1652,16 +1661,16 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
 #ifdef NAUDIO_SUPPORT
     AudioManager::Instance = new AudioManager();
 #endif
-    BinaryWrapper* wrapper = nullptr;
+    std::unique_ptr<BinaryWrapper> wrapper;
 
     if (mShouldProcess) {
         if (this->gConfig.exporterType == ExportType::Binary) {
             switch (this->gConfig.otrMode) {
                 case ArchiveType::OTR:
-                    wrapper = new SWrapper(this->gConfig.outputPath);
+                    wrapper.reset(new SWrapper(this->gConfig.outputPath));
                     break;
                 case ArchiveType::O2R:
-                    wrapper = new ZWrapper(this->gConfig.outputPath);
+                    wrapper.reset(new ZWrapper(this->gConfig.outputPath));
                     break;
                 default:
                     throw std::runtime_error("Invalid archive type for export type Binary");
@@ -1672,7 +1681,7 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
             wrapper->CreateArchive();
         }
 
-        this->gCurrentWrapper = wrapper;
+        this->gCurrentWrapper = wrapper.get();
 
         for (auto& entry : this->gCompanionFiles) {
             auto output = entry.first;
@@ -1910,6 +1919,7 @@ void Companion::Process(std::atomic<size_t>& assetCount) {
         this->gCurrentAssetName = "Writing archive to disk...";
         wrapper->Close();
     }
+    this->gCurrentWrapper = nullptr;
 
     // Write entries hash
     std::ofstream file(this->gDestinationDirectory / "torch.hash.yml", std::ios::binary);
diff --git a/src/archive/ZWrapper.cpp b/src/archive/ZWrapper.cpp
index 480dafc..335a3a9 100644
--- a/src/archive/ZWrapper.cpp
+++ b/src/archive/ZWrapper.cpp
@@ -2,6 +2,8 @@
 #include <filesystem>
 #include <iostream>
 #include <fstream>
+#include <cstring>
+#include <stdexcept>
 
 #include "spdlog/spdlog.h"
 #include <Companion.h>
@@ -11,17 +13,40 @@ namespace fs = std::filesystem;
 
 ZWrapper::ZWrapper(const std::string& path) {
     this->mPath = path;
-    this->mZip = new miniz_cpp::zip_file;
+    this->mTempPath = path + ".tmp";
 }
 
-int32_t ZWrapper::CreateArchive() {
-    SPDLOG_INFO("Loaded ZIP (O2R) archive: {}", mPath.c_str());
+ZWrapper::~ZWrapper() {
+    if (this->mZip != nullptr) {
+        if (this->mOpen) {
+            mz_zip_writer_end(reinterpret_cast<mz_zip_archive*>(this->mZip));
+        }
+        delete reinterpret_cast<mz_zip_archive*>(this->mZip);
+        this->mZip = nullptr;
+    }
+    if (!this->mFinalized) {
+        std::error_code ec;
+        fs::remove(this->mTempPath, ec);
+    }
+}
+
+int32_t ZWrapper::CreateArchive(void) {
+    auto* zip = new mz_zip_archive;
+    std::memset(zip, 0, sizeof(mz_zip_archive));
+
+    if (!mz_zip_writer_init_file(zip, this->mTempPath.c_str(), 0)) {
+        delete zip;
+        throw std::runtime_error("[ZWrapper] Could not open '" + this->mTempPath + "' for writing");
+    }
+
+    this->mZip = reinterpret_cast<mz_zip_archive_tag*>(zip);
+    this->mOpen = true;
+    SPDLOG_INFO("Loaded ZIP (O2R) archive: {}", this->mPath.c_str());
     return 0;
 }
 
 bool ZWrapper::AddFile(const std::string& path, std::vector<char> data) {
-    char* fileData = data.data();
-    size_t fileSize = data.size();
+    const size_t fileSize = data.size();
 
     if (Companion::Instance != nullptr && Companion::Instance->IsDebug()) {
         SPDLOG_INFO("Creating debug file: debug/{}", path);
@@ -30,21 +55,47 @@ bool ZWrapper::AddFile(const std::string& path, std::vector<char> data) {
             fs::create_directories(fs::path(dpath).parent_path());
         }
         std::ofstream stream(dpath, std::ios::binary);
-        stream.write(fileData, fileSize);
+        stream.write(data.data(), fileSize);
         stream.close();
     }
 
-    try {
-        this->mZip->writebytes(path, data);
-    } catch (const std::exception& e) {
+    if (this->mZip == nullptr || !this->mOpen) {
+        throw std::runtime_error("[ZWrapper] Cannot add '" + path + "': the archive is not open");
+    }
+
+    std::lock_guard<std::mutex> guard(this->mMutex);
+
+    if (!mz_zip_writer_add_mem(reinterpret_cast<mz_zip_archive*>(this->mZip), path.c_str(), data.data(), fileSize,
+                               MZ_BEST_COMPRESSION)) {
         // miniz's own error says nothing useful, so log which entry blew up and how big.
-        SPDLOG_ERROR("[ZWrapper] Failed to add '{}' ({} bytes): {}", path, fileSize, e.what());
-        throw;
+        SPDLOG_ERROR("[ZWrapper] Failed to add '{}' ({} bytes)", path, fileSize);
+        throw std::runtime_error("[ZWrapper] Failed to add '" + path + "'");
     }
+
     return true;
 }
 
 int32_t ZWrapper::Close(void) {
-    this->mZip->save(this->mPath);
+    if (this->mZip == nullptr || !this->mOpen) {
+        return -1;
+    }
+
+    auto* zip = reinterpret_cast<mz_zip_archive*>(this->mZip);
+    const bool finalized = mz_zip_writer_finalize_archive(zip);
+    mz_zip_writer_end(zip);
+    this->mOpen = false;
+
+    if (!finalized) {
+        throw std::runtime_error("[ZWrapper] Failed to finalize '" + this->mPath + "'");
+    }
+
+    std::error_code ec;
+    fs::rename(this->mTempPath, this->mPath, ec);
+    if (ec) {
+        throw std::runtime_error("[ZWrapper] Could not move '" + this->mTempPath + "' to '" + this->mPath +
+                                 "': " + ec.message());
+    }
+    this->mFinalized = true;
+
     return 0;
 }
diff --git a/src/archive/ZWrapper.h b/src/archive/ZWrapper.h
index d1fe867..b543082 100644
--- a/src/archive/ZWrapper.h
+++ b/src/archive/ZWrapper.h
@@ -4,17 +4,20 @@
 #include <string>
 #include "BinaryWrapper.h"
 
-namespace miniz_cpp {
-class zip_file;
-}
+struct mz_zip_archive_tag;
 
 class ZWrapper : public BinaryWrapper {
 public:
     explicit ZWrapper(const std::string& path);
+    ~ZWrapper() override;
 
     int32_t CreateArchive(void) override;
     bool AddFile(const std::string& path, std::vector<char> data) override;
     int32_t Close(void) override;
 
-    miniz_cpp::zip_file* mZip;
+private:
+    struct mz_zip_archive_tag* mZip = nullptr;
+    std::string mTempPath;
+    bool mOpen = false;
+    bool mFinalized = false;
 };