Fixes for SoH (#12)

* build ZAPDTR

* Fixed crashes

---------

Co-authored-by: Nicholas Estelami <NEstelami@users.noreply.github.com>

9 files changed, 142 insertions, 38 deletions

diff --git a/ZAPD/CMakeLists.txt b/ZAPD/CMakeLists.txt
index 96c42d0..12be637 100644
--- a/ZAPD/CMakeLists.txt
+++ b/ZAPD/CMakeLists.txt
@@ -42,9 +42,10 @@ set(Header_Files__Yaz0
 source_group("Header Files\\Yaz0" FILES ${Header_Files__Yaz0})

 

 set(Header_Files__Z64

-    "OtherStructs/SkinLimbStructs.h"

+    "OtherStructs/Cutscene_Common.h"

     "OtherStructs/CutsceneMM_Commands.h"

-    "OtherStructs/Cutscene_Commands.h"

+    "OtherStructs/CutsceneOoT_Commands.h"

+    "OtherStructs/SkinLimbStructs.h"

     "ZAnimation.h"

     "ZActorList.h"

     "ZArray.h"

@@ -71,6 +72,7 @@ set(Header_Files__Z64
     "ZTexture.h"

     "ZTextureAnimation.h"

     "ZVector.h"

+    "ZWaterbox.h"

     "ZVtx.h"

 )

 source_group("Header Files\\Z64" FILES ${Header_Files__Z64})

@@ -83,7 +85,6 @@ source_group("Header Files\\Z64\\ZRoom" FILES ${Header_Files__Z64__ZRoom})
 

 set(Header_Files__Z64__ZRoom__Commands

     "ZRoom/Commands/EndMarker.h"

-    "ZRoom/Commands/SetActorCutsceneList.h"

     "ZRoom/Commands/SetActorList.h"

     "ZRoom/Commands/SetAlternateHeaders.h"

     "ZRoom/Commands/SetAnimatedMaterialList.h"

@@ -91,6 +92,7 @@ set(Header_Files__Z64__ZRoom__Commands
     "ZRoom/Commands/SetCollisionHeader.h"

     "ZRoom/Commands/SetCsCamera.h"

     "ZRoom/Commands/SetCutscenes.h"

+    "ZRoom/Commands/SetCutsceneEntryList.cpp"

     "ZRoom/Commands/SetEchoSettings.h"

     "ZRoom/Commands/SetEntranceList.h"

     "ZRoom/Commands/SetExitList.h"

@@ -153,9 +155,10 @@ set(Source_Files__Yaz0
 source_group("Source Files\\Yaz0" FILES ${Source_Files__Yaz0})

 

 set(Source_Files__Z64

-    "OtherStructs/SkinLimbStructs.cpp"

+    "OtherStructs/Cutscene_Common.cpp"

     "OtherStructs/CutsceneMM_Commands.cpp"

-    "OtherStructs/Cutscene_Commands.cpp"

+    "OtherStructs/CutsceneOoT_Commands.cpp"

+    "OtherStructs/SkinLimbStructs.cpp"

     "ZAnimation.cpp"

     "ZActorList.cpp"

     "ZArray.cpp"

@@ -183,6 +186,7 @@ set(Source_Files__Z64
     "ZTexture.cpp"

     "ZTextureAnimation.cpp"

     "ZVector.cpp"

+    "ZWaterbox.cpp"

     "ZVtx.cpp"

 )

 source_group("Source Files\\Z64" FILES ${Source_Files__Z64})

@@ -195,7 +199,6 @@ source_group("Source Files\\Z64\\ZRoom" FILES ${Source_Files__Z64__ZRoom})
 

 set(Source_Files__Z64__ZRoom__Commands

     "ZRoom/Commands/EndMarker.cpp"

-    "ZRoom/Commands/SetActorCutsceneList.cpp"

     "ZRoom/Commands/SetActorList.cpp"

     "ZRoom/Commands/SetAlternateHeaders.cpp"

     "ZRoom/Commands/SetAnimatedMaterialList.cpp"

@@ -203,6 +206,7 @@ set(Source_Files__Z64__ZRoom__Commands
     "ZRoom/Commands/SetCollisionHeader.cpp"

     "ZRoom/Commands/SetCsCamera.cpp"

     "ZRoom/Commands/SetCutscenes.cpp"

+    "ZRoom/Commands/SetCutsceneEntryList.cpp"

     "ZRoom/Commands/SetEchoSettings.cpp"

     "ZRoom/Commands/SetEntranceList.cpp"

     "ZRoom/Commands/SetExitList.cpp"

diff --git a/ZAPD/ExporterSet.h b/ZAPD/ExporterSet.h
index c4dd934..b867478 100644
--- a/ZAPD/ExporterSet.h
+++ b/ZAPD/ExporterSet.h
@@ -5,8 +5,9 @@ typedef bool (*ExporterSetFuncBool)(ZFileMode fileMode);
 typedef void (*ExporterSetFuncVoid)(int argc, char* argv[], int& i);
 typedef void (*ExporterSetFuncVoid2)(const std::string& buildMode, ZFileMode& fileMode);
 typedef void (*ExporterSetFuncVoid3)();
+typedef void (*ExporterSetFuncVoid4)(tinyxml2::XMLElement* reader);
 typedef void (*ExporterSetResSave)(ZResource* res, BinaryWriter& writer);
-
+//processCompilableFunc
 class ExporterSet
 {
 public:
@@ -21,4 +22,6 @@ public:
 	ExporterSetFuncVoid3 beginXMLFunc = nullptr;
 	ExporterSetFuncVoid3 endXMLFunc = nullptr;
 	ExporterSetResSave resSaveFunc = nullptr;
+	ExporterSetFuncVoid3 endProgramFunc = nullptr;
+	ExporterSetFuncVoid4 processCompilableFunc = nullptr;
 };
\ No newline at end of file
diff --git a/ZAPD/GameConfig.cpp b/ZAPD/GameConfig.cpp
index 6068280..c890b4f 100644
--- a/ZAPD/GameConfig.cpp
+++ b/ZAPD/GameConfig.cpp
@@ -167,7 +167,7 @@ void GameConfig::ConfigFunc_ExternalFile(const tinyxml2::XMLElement& element)
 
 void GameConfig::ConfigFunc_EnumData(const tinyxml2::XMLElement& element)
 {
-	std::string path = Path::GetDirectoryName(configFilePath);
+	std::string path = Path::GetDirectoryName(configFilePath).string();
 	path = path.append("/").append(element.Attribute("File"));
 	tinyxml2::XMLDocument doc;
 	tinyxml2::XMLError eResult = doc.LoadFile(path.c_str());
diff --git a/ZAPD/Globals.h b/ZAPD/Globals.h
index c707c54..4cd8fe1 100644
--- a/ZAPD/Globals.h
+++ b/ZAPD/Globals.h
@@ -5,6 +5,8 @@
 #include <vector>
 #include "GameConfig.h"
 #include "ZFile.h"
+#include "ZRom.h"
+#include "FileWorker.h"
 #include "ExporterSet.h"
 
 class ZRoom;
diff --git a/ZAPD/Main.cpp b/ZAPD/Main.cpp
index 47f949e..a388087 100644
--- a/ZAPD/Main.cpp
+++ b/ZAPD/Main.cpp
@@ -1,7 +1,7 @@
 #include "Globals.h"
-#include "Utils/Directory.h"
-#include "Utils/File.h"
-#include "Utils/Path.h"
+#include <Utils/DiskFile.h>
+#include <Utils/Path.h>
+#include <Utils/Directory.h>
 #include "WarningHandler.h"
 
 // Linker Hacks Begin
@@ -103,6 +103,9 @@ void Arg_SetExporter(int& i, char* argv[]);
 void Arg_EnableGCCCompat(int& i, char* argv[]);
 void Arg_ForceStatic(int& i, char* argv[]);
 void Arg_ForceUnaccountedStatic(int& i, char* argv[]);
+void Arg_SetFileListPath(int& i, char* argv[]);
+void Arg_SetBuildRawTexture(int& i, char* argv[]);
+void Arg_SetNoRomMode(int& i, char* argv[]);
 
 int main(int argc, char* argv[]);
 
@@ -205,6 +208,9 @@ extern "C" int zapd_main(int argc, char* argv[])
 	else if (fileMode == ZFileMode::BuildBlob)
 		BuildAssetBlob(Globals::Instance->inputPath, Globals::Instance->outputPath);
 
+	if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
+		exporterSet->endProgramFunc();
+
 	delete g;
 	return returnCode;
 }
@@ -405,6 +411,9 @@ void ParseArgs(int& argc, char* argv[])
 		{"--static", &Arg_ForceStatic},
 		{"-us", &Arg_ForceUnaccountedStatic},
 		{"--unaccounted-static", &Arg_ForceUnaccountedStatic},
+		{"-fl", &Arg_SetFileListPath},
+		{"-brt", &Arg_SetBuildRawTexture},
+		{"--norom", &Arg_SetNoRomMode},
 	};
 
 	for (int32_t i = 2; i < argc; i++)
@@ -421,6 +430,9 @@ void ParseArgs(int& argc, char* argv[])
 		if (it == ArgFuncDictionary.end())
 		{
 			fprintf(stderr, "Unsupported argument: %s\n", arg.c_str());
+			ExporterSet* exporterSet = Globals::Instance->GetExporterSet();
+			if (exporterSet != nullptr)
+				exporterSet->parseArgsFunc(argc, argv, i);
 			continue;
 		}
 
@@ -442,6 +454,8 @@ ZFileMode ParseFileMode(const std::string& buildMode, ExporterSet* exporterSet)
 		fileMode = ZFileMode::BuildBlob;
 	else if (buildMode == "e")
 		fileMode = ZFileMode::Extract;
+	else if (buildMode == "ed")
+		fileMode = ZFileMode::ExtractDirectory;
 	else if (exporterSet != nullptr && exporterSet->parseFileModeFunc != nullptr)
 		exporterSet->parseFileModeFunc(buildMode, fileMode);
 
@@ -527,6 +541,7 @@ void Arg_VerboseUnaccounted([[maybe_unused]] int& i, [[maybe_unused]] char* argv
 
 void Arg_SetExporter(int& i, char* argv[])
 {
+	ImportExporters();
 	Globals::Instance->currentExporter = argv[++i];
 }
 
@@ -545,6 +560,21 @@ void Arg_ForceUnaccountedStatic([[maybe_unused]] int& i, [[maybe_unused]] char*
 	Globals::Instance->forceUnaccountedStatic = true;
 }
 
+void Arg_SetFileListPath(int& i,  char* argv[])
+{
+	Globals::Instance->fileListPath = argv[++i];
+}
+
+void Arg_SetBuildRawTexture([[maybe_unused]] int& i, [[maybe_unused]] char* argv[])
+{
+	Globals::Instance->buildRawTexture = true;
+}
+
+void Arg_SetNoRomMode([[maybe_unused]] int& i, [[maybe_unused]] char* argv[])
+{
+	Globals::Instance->onlyGenSohOtr = true;
+}
+
 int HandleExtract(ZFileMode fileMode, ExporterSet* exporterSet)
 {
 	bool procFileModeSuccess = false;
@@ -558,23 +588,72 @@ int HandleExtract(ZFileMode fileMode, ExporterSet* exporterSet)
 
 		for (auto& extFile : Globals::Instance->cfg.externalFiles)
 		{
-			fs::path externalXmlFilePath =
-				Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
+			if (fileMode == ZFileMode::ExtractDirectory)
+			{
+				std::vector<std::string> fileList =
+					Directory::ListFiles(Globals::Instance->inputPath.string());
+
+				const int num_threads = std::thread::hardware_concurrency();
+				ctpl::thread_pool pool(num_threads > 1 ? num_threads / 2 : 1);
+
+				bool parseSuccessful;
+
+				auto start = std::chrono::steady_clock::now();
+				int fileListSize = fileList.size();
+				Globals::Instance->singleThreaded = false;
+
+				for (int i = 0; i < fileListSize; i++)
+					Globals::Instance->workerData[i] = new FileWorker();
+
+				numWorkersLeft = fileListSize;
+
+				for (int i = 0; i < fileListSize; i++)
+				{
+					if (Globals::Instance->singleThreaded)
+					{
+						ExtractFunc(i, fileList.size(), fileList[i], fileMode);
+					}
+					else
+					{
+						std::string fileListItem = fileList[i];
+						pool.push([i, fileListSize, fileListItem, fileMode](int) {
+							ExtractFunc(i, fileListSize, fileListItem, fileMode);
+						});
+					}
+				}
+
+				if (!Globals::Instance->singleThreaded)
+				{
+					while (true)
+					{
+						if (numWorkersLeft <= 0)
+							break;
+
+						std::this_thread::sleep_for(std::chrono::milliseconds(250));
+					}
+				}
+
+				auto end = std::chrono::steady_clock::now();
+				auto diff = std::chrono::duration_cast<std::chrono::seconds>(end - start).count();
+
+				printf("Generated OTR File Data in %i seconds\n", diff);
+			}
+			else
+			{
+				fs::path externalXmlFilePath =
+					Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
 
-			if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
-				printf("Parsing external file from config: '%s'\n", externalXmlFilePath.c_str());
+				if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
+					printf("Parsing external file from config: '%s'\n",
+					       externalXmlFilePath.c_str());
 
-			parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
-			                        extFile.outPath, ZFileMode::ExternalFile);
+				parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
+				                        extFile.outPath, ZFileMode::ExternalFile, 0);
 
-			if (!parseSuccessful)
-				return 1;
+				if (!parseSuccessful)
+					return 1;
+			}
 		}
-
-		parseSuccessful = Parse(Globals::Instance->inputPath, Globals::Instance->baseRomPath,
-		                        Globals::Instance->outputPath, fileMode);
-		if (!parseSuccessful)
-			return 1;
 	}
 
 	return 0;
diff --git a/ZAPD/OtherStructs/CutsceneOoT_Commands.cpp b/ZAPD/OtherStructs/CutsceneOoT_Commands.cpp
index e38bfb6..68d472a 100644
--- a/ZAPD/OtherStructs/CutsceneOoT_Commands.cpp
+++ b/ZAPD/OtherStructs/CutsceneOoT_Commands.cpp
@@ -64,13 +64,19 @@ std::string CutsceneOoTSubCommandEntry_GenericCmd::GetBodySourceCode() const
 			bool baseOne = (commandId == CutsceneOoT_CommandType::CS_CMD_LIGHT_SETTING ||
 			                commandId == CutsceneOoT_CommandType::CS_CMD_START_SEQ ||
 			                commandId == CutsceneOoT_CommandType::CS_CMD_STOP_SEQ);
-			return StringHelper::Sprintf(entryFmt.c_str(), baseOne ? base - 1 : base, startFrame,
-			                             endFrame, pad, unused1, unused2, unused3, unused4, unused5,
-			                             unused6, unused7, unused8, unused9, unused10);
+			
+			return StringHelper::Sprintf(entryFmt.c_str(), firstArg.c_str(), startFrame, endFrame,
+			                             pad, unused1, unused2, unused3, unused4, unused5, unused6,
+			                             unused7, unused8, unused9, unused10);
+			
+			//return StringHelper::Sprintf(entryFmt.c_str(), baseOne ? base - 1 : base, startFrame,
+			                             //endFrame, pad, unused1, unused2, unused3, unused4, unused5,
+			                             //unused6, unused7, unused8, unused9, unused10);
 		}
-		return StringHelper::Sprintf(entryFmt.c_str(), firstArg.c_str(), startFrame, endFrame, pad,
-		                             unused1, unused2, unused3, unused4, unused5, unused6, unused7,
-		                             unused8, unused9, unused10);
+
+		//return StringHelper::Sprintf(entryFmt.c_str(), firstArg.c_str(), startFrame, endFrame, pad,
+		                             //unused1, unused2, unused3, unused4, unused5, unused6, unused7,
+		                             //unused8, unused9, unused10);
 	}
 	return StringHelper::Sprintf("CS_UNK_DATA(0x%08X, 0x%08X, 0x%08X, 0x%08X, 0x%08X, 0x%08X, "
 	                             "0x%08X, 0x%08X, 0x%08X, 0x%08X, 0x%08X, 0x%08X)",
diff --git a/ZAPD/ZCollision.cpp b/ZAPD/ZCollision.cpp
index 3424783..7a724e5 100644
--- a/ZAPD/ZCollision.cpp
+++ b/ZAPD/ZCollision.cpp
@@ -308,7 +308,7 @@ size_t ZCollisionHeader::GetRawDataSize() const
 {
 	return 44;
 }
-
+#if 0
 WaterBoxHeader::WaterBoxHeader(const std::vector<uint8_t>& rawData, uint32_t rawDataIndex)
 {
 	xMin = BitConverter::ToInt16BE(rawData, rawDataIndex + 0);
@@ -328,7 +328,7 @@ std::string WaterBoxHeader::GetBodySourceCode() const
 	return StringHelper::Sprintf("%i, %i, %i, %i, %i, 0x%08X", xMin, ySurface, zMin, xLength,
 	                             zLength, properties);
 }
-
+#endif
 CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix,
                                const std::vector<uint8_t>& rawData, offset_t rawDataIndex,
                                offset_t upperCameraBoundary)
@@ -436,11 +436,11 @@ CameraDataList::CameraDataList(ZFile* parent, const std::string& prefix,
 
 CameraDataList::~CameraDataList()
 {
-	for (auto entry : entries)
-		delete entry;
-
-	for (auto camPosData : cameraPositionData)
-		delete camPosData;
+	//for (auto entry : entries)
+	//	delete entry;
+//
+	//for (auto camPosData : cameraPositionData)
+	//	delete camPosData;
 }
 
 CameraPositionData::CameraPositionData(const std::vector<uint8_t>& rawData, uint32_t rawDataIndex)
diff --git a/ZAPD/ZRoom/Commands/SetMesh.cpp b/ZAPD/ZRoom/Commands/SetMesh.cpp
index 68a8ab9..2a39766 100644
--- a/ZAPD/ZRoom/Commands/SetMesh.cpp
+++ b/ZAPD/ZRoom/Commands/SetMesh.cpp
@@ -597,7 +597,7 @@ void RoomShapeCullable::DeclareReferences(const std::string& prefix)
 std::string RoomShapeCullable::GetBodySourceCode() const
 {
 	std::string listName;
-	Globals::Instance->GetSegmentedPtrName(start, parent, "", listName);
+	Globals::Instance->GetSegmentedPtrName(start, parent, "", listName, 0, true);
 
 	std::string body = StringHelper::Sprintf("\n    %i, %i,\n", type, polyDLists.size());
 	body += StringHelper::Sprintf("    %s,\n", listName.c_str());
diff --git a/ZAPD/ZTexture.cpp b/ZAPD/ZTexture.cpp
index cd37e72..7d939f7 100644
--- a/ZAPD/ZTexture.cpp
+++ b/ZAPD/ZTexture.cpp
@@ -10,6 +10,12 @@
 #include "Utils/Path.h"
 #include "WarningHandler.h"
 
+// OTRTODO put in LUS
+
+#define ALIGN8(val) (((val) + 7) & ~7)
+#define ALIGN16(val) (((val) + 0xF) & ~0xF)
+#define ALIGN64(val) (((val) + 0x3F) & ~0x3F)
+
 REGISTER_ZFILENODE(Texture, ZTexture);
 
 ZTexture::ZTexture(ZFile* nParent) : ZResource(nParent)
@@ -871,6 +877,8 @@ std::string ZTexture::GetBodySourceCode() const
 
 std::string ZTexture::GetHeaderDefines() const
 {
+	// OTRTODO add to LUS
+	#if 0
 	std::string definePrefix = StringHelper::camelCaseTo_SCREAMING_SNAKE_CASE(name.c_str(), true);
 	std::string ret = StringHelper::Sprintf("#define %s_WIDTH %d\n", definePrefix.c_str(), width);
 
@@ -878,6 +886,8 @@ std::string ZTexture::GetHeaderDefines() const
 	ret += StringHelper::Sprintf("#define %s_SIZE 0x%zX\n", definePrefix.c_str(), GetRawDataSize());
 
 	return ret;
+	#endif
+	return "";
 }
 
 bool ZTexture::IsExternalResource() const