diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | OTRExporter/BackgroundExporter.cpp | 6 | ||||
| -rw-r--r-- | OTRExporter/DisplayListExporter.cpp | 123 | ||||
| -rw-r--r-- | OTRExporter/Main.cpp | 56 | ||||
| -rw-r--r-- | OTRExporter/MtxExporter.cpp | 13 | ||||
| -rw-r--r-- | OTRExporter/MtxExporter.h | 10 | ||||
| -rw-r--r-- | OTRExporter/OTRExporter.vcxproj | 2 | ||||
| -rw-r--r-- | OTRExporter/OTRExporter.vcxproj.filters | 6 | ||||
| -rw-r--r-- | OTRExporter/RoomExporter.cpp | 8 | ||||
| -rw-r--r-- | OTRExporter/TextureExporter.cpp | 4 | ||||
| -rwxr-xr-x | extract_assets.py | 53 |
11 files changed, 221 insertions, 61 deletions
@@ -346,5 +346,6 @@ baserom/ *.otr *.swp *.a +Extract/ tmp.txt diff --git a/OTRExporter/BackgroundExporter.cpp b/OTRExporter/BackgroundExporter.cpp index 2bbee3f..956129c 100644 --- a/OTRExporter/BackgroundExporter.cpp +++ b/OTRExporter/BackgroundExporter.cpp @@ -11,8 +11,10 @@ void OTRExporter_Background::Save(ZResource* res, const fs::path& outPath, Binar auto data = bg->parent->GetRawData(); - for (size_t i = bg->GetRawDataIndex(); i < bg->GetRawDataIndex() + bg->GetRawDataSize(); i++) - writer->Write(data[i]); + //for (size_t i = bg->GetRawDataIndex(); i < bg->GetRawDataIndex() + bg->GetRawDataSize(); i++) + //writer->Write(data[i]); + + writer->Write((char*)data.data() + bg->GetRawDataIndex(), bg->GetRawDataSize()); auto end = std::chrono::steady_clock::now(); size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count(); diff --git a/OTRExporter/DisplayListExporter.cpp b/OTRExporter/DisplayListExporter.cpp index ee4a385..a505a70 100644 --- a/OTRExporter/DisplayListExporter.cpp +++ b/OTRExporter/DisplayListExporter.cpp @@ -9,6 +9,8 @@ #include <Globals.h> #include <iostream> #include <string> +#include "MtxExporter.h" +#include <Utils/File.h> //#include "gbi.h" //#include "Lib/Fast3D/U64/PR/gbi.h" @@ -91,6 +93,9 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina if ((int)opF3D == G_DL)// || (int)opF3D == G_BRANCH_Z) opcode = (uint8_t)G_DL_OTR; + if ((int)opF3D == G_MTX) + opcode = (uint8_t)G_MTX_OTR; + if ((int)opF3D == G_BRANCH_Z) opcode = (uint8_t)G_BRANCH_Z_OTR; @@ -205,16 +210,72 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina break; case G_MTX: { - uint32_t pp = (data & 0x000000FF00000000) >> 32; - uint32_t mm = (data & 0x00000000FFFFFFFF); + if ((!Globals::Instance->HasSegment(GETSEGNUM(data))) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7 + { + uint32_t pp = (data & 0x000000FF00000000) >> 32; + uint32_t mm = (data & 0x00000000FFFFFFFF); - pp ^= G_MTX_PUSH; + pp ^= G_MTX_PUSH; - mm = (mm & 0x0FFFFFFF) + 0xF0000000; + mm = (mm & 0x0FFFFFFF) + 0xF0000000; - Gfx value = gsSPMatrix(mm, pp); - word0 = value.words.w0; - word1 = value.words.w1; + Gfx value = gsSPMatrix(mm, pp); + word0 = value.words.w0; + word1 = value.words.w1; + } + else + { + uint32_t pp = (data & 0x000000FF00000000) >> 32; + uint32_t mm = (data & 0x00000000FFFFFFFF); + pp ^= G_MTX_PUSH; + + Gfx value = gsSPMatrix(mm, pp); + word0 = value.words.w0; + word1 = value.words.w1; + + word0 = (word0 & 0x00FFFFFF) + (G_MTX_OTR << 24); + + Declaration* mtxDecl = dList->parent->GetDeclaration(GETSEGOFFSET(mm)); + + int bp = 0; + + writer->Write(word0); + writer->Write(word1); + + if (mtxDecl != nullptr) + { + std::string vName = StringHelper::Sprintf("%s\\%s", (GetParentFolderName(res).c_str()), mtxDecl->varName.c_str()); + + + + + + /*std::string vName = ""; + + if (GETSEGNUM(mm) == SEGMENT_SCENE || GETSEGNUM(mm) == SEGMENT_ROOM) + vName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), texName.c_str()); + else + vName = StringHelper::Sprintf("%s\\%s", assocFileName.c_str(), texName.c_str());*/ + + //std::string vName = StringHelper::Sprintf("%s\\%s", (GetParentFolderName(res).c_str()), mtxDecl->varName.c_str()); + + uint64_t hash = CRC64(vName.c_str()); + + word0 = hash >> 32; + word1 = hash & 0xFFFFFFFF; + + + + //OTRExporter_MtxExporter mtxExporter; + //mtxExporter.Save(outPath, vName, writer); + } + else + { + word0 = 0; + word1 = 0; + spdlog::error(StringHelper::Sprintf("dListDecl == nullptr! Addr = {:08X}", GETSEGOFFSET(data))); + } + } } break; case G_LOADBLOCK: @@ -326,19 +387,24 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina if (dListDecl2 != nullptr) { - MemoryStream* dlStream = new MemoryStream(); - BinaryWriter dlWriter = BinaryWriter(dlStream); + std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str()); - Save(dList->otherDLists[i], outPath, &dlWriter); + if (!File::Exists("Extract\\" + fName)) + { + MemoryStream* dlStream = new MemoryStream(); + BinaryWriter dlWriter = BinaryWriter(dlStream); - std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str()); + Save(dList->otherDLists[i], outPath, &dlWriter); #ifdef _DEBUG - if (otrArchive->HasFile(fName)) - otrArchive->RemoveFile(fName); + //if (otrArchive->HasFile(fName)) + //otrArchive->RemoveFile(fName); #endif - otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress()); + File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector()); + + //otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress()); + } } else { @@ -414,19 +480,24 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina if (dListDecl2 != nullptr) { - MemoryStream* dlStream = new MemoryStream(); - BinaryWriter dlWriter = BinaryWriter(dlStream); + std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str()); - Save(dList->otherDLists[i], outPath, &dlWriter); + if (!File::Exists("Extract\\" + fName)) + { + MemoryStream* dlStream = new MemoryStream(); + BinaryWriter dlWriter = BinaryWriter(dlStream); - std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str()); + Save(dList->otherDLists[i], outPath, &dlWriter); #ifdef _DEBUG - if (otrArchive->HasFile(fName)) - otrArchive->RemoveFile(fName); + //if (otrArchive->HasFile(fName)) + //otrArchive->RemoveFile(fName); #endif - otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress()); + File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector()); + + //otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress()); + } } else { @@ -769,7 +840,8 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina word0 = hash >> 32; word1 = hash & 0xFFFFFFFF; - if (!otrArchive->HasFile(fName)) + //if (!otrArchive->HasFile(fName)) + if (!File::Exists("Extract\\" + fName)) { //printf("Exporting VTX Data %s\n", fName.c_str()); // Write vertices to file @@ -827,11 +899,12 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a - - int bp = 0; } } - otrArchive->AddFile(fName, (uintptr_t)vtxStream->ToVector().data(), vtxWriter.GetBaseAddress()); + + File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector()); + + //otrArchive->AddFile(fName, (uintptr_t)vtxStream->ToVector().data(), vtxWriter.GetBaseAddress()); auto end = std::chrono::steady_clock::now(); size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count(); diff --git a/OTRExporter/Main.cpp b/OTRExporter/Main.cpp index 06b1a22..af13477 100644 --- a/OTRExporter/Main.cpp +++ b/OTRExporter/Main.cpp @@ -15,6 +15,7 @@ #include "PathExporter.h" #include "TextExporter.h" #include "BlobExporter.h" +#include "MtxExporter.h" #include <Globals.h> #include <Utils/File.h> #include <Utils/Directory.h> @@ -27,19 +28,29 @@ std::chrono::steady_clock::time_point fileStart, resStart; enum class ExporterFileMode { - ModeExample1 = (int)ZFileMode::Custom + 1, - ModeExample2 = (int)ZFileMode::Custom + 2, - ModeExample3 = (int)ZFileMode::Custom + 3, + BuildOTR = (int)ZFileMode::Custom + 1, }; -static void ExporterParseFileMode(std::string buildMode, ZFileMode& fileMode) +static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileMode) { - if (buildMode == "me1") - fileMode = (ZFileMode)ExporterFileMode::ModeExample1; - else if (buildMode == "me2") - fileMode = (ZFileMode)ExporterFileMode::ModeExample2; - else if (buildMode == "me3") - fileMode = (ZFileMode)ExporterFileMode::ModeExample3; + if (buildMode == "botr") + { + fileMode = (ZFileMode)ExporterFileMode::BuildOTR; + + // Do the magic... + if (File::Exists("oot.otr")) + otrArchive = std::shared_ptr<Ship::Archive>(new Ship::Archive("oot.otr", true)); + else + otrArchive = Ship::Archive::CreateArchive("oot.otr"); + + auto lst = Directory::ListFiles("Extract"); + + for (auto item : lst) + { + auto fileData = File::ReadAllBytes(item); + otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size()); + } + } } static void ExporterParseArgs(int argc, char* argv[], int& i) @@ -60,11 +71,7 @@ static bool ExporterProcessFileMode(ZFileMode fileMode) { // Do whatever work is associated with these custom file modes... // Return true to indicate one of our own file modes is being processed - if (fileMode == (ZFileMode)ExporterFileMode::ModeExample1) - return true; - else if (fileMode == (ZFileMode)ExporterFileMode::ModeExample2) - return true; - else if (fileMode == (ZFileMode)ExporterFileMode::ModeExample3) + if (fileMode == (ZFileMode)ExporterFileMode::BuildOTR) return true; return false; @@ -126,12 +133,14 @@ static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer) std::string fName = StringHelper::Sprintf("%s\\%s", oName.c_str(), rName.c_str()); #ifdef _DEBUG - if (otrArchive->HasFile(fName)) - otrArchive->RemoveFile(fName); + //if (otrArchive->HasFile(fName)) + //otrArchive->RemoveFile(fName); #endif + File::WriteAllBytes("Extract\\" + fName, strem->ToVector()); + //if (!otrArchive->HasFile(fName)) - otrArchive->AddFile(fName, (uintptr_t)strem->ToVector().data(), writer.GetBaseAddress()); + //otrArchive->AddFile(fName, (uintptr_t)strem->ToVector().data(), writer.GetBaseAddress()); } auto end = std::chrono::steady_clock::now(); @@ -145,10 +154,10 @@ static void ExporterXMLBegin() { //printf("ExporterXMLBegin() called.\n"); - if (File::Exists("oot.otr")) - otrArchive = std::shared_ptr<Ship::Archive>(new Ship::Archive("oot.otr", true)); - else - otrArchive = Ship::Archive::CreateArchive("oot.otr"); + //if (File::Exists("oot.otr")) + //otrArchive = std::shared_ptr<Ship::Archive>(new Ship::Archive("oot.otr", true)); + //else + //otrArchive = Ship::Archive::CreateArchive("oot.otr"); } static void ExporterXMLEnd() @@ -164,7 +173,7 @@ static void ImportExporters() // By running ZAPD with the argument -se EXAMPLE, we tell it that we want to use this exporter for our resources. ExporterSet* exporterSet = new ExporterSet(); exporterSet->processFileModeFunc = ExporterProcessFileMode; - //exporterSet->parseFileModeFunc = ExporterParseFileMode; + exporterSet->parseFileModeFunc = ExporterParseFileMode; exporterSet->parseArgsFunc = ExporterParseArgs; exporterSet->beginFileFunc = ExporterFileBegin; exporterSet->endFileFunc = ExporterFileEnd; @@ -188,6 +197,7 @@ static void ImportExporters() exporterSet->exporters[ZResourceType::Path] = new OTRExporter_Path(); exporterSet->exporters[ZResourceType::Text] = new OTRExporter_Text(); exporterSet->exporters[ZResourceType::Blob] = new OTRExporter_Blob(); + exporterSet->exporters[ZResourceType::Mtx] = new OTRExporter_MtxExporter(); Globals::AddExporter("OTR", exporterSet); } diff --git a/OTRExporter/MtxExporter.cpp b/OTRExporter/MtxExporter.cpp new file mode 100644 index 0000000..a329c43 --- /dev/null +++ b/OTRExporter/MtxExporter.cpp @@ -0,0 +1,13 @@ +#include "MtxExporter.h" + +void OTRExporter_MtxExporter::Save(ZResource* res, const fs::path& outPath, BinaryWriter* writer) +{ + ZMtx* mtx = (ZMtx*)res; + + WriteHeader(res, outPath, writer, Ship::ResourceType::Matrix); + + for (size_t i = 0; i < 4; i++) + for (size_t j = 0; j < 4; j++) + //TODO possibly utilize the array class better + writer->Write(mtx->mtx[i][j]); +}
\ No newline at end of file diff --git a/OTRExporter/MtxExporter.h b/OTRExporter/MtxExporter.h new file mode 100644 index 0000000..895f2a4 --- /dev/null +++ b/OTRExporter/MtxExporter.h @@ -0,0 +1,10 @@ +#include "ZResource.h" +#include "ZMtx.h" +#include "Exporter.h" +#include <Utils/BinaryWriter.h> + +class OTRExporter_MtxExporter : public OTRExporter +{ +public: + virtual void Save(ZResource* res, const fs::path& outPath, BinaryWriter* writer) override; +}; diff --git a/OTRExporter/OTRExporter.vcxproj b/OTRExporter/OTRExporter.vcxproj index 03a6e9d..828ca5f 100644 --- a/OTRExporter/OTRExporter.vcxproj +++ b/OTRExporter/OTRExporter.vcxproj @@ -29,6 +29,7 @@ <ClInclude Include="AnimationExporter.h" />
<ClInclude Include="Main.h" />
<ClInclude Include="Exporter.h" />
+ <ClInclude Include="MtxExporter.h" />
<ClInclude Include="SkeletonExporter.h" />
<ClInclude Include="SkeletonLimbExporter.h" />
<ClInclude Include="PathExporter.h" />
@@ -50,6 +51,7 @@ <ClCompile Include="AnimationExporter.cpp" />
<ClCompile Include="Main.cpp" />
<ClCompile Include="Exporter.cpp" />
+ <ClCompile Include="MtxExporter.cpp" />
<ClCompile Include="SkeletonExporter.cpp" />
<ClCompile Include="SkeletonLimbExporter.cpp" />
<ClCompile Include="PathExporter.cpp" />
diff --git a/OTRExporter/OTRExporter.vcxproj.filters b/OTRExporter/OTRExporter.vcxproj.filters index 97cae0b..5ea7088 100644 --- a/OTRExporter/OTRExporter.vcxproj.filters +++ b/OTRExporter/OTRExporter.vcxproj.filters @@ -75,6 +75,9 @@ <ClInclude Include="BlobExporter.h">
<Filter>Header Files</Filter>
</ClInclude>
+ <ClInclude Include="MtxExporter.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="CollisionExporter.cpp">
@@ -128,5 +131,8 @@ <ClCompile Include="BlobExporter.cpp">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="MtxExporter.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
</ItemGroup>
</Project>
\ No newline at end of file diff --git a/OTRExporter/RoomExporter.cpp b/OTRExporter/RoomExporter.cpp index ac770f4..7c28d07 100644 --- a/OTRExporter/RoomExporter.cpp +++ b/OTRExporter/RoomExporter.cpp @@ -453,8 +453,10 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite OTRExporter_Cutscene cs; cs.Save(cmdSetCutscenes->cutscenes[0], "", &csWriter); + File::WriteAllBytes("Extract\\" + fName, csStream->ToVector()); + //std::string fName = OTRExporter_DisplayList::GetPathToRes(res, vtxDecl->varName); - otrArchive->AddFile(fName, (uintptr_t)csStream->ToVector().data(), csWriter.GetBaseAddress()); + //otrArchive->AddFile(fName, (uintptr_t)csStream->ToVector().data(), csWriter.GetBaseAddress()); } break; case RoomCommand::SetPathways: @@ -474,7 +476,9 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite OTRExporter_Path pathExp; pathExp.Save(&cmdSetPathways->pathwayList, outPath, &pathWriter); - otrArchive->AddFile(path, (uintptr_t)pathStream->ToVector().data(), pathWriter.GetBaseAddress()); + File::WriteAllBytes("Extract\\" + path, pathStream->ToVector()); + + //otrArchive->AddFile(path, (uintptr_t)pathStream->ToVector().data(), pathWriter.GetBaseAddress()); int bp = 0; } diff --git a/OTRExporter/TextureExporter.cpp b/OTRExporter/TextureExporter.cpp index ab6a5a0..a9fc8bf 100644 --- a/OTRExporter/TextureExporter.cpp +++ b/OTRExporter/TextureExporter.cpp @@ -22,6 +22,8 @@ void OTRExporter_Texture::Save(ZResource* res, const fs::path& outPath, BinaryWr for (size_t i = tex->GetRawDataIndex(); i < tex->GetRawDataIndex() + tex->GetRawDataSize(); i++) writer->Write(data[i]); + writer->Write((char*)data.data() + tex->GetRawDataIndex(), tex->GetRawDataSize()); + auto end = std::chrono::steady_clock::now(); size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count(); @@ -29,6 +31,4 @@ void OTRExporter_Texture::Save(ZResource* res, const fs::path& outPath, BinaryWr //if (diff > 2) //printf("Export took %lms\n", diff); - - //otrArchive->AddFile(file->GetName(), (uintptr_t)strem->ToVector().data(), strem->GetLength()); }
\ No newline at end of file diff --git a/extract_assets.py b/extract_assets.py index f5629f8..7c2c3d5 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -11,6 +11,24 @@ def SignalHandler(sig, frame): mainAbort.set() # Don't exit immediately to update the extracted assets file. +def BuildOTR(): + #if globalAbort.is_set(): + # Don't extract if another file wasn't extracted properly. + # return + + execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out" + + execStr += " botr -se OTR" + + print(execStr) + exitValue = os.system(execStr) + if exitValue != 0: + # globalAbort.set() + print("\n") + print("Error when building the OTR file...", file=os.sys.stderr) + print("Aborting...", file=os.sys.stderr) + print("\n") + def ExtractFile(xmlPath, outputPath, outputSourcePath): #if globalAbort.is_set(): # Don't extract if another file wasn't extracted properly. @@ -24,9 +42,6 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath): if "overlays" in xmlPath: execStr += " --static" - #if globalUnaccounted: - # execStr += " -wu" - print(execStr) exitValue = os.system(execStr) #exitValue = 0 @@ -62,6 +77,16 @@ def ExtractFunc(fullPath): # globalExtractedAssetsTracker[fullPath] = globalManager.dict() # globalExtractedAssetsTracker[fullPath]["timestamp"] = currentTimeStamp +#def initializeWorker(abort, unaccounted: bool, extractedAssetsTracker: dict, manager): +def initializeWorker(abort, test): + global globalAbort + #global globalUnaccounted + #global globalExtractedAssetsTracker + #global globalManager + globalAbort = abort + #globalUnaccounted = unaccounted + #globalExtractedAssetsTracker = extractedAssetsTracker + #globalManager = manager def main(): @@ -76,7 +101,7 @@ def main(): manager = Manager() signal.signal(signal.SIGINT, SignalHandler) - #extractedAssetsTracker = manager.dict() + extractedAssetsTracker = manager.dict() #if os.path.exists(EXTRACTED_ASSETS_NAMEFILE) and not args.force: # with open(EXTRACTED_ASSETS_NAMEFILE, encoding='utf-8') as f: # extractedAssetsTracker.update(json.load(f, object_hook=manager.dict)) @@ -107,9 +132,23 @@ def main(): if file.endswith(".xml"): xmlFiles.append(fullPath) - for singlePath in xmlFiles: - ExtractFunc(singlePath) - #return + try: + numCores = cpu_count() + print("Extracting assets with " + str(numCores) + " CPU cores.") + #with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, args.unaccounted, extractedAssetsTracker, manager)) as p: + with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p: + p.map(ExtractFunc, xmlFiles) + except Exception as e: + print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr) + print("Disabling mutliprocessing.", file=os.sys.stderr) + + initializeWorker(mainAbort, 0) + for singlePath in xmlFiles: + ExtractFunc(singlePath) + + + BuildOTR() + os.rmdir("Extract") # OTRTODO: This line does not work... #with open(EXTRACTED_ASSETS_NAMEFILE, 'w', encoding='utf-8') as f: # serializableDict = dict() |
