Extract palette data from external file (#249)

* WIP

* Done

* Fix

* Fixes

* More fixes

* Fixes for backgrounds.

* Roman's fixes and format.

* Update docs and use tinyxml function for validation.

* Update docs/zapd_extraction_xml_reference.md

Co-authored-by: EllipticEllipsis <elliptic.ellipsis@gmail.com>

* move tlut texture data to stack variable

Co-authored-by: EllipticEllipsis <elliptic.ellipsis@gmail.com>

10 files changed, 128 insertions, 49 deletions

diff --git a/ZAPD/ImageBackend.cpp b/ZAPD/ImageBackend.cpp
index 8accb6b..307945b 100644
--- a/ZAPD/ImageBackend.cpp
+++ b/ZAPD/ImageBackend.cpp
@@ -398,7 +398,7 @@ void ImageBackend::SetPaletteIndex(size_t index, uint8_t nR, uint8_t nG, uint8_t
 	alphaPalette[index] = nA;
 }
 
-void ImageBackend::SetPalette(const ImageBackend& pal)
+void ImageBackend::SetPalette(const ImageBackend& pal, uint32_t offset)
 {
 	assert(isColorIndexed);
 	size_t bytePerPixel = pal.GetBytesPerPixel();
@@ -422,7 +422,7 @@ void ImageBackend::SetPalette(const ImageBackend& pal)
 			uint8_t g = pal.pixelMatrix[y][x * bytePerPixel + 1];
 			uint8_t b = pal.pixelMatrix[y][x * bytePerPixel + 2];
 			uint8_t a = pal.pixelMatrix[y][x * bytePerPixel + 3];
-			SetPaletteIndex(index, r, g, b, a);
+			SetPaletteIndex(index + offset, r, g, b, a);
 		}
 	}
 }
diff --git a/ZAPD/ImageBackend.h b/ZAPD/ImageBackend.h
index 9e05a2e..0b1f480 100644
--- a/ZAPD/ImageBackend.h
+++ b/ZAPD/ImageBackend.h
@@ -42,8 +42,9 @@ public:
 	void SetGrayscalePixel(size_t y, size_t x, uint8_t grayscale, uint8_t alpha = 0);
 
 	void SetIndexedPixel(size_t y, size_t x, uint8_t index, uint8_t grayscale);
+	void SetIndexedPixel(size_t y, size_t x, uint8_t index, RGBAPixel pixel);
 	void SetPaletteIndex(size_t index, uint8_t nR, uint8_t nG, uint8_t nB, uint8_t nA);
-	void SetPalette(const ImageBackend& pal);
+	void SetPalette(const ImageBackend& pal, uint32_t offset = 0);
 
 	uint32_t GetWidth() const;
 	uint32_t GetHeight() const;
diff --git a/ZAPD/ZAPD.vcxproj b/ZAPD/ZAPD.vcxproj
index f3bb35a..26362c4 100644
--- a/ZAPD/ZAPD.vcxproj
+++ b/ZAPD/ZAPD.vcxproj
@@ -91,6 +91,7 @@
       <LanguageStandard>stdcpp17</LanguageStandard>

       <LanguageStandard_C>stdc11</LanguageStandard_C>

       <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

     </ClCompile>

     <Link>

       <Profile>true</Profile>

@@ -112,6 +113,7 @@ mkdir build\ZAPD
       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>

       <LanguageStandard_C>stdc11</LanguageStandard_C>

       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

     </ClCompile>

     <Link>

       <Profile>true</Profile>

@@ -132,6 +134,7 @@ mkdir build\ZAPD
       <IntrinsicFunctions>true</IntrinsicFunctions>

       <SDLCheck>true</SDLCheck>

       <ConformanceMode>true</ConformanceMode>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

     </ClCompile>

     <Link>

       <EnableCOMDATFolding>true</EnableCOMDATFolding>

@@ -153,6 +156,7 @@ mkdir build\ZAPD
       <ConformanceMode>true</ConformanceMode>

       <LanguageStandard>stdcpplatest</LanguageStandard>

       <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

     </ClCompile>

     <Link>

       <EnableCOMDATFolding>true</EnableCOMDATFolding>

diff --git a/ZAPD/ZCollision.cpp b/ZAPD/ZCollision.cpp
index 992332d..debf037 100644
--- a/ZAPD/ZCollision.cpp
+++ b/ZAPD/ZCollision.cpp
@@ -61,7 +61,7 @@ void ZCollisionHeader::ParseRawData()
 		currentPtr += vec.GetRawDataSize();
 		vertices.push_back(vec);
 	}
-	
+
 	for (uint16_t i = 0; i < numPolygons; i++)
 	{
 		ZCollisionPoly poly(parent);
@@ -158,9 +158,10 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
 				declaration += "\n";
 		}
 
-		parent->AddDeclarationArray(
-			polySegmentOffset, DeclarationAlignment::Align4, polygons.size() * 16, polygons[0].GetSourceTypeName().c_str(),
-			StringHelper::Sprintf("%sPolygons", auxName.c_str()), polygons.size(), declaration);
+		parent->AddDeclarationArray(polySegmentOffset, DeclarationAlignment::Align4,
+		                            polygons.size() * 16, polygons[0].GetSourceTypeName().c_str(),
+		                            StringHelper::Sprintf("%sPolygons", auxName.c_str()),
+		                            polygons.size(), declaration);
 	}
 
 	declaration.clear();
diff --git a/ZAPD/ZCollision.h b/ZAPD/ZCollision.h
index 7481a85..6a49484 100644
--- a/ZAPD/ZCollision.h
+++ b/ZAPD/ZCollision.h
@@ -1,10 +1,10 @@
 #pragma once
 
+#include "ZCollisionPoly.h"
 #include "ZFile.h"
 #include "ZResource.h"
 #include "ZRoom/ZRoom.h"
 #include "ZVector.h"
-#include "ZCollisionPoly.h"
 
 #if 0
 class PolygonEntry
diff --git a/ZAPD/ZCollisionPoly.cpp b/ZAPD/ZCollisionPoly.cpp
index eb2c14a..c4eb2d4 100644
--- a/ZAPD/ZCollisionPoly.cpp
+++ b/ZAPD/ZCollisionPoly.cpp
@@ -16,61 +16,63 @@ ZCollisionPoly::~ZCollisionPoly()
 
 void ZCollisionPoly::ParseRawData()
 {
-    const auto& rawData = parent->GetRawData();
-    type = BitConverter::ToUInt16BE(rawData, rawDataIndex + 0);
-    
-    vtxA = BitConverter::ToUInt16BE(rawData, rawDataIndex + 2);
-    vtxB = BitConverter::ToUInt16BE(rawData, rawDataIndex + 4);
-    vtxC = BitConverter::ToUInt16BE(rawData, rawDataIndex + 6);
-
-    normX = BitConverter::ToUInt16BE(rawData, rawDataIndex + 8);
-    normY = BitConverter::ToUInt16BE(rawData, rawDataIndex + 10);
-    normZ = BitConverter::ToUInt16BE(rawData, rawDataIndex + 12);
-
-    dist = BitConverter::ToUInt16BE(rawData, rawDataIndex + 14);
+	const auto& rawData = parent->GetRawData();
+	type = BitConverter::ToUInt16BE(rawData, rawDataIndex + 0);
+
+	vtxA = BitConverter::ToUInt16BE(rawData, rawDataIndex + 2);
+	vtxB = BitConverter::ToUInt16BE(rawData, rawDataIndex + 4);
+	vtxC = BitConverter::ToUInt16BE(rawData, rawDataIndex + 6);
+
+	normX = BitConverter::ToUInt16BE(rawData, rawDataIndex + 8);
+	normY = BitConverter::ToUInt16BE(rawData, rawDataIndex + 10);
+	normZ = BitConverter::ToUInt16BE(rawData, rawDataIndex + 12);
+
+	dist = BitConverter::ToUInt16BE(rawData, rawDataIndex + 14);
 }
 
 void ZCollisionPoly::DeclareReferences(const std::string& prefix)
 {
-    std::string declaration;
-    std::string auxName = name;
+	std::string declaration;
+	std::string auxName = name;
 
-    if (name == "")
-        auxName = GetDefaultName(prefix);
+	if (name == "")
+		auxName = GetDefaultName(prefix);
 
-    parent->AddDeclaration(rawDataIndex, DeclarationAlignment::Align4,GetRawDataSize(), GetSourceTypeName(), name.c_str(),GetBodySourceCode());
+	parent->AddDeclaration(rawDataIndex, DeclarationAlignment::Align4, GetRawDataSize(),
+	                       GetSourceTypeName(), name.c_str(), GetBodySourceCode());
 }
 
 std::string ZCollisionPoly::GetBodySourceCode() const
 {
-    std::string declaration;
+	std::string declaration;
 
-    declaration += StringHelper::Sprintf("{0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X}", type, vtxA, vtxB, vtxC, normX, normY, normZ, dist);
-    return declaration;
+	declaration +=
+		StringHelper::Sprintf("{0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X}",
+	                          type, vtxA, vtxB, vtxC, normX, normY, normZ, dist);
+	return declaration;
 }
 
 std::string ZCollisionPoly::GetDefaultName(const std::string& prefix) const
 {
-    return StringHelper::Sprintf("%sCollisionPoly_%06X", prefix.c_str(), rawDataIndex);
+	return StringHelper::Sprintf("%sCollisionPoly_%06X", prefix.c_str(), rawDataIndex);
 }
 
-
 ZResourceType ZCollisionPoly::GetResourceType() const
 {
-    return ZResourceType::CollisionPoly;
+	return ZResourceType::CollisionPoly;
 }
 
 size_t ZCollisionPoly::GetRawDataSize() const
 {
-    return 16;
+	return 16;
 }
 
 std::string ZCollisionPoly::GetSourceTypeName() const
 {
-    return "CollisionPoly";
+	return "CollisionPoly";
 }
 
 bool ZCollisionPoly::DoesSupportArray() const
 {
-    return true;
+	return true;
 }
\ No newline at end of file
diff --git a/ZAPD/ZCollisionPoly.h b/ZAPD/ZCollisionPoly.h
index 9596d22..64b98ba 100644
--- a/ZAPD/ZCollisionPoly.h
+++ b/ZAPD/ZCollisionPoly.h
@@ -3,28 +3,27 @@
 #include "ZFile.h"
 #include "ZResource.h"
 
-
 class ZCollisionPoly : public ZResource
 {
-public: 
-    uint16_t type;
+public:
+	uint16_t type;
 	uint16_t vtxA, vtxB, vtxC;
 	uint16_t normX, normY, normZ;
-    uint16_t dist;
+	uint16_t dist;
 
-    ZCollisionPoly(ZFile* nParent);
-    ~ZCollisionPoly();
+	ZCollisionPoly(ZFile* nParent);
+	~ZCollisionPoly();
 
-    void ParseRawData() override;
-    void DeclareReferences(const std::string& prefix) override;
+	void ParseRawData() override;
+	void DeclareReferences(const std::string& prefix) override;
 
-    std::string GetBodySourceCode() const override;
-    std::string GetDefaultName(const std::string& prefix) const override;
+	std::string GetBodySourceCode() const override;
+	std::string GetDefaultName(const std::string& prefix) const override;
 
-    std::string GetSourceTypeName() const override;
-    ZResourceType GetResourceType() const override;
+	std::string GetSourceTypeName() const override;
+	ZResourceType GetResourceType() const override;
 
-    bool DoesSupportArray() const override;
+	bool DoesSupportArray() const override;
 
-    size_t GetRawDataSize() const override;
+	size_t GetRawDataSize() const override;
 };
\ No newline at end of file
diff --git a/ZAPD/ZTexture.cpp b/ZAPD/ZTexture.cpp
index a9cb539..74b7c7d 100644
--- a/ZAPD/ZTexture.cpp
+++ b/ZAPD/ZTexture.cpp
@@ -17,11 +17,15 @@ ZTexture::ZTexture(ZFile* nParent) : ZResource(nParent)
 	width = 0;
 	height = 0;
 	dWordAligned = true;
+	splitTlut = false;
 
 	RegisterRequiredAttribute("Width");
 	RegisterRequiredAttribute("Height");
 	RegisterRequiredAttribute("Format");
 	RegisterOptionalAttribute("TlutOffset");
+	RegisterOptionalAttribute("ExternalTlut");
+	RegisterOptionalAttribute("ExternalTlutOffset");
+	RegisterOptionalAttribute("SplitTlut");
 }
 
 void ZTexture::ExtractFromBinary(uint32_t nRawDataIndex, int32_t nWidth, int32_t nHeight,
@@ -56,6 +60,7 @@ void ZTexture::ParseXML(tinyxml2::XMLElement* reader)
 
 	std::string widthXml = registeredAttributes.at("Width").value;
 	std::string heightXml = registeredAttributes.at("Height").value;
+	std::string SplitTlutXml = registeredAttributes.at("SplitTlut").value;
 
 	if (!StringHelper::HasOnlyDigits(widthXml))
 	{
@@ -72,6 +77,27 @@ void ZTexture::ParseXML(tinyxml2::XMLElement* reader)
 		                      errorHeader, "");
 	}
 
+	if (!registeredAttributes.at("ExternalTlut").wasSet &&
+	    registeredAttributes.at("SplitTlut").wasSet)
+	{
+		std::string errorHeader =
+			StringHelper::Sprintf("SplitTlut set without using an external tlut");
+		HANDLE_WARNING_RESOURCE(WarningType::InvalidAttributeValue, parent, this, rawDataIndex,
+		                        errorHeader, "");
+	}
+
+	if (!SplitTlutXml.empty())
+	{
+		if (!tinyxml2::XMLUtil::ToBool(SplitTlutXml.c_str(), &splitTlut))
+		{
+			std::string errorHeader = StringHelper::Sprintf(
+				"Invalid value passed to SplitTlut: '%s'. Valid values are true, false, 1, 0",
+				SplitTlutXml.c_str());
+			HANDLE_ERROR_RESOURCE(WarningType::InvalidAttributeValue, parent, this, rawDataIndex,
+			                      errorHeader, "");
+		}
+	}
+
 	width = StringHelper::StrToL(widthXml);
 	height = StringHelper::StrToL(heightXml);
 
@@ -145,6 +171,47 @@ void ZTexture::ParseRawData()
 	}
 }
 
+void ZTexture::ParseRawDataLate()
+{
+	if (registeredAttributes["ExternalTlut"].wasSet)
+	{
+		const std::string externPalette = registeredAttributes["ExternalTlut"].value;
+		for (const auto& file : Globals::Instance->files)
+		{
+			if (file->GetName() == externPalette)
+			{
+				offset_t palOffset = 0;
+				if (registeredAttributes["ExternalTlutOffset"].wasSet)
+				{
+					palOffset =
+						StringHelper::StrToL(registeredAttributes["ExternalTlutOffset"].value, 16);
+				}
+				else
+				{
+					HANDLE_WARNING_RESOURCE(
+						WarningType::MissingOffsets, parent, this, rawDataIndex,
+						StringHelper::Sprintf(
+							"No ExternalTlutOffset Given. Assuming offset of 0x0"),
+						"");
+				}
+				for (const auto& res : file->resources)
+				{
+					if (res->GetRawDataIndex() == palOffset)
+					{
+						ZTexture* palette = (ZTexture*)res;
+						ZTexture tlutTemp(file);
+
+						tlut = &tlutTemp;
+						tlut->ExtractFromBinary(palOffset, palette->width, palette->height,
+						                        TextureType::RGBA16bpp, true);
+						SetTlut(tlut);
+					}
+				}
+			}
+		}
+	}
+}
+
 void ZTexture::PrepareBitmapRGBA16()
 {
 	textureData.InitEmptyRGBImage(width, height, true);
@@ -871,7 +938,7 @@ void ZTexture::SetTlut(ZTexture* nTlut)
 	assert(nTlut->isPalette);
 	tlut = nTlut;
 
-	textureData.SetPalette(tlut->textureData);
+	textureData.SetPalette(tlut->textureData, splitTlut ? 128 : 0);
 }
 
 bool ZTexture::HasTlut() const
diff --git a/ZAPD/ZTexture.h b/ZAPD/ZTexture.h
index 6f2c499..8cece3f 100644
--- a/ZAPD/ZTexture.h
+++ b/ZAPD/ZTexture.h
@@ -28,6 +28,7 @@ protected:
 	std::vector<uint8_t> textureDataRaw;  // When reading from a PNG file.
 	uint32_t tlutOffset = static_cast<uint32_t>(-1);
 	ZTexture* tlut = nullptr;
+	bool splitTlut;
 
 	void PrepareBitmapRGBA16();
 	void PrepareBitmapRGBA32();
@@ -88,4 +89,5 @@ public:
 	bool IsColorIndexed() const;
 	void SetTlut(ZTexture* nTlut);
 	bool HasTlut() const;
+	void ParseRawDataLate() override;
 };
diff --git a/docs/zapd_extraction_xml_reference.md b/docs/zapd_extraction_xml_reference.md
index 89ce33a..b2a679c 100644
--- a/docs/zapd_extraction_xml_reference.md
+++ b/docs/zapd_extraction_xml_reference.md
@@ -155,6 +155,9 @@ u64 gCraterSmokeConeTex[] = {
   - `Width`: Required. Width in pixels of the image.
   - `Height`: Required. Height in pixels of the image.
   - `TlutOffset`: Optional. Specifies the tlut's offset used by this texture. This attribute is only valid if `Format` is either `ci4` or `ci8`, otherwise an exception would be thrown.
+  - `ExternalTlut`: Optional. Specifies that the texture's tlut is in a different file. Takes the filename of the file that contains the tlut.
+  - `ExternalTlutOffset`: Optional. Specifies the offset in the `ExternalTlut` of the tlut for the given texture.
+  - `SplitTlut`: Optional. Specifies that the given texture should take from the upper half of the tlut. Takes a bool, i.e. one of `true`, `false`, `1`, `0`.
 
 The following is a list of the texture formats the Nintendo 64 supports, with their gfxdis names and ZAPD format names.