Merge pull request #2369 from Armada651/depth-percision

VideoBackends: Use proper floating point depth precision.

2 files changed, 12 insertions, 11 deletions

diff --git a/Source/Core/VideoCommon/PixelShaderGen.cpp b/Source/Core/VideoCommon/PixelShaderGen.cpp
index 4873afc567..562cb9be07 100644
--- a/Source/Core/VideoCommon/PixelShaderGen.cpp
+++ b/Source/Core/VideoCommon/PixelShaderGen.cpp
@@ -563,13 +563,14 @@ static inline void GeneratePixelShader(T& out, DSTALPHA_MODE dstAlphaMode, API_T
 	// The performance impact of this additional calculation doesn't matter, but it prevents
 	// the host GPU driver from performing any early depth test optimizations.
 	if (g_ActiveConfig.bFastDepthCalc)
-		out.Write("\tint zCoord = iround(rawpos.z * float(0xFFFFFF));\n");
+		out.Write("\tint zCoord = int(rawpos.z * 16777216.0);\n");
 	else
 	{
 		out.SetConstantsUsed(C_ZBIAS+1, C_ZBIAS+1);
 		// the screen space depth value = far z + (clip z / clip w) * z range
-		out.Write("\tint zCoord = " I_ZBIAS"[1].x + iround((clipPos.z / clipPos.w) * float(" I_ZBIAS"[1].y));\n");
+		out.Write("\tint zCoord = " I_ZBIAS"[1].x + int((clipPos.z / clipPos.w) * float(" I_ZBIAS"[1].y));\n");
 	}
+	out.Write("\tzCoord = clamp(zCoord, " I_ZBIAS"[1].x - " I_ZBIAS"[1].y, " I_ZBIAS"[1].x);\n");
 
 	// depth texture can safely be ignored if the result won't be written to the depth buffer (early_ztest) and isn't used for fog either
 	const bool skip_ztexture = !per_pixel_depth && !bpmem.fog.c_proj_fsel.fsel;
@@ -1169,13 +1170,13 @@ static inline void WriteFog(T& out, pixel_shader_uid_data* uid_data)
 		// TODO: Verify that we want to drop lower bits here! (currently taken over from software renderer)
 		//       Maybe we want to use "ze = (A << B_SHF)/((B << B_SHF) - Zs)" instead?
 		//       That's equivalent, but keeps the lower bits of Zs.
-		out.Write("\tfloat ze = (" I_FOGF"[1].x * 16777215.0) / float(" I_FOGI".y - (zCoord >> " I_FOGI".w));\n");
+		out.Write("\tfloat ze = (" I_FOGF"[1].x * 16777216.0) / float(" I_FOGI".y - (zCoord >> " I_FOGI".w));\n");
 	}
 	else
 	{
 		// orthographic
 		// ze = a*Zs    (here, no B_SHF)
-		out.Write("\tfloat ze = " I_FOGF"[1].x * float(zCoord) / 16777215.0;\n");
+		out.Write("\tfloat ze = " I_FOGF"[1].x * float(zCoord) / 16777216.0;\n");
 	}
 
 	// x_adjust = sqrt((x-center)^2 + k^2)/k
@@ -1224,11 +1225,11 @@ static inline void WritePerPixelDepth(T& out, pixel_shader_uid_data* uid_data, A
 		if (ApiType == API_OPENGL)
 			out.Write("\tscreenpos.y = %i - screenpos.y;\n", EFB_HEIGHT);
 
-		out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * screenpos.x + " I_ZSLOPE".y * screenpos.y) / float(0xFFFFFF);\n");
+		out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * screenpos.x + " I_ZSLOPE".y * screenpos.y) / 16777216.0;\n");
 	}
 	else
 	{
-		out.Write("\tdepth = float(zCoord) / float(0xFFFFFF);\n");
+		out.Write("\tdepth = float(zCoord) / 16777216.0;\n");
 	}
 }
 
diff --git a/Source/Core/VideoCommon/TextureConversionShader.cpp b/Source/Core/VideoCommon/TextureConversionShader.cpp
index 99559db08b..9d8c4eab5c 100644
--- a/Source/Core/VideoCommon/TextureConversionShader.cpp
+++ b/Source/Core/VideoCommon/TextureConversionShader.cpp
@@ -487,7 +487,7 @@ static void WriteZ16Encoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 0, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -497,7 +497,7 @@ static void WriteZ16Encoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 1, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -519,7 +519,7 @@ static void WriteZ16LEncoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 0, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -531,7 +531,7 @@ static void WriteZ16LEncoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 1, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth = clamp(depth * 16777216.0, 0.0, float(0xFFFFFF));\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -558,7 +558,7 @@ static void WriteZ24Encoder(char*& p, API_TYPE ApiType)
 
 	for (int i = 0; i < 2; i++)
 	{
-		WRITE(p, "  depth%i *= 16777215.0;\n", i);
+		WRITE(p, "  depth%i = clamp(depth%i * 16777216.0, 0.0, float(0xFFFFFF));\n", i, i);
 
 		WRITE(p, "  expanded%i.r = floor(depth%i / (256.0 * 256.0));\n", i, i);
 		WRITE(p, "  depth%i -= expanded%i.r * 256.0 * 256.0;\n", i, i);