VideoBackends: Use proper floating point depth precision.

5 files changed, 12 insertions, 12 deletions

diff --git a/Source/Core/VideoBackends/D3D/PixelShaderCache.cpp b/Source/Core/VideoBackends/D3D/PixelShaderCache.cpp
index 3e2c6292c0..4a2def325c 100644
--- a/Source/Core/VideoBackends/D3D/PixelShaderCache.cpp
+++ b/Source/Core/VideoBackends/D3D/PixelShaderCache.cpp
@@ -146,7 +146,7 @@ const char depth_matrix_program[] = {
 	" in float4 pos : SV_Position,\n"
 	" in float3 uv0 : TEXCOORD0){\n"
 	"	float4 texcol = Tex0.Sample(samp0,uv0);\n"
-	"	int depth = int(round(texcol.x * float(0xFFFFFF)));\n"
+	"	int depth = int(round(texcol.x * 16777216.0));\n"
 
 	// Convert to Z24 format
 	"	int4 workspace;\n"
@@ -180,7 +180,7 @@ const char depth_matrix_program_msaa[] = {
 	"	for(int i = 0; i < SAMPLES; ++i)\n"
 	"		texcol += Tex0.Load(int3(uv0.x*(width), uv0.y*(height), uv0.z), i);\n"
 	"	texcol /= SAMPLES;\n"
-	"	int depth = int(round(texcol.x * float(0xFFFFFF)));\n"
+	"	int depth = int(round(texcol.x * 16777216.0));\n"
 
 	// Convert to Z24 format
 	"	int4 workspace;\n"
diff --git a/Source/Core/VideoBackends/D3D/Render.cpp b/Source/Core/VideoBackends/D3D/Render.cpp
index ec73313476..67046ecf91 100644
--- a/Source/Core/VideoBackends/D3D/Render.cpp
+++ b/Source/Core/VideoBackends/D3D/Render.cpp
@@ -419,11 +419,11 @@ u32 Renderer::AccessEFB(EFBAccessType type, u32 x, u32 y, u32 poke_data)
 		if (bpmem.zcontrol.pixel_format == PEControl::RGB565_Z16)
 		{
 			// if Z is in 16 bit format you must return a 16 bit integer
-			ret = ((u32)(val * 0xffff));
+			ret = ((u32)(val * 65536.0f));
 		}
 		else
 		{
-			ret = ((u32)(val * 0xffffff));
+			ret = ((u32)(val * 16777216.0f));
 		}
 		D3D::context->Unmap(read_tex, 0);
 
diff --git a/Source/Core/VideoBackends/OGL/TextureCache.cpp b/Source/Core/VideoBackends/OGL/TextureCache.cpp
index abc1a73ba9..31979b8347 100644
--- a/Source/Core/VideoBackends/OGL/TextureCache.cpp
+++ b/Source/Core/VideoBackends/OGL/TextureCache.cpp
@@ -294,7 +294,7 @@ void TextureCache::CompileShaders()
 		"\n"
 		"void main(){\n"
 		"	vec4 texcol = texture(samp9, vec3(f_uv0.xy, %s));\n"
-		"	int depth = int(round(texcol.x * float(0xFFFFFF)));\n"
+		"	int depth = int(round(texcol.x * 16777216.0));\n"
 
 		// Convert to Z24 format
 		"	ivec4 workspace;\n"
diff --git a/Source/Core/VideoCommon/PixelShaderGen.cpp b/Source/Core/VideoCommon/PixelShaderGen.cpp
index 4873afc567..5581aa1f1d 100644
--- a/Source/Core/VideoCommon/PixelShaderGen.cpp
+++ b/Source/Core/VideoCommon/PixelShaderGen.cpp
@@ -1224,11 +1224,11 @@ static inline void WritePerPixelDepth(T& out, pixel_shader_uid_data* uid_data, A
 		if (ApiType == API_OPENGL)
 			out.Write("\tscreenpos.y = %i - screenpos.y;\n", EFB_HEIGHT);
 
-		out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * screenpos.x + " I_ZSLOPE".y * screenpos.y) / float(0xFFFFFF);\n");
+		out.Write("\tdepth = float(" I_ZSLOPE".z + " I_ZSLOPE".x * screenpos.x + " I_ZSLOPE".y * screenpos.y) / 16777216.0;\n");
 	}
 	else
 	{
-		out.Write("\tdepth = float(zCoord) / float(0xFFFFFF);\n");
+		out.Write("\tdepth = float(zCoord) / 16777216.0;\n");
 	}
 }
 
diff --git a/Source/Core/VideoCommon/TextureConversionShader.cpp b/Source/Core/VideoCommon/TextureConversionShader.cpp
index 99559db08b..18e5c258c0 100644
--- a/Source/Core/VideoCommon/TextureConversionShader.cpp
+++ b/Source/Core/VideoCommon/TextureConversionShader.cpp
@@ -487,7 +487,7 @@ static void WriteZ16Encoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 0, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth *= 16777216.0;\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -497,7 +497,7 @@ static void WriteZ16Encoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 1, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth *= 16777216.0;\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -519,7 +519,7 @@ static void WriteZ16LEncoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 0, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth *= 16777216.0;\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -531,7 +531,7 @@ static void WriteZ16LEncoder(char*& p,API_TYPE ApiType)
 
 	WriteSampleColor(p, "r", "depth", 1, ApiType);
 
-	WRITE(p, "  depth *= 16777215.0;\n");
+	WRITE(p, "  depth *= 16777216.0;\n");
 	WRITE(p, "  expanded.r = floor(depth / (256.0 * 256.0));\n");
 	WRITE(p, "  depth -= expanded.r * 256.0 * 256.0;\n");
 	WRITE(p, "  expanded.g = floor(depth / 256.0);\n");
@@ -558,7 +558,7 @@ static void WriteZ24Encoder(char*& p, API_TYPE ApiType)
 
 	for (int i = 0; i < 2; i++)
 	{
-		WRITE(p, "  depth%i *= 16777215.0;\n", i);
+		WRITE(p, "  depth%i *= 16777216.0;\n", i);
 
 		WRITE(p, "  expanded%i.r = floor(depth%i / (256.0 * 256.0));\n", i, i);
 		WRITE(p, "  depth%i -= expanded%i.r * 256.0 * 256.0;\n", i, i);