Fix various issues with zfreeze implemntation.

Results are still not correct, but things are getting closer.

 * Don't cull CULLALL primitives so early so they can be used as reference
        planes.
 * Convert CalculateZSlope to screenspace coordinates.
 * Convert Pixelshader to screenspace coordinates (instead of worldspace
        xy coordinates, which is totally wrong)
 * Divide depth by 2^24 instead of clamping to 0.0-1.0 as was done
        before.

Progress:
 * Rouge Squadron 2/3 appear correct in game (videos in rs2 save file
         selection are missing)
 * Shadows draw 100% correctly in NHL 2003.
 * Mario golf menu renders correctly.
 * NFS: HP2, shadows sometimes render on top of car or below the road.
 * Mario Tennis, courts and shadows render correctly, but at wrong depth
 * Blood Omen 2, doesn't work.

1 files changed, 5 insertions, 4 deletions

diff --git a/Source/Core/VideoCommon/VertexManagerBase.cpp b/Source/Core/VideoCommon/VertexManagerBase.cpp
index 80ea3b5bb9..bcdaf466a4 100644
--- a/Source/Core/VideoCommon/VertexManagerBase.cpp
+++ b/Source/Core/VideoCommon/VertexManagerBase.cpp
@@ -259,11 +259,12 @@ void VertexManager::CalculateZSlope(u32 stride)
 
 		VertexShaderManager::TransformToClipSpace(&vtx[i * 3], &out[i * 4]);
 
-		// viewport offset ignored because we only look at coordinate differences.
-		out[0 + i * 4] = out[0 + i * 4] / out[3 + i * 4] * xfmem.viewport.wd;
-		out[1 + i * 4] = out[1 + i * 4] / out[3 + i * 4] * xfmem.viewport.ht;
+		// Transform to Screenspace
+		out[0 + i * 4] = out[0 + i * 4] / out[3 + i * 4] * xfmem.viewport.wd + (xfmem.viewport.xOrig - 342);
+		out[1 + i * 4] = out[1 + i * 4] / out[3 + i * 4] * xfmem.viewport.ht + (xfmem.viewport.yOrig - 342);
 		out[2 + i * 4] = out[2 + i * 4] / out[3 + i * 4] * xfmem.viewport.zRange + xfmem.viewport.farZ;
 	}
+
 	float dx31 = out[8] - out[0];
 	float dx12 = out[0] - out[4];
 	float dy12 = out[1] - out[5];
@@ -277,7 +278,7 @@ void VertexManager::CalculateZSlope(u32 stride)
 
 	float slope_dfdx = -a / c;
 	float slope_dfdy = -b / c;
-	float slope_f0 = out[2];
+	float slope_f0 = out[2] - (out[0] * slope_dfdx + out[1] * slope_dfdy);
 
 	PixelShaderManager::SetZSlope(slope_dfdx, slope_dfdy, slope_f0);
 }