VideoSW: rewrite lighting attenuation

- Fixes remaining lighting issues (Mario Tennis, etc)
- Apply same fixes to Software Renderer
- Corrected zero length light direction vector to resolve with normal direction (essentially becomes LIGHTDIF_NONE which was what I was after)

1 files changed, 81 insertions, 121 deletions

diff --git a/Source/Core/VideoBackends/Software/TransformUnit.cpp b/Source/Core/VideoBackends/Software/TransformUnit.cpp
index f170e7d9c4..73eab84440 100644
--- a/Source/Core/VideoBackends/Software/TransformUnit.cpp
+++ b/Source/Core/VideoBackends/Software/TransformUnit.cpp
@@ -212,38 +212,33 @@ static void LightColor(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 {
 	const LightPointer *light = (const LightPointer*)&xfmem.lights[lightNum];
 
-	if (!(chan.attnfunc & 1))
+	Vec3 ldir = light->pos - pos;
+	float attn = 1.0f;
+
+	switch (chan.attnfunc)
 	{
-		// atten disabled
-		switch (chan.diffusefunc)
+		case LIGHTATTN_NONE:
+		case LIGHTATTN_DIR:
 		{
-			case LIGHTDIF_NONE:
-				AddIntegerColor(light->color, lightCol);
-				break;
-			case LIGHTDIF_SIGN:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = ldir * normal;
-					AddScaledIntegerColor(light->color, diffuse, lightCol);
-				}
-				break;
-			case LIGHTDIF_CLAMP:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = std::max(0.0f, ldir * normal);
-					AddScaledIntegerColor(light->color, diffuse, lightCol);
-				}
-				break;
-			default: _assert_(0);
+			ldir = ldir.Normalized();
+			if (ldir == Vec3(0, 0, 0))
+				ldir = Vec3(1, 1, 1);
+			break;
 		}
-	}
-	else // spec and spot
-	{
-		// not sure about divide by zero checks
-		Vec3 ldir = light->pos - pos;
-		float attn;
-
-		if (chan.attnfunc == 3) // spot
+		case LIGHTATTN_SPEC:
+		{
+			ldir = ldir.Normalized();
+			attn = (ldir * normal) >= 0.0 ? std::max(0.0f, light->dir * normal) : 0;
+			Vec3 attLen = Vec3(1.0, attn, attn*attn);
+			Vec3 cosAttn = light->cosatt;
+			Vec3 distAttn = light->distatt;
+			if (chan.diffusefunc != LIGHTDIF_NONE)
+				distAttn = distAttn.Normalized();
+
+			attn = SafeDivide(std::max(0.0f, attLen * cosAttn), attLen * distAttn);
+			break;
+		}
+		case LIGHTATTN_SPOT:
 		{
 			float dist2 = ldir.Length2();
 			float dist = sqrtf(dist2);
@@ -253,43 +248,26 @@ static void LightColor(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 			float cosAtt = light->cosatt.x + (light->cosatt.y * attn) + (light->cosatt.z * attn * attn);
 			float distAtt = light->distatt.x + (light->distatt.y * dist) + (light->distatt.z * dist2);
 			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
+			break;
 		}
-		else if (chan.attnfunc == 1) // specular
-		{
-			// donko - what is going on here?  655.36 is a guess but seems about right.
-			attn = (light->pos * normal) > -655.36 ? std::max(0.0f, (light->dir * normal)) : 0;
-			ldir.set(1.0f, attn, attn * attn);
-
-			float cosAtt = std::max(0.0f, light->cosatt * ldir);
-			float distAtt = light->distatt * ldir;
-			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
-		}
-		else
-		{
+		default:
 			PanicAlert("LightColor");
-			return;
-		}
+	}
 
-		switch (chan.diffusefunc)
-		{
-			case LIGHTDIF_NONE:
-				AddScaledIntegerColor(light->color, attn, lightCol);
-				break;
-			case LIGHTDIF_SIGN:
-				{
-					float difAttn = ldir * normal;
-					AddScaledIntegerColor(light->color, attn * difAttn, lightCol);
-				}
-				break;
-
-			case LIGHTDIF_CLAMP:
-				{
-					float difAttn = std::max(0.0f, ldir * normal);
-					AddScaledIntegerColor(light->color, attn * difAttn, lightCol);
-				}
-				break;
-			default: _assert_(0);
-		}
+	float difAttn = ldir * normal;
+	switch (chan.diffusefunc)
+	{
+		case LIGHTDIF_NONE:
+			AddScaledIntegerColor(light->color, attn, lightCol);
+			break;
+		case LIGHTDIF_SIGN:
+			AddScaledIntegerColor(light->color, attn * difAttn, lightCol);
+			break;
+		case LIGHTDIF_CLAMP:
+			difAttn = std::max(0.0f, difAttn);
+			AddScaledIntegerColor(light->color, attn * difAttn, lightCol);
+			break;
+		default: _assert_(0);
 	}
 }
 
@@ -297,37 +275,31 @@ static void LightAlpha(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 {
 	const LightPointer *light = (const LightPointer*)&xfmem.lights[lightNum];
 
-	if (!(chan.attnfunc & 1))
+	Vec3 ldir = light->pos - pos;
+	float attn = 1.0f;
+
+	switch (chan.attnfunc)
 	{
-		// atten disabled
-		switch (chan.diffusefunc)
+		case LIGHTATTN_NONE:
+		case LIGHTATTN_DIR:
 		{
-			case LIGHTDIF_NONE:
-				lightCol += light->color[0];
-				break;
-			case LIGHTDIF_SIGN:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = ldir * normal;
-					lightCol += light->color[0] * diffuse;
-				}
-				break;
-			case LIGHTDIF_CLAMP:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = std::max(0.0f, ldir * normal);
-					lightCol += light->color[0] * diffuse;
-				}
-				break;
-			default: _assert_(0);
+			ldir = ldir.Normalized();
+			break;
 		}
-	}
-	else // spec and spot
-	{
-		Vec3 ldir = light->pos - pos;
-		float attn;
-
-		if (chan.attnfunc == 3) // spot
+		case LIGHTATTN_SPEC:
+		{
+			ldir = ldir.Normalized();
+			attn = (ldir * normal) >= 0.0 ? std::max(0.0f, light->dir * normal) : 0;
+			Vec3 attLen = Vec3(1.0, attn, attn*attn);
+			Vec3 cosAttn = light->cosatt;
+			Vec3 distAttn = light->distatt;
+			if (chan.diffusefunc != LIGHTDIF_NONE)
+				distAttn = distAttn.Normalized();
+
+			attn = SafeDivide(std::max(0.0f, attLen * cosAttn), attLen * distAttn);
+			break;
+		}
+		case LIGHTATTN_SPOT:
 		{
 			float dist2 = ldir.Length2();
 			float dist = sqrtf(dist2);
@@ -337,38 +309,26 @@ static void LightAlpha(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 			float cosAtt = light->cosatt.x + (light->cosatt.y * attn) + (light->cosatt.z * attn * attn);
 			float distAtt = light->distatt.x + (light->distatt.y * dist) + (light->distatt.z * dist2);
 			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
+			break;
 		}
-		else /* if (chan.attnfunc == 1) */ // specular
-		{
-			// donko - what is going on here?  655.36 is a guess but seems about right.
-			attn = (light->pos * normal) > -655.36 ? std::max(0.0f, (light->dir * normal)) : 0;
-			ldir.set(1.0f, attn, attn * attn);
-
-			float cosAtt = light->cosatt * ldir;
-			float distAtt = light->distatt * ldir;
-			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
-		}
+		default:
+			PanicAlert("LightColor");
+	}
 
-		switch (chan.diffusefunc)
-		{
-			case LIGHTDIF_NONE:
-				lightCol += light->color[0] * attn;
-				break;
-			case LIGHTDIF_SIGN:
-				{
-					float difAttn = ldir * normal;
-					lightCol += light->color[0] * attn * difAttn;
-				}
-				break;
-
-			case LIGHTDIF_CLAMP:
-				{
-					float difAttn = std::max(0.0f, ldir * normal);
-					lightCol += light->color[0] * attn * difAttn;
-				}
-				break;
-			default: _assert_(0);
-		}
+	float difAttn = ldir * normal;
+	switch (chan.diffusefunc)
+	{
+		case LIGHTDIF_NONE:
+			lightCol += light->color[0] * attn;
+			break;
+		case LIGHTDIF_SIGN:
+			lightCol += light->color[0] * attn * difAttn;
+			break;
+		case LIGHTDIF_CLAMP:
+			difAttn = std::max(0.0f, difAttn);
+			lightCol += light->color[0] * attn * difAttn;
+			break;
+		default: _assert_(0);
 	}
 }