LightingShaderGen: Always calculate lighting for both color channels

Cel-damage depends on lighting being calculated for the first channel
even though there is no color in the vertex format (defaults to the
material color). If lighting for the channel is not enabled, the vertex
will use the default color as before.

The default value of the color is determined by the number of elements in
the vertex format. This fixes the grey cubes in Super Mario Sunshine.

If the color channel count is zero, we set the color to black before the
end of the vertex shader. It's possible that this would be undefined
behavior on hardware if a vertex color index that was greater than the
channel count was used within TEV.

1 files changed, 11 insertions, 1 deletions

diff --git a/Source/Core/VideoCommon/VertexShaderManager.cpp b/Source/Core/VideoCommon/VertexShaderManager.cpp
index aa57161d71..04cb3b2f6f 100644
--- a/Source/Core/VideoCommon/VertexShaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexShaderManager.cpp
@@ -452,7 +452,6 @@ void VertexShaderManager::SetConstants()
       constants.xfmem_pack1[i][3] = xfmem.alpha[i].hex;
     }
     constants.xfmem_numColorChans = xfmem.numChan.numColorChans;
-
     dirty = true;
   }
 }
@@ -617,6 +616,17 @@ void VertexShaderManager::SetVertexFormat(u32 components)
     constants.components = components;
     dirty = true;
   }
+
+  // The default alpha channel seems to depend on the number of components in the vertex format.
+  // If the vertex attribute has an alpha channel, zero is used, otherwise one.
+  const u32 color_chan_alpha =
+      (g_main_cp_state.vtx_attr[g_main_cp_state.last_id].g0.Color0Elements ^ 1) |
+      ((g_main_cp_state.vtx_attr[g_main_cp_state.last_id].g0.Color1Elements ^ 1) << 1);
+  if (color_chan_alpha != constants.color_chan_alpha)
+  {
+    constants.color_chan_alpha = color_chan_alpha;
+    dirty = true;
+  }
 }
 
 void VertexShaderManager::SetTexMatrixInfoChanged(int index)