VideoCommon: Fix color channel logic when per-pixel lighting is in use

This was broken in #10012 (specifically by 06579e4d53844e5a45ae18f8eda6c4ee69ebad2c and c3dec343918ff44e90886daf71a6bf0c49033de1).

4 files changed, 44 insertions, 52 deletions

diff --git a/Source/Core/VideoCommon/PixelShaderGen.cpp b/Source/Core/VideoCommon/PixelShaderGen.cpp
index 5f5b41e081..a380937ae2 100644
--- a/Source/Core/VideoCommon/PixelShaderGen.cpp
+++ b/Source/Core/VideoCommon/PixelShaderGen.cpp
@@ -787,6 +787,10 @@ ShaderCode GeneratePixelShaderCode(APIType api_type, const ShaderHostConfig& hos
     // out.SetConstantsUsed(C_PLIGHTS, C_PLIGHTS+31); // TODO: Can be optimized further
     // out.SetConstantsUsed(C_PMATERIALS, C_PMATERIALS+3);
     GenerateLightingShaderCode(out, uid_data->lighting, "colors_", "col");
+    // The number of colors available to TEV is determined by numColorChans.
+    // Normally this is performed in the vertex shader after lighting, but with per-pixel lighting,
+    // we need to perform it here.  (It needs to be done after lighting, as what was originally
+    // black might become a different color after lighting).
     if (uid_data->numColorChans == 0)
       out.Write("col0 = float4(0.0, 0.0, 0.0, 0.0);\n");
     if (uid_data->numColorChans <= 1)
diff --git a/Source/Core/VideoCommon/UberShaderPixel.cpp b/Source/Core/VideoCommon/UberShaderPixel.cpp
index f7c5d4c309..4541f15b10 100644
--- a/Source/Core/VideoCommon/UberShaderPixel.cpp
+++ b/Source/Core/VideoCommon/UberShaderPixel.cpp
@@ -748,6 +748,14 @@ ShaderCode GenPixelShader(APIType api_type, const ShaderHostConfig& host_config,
     WriteVertexLighting(out, api_type, "lit_pos", "lit_normal", "colors_0", "colors_1",
                         "lit_colors_0", "lit_colors_1");
     color_input_prefix = "lit_";
+    out.Write("  // The number of colors available to TEV is determined by numColorChans.\n"
+              "  // Normally this is performed in the vertex shader after lighting,\n"
+              "  // but with per-pixel lighting, we need to perform it here.\n"
+              "  // TODO: Actually implement this for ubershaders\n"
+              "  // if (xfmem_numColorChans == 0u)\n"
+              "  //   o.colors_0 = float4(0.0, 0.0, 0.0, 0.0);\n"
+              "  // if (xfmem_numColorChans <= 1u)\n"
+              "  //   o.colors_1 = float4(0.0, 0.0, 0.0, 0.0);\n");
   }
 
   out.Write("  uint num_stages = {};\n\n",
diff --git a/Source/Core/VideoCommon/UberShaderVertex.cpp b/Source/Core/VideoCommon/UberShaderVertex.cpp
index 90871a45eb..8d3b128688 100644
--- a/Source/Core/VideoCommon/UberShaderVertex.cpp
+++ b/Source/Core/VideoCommon/UberShaderVertex.cpp
@@ -215,37 +215,29 @@ ShaderCode GenVertexShader(APIType api_type, const ShaderHostConfig& host_config
   if (num_texgen > 0)
     GenVertexShaderTexGens(api_type, num_texgen, out);
 
-  out.Write("// The number of colors available to TEV is determined by numColorChans.\n"
-            "// We have to provide the fields to match the interface, so set to zero\n"
-            "// if it's not enabled.\n");
-
-  const char* color_prefix;
   if (per_pixel_lighting)
   {
-    out.Write("\n// Since per-pixel lighting is enabled, the vertex colors are passed through\n"
-              "// unmodified so we can evaluate the lighting in the pixel shader.\n");
-    color_prefix = "vertex_color_";
+    out.Write("// When per-pixel lighting is enabled, the vertex colors are passed through\n"
+              "// unmodified so we can evaluate the lighting in the pixel shader.\n"
+              "// Lighting is also still computed in the vertex shader since it can be used to\n"
+              "// generate texture coordinates. We generated them above, so now the colors can\n"
+              "// be reverted to their previous stage.\n"
+              "o.colors_0 = vertex_color_0;\n"
+              "o.colors_1 = vertex_color_1;\n"
+              "// Note that the numColorChans logic should be (but currently isn't)\n"
+              "// performed in the pixel shader.\n");
   }
   else
   {
-    color_prefix = "rawcolor";
+    out.Write("// The number of colors available to TEV is determined by numColorChans.\n"
+              "// We have to provide the fields to match the interface, so set to zero\n"
+              "// if it's not enabled.\n"
+              "if (xfmem_numColorChans == 0u)\n"
+              "  o.colors_0 = float4(0.0, 0.0, 0.0, 0.0);\n"
+              "if (xfmem_numColorChans <= 1u)\n"
+              "  o.colors_1 = float4(0.0, 0.0, 0.0, 0.0);\n");
   }
 
-  out.Write("if (xfmem_numColorChans == 0u) {{\n"
-            "  if ((components & {}u) != 0u) // VB_HAS_COL0\n"
-            "    o.colors_0 = {}0;\n"
-            "  else\n"
-            "    o.colors_0 = float4(1.0, 1.0, 1.0, 1.0);\n"
-            "}}\n",
-            VB_HAS_COL0, color_prefix);
-  out.Write("if (xfmem_numColorChans <= 1u) {{\n"
-            "  if ((components & {}u) != 0u) // VB_HAS_COL1\n"
-            "    o.colors_1 = {}1;\n"
-            "  else\n"
-            "    o.colors_1 = o.colors_0;\n"
-            "}}\n",
-            VB_HAS_COL1, color_prefix);
-
   if (!host_config.fast_depth_calc)
   {
     // clipPos/w needs to be done in pixel shader, not here
diff --git a/Source/Core/VideoCommon/VertexShaderGen.cpp b/Source/Core/VideoCommon/VertexShaderGen.cpp
index 5795597ff7..7f3e00609a 100644
--- a/Source/Core/VideoCommon/VertexShaderGen.cpp
+++ b/Source/Core/VideoCommon/VertexShaderGen.cpp
@@ -443,38 +443,26 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
     out.Write("}}\n");
   }
 
-  // The number of colors available to TEV is determined by numColorChans.
-  // We have to provide the fields to match the interface, so set to zero if it's not enabled.
-
-  // When per-pixel lighting is enabled, the vertex colors are passed through unmodified so we can
-  // evaluate the lighting in the same manner in the pixel shader.
-  if (uid_data->numColorChans == 0)
+  if (per_pixel_lighting)
   {
-    if ((uid_data->components & VB_HAS_COL0) != 0)
-    {
-      if (per_pixel_lighting)
-        out.Write("o.colors_0 = vertex_color_0;\n");
-      else
-        out.Write("o.colors_0 = rawcolor0;\n");
-    }
-    else
-    {
-      out.Write("o.colors_0 = float4(0.0, 0.0, 0.0, 0.0);\n");
-    }
+    // When per-pixel lighting is enabled, the vertex colors are passed through
+    // unmodified so we can evaluate the lighting in the pixel shader.
+
+    // Lighting is also still computed in the vertex shader since it can be used to
+    // generate texture coordinates. We generated them above, so now the colors can
+    // be reverted to their previous stage.
+    out.Write("o.colors_0 = vertex_color_0;\n");
+    out.Write("o.colors_1 = vertex_color_1;\n");
+    // Note that the numColorChans logic is performed in the pixel shader.
   }
-  if (uid_data->numColorChans <= 1)
+  else
   {
-    if ((uid_data->components & VB_HAS_COL1) != 0)
-    {
-      if (per_pixel_lighting)
-        out.Write("o.colors_1 = vertex_color_1;\n");
-      else
-        out.Write("o.colors_1 = rawcolor1;\n");
-    }
-    else
-    {
+    // The number of colors available to TEV is determined by numColorChans.
+    // We have to provide the fields to match the interface, so set to zero if it's not enabled.
+    if (uid_data->numColorChans == 0)
+      out.Write("o.colors_0 = float4(0.0, 0.0, 0.0, 0.0);\n");
+    if (uid_data->numColorChans <= 1)
       out.Write("o.colors_1 = float4(0.0, 0.0, 0.0, 0.0);\n");
-    }
   }
 
   // clipPos/w needs to be done in pixel shader, not here