Merge pull request #13350 from iwubcode/custom_shader_overhaul_vertex_pos

VideoCommon: move transform matrices to accessible functions in VertexShaderGen

1 files changed, 67 insertions, 45 deletions

diff --git a/Source/Core/VideoCommon/VertexShaderGen.cpp b/Source/Core/VideoCommon/VertexShaderGen.cpp
index ace5cb514e..053510f68b 100644
--- a/Source/Core/VideoCommon/VertexShaderGen.cpp
+++ b/Source/Core/VideoCommon/VertexShaderGen.cpp
@@ -74,6 +74,58 @@ VertexShaderUid GetVertexShaderUid()
   return out;
 }
 
+static void WriteTransformMatrices(APIType api_type, const ShaderHostConfig& host_config,
+                                   const vertex_shader_uid_data* uid_data, ShaderCode& out)
+{
+  out.Write("mat3x4 dolphin_position_matrix()\n");
+  out.Write("{{\n");
+  out.Write("\tmat3x4 result;\n");
+  if ((uid_data->components & VB_HAS_POSMTXIDX) != 0)
+  {
+    // Vertex format has a per-vertex matrix
+    out.Write("\tint posidx = int(posmtx.r);\n"
+              "\tresult[0] = " I_TRANSFORMMATRICES "[posidx];\n"
+              "\tresult[1] = " I_TRANSFORMMATRICES "[posidx + 1];\n"
+              "\tresult[2] = " I_TRANSFORMMATRICES "[posidx + 2];\n");
+  }
+  else
+  {
+    // One shared matrix
+    out.Write("\tresult[0] = " I_POSNORMALMATRIX "[0];\n"
+              "\tresult[1] = " I_POSNORMALMATRIX "[1];\n"
+              "\tresult[2] = " I_POSNORMALMATRIX "[2];\n");
+  }
+  out.Write("\treturn result;\n");
+  out.Write("}}\n\n");
+
+  // The scale of the transform matrix is used to control the size of the emboss map effect, by
+  // changing the scale of the transformed binormals (which only get used by emboss map texgens).
+  // By normalising the first transformed normal (which is used by lighting calculations and needs
+  // to be unit length), the same transform matrix can do double duty, scaling for emboss mapping,
+  // and not scaling for lighting.
+  out.Write("mat3 dolphin_normal_matrix()\n");
+  out.Write("{{\n");
+  out.Write("\tmat3 result;\n");
+  if ((uid_data->components & VB_HAS_POSMTXIDX) != 0)
+  {
+    // Vertex format has a per-vertex matrix
+    out.Write("\tint posidx = int(posmtx.r);\n");
+    out.Write("\tint normidx = posidx & 31;\n"
+              "\tresult[0] = " I_NORMALMATRICES "[normidx].xyz;\n"
+              "\tresult[1] = " I_NORMALMATRICES "[normidx + 1].xyz;\n"
+              "\tresult[2] = " I_NORMALMATRICES "[normidx + 2].xyz;\n");
+  }
+  else
+  {
+    // One shared matrix
+    out.Write("\tresult[0] = " I_POSNORMALMATRIX "[3].xyz;\n"
+              "\tresult[1] = " I_POSNORMALMATRIX "[4].xyz;\n"
+              "\tresult[2] = " I_POSNORMALMATRIX "[5].xyz;\n");
+  }
+  out.Write("\treturn result;\n");
+  out.Write("}}\n\n");
+}
+
 static void WriteTexCoordTransforms(APIType api_type, const ShaderHostConfig& host_config,
                                     const vertex_shader_uid_data* uid_data, ShaderCode& out)
 {
@@ -336,6 +388,8 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
     }
   }
 
+  // Note: this is done after to ensure above global variables are accessible
+  WriteTransformMatrices(api_type, host_config, uid_data, out);
   WriteTexCoordTransforms(api_type, host_config, uid_data, out);
 
   out.Write("void main()\n{{\n");
@@ -383,50 +437,17 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
     }
   }
 
-  // transforms
-  if ((uid_data->components & VB_HAS_POSMTXIDX) != 0)
-  {
-    // Vertex format has a per-vertex matrix
-    out.Write("int posidx = int(posmtx.r);\n"
-              "float4 P0 = " I_TRANSFORMMATRICES "[posidx];\n"
-              "float4 P1 = " I_TRANSFORMMATRICES "[posidx + 1];\n"
-              "float4 P2 = " I_TRANSFORMMATRICES "[posidx + 2];\n"
-              "int normidx = posidx & 31;\n"
-              "float3 N0 = " I_NORMALMATRICES "[normidx].xyz;\n"
-              "float3 N1 = " I_NORMALMATRICES "[normidx + 1].xyz;\n"
-              "float3 N2 = " I_NORMALMATRICES "[normidx + 2].xyz;\n");
-  }
-  else
-  {
-    // One shared matrix
-    out.Write("float4 P0 = " I_POSNORMALMATRIX "[0];\n"
-              "float4 P1 = " I_POSNORMALMATRIX "[1];\n"
-              "float4 P2 = " I_POSNORMALMATRIX "[2];\n"
-              "float3 N0 = " I_POSNORMALMATRIX "[3].xyz;\n"
-              "float3 N1 = " I_POSNORMALMATRIX "[4].xyz;\n"
-              "float3 N2 = " I_POSNORMALMATRIX "[5].xyz;\n");
-  }
+  out.Write("\tvec4 pos = vec4(rawpos * dolphin_position_matrix(), 1.0);\n");
 
-  out.Write("// Multiply the position vector by the position matrix\n"
-            "float4 pos = float4(dot(P0, rawpos), dot(P1, rawpos), dot(P2, rawpos), 1.0);\n");
   if ((uid_data->components & VB_HAS_NORMAL) == 0)
-    out.Write("float3 rawnormal = " I_CACHED_NORMAL ".xyz;\n");
+    out.Write("\tvec3 rawnormal = " I_CACHED_NORMAL ".xyz;\n");
+  out.Write("\tvec3 _normal = normalize(rawnormal * dolphin_normal_matrix());\n");
+
   if ((uid_data->components & VB_HAS_TANGENT) == 0)
-    out.Write("float3 rawtangent = " I_CACHED_TANGENT ".xyz;\n");
-  if ((uid_data->components & VB_HAS_BINORMAL) == 0)
-    out.Write("float3 rawbinormal = " I_CACHED_BINORMAL ".xyz;\n");
+    out.Write("\tvec3 rawtangent = " I_CACHED_TANGENT ".xyz;\n");
 
-  // The scale of the transform matrix is used to control the size of the emboss map effect, by
-  // changing the scale of the transformed binormals (which only get used by emboss map texgens).
-  // By normalising the first transformed normal (which is used by lighting calculations and needs
-  // to be unit length), the same transform matrix can do double duty, scaling for emboss mapping,
-  // and not scaling for lighting.
-  out.Write("float3 _normal = normalize(float3(dot(N0, rawnormal), dot(N1, rawnormal), dot(N2, "
-            "rawnormal)));\n"
-            "float3 _tangent = float3(dot(N0, rawtangent), dot(N1, rawtangent), dot(N2, "
-            "rawtangent));\n"
-            "float3 _binormal = float3(dot(N0, rawbinormal), dot(N1, rawbinormal), dot(N2, "
-            "rawbinormal));\n");
+  if ((uid_data->components & VB_HAS_BINORMAL) == 0)
+    out.Write("\tvec3 rawbinormal = " I_CACHED_BINORMAL ".xyz;\n");
 
   out.Write("o.pos = float4(dot(" I_PROJECTION "[0], pos), dot(" I_PROJECTION
             "[1], pos), dot(" I_PROJECTION "[2], pos), dot(" I_PROJECTION "[3], pos));\n");
@@ -505,9 +526,11 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
       // transform the light dir into tangent space
       out.Write("ldir = normalize(" LIGHT_POS ".xyz - pos.xyz);\n",
                 LIGHT_POS_PARAMS(texinfo.embosslightshift));
-      out.Write(
-          "o.tex{}.xyz = o.tex{}.xyz + float3(dot(ldir, _tangent), dot(ldir, _binormal), 0.0);\n",
-          i, texinfo.embosssourceshift);
+      out.Write("vec3 tangent = rawtangent * dolphin_normal_matrix();\n");
+      out.Write("vec3 binormal = rawbinormal * dolphin_normal_matrix();\n");
+      out.Write("o.tex{}.xyz = o.tex{}.xyz + vec3(dot(ldir, tangent), "
+                "dot(ldir, binormal), 0.0);\n",
+                i, texinfo.embosssourceshift);
 
       break;
     case TexGenType::Color0:
@@ -551,8 +574,7 @@ ShaderCode GenerateVertexShaderCode(APIType api_type, const ShaderHostConfig& ho
     }
     else
     {
-      out.Write("other_pos = float4(dot(P0, other_pos), dot(P1, other_pos), dot(P2, other_pos), "
-                "1.0f);\n");
+      out.Write("other_pos = vec4(other_pos * dolphin_position_matrix(), 1.0);\n");
     }
     GenerateVSLineExpansion(out, "", uid_data->numTexGens);
   }