EFB2RAM: Apply copy filter as a float coefficient after sampling

Using 8-bit integer math here lead to precision loss for depth copies,
which broke various effects in games, e.g. lens flare in MK:DD.

It's unlikely the console implements this as a floating-point multiply
(fixed-point perhaps), but since we have the float round trip in our
EFB2RAM shaders anyway, it's not going to make things any worse. If we
do rewrite our shaders to use integer math completely, then it might be
worth switching this conversion back to integers.

However, the range of the values (format) should be known, or we should
expand all values out to 24-bits first.

4 files changed, 25 insertions, 22 deletions

diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp
index 48518911bd..c1e112e69a 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.cpp
+++ b/Source/Core/VideoCommon/TextureCacheBase.cpp
@@ -1506,10 +1506,14 @@ TextureCacheBase::CopyFilterCoefficientArray TextureCacheBase::GetRAMCopyFilterC
 {
   // To simplify the backend, we precalculate the three coefficients in common. Coefficients 0, 1
   // are for the row above, 2, 3, 4 are for the current pixel, and 5, 6 are for the row below.
-  return {static_cast<u32>(coefficients[0]) + static_cast<u32>(coefficients[1]),
-          static_cast<u32>(coefficients[2]) + static_cast<u32>(coefficients[3]) +
-              static_cast<u32>(coefficients[4]),
-          static_cast<u32>(coefficients[5]) + static_cast<u32>(coefficients[6])};
+  return {
+      static_cast<float>(static_cast<u32>(coefficients[0]) + static_cast<u32>(coefficients[1])) /
+          64.0f,
+      static_cast<float>(static_cast<u32>(coefficients[2]) + static_cast<u32>(coefficients[3]) +
+                         static_cast<u32>(coefficients[4])) /
+          64.0f,
+      static_cast<float>(static_cast<u32>(coefficients[5]) + static_cast<u32>(coefficients[6])) /
+          64.0f};
 }
 
 TextureCacheBase::CopyFilterCoefficientArray TextureCacheBase::GetVRAMCopyFilterCoefficients(
diff --git a/Source/Core/VideoCommon/TextureCacheBase.h b/Source/Core/VideoCommon/TextureCacheBase.h
index 68477a4a37..c4a17efc88 100644
--- a/Source/Core/VideoCommon/TextureCacheBase.h
+++ b/Source/Core/VideoCommon/TextureCacheBase.h
@@ -109,7 +109,7 @@ private:
 
 public:
   // Reduced version of the full coefficient array, reduced to a single value for each row.
-  using CopyFilterCoefficientArray = std::array<u32, 3>;
+  using CopyFilterCoefficientArray = std::array<float, 3>;
 
   struct TCacheEntry
   {
diff --git a/Source/Core/VideoCommon/TextureConversionShader.cpp b/Source/Core/VideoCommon/TextureConversionShader.cpp
index 8a2588908c..e57cbdf603 100644
--- a/Source/Core/VideoCommon/TextureConversionShader.cpp
+++ b/Source/Core/VideoCommon/TextureConversionShader.cpp
@@ -67,7 +67,7 @@ static void WriteHeader(char*& p, APIType ApiType)
     WRITE(p, "uniform float y_scale;\n");
     WRITE(p, "uniform float gamma_rcp;\n");
     WRITE(p, "uniform float2 clamp_tb;\n");
-    WRITE(p, "uniform int3 filter_coefficients;\n");
+    WRITE(p, "uniform float3 filter_coefficients;\n");
     WRITE(p, "#define samp0 samp9\n");
     WRITE(p, "SAMPLER_BINDING(9) uniform sampler2DArray samp0;\n");
     WRITE(p, "FRAGMENT_OUTPUT_LOCATION(0) out vec4 ocol0;\n");
@@ -79,7 +79,7 @@ static void WriteHeader(char*& p, APIType ApiType)
     WRITE(p, "  float y_scale;\n");
     WRITE(p, "  float gamma_rcp;\n");
     WRITE(p, "  float2 clamp_tb;\n");
-    WRITE(p, "  int3 filter_coefficients;\n");
+    WRITE(p, "  float3 filter_coefficients;\n");
     WRITE(p, "};\n");
     WRITE(p, "SAMPLER_BINDING(0) uniform sampler2DArray samp0;\n");
     WRITE(p, "FRAGMENT_OUTPUT_LOCATION(0) out vec4 ocol0;\n");
@@ -91,7 +91,7 @@ static void WriteHeader(char*& p, APIType ApiType)
     WRITE(p, "  float y_scale;\n");
     WRITE(p, "  float gamma_rcp;\n");
     WRITE(p, "  float2 clamp_tb;\n");
-    WRITE(p, "  int3 filter_coefficients;\n");
+    WRITE(p, "  float3 filter_coefficients;\n");
     WRITE(p, "};\n");
     WRITE(p, "sampler samp0 : register(s0);\n");
     WRITE(p, "Texture2DArray Tex0 : register(t0);\n");
@@ -191,21 +191,18 @@ static void WriteSampleFunction(char*& p, const EFBCopyParams& params, APIType A
     WRITE(p, "  float4 next_row = ");
     WriteSampleOp(1);
     WRITE(p, ";\n");
-    WRITE(
-        p,
-        "  float3 col = float3(clamp((int3(prev_row.rgb * 255.0) * filter_coefficients[0] +\n"
-        "                             int3(current_row.rgb * 255.0) * filter_coefficients[1] +\n"
-        "                             int3(next_row.rgb * 255.0) * filter_coefficients[2]) >> 6,\n"
-        "                            int3(0, 0, 0), int3(255, 255, 255))) / 255.0;\n");
-    WRITE(p, "  return float4(col, current_row.a);\n");
+    WRITE(p, "  return float4(min(prev_row.rgb * filter_coefficients[0] +\n"
+             "                      current_row.rgb * filter_coefficients[1] +\n"
+             "                      next_row.rgb * filter_coefficients[2], \n"
+             "                    float3(1, 1, 1)), current_row.a);\n");
   }
   else
   {
     WRITE(p, "  float4 current_row = ");
     WriteSampleOp(0);
     WRITE(p, ";\n");
-    WRITE(p, "  return float4(clamp(int3(current_row.rgb * 255.0) * filter_coefficients[1], "
-             "int3(0, 0, 0), int3(255, 255, 255)), current_row.a);\n");
+    WRITE(p, "return float4(min(current_row.rgb * filter_coefficients[1], float3(1, 1, 1)),\n"
+             "              current_row.a);\n");
   }
   WRITE(p, "}\n");
 }
@@ -1422,4 +1419,4 @@ std::string GenerateDecodingShader(TextureFormat format, TLUTFormat palette_form
   return ss.str();
 }
 
-}  // namespace
+}  // namespace TextureConversionShaderTiled
diff --git a/Source/Core/VideoCommon/TextureConverterShaderGen.cpp b/Source/Core/VideoCommon/TextureConverterShaderGen.cpp
index 748afa29b6..5cfb3e7c76 100644
--- a/Source/Core/VideoCommon/TextureConverterShaderGen.cpp
+++ b/Source/Core/VideoCommon/TextureConverterShaderGen.cpp
@@ -97,15 +97,17 @@ ShaderCode GenerateShader(APIType api_type, const UidData* uid_data)
     out.Write("  float4 prev_row = SampleEFB(uv0, -1.0f);\n"
               "  float4 current_row = SampleEFB(uv0, 0.0f);\n"
               "  float4 next_row = SampleEFB(uv0, 1.0f);\n"
-              "  float4 texcol = float4(prev_row.rgb * filter_coefficients[0] +\n"
-              "                         current_row.rgb * filter_coefficients[1] +\n"
-              "                         next_row.rgb * filter_coefficients[2], current_row.a);\n");
+              "  float4 texcol = float4(min(prev_row.rgb * filter_coefficients[0] +\n"
+              "                               current_row.rgb * filter_coefficients[1] +\n"
+              "                               next_row.rgb * filter_coefficients[2], \n"
+              "                             float3(1, 1, 1)), current_row.a);\n");
   }
   else
   {
     out.Write(
         "  float4 current_row = SampleEFB(uv0, 0.0f);\n"
-        "  float4 texcol = float4(current_row.rgb * filter_coefficients[1], current_row.a);\n");
+        "  float4 texcol = float4(min(current_row.rgb * filter_coefficients[1], float3(1, 1, 1)),\n"
+        "                         current_row.a);\n");
   }
 
   if (uid_data->is_depth_copy)