Merge pull request #6348 from stenzek/fog

BPMemory: Handle fog configuration where both A and C are infinity/NaN

1 files changed, 39 insertions, 5 deletions

diff --git a/Source/Core/VideoCommon/BPMemory.cpp b/Source/Core/VideoCommon/BPMemory.cpp
index 8c0a2ec43a..fe6e62ed6e 100644
--- a/Source/Core/VideoCommon/BPMemory.cpp
+++ b/Source/Core/VideoCommon/BPMemory.cpp
@@ -23,21 +23,55 @@ bool BlendMode::UseLogicOp() const
   return true;
 }
 
-float FogParam0::GetA() const
+bool FogParams::IsNaNCase() const
 {
+  // Check for the case where both a and c are infinity or NaN.
+  // On hardware, this results in the following colors:
+  //
+  // -------------------------------------------------------
+  // |   A   |   C   |  Result  |   A   |   C   |  Result  |
+  // -------------------------------------------------------
+  // |  inf  |  inf  |  Fogged  |  inf  |  nan  |  Fogged  |
+  // |  inf  | -inf  | Unfogged |  inf  | -nan  | Unfogged |
+  // | -inf  |  inf  | Unfogged | -inf  |  nan  | Unfogged |
+  // | -inf  | -inf  | Unfogged | -inf  | -nan  | Unfogged |
+  // -------------------------------------------------------
+  // |  nan  |  inf  |  Fogged  |  nan  |  nan  |  Fogged  |
+  // |  nan  | -inf  | Unfogged |  nan  | -nan  | Unfogged |
+  // | -nan  |  inf  | Unfogged | -nan  |  nan  | Unfogged |
+  // | -nan  | -inf  | Unfogged | -nan  | -nan  | Unfogged |
+  // -------------------------------------------------------
+  //
+  // We replicate this by returning A=0, and C=inf for the inf/inf case, otherwise -inf.
+  // This ensures we do not pass a NaN to the GPU, and -inf/inf clamp to 0/1 respectively.
+  return a.exp == 255 && c_proj_fsel.c_exp == 255;
+}
+
+float FogParams::GetA() const
+{
+  if (IsNaNCase())
+    return 0.0f;
+
   // scale mantissa from 11 to 23 bits
-  const u32 integral = (static_cast<u32>(sign) << 31) | (static_cast<u32>(exponent) << 23) |
-                       (static_cast<u32>(mantissa) << 12);
+  const u32 integral = (static_cast<u32>(a.sign) << 31) | (static_cast<u32>(a.exp) << 23) |
+                       (static_cast<u32>(a.mant) << 12);
 
   float real;
   std::memcpy(&real, &integral, sizeof(u32));
   return real;
 }
 
-float FogParam3::GetC() const
+float FogParams::GetC() const
 {
+  if (IsNaNCase())
+  {
+    constexpr float inf = std::numeric_limits<float>::infinity();
+    return !a.sign && !c_proj_fsel.c_sign ? -inf : inf;
+  }
+
   // scale mantissa from 11 to 23 bits
-  const u32 integral = (c_sign.Value() << 31) | (c_exp.Value() << 23) | (c_mant.Value() << 12);
+  const u32 integral = (c_proj_fsel.c_sign.Value() << 31) | (c_proj_fsel.c_exp.Value() << 23) |
+                       (c_proj_fsel.c_mant.Value() << 12);
 
   float real;
   std::memcpy(&real, &integral, sizeof(u32));