Use custom isnan implementation to avoid HLSL optimizer issues

This adjusts the NaN replacement logic introduced in #9928 to work around the HLSL compiler optimizing away calls to isnan, which caused that functionality to not work with ubershaders on D3D11 and D3D12 (it did work with specialized shaders, despite a warning being logged for both; that warning is also now gone).  Note that the `D3DCOMPILE_IEEE_STRICTNESS` flag did not solve this issue, despite the warning suggesting that it might.

Suggested by @kayru and @jamiehayes.

1 files changed, 4 insertions, 3 deletions

diff --git a/Source/Core/VideoCommon/UberShaderVertex.cpp b/Source/Core/VideoCommon/UberShaderVertex.cpp
index 5b02b1f989..70b434b786 100644
--- a/Source/Core/VideoCommon/UberShaderVertex.cpp
+++ b/Source/Core/VideoCommon/UberShaderVertex.cpp
@@ -50,6 +50,7 @@ ShaderCode GenVertexShader(APIType api_type, const ShaderHostConfig& host_config
   out.Write("}};\n\n");
 
   WriteUberShaderCommonHeader(out, api_type, host_config);
+  WriteIsNanHeader(out, api_type);
   WriteLightingFunction(out);
 
   if (api_type == APIType::OpenGL || api_type == APIType::Vulkan)
@@ -438,9 +439,9 @@ static void GenVertexShaderTexGens(APIType api_type, u32 num_texgen, ShaderCode&
   // Convert NaNs to 1 - needed to fix eyelids in Shadow the Hedgehog during cutscenes
   // See https://bugs.dolphin-emu.org/issues/11458
   out.Write("  // Convert NaN to 1\n");
-  out.Write("  if (isnan(coord.x)) coord.x = 1.0;\n");
-  out.Write("  if (isnan(coord.y)) coord.y = 1.0;\n");
-  out.Write("  if (isnan(coord.z)) coord.z = 1.0;\n");
+  out.Write("  if (dolphin_isnan(coord.x)) coord.x = 1.0;\n");
+  out.Write("  if (dolphin_isnan(coord.y)) coord.y = 1.0;\n");
+  out.Write("  if (dolphin_isnan(coord.z)) coord.z = 1.0;\n");
 
   out.Write("  // first transformation\n");
   out.Write("  uint texgentype = {};\n", BitfieldExtract<&TexMtxInfo::texgentype>("texMtxInfo"));