Merge pull request #8378 from stenzek/quad-buffer-stereo

Various quad-buffered ("HDMI 3D") stereo fixes

6 files changed, 169 insertions, 137 deletions

diff --git a/Source/Core/VideoBackends/D3D/main.cpp b/Source/Core/VideoBackends/D3D/main.cpp
index 33c9117f25..816c896478 100644
--- a/Source/Core/VideoBackends/D3D/main.cpp
+++ b/Source/Core/VideoBackends/D3D/main.cpp
@@ -154,8 +154,8 @@ bool VideoBackend::Initialize(const WindowSystemInfo& wsi)
   g_framebuffer_manager = std::make_unique<FramebufferManager>();
   g_texture_cache = std::make_unique<TextureCacheBase>();
   g_perf_query = std::make_unique<PerfQuery>();
-  if (!g_renderer->Initialize() || !g_vertex_manager->Initialize() ||
-      !g_shader_cache->Initialize() || !g_framebuffer_manager->Initialize() ||
+  if (!g_vertex_manager->Initialize() || !g_shader_cache->Initialize() ||
+      !g_renderer->Initialize() || !g_framebuffer_manager->Initialize() ||
       !g_texture_cache->Initialize())
   {
     Shutdown();
diff --git a/Source/Core/VideoCommon/FramebufferShaderGen.cpp b/Source/Core/VideoCommon/FramebufferShaderGen.cpp
index 4917b2dba7..ca00f2c332 100644
--- a/Source/Core/VideoCommon/FramebufferShaderGen.cpp
+++ b/Source/Core/VideoCommon/FramebufferShaderGen.cpp
@@ -14,9 +14,9 @@ static APIType GetAPIType()
 static void EmitUniformBufferDeclaration(std::stringstream& ss)
 {
   if (GetAPIType() == APIType::D3D)
-    ss << "cbuffer UBO : register(b0)\n";
+    ss << "cbuffer PSBlock : register(b0)\n";
   else
-    ss << "UBO_BINDING(std140, 1) uniform UBO\n";
+    ss << "UBO_BINDING(std140, 1) uniform PSBlock\n";
 }
 
 static void EmitSamplerDeclarations(std::stringstream& ss, u32 start = 0, u32 end = 1,
@@ -661,4 +661,43 @@ std::string GenerateEFBRestorePixelShader()
   return ss.str();
 }
 
+std::string GenerateImGuiVertexShader()
+{
+  std::stringstream ss;
+
+  // Uniform buffer contains the viewport size, and we transform in the vertex shader.
+  EmitUniformBufferDeclaration(ss);
+  ss << "{\n";
+  ss << "float2 u_rcp_viewport_size_mul2;\n";
+  ss << "};\n\n";
+
+  EmitVertexMainDeclaration(ss, 1, 1, true, 1, 1);
+  ss << "{\n"
+     << "  v_tex0 = float3(rawtex0.xy, 0.0);\n"
+     << "  v_col0 = rawcolor0;\n"
+     << "  opos = float4(rawpos.x * u_rcp_viewport_size_mul2.x - 1.0,"
+     << "                1.0 - rawpos.y * u_rcp_viewport_size_mul2.y, 0.0, 1.0);\n";
+
+  // NDC space is flipped in Vulkan.
+  if (GetAPIType() == APIType::Vulkan)
+    ss << "  opos.y = -opos.y;\n";
+
+  ss << "}\n";
+  return ss.str();
+}
+
+std::string GenerateImGuiPixelShader()
+{
+  std::stringstream ss;
+  EmitSamplerDeclarations(ss, 0, 1, false);
+  EmitPixelMainDeclaration(ss, 1, 1);
+  ss << "{\n";
+  ss << "  ocol0 = ";
+  EmitSampleTexture(ss, 0, "float3(v_tex0.xy, 0.0)");
+  ss << " * v_col0;\n";
+  ss << "}\n";
+
+  return ss.str();
+}
+
 }  // namespace FramebufferShaderGen
diff --git a/Source/Core/VideoCommon/FramebufferShaderGen.h b/Source/Core/VideoCommon/FramebufferShaderGen.h
index 2ec50b4d76..c6186a4aa3 100644
--- a/Source/Core/VideoCommon/FramebufferShaderGen.h
+++ b/Source/Core/VideoCommon/FramebufferShaderGen.h
@@ -31,5 +31,7 @@ std::string GenerateColorPixelShader();
 std::string GenerateFormatConversionShader(EFBReinterpretType convtype, u32 samples);
 std::string GenerateTextureReinterpretShader(TextureFormat from_format, TextureFormat to_format);
 std::string GenerateEFBRestorePixelShader();
+std::string GenerateImGuiVertexShader();
+std::string GenerateImGuiPixelShader();
 
 }  // namespace FramebufferShaderGen
diff --git a/Source/Core/VideoCommon/PostProcessing.cpp b/Source/Core/VideoCommon/PostProcessing.cpp
index 859715542b..23c0ded751 100644
--- a/Source/Core/VideoCommon/PostProcessing.cpp
+++ b/Source/Core/VideoCommon/PostProcessing.cpp
@@ -441,8 +441,8 @@ std::string PostProcessing::GetUniformBufferHeader() const
   ss << "  float4 resolution;\n";
   ss << "  float4 window_resolution;\n";
   ss << "  float4 src_rect;\n";
+  ss << "  int src_layer;\n";
   ss << "  uint time;\n";
-  ss << "  int layer;\n";
   for (u32 i = 0; i < 2; i++)
     ss << "  uint ubo_align_" << unused_counter++ << "_;\n";
   ss << "\n";
@@ -499,7 +499,18 @@ std::string PostProcessing::GetHeader() const
   else
   {
     ss << "SAMPLER_BINDING(0) uniform sampler2DArray samp0;\n";
-    ss << "VARYING_LOCATION(0) in float3 v_tex0;\n";
+
+    if (g_ActiveConfig.backend_info.bSupportsGeometryShaders)
+    {
+      ss << "VARYING_LOCATION(0) in VertexData {\n";
+      ss << "  float3 v_tex0;\n";
+      ss << "};\n";
+    }
+    else
+    {
+      ss << "VARYING_LOCATION(0) in float3 v_tex0;\n";
+    }
+
     ss << "FRAGMENT_OUTPUT_LOCATION(0) out float4 ocol0;\n";
   }
 
@@ -518,10 +529,10 @@ static float4 ocol0;
   }
 
   ss << R"(
-float4 Sample() { return texture(samp0, float3(v_tex0.xy, float(layer))); }
-float4 SampleLocation(float2 location) { return texture(samp0, float3(location, float(layer))); }
+float4 Sample() { return texture(samp0, v_tex0); }
+float4 SampleLocation(float2 location) { return texture(samp0, float3(location, float(v_tex0.z))); }
 float4 SampleLayer(int layer) { return texture(samp0, float3(v_tex0.xy, float(layer))); }
-#define SampleOffset(offset) textureOffset(samp0, float3(v_tex0.xy, float(layer)), offset)
+#define SampleOffset(offset) textureOffset(samp0, v_tex0, offset)
 
 float2 GetWindowResolution()
 {
@@ -543,6 +554,11 @@ float2 GetCoordinates()
   return v_tex0.xy;
 }
 
+float GetLayer()
+{
+  return v_tex0.z;
+}
+
 uint GetTime()
 {
   return time;
@@ -592,14 +608,24 @@ bool PostProcessing::CompileVertexShader()
   }
   else
   {
-    ss << "VARYING_LOCATION(0) out float3 v_tex0;\n";
+    if (g_ActiveConfig.backend_info.bSupportsGeometryShaders)
+    {
+      ss << "VARYING_LOCATION(0) out VertexData {\n";
+      ss << "  float3 v_tex0;\n";
+      ss << "};\n";
+    }
+    else
+    {
+      ss << "VARYING_LOCATION(0) out float3 v_tex0;\n";
+    }
+
     ss << "#define id gl_VertexID\n";
     ss << "#define opos gl_Position\n";
     ss << "void main() {\n";
   }
   ss << "  v_tex0 = float3(float((id << 1) & 2), float(id & 2), 0.0f);\n";
   ss << "  opos = float4(v_tex0.xy * float2(2.0f, -2.0f) + float2(-1.0f, 1.0f), 0.0f, 1.0f);\n";
-  ss << "  v_tex0 = float3(src_rect.xy + (src_rect.zw * v_tex0.xy), 0.0f);\n";
+  ss << "  v_tex0 = float3(src_rect.xy + (src_rect.zw * v_tex0.xy), float(src_layer));\n";
 
   if (g_ActiveConfig.backend_info.api_type == APIType::Vulkan)
     ss << "  opos.y = -opos.y;\n";
@@ -621,8 +647,8 @@ struct BuiltinUniforms
   float resolution[4];
   float window_resolution[4];
   float src_rect[4];
-  s32 time;
-  u32 layer;
+  s32 src_layer;
+  u32 time;
   u32 padding[2];
 };
 
@@ -646,8 +672,8 @@ void PostProcessing::FillUniformBuffer(const MathUtil::Rectangle<int>& src,
       {static_cast<float>(src.left) * rcp_src_width, static_cast<float>(src.top) * rcp_src_height,
        static_cast<float>(src.GetWidth()) * rcp_src_width,
        static_cast<float>(src.GetHeight()) * rcp_src_height},
-      static_cast<s32>(m_timer.GetTimeElapsed()),
-      static_cast<u32>(src_layer),
+      static_cast<s32>(src_layer),
+      static_cast<u32>(m_timer.GetTimeElapsed()),
   };
 
   u8* buf = m_uniform_staging_buffer.data();
@@ -716,9 +742,8 @@ bool PostProcessing::CompilePipeline()
 {
   AbstractPipelineConfig config = {};
   config.vertex_shader = m_vertex_shader.get();
-  config.geometry_shader = g_ActiveConfig.stereo_mode == StereoMode::QuadBuffer ?
-                               g_shader_cache->GetTexcoordGeometryShader() :
-                               nullptr;
+  config.geometry_shader =
+      g_renderer->UseGeometryShaderForUI() ? g_shader_cache->GetTexcoordGeometryShader() : nullptr;
   config.pixel_shader = m_pixel_shader.get();
   config.rasterization_state = RenderState::GetNoCullRasterizationState(PrimitiveType::Triangles);
   config.depth_state = RenderState::GetNoDepthTestingDepthState();
diff --git a/Source/Core/VideoCommon/RenderBase.cpp b/Source/Core/VideoCommon/RenderBase.cpp
index 31d3192933..8559ca86b6 100644
--- a/Source/Core/VideoCommon/RenderBase.cpp
+++ b/Source/Core/VideoCommon/RenderBase.cpp
@@ -59,6 +59,7 @@
 #include "VideoCommon/FPSCounter.h"
 #include "VideoCommon/FrameDump.h"
 #include "VideoCommon/FramebufferManager.h"
+#include "VideoCommon/FramebufferShaderGen.h"
 #include "VideoCommon/ImageWrite.h"
 #include "VideoCommon/NetPlayChatUI.h"
 #include "VideoCommon/NetPlayGolfUI.h"
@@ -442,6 +443,13 @@ void Renderer::CheckForConfigChanges()
   // Notify the backend of the changes, if any.
   OnConfigChanged(changed_bits);
 
+  // If there's any shader changes, wait for the GPU to finish before destroying anything.
+  if (changed_bits & (CONFIG_CHANGE_BIT_HOST_CONFIG | CONFIG_CHANGE_BIT_MULTISAMPLES))
+  {
+    WaitForGPUIdle();
+    SetPipeline(nullptr);
+  }
+
   // Framebuffer changed?
   if (changed_bits & (CONFIG_CHANGE_BIT_MULTISAMPLES | CONFIG_CHANGE_BIT_STEREO_MODE |
                       CONFIG_CHANGE_BIT_TARGET_SIZE))
@@ -453,8 +461,6 @@ void Renderer::CheckForConfigChanges()
   if (changed_bits & (CONFIG_CHANGE_BIT_HOST_CONFIG | CONFIG_CHANGE_BIT_MULTISAMPLES))
   {
     OSD::AddMessage("Video config changed, reloading shaders.", OSD::Duration::NORMAL);
-    WaitForGPUIdle();
-    SetPipeline(nullptr);
     g_vertex_manager->InvalidatePipelineObject();
     g_shader_cache->SetHostConfig(new_host_config);
     g_shader_cache->Reload();
@@ -467,6 +473,14 @@ void Renderer::CheckForConfigChanges()
     BPFunctions::SetViewport();
     BPFunctions::SetScissor();
   }
+
+  // Stereo mode change requires recompiling our post processing pipeline and imgui pipelines for
+  // rendering the UI.
+  if (changed_bits & CONFIG_CHANGE_BIT_STEREO_MODE)
+  {
+    RecompileImGuiPipeline();
+    m_post_processor->RecompilePipeline();
+  }
 }
 
 // Create On-Screen-Messages
@@ -896,90 +910,6 @@ void Renderer::RecordVideoMemory()
                                              texMem);
 }
 
-static std::string GenerateImGuiVertexShader()
-{
-  const APIType api_type = g_ActiveConfig.backend_info.api_type;
-  std::stringstream ss;
-
-  // Uniform buffer contains the viewport size, and we transform in the vertex shader.
-  if (api_type == APIType::D3D)
-    ss << "cbuffer PSBlock : register(b0) {\n";
-  else if (api_type == APIType::OpenGL)
-    ss << "UBO_BINDING(std140, 1) uniform PSBlock {\n";
-  else if (api_type == APIType::Vulkan)
-    ss << "UBO_BINDING(std140, 1) uniform PSBlock {\n";
-  ss << "float2 u_rcp_viewport_size_mul2;\n";
-  ss << "};\n";
-
-  if (api_type == APIType::D3D)
-  {
-    ss << "void main(in float2 rawpos : POSITION,\n"
-       << "          in float2 rawtex0 : TEXCOORD,\n"
-       << "          in float4 rawcolor0 : COLOR,\n"
-       << "          out float2 frag_uv : TEXCOORD,\n"
-       << "          out float4 frag_color : COLOR,\n"
-       << "          out float4 out_pos : SV_Position)\n";
-  }
-  else
-  {
-    ss << "ATTRIBUTE_LOCATION(" << SHADER_POSITION_ATTRIB << ") in float2 rawpos;\n"
-       << "ATTRIBUTE_LOCATION(" << SHADER_TEXTURE0_ATTRIB << ") in float2 rawtex0;\n"
-       << "ATTRIBUTE_LOCATION(" << SHADER_COLOR0_ATTRIB << ") in float4 rawcolor0;\n"
-       << "VARYING_LOCATION(0) out float2 frag_uv;\n"
-       << "VARYING_LOCATION(1) out float4 frag_color;\n"
-       << "void main()\n";
-  }
-
-  ss << "{\n"
-     << "  frag_uv = rawtex0;\n"
-     << "  frag_color = rawcolor0;\n";
-
-  ss << "  " << (api_type == APIType::D3D ? "out_pos" : "gl_Position")
-     << "= float4(rawpos.x * u_rcp_viewport_size_mul2.x - 1.0, 1.0 - rawpos.y * "
-        "u_rcp_viewport_size_mul2.y, 0.0, 1.0);\n";
-
-  // Clip-space is flipped in Vulkan
-  if (api_type == APIType::Vulkan)
-    ss << "  gl_Position.y = -gl_Position.y;\n";
-
-  ss << "}\n";
-  return ss.str();
-}
-
-static std::string GenerateImGuiPixelShader()
-{
-  const APIType api_type = g_ActiveConfig.backend_info.api_type;
-
-  std::stringstream ss;
-  if (api_type == APIType::D3D)
-  {
-    ss << "Texture2DArray tex0 : register(t0);\n"
-       << "SamplerState samp0 : register(s0);\n"
-       << "void main(in float2 frag_uv : TEXCOORD,\n"
-       << "          in float4 frag_color : COLOR,\n"
-       << "          out float4 ocol0 : SV_Target)\n";
-  }
-  else
-  {
-    ss << "SAMPLER_BINDING(0) uniform sampler2DArray samp0;\n"
-       << "VARYING_LOCATION(0) in float2 frag_uv; \n"
-       << "VARYING_LOCATION(1) in float4 frag_color;\n"
-       << "FRAGMENT_OUTPUT_LOCATION(0) out float4 ocol0;\n"
-       << "void main()\n";
-  }
-
-  ss << "{\n";
-
-  if (api_type == APIType::D3D)
-    ss << "  ocol0 = tex0.Sample(samp0, float3(frag_uv, 0.0)) * frag_color;\n";
-  else
-    ss << "  ocol0 = texture(samp0, float3(frag_uv, 0.0)) * frag_color;\n";
-
-  ss << "}\n";
-
-  return ss.str();
-}
-
 bool Renderer::InitializeImGui()
 {
   if (!ImGui::CreateContext())
@@ -1007,21 +937,66 @@ bool Renderer::InitializeImGui()
     return false;
   }
 
-  const std::string vertex_shader_source = GenerateImGuiVertexShader();
-  const std::string pixel_shader_source = GenerateImGuiPixelShader();
-  std::unique_ptr<AbstractShader> vertex_shader =
-      CreateShaderFromSource(ShaderStage::Vertex, vertex_shader_source);
+  // Font texture(s).
+  {
+    ImGuiIO& io = ImGui::GetIO();
+    u8* font_tex_pixels;
+    int font_tex_width, font_tex_height;
+    io.Fonts->GetTexDataAsRGBA32(&font_tex_pixels, &font_tex_width, &font_tex_height);
+
+    TextureConfig font_tex_config(font_tex_width, font_tex_height, 1, 1, 1,
+                                  AbstractTextureFormat::RGBA8, 0);
+    std::unique_ptr<AbstractTexture> font_tex = CreateTexture(font_tex_config);
+    if (!font_tex)
+    {
+      PanicAlert("Failed to create imgui texture");
+      return false;
+    }
+    font_tex->Load(0, font_tex_width, font_tex_height, font_tex_width, font_tex_pixels,
+                   sizeof(u32) * font_tex_width * font_tex_height);
+
+    io.Fonts->TexID = font_tex.get();
+
+    m_imgui_textures.push_back(std::move(font_tex));
+  }
+
+  if (!RecompileImGuiPipeline())
+    return false;
+
+  m_imgui_last_frame_time = Common::Timer::GetTimeUs();
+  BeginImGuiFrame();
+  return true;
+}
+
+bool Renderer::RecompileImGuiPipeline()
+{
+  std::unique_ptr<AbstractShader> vertex_shader = CreateShaderFromSource(
+      ShaderStage::Vertex, FramebufferShaderGen::GenerateImGuiVertexShader());
   std::unique_ptr<AbstractShader> pixel_shader =
-      CreateShaderFromSource(ShaderStage::Pixel, pixel_shader_source);
+      CreateShaderFromSource(ShaderStage::Pixel, FramebufferShaderGen::GenerateImGuiPixelShader());
   if (!vertex_shader || !pixel_shader)
   {
     PanicAlert("Failed to compile imgui shaders");
     return false;
   }
 
+  // GS is used to render the UI to both eyes in stereo modes.
+  std::unique_ptr<AbstractShader> geometry_shader;
+  if (UseGeometryShaderForUI())
+  {
+    geometry_shader = CreateShaderFromSource(
+        ShaderStage::Geometry, FramebufferShaderGen::GeneratePassthroughGeometryShader(1, 1));
+    if (!geometry_shader)
+    {
+      PanicAlert("Failed to compile imgui geometry shader");
+      return false;
+    }
+  }
+
   AbstractPipelineConfig pconfig = {};
   pconfig.vertex_format = m_imgui_vertex_format.get();
   pconfig.vertex_shader = vertex_shader.get();
+  pconfig.geometry_shader = geometry_shader.get();
   pconfig.pixel_shader = pixel_shader.get();
   pconfig.rasterization_state = RenderState::GetNoCullRasterizationState(PrimitiveType::Triangles);
   pconfig.depth_state = RenderState::GetNoDepthTestingDepthState();
@@ -1043,31 +1018,6 @@ bool Renderer::InitializeImGui()
     return false;
   }
 
-  // Font texture(s).
-  {
-    ImGuiIO& io = ImGui::GetIO();
-    u8* font_tex_pixels;
-    int font_tex_width, font_tex_height;
-    io.Fonts->GetTexDataAsRGBA32(&font_tex_pixels, &font_tex_width, &font_tex_height);
-
-    TextureConfig font_tex_config(font_tex_width, font_tex_height, 1, 1, 1,
-                                  AbstractTextureFormat::RGBA8, 0);
-    std::unique_ptr<AbstractTexture> font_tex = CreateTexture(font_tex_config);
-    if (!font_tex)
-    {
-      PanicAlert("Failed to create imgui texture");
-      return false;
-    }
-    font_tex->Load(0, font_tex_width, font_tex_height, font_tex_width, font_tex_pixels,
-                   sizeof(u32) * font_tex_width * font_tex_height);
-
-    io.Fonts->TexID = font_tex.get();
-
-    m_imgui_textures.push_back(std::move(font_tex));
-  }
-
-  m_imgui_last_frame_time = Common::Timer::GetTimeUs();
-  BeginImGuiFrame();
   return true;
 }
 
@@ -1160,6 +1110,15 @@ void Renderer::DrawImGui()
       m_current_framebuffer));
 }
 
+bool Renderer::UseGeometryShaderForUI() const
+{
+  // OpenGL doesn't render to a 2-layer backbuffer like D3D/Vulkan for quad-buffered stereo,
+  // instead drawing twice and the eye selected by glDrawBuffer() (see
+  // OGL::Renderer::RenderXFBToScreen).
+  return g_ActiveConfig.stereo_mode == StereoMode::QuadBuffer &&
+         g_ActiveConfig.backend_info.api_type != APIType::OpenGL;
+}
+
 std::unique_lock<std::mutex> Renderer::GetImGuiLock()
 {
   return std::unique_lock<std::mutex>(m_imgui_mutex);
diff --git a/Source/Core/VideoCommon/RenderBase.h b/Source/Core/VideoCommon/RenderBase.h
index 16ce52aaa4..b323946c5a 100644
--- a/Source/Core/VideoCommon/RenderBase.h
+++ b/Source/Core/VideoCommon/RenderBase.h
@@ -199,9 +199,6 @@ public:
   void SaveScreenshot(std::string filename, bool wait_for_completion);
   void DrawDebugText();
 
-  // ImGui initialization depends on being able to create textures and pipelines, so do it last.
-  bool InitializeImGui();
-
   virtual void ClearScreen(const MathUtil::Rectangle<int>& rc, bool colorEnable, bool alphaEnable,
                            bool zEnable, u32 color, u32 z);
   virtual void ReinterpretPixelData(EFBReinterpretType convtype);
@@ -243,6 +240,10 @@ public:
 
   virtual std::unique_ptr<VideoCommon::AsyncShaderCompiler> CreateAsyncShaderCompiler();
 
+  // Returns true if a layer-expanding geometry shader should be used when rendering the user
+  // interface and final XFB.
+  bool UseGeometryShaderForUI() const;
+
   // Returns a lock for the ImGui mutex, enabling data structures to be modified from outside.
   // Use with care, only non-drawing functions should be called from outside the video thread,
   // as the drawing is tied to a "frame".
@@ -275,6 +276,12 @@ protected:
   void CheckFifoRecording();
   void RecordVideoMemory();
 
+  // ImGui initialization depends on being able to create textures and pipelines, so do it last.
+  bool InitializeImGui();
+
+  // Recompiles ImGui pipeline - call when stereo mode changes.
+  bool RecompileImGuiPipeline();
+
   // Sets up ImGui state for the next frame.
   // This function itself acquires the ImGui lock, so it should not be held.
   void BeginImGuiFrame();