OGL: implement Bounding Box on systems w/o SSBO

This commit should have zero performance effect if SSBOs are supported.

If they aren't (e.g. on all Macs), this commit alters FramebufferManager
to attach a new stencil buffer and VertexManager to draw to it when
bounding box is active. `BBoxRead` gets the pixel data from the buffer
and dumbly loops through it to find the bounding box.

This patch can run Paper Mario: The Thousand-Year Door at almost full
speed (50–60 FPS) without Dual-Core enabled for all common bounding
box-using actions I tested (going through pipes, Plane Mode, Paper
Mode, Prof. Frankly's gate, combat, walking around the overworld, etc.)
on my computer (macOS 10.12.3, 2.8 GHz Intel Core i7, 16 GB 1600 MHz
DDR3, and Intel Iris 1536 MB).

A few more demanding scenes (e.g. the self-building bridge on the way
to Petalburg) slow to ~15% of their speed without this patch (though
they don't run quite at full speed even on master). The slowdown is
caused almost solely by `glReadPixels` in `OGL::BoundingBox::Get`.

Other implementation ideas:

- Use a stencil buffer that's separate from the depth buffer. This would
  require ARB_texture_stencil8 / OpenGL 4.4, which isn't available on
  macOS.

- Use `glGetTexImage` instead of `glReadPixels`. This is ~5 FPS slower
  on my computer, presumably because it has to transfer the entire
  combined depth-stencil buffer instead of only the stencil data.
  Getting only stencil data from `glGetTexImage` requires
  ARB_texture_stencil8 / OpenGL 4.4, which (again) is not available on
  macOS.

- Don't use a PBO, and use `glReadPixels` synchronously. This has no
  visible performance effect on my computer, and is theoretically
  slower.

1 files changed, 30 insertions, 2 deletions

diff --git a/Source/Core/VideoBackends/OGL/FramebufferManager.cpp b/Source/Core/VideoBackends/OGL/FramebufferManager.cpp
index 001cbb59fc..13b6759a01 100644
--- a/Source/Core/VideoBackends/OGL/FramebufferManager.cpp
+++ b/Source/Core/VideoBackends/OGL/FramebufferManager.cpp
@@ -152,12 +152,13 @@ FramebufferManager::FramebufferManager(int targetWidth, int targetHeight, int ms
 
     m_resolvedColorTexture = CreateTexture(resolvedType, GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE);
     m_resolvedDepthTexture =
-        CreateTexture(resolvedType, GL_DEPTH_COMPONENT32F, GL_DEPTH_COMPONENT, GL_FLOAT);
+        CreateTexture(resolvedType, GL_DEPTH32F_STENCIL8, GL_DEPTH_STENCIL, GL_FLOAT_32_UNSIGNED_INT_24_8_REV);
 
     // Bind resolved textures to resolved framebuffer.
     glGenFramebuffers(m_EFBLayers, m_resolvedFramebuffer.data());
     BindLayeredTexture(m_resolvedColorTexture, m_resolvedFramebuffer, GL_COLOR_ATTACHMENT0, resolvedType);
     BindLayeredTexture(m_resolvedDepthTexture, m_resolvedFramebuffer, GL_DEPTH_ATTACHMENT, resolvedType);
+    BindLayeredTexture(m_resolvedDepthTexture, m_resolvedFramebuffer, GL_STENCIL_ATTACHMENT, resolvedType);
   }
 
   m_efbColor = CreateTexture(m_textureType, GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE);
@@ -172,13 +173,15 @@ FramebufferManager::FramebufferManager(int targetWidth, int targetHeight, int ms
   glGenFramebuffers(m_EFBLayers, m_efbFramebuffer.data());
   BindLayeredTexture(m_efbColor, m_efbFramebuffer, GL_COLOR_ATTACHMENT0, m_textureType);
   BindLayeredTexture(m_efbDepth, m_efbFramebuffer, GL_DEPTH_ATTACHMENT, m_textureType);
+  BindLayeredTexture(m_efbDepth, m_efbFramebuffer, GL_STENCIL_ATTACHMENT, m_textureType);
 
   // EFB framebuffer is currently bound, make sure to clear it before use.
   glViewport(0, 0, m_targetWidth, m_targetHeight);
   glScissor(0, 0, m_targetWidth, m_targetHeight);
   glClearColor(0.f, 0.f, 0.f, 0.f);
   glClearDepthf(1.0f);
-  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+  glClearStencil(0);
+  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 
   // reinterpret pixel format
   const char* vs = m_EFBLayers > 1 ? "void main(void) {\n"
@@ -478,6 +481,24 @@ GLuint FramebufferManager::GetEFBDepthTexture(const EFBRectangle& sourceRc)
   }
 }
 
+void FramebufferManager::ResolveEFBStencilTexture()
+{
+  if (m_msaaSamples <= 1)
+    return;
+
+  // Resolve.
+  for (unsigned int i = 0; i < m_EFBLayers; i++)
+  {
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, m_efbFramebuffer[i]);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, m_resolvedFramebuffer[i]);
+    glBlitFramebuffer(0, 0, m_targetWidth, m_targetHeight, 0, 0, m_targetWidth, m_targetHeight,
+                      GL_STENCIL_BUFFER_BIT, GL_NEAREST);
+  }
+
+  // Return to EFB.
+  glBindFramebuffer(GL_FRAMEBUFFER, m_efbFramebuffer[0]);
+}
+
 void FramebufferManager::CopyToRealXFB(u32 xfbAddr, u32 fbStride, u32 fbHeight,
                                        const EFBRectangle& sourceRc, float Gamma)
 {
@@ -493,6 +514,13 @@ void FramebufferManager::CopyToRealXFB(u32 xfbAddr, u32 fbStride, u32 fbHeight,
                                     sourceRc.GetWidth(), fbStride, fbHeight);
 }
 
+GLuint FramebufferManager::GetResolvedFramebuffer()
+{
+  if (m_msaaSamples <= 1)
+    return m_efbFramebuffer[0];
+  return m_resolvedFramebuffer[0];
+}
+
 void FramebufferManager::SetFramebuffer(GLuint fb)
 {
   glBindFramebuffer(GL_FRAMEBUFFER, fb != 0 ? fb : GetEFBFramebuffer());