Merge pull request #5343 from stenzek/videocommon-states

Move depth, rasterization and sampler states to VideoCommon

7 files changed, 78 insertions, 194 deletions

diff --git a/Source/Core/VideoBackends/D3D/D3DState.cpp b/Source/Core/VideoBackends/D3D/D3DState.cpp
index b21af1f217..56bcb58fa1 100644
--- a/Source/Core/VideoBackends/D3D/D3DState.cpp
+++ b/Source/Core/VideoBackends/D3D/D3DState.cpp
@@ -12,7 +12,7 @@
 
 #include "VideoBackends/D3D/D3DBase.h"
 #include "VideoBackends/D3D/D3DState.h"
-#include "VideoCommon/SamplerCommon.h"
+#include "VideoCommon/VideoConfig.h"
 
 namespace DX11
 {
@@ -265,90 +265,55 @@ void StateManager::SetTextureByMask(u32 textureSlotMask, ID3D11ShaderResourceVie
 
 ID3D11SamplerState* StateCache::Get(SamplerState state)
 {
-  auto it = m_sampler.find(state.packed);
-
+  auto it = m_sampler.find(state.hex);
   if (it != m_sampler.end())
-  {
     return it->second;
-  }
-
-  const unsigned int d3dMipFilters[4] = {
-      TexMode0::TEXF_NONE, TexMode0::TEXF_POINT, TexMode0::TEXF_LINEAR,
-      TexMode0::TEXF_NONE,  // reserved
-  };
-  const D3D11_TEXTURE_ADDRESS_MODE d3dClamps[4] = {
-      D3D11_TEXTURE_ADDRESS_CLAMP, D3D11_TEXTURE_ADDRESS_WRAP, D3D11_TEXTURE_ADDRESS_MIRROR,
-      D3D11_TEXTURE_ADDRESS_WRAP  // reserved
-  };
 
   D3D11_SAMPLER_DESC sampdc = CD3D11_SAMPLER_DESC(CD3D11_DEFAULT());
-
-  unsigned int mip = d3dMipFilters[state.min_filter & 3];
-
-  if (state.max_anisotropy > 1 && !SamplerCommon::IsBpTexMode0PointFiltering(state))
-  {
-    sampdc.Filter = D3D11_FILTER_ANISOTROPIC;
-    sampdc.MaxAnisotropy = (u32)state.max_anisotropy;
-  }
-  else if (state.min_filter & 4)  // linear min filter
+  if (state.mipmap_filter == SamplerState::Filter::Linear)
   {
-    if (state.mag_filter)  // linear mag filter
-    {
-      if (mip == TexMode0::TEXF_NONE)
-        sampdc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT;
-      else if (mip == TexMode0::TEXF_POINT)
-        sampdc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT;
-      else if (mip == TexMode0::TEXF_LINEAR)
-        sampdc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR;
-    }
-    else  // point mag filter
-    {
-      if (mip == TexMode0::TEXF_NONE)
-        sampdc.Filter = D3D11_FILTER_MIN_LINEAR_MAG_MIP_POINT;
-      else if (mip == TexMode0::TEXF_POINT)
-        sampdc.Filter = D3D11_FILTER_MIN_LINEAR_MAG_MIP_POINT;
-      else if (mip == TexMode0::TEXF_LINEAR)
-        sampdc.Filter = D3D11_FILTER_MIN_LINEAR_MAG_POINT_MIP_LINEAR;
-    }
+    if (state.min_filter == SamplerState::Filter::Linear)
+      sampdc.Filter = (state.mag_filter == SamplerState::Filter::Linear) ?
+                          D3D11_FILTER_MIN_MAG_MIP_LINEAR :
+                          D3D11_FILTER_MIN_LINEAR_MAG_POINT_MIP_LINEAR;
+    else
+      sampdc.Filter = (state.mag_filter == SamplerState::Filter::Linear) ?
+                          D3D11_FILTER_MIN_POINT_MAG_MIP_LINEAR :
+                          D3D11_FILTER_MIN_MAG_POINT_MIP_LINEAR;
   }
-  else  // point min filter
+  else
   {
-    if (state.mag_filter)  // linear mag filter
-    {
-      if (mip == TexMode0::TEXF_NONE)
-        sampdc.Filter = D3D11_FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT;
-      else if (mip == TexMode0::TEXF_POINT)
-        sampdc.Filter = D3D11_FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT;
-      else if (mip == TexMode0::TEXF_LINEAR)
-        sampdc.Filter = D3D11_FILTER_MIN_POINT_MAG_MIP_LINEAR;
-    }
-    else  // point mag filter
-    {
-      if (mip == TexMode0::TEXF_NONE)
-        sampdc.Filter = D3D11_FILTER_MIN_MAG_MIP_POINT;
-      else if (mip == TexMode0::TEXF_POINT)
-        sampdc.Filter = D3D11_FILTER_MIN_MAG_MIP_POINT;
-      else if (mip == TexMode0::TEXF_LINEAR)
-        sampdc.Filter = D3D11_FILTER_MIN_MAG_POINT_MIP_LINEAR;
-    }
+    if (state.min_filter == SamplerState::Filter::Linear)
+      sampdc.Filter = (state.mag_filter == SamplerState::Filter::Linear) ?
+                          D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT :
+                          D3D11_FILTER_MIN_LINEAR_MAG_MIP_POINT;
+    else
+      sampdc.Filter = (state.mag_filter == SamplerState::Filter::Linear) ?
+                          D3D11_FILTER_MIN_POINT_MAG_LINEAR_MIP_POINT :
+                          D3D11_FILTER_MIN_MAG_MIP_POINT;
   }
 
-  sampdc.AddressU = d3dClamps[state.wrap_s];
-  sampdc.AddressV = d3dClamps[state.wrap_t];
-
-  sampdc.MaxLOD = SamplerCommon::AreBpTexMode0MipmapsEnabled(state) ? state.max_lod / 16.f : 0.f;
-  sampdc.MinLOD = std::min(state.min_lod / 16.f, sampdc.MaxLOD);
+  static constexpr std::array<D3D11_TEXTURE_ADDRESS_MODE, 3> address_modes = {
+      {D3D11_TEXTURE_ADDRESS_CLAMP, D3D11_TEXTURE_ADDRESS_WRAP, D3D11_TEXTURE_ADDRESS_MIRROR}};
+  sampdc.AddressU = address_modes[static_cast<u32>(state.wrap_u.Value())];
+  sampdc.AddressV = address_modes[static_cast<u32>(state.wrap_v.Value())];
+  sampdc.MaxLOD = state.max_lod / 16.f;
+  sampdc.MinLOD = state.min_lod / 16.f;
   sampdc.MipLODBias = (s32)state.lod_bias / 32.0f;
 
-  ID3D11SamplerState* res = nullptr;
+  if (state.anisotropic_filtering)
+  {
+    sampdc.Filter = D3D11_FILTER_ANISOTROPIC;
+    sampdc.MaxAnisotropy = 1u << g_ActiveConfig.iMaxAnisotropy;
+  }
 
+  ID3D11SamplerState* res = nullptr;
   HRESULT hr = D3D::device->CreateSamplerState(&sampdc, &res);
   if (FAILED(hr))
     PanicAlert("Fail %s %d\n", __FILE__, __LINE__);
 
   D3D::SetDebugObjectName(res, "sampler state used to emulate the GX pipeline");
-  m_sampler.emplace(state.packed, res);
-
+  m_sampler.emplace(state.hex, res);
   return res;
 }
 
@@ -434,32 +399,33 @@ ID3D11BlendState* StateCache::Get(BlendingState state)
   return res;
 }
 
-ID3D11RasterizerState* StateCache::Get(RasterizerState state)
+ID3D11RasterizerState* StateCache::Get(RasterizationState state)
 {
-  auto it = m_raster.find(state.packed);
-
+  auto it = m_raster.find(state.hex);
   if (it != m_raster.end())
     return it->second;
 
-  D3D11_RASTERIZER_DESC rastdc = CD3D11_RASTERIZER_DESC(D3D11_FILL_SOLID, state.cull_mode, false, 0,
-                                                        0.f, 0, false, true, false, false);
+  static constexpr std::array<D3D11_CULL_MODE, 4> cull_modes = {
+      {D3D11_CULL_NONE, D3D11_CULL_BACK, D3D11_CULL_FRONT, D3D11_CULL_BACK}};
 
-  ID3D11RasterizerState* res = nullptr;
+  D3D11_RASTERIZER_DESC desc = {};
+  desc.FillMode = D3D11_FILL_SOLID;
+  desc.CullMode = cull_modes[state.cullmode];
+  desc.ScissorEnable = TRUE;
 
-  HRESULT hr = D3D::device->CreateRasterizerState(&rastdc, &res);
+  ID3D11RasterizerState* res = nullptr;
+  HRESULT hr = D3D::device->CreateRasterizerState(&desc, &res);
   if (FAILED(hr))
     PanicAlert("Failed to create rasterizer state at %s %d\n", __FILE__, __LINE__);
 
   D3D::SetDebugObjectName(res, "rasterizer state used to emulate the GX pipeline");
-  m_raster.emplace(state.packed, res);
-
+  m_raster.emplace(state.hex, res);
   return res;
 }
 
-ID3D11DepthStencilState* StateCache::Get(ZMode state)
+ID3D11DepthStencilState* StateCache::Get(DepthState state)
 {
   auto it = m_depth.find(state.hex);
-
   if (it != m_depth.end())
     return it->second;
 
@@ -472,6 +438,7 @@ ID3D11DepthStencilState* StateCache::Get(ZMode state)
   depthdc.StencilReadMask = D3D11_DEFAULT_STENCIL_READ_MASK;
   depthdc.StencilWriteMask = D3D11_DEFAULT_STENCIL_WRITE_MASK;
 
+  // Less/greater are swapped due to inverted depth.
   const D3D11_COMPARISON_FUNC d3dCmpFuncs[8] = {
       D3D11_COMPARISON_NEVER,         D3D11_COMPARISON_GREATER, D3D11_COMPARISON_EQUAL,
       D3D11_COMPARISON_GREATER_EQUAL, D3D11_COMPARISON_LESS,    D3D11_COMPARISON_NOT_EQUAL,
@@ -531,4 +498,12 @@ void StateCache::Clear()
   m_sampler.clear();
 }
 
+D3D11_PRIMITIVE_TOPOLOGY StateCache::GetPrimitiveTopology(PrimitiveType primitive)
+{
+  static constexpr std::array<D3D11_PRIMITIVE_TOPOLOGY, 4> primitives = {
+      {D3D11_PRIMITIVE_TOPOLOGY_POINTLIST, D3D11_PRIMITIVE_TOPOLOGY_LINELIST,
+       D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST, D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP}};
+  return primitives[static_cast<u32>(primitive)];
+}
+
 }  // namespace DX11
diff --git a/Source/Core/VideoBackends/D3D/D3DState.h b/Source/Core/VideoBackends/D3D/D3DState.h
index 6b3838c661..fc7cda88b3 100644
--- a/Source/Core/VideoBackends/D3D/D3DState.h
+++ b/Source/Core/VideoBackends/D3D/D3DState.h
@@ -20,27 +20,6 @@ struct ID3D11RasterizerState;
 
 namespace DX11
 {
-union RasterizerState
-{
-  BitField<0, 2, D3D11_CULL_MODE> cull_mode;
-
-  u32 packed;
-};
-
-union SamplerState
-{
-  BitField<0, 3, u64> min_filter;
-  BitField<3, 1, u64> mag_filter;
-  BitField<4, 8, u64> min_lod;
-  BitField<12, 8, u64> max_lod;
-  BitField<20, 8, s64> lod_bias;
-  BitField<28, 2, u64> wrap_s;
-  BitField<30, 2, u64> wrap_t;
-  BitField<32, 5, u64> max_anisotropy;
-
-  u64 packed;
-};
-
 class StateCache
 {
 public:
@@ -48,17 +27,20 @@ public:
   // Returned objects is owned by the cache and does not need to be released.
   ID3D11SamplerState* Get(SamplerState state);
   ID3D11BlendState* Get(BlendingState state);
-  ID3D11RasterizerState* Get(RasterizerState state);
-  ID3D11DepthStencilState* Get(ZMode state);
+  ID3D11RasterizerState* Get(RasterizationState state);
+  ID3D11DepthStencilState* Get(DepthState state);
 
   // Release all cached states and clear hash tables.
   void Clear();
 
+  // Convert RasterState primitive type to D3D11 primitive topology.
+  static D3D11_PRIMITIVE_TOPOLOGY GetPrimitiveTopology(PrimitiveType primitive);
+
 private:
   std::unordered_map<u32, ID3D11DepthStencilState*> m_depth;
   std::unordered_map<u32, ID3D11RasterizerState*> m_raster;
   std::unordered_map<u32, ID3D11BlendState*> m_blend;
-  std::unordered_map<u64, ID3D11SamplerState*> m_sampler;
+  std::unordered_map<u32, ID3D11SamplerState*> m_sampler;
 };
 
 namespace D3D
diff --git a/Source/Core/VideoBackends/D3D/GeometryShaderCache.cpp b/Source/Core/VideoBackends/D3D/GeometryShaderCache.cpp
index 5f4072022b..ecb8b7b1bb 100644
--- a/Source/Core/VideoBackends/D3D/GeometryShaderCache.cpp
+++ b/Source/Core/VideoBackends/D3D/GeometryShaderCache.cpp
@@ -207,7 +207,7 @@ void GeometryShaderCache::Shutdown()
   g_gs_disk_cache.Close();
 }
 
-bool GeometryShaderCache::SetShader(u32 primitive_type)
+bool GeometryShaderCache::SetShader(PrimitiveType primitive_type)
 {
   GeometryShaderUid uid = GetGeometryShaderUid(primitive_type);
   if (last_entry && uid == last_uid)
diff --git a/Source/Core/VideoBackends/D3D/GeometryShaderCache.h b/Source/Core/VideoBackends/D3D/GeometryShaderCache.h
index 6bebf0a2a4..fdd3d053f2 100644
--- a/Source/Core/VideoBackends/D3D/GeometryShaderCache.h
+++ b/Source/Core/VideoBackends/D3D/GeometryShaderCache.h
@@ -18,7 +18,7 @@ public:
   static void Reload();
   static void Clear();
   static void Shutdown();
-  static bool SetShader(u32 primitive_type);
+  static bool SetShader(PrimitiveType primitive_type);
   static bool CompileShader(const GeometryShaderUid& uid);
   static bool InsertByteCode(const GeometryShaderUid& uid, const u8* bytecode, size_t len);
   static void PrecompileShaders();
diff --git a/Source/Core/VideoBackends/D3D/Render.cpp b/Source/Core/VideoBackends/D3D/Render.cpp
index ae50045c26..5fce3ab961 100644
--- a/Source/Core/VideoBackends/D3D/Render.cpp
+++ b/Source/Core/VideoBackends/D3D/Render.cpp
@@ -60,8 +60,8 @@ struct GXPipelineState
 {
   std::array<SamplerState, 8> samplers;
   BlendingState blend;
-  ZMode zmode;
-  RasterizerState raster;
+  DepthState zmode;
+  RasterizationState raster;
 };
 
 static u32 s_last_multisamples = 1;
@@ -249,13 +249,12 @@ Renderer::Renderer() : ::Renderer(D3D::GetBackBufferWidth(), D3D::GetBackBufferH
 
   // Setup GX pipeline state
   for (auto& sampler : s_gx_state.samplers)
-    sampler.packed = 0;
+    sampler.hex = RenderState::GetPointSamplerState().hex;
 
   s_gx_state.zmode.testenable = false;
   s_gx_state.zmode.updateenable = false;
   s_gx_state.zmode.func = ZMode::NEVER;
-
-  s_gx_state.raster.cull_mode = D3D11_CULL_NONE;
+  s_gx_state.raster.cullmode = GenMode::CULL_NONE;
 
   // Clear EFB textures
   constexpr std::array<float, 4> clear_color{{0.f, 0.f, 0.f, 1.f}};
@@ -867,14 +866,12 @@ void Renderer::ApplyState()
   D3D::stateman->PushBlendState(s_gx_state_cache.Get(s_gx_state.blend));
   D3D::stateman->PushDepthState(s_gx_state_cache.Get(s_gx_state.zmode));
   D3D::stateman->PushRasterizerState(s_gx_state_cache.Get(s_gx_state.raster));
+  D3D::stateman->SetPrimitiveTopology(
+      StateCache::GetPrimitiveTopology(s_gx_state.raster.primitive));
   FramebufferManager::SetIntegerEFBRenderTarget(s_gx_state.blend.logicopenable);
 
-  for (size_t stage = 0; stage < s_gx_state.samplers.size(); stage++)
-  {
-    // TODO: cache SamplerState directly, not d3d object
-    s_gx_state.samplers[stage].max_anisotropy = UINT64_C(1) << g_ActiveConfig.iMaxAnisotropy;
+  for (u32 stage = 0; stage < static_cast<u32>(s_gx_state.samplers.size()); stage++)
     D3D::stateman->SetSampler(stage, s_gx_state_cache.Get(s_gx_state.samplers[stage]));
-  }
 
   ID3D11Buffer* vertexConstants = VertexShaderCache::GetConstantBuffer();
 
@@ -891,68 +888,19 @@ void Renderer::RestoreState()
   D3D::stateman->PopRasterizerState();
 }
 
-void Renderer::ApplyCullDisable()
-{
-  RasterizerState rast = s_gx_state.raster;
-  rast.cull_mode = D3D11_CULL_NONE;
-
-  ID3D11RasterizerState* raststate = s_gx_state_cache.Get(rast);
-  D3D::stateman->PushRasterizerState(raststate);
-}
-
-void Renderer::RestoreCull()
-{
-  D3D::stateman->PopRasterizerState();
-}
-
-void Renderer::SetGenerationMode()
+void Renderer::SetRasterizationState(const RasterizationState& state)
 {
-  constexpr std::array<D3D11_CULL_MODE, 4> d3d_cull_modes{{
-      D3D11_CULL_NONE, D3D11_CULL_BACK, D3D11_CULL_FRONT, D3D11_CULL_BACK,
-  }};
-
-  // rastdc.FrontCounterClockwise must be false for this to work
-  // TODO: GX_CULL_ALL not supported, yet!
-  s_gx_state.raster.cull_mode = d3d_cull_modes[bpmem.genMode.cullmode];
+  s_gx_state.raster.hex = state.hex;
 }
 
-void Renderer::SetDepthMode()
+void Renderer::SetDepthState(const DepthState& state)
 {
-  s_gx_state.zmode.hex = bpmem.zmode.hex;
+  s_gx_state.zmode.hex = state.hex;
 }
 
-void Renderer::SetSamplerState(int stage, int texindex, bool custom_tex)
+void Renderer::SetSamplerState(u32 index, const SamplerState& state)
 {
-  const FourTexUnits& tex = bpmem.tex[texindex];
-  const TexMode0& tm0 = tex.texMode0[stage];
-  const TexMode1& tm1 = tex.texMode1[stage];
-
-  if (texindex)
-    stage += 4;
-
-  if (g_ActiveConfig.bForceFiltering)
-  {
-    // Only use mipmaps if the game says they are available.
-    s_gx_state.samplers[stage].min_filter = SamplerCommon::AreBpTexMode0MipmapsEnabled(tm0) ? 6 : 4;
-    s_gx_state.samplers[stage].mag_filter = 1;  // linear mag
-  }
-  else
-  {
-    s_gx_state.samplers[stage].min_filter = (u32)tm0.min_filter;
-    s_gx_state.samplers[stage].mag_filter = (u32)tm0.mag_filter;
-  }
-
-  s_gx_state.samplers[stage].wrap_s = (u32)tm0.wrap_s;
-  s_gx_state.samplers[stage].wrap_t = (u32)tm0.wrap_t;
-  s_gx_state.samplers[stage].max_lod = (u32)tm1.max_lod;
-  s_gx_state.samplers[stage].min_lod = (u32)tm1.min_lod;
-  s_gx_state.samplers[stage].lod_bias = (s32)tm0.lod_bias;
-
-  // custom textures may have higher resolution, so disable the max_lod
-  if (custom_tex)
-  {
-    s_gx_state.samplers[stage].max_lod = 255;
-  }
+  s_gx_state.samplers[index].hex = state.hex;
 }
 
 void Renderer::SetInterlacingMode()
diff --git a/Source/Core/VideoBackends/D3D/Render.h b/Source/Core/VideoBackends/D3D/Render.h
index 907c9e2b18..a33eba7cd2 100644
--- a/Source/Core/VideoBackends/D3D/Render.h
+++ b/Source/Core/VideoBackends/D3D/Render.h
@@ -21,9 +21,9 @@ public:
 
   void SetBlendingState(const BlendingState& state) override;
   void SetScissorRect(const EFBRectangle& rc) override;
-  void SetGenerationMode() override;
-  void SetDepthMode() override;
-  void SetSamplerState(int stage, int texindex, bool custom_tex) override;
+  void SetRasterizationState(const RasterizationState& state) override;
+  void SetDepthState(const DepthState& state) override;
+  void SetSamplerState(u32 index, const SamplerState& state) override;
   void SetInterlacingMode() override;
   void SetViewport() override;
   void SetFullscreen(bool enable_fullscreen) override;
@@ -33,9 +33,6 @@ public:
   void ApplyState() override;
   void RestoreState() override;
 
-  void ApplyCullDisable();
-  void RestoreCull();
-
   void RenderText(const std::string& text, int left, int top, u32 color) override;
 
   u32 AccessEFB(EFBAccessType type, u32 x, u32 y, u32 poke_data) override;
diff --git a/Source/Core/VideoBackends/D3D/VertexManager.cpp b/Source/Core/VideoBackends/D3D/VertexManager.cpp
index 4edcf78165..d4aec17bf4 100644
--- a/Source/Core/VideoBackends/D3D/VertexManager.cpp
+++ b/Source/Core/VideoBackends/D3D/VertexManager.cpp
@@ -127,28 +127,10 @@ void VertexManager::Draw(u32 stride)
   u32 baseVertex = m_vertexDrawOffset / stride;
   u32 startIndex = m_indexDrawOffset / sizeof(u16);
 
-  switch (m_current_primitive_type)
-  {
-  case PRIMITIVE_POINTS:
-    D3D::stateman->SetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_POINTLIST);
-    static_cast<Renderer*>(g_renderer.get())->ApplyCullDisable();
-    break;
-  case PRIMITIVE_LINES:
-    D3D::stateman->SetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_LINELIST);
-    static_cast<Renderer*>(g_renderer.get())->ApplyCullDisable();
-    break;
-  case PRIMITIVE_TRIANGLES:
-    D3D::stateman->SetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
-    break;
-  }
-
   D3D::stateman->Apply();
   D3D::context->DrawIndexed(indices, startIndex, baseVertex);
 
   INCSTAT(stats.thisFrame.numDrawCalls);
-
-  if (m_current_primitive_type != PRIMITIVE_TRIANGLES)
-    static_cast<Renderer*>(g_renderer.get())->RestoreCull();
 }
 
 void VertexManager::vFlush()