From ac9079f2cad9953958112da158b2ffb6c5f2e4a8 Mon Sep 17 00:00:00 2001
From: Tillmann Karras <tilkax@gmail.com>
Date: Sun, 12 Nov 2023 05:47:22 +0000
Subject: VideoCommon: copy software renderer logic for blend mode priorities

I've not tested this on hardware, but it fixes issue 12271 (shadow
people in Deal or No Deal - Special Edition).
---
 Source/Core/VideoCommon/RenderState.cpp | 66 ++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 34 deletions(-)

(limited to 'Source/Core/VideoCommon/RenderState.cpp')

diff --git a/Source/Core/VideoCommon/RenderState.cpp b/Source/Core/VideoCommon/RenderState.cpp
index ec010756ba..89f36f2b18 100644
--- a/Source/Core/VideoCommon/RenderState.cpp
+++ b/Source/Core/VideoCommon/RenderState.cpp
@@ -120,47 +120,45 @@ void BlendingState::Generate(const BPMemory& bp)
   const bool dstalpha = bp.dstalpha.enable && alphaupdate;
   usedualsrc = true;
 
-  // The subtract bit has the highest priority
-  if (bp.blendmode.subtract)
+  if (bp.blendmode.blendenable)
   {
-    blendenable = true;
-    subtractAlpha = subtract = true;
-    srcfactoralpha = srcfactor = SrcBlendFactor::One;
-    dstfactoralpha = dstfactor = DstBlendFactor::One;
-
-    if (dstalpha)
+    if (bp.blendmode.subtract)
     {
-      subtractAlpha = false;
-      srcfactoralpha = SrcBlendFactor::One;
-      dstfactoralpha = DstBlendFactor::Zero;
-    }
-  }
+      blendenable = true;
+      subtractAlpha = subtract = true;
+      srcfactoralpha = srcfactor = SrcBlendFactor::One;
+      dstfactoralpha = dstfactor = DstBlendFactor::One;
 
-  // The blendenable bit has the middle priority
-  else if (bp.blendmode.blendenable)
-  {
-    blendenable = true;
-    srcfactor = bp.blendmode.srcfactor;
-    dstfactor = bp.blendmode.dstfactor;
-    if (!target_has_alpha)
-    {
-      // uses ONE instead of DSTALPHA
-      srcfactor = RemoveDstAlphaUsage(srcfactor);
-      dstfactor = RemoveDstAlphaUsage(dstfactor);
+      if (dstalpha)
+      {
+        subtractAlpha = false;
+        srcfactoralpha = SrcBlendFactor::One;
+        dstfactoralpha = DstBlendFactor::Zero;
+      }
     }
-    // replaces SrcClr with SrcAlpha and DstClr with DstAlpha, it is important to
-    // use the dst function for the src factor and vice versa
-    srcfactoralpha = RemoveDstColorUsage(srcfactor);
-    dstfactoralpha = RemoveSrcColorUsage(dstfactor);
-
-    if (dstalpha)
+    else
     {
-      srcfactoralpha = SrcBlendFactor::One;
-      dstfactoralpha = DstBlendFactor::Zero;
+      blendenable = true;
+      srcfactor = bp.blendmode.srcfactor;
+      dstfactor = bp.blendmode.dstfactor;
+      if (!target_has_alpha)
+      {
+        // uses ONE instead of DSTALPHA
+        srcfactor = RemoveDstAlphaUsage(srcfactor);
+        dstfactor = RemoveDstAlphaUsage(dstfactor);
+      }
+      // replaces SrcClr with SrcAlpha and DstClr with DstAlpha, it is important to
+      // use the dst function for the src factor and vice versa
+      srcfactoralpha = RemoveDstColorUsage(srcfactor);
+      dstfactoralpha = RemoveSrcColorUsage(dstfactor);
+
+      if (dstalpha)
+      {
+        srcfactoralpha = SrcBlendFactor::One;
+        dstfactoralpha = DstBlendFactor::Zero;
+      }
     }
   }
-
-  // The logicop bit has the lowest priority
   else if (bp.blendmode.logicopenable)
   {
     if (bp.blendmode.logicmode == LogicOp::NoOp)
-- 
cgit v1.2.3