Fix EFB format change emulation in D3D11 if MSAA is enabled.

Fixes issue 4346.

git-svn-id: https://dolphin-emu.googlecode.com/svn/trunk@7496 8ced0084-cf51-0410-be5f-012b33b47a6e

3 files changed, 137 insertions, 62 deletions

diff --git a/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.cpp b/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.cpp
index d5b1628f11..d1b20cd3a4 100644
--- a/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.cpp
+++ b/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.cpp
@@ -48,8 +48,8 @@ SharedPtr<ID3D11PixelShader> s_ColorMatrixProgram[2];
 SharedPtr<ID3D11PixelShader> s_ColorCopyProgram[2];
 SharedPtr<ID3D11PixelShader> s_DepthMatrixProgram[2];
 SharedPtr<ID3D11PixelShader> s_ClearProgram;
-SharedPtr<ID3D11PixelShader> s_rgba6_to_rgb8;
-SharedPtr<ID3D11PixelShader> s_rgb8_to_rgba6;
+SharedPtr<ID3D11PixelShader> s_rgba6_to_rgb8[2];
+SharedPtr<ID3D11PixelShader> s_rgb8_to_rgba6[2];
 
 SharedPtr<ID3D11Buffer> pscbuf;
 
@@ -62,6 +62,7 @@ const char clear_program_code[] = {
 	"}\n"
 };
 
+// TODO: Find some way to avoid having separate shaders for non-MSAA and MSAA...
 const char color_copy_program_code[] = {
 	"sampler samp0 : register(s0);\n"
 	"Texture2D Tex0 : register(t0);\n"
@@ -158,66 +159,141 @@ const char depth_matrix_program_msaa[] = {
 	"}\n"
 };
 
-ID3D11PixelShader* PixelShaderCache::ReinterpRGBA6ToRGB8()
+const char reint_rgba6_to_rgb8[] = {
+	"sampler samp0 : register(s0);\n"
+	"Texture2D Tex0 : register(t0);\n"
+	"void main(\n"
+	"	out float4 ocol0 : SV_Target,\n"
+	"	in float4 pos : SV_Position,\n"
+	"	in float2 uv0 : TEXCOORD0)\n"
+	"{\n"
+	"	int4 src6 = round(Tex0.Sample(samp0,uv0) * 63.f);\n"
+	"	int4 dst8;\n"
+	"	dst8.r = (src6.r << 2) | (src6.g >> 4);\n"
+	"	dst8.g = ((src6.g & 0xF) << 4) | (src6.b >> 2);\n"
+	"	dst8.b = ((src6.b & 0x3) << 6) | src6.a;\n"
+	"	dst8.a = 255;\n"
+	"	ocol0 = (float4)dst8 / 255.f;\n"
+	"}"
+};
+
+const char reint_rgba6_to_rgb8_msaa[] = {
+	"sampler samp0 : register(s0);\n"
+	"Texture2DMS<float4, %d> Tex0 : register(t0);\n"
+	"void main(\n"
+	"	out float4 ocol0 : SV_Target,\n"
+	"	in float4 pos : SV_Position,\n"
+	"	in float2 uv0 : TEXCOORD0)\n"
+	"{\n"
+	"	int width, height, samples;\n"
+	"	Tex0.GetDimensions(width, height, samples);\n"
+	"	float4 texcol = 0;\n"
+	"	for(int i = 0; i < samples; ++i)\n"
+	"		texcol += Tex0.Load(int2(uv0.x*(width), uv0.y*(height)), i);\n"
+	"	texcol /= samples;\n"
+	"	int4 src6 = round(texcol * 63.f);\n"
+	"	int4 dst8;\n"
+	"	dst8.r = (src6.r << 2) | (src6.g >> 4);\n"
+	"	dst8.g = ((src6.g & 0xF) << 4) | (src6.b >> 2);\n"
+	"	dst8.b = ((src6.b & 0x3) << 6) | src6.a;\n"
+	"	dst8.a = 255;\n"
+	"	ocol0 = (float4)dst8 / 255.f;\n"
+	"}"
+};
+
+const char reint_rgb8_to_rgba6[] = {
+	"sampler samp0 : register(s0);\n"
+	"Texture2D Tex0 : register(t0);\n"
+	"void main(\n"
+	"	out float4 ocol0 : SV_Target,\n"
+	"	in float4 pos : SV_Position,\n"
+	"	in float2 uv0 : TEXCOORD0)\n"
+	"{\n"
+	"	int4 src8 = round(Tex0.Sample(samp0,uv0) * 255.f);\n"
+	"	int4 dst6;\n"
+	"	dst6.r = src8.r >> 2;\n"
+	"	dst6.g = ((src8.r & 0x3) << 4) | (src8.g >> 4);\n"
+	"	dst6.b = ((src8.g & 0xF) << 2) | (src8.b >> 6);\n"
+	"	dst6.a = src8.b & 0x3F;\n"
+	"	ocol0 = (float4)dst6 / 63.f;\n"
+	"}\n"
+};
+
+const char reint_rgb8_to_rgba6_msaa[] = {
+	"sampler samp0 : register(s0);\n"
+	"Texture2DMS<float4, %d> Tex0 : register(t0);\n"
+	"void main(\n"
+	"	out float4 ocol0 : SV_Target,\n"
+	"	in float4 pos : SV_Position,\n"
+	"	in float2 uv0 : TEXCOORD0)\n"
+	"{\n"
+	"	int width, height, samples;\n"
+	"	Tex0.GetDimensions(width, height, samples);\n"
+	"	float4 texcol = 0;\n"
+	"	for(int i = 0; i < samples; ++i)\n"
+	"		texcol += Tex0.Load(int2(uv0.x*(width), uv0.y*(height)), i);\n"
+	"	texcol /= samples;\n"
+	"	int4 src8 = round(texcol * 255.f);\n"
+	"	int4 dst6;\n"
+	"	dst6.r = src8.r >> 2;\n"
+	"	dst6.g = ((src8.r & 0x3) << 4) | (src8.g >> 4);\n"
+	"	dst6.b = ((src8.g & 0xF) << 2) | (src8.b >> 6);\n"
+	"	dst6.a = src8.b & 0x3F;\n"
+	"	ocol0 = (float4)dst6 / 63.f;\n"
+	"}\n"
+};
+
+
+ID3D11PixelShader* PixelShaderCache::ReinterpRGBA6ToRGB8(bool multisampled)
 {
-	// TODO: MSAA support..
-	const char code[] = {
-		"sampler samp0 : register(s0);\n"
-		"Texture2D Tex0 : register(t0);\n"
-		"void main(\n"
-		"	out float4 ocol0 : SV_Target,\n"
-		"	in float4 pos : SV_Position,\n"
-		"	in float2 uv0 : TEXCOORD0)\n"
-		"{\n"
-		"	int4 src6 = round(Tex0.Sample(samp0,uv0) * 63.f);\n"
-		"	int4 dst8;\n"
-		"	dst8.r = (src6.r << 2) | (src6.g >> 4);\n"
-		"	dst8.g = ((src6.g & 0xF) << 4) | (src6.b >> 2);\n"
-		"	dst8.b = ((src6.b & 0x3) << 6) | src6.a;\n"
-		"	dst8.a = 255;\n"
-		"	ocol0 = (float4)dst8 / 255.f;\n"
-		"}"
-	};
-
-	if (!s_rgba6_to_rgb8)
+	if (!multisampled || D3D::GetAAMode(g_ActiveConfig.iMultisampleMode).Count == 1)
 	{
-		s_rgba6_to_rgb8 = D3D::CompileAndCreatePixelShader(code, sizeof(code));
-		CHECK(s_rgba6_to_rgb8, "Create RGBA6 to RGB8 pixel shader");
-		D3D::SetDebugObjectName(s_rgba6_to_rgb8, "RGBA6 to RGB8 pixel shader");
+		if (!s_rgba6_to_rgb8[0])
+		{
+			s_rgba6_to_rgb8[0] = D3D::CompileAndCreatePixelShader(reint_rgba6_to_rgb8, sizeof(reint_rgba6_to_rgb8));
+			CHECK(s_rgba6_to_rgb8[0], "Create RGBA6 to RGB8 pixel shader");
+			D3D::SetDebugObjectName(s_rgba6_to_rgb8[0], "RGBA6 to RGB8 pixel shader");
+		}
+		return s_rgba6_to_rgb8[0];
 	}
-	
-	return s_rgba6_to_rgb8;
+	else if (!s_rgba6_to_rgb8[1])
+	{
+		// create MSAA shader for current AA mode
+		char buf[1024];
+		const int l = sprintf_s(buf, 1024, reint_rgba6_to_rgb8_msaa, D3D::GetAAMode(g_ActiveConfig.iMultisampleMode));
+
+		s_rgba6_to_rgb8[1] = D3D::CompileAndCreatePixelShader(buf, l);
+
+		CHECK(s_rgba6_to_rgb8[1], "Create RGBA6 to RGB8 MSAA pixel shader");
+		D3D::SetDebugObjectName(s_rgba6_to_rgb8[1], "RGBA6 to RGB8 MSAA pixel shader");
+	}
+	return s_rgba6_to_rgb8[1];
 }
 
-ID3D11PixelShader* PixelShaderCache::ReinterpRGB8ToRGBA6()
+ID3D11PixelShader* PixelShaderCache::ReinterpRGB8ToRGBA6(bool multisampled)
 {
-	// TODO: MSAA support..
-	const char code[] = {
-		"sampler samp0 : register(s0);\n"
-		"Texture2D Tex0 : register(t0);\n"
-		"void main(\n"
-		"	out float4 ocol0 : SV_Target,\n"
-		"	in float4 pos : SV_Position,\n"
-		"	in float2 uv0 : TEXCOORD0)\n"
-		"{\n"
-		"	int4 src8 = round(Tex0.Sample(samp0,uv0) * 255.f);\n"
-		"	int4 dst6;\n"
-		"	dst6.r = src8.r >> 2;\n"
-		"	dst6.g = ((src8.r & 0x3) << 4) | (src8.g >> 4);\n"
-		"	dst6.b = ((src8.g & 0xF) << 2) | (src8.b >> 6);\n"
-		"	dst6.a = src8.b & 0x3F;\n"
-		"	ocol0 = (float4)dst6 / 63.f;\n"
-		"}\n"
-	};
-
-	if (!s_rgb8_to_rgba6)
+	if (!multisampled || D3D::GetAAMode(g_ActiveConfig.iMultisampleMode).Count == 1)
 	{
-		s_rgb8_to_rgba6 = D3D::CompileAndCreatePixelShader(code, sizeof(code));
-		CHECK(s_rgb8_to_rgba6, "Create RGB8 to RGBA6 pixel shader");
-		D3D::SetDebugObjectName(s_rgb8_to_rgba6, "RGB8 to RGBA6 pixel shader");
+		if (!s_rgb8_to_rgba6[0])
+		{
+			s_rgb8_to_rgba6[0] = D3D::CompileAndCreatePixelShader(reint_rgb8_to_rgba6, sizeof(reint_rgb8_to_rgba6));
+			CHECK(s_rgb8_to_rgba6[0], "Create RGB8 to RGBA6 pixel shader");
+			D3D::SetDebugObjectName(s_rgb8_to_rgba6[0], "RGB8 to RGBA6 pixel shader");
+		}
+		return s_rgb8_to_rgba6[0];
 	}
+	else if (!s_rgb8_to_rgba6[1])
+	{
+		// create MSAA shader for current AA mode
+		char buf[1024];
+		const int l = sprintf_s(buf, 1024, reint_rgb8_to_rgba6_msaa, D3D::GetAAMode(g_ActiveConfig.iMultisampleMode));
 
-	return s_rgb8_to_rgba6;
+		s_rgb8_to_rgba6[1] = D3D::CompileAndCreatePixelShader(buf, l);
+
+		CHECK(s_rgb8_to_rgba6[1], "Create RGB8 to RGBA6 MSAA pixel shader");
+		D3D::SetDebugObjectName(s_rgb8_to_rgba6[1], "RGB8 to RGBA6 MSAA pixel shader");
+	}
+	return s_rgb8_to_rgba6[1];
 }
 
 ID3D11PixelShader* PixelShaderCache::GetColorCopyProgram(bool multisampled)
@@ -228,7 +304,6 @@ ID3D11PixelShader* PixelShaderCache::GetColorCopyProgram(bool multisampled)
 	}
 	else if (!s_ColorCopyProgram[1])
 	{
-		// TODO: recreate shader on AA mode change!
 		// create MSAA shader for current AA mode
 		char buf[1024];
 		const int l = sprintf_s(buf, 1024, color_copy_program_code_msaa, D3D::GetAAMode(g_ActiveConfig.iMultisampleMode));
@@ -250,7 +325,6 @@ ID3D11PixelShader* PixelShaderCache::GetColorMatrixProgram(bool multisampled)
 	}
 	else if (!s_ColorMatrixProgram[1])
 	{
-		// TODO: recreate shader on AA mode change!
 		// create MSAA shader for current AA mode
 		char buf[1024];
 		const int l = sprintf_s(buf, 1024, color_matrix_program_code_msaa, D3D::GetAAMode(g_ActiveConfig.iMultisampleMode));
@@ -272,7 +346,6 @@ ID3D11PixelShader* PixelShaderCache::GetDepthMatrixProgram(bool multisampled)
 	}
 	else if (!s_DepthMatrixProgram[1])
 	{
-		// TODO: recreate shader on AA mode change!
 		// create MSAA shader for current AA mode
 		char buf[1024];
 		const int l = sprintf_s(buf, 1024, depth_matrix_program_msaa, D3D::GetAAMode(g_ActiveConfig.iMultisampleMode));
@@ -374,20 +447,22 @@ void PixelShaderCache::InvalidateMSAAShaders()
 	s_ColorCopyProgram[1].reset();
 	s_ColorMatrixProgram[1].reset();
 	s_DepthMatrixProgram[1].reset();
+	s_rgb8_to_rgba6[1].reset();
+	s_rgba6_to_rgb8[1].reset();
 }
 
 void PixelShaderCache::Shutdown()
 {
 	pscbuf.reset();
 
-	s_rgba6_to_rgb8.reset();
-	s_rgb8_to_rgba6.reset();
 	s_ClearProgram.reset();
 	for (int i = 0; i < 2; ++i)
 	{
 		s_ColorCopyProgram[i].reset();
 		s_ColorMatrixProgram[i].reset();
 		s_DepthMatrixProgram[i].reset();
+		s_rgba6_to_rgb8[i].reset();
+		s_rgb8_to_rgba6[i].reset();
 	}
 	
 	Clear();
diff --git a/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.h b/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.h
index 332c307faa..5377ad47a8 100644
--- a/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.h
+++ b/Source/Plugins/Plugin_VideoDX11/Src/PixelShaderCache.h
@@ -44,8 +44,8 @@ public:
 	static ID3D11PixelShader* GetColorCopyProgram(bool multisampled);
 	static ID3D11PixelShader* GetDepthMatrixProgram(bool multisampled);
 	static ID3D11PixelShader* GetClearProgram();
-	static ID3D11PixelShader* ReinterpRGBA6ToRGB8();
-	static ID3D11PixelShader* ReinterpRGB8ToRGBA6();
+	static ID3D11PixelShader* ReinterpRGBA6ToRGB8(bool multisampled);
+	static ID3D11PixelShader* ReinterpRGB8ToRGBA6(bool multisampled);
 
 	static void InvalidateMSAAShaders();
 
diff --git a/Source/Plugins/Plugin_VideoDX11/Src/Render.cpp b/Source/Plugins/Plugin_VideoDX11/Src/Render.cpp
index e03a7d2ea0..418b03ea40 100644
--- a/Source/Plugins/Plugin_VideoDX11/Src/Render.cpp
+++ b/Source/Plugins/Plugin_VideoDX11/Src/Render.cpp
@@ -781,8 +781,8 @@ void Renderer::ReinterpretPixelData(unsigned int convtype)
 	D3D11_RECT source = CD3D11_RECT(0, 0, g_renderer->GetFullTargetWidth(), g_renderer->GetFullTargetHeight());
 
 	ID3D11PixelShader* pixel_shader;
-	if (convtype == 0) pixel_shader = PixelShaderCache::ReinterpRGB8ToRGBA6();
-	else if (convtype == 2) pixel_shader = PixelShaderCache::ReinterpRGBA6ToRGB8();
+	if (convtype == 0) pixel_shader = PixelShaderCache::ReinterpRGB8ToRGBA6(true);
+	else if (convtype == 2) pixel_shader = PixelShaderCache::ReinterpRGBA6ToRGB8(true);
 	else
 	{
 		PanicAlert("Trying to reinterpret pixel data with unsupported conversion type %d", convtype);