VideoCommon: update custom pipeline action to support a variety of texture samplers (2D, 2D array, and cube maps)

3 files changed, 210 insertions, 150 deletions

diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp
index 0a51bb995e..f457300847 100644
--- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp
+++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.cpp
@@ -11,6 +11,7 @@
 #include "Common/FileUtil.h"
 #include "Common/Logging/Log.h"
 #include "Common/StringUtil.h"
+#include "Common/VariantUtil.h"
 #include "Core/System.h"
 
 #include "VideoCommon/AbstractGfx.h"
@@ -175,20 +176,6 @@ std::vector<std::string> GlobalConflicts(std::string_view source)
   return global_result;
 }
 
-void WriteDefines(ShaderCode* out, const std::vector<std::string>& texture_code_names,
-                  u32 texture_unit)
-{
-  for (std::size_t i = 0; i < texture_code_names.size(); i++)
-  {
-    const auto& code_name = texture_code_names[i];
-    out->Write("#define {}_UNIT_{{0}} {}\n", code_name, texture_unit);
-    out->Write(
-        "#define {0}_COORD_{{0}} float3(data.texcoord[data.texmap_to_texcoord_index[{1}]].xy, "
-        "{2})\n",
-        code_name, texture_unit, i + 1);
-  }
-}
-
 }  // namespace
 
 std::unique_ptr<CustomPipelineAction>
@@ -267,186 +254,251 @@ void CustomPipelineAction::OnDrawStarted(GraphicsModActionData::DrawStarted* dra
   if (!draw_started->custom_pixel_shader) [[unlikely]]
     return;
 
-  if (!m_valid)
+  if (!draw_started->material_uniform_buffer) [[unlikely]]
     return;
 
   if (m_passes.empty()) [[unlikely]]
     return;
 
-  // For now assume a single pass
-  auto& pass = m_passes[0];
-
-  if (!pass.m_pixel_shader.m_asset) [[unlikely]]
-    return;
-
-  const auto shader_data = pass.m_pixel_shader.m_asset->GetData();
-  if (shader_data)
-  {
-    if (m_last_generated_shader_code.GetBuffer().empty())
-    {
-      // Calculate shader details
-      std::string color_shader_data =
-          ReplaceAll(shader_data->m_shader_source, "custom_main", CUSTOM_PIXELSHADER_COLOR_FUNC);
-      const auto global_conflicts = GlobalConflicts(color_shader_data);
-      color_shader_data = ReplaceAll(color_shader_data, "\r\n", "\n");
-      color_shader_data = ReplaceAll(color_shader_data, "{", "{{");
-      color_shader_data = ReplaceAll(color_shader_data, "}", "}}");
-      // First replace global conflicts with dummy strings
-      // This avoids the problem where a shorter word
-      // is in a longer word, ex two functions:  'execute' and 'execute_fast'
-      for (std::size_t i = 0; i < global_conflicts.size(); i++)
-      {
-        const std::string& identifier = global_conflicts[i];
-        color_shader_data =
-            ReplaceAll(color_shader_data, identifier, fmt::format("_{0}_DOLPHIN_TEMP_{0}_", i));
-      }
-      // Now replace the temporaries with the actual value
-      for (std::size_t i = 0; i < global_conflicts.size(); i++)
-      {
-        const std::string& identifier = global_conflicts[i];
-        color_shader_data = ReplaceAll(color_shader_data, fmt::format("_{0}_DOLPHIN_TEMP_{0}_", i),
-                                       fmt::format("{}_{{0}}", identifier));
-      }
-
-      for (const auto& texture_code_name : m_texture_code_names)
-      {
-        color_shader_data =
-            ReplaceAll(color_shader_data, fmt::format("{}_COORD", texture_code_name),
-                       fmt::format("{}_COORD_{{0}}", texture_code_name));
-        color_shader_data = ReplaceAll(color_shader_data, fmt::format("{}_UNIT", texture_code_name),
-                                       fmt::format("{}_UNIT_{{0}}", texture_code_name));
-      }
-
-      WriteDefines(&m_last_generated_shader_code, m_texture_code_names, draw_started->texture_unit);
-      m_last_generated_shader_code.Write("{}", color_shader_data);
-    }
-    CustomPixelShader custom_pixel_shader;
-    custom_pixel_shader.custom_shader = m_last_generated_shader_code.GetBuffer();
-    *draw_started->custom_pixel_shader = custom_pixel_shader;
-  }
-}
-
-void CustomPipelineAction::OnTextureCreate(GraphicsModActionData::TextureCreate* create)
-{
-  if (!create->custom_textures) [[unlikely]]
-    return;
-
-  if (!create->additional_dependencies) [[unlikely]]
-    return;
-
-  if (m_passes_config.empty()) [[unlikely]]
-    return;
-
-  if (m_passes.empty()) [[unlikely]]
-    return;
-
-  m_valid = true;
   auto& loader = Core::System::GetInstance().GetCustomAssetLoader();
 
   // For now assume a single pass
   const auto& pass_config = m_passes_config[0];
   auto& pass = m_passes[0];
 
-  if (!pass.m_pixel_material.m_asset)
+  if (!pass.m_pixel_material.m_asset ||
+      pass_config.m_pixel_material_asset != pass.m_pixel_material.m_asset->GetAssetId())
   {
     pass.m_pixel_material.m_asset =
         loader.LoadMaterial(pass_config.m_pixel_material_asset, m_library);
-    pass.m_pixel_material.m_cached_write_time = pass.m_pixel_material.m_asset->GetLastLoadedTime();
   }
-  create->additional_dependencies->push_back(VideoCommon::CachedAsset<VideoCommon::CustomAsset>{
-      pass.m_pixel_material.m_asset, pass.m_pixel_material.m_asset->GetLastLoadedTime()});
 
   const auto material_data = pass.m_pixel_material.m_asset->GetData();
   if (!material_data)
+  {
     return;
+  }
 
-  if (!pass.m_pixel_shader.m_asset || pass.m_pixel_material.m_asset->GetLastLoadedTime() >
-                                          pass.m_pixel_material.m_cached_write_time)
+  std::size_t max_material_data_size = 0;
+  if (pass.m_pixel_material.m_asset->GetLastLoadedTime() >
+      pass.m_pixel_material.m_cached_write_time)
+  {
+    m_last_generated_material_code = ShaderCode{};
+    pass.m_pixel_material.m_cached_write_time = pass.m_pixel_material.m_asset->GetLastLoadedTime();
+    std::size_t texture_count = 0;
+    for (const auto& property : material_data->properties)
+    {
+      max_material_data_size += VideoCommon::MaterialProperty::GetMemorySize(property);
+      VideoCommon::MaterialProperty::WriteAsShaderCode(m_last_generated_material_code, property);
+      if (auto* texture_asset_id =
+              std::get_if<VideoCommon::CustomAssetLibrary::AssetID>(&property.m_value))
+      {
+        texture_count++;
+      }
+    }
+    m_material_data.resize(max_material_data_size);
+    pass.m_game_textures.resize(texture_count);
+  }
+
+  if (!pass.m_pixel_shader.m_asset ||
+      pass.m_pixel_shader.m_asset->GetLastLoadedTime() > pass.m_pixel_shader.m_cached_write_time ||
+      material_data->shader_asset != pass.m_pixel_shader.m_asset->GetAssetId())
   {
-    m_last_generated_shader_code = ShaderCode{};
     pass.m_pixel_shader.m_asset = loader.LoadPixelShader(material_data->shader_asset, m_library);
     pass.m_pixel_shader.m_cached_write_time = pass.m_pixel_shader.m_asset->GetLastLoadedTime();
+
+    m_last_generated_shader_code = ShaderCode{};
   }
-  create->additional_dependencies->push_back(VideoCommon::CachedAsset<VideoCommon::CustomAsset>{
-      pass.m_pixel_shader.m_asset, pass.m_pixel_shader.m_asset->GetLastLoadedTime()});
 
   const auto shader_data = pass.m_pixel_shader.m_asset->GetData();
   if (!shader_data)
   {
-    m_valid = false;
     return;
   }
 
-  m_texture_code_names.clear();
-  std::vector<VideoCommon::CachedAsset<VideoCommon::GameTextureAsset>> game_assets;
-  for (const auto& property : material_data->properties)
+  if (shader_data->m_properties.size() != material_data->properties.size())
+  {
+    return;
+  }
+
+  u8* material_buffer = m_material_data.data();
+  u32 sampler_index = 8;
+  for (std::size_t index = 0; index < material_data->properties.size(); index++)
   {
+    auto& property = material_data->properties[index];
     const auto shader_it = shader_data->m_properties.find(property.m_code_name);
     if (shader_it == shader_data->m_properties.end())
     {
       ERROR_LOG_FMT(VIDEO,
-                    "Custom pipeline for texture '{}' has material asset '{}' that uses a "
+                    "Custom pipeline, has material asset '{}' that uses a "
                     "code name of '{}' but that can't be found on shader asset '{}'!",
-                    create->texture_name, pass.m_pixel_material.m_asset->GetAssetId(),
-                    property.m_code_name, pass.m_pixel_shader.m_asset->GetAssetId());
-      m_valid = false;
+                    pass.m_pixel_material.m_asset->GetAssetId(), property.m_code_name,
+                    pass.m_pixel_shader.m_asset->GetAssetId());
       return;
     }
 
-    if (auto* value = std::get_if<std::string>(&property.m_value))
+    if (auto* texture_asset_id =
+            std::get_if<VideoCommon::CustomAssetLibrary::AssetID>(&property.m_value))
     {
-      auto asset = loader.LoadGameTexture(*value, m_library);
-      if (asset)
+      if (*texture_asset_id != "")
       {
-        const auto loaded_time = asset->GetLastLoadedTime();
-        game_assets.push_back(
-            VideoCommon::CachedAsset<VideoCommon::GameTextureAsset>{std::move(asset), loaded_time});
-        m_texture_code_names.push_back(property.m_code_name);
-      }
-    }
-  }
-  // Note: we swap here instead of doing a clear + append of the member
-  // variable so that any loaded assets from previous iterations
-  // won't be let go
-  std::swap(pass.m_game_textures, game_assets);
+        auto asset = loader.LoadGameTexture(*texture_asset_id, m_library);
+        if (!asset)
+        {
+          return;
+        }
 
-  for (auto& game_texture : pass.m_game_textures)
-  {
-    if (game_texture.m_asset)
-    {
-      auto data = game_texture.m_asset->GetData();
-      if (data)
-      {
-        if (data->m_texture.m_slices.empty() || data->m_texture.m_slices[0].m_levels.empty())
+        auto& texture_asset = pass.m_game_textures[index];
+        if (!texture_asset ||
+            texture_asset->m_cached_asset.m_asset->GetLastLoadedTime() >
+                texture_asset->m_cached_asset.m_cached_write_time ||
+            *texture_asset_id != texture_asset->m_cached_asset.m_asset->GetAssetId())
+        {
+          if (!texture_asset)
+          {
+            texture_asset = PipelinePass::CachedTextureAsset{};
+          }
+          const auto loaded_time = asset->GetLastLoadedTime();
+          texture_asset->m_cached_asset = VideoCommon::CachedAsset<VideoCommon::GameTextureAsset>{
+              std::move(asset), loaded_time};
+          texture_asset->m_texture.reset();
+
+          if (std::holds_alternative<VideoCommon::ShaderProperty::Sampler2D>(
+                  shader_it->second.m_default))
+          {
+            texture_asset->m_sampler_code =
+                fmt::format("SAMPLER_BINDING({}) uniform sampler2D samp_{};\n", sampler_index,
+                            property.m_code_name);
+            texture_asset->m_define_code = fmt::format("#define HAS_{} 1\n", property.m_code_name);
+          }
+          else if (std::holds_alternative<VideoCommon::ShaderProperty::Sampler2DArray>(
+                       shader_it->second.m_default))
+          {
+            texture_asset->m_sampler_code =
+                fmt::format("SAMPLER_BINDING({}) uniform sampler2DArray samp_{};\n", sampler_index,
+                            property.m_code_name);
+            texture_asset->m_define_code = fmt::format("#define HAS_{} 1\n", property.m_code_name);
+          }
+          else if (std::holds_alternative<VideoCommon::ShaderProperty::SamplerCube>(
+                       shader_it->second.m_default))
+          {
+            texture_asset->m_sampler_code =
+                fmt::format("SAMPLER_BINDING({}) uniform samplerCube samp_{};\n", sampler_index,
+                            property.m_code_name);
+            texture_asset->m_define_code = fmt::format("#define HAS_{} 1\n", property.m_code_name);
+          }
+        }
+
+        const auto texture_data = texture_asset->m_cached_asset.m_asset->GetData();
+        if (!texture_data)
         {
-          ERROR_LOG_FMT(
-              VIDEO,
-              "Custom pipeline for texture '{}' has asset '{}' that does not have any texture data",
-              create->texture_name, game_texture.m_asset->GetAssetId());
-          m_valid = false;
+          return;
         }
-        else if (create->texture_width != data->m_texture.m_slices[0].m_levels[0].width ||
-                 create->texture_height != data->m_texture.m_slices[0].m_levels[0].height)
+
+        if (texture_asset->m_texture)
         {
-          ERROR_LOG_FMT(VIDEO,
-                        "Custom pipeline for texture '{}' has asset '{}' that does not match "
-                        "the width/height of the texture loaded.  Texture {}x{} vs asset {}x{}",
-                        create->texture_name, game_texture.m_asset->GetAssetId(),
-                        create->texture_width, create->texture_height,
-                        data->m_texture.m_slices[0].m_levels[0].width,
-                        data->m_texture.m_slices[0].m_levels[0].height);
-          m_valid = false;
+          g_gfx->SetTexture(sampler_index, texture_asset->m_texture.get());
+          g_gfx->SetSamplerState(sampler_index, texture_data->m_sampler);
         }
+        else
+        {
+          AbstractTextureType texture_usage = AbstractTextureType::Texture_2DArray;
+          if (std::holds_alternative<VideoCommon::ShaderProperty::SamplerCube>(
+                  shader_it->second.m_default))
+          {
+            texture_usage = AbstractTextureType::Texture_CubeMap;
+          }
+          else if (std::holds_alternative<VideoCommon::ShaderProperty::Sampler2D>(
+                       shader_it->second.m_default))
+          {
+            texture_usage = AbstractTextureType::Texture_2D;
+          }
+
+          if (texture_data->m_texture.m_slices.empty() ||
+              texture_data->m_texture.m_slices[0].m_levels.empty())
+          {
+            return;
+          }
+
+          auto& first_slice = texture_data->m_texture.m_slices[0];
+          const TextureConfig texture_config(
+              first_slice.m_levels[0].width, first_slice.m_levels[0].height,
+              static_cast<u32>(first_slice.m_levels.size()),
+              static_cast<u32>(texture_data->m_texture.m_slices.size()), 1,
+              first_slice.m_levels[0].format, 0, texture_usage);
+          texture_asset->m_texture = g_gfx->CreateTexture(
+              texture_config, fmt::format("Custom shader texture '{}'", property.m_code_name));
+          for (std::size_t slice_index = 0; slice_index < texture_data->m_texture.m_slices.size();
+               slice_index++)
+          {
+            auto& slice = texture_data->m_texture.m_slices[slice_index];
+            for (u32 level_index = 0; level_index < static_cast<u32>(slice.m_levels.size());
+                 ++level_index)
+            {
+              auto& level = slice.m_levels[level_index];
+              texture_asset->m_texture->Load(level_index, level.width, level.height,
+                                             level.row_length, level.data.data(), level.data.size(),
+                                             static_cast<u32>(slice_index));
+            }
+          }
+        }
+
+        sampler_index++;
       }
-      else
-      {
-        m_valid = false;
-      }
+    }
+    else
+    {
+      VideoCommon::MaterialProperty::WriteToMemory(material_buffer, property);
     }
   }
 
-  // TODO: compare game textures and shader requirements
+  if (m_last_generated_shader_code.GetBuffer().empty())
+  {
+    // Calculate shader details
+    std::string color_shader_data =
+        ReplaceAll(shader_data->m_shader_source, "custom_main", CUSTOM_PIXELSHADER_COLOR_FUNC);
+    const auto global_conflicts = GlobalConflicts(color_shader_data);
+    color_shader_data = ReplaceAll(color_shader_data, "\r\n", "\n");
+    color_shader_data = ReplaceAll(color_shader_data, "{", "{{");
+    color_shader_data = ReplaceAll(color_shader_data, "}", "}}");
+    // First replace global conflicts with dummy strings
+    // This avoids the problem where a shorter word
+    // is in a longer word, ex two functions:  'execute' and 'execute_fast'
+    for (std::size_t i = 0; i < global_conflicts.size(); i++)
+    {
+      const std::string& identifier = global_conflicts[i];
+      color_shader_data =
+          ReplaceAll(color_shader_data, identifier, fmt::format("_{0}_DOLPHIN_TEMP_{0}_", i));
+    }
+    // Now replace the temporaries with the actual value
+    for (std::size_t i = 0; i < global_conflicts.size(); i++)
+    {
+      const std::string& identifier = global_conflicts[i];
+      color_shader_data = ReplaceAll(color_shader_data, fmt::format("_{0}_DOLPHIN_TEMP_{0}_", i),
+                                     fmt::format("{}_{{0}}", identifier));
+    }
 
-  create->custom_textures->insert(create->custom_textures->end(), pass.m_game_textures.begin(),
-                                  pass.m_game_textures.end());
+    for (const auto& game_texture : pass.m_game_textures)
+    {
+      if (!game_texture)
+        continue;
+
+      m_last_generated_shader_code.Write("{}", game_texture->m_sampler_code);
+      m_last_generated_shader_code.Write("{}", game_texture->m_define_code);
+    }
+
+    for (std::size_t i = 0; i < draw_started->texture_units.size(); i++)
+    {
+      const auto& texture_unit = draw_started->texture_units[i];
+      m_last_generated_shader_code.Write(
+          "#define TEX_COORD{} data.texcoord[data.texmap_to_texcoord_index[{}]].xy\n", i,
+          texture_unit);
+    }
+    m_last_generated_shader_code.Write("{}", color_shader_data);
+  }
+  CustomPixelShader custom_pixel_shader;
+  custom_pixel_shader.custom_shader = m_last_generated_shader_code.GetBuffer();
+  custom_pixel_shader.material_uniform_block = m_last_generated_material_code.GetBuffer();
+  *draw_started->custom_pixel_shader = custom_pixel_shader;
+  *draw_started->material_uniform_buffer = m_material_data;
 }
diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.h b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.h
index 4760da3124..d609f065fb 100644
--- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.h
+++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/Actions/CustomPipelineAction.h
@@ -4,6 +4,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -33,7 +34,6 @@ public:
                        std::vector<PipelinePassPassDescription> pass_descriptions);
   ~CustomPipelineAction();
   void OnDrawStarted(GraphicsModActionData::DrawStarted*) override;
-  void OnTextureCreate(GraphicsModActionData::TextureCreate*) override;
 
 private:
   std::shared_ptr<VideoCommon::CustomAssetLibrary> m_library;
@@ -42,13 +42,20 @@ private:
   {
     VideoCommon::CachedAsset<VideoCommon::MaterialAsset> m_pixel_material;
     VideoCommon::CachedAsset<VideoCommon::PixelShaderAsset> m_pixel_shader;
-    std::vector<VideoCommon::CachedAsset<VideoCommon::GameTextureAsset>> m_game_textures;
+
+    struct CachedTextureAsset
+    {
+      VideoCommon::CachedAsset<VideoCommon::GameTextureAsset> m_cached_asset;
+      std::unique_ptr<AbstractTexture> m_texture;
+      std::string m_sampler_code;
+      std::string m_define_code;
+    };
+    std::vector<std::optional<CachedTextureAsset>> m_game_textures;
   };
   std::vector<PipelinePass> m_passes;
 
   ShaderCode m_last_generated_shader_code;
+  ShaderCode m_last_generated_material_code;
 
-  bool m_valid = true;
-
-  std::vector<std::string> m_texture_code_names;
+  std::vector<u8> m_material_data;
 };
diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h
index 1b9115e6d5..cecd0ce94f 100644
--- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h
+++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/GraphicsModActionData.h
@@ -11,6 +11,7 @@
 
 #include "Common/CommonTypes.h"
 #include "Common/Matrix.h"
+#include "Common/SmallVector.h"
 #include "VideoCommon/Assets/TextureAsset.h"
 #include "VideoCommon/PixelShaderGen.h"
 
@@ -18,7 +19,7 @@ namespace GraphicsModActionData
 {
 struct DrawStarted
 {
-  u32 texture_unit;
+  const Common::SmallVector<u32, 8>& texture_units;
   bool* skip;
   std::optional<CustomPixelShader>* custom_pixel_shader;
   std::span<u8>* material_uniform_buffer;