HiresTextures: Load full mipmap chain from DDS files

This removes the need for multiple texture files to store the mipmap
chain for a texture. As many mipmaps will be loaded as are present in
the DDS file, and any remaining mipmaps will fall back to the old
behavior.

3 files changed, 231 insertions, 80 deletions

diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp
index d087ee8575..02c328c922 100644
--- a/Source/Core/VideoCommon/HiresTextures.cpp
+++ b/Source/Core/VideoCommon/HiresTextures.cpp
@@ -395,71 +395,92 @@ std::shared_ptr<HiresTexture> HiresTexture::Search(const u8* texture, size_t tex
 std::unique_ptr<HiresTexture> HiresTexture::Load(const std::string& base_filename, u32 width,
                                                  u32 height)
 {
-  std::unique_ptr<HiresTexture> ret;
-  for (int level = 0;; level++)
+  // We need to have a level 0 custom texture to even consider loading.
+  auto filename_iter = s_textureMap.find(base_filename);
+  if (filename_iter == s_textureMap.end())
+    return nullptr;
+
+  // Try to load level 0 (and any mipmaps) from a DDS file.
+  // If this fails, it's fine, we'll just load level0 again using SOIL.
+  // Can't use make_unique due to private constructor.
+  std::unique_ptr<HiresTexture> ret = std::unique_ptr<HiresTexture>(new HiresTexture());
+  const std::string& first_mip_filename = filename_iter->second;
+  LoadDDSTexture(ret.get(), first_mip_filename);
+
+  // Load remaining mip levels, or from the start if it's not a DDS texture.
+  for (u32 mip_level = static_cast<u32>(ret->m_levels.size());; mip_level++)
   {
     std::string filename = base_filename;
-    if (level)
-    {
-      filename += StringFromFormat("_mip%u", level);
-    }
+    if (mip_level != 0)
+      filename += StringFromFormat("_mip%u", mip_level);
 
-    auto filename_iter = s_textureMap.find(filename);
-    if (filename_iter != s_textureMap.end())
-    {
-      Level l;
+    filename_iter = s_textureMap.find(filename);
+    if (filename_iter == s_textureMap.end())
+      break;
 
+    // Try loading DDS textures first, that way we maintain compression of DXT formats.
+    // TODO: Reduce the number of open() calls here. We could use one fd.
+    Level level;
+    if (!LoadDDSTexture(level, filename_iter->second))
+    {
       File::IOFile file;
       file.Open(filename_iter->second, "rb");
       std::vector<u8> buffer(file.GetSize());
       file.ReadBytes(buffer.data(), file.GetSize());
-
-      // Try loading DDS textures first, that way we maintain compression of DXT formats.
-      if (!LoadDDSTexture(l, buffer) && !LoadTexture(l, buffer))
+      if (!LoadTexture(level, buffer))
       {
         ERROR_LOG(VIDEO, "Custom texture %s failed to load", filename.c_str());
         break;
       }
+    }
 
-      if (!level)
-      {
-        if (l.width * height != l.height * width)
-          ERROR_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. The aspect differs "
-                           "from the native size %dx%d.",
-                    l.width, l.height, filename.c_str(), width, height);
-        if (width && height && (l.width % width || l.height % height))
-          WARN_LOG(VIDEO, "Invalid custom texture size %dx%d for texture %s. Please use an integer "
-                          "upscaling factor based on the native size %dx%d.",
-                   l.width, l.height, filename.c_str(), width, height);
-        width = l.width;
-        height = l.height;
-      }
-      else if (width != l.width || height != l.height)
-      {
-        ERROR_LOG(
-            VIDEO,
-            "Invalid custom texture size %dx%d for texture %s. This mipmap layer _must_ be %dx%d.",
-            l.width, l.height, filename.c_str(), width, height);
-        l.data.reset();
-        break;
-      }
+    ret->m_levels.push_back(std::move(level));
+  }
 
-      if (!ret)
-        ret = std::unique_ptr<HiresTexture>(new HiresTexture);
-      ret->m_levels.push_back(std::move(l));
+  // If we failed to load any mip levels, we can't use this texture at all.
+  if (ret->m_levels.empty())
+    return nullptr;
 
-      // no more mipmaps available
-      if (width == 1 && height == 1)
-        break;
+  // Verify that the aspect ratio of the texture hasn't changed, as this could have side-effects.
+  const Level& first_mip = ret->m_levels[0];
+  if (first_mip.width * height != first_mip.height * width)
+  {
+    ERROR_LOG(VIDEO, "Invalid custom texture size %ux%u for texture %s. The aspect differs "
+                     "from the native size %ux%u.",
+              first_mip.width, first_mip.height, first_mip_filename.c_str(), width, height);
+  }
 
-      // calculate the size of the next mipmap
-      width = std::max(1u, width >> 1);
-      height = std::max(1u, height >> 1);
-    }
-    else
+  // Same deal if the custom texture isn't a multiple of the native size.
+  if (width != 0 && height != 0 && (first_mip.width % width || first_mip.height % height))
+  {
+    WARN_LOG(VIDEO, "Invalid custom texture size %ux%u for texture %s. Please use an integer "
+                    "upscaling factor based on the native size %ux%u.",
+             first_mip.width, first_mip.height, first_mip_filename.c_str(), width, height);
+  }
+
+  // Verify that each mip level is the correct size (divide by 2 each time).
+  u32 current_mip_width = std::max(first_mip.width / 2, 1u);
+  u32 current_mip_height = std::max(first_mip.height / 2, 1u);
+  for (u32 mip_level = 1; mip_level < static_cast<u32>(ret->m_levels.size()); mip_level++)
+  {
+    const Level& level = ret->m_levels[mip_level];
+    if (current_mip_width == level.width && current_mip_height == level.height)
     {
-      break;
+      current_mip_width = std::max(current_mip_width / 2, 1u);
+      current_mip_height = std::max(current_mip_height / 2, 1u);
+      continue;
     }
+
+    ERROR_LOG(VIDEO,
+              "Invalid custom texture size %dx%d for texture %s. Mipmap level %u _must_ be %dx%d.",
+              level.width, level.height, first_mip_filename.c_str(), mip_level, current_mip_width,
+              current_mip_height);
+
+    // Drop this mip level and any others after it.
+    while (ret->m_levels.size() > mip_level)
+      ret->m_levels.pop_back();
+
+    break;
   }
 
   // All levels have to have the same format.
@@ -467,8 +488,9 @@ std::unique_ptr<HiresTexture> HiresTexture::Load(const std::string& base_filenam
                          [&ret](const Level& l) { return l.format != ret->m_levels[0].format; }))
   {
     ERROR_LOG(VIDEO, "Custom texture %s has inconsistent formats across mip levels.",
-              base_filename.c_str());
-    ret.reset();
+              first_mip_filename.c_str());
+
+    return nullptr;
   }
 
   return ret;
diff --git a/Source/Core/VideoCommon/HiresTextures.h b/Source/Core/VideoCommon/HiresTextures.h
index c11b6f140b..95ad65ce7f 100644
--- a/Source/Core/VideoCommon/HiresTextures.h
+++ b/Source/Core/VideoCommon/HiresTextures.h
@@ -47,7 +47,8 @@ public:
 private:
   static std::unique_ptr<HiresTexture> Load(const std::string& base_filename, u32 width,
                                             u32 height);
-  static bool LoadDDSTexture(Level& level, const std::vector<u8>& buffer);
+  static bool LoadDDSTexture(HiresTexture* tex, const std::string& filename);
+  static bool LoadDDSTexture(Level& level, const std::string& filename);
   static bool LoadTexture(Level& level, const std::vector<u8>& buffer);
   static void Prefetch();
 
diff --git a/Source/Core/VideoCommon/HiresTextures_DDSLoader.cpp b/Source/Core/VideoCommon/HiresTextures_DDSLoader.cpp
index d7e32443a9..8abdffdd0b 100644
--- a/Source/Core/VideoCommon/HiresTextures_DDSLoader.cpp
+++ b/Source/Core/VideoCommon/HiresTextures_DDSLoader.cpp
@@ -8,6 +8,7 @@
 #include <cstdint>
 #include <cstring>
 #include "Common/Align.h"
+#include "Common/FileUtil.h"
 #include "VideoCommon/VideoConfig.h"
 
 namespace
@@ -153,19 +154,49 @@ static_assert(sizeof(DDS_HEADER_DXT10) == 20, "DDS DX10 Extended Header size mis
 
 }  // namespace
 
-bool HiresTexture::LoadDDSTexture(Level& level, const std::vector<u8>& buffer)
+struct DDSLoadInfo
 {
-  u32 magic;
-  std::memcpy(&magic, buffer.data(), sizeof(magic));
+  u32 block_size = 1;
+  u32 bytes_per_block = 4;
+  u32 width = 0;
+  u32 height = 0;
+  u32 mip_count = 0;
+  HostTextureFormat format = HostTextureFormat::RGBA8;
+  size_t first_mip_offset = 0;
+  size_t first_mip_size = 0;
+  u32 first_mip_row_length = 0;
+};
+
+static u32 GetBlockCount(u32 extent, u32 block_size)
+{
+  return std::max(Common::AlignUp(extent, block_size) / block_size, 1u);
+}
 
+static u32 CalculateMipCount(u32 width, u32 height)
+{
+  u32 mip_width = std::max(width / 2, 1u);
+  u32 mip_height = std::max(height / 2, 1u);
+  u32 mip_count = 1;
+  while (mip_width > 1 || mip_height > 1)
+  {
+    mip_width = std::max(mip_width / 2, 1u);
+    mip_height = std::max(mip_height / 2, 1u);
+    mip_count++;
+  }
+
+  return mip_count;
+}
+
+static bool ParseDDSHeader(File::IOFile& file, DDSLoadInfo* info)
+{
   // Exit as early as possible for non-DDS textures, since all extensions are currently
   // passed through this function.
-  if (magic != DDS_MAGIC)
+  u32 magic;
+  if (!file.ReadBytes(&magic, sizeof(magic)) || magic != DDS_MAGIC)
     return false;
 
   DDS_HEADER header;
-  std::memcpy(&header, &buffer[sizeof(magic)], sizeof(header));
-  if (header.dwSize < sizeof(header))
+  if (!file.ReadBytes(&header, sizeof(header)) || header.dwSize < sizeof(header))
     return false;
 
   // Required fields.
@@ -177,34 +208,51 @@ bool HiresTexture::LoadDDSTexture(Level& level, const std::vector<u8>& buffer)
     return false;
 
   // Presence of width/height fields is already tested by DDS_HEADER_FLAGS_TEXTURE.
-  level.width = header.dwWidth;
-  level.height = header.dwHeight;
+  info->width = header.dwWidth;
+  info->height = header.dwHeight;
+  if (info->width == 0 || info->height == 0)
+    return false;
+
+  // Check for mip levels.
+  if (header.dwFlags & DDS_HEADER_FLAGS_MIPMAP)
+  {
+    // Miplevels = 0 means full mip chain?
+    // Some files may specify a number too large here, which doesn't play well with the backends.
+    info->mip_count = header.dwMipMapCount;
+    if (info->mip_count != 0)
+      info->mip_count = std::min(info->mip_count, CalculateMipCount(info->width, info->height));
+    else
+      info->mip_count = CalculateMipCount(info->width, info->height);
+  }
+  else
+  {
+    info->mip_count = 1;
+  }
 
   // Currently, we only handle compressed textures here, and leave the rest to the SOIL loader.
   // In the future, this could be extended, but these isn't much benefit in doing so currently.
   // TODO: DX10 extension header handling.
-  u32 block_size = 1;
-  u32 bytes_per_block = 4;
+  // TODO: Support RGBA8 and friends.
   bool needs_s3tc = false;
   if (header.ddspf.dwFourCC == MAKEFOURCC('D', 'X', 'T', '1'))
   {
-    level.format = HostTextureFormat::DXT1;
-    block_size = 4;
-    bytes_per_block = 8;
+    info->format = HostTextureFormat::DXT1;
+    info->block_size = 4;
+    info->bytes_per_block = 8;
     needs_s3tc = true;
   }
   else if (header.ddspf.dwFourCC == MAKEFOURCC('D', 'X', 'T', '3'))
   {
-    level.format = HostTextureFormat::DXT3;
-    block_size = 4;
-    bytes_per_block = 16;
+    info->format = HostTextureFormat::DXT3;
+    info->block_size = 4;
+    info->bytes_per_block = 16;
     needs_s3tc = true;
   }
   else if (header.ddspf.dwFourCC == MAKEFOURCC('D', 'X', 'T', '5'))
   {
-    level.format = HostTextureFormat::DXT5;
-    block_size = 4;
-    bytes_per_block = 16;
+    info->format = HostTextureFormat::DXT5;
+    info->block_size = 4;
+    info->bytes_per_block = 16;
     needs_s3tc = true;
   }
   else
@@ -219,8 +267,8 @@ bool HiresTexture::LoadDDSTexture(Level& level, const std::vector<u8>& buffer)
     return false;
 
   // Mip levels smaller than the block size are padded to multiples of the block size.
-  u32 blocks_wide = std::max(level.width / block_size, 1u);
-  u32 blocks_high = std::max(level.height / block_size, 1u);
+  u32 blocks_wide = GetBlockCount(info->width, info->block_size);
+  u32 blocks_high = GetBlockCount(info->height, info->block_size);
 
   // Pitch can be specified in the header, otherwise we can derive it from the dimensions. For
   // compressed formats, both DDS_HEADER_FLAGS_LINEARSIZE and DDS_HEADER_FLAGS_PITCH should be
@@ -228,30 +276,110 @@ bool HiresTexture::LoadDDSTexture(Level& level, const std::vector<u8>& buffer)
   if (header.dwFlags & DDS_HEADER_FLAGS_PITCH && header.dwFlags & DDS_HEADER_FLAGS_LINEARSIZE)
   {
     // Convert pitch (in bytes) to texels/row length.
-    if (header.dwPitchOrLinearSize < bytes_per_block)
+    if (header.dwPitchOrLinearSize < info->bytes_per_block)
     {
       // Likely a corrupted or invalid file.
       return false;
     }
 
-    level.row_length = std::max(header.dwPitchOrLinearSize / bytes_per_block, 1u) * block_size;
-    level.data_size = static_cast<size_t>(level.row_length / block_size) * block_size * blocks_high;
+    info->first_mip_row_length =
+        std::max(header.dwPitchOrLinearSize / info->bytes_per_block, 1u) * info->block_size;
+    info->first_mip_size = static_cast<size_t>(info->first_mip_row_length / info->block_size) *
+                           info->block_size * blocks_high;
   }
   else
   {
     // Assume no padding between rows of blocks.
-    level.row_length = blocks_wide * block_size;
-    level.data_size = blocks_wide * static_cast<size_t>(bytes_per_block) * blocks_high;
+    info->first_mip_row_length = blocks_wide * info->block_size;
+    info->first_mip_size = blocks_wide * static_cast<size_t>(info->bytes_per_block) * blocks_high;
   }
 
   // Check for truncated or corrupted files.
-  size_t data_offset = sizeof(magic) + sizeof(DDS_HEADER);
-  if ((data_offset + level.data_size) > buffer.size())
+  info->first_mip_offset = sizeof(magic) + sizeof(DDS_HEADER);
+  if (info->first_mip_offset >= file.GetSize())
     return false;
 
+  return true;
+}
+
+static bool ReadMipLevel(HiresTexture::Level& level, File::IOFile& file, u32 width, u32 height,
+                         HostTextureFormat format, u32 row_length, size_t size)
+{
   // Copy to the final storage location. The deallocator here is simple, nothing extra is
   // needed, compared to the SOIL-based loader.
-  level.data = ImageDataPointer(new u8[level.data_size], [](u8* data) { delete[] data; });
-  std::memcpy(level.data.get(), &buffer[data_offset], level.data_size);
+  level.width = width;
+  level.height = height;
+  level.format = format;
+  level.row_length = row_length;
+  level.data_size = size;
+  level.data =
+      HiresTexture::ImageDataPointer(new u8[level.data_size], [](u8* data) { delete[] data; });
+  if (!file.ReadBytes(level.data.get(), level.data_size))
+  {
+    level.data.reset();
+    return false;
+  }
+
   return true;
 }
+
+bool HiresTexture::LoadDDSTexture(HiresTexture* tex, const std::string& filename)
+{
+  File::IOFile file;
+  file.Open(filename, "rb");
+  if (!file.IsOpen())
+    return false;
+
+  DDSLoadInfo info;
+  if (!ParseDDSHeader(file, &info))
+    return false;
+
+  // Read first mip level, as it may have a custom pitch.
+  Level first_level;
+  if (!file.Seek(info.first_mip_offset, SEEK_SET) ||
+      !ReadMipLevel(first_level, file, info.width, info.height, info.format,
+                    info.first_mip_row_length, info.first_mip_size))
+  {
+    return false;
+  }
+
+  tex->m_levels.push_back(std::move(first_level));
+
+  // Read in any remaining mip levels in the file.
+  // If the .dds file does not contain a full mip chain, we'll fall back to the old path.
+  u32 mip_width = std::max(info.width / 2, 1u);
+  u32 mip_height = std::max(info.height / 2, 1u);
+  for (u32 i = 1; i < info.mip_count; i++)
+  {
+    // Pitch can't be specified with each mip level, so we have to calculate it ourselves.
+    u32 blocks_wide = GetBlockCount(mip_width, info.block_size);
+    u32 blocks_high = GetBlockCount(mip_height, info.block_size);
+    u32 mip_row_length = blocks_wide * info.block_size;
+    size_t mip_size = blocks_wide * static_cast<size_t>(info.bytes_per_block) * blocks_high;
+    Level level;
+    if (!ReadMipLevel(level, file, mip_width, mip_height, info.format, mip_row_length, mip_size))
+      break;
+
+    tex->m_levels.push_back(std::move(level));
+    mip_width = std::max(mip_width / 2, 1u);
+    mip_height = std::max(mip_height / 2, 1u);
+  }
+
+  return true;
+}
+
+bool HiresTexture::LoadDDSTexture(Level& level, const std::string& filename)
+{
+  // Only loading a single mip level.
+  File::IOFile file;
+  file.Open(filename, "rb");
+  if (!file.IsOpen())
+    return false;
+
+  DDSLoadInfo info;
+  if (!ParseDDSHeader(file, &info))
+    return false;
+
+  return ReadMipLevel(level, file, info.width, info.height, info.format, info.first_mip_row_length,
+                      info.first_mip_size);
+}