Merge pull request #5659 from shuffle2/gamelist-speedup-with-emustate

Gamelist speedup with emustate

3 files changed, 81 insertions, 134 deletions

diff --git a/Source/Core/Common/ChunkFile.h b/Source/Core/Common/ChunkFile.h
index 72832c5e97..497a6d439a 100644
--- a/Source/Core/Common/ChunkFile.h
+++ b/Source/Core/Common/ChunkFile.h
@@ -296,125 +296,3 @@ private:
     *ptr += size;
   }
 };
-
-// NOTE: this class is only used in DolphinWX/ISOFile.cpp for caching loaded
-// ISO data. It will be removed when DolphinWX is, so please don't use it.
-class CChunkFileReader
-{
-public:
-  // Load file template
-  template <class T>
-  static bool Load(const std::string& _rFilename, u32 _Revision, T& _class)
-  {
-    INFO_LOG(COMMON, "ChunkReader: Loading %s", _rFilename.c_str());
-
-    if (!File::Exists(_rFilename))
-      return false;
-
-    // Check file size
-    const u64 fileSize = File::GetSize(_rFilename);
-    static const u64 headerSize = sizeof(SChunkHeader);
-    if (fileSize < headerSize)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: File too small");
-      return false;
-    }
-
-    File::IOFile pFile(_rFilename, "rb");
-    if (!pFile)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Can't open file for reading");
-      return false;
-    }
-
-    // read the header
-    SChunkHeader header;
-    if (!pFile.ReadArray(&header, 1))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Bad header size");
-      return false;
-    }
-
-    // Check revision
-    if (header.Revision != _Revision)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Wrong file revision, got %d expected %d", header.Revision,
-                _Revision);
-      return false;
-    }
-
-    // get size
-    const u32 sz = (u32)(fileSize - headerSize);
-    if (header.ExpectedSize != sz)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Bad file size, got %d expected %d", sz, header.ExpectedSize);
-      return false;
-    }
-
-    // read the state
-    std::vector<u8> buffer(sz);
-    if (!pFile.ReadArray(&buffer[0], sz))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Error reading file");
-      return false;
-    }
-
-    u8* ptr = &buffer[0];
-    PointerWrap p(&ptr, PointerWrap::MODE_READ);
-    _class.DoState(p);
-
-    INFO_LOG(COMMON, "ChunkReader: Done loading %s", _rFilename.c_str());
-    return true;
-  }
-
-  // Save file template
-  template <class T>
-  static bool Save(const std::string& _rFilename, u32 _Revision, T& _class)
-  {
-    INFO_LOG(COMMON, "ChunkReader: Writing %s", _rFilename.c_str());
-    File::IOFile pFile(_rFilename, "wb");
-    if (!pFile)
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Error opening file for write");
-      return false;
-    }
-
-    // Get data
-    u8* ptr = nullptr;
-    PointerWrap p(&ptr, PointerWrap::MODE_MEASURE);
-    _class.DoState(p);
-    size_t const sz = (size_t)ptr;
-    std::vector<u8> buffer(sz);
-    ptr = &buffer[0];
-    p.SetMode(PointerWrap::MODE_WRITE);
-    _class.DoState(p);
-
-    // Create header
-    SChunkHeader header;
-    header.Revision = _Revision;
-    header.ExpectedSize = (u32)sz;
-
-    // Write to file
-    if (!pFile.WriteArray(&header, 1))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Failed writing header");
-      return false;
-    }
-
-    if (!pFile.WriteArray(&buffer[0], sz))
-    {
-      ERROR_LOG(COMMON, "ChunkReader: Failed writing data");
-      return false;
-    }
-
-    INFO_LOG(COMMON, "ChunkReader: Done writing %s", _rFilename.c_str());
-    return true;
-  }
-
-private:
-  struct SChunkHeader
-  {
-    u32 Revision;
-    u32 ExpectedSize;
-  };
-};
diff --git a/Source/Core/Common/FileSearch.cpp b/Source/Core/Common/FileSearch.cpp
index 29b8624b15..783c22d864 100644
--- a/Source/Core/Common/FileSearch.cpp
+++ b/Source/Core/Common/FileSearch.cpp
@@ -7,10 +7,19 @@
 
 #include "Common/CommonPaths.h"
 #include "Common/FileSearch.h"
+
+#ifdef _MSC_VER
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#define HAS_STD_FILESYSTEM
+#else
 #include "Common/FileUtil.h"
+#endif
 
 namespace Common
 {
+#ifndef HAS_STD_FILESYSTEM
+
 static std::vector<std::string>
 FileSearchWithTest(const std::vector<std::string>& directories, bool recursive,
                    std::function<bool(const File::FSTEntry&)> callback)
@@ -36,10 +45,10 @@ FileSearchWithTest(const std::vector<std::string>& directories, bool recursive,
   return result;
 }
 
-std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
-                                      const std::vector<std::string>& directories, bool recursive)
+std::vector<std::string> DoFileSearchNoSTL(const std::vector<std::string>& directories,
+                                           const std::vector<std::string>& exts, bool recursive)
 {
-  bool accept_all = std::find(exts.begin(), exts.end(), "") != exts.end();
+  bool accept_all = exts.empty();
   return FileSearchWithTest(directories, recursive, [&](const File::FSTEntry& entry) {
     if (accept_all)
       return true;
@@ -52,11 +61,71 @@ std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
   });
 }
 
-// Result includes the passed directories themselves as well as their subdirectories.
-std::vector<std::string> FindSubdirectories(const std::vector<std::string>& directories,
-                                            bool recursive)
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts, bool recursive)
 {
-  return FileSearchWithTest(directories, true,
-                            [&](const File::FSTEntry& entry) { return entry.isDirectory; });
+  return DoFileSearchNoSTL(directories, exts, recursive);
 }
+
+#else
+
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts, bool recursive)
+{
+  bool accept_all = exts.empty();
+
+  std::vector<fs::path> native_exts;
+  for (const auto& ext : exts)
+    native_exts.push_back(ext);
+
+  // N.B. This avoids doing any copies
+  auto ext_matches = [&native_exts](const fs::path& path) {
+    const auto& native_path = path.native();
+    return std::any_of(native_exts.cbegin(), native_exts.cend(), [&native_path](const auto& ext) {
+      // TODO provide cross-platform compat for the comparison function, once more platforms
+      // support std::filesystem
+      return native_path.length() >= ext.native().length() &&
+             _wcsicmp(&native_path.c_str()[native_path.length() - ext.native().length()],
+                      ext.c_str()) == 0;
+    });
+  };
+
+  std::vector<std::string> result;
+  auto add_filtered = [&](const fs::directory_entry& entry) {
+    auto& path = entry.path();
+    if (accept_all || (ext_matches(path) && !fs::is_directory(path)))
+      result.emplace_back(path.u8string());
+  };
+  for (const auto& directory : directories)
+  {
+    if (recursive)
+    {
+      // TODO use fs::directory_options::follow_directory_symlink ?
+      for (auto& entry : fs::recursive_directory_iterator(fs::path(directory.c_str())))
+        add_filtered(entry);
+    }
+    else
+    {
+      for (auto& entry : fs::directory_iterator(fs::path(directory.c_str())))
+        add_filtered(entry);
+    }
+  }
+
+  // Remove duplicates (occurring because caller gave e.g. duplicate or overlapping directories -
+  // not because std::filesystem returns duplicates). Also note that this pathname-based uniqueness
+  // isn't as thorough as std::filesystem::equivalent.
+  std::sort(result.begin(), result.end());
+  result.erase(std::unique(result.begin(), result.end()), result.end());
+
+  // Dolphin expects to be able to use "/" (DIR_SEP) everywhere. std::filesystem uses the OS
+  // separator.
+  if (fs::path::preferred_separator != DIR_SEP_CHR)
+    for (auto& path : result)
+      std::replace(path.begin(), path.end(), '\\', DIR_SEP_CHR);
+
+  return result;
+}
+
+#endif
+
 }  // namespace Common
diff --git a/Source/Core/Common/FileSearch.h b/Source/Core/Common/FileSearch.h
index dc2f7f6b62..dc38db945a 100644
--- a/Source/Core/Common/FileSearch.h
+++ b/Source/Core/Common/FileSearch.h
@@ -9,9 +9,9 @@
 
 namespace Common
 {
-std::vector<std::string> DoFileSearch(const std::vector<std::string>& exts,
-                                      const std::vector<std::string>& directories,
+// Callers can pass empty "exts" to indicate they want all files + directories in results
+// Otherwise, only files matching the extensions are returned
+std::vector<std::string> DoFileSearch(const std::vector<std::string>& directories,
+                                      const std::vector<std::string>& exts = {},
                                       bool recursive = false);
-std::vector<std::string> FindSubdirectories(const std::vector<std::string>& directories,
-                                            bool recursive);
 }  // namespace Common