Merge branch 'windows-unicode'

Fixed unicode filename handling on Windows.
Made all significant projects build with "Unicode" option on Windows.
Fixed unicode string handling in GUI code on all OSes.

From now on: std::string == UTF-8.

Fixed issue 4111.
Fixed issue 5178.
Fixed issue 5980.

1 files changed, 144 insertions, 0 deletions

diff --git a/Source/Core/Common/Src/StringUtil.cpp b/Source/Core/Common/Src/StringUtil.cpp
index cd08863877..c38184776b 100644
--- a/Source/Core/Common/Src/StringUtil.cpp
+++ b/Source/Core/Common/Src/StringUtil.cpp
@@ -17,11 +17,21 @@
 
 #include <stdlib.h>
 #include <stdio.h>
+#include <algorithm>
 
 #include "Common.h"
 #include "CommonPaths.h"
 #include "StringUtil.h"
 
+#ifdef _WIN32
+	#include <Windows.h>
+#elif defined(ANDROID)
+
+#else
+	#include <iconv.h>
+	#include <errno.h>
+#endif
+
 // faster than sscanf
 bool AsciiToHex(const char* _szValue, u32& result)
 {
@@ -375,3 +385,137 @@ std::string UriEncode(const std::string & sSrc)
 	delete [] pStart;
 	return sResult;
 }
+
+#ifdef _WIN32
+
+std::string UTF16ToUTF8(const std::wstring& input)
+{
+	auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), nullptr, 0, nullptr, nullptr);
+
+	std::string output;
+	output.resize(size);
+
+	if (size != WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), &output[0], output.size(), nullptr, nullptr))
+		output.clear();
+
+	return output;
+}
+
+std::wstring CPToUTF16(u32 code_page, const std::string& input)
+{
+	auto const size = MultiByteToWideChar(code_page, 0, input.data(), input.size(), nullptr, 0);
+
+	std::wstring output;
+	output.resize(size);
+
+	if (size != MultiByteToWideChar(code_page, 0, input.data(), input.size(), &output[0], output.size()))
+		output.clear();
+
+	return output;
+}
+
+std::wstring UTF8ToUTF16(const std::string& input)
+{
+	return CPToUTF16(CP_UTF8, input);
+}
+
+std::string SHIFTJISToUTF8(const std::string& input)
+{
+	return UTF16ToUTF8(CPToUTF16(932, input));
+}
+
+std::string CP1252ToUTF8(const std::string& input)
+{
+	return UTF16ToUTF8(CPToUTF16(1252, input));
+}
+
+#else
+
+template <typename T>
+std::string CodeToUTF8(const char* fromcode, const std::basic_string<T>& input)
+{
+	std::string result;
+	
+#if defined(ANDROID)
+	result = "Not implemented on Android!";
+	
+#else
+	iconv_t const conv_desc = iconv_open("UTF-8", fromcode);
+	if ((iconv_t)-1 == conv_desc)
+	{
+		ERROR_LOG(COMMON, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno));
+	}
+	else
+	{
+		size_t const in_bytes = sizeof(T) * input.size();
+		size_t const out_buffer_size = 4 * in_bytes;
+		
+		std::string out_buffer;
+		out_buffer.resize(out_buffer_size);
+
+		auto src_buffer = &input[0];
+		size_t src_bytes = in_bytes;
+		auto dst_buffer = &out_buffer[0];
+		size_t dst_bytes = out_buffer.size();
+		
+		while (src_bytes != 0)
+		{
+			size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes,
+				&dst_buffer, &dst_bytes);
+			
+			if ((size_t)-1 == iconv_result)
+			{
+				if (EILSEQ == errno || EINVAL == errno)
+				{
+					// Try to skip the bad character
+					if (src_bytes != 0)
+					{
+						--src_bytes;
+						++src_buffer;
+					}
+				}
+				else
+				{
+					ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno));
+					break;
+				}
+			}
+		}
+
+		out_buffer.resize(out_buffer_size - dst_bytes);
+		out_buffer.swap(result);
+		
+		iconv_close(conv_desc);
+	}
+	
+#endif
+	return result;
+}
+
+std::string CP1252ToUTF8(const std::string& input)
+{
+	//return CodeToUTF8("CP1252//TRANSLIT", input);
+	//return CodeToUTF8("CP1252//IGNORE", input);
+	return CodeToUTF8("CP1252", input);
+}
+
+std::string SHIFTJISToUTF8(const std::string& input)
+{
+	//return CodeToUTF8("CP932", input);
+	return CodeToUTF8("SJIS", input);
+}
+
+std::string UTF16ToUTF8(const std::wstring& input)
+{
+	std::string result =
+	//	CodeToUTF8("UCS-2", input);
+	//	CodeToUTF8("UCS-2LE", input);
+	//	CodeToUTF8("UTF-16", input);
+		CodeToUTF8("UTF-16LE", input);
+	
+	// TODO: why is this needed?
+	result.erase(std::remove(result.begin(), result.end(), 0x00), result.end());
+	return result;
+}
+
+#endif