From 15d9fab0bbe5cae41116c7eb24dc1797f05957c1 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 28 Jun 2020 18:15:30 +0200 Subject: Common: Rename UTF16ToUTF8 This function does *not* always convert from UTF-16. It converts from UTF-16 on Windows and UTF-32 on other operating systems. Also renaming UTF8ToUTF16 for consistency, even though it technically doesn't have the same problem since it only was implemented on Windows. --- Source/Core/Common/StringUtil.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'Source/Core/Common/StringUtil.cpp') diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index c44f3fa7b1..a8880d371b 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -457,29 +457,29 @@ std::string UTF16ToCP(u32 code_page, std::wstring_view input) return output; } -std::wstring UTF8ToUTF16(std::string_view input) +std::wstring UTF8ToWString(std::string_view input) { return CPToUTF16(CP_UTF8, input); } -std::string UTF16ToUTF8(std::wstring_view input) +std::string WStringToUTF8(std::wstring_view input) { return UTF16ToCP(CP_UTF8, input); } std::string SHIFTJISToUTF8(std::string_view input) { - return UTF16ToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input)); + return WStringToUTF8(CPToUTF16(CODEPAGE_SHIFT_JIS, input)); } std::string UTF8ToSHIFTJIS(std::string_view input) { - return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToUTF16(input)); + return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToWString(input)); } std::string CP1252ToUTF8(std::string_view input) { - return UTF16ToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input)); + return WStringToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input)); } std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) @@ -487,7 +487,7 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) const char16_t* str_end = std::find(str, str + max_size, '\0'); std::wstring result(static_cast(str_end - str), '\0'); std::transform(str, str_end, result.begin(), static_cast(Common::swap16)); - return UTF16ToUTF8(result); + return WStringToUTF8(result); } #else @@ -572,7 +572,7 @@ std::string UTF8ToSHIFTJIS(std::string_view input) return CodeTo("SJIS", "UTF-8", input); } -std::string UTF16ToUTF8(std::wstring_view input) +std::string WStringToUTF8(std::wstring_view input) { std::wstring_convert, wchar_t> converter; return converter.to_bytes(input.data(), input.data() + input.size()); @@ -591,7 +591,7 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) std::filesystem::path StringToPath(std::string_view path) { #ifdef _MSC_VER - return std::filesystem::path(UTF8ToUTF16(path)); + return std::filesystem::path(UTF8ToWString(path)); #else return std::filesystem::path(path); #endif @@ -602,7 +602,7 @@ std::filesystem::path StringToPath(std::string_view path) std::string PathToString(const std::filesystem::path& path) { #ifdef _MSC_VER - return UTF16ToUTF8(path.native()); + return WStringToUTF8(path.native()); #else return path.native(); #endif -- cgit v1.2.3 From 38791eec1879b0a66d6ae920cfd6b42d9ae66dee Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 28 Jun 2020 18:22:18 +0200 Subject: Common: Never convert from UCS-2 in WStringToUTF8 Probably not something we would run into in practice since Windows uses a separate implementation, but let's do it for the sake of correctness. --- Source/Core/Common/StringUtil.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'Source/Core/Common/StringUtil.cpp') diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index a8880d371b..e3c6f692f6 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -574,7 +575,10 @@ std::string UTF8ToSHIFTJIS(std::string_view input) std::string WStringToUTF8(std::wstring_view input) { - std::wstring_convert, wchar_t> converter; + using codecvt = std::conditional_t, + std::codecvt_utf8>; + + std::wstring_convert converter; return converter.to_bytes(input.data(), input.data() + input.size()); } -- cgit v1.2.3 From f5da6e07d7405d89cc550b8c02f39a432f7888a3 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sun, 28 Jun 2020 19:16:23 +0200 Subject: Android: Use correct encoding when converting strings The functions with "UTF" in the name use "modified UTF-8" rather than the standard UTF-8 which Dolphin uses, at least according to Oracle's documentation, so it is incorrect for us to use them. This change fixes the problem by converting between UTF-8 and UTF-16 manually instead of letting JNI do it for us. --- Source/Core/Common/StringUtil.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'Source/Core/Common/StringUtil.cpp') diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index e3c6f692f6..6c236de587 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -5,6 +5,7 @@ #include "Common/StringUtil.h" #include +#include #include #include #include @@ -33,7 +34,6 @@ constexpr u32 CODEPAGE_SHIFT_JIS = 932; constexpr u32 CODEPAGE_WINDOWS_1252 = 1252; #else -#include #include #include #include @@ -590,6 +590,18 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) #endif +std::string UTF16ToUTF8(std::u16string_view input) +{ + std::wstring_convert, char16_t> converter; + return converter.to_bytes(input.data(), input.data() + input.size()); +} + +std::u16string UTF8ToUTF16(std::string_view input) +{ + std::wstring_convert, char16_t> converter; + return converter.from_bytes(input.data(), input.data() + input.size()); +} + #ifdef HAS_STD_FILESYSTEM // This is a replacement for path::u8path, which is deprecated starting with C++20. std::filesystem::path StringToPath(std::string_view path) -- cgit v1.2.3