From 0ea458b4dc9241a59fd18456c846be44694fdb7d Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Wed, 27 Feb 2013 18:00:42 -0600 Subject: Add functions for converting between UTF-8/16. --- Source/Core/Common/Src/StringUtil.cpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'Source/Core/Common/Src/StringUtil.cpp') diff --git a/Source/Core/Common/Src/StringUtil.cpp b/Source/Core/Common/Src/StringUtil.cpp index 6ef3d91a73..503c2810c3 100644 --- a/Source/Core/Common/Src/StringUtil.cpp +++ b/Source/Core/Common/Src/StringUtil.cpp @@ -22,6 +22,10 @@ #include "CommonPaths.h" #include "StringUtil.h" +#ifdef _WIN32 +#include +#endif + // faster than sscanf bool AsciiToHex(const char* _szValue, u32& result) { @@ -375,3 +379,33 @@ std::string UriEncode(const std::string & sSrc) delete [] pStart; return sResult; } + +#ifdef _WIN32 + +std::string UTF16ToUTF8(const std::wstring& input) +{ + auto const size = WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), nullptr, 0, nullptr, nullptr); + + std::string output; + output.resize(size); + + if (size != WideCharToMultiByte(CP_UTF8, 0, input.data(), input.size(), &output[0], output.size(), nullptr, nullptr)) + output.clear(); + + return output; +} + +std::wstring UTF8ToUTF16(const std::string& input) +{ + auto const size = MultiByteToWideChar(CP_UTF8, 0, input.data(), input.size(), nullptr, 0); + + std::wstring output; + output.resize(size); + + if (size != MultiByteToWideChar(CP_UTF8, 0, input.data(), input.size(), &output[0], output.size())) + output.clear(); + + return output; +} + +#endif -- cgit v1.2.3 From 6c8adf6130b32a0ffc113f4ce30f70b9d5994f10 Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Sat, 2 Mar 2013 19:46:55 -0600 Subject: Eliminate the wstring game name. Some cleanup throughout related code. (try to make logic in ISOFile understandable by a human) Encode strings in UTF-8 rather than somehow trying to determine the encoding in the GUI code. Non-windows OSes temporarily broken. --- Source/Core/Common/Src/StringUtil.cpp | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'Source/Core/Common/Src/StringUtil.cpp') diff --git a/Source/Core/Common/Src/StringUtil.cpp b/Source/Core/Common/Src/StringUtil.cpp index 38ac107ec1..aab0f3bd58 100644 --- a/Source/Core/Common/Src/StringUtil.cpp +++ b/Source/Core/Common/Src/StringUtil.cpp @@ -395,17 +395,41 @@ std::string UTF16ToUTF8(const std::wstring& input) return output; } -std::wstring UTF8ToUTF16(const std::string& input) +std::wstring CPToUTF16(u32 code_page, const std::string& input) { - auto const size = MultiByteToWideChar(CP_UTF8, 0, input.data(), input.size(), nullptr, 0); + auto const size = MultiByteToWideChar(code_page, 0, input.data(), input.size(), nullptr, 0); std::wstring output; output.resize(size); - if (size != MultiByteToWideChar(CP_UTF8, 0, input.data(), input.size(), &output[0], output.size())) + if (size != MultiByteToWideChar(code_page, 0, input.data(), input.size(), &output[0], output.size())) output.clear(); return output; } +std::wstring UTF8ToUTF16(const std::string& input) +{ + return CPToUTF16(CP_UTF8, input); +} + +std::string SHIFTJISToUTF8(const std::string& input) +{ + return UTF16ToUTF8(CPToUTF16(932, input)); +} + +#else + +std::string UTF16ToUTF8(const std::wstring& input) +{ + // TODO: implement + return std::string(); +} + +std::string SHIFTJISToUTF8(const std::string& input) +{ + // TODO: implement + return std::string(); +} + #endif -- cgit v1.2.3 From aeb4fc9846f19d78248994b72c0902c17df3947b Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Sat, 2 Mar 2013 22:57:49 -0600 Subject: Fix what I broke. --- Source/Core/Common/Src/StringUtil.cpp | 80 ++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 6 deletions(-) (limited to 'Source/Core/Common/Src/StringUtil.cpp') diff --git a/Source/Core/Common/Src/StringUtil.cpp b/Source/Core/Common/Src/StringUtil.cpp index aab0f3bd58..e499fc40c3 100644 --- a/Source/Core/Common/Src/StringUtil.cpp +++ b/Source/Core/Common/Src/StringUtil.cpp @@ -17,13 +17,19 @@ #include #include +#include #include "Common.h" #include "CommonPaths.h" #include "StringUtil.h" #ifdef _WIN32 -#include + #include +#elif defined(ANDROID) + +#else + #include + #include #endif // faster than sscanf @@ -418,18 +424,80 @@ std::string SHIFTJISToUTF8(const std::string& input) return UTF16ToUTF8(CPToUTF16(932, input)); } +std::string CP1252ToUTF8(const std::string& input) +{ + return UTF16ToUTF8(CPToUTF16(1252, input)); +} + #else -std::string UTF16ToUTF8(const std::wstring& input) +template +std::string CodeToUTF8(const char* fromcode, const std::basic_string& input) +{ + std::string result; + +#if defined(ANDROID) + result = "Not implemented on Android!"; + +#else + iconv_t const conv_desc = iconv_open("UTF-8", fromcode); + if ((iconv_t)-1 == conv_desc) + { + ERROR_LOG(COMMON, "Iconv initialization failure [%s]: %s", fromcode, strerror(errno)); + } + else + { + size_t const in_bytes = sizeof(T) * input.size(); + size_t const out_buffer_size = 4 * in_bytes; + + std::string out_buffer; + out_buffer.resize(out_buffer_size); + + auto src_buffer = &input[0]; + size_t src_bytes = in_bytes; + auto dst_buffer = &out_buffer[0]; + size_t dst_bytes = out_buffer.size(); + + size_t const iconv_size = iconv(conv_desc, (char**)(&src_buffer), &src_bytes, + &dst_buffer, &dst_bytes); + + if ((size_t)-1 == iconv_size) + { + ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno)); + } + else + { + out_buffer.resize(out_buffer_size - dst_bytes); + out_buffer.swap(result); + + // TODO: why is this needed? + result.erase(std::remove(result.begin(), result.end(), 0x00), result.end()); + } + + iconv_close(conv_desc); + } + +#endif + return result; +} + +std::string CP1252ToUTF8(const std::string& input) { - // TODO: implement - return std::string(); + return CodeToUTF8("CP1252", input); } std::string SHIFTJISToUTF8(const std::string& input) { - // TODO: implement - return std::string(); + //return CodeToUTF8("CP932", input); + return CodeToUTF8("SJIS", input); +} + +std::string UTF16ToUTF8(const std::wstring& input) +{ + //return CodeToUTF8("UCS-2", input); + //return CodeToUTF8("UCS-2LE", input); + //return CodeToUTF8("UTF-16", input); + return CodeToUTF8("UTF-16LE", input); } #endif -- cgit v1.2.3 From c07b8a6e3754abb4be5d962d84550d4b10827240 Mon Sep 17 00:00:00 2001 From: Jordan Woyak Date: Sun, 3 Mar 2013 16:51:26 -0600 Subject: Fix more of what I broke. --- Source/Core/Common/Src/StringUtil.cpp | 52 +++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 17 deletions(-) (limited to 'Source/Core/Common/Src/StringUtil.cpp') diff --git a/Source/Core/Common/Src/StringUtil.cpp b/Source/Core/Common/Src/StringUtil.cpp index e499fc40c3..c38184776b 100644 --- a/Source/Core/Common/Src/StringUtil.cpp +++ b/Source/Core/Common/Src/StringUtil.cpp @@ -457,22 +457,33 @@ std::string CodeToUTF8(const char* fromcode, const std::basic_string& input) size_t src_bytes = in_bytes; auto dst_buffer = &out_buffer[0]; size_t dst_bytes = out_buffer.size(); - - size_t const iconv_size = iconv(conv_desc, (char**)(&src_buffer), &src_bytes, - &dst_buffer, &dst_bytes); - - if ((size_t)-1 == iconv_size) - { - ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno)); - } - else + + while (src_bytes != 0) { - out_buffer.resize(out_buffer_size - dst_bytes); - out_buffer.swap(result); + size_t const iconv_result = iconv(conv_desc, (char**)(&src_buffer), &src_bytes, + &dst_buffer, &dst_bytes); - // TODO: why is this needed? - result.erase(std::remove(result.begin(), result.end(), 0x00), result.end()); + if ((size_t)-1 == iconv_result) + { + if (EILSEQ == errno || EINVAL == errno) + { + // Try to skip the bad character + if (src_bytes != 0) + { + --src_bytes; + ++src_buffer; + } + } + else + { + ERROR_LOG(COMMON, "iconv failure [%s]: %s", fromcode, strerror(errno)); + break; + } + } } + + out_buffer.resize(out_buffer_size - dst_bytes); + out_buffer.swap(result); iconv_close(conv_desc); } @@ -483,6 +494,8 @@ std::string CodeToUTF8(const char* fromcode, const std::basic_string& input) std::string CP1252ToUTF8(const std::string& input) { + //return CodeToUTF8("CP1252//TRANSLIT", input); + //return CodeToUTF8("CP1252//IGNORE", input); return CodeToUTF8("CP1252", input); } @@ -494,10 +507,15 @@ std::string SHIFTJISToUTF8(const std::string& input) std::string UTF16ToUTF8(const std::wstring& input) { - //return CodeToUTF8("UCS-2", input); - //return CodeToUTF8("UCS-2LE", input); - //return CodeToUTF8("UTF-16", input); - return CodeToUTF8("UTF-16LE", input); + std::string result = + // CodeToUTF8("UCS-2", input); + // CodeToUTF8("UCS-2LE", input); + // CodeToUTF8("UTF-16", input); + CodeToUTF8("UTF-16LE", input); + + // TODO: why is this needed? + result.erase(std::remove(result.begin(), result.end(), 0x00), result.end()); + return result; } #endif -- cgit v1.2.3