#include "dl_code_cvt.h" #include // windows使用预编译的库、linux通过apt安装、android使用ndk带的 #include #include "io/dl_log.h" #include "memory/dl_memory.h" #include "system/dl_platform.h" #include "base/dl_string.h" namespace dl::CodeCvt { //#if DL_PLATFORM == DL_PLATFORM_ANDROID //Buffer CvtString(std::string_view from, std::string_view to, BufferView buf) //{ // //前置条件检查 // if (buf.empty()) // return {}; // // return CreateBuffer(buf); //} //#else Buffer CvtString(std::string_view from, std::string_view to, BufferView buf) { //前置条件检查 if (buf.empty()) return {}; iconv_t cd = iconv_open(to.data(), from.data()); if ((iconv_t)(-1) == cd) { log_easy("iconv_open失败!"); return {}; } char temp[SIZE_BUFFER_STRING_CONVERT];//缓冲区 Buffer ret;//输出 char* in_buf = (char*)buf.data(); //输入指针 size_t in_size = buf.size(); //输入大小,直到0停止转换 char* out_buf = temp; //输出缓冲区指针 size_t out_size = SIZE_BUFFER_STRING_CONVERT;//输出缓冲区大小 do {//直到转换完 if (-1 == iconv(cd, &in_buf, &in_size, &out_buf, &out_size)) { if (errno != E2BIG) { log_easy("转换编码失败!"); iconv_close(cd); return {}; } else { size_t num = SIZE_BUFFER_STRING_CONVERT - out_size; BufferAppend(ret, CreateBufferView(temp, num)); //没有转换完,重设输出位置 out_size = SIZE_BUFFER_STRING_CONVERT; out_buf = temp; } } else {//完整转换,增加num assert(in_size == 0); size_t num = SIZE_BUFFER_STRING_CONVERT - out_size; if (ret.empty())//第一次就结束了,优化直接return { iconv_close(cd); BufferAppend(ret, CreateBufferView(temp, num)); return ret; } BufferAppend(ret, CreateBufferView(temp, num)); } } while (in_size); iconv_close(cd); return ret; } //#endif namespace Utf8 { int Check(std::string_view str) { unsigned n_bytes = 0;//UFT8可用1-6个字节编码,ASCII用一个字节 unsigned char ch; bool b_all_ascii = true; //如果全部都是ASCII, 说明不是UTF-8 for (size_t i = 0; i < str.size(); ++i) { ch = str[i]; if ((ch & 0x80) != 0) // 判断是否ASCII编码,如果不是,说明有可能是UTF-8,ASCII用7位编码,但用一个字节存,最高位标记为0,o0xxxxxxx b_all_ascii = false; if (n_bytes == 0) //如果不是ASCII码,应该是多字节符,计算字节数 { if (ch >= 0x80) { if (ch >= 0xFC && ch <= 0xFD) n_bytes = 6; else if (ch >= 0xF8) n_bytes = 5; else if (ch >= 0xF0) n_bytes = 4; else if (ch >= 0xE0) n_bytes = 3; else if (ch >= 0xC0) n_bytes = 2; else return 0; --n_bytes; } } else //多字节符的非首字节,应为 10xxxxxx { if ((ch & 0xC0) != 0x80) return 0; --n_bytes; } } if (n_bytes > 0) //违返规则mb return 0; if (b_all_ascii) //如果全部都是ASCII, 说明不是UTF-8 return 1; return 2;//utf8 } void Copy(char* target, size_t size, std::string_view source, bool show_log /*= true*/) { if (size == 0) { log_warn0("缓冲区大小为0!"); return; } //如果加上结束符 超过缓冲区 size_t size_source = source.size(); if (size_source + 1 > size) { //std::string str_mb = CodeCvt::cvt_u8_mb(source); if (show_log) log_warn("字符串过比缓冲区大:{}", source); size_t sub_size = Sub(source, size - 1); Memory::Copy(target, size - 1, source.data(), sub_size); target[sub_size] = 0; } else { memcpy(target, source.data(), size_source); target[size_source] = 0; } } size_t Sub(std::string_view str, size_t max) { if (max == 0) return 0; //源下标(与子串下标相同) size_t i = 0; size_t size = str.length(); size_t cut_len = 0; unsigned char ch; while (i < size) { ch = (unsigned char)str[i]; if (ch >= 252) cut_len = 6; else if (ch >= 248) cut_len = 5; else if (ch >= 240) cut_len = 4; else if (ch >= 224) cut_len = 3; else if (ch >= 192) cut_len = 2; else if (ch >= 65 && ch <= 90) cut_len = 1; else cut_len = 1; if (i + cut_len > max) return i; i += cut_len; } return size; } } std::string Utf8ToMultiByte(std::string_view str, std::string_view code_page /*= ""*/) { Buffer buf = CvtString("UTF-8", code_page, CreateBufferView(str)); return to_string(buf); } std::string MultiByteToUtf8(const char* str, std::string_view code_page) { Buffer buf = CvtString(code_page, "UTF-8", CreateBufferView((char*)str, strlen(str))); return to_string(buf); } std::string MultiByteToUtf8(std::string_view str, std::string_view code_page) { Buffer buf = CvtString(code_page, "UTF-8", CreateBufferView(str)); return to_string(buf); } // UTF-16 UTF-32 会带上bom,参考:https://zzz.buzz/2016/07/30/bom-in-iconv/ std::string Utf32ToUtf8(std::u32string_view str) { Buffer buf = CvtString("UTF-32LE", "UTF-8", CreateBufferView((char*)str.data(), str.size() * sizeof(char32_t))); return to_string(buf); } std::u32string Utf8ToUtf32(std::string_view str) { Buffer buf = CvtString("UTF-8", "UTF-32LE", CreateBufferView(str)); assert(buf.size() % sizeof(char32_t) == 0);// 必须是sizeof(char32_t)的倍数 std::u32string ret((char32_t*)buf.data(), buf.size() / sizeof(char32_t)); return ret; } std::string WideCharToUtf8(std::wstring_view str) { Buffer buf = CvtString("wchar_t", "UTF-8", CreateBufferView((char*)str.data(), str.size() * sizeof(wchar_t))); return to_string(buf); } std::wstring Utf8ToWideChar(std::string_view str) { Buffer buf = CvtString("UTF-8", "wchar_t", CreateBufferView(str)); std::wstring ret((wchar_t*)buf.data(), buf.size() / sizeof(wchar_t)); return ret; } // std::wstring cvt_u8_wc(string_view str) // { // string out = CvtString("UTF-8", "wchar_t", str.data(), str.size()); // // dnd_assert(out.size() % sizeof(wchar_t) == 0);//必须是sizeof(wchar_t)的倍数 // std::wstring ret((wchar_t*)out.c_str(), out.size() / sizeof(wchar_t)); // return ret; // } // // string cvt_u8_mb(string_view str) // { // return CvtString("UTF-8", "", str.data(), str.size()); // } // // dnd::utf_string cvt_u8_utf(string_view str) // { // string out = CvtString("UTF-8", "UCS-4-INTERNAL", str.data(), str.size()); // // dnd_assert(out.size() % sizeof(char32_t) == 0);//必须是sizeof(char32_t)的倍数 // dnd::utf_string ret((char32_t*)out.c_str(), out.size() / sizeof(char32_t)); // return ret; // } // // string cvt_wc_u8(const wchar_t* str) // { // size_t size = wcslen(str) * sizeof(wchar_t); // return CvtString("wchar_t", "UTF-8", (char*)str, size); // } // // std::string cvt_wc_u8(std::wstring_view str) // { // return CvtString("wchar_t", "UTF-8", (char*)str.data(), str.size() * sizeof(wchar_t)); // } // // // // string cvt_wc_mb(const wchar_t* str, string_view code_name) // { // size_t size = wcslen(str) * sizeof(wchar_t); // return CvtString("wchar_t", code_name, (char*)str, size); // } // // std::string cvt_mb_u8(string_view str, string_view code_name /*= ""*/) // { // return CvtString(code_name, "UTF-8", str.data(), str.size()); // } // // // // std::wstring cvt_mb_wc(const char* str, string_view code_name) // { // string out = CvtString(code_name, "wchar_t", str, strlen(str)); // // dnd_assert(out.size() % sizeof(wchar_t) == 0);//必须是sizeof(wchar_t)的倍数 // std::wstring ret((wchar_t*)out.c_str(), out.size() / sizeof(wchar_t)); // return ret; // } // // string cvt_utf_u8(const utf_string& str) // { // return CvtString("UCS-4-INTERNAL", "UTF-8", (char*)str.c_str(), str.size() * sizeof(wchar_t)); // } // // //size_t ANSIExtend::GbkSubIndex(std::string_view str, size_t max) //{ // // 足够容纳,则全部返回 // if (str.size() <= max) // return str.size(); // // // end为普通字符 // if (str[max] > 0) // return max; // // // end-1为普通字符 // if (str[max - 1] > 0) // return max; // // // end-1不为普通字符,end-2为普通字符 // if (str[max - 2] > 0) // return max - 1; // // // end-1不为普通字符,end-2不为普通字符,end不为普通字符 // // 往前遍历,直到普通字符,或者头部 // size_t cur = max - 3; // while (true) // { // if (str[cur] > 0) // { // size_t dis = // } // } // return max - 2; //} }