355 lines
8.1 KiB
C++
355 lines
8.1 KiB
C++
#include "dl_code_cvt.h"
|
||
|
||
#include <cassert>
|
||
|
||
// windows使用预编译的库、linux通过apt安装、android使用ndk带的
|
||
#include <iconv.h>
|
||
|
||
#include "io/dl_log.h"
|
||
#include "memory/dl_memory.h"
|
||
#include "system/dl_platform.h"
|
||
#include "base/dl_string.h"
|
||
|
||
|
||
namespace dl::CodeCvt
|
||
{
|
||
//#if DL_PLATFORM == DL_PLATFORM_ANDROID
|
||
//Buffer CvtString(std::string_view from, std::string_view to, BufferView buf)
|
||
//{
|
||
// //前置条件检查
|
||
// if (buf.empty())
|
||
// return {};
|
||
//
|
||
// return CreateBuffer(buf);
|
||
//}
|
||
//#else
|
||
Buffer CvtString(std::string_view from, std::string_view to, BufferView buf)
|
||
{
|
||
//前置条件检查
|
||
if (buf.empty())
|
||
return {};
|
||
|
||
iconv_t cd = iconv_open(to.data(), from.data());
|
||
if ((iconv_t)(-1) == cd)
|
||
{
|
||
log_easy("iconv_open失败！");
|
||
return {};
|
||
}
|
||
|
||
char temp[SIZE_BUFFER_STRING_CONVERT];//缓冲区
|
||
Buffer ret;//输出
|
||
|
||
char* in_buf = (char*)buf.data(); //输入指针
|
||
size_t in_size = buf.size(); //输入大小，直到0停止转换
|
||
|
||
char* out_buf = temp; //输出缓冲区指针
|
||
size_t out_size = SIZE_BUFFER_STRING_CONVERT;//输出缓冲区大小
|
||
|
||
do
|
||
{//直到转换完
|
||
if (-1 == iconv(cd, &in_buf, &in_size, &out_buf, &out_size))
|
||
{
|
||
if (errno != E2BIG)
|
||
{
|
||
log_easy("转换编码失败！");
|
||
iconv_close(cd);
|
||
return {};
|
||
}
|
||
else
|
||
{
|
||
size_t num = SIZE_BUFFER_STRING_CONVERT - out_size;
|
||
BufferAppend(ret, CreateBufferView(temp, num));
|
||
//没有转换完，重设输出位置
|
||
out_size = SIZE_BUFFER_STRING_CONVERT;
|
||
out_buf = temp;
|
||
}
|
||
}
|
||
else
|
||
{//完整转换，增加num
|
||
assert(in_size == 0);
|
||
size_t num = SIZE_BUFFER_STRING_CONVERT - out_size;
|
||
if (ret.empty())//第一次就结束了，优化直接return
|
||
{
|
||
iconv_close(cd);
|
||
BufferAppend(ret, CreateBufferView(temp, num));
|
||
return ret;
|
||
}
|
||
BufferAppend(ret, CreateBufferView(temp, num));
|
||
}
|
||
} while (in_size);
|
||
|
||
iconv_close(cd);
|
||
return ret;
|
||
}
|
||
//#endif
|
||
|
||
namespace Utf8
|
||
{
|
||
int Check(std::string_view str)
|
||
{
|
||
unsigned n_bytes = 0;//UFT8可用1-6个字节编码,ASCII用一个字节
|
||
unsigned char ch;
|
||
bool b_all_ascii = true; //如果全部都是ASCII, 说明不是UTF-8
|
||
|
||
for (size_t i = 0; i < str.size(); ++i)
|
||
{
|
||
ch = str[i];
|
||
if ((ch & 0x80) != 0) // 判断是否ASCII编码,如果不是,说明有可能是UTF-8,ASCII用7位编码,但用一个字节存,最高位标记为0,o0xxxxxxx
|
||
b_all_ascii = false;
|
||
if (n_bytes == 0) //如果不是ASCII码,应该是多字节符,计算字节数
|
||
{
|
||
if (ch >= 0x80)
|
||
{
|
||
if (ch >= 0xFC && ch <= 0xFD)
|
||
n_bytes = 6;
|
||
else if (ch >= 0xF8)
|
||
n_bytes = 5;
|
||
else if (ch >= 0xF0)
|
||
n_bytes = 4;
|
||
else if (ch >= 0xE0)
|
||
n_bytes = 3;
|
||
else if (ch >= 0xC0)
|
||
n_bytes = 2;
|
||
else
|
||
return 0;
|
||
|
||
--n_bytes;
|
||
}
|
||
}
|
||
else //多字节符的非首字节,应为 10xxxxxx
|
||
{
|
||
if ((ch & 0xC0) != 0x80)
|
||
return 0;
|
||
|
||
--n_bytes;
|
||
}
|
||
}
|
||
if (n_bytes > 0) //违返规则mb
|
||
return 0;
|
||
if (b_all_ascii) //如果全部都是ASCII, 说明不是UTF-8
|
||
return 1;
|
||
|
||
return 2;//utf8
|
||
}
|
||
|
||
void Copy(char* target, size_t size, std::string_view source, bool show_log /*= true*/)
|
||
{
|
||
if (size == 0)
|
||
{
|
||
log_warn0("缓冲区大小为0！");
|
||
return;
|
||
}
|
||
|
||
//如果加上结束符 超过缓冲区
|
||
size_t size_source = source.size();
|
||
if (size_source + 1 > size)
|
||
{
|
||
//std::string str_mb = CodeCvt::cvt_u8_mb(source);
|
||
|
||
if (show_log)
|
||
log_warn("字符串过比缓冲区大：{}", source);
|
||
|
||
size_t sub_size = Sub(source, size - 1);
|
||
|
||
Memory::Copy(target, size - 1, source.data(), sub_size);
|
||
|
||
target[sub_size] = 0;
|
||
}
|
||
else
|
||
{
|
||
memcpy(target, source.data(), size_source);
|
||
target[size_source] = 0;
|
||
}
|
||
}
|
||
|
||
size_t Sub(std::string_view str, size_t max)
|
||
{
|
||
if (max == 0)
|
||
return 0;
|
||
|
||
//源下标（与子串下标相同）
|
||
size_t i = 0;
|
||
|
||
size_t size = str.length();
|
||
size_t cut_len = 0;
|
||
|
||
unsigned char ch;
|
||
while (i < size)
|
||
{
|
||
ch = (unsigned char)str[i];
|
||
if (ch >= 252)
|
||
cut_len = 6;
|
||
else if (ch >= 248)
|
||
cut_len = 5;
|
||
else if (ch >= 240)
|
||
cut_len = 4;
|
||
else if (ch >= 224)
|
||
cut_len = 3;
|
||
else if (ch >= 192)
|
||
cut_len = 2;
|
||
else if (ch >= 65 && ch <= 90)
|
||
cut_len = 1;
|
||
else
|
||
cut_len = 1;
|
||
|
||
if (i + cut_len > max)
|
||
return i;
|
||
i += cut_len;
|
||
}
|
||
|
||
return size;
|
||
}
|
||
|
||
|
||
}
|
||
|
||
|
||
|
||
std::string Utf8ToMultiByte(std::string_view str, std::string_view code_page /*= ""*/)
|
||
{
|
||
Buffer buf = CvtString("UTF-8", code_page, CreateBufferView(str));
|
||
return to_string(buf);
|
||
}
|
||
|
||
std::string MultiByteToUtf8(const char* str, std::string_view code_page)
|
||
{
|
||
Buffer buf = CvtString(code_page, "UTF-8", CreateBufferView((char*)str, strlen(str)));
|
||
return to_string(buf);
|
||
}
|
||
|
||
std::string MultiByteToUtf8(std::string_view str, std::string_view code_page)
|
||
{
|
||
Buffer buf = CvtString(code_page, "UTF-8", CreateBufferView(str));
|
||
return to_string(buf);
|
||
}
|
||
|
||
// UTF-16 UTF-32 会带上bom，参考：https://zzz.buzz/2016/07/30/bom-in-iconv/
|
||
|
||
std::string Utf32ToUtf8(std::u32string_view str)
|
||
{
|
||
Buffer buf = CvtString("UTF-32LE", "UTF-8",
|
||
CreateBufferView((char*)str.data(), str.size() * sizeof(char32_t)));
|
||
return to_string(buf);
|
||
}
|
||
|
||
std::u32string Utf8ToUtf32(std::string_view str)
|
||
{
|
||
Buffer buf = CvtString("UTF-8", "UTF-32LE", CreateBufferView(str));
|
||
|
||
assert(buf.size() % sizeof(char32_t) == 0);// 必须是sizeof(char32_t)的倍数
|
||
std::u32string ret((char32_t*)buf.data(), buf.size() / sizeof(char32_t));
|
||
return ret;
|
||
}
|
||
|
||
std::string WideCharToUtf8(std::wstring_view str)
|
||
{
|
||
Buffer buf = CvtString("wchar_t", "UTF-8",
|
||
CreateBufferView((char*)str.data(), str.size() * sizeof(wchar_t)));
|
||
return to_string(buf);
|
||
}
|
||
|
||
std::wstring Utf8ToWideChar(std::string_view str)
|
||
{
|
||
Buffer buf = CvtString("UTF-8", "wchar_t", CreateBufferView(str));
|
||
|
||
std::wstring ret((wchar_t*)buf.data(), buf.size() / sizeof(wchar_t));
|
||
return ret;
|
||
}
|
||
|
||
// std::wstring cvt_u8_wc(string_view str)
|
||
// {
|
||
// string out = CvtString("UTF-8", "wchar_t", str.data(), str.size());
|
||
//
|
||
// dnd_assert(out.size() % sizeof(wchar_t) == 0);//必须是sizeof(wchar_t)的倍数
|
||
// std::wstring ret((wchar_t*)out.c_str(), out.size() / sizeof(wchar_t));
|
||
// return ret;
|
||
// }
|
||
//
|
||
// string cvt_u8_mb(string_view str)
|
||
// {
|
||
// return CvtString("UTF-8", "", str.data(), str.size());
|
||
// }
|
||
//
|
||
// dnd::utf_string cvt_u8_utf(string_view str)
|
||
// {
|
||
// string out = CvtString("UTF-8", "UCS-4-INTERNAL", str.data(), str.size());
|
||
//
|
||
// dnd_assert(out.size() % sizeof(char32_t) == 0);//必须是sizeof(char32_t)的倍数
|
||
// dnd::utf_string ret((char32_t*)out.c_str(), out.size() / sizeof(char32_t));
|
||
// return ret;
|
||
// }
|
||
//
|
||
// string cvt_wc_u8(const wchar_t* str)
|
||
// {
|
||
// size_t size = wcslen(str) * sizeof(wchar_t);
|
||
// return CvtString("wchar_t", "UTF-8", (char*)str, size);
|
||
// }
|
||
//
|
||
// std::string cvt_wc_u8(std::wstring_view str)
|
||
// {
|
||
// return CvtString("wchar_t", "UTF-8", (char*)str.data(), str.size() * sizeof(wchar_t));
|
||
// }
|
||
//
|
||
//
|
||
//
|
||
// string cvt_wc_mb(const wchar_t* str, string_view code_name)
|
||
// {
|
||
// size_t size = wcslen(str) * sizeof(wchar_t);
|
||
// return CvtString("wchar_t", code_name, (char*)str, size);
|
||
// }
|
||
//
|
||
|
||
// std::string cvt_mb_u8(string_view str, string_view code_name /*= ""*/)
|
||
// {
|
||
// return CvtString(code_name, "UTF-8", str.data(), str.size());
|
||
// }
|
||
//
|
||
//
|
||
//
|
||
// std::wstring cvt_mb_wc(const char* str, string_view code_name)
|
||
// {
|
||
// string out = CvtString(code_name, "wchar_t", str, strlen(str));
|
||
//
|
||
// dnd_assert(out.size() % sizeof(wchar_t) == 0);//必须是sizeof(wchar_t)的倍数
|
||
// std::wstring ret((wchar_t*)out.c_str(), out.size() / sizeof(wchar_t));
|
||
// return ret;
|
||
// }
|
||
//
|
||
// string cvt_utf_u8(const utf_string& str)
|
||
// {
|
||
// return CvtString("UCS-4-INTERNAL", "UTF-8", (char*)str.c_str(), str.size() * sizeof(wchar_t));
|
||
// }
|
||
//
|
||
//
|
||
//size_t ANSIExtend::GbkSubIndex(std::string_view str, size_t max)
|
||
//{
|
||
// // 足够容纳，则全部返回
|
||
// if (str.size() <= max)
|
||
// return str.size();
|
||
//
|
||
// // end为普通字符
|
||
// if (str[max] > 0)
|
||
// return max;
|
||
//
|
||
// // end-1为普通字符
|
||
// if (str[max - 1] > 0)
|
||
// return max;
|
||
//
|
||
// // end-1不为普通字符，end-2为普通字符
|
||
// if (str[max - 2] > 0)
|
||
// return max - 1;
|
||
//
|
||
// // end-1不为普通字符，end-2不为普通字符，end不为普通字符
|
||
// // 往前遍历，直到普通字符，或者头部
|
||
// size_t cur = max - 3;
|
||
// while (true)
|
||
// {
|
||
// if (str[cur] > 0)
|
||
// {
|
||
// size_t dis =
|
||
// }
|
||
// }
|
||
// return max - 2;
|
||
//}
|
||
|
||
} |