hyzboy/CMCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support "u8char" type

Browse Source
This commit is contained in:
hyzboy
2020-07-07 19:14:42 +08:00
parent 5e67c0cef7
commit f6a846c655
22 changed files with 92 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
inc/hgl/CodePage.h
View File

@@ -85,7 +85,7 @@ namespace hgl
constexpr int CharPageCount=sizeof(CodeSet2CharPage)/sizeof(CodePageAndCharSet);
inline CharCodePage FindCodePage(const char *char_set)
inline CharCodePage FindCodePage(const u8char *char_set)
{
for(int i=0;i<CharPageCount;i++)
if(!charset_cmp(CodePage2CharSet[i].charset,char_set))
@@ -104,17 +104,17 @@ namespace hgl
CharSet()
{
codepage=ccpNone;
strcpy(charset,CharSetNameLength,"us-ascii");
hgl::strcpy(charset,CharSetNameLength,"us-ascii");
}
CharSet(CharCodePage ccp,const char *cs)
{
codepage=ccp;
strcpy(charset,CharSetNameLength,cs);
hgl::strcpy(charset,CharSetNameLength,cs);
}
CharSet(CharCodePage);
CharSet(const char *);
CharSet(const u8char *);
CharSet(const CodePageAndCharSet &cs)
{
@@ -129,13 +129,13 @@ namespace hgl
inline CharSet::CharSet(CharCodePage ccp)
{
codepage=ccp;
strcpy(charset,CharSetNameLength,FindCharSet(ccp));
hgl::strcpy(charset,CharSetNameLength,FindCharSet(ccp));
}
inline CharSet::CharSet(const char *cs)
inline CharSet::CharSet(const u8char *cs)
{
codepage=FindCodePage(cs);
strcpy(charset,CharSetNameLength,FindCharSet(codepage));
hgl::strcpy(charset,CharSetNameLength,FindCharSet(codepage));
}
extern CharSet DefaultCharSet();
@@ -152,9 +152,9 @@ namespace hgl
* @param src_size 字符串长度,-1表示全部
* @return 转换成功后的字符串字符数
*/
int to_utf16(const CharSet &charset,u16char **dst,const char *src,const int src_size=-1);
int to_utf16(const CharSet &charset,u16char **dst,const void *src,const int src_size=-1);
int to_utf8(const CharSet &charset,char **dst,const char *src,const int src_size=-1);
int to_utf8(const CharSet &charset,u8char **dst,const void *src,const int src_size=-1);
/**
* 转换u16char *字符串到指定字符集的字符串
@@ -164,29 +164,29 @@ namespace hgl
* @param src_size 字符串长度,-1表示全部
* @return 转换成功后的字符串字符数
*/
int utf16_to(const CharSet &charset,char **dst,const u16char *src,const int src_size=-1);
int utf16_to(const CharSet &charset,u8char **dst,const u16char *src,const int src_size=-1);
int utf8_to(const CharSet &charset,char **dst,const char *src,const int src_size=-1);
int utf8_to(const CharSet &charset,u8char **dst,const u8char *src,const int src_size=-1);
int u16_to_u8(char *,int,const u16char *,const int=-1); ///<转换u16char *到utf8格式的char *
int u8_to_u16(u16char *,int,const char *,const int=-1); ///<转换utf8格式的char *到u16char *
int u16_to_u8(u8char *,int,const u16char *,const int=-1); ///<转换u16char *到utf8格式的u8char *
int u8_to_u16(u16char *,int,const u8char *,const int=-1); ///<转换utf8格式的u8char *到u16char *
char * u16_to_u8(const u16char *,const int,int &); ///<转换u16char *到utf8格式的char *
u16char * u8_to_u16(const char *,const int,int &); ///<转换utf8格式的char *到u16char *
u8char * u16_to_u8(const u16char *,const int,int &); ///<转换u16char *到utf8格式的u8char *
u16char * u8_to_u16(const u8char *,const int,int &); ///<转换utf8格式的u8char *到u16char *
inline char * u16_to_u8(const u16char *str)
inline u8char * u16_to_u8(const u16char *str)
{
int len;
return u16_to_u8(str,hgl::strlen<u16char>(str)+1,len);
}
inline u16char *u8_to_u16(const char *str)
inline u16char *u8_to_u16(const u8char *str)
{
int len;
return u8_to_u16(str,hgl::strlen<char>(str)+1,len);
return u8_to_u16(str,hgl::strlen<u8char>(str)+1,len);
}
inline UTF16String to_u16(const char *u8_str,int length)
inline UTF16String to_u16(const u8char *u8_str,int length)
{
int wlen;
u16char *ws=u8_to_u16(u8_str,length,wlen);
@@ -199,7 +199,7 @@ namespace hgl
return to_u16(u8str.c_str(),u8str.Length());
}
inline UTF16String to_u16(const char *str)
inline UTF16String to_u16(const u8char *str)
{
int wlen;
@@ -212,7 +212,7 @@ namespace hgl
{
int ulen;
char *us=u16_to_u8(wide_str,length,ulen);
u8char *us=u16_to_u8(wide_str,length,ulen);
return UTF8String::newOf(us,ulen);
}
@@ -223,7 +223,7 @@ namespace hgl
}
#if HGL_OS == HGL_OS_Windows
inline OSString ToOSString(const char *str){return to_u16(str);}
inline OSString ToOSString(const u8char *str){return to_u16(str);}
inline OSString ToOSString(const UTF8String &str){return to_u16(str.c_str(), (int)(str.Length()));}
inline UTF8String ToUTF8String(const os_char *str){return to_u8(str,strlen(str));}

8
inc/hgl/Endian.h
View File

@@ -58,10 +58,10 @@ namespace hgl
bomEnd
};
constexpr uint CharSetNameLength=32; ///<字符集名称长度
using CharSetName=char[CharSetNameLength]; ///<字符集名称类型定义
constexpr uint CharSetNameLength=32; ///<字符集名称长度
using CharSetName=char[CharSetNameLength]; ///<字符集名称类型定义
template<int,char> const CharSetName &GetCurCharSet(); ///<取得当前程序编码字符集
template<int,char> const CharSetName &GetCurCharSet(); ///<取得当前程序编码字符集
constexpr CharSetName utf8_charset="utf8";
constexpr CharSetName utf16le_charset="utf-16le";
@@ -85,7 +85,7 @@ namespace hgl
struct BOMFileHeader
{
int size; ///<字节序文件头长度
unsigned char data[4]; ///<字节序数据
uint8 data[4]; ///<字节序数据
ByteOrderMask bom; ///<字节序枚举
const CharSetName *char_set;///<字符集名称

2
inc/hgl/filesystem/FileSystem.h
View File

@@ -147,7 +147,7 @@ namespace hgl
}
inline UTF8String MergeFilename(const UTF8String &pathname,const UTF8String &filename) ///<组合路径名与文件名
{return MergeFilename<char>(pathname,filename,HGL_DIRECTORY_SEPARATOR,HGL_DIRECTORY_SEPARATOR_U8STR);}
{return MergeFilename<u8char>(pathname,filename,HGL_DIRECTORY_SEPARATOR,HGL_DIRECTORY_SEPARATOR_U8STR);}
inline WideString MergeFilename(const WideString &pathname,const WideString &filename) ///<组合路径名与文件名
{return MergeFilename<wchar_t>(pathname,filename,L'\\',L"\\");}

14
inc/hgl/io/DataOutputStream.h
View File

@@ -140,7 +140,7 @@ namespace hgl
template<typename T> bool WriteUTF16LEChars (const BaseString<T> &str){return WriteUTF16LEChars (str.c_str(),str.Length());}
template<typename T> bool WriteUTF16BEChars (const BaseString<T> &str){return WriteUTF16BEChars (str.c_str(),str.Length());}
template<typename T> bool WriteUTF8StringWithLength (const char *str,const uint length);
template<typename T> bool WriteUTF8StringWithLength (const u8char *str,const uint length);
template<typename T> bool WriteUTF8StringWithLength (const UTF16String &str);
template<uchar C,typename T> bool WriteUTF16StringWithLength(const u16char *str,const uint len);
@@ -148,8 +148,8 @@ namespace hgl
public:
//32 bit length
bool WriteUTF8String (const char *str,uint length); ///<按utf16-le格式写入字符串(前置4字节字符串长度,再写入字符阵列)
bool WriteUTF8String (const char *str ); ///<按utf8格式写入字符串(前置4字节字符串长度,再写入字符阵列)
bool WriteUTF8String (const u8char *str,uint length); ///<按utf16-le格式写入字符串(前置4字节字符串长度,再写入字符阵列)
bool WriteUTF8String (const u8char *str ); ///<按utf8格式写入字符串(前置4字节字符串长度,再写入字符阵列)
bool WriteUTF8String (const UTF8String &str ); ///<按utf8格式写入字符串(前置4字节字符串长度,再写入字符阵列)
bool WriteUTF8String (const UTF16String &str ); ///<按utf8格式写入字符串(前置4字节字符串长度,再写入字符阵列)
@@ -166,8 +166,8 @@ namespace hgl
bool WriteUTF16BEString (const UTF8String &str ){return WriteUTF16BEString(to_u16(str));} ///<按utf16-be格式写入字符串(前置4字节字符串长度,再写入字符阵列)
//16 bit length
bool WriteUTF8ShortString (const char *str,uint length); ///<按utf16-le格式写入字符串(前置2字节字符串长度,再写入字符阵列)
bool WriteUTF8ShortString (const char *str ); ///<按utf8格式写入字符串(前置2字节字符串长度,再写入字符阵列)
bool WriteUTF8ShortString (const u8char *str,uint length); ///<按utf16-le格式写入字符串(前置2字节字符串长度,再写入字符阵列)
bool WriteUTF8ShortString (const u8char *str ); ///<按utf8格式写入字符串(前置2字节字符串长度,再写入字符阵列)
bool WriteUTF8ShortString (const UTF8String &str ); ///<按utf8格式写入字符串(前置2字节字符串长度,再写入字符阵列)
bool WriteUTF8ShortString (const UTF16String &str ); ///<按utf8格式写入字符串(前置2字节字符串长度,再写入字符阵列)
@@ -184,8 +184,8 @@ namespace hgl
bool WriteUTF16BEShortString(const UTF8String &str ){return WriteUTF16BEShortString(to_u16(str));} ///<按utf16-be格式写入字符串(前置2字节字符串长度,再写入字符阵列)
//8 bit length
bool WriteUTF8TinyString (const char *str,uint length); ///<按utf16-le格式写入字符串(前置1字节字符串长度,再写入字符阵列)
bool WriteUTF8TinyString (const char *str ); ///<按utf8格式写入字符串(前置1字节字符串长度,再写入字符阵列)
bool WriteUTF8TinyString (const u8char *str,uint length); ///<按utf16-le格式写入字符串(前置1字节字符串长度,再写入字符阵列)
bool WriteUTF8TinyString (const u8char *str ); ///<按utf8格式写入字符串(前置1字节字符串长度,再写入字符阵列)
bool WriteUTF8TinyString (const UTF8String &str ); ///<按utf8格式写入字符串(前置1字节字符串长度,再写入字符阵列)
bool WriteUTF8TinyString (const UTF16String &str ); ///<按utf8格式写入字符串(前置1字节字符串长度,再写入字符阵列)

4
inc/hgl/io/TextOutputStream.h
View File

@@ -55,7 +55,7 @@ namespace hgl
return(out->Write(bom_item->data,bom_item->size)==bom_item->size);
}
virtual bool WriteChars(const char *,int64)=0; ///<写入一个字符串
virtual bool WriteChars(const u8char *,int64)=0; ///<写入一个字符串
virtual bool WriteChars(const u16char *,int64)=0; ///<写入一个字符串
template<typename N>
@@ -162,7 +162,7 @@ namespace hgl
EndianTextOutputStream(OutputStream *);
bool WriteChars(const char *,int64); ///<写入一个字符串
bool WriteChars(const u8char *,int64); ///<写入一个字符串
bool WriteChars(const u16char *,int64); ///<写入一个字符串
};//template<ByteOrderMask BOM> class EndianTextOutputStream

9
inc/hgl/log/LogInfo.h
View File

@@ -30,18 +30,23 @@ namespace hgl
}
inline void Log(LogLevel ll,const UTF8String &str)
{
std::cout<<(char *)str.c_str()<<std::endl;
}
inline void Log(LogLevel ll,const AnsiString &str)
{
std::cout<<str.c_str()<<std::endl;
}
inline void DebugLog(LogLevel ll,const UTF16String &str,const char *filename,int line,const char *funcname)
{
Log(ll,str+U16_TEXT(">>LogFrom(\"")+to_u16(filename)+U16_TEXT("\", ")+UTF16String::valueOf(line)+U16_TEXT(" line,func:\"")+to_u16(funcname)+U16_TEXT("\")"));
Log(ll,str+U16_TEXT(">>LogFrom(\"")+to_u16((u8char *)filename)+U16_TEXT("\", ")+UTF16String::valueOf(line)+U16_TEXT(" line,func:\"")+to_u16((u8char *)funcname)+U16_TEXT("\")"));
}
inline void DebugLog(LogLevel ll,const UTF8String &str,const char *filename,int line,const char *funcname)
{
Log(ll,str+U8_TEXT(">>LogFrom(\"")+UTF8String(filename)+U8_TEXT("\", ")+UTF8String::valueOf(line)+U8_TEXT(" line,func:\"")+UTF8String(funcname)+U8_TEXT("\")"));
Log(ll,str+U8_TEXT(">>LogFrom(\"")+UTF8String((u8char *)filename)+U8_TEXT("\", ")+UTF8String::valueOf(line)+U8_TEXT(" line,func:\"")+UTF8String((u8char *)funcname)+U8_TEXT("\")"));
}
#define LOG_INFO(str) {Log(llLog, str);}

2
inc/hgl/log/Logger.h
View File

@@ -35,7 +35,7 @@ namespace hgl
virtual void Close()=0; ///<关闭日志
virtual void Write(const u16char *,int)=0; ///<输出一行u16char日志
virtual void Write(const char *,int)=0; ///<输出一行char(utf8)日志
virtual void Write(const u8char *,int)=0; ///<输出一行u8char日志
};//class Logger
}//namespace logger

2
inc/hgl/platform/os/CharType.h
View File

@@ -3,7 +3,9 @@
using u32char =char32_t;
using u16char =char16_t;
using u8char =char8_t;
using os_char =char;
#define to_oschar to_u8
#define OS_TEXT(str) u8##str
#define U8_TEXT(str) u8##str

3
inc/hgl/platform/os/MSWindows.h
View File

@@ -4,13 +4,16 @@
#include<malloc.h>
#include<winsock2.h> //winsock2必须在Windows.h前面，不然会报错
#include<windows.h>
#include<uchar.h>
#undef min
#undef max
//--------------------------------------------------------------------------------------------------
using u32char =char32_t;
using u16char =wchar_t;
using u8char =char8_t;
using os_char =wchar_t;
#define to_oschar to_u16
#define OS_TEXT(str) L##str
#define U8_TEXT(str) u8##str

2
inc/hgl/thread/Thread.h
View File

@@ -58,7 +58,7 @@ namespace hgl
virtual bool ProcStartThread() ///<线程启动运行函数,在Execute前被调用
{
#ifdef _DEBUG
char thread_addr[(sizeof(thread_ptr)<<1)+1];
u8char thread_addr[(sizeof(thread_ptr)<<1)+1];
DataToUpperHexStr(thread_addr,(uint8 *)&tp,sizeof(thread_ptr));

6
inc/hgl/type/BaseString.h
View File

@@ -1289,12 +1289,12 @@ namespace hgl
return BaseString<T>(value)+str;
}
using UTF8String =BaseString<char>;
using AnsiString =BaseString<char>;
using UTF8String =BaseString<u8char>;
using UTF16String =BaseString<u16char>;
using OSString =BaseString<os_char>;
using UTF32String =BaseString<char32_t>;
using OSString =BaseString<os_char>;
using WideString =BaseString<wchar_t>;
template<typename C> bool ToNumber(const BaseString<C> &str,int &value){return str.ToInt(value);}

7
inc/hgl/type/StdString.h
View File

@@ -7,14 +7,14 @@
#if HGL_OS == HGL_OS_Windows
inline hgl::OSString std_to_os(const std::string &str)
{
return hgl::to_u16(str.c_str(),(int)(str.length()));
return hgl::to_u16((u8char *)str.c_str(),(int)(str.length()));
}
inline std::string to_std(const hgl::OSString &str)
{
hgl::UTF8String u8_str=hgl::to_u8(str);
return std::string(u8_str.c_str(),str.Length());
return std::string((char *)u8_str.c_str(),str.Length());
}
#else
inline hgl::OSString std_to_os(const std::string &str)
@@ -30,6 +30,5 @@ inline std::string to_std(const hgl::OSString &str)
inline hgl::UTF8String std_to_u8(const std::string &str)
{
return hgl::UTF8String(str.c_str(),int(str.size()));
return hgl::UTF8String((u8char *)str.c_str(),int(str.size()));
}

2
inc/hgl/type/StringList.h
View File

@@ -639,7 +639,7 @@ namespace hgl
return(result);
}//int LoadStringList
inline int LoadUTF8StringList (UTF8StringList & sl,io::DataInputStream *dis){return LoadStringList<char, bomUTF8 >(sl,dis);}
inline int LoadUTF8StringList (UTF8StringList & sl,io::DataInputStream *dis){return LoadStringList<u8char, bomUTF8 >(sl,dis);}
inline int LoadUTF16LEStringList (UTF16StringList & sl,io::DataInputStream *dis){return LoadStringList<u16char, bomUTF16LE >(sl,dis);}
inline int LoadUTF16BEStringList (UTF16StringList & sl,io::DataInputStream *dis){return LoadStringList<u16char, bomUTF16BE >(sl,dis);}