1
2
3

wchar_t wc = L'中';
wchar_t wc_array[] = L"破曉的博客";
std::wstring wstr = L"破曉的博客";

1

wchar_t wc = L'??'; // U+2A6A5

1
2
3
4
5
6
7
8
9
10
11

// utf-8
char u8_array[] = u8"破曉的博客";
std::string u8_str = u8"破曉的博客";
// utf-16
char16_t u16_c = u'中';
char16_t u16_array[] = u"破曉的博客";
std::u16string u16_str = u"破曉的博客";
// ucs4
char32_t u32_c = U'??';
char32_t u32_array[] = U"破曉的博客";
std::u32string u32_str = U"破曉的博客";

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

#include <string>
#include <iostream>
#include <iomanip>
#include <type_traits>

template<typename tStringType, typename tTraits = typename tStringType::traits_type>
void print_code_unit_sequence(tStringType str)
{
  using char_type = typename tTraits::char_type;
  static_assert(std::is_same<char_type, char>::value || std::is_same<char_type, char16_t>::value || std::is_same<char_type, char32_t>::value, "error");
  using unsigned_char_type = typename std::make_unsigned<char_type>::type;
  using unsigned_int_type = typename std::make_unsigned<typename tTraits::int_type>::type;
  int w = std::is_same<char, char_type>::value ? 2 : std::is_same<char16_t, char_type>::value ? 4 : 8;
  for(auto c : str) {
    auto value = static_cast<unsigned_int_type>(static_cast<unsigned_char_type>(c));
    std::cout << "0x" << std::hex << std::uppercase << std::setw(w) << std::setfill('0') << value << ' ';
  }
}

int main()
{
  std::string u8_str = u8"??"; // utf-8
  std::u16string u16_str = u"??"; // utf-16
  std::u32string u32_str = U"??"; // ucs4
  std::cout << "utf-8: ";
  print_code_unit_sequence(u8_str);
  std::cout << std::endl;
  std::cout << "utf-16: ";
  print_code_unit_sequence(u16_str);
  std::cout << std::endl;
  std::cout << "ucs4: ";
  print_code_unit_sequence(u32_str);
  std::cout << std::endl;
}

1
2
3

utf-8: 0xF0 0xAA 0x9A 0xA5
utf-16: 0xD869 0xDEA5
ucs4: 0x0002A6A5

1
2
3
4
5

template<class Codecvt,
  class Elem = wchar_t,
  class Wide_alloc = std::allocator<Elem>,
  class Byte_alloc = std::allocator<char>>
class wstring_convert;

1
2
3
4

template<class Elem,
  unsigned long Maxcode = 0x10ffff,
  std::codecvt_mode Mode = (std::codecvt_mode)0>
class codecvt_utf8_utf16 : public std::codecvt<Elem, char, std::mbstate_t>;

1
2
3
4
5
6
7
8
9
10
11
12
13

std::string u8_source_str = u8"破曉的博客"; // utf-8
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> cvt;
std::u16string u16_cvt_str = cvt.from_bytes(u8_source_str); // utf-8 to utf-16
std::string u8_cvt_str = cvt.to_bytes(u16_cvt_str); // utf-16 to utf-8
std::cout << "u8_source_str = ";
print_code_unit_sequence(u8_source_str);
std::cout << std::endl;
std::cout << "u16_cvt_str = ";
print_code_unit_sequence(u16_cvt_str);
std::cout << std::endl;
std::cout << "u8_cvt_str = ";
print_code_unit_sequence(u8_cvt_str);
std::cout << std::endl;

1
2
3

u8_source_str = 0xE7 0xA0 0xB4 0xE6 0x99 0x93 0xE7 0x9A 0x84 0xE5 0x8D 0x9A 0xE5 0xAE 0xA2
u16_cvt_str = 0x7834 0x6653 0x7684 0x535A 0x5BA2
u8_cvt_str = 0xE7 0xA0 0xB4 0xE6 0x99 0x93 0xE7 0x9A 0x84 0xE5 0x8D 0x9A 0xE5 0xAE 0xA2

1
2
3
4

template<class Elem,
    unsigned long Maxcode = 0x10ffff,
    std::codecvt_mode Mode = (std::codecvt_mode)0>
class codecvt_utf8 : public std::codecvt<Elem, char, std::mbstate_t>;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

std::string u8_source_str = u8"破曉的博客"; // utf-8
std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> utf8_ucs2_cvt;
std::u16string ucs2_cvt_str = utf8_ucs2_cvt.from_bytes(u8_source_str); // utf-8 to ucs2
std::string u8_str_from_ucs2 = utf8_ucs2_cvt.to_bytes(ucs2_cvt_str); // ucs2 to utf-8
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8_ucs4_cvt;
std::u32string ucs4_cvt_str = utf8_ucs4_cvt.from_bytes(u8_source_str); // utf-8 to ucs4
std::string u8_str_from_ucs4 = utf8_ucs4_cvt.to_bytes(ucs4_cvt_str); // ucs4 to utf-8
std::cout << "u8_source_str: ";
print_code_unit_sequence(u8_source_str);
std::cout << std::endl;
std::cout << "ucs2_cvt_str: ";
print_code_unit_sequence(ucs2_cvt_str);
std::cout << std::endl;
std::cout << "u8_str_from_ucs2: ";
print_code_unit_sequence(u8_str_from_ucs2);
std::cout << std::endl;
std::cout << "ucs4_cvt_str: ";
print_code_unit_sequence(ucs4_cvt_str);
std::cout << std::endl;
std::cout << "u8_str_from_ucs4: ";
print_code_unit_sequence(u8_str_from_ucs4);
std::cout << std::endl;

1
2
3
4
5

u8_source_str: 0xE7 0xA0 0xB4 0xE6 0x99 0x93 0xE7 0x9A 0x84 0xE5 0x8D 0x9A 0xE5 0xAE 0xA2
ucs2_cvt_str: 0x7834 0x6653 0x7684 0x535A 0x5BA2
u8_str_from_ucs2: 0xE7 0xA0 0xB4 0xE6 0x99 0x93 0xE7 0x9A 0x84 0xE5 0x8D 0x9A 0xE5 0xAE 0xA2
ucs4_cvt_str: 0x00007834 0x00006653 0x00007684 0x0000535A 0x00005BA2
u8_str_from_ucs4: 0xE7 0xA0 0xB4 0xE6 0x99 0x93 0xE7 0x9A 0x84 0xE5 0x8D 0x9A 0xE5 0xAE 0xA2

1
2
3
4

template<class Elem,
  unsigned long Maxcode = 0x10ffff,
  std::codecvt_mode Mode = (std::codecvt_mode)0>
class codecvt_utf16 : public std::codecvt<Elem, char, std::mbstate_t>;

1
2
3
4
5

enum codecvt_mode {
  consume_header = 4,
  generate_header = 2,
  little_endian = 1
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

std::string u16le_byte_str = "\x34\x78\x53\x66"; // utf-16 Little Endian
std::string u16be_byte_str = "\x78\x34\x66\x53"; // utf-16 Big Endian
std::wstring_convert<std::codecvt_utf16<char32_t, 0x10ffff, std::little_endian>, char32_t> utf16le_cvt; // little endian
std::wstring_convert<std::codecvt_utf16<char32_t>, char32_t> utf16be_cvt; // default big endian
std::u32string u32_str_from_le = utf16le_cvt.from_bytes(u16le_byte_str); // utf-16 to ucs4
std::u32string u32_str_from_be = utf16be_cvt.from_bytes(u16be_byte_str); // utf-16 to ucs4
std::cout << "u16le_byte_str: ";
print_code_unit_sequence(u16le_byte_str);
std::cout << std::endl;
std::cout << "u16be_byte_str: ";
print_code_unit_sequence(u16be_byte_str);
std::cout << std::endl;
std::cout << "u32_str_from_le: ";
print_code_unit_sequence(u32_str_from_le);
std::cout << std::endl;
std::cout << "u32_str_from_be: ";
print_code_unit_sequence(u32_str_from_be);
std::cout << std::endl;

1
2
3
4

u16le_byte_str: 0x34 0x78 0x53 0x66
u16be_byte_str: 0x78 0x34 0x66 0x53
u32_str_from_le: 0x00007834 0x00006653
u32_str_from_be: 0x00007834 0x00006653

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

std::string u16le_byte_str = "\xff\xfe\x34\x78\x53\x66"; // utf-16 little endian with BOM
std::string u16be_byte_str = "\xfe\xff\x78\x34\x66\x53"; // utf-16 big endian with BOM
std::wstring_convert<std::codecvt_utf16<char32_t, 0x10ffff, std::consume_header>, char32_t> utf16le_cvt; // little endian
std::wstring_convert<std::codecvt_utf16<char32_t, 0x10ffff, std::consume_header>, char32_t> utf16be_cvt; // default big endian
std::u32string u32_str_from_le = utf16le_cvt.from_bytes(u16le_byte_str); // utf-16 to ucs4
std::u32string u32_str_from_be = utf16be_cvt.from_bytes(u16be_byte_str); // utf-16 to ucs4
std::cout << "u16le_byte_str: ";
print_code_unit_sequence(u16le_byte_str);
std::cout << std::endl;
std::cout << "u16be_byte_str: ";
print_code_unit_sequence(u16be_byte_str);
std::cout << std::endl;
std::cout << "u32_str_from_le: ";
print_code_unit_sequence(u32_str_from_le);
std::cout << std::endl;
std::cout << "u32_str_from_be: ";
print_code_unit_sequence(u32_str_from_be);
std::cout << std::endl;

1
2
3
4

u16le_byte_str: 0xFF 0xFE 0x34 0x78 0x53 0x66
u16be_byte_str: 0xFE 0xFF 0x78 0x34 0x66 0x53
u32_str_from_le: 0x00003478 0x00005366
u32_str_from_be: 0x00007834 0x00006653

1
2
3
4
5
6
7
8
9
10
11
12
13
14

std::u32string u32_str = U"破曉"; // ucs4
std::wstring_convert<std::codecvt_utf16<char32_t, 0x10ffff, static_cast<std::codecvt_mode>(std::generate_header | std::little_endian)>, char32_t> utf16le_cvt; // little endian
std::wstring_convert<std::codecvt_utf16<char32_t, 0x10ffff, static_cast<std::codecvt_mode>(std::generate_header)>, char32_t> utf16be_cvt; // default big endian
std::string u16le_byte_str = utf16le_cvt.to_bytes(u32_str); // ucs4 to utf-16 little endian with BOM
std::string u16be_byte_str = utf16be_cvt.to_bytes(u32_str); // ucs4 to utf-16 big endian with BOM
std::cout << "u32_str: ";
print_code_unit_sequence(u32_str);
std::cout << std::endl;
std::cout << "u16le_byte_str: ";
print_code_unit_sequence(u16le_byte_str);
std::cout << std::endl;
std::cout << "u16be_byte_str: ";
print_code_unit_sequence(u16be_byte_str);
std::cout << std::endl;

1
2
3

u32_str: 0x00007834 0x00006653
u16le_byte_str: 0xFF 0xFE 0x34 0x78 0x53 0x66
u16be_byte_str: 0xFE 0xFF 0x78 0x34 0x66 0x53