utf8.c (3366B)
1 /* This program is licensed under the terms of GNU GPL v3 or (at your option) 2 * any later version. Copyright (C) 2021-2025 Страхиња Радић. 3 * See the file LICENSE for exact copyright and license details. */ 4 5 #include "utf8.h" 6 7 /* 8 * 00000000 -- 0000007F: 0xxxxxxx 9 * (2^7 = 128 chars) 10 * 00000080 -- 000007FF: 110xxxxx 10xxxxxx 11 * (2^5 = 32 chars) 12 * 00000800 -- 0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx 13 * (2^4 = 16 chars) 14 * 00010000 -- 001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 15 * (2^3 = 8 chars) 16 * 00200000 -- 007FFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 17 * (2^2 = 4 chars) 18 * 00800000 -- 00FFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 19 * 10xxxxxx 20 * (2^1 = 2 chars) 21 */ 22 23 const int 24 utf_length_table[256] = { 25 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 26 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 32 */ 27 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 64 */ 29 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 30 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 96 */ 31 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 32 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 128 */ 33 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 34 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 160 */ 35 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 36 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 192 */ 37 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 38 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 224 */ 39 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 40 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, /* 256 */ 41 }; 42 43 const int 44 extract_masks[6] = { 45 XMASK1, XMASK2, XMASK3, XMASK4, XMASK5, XMASK6 46 }; 47 48 int 49 u8_char_to_u32(u32* to, const u8 *from, size_t* from_delta) 50 { 51 const u8* pfrom = from; 52 if (!from) 53 return 1; 54 *from_delta = 0; 55 int len = utf_length_table[*from]; 56 *to = *from & extract_masks[len-1]; 57 for (int i = 1; i < len; i++) 58 { 59 pfrom = from + i; 60 if (!*pfrom) 61 return 1; 62 *to <<= 6; 63 *to |= *pfrom & XMASKR; 64 } 65 *from_delta = len; 66 67 return 0; 68 } 69 70 size_t 71 u32_char_to_u8(u8 *to, const u32 from) 72 { 73 size_t len = 0; 74 u8 start = 0; 75 u32 cfrom = from; 76 if (from >= BOUND6) 77 { 78 start = START6; 79 len = 6; 80 } 81 else if (from >= BOUND5) 82 { 83 start = START5; 84 len = 5; 85 } 86 else if (from >= BOUND4) 87 { 88 start = START4; 89 len = 4; 90 } 91 else if (from >= BOUND3) 92 { 93 start = START3; 94 len = 3; 95 } 96 else if (from >= BOUND2) 97 { 98 start = START2; 99 len = 2; 100 } 101 else 102 { 103 start = START1; 104 len = 1; 105 } 106 for (int i = len-1; i > 0; i--) 107 { 108 to[i] = STARTR | (cfrom & XMASKR); 109 cfrom >>= 6; 110 } 111 to[0] = start | cfrom; 112 return len; 113 } 114 115 int 116 u8_to_u32(u32 *to, const u8 *from, size_t* from_delta) 117 { 118 const u8* pfrom = from; 119 u32* pto = to; 120 if (!from) 121 return 1; 122 size_t delta = 0; 123 *from_delta = 0; 124 while (*pfrom) 125 { 126 int result = u8_char_to_u32(pto, pfrom, &delta); 127 if (result) 128 return result; 129 pto++; 130 pfrom += delta; 131 *from_delta += delta; 132 } 133 *pto = 0; 134 return 0; 135 } 136 137 size_t 138 u32_to_u8(u8 *to, const u32* from) 139 { 140 u8* pto = to; 141 const u32* pfrom = from; 142 size_t len = 0; 143 if (!from) 144 return 0; 145 while (*pfrom) 146 { 147 size_t delta = u32_char_to_u8(pto, *pfrom); 148 pto += delta; 149 len += delta; 150 pfrom++; 151 } 152 return len; 153 } 154 155 size_t 156 u32_strlen(const u32* s) 157 { 158 const u32* ps = s; 159 while (ps && *ps) 160 ps++; 161 return ps - s; 162 }