utf8.c (3580B)
1 /* This program is licensed under the terms of GNU GPL v3 or (at your option) 2 * any later version. Copyright (C) 2020-2026 Страхиња Радић. 3 * See the file LICENSE for exact copyright and license details. */ 4 5 #include "utf8.h" 6 7 /* 8 * 00000000 -- 0000007F: 0xxxxxxx 9 * (2^7 = 128 chars) 10 * 00000080 -- 000007FF: 110xxxxx 10xxxxxx 11 * (2^5 = 32 chars) 12 * 00000800 -- 0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx 13 * (2^4 = 16 chars) 14 * 00010000 -- 001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 15 * (2^3 = 8 chars) 16 * 00200000 -- 007FFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 17 * (2^2 = 4 chars) 18 * 00800000 -- 00FFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 19 * 10xxxxxx 20 * (2^1 = 2 chars) 21 */ 22 23 /* clang-format off */ 24 const int utf_length_table[256] = { 25 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 26 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 32 */ 27 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 64 */ 29 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 30 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 96 */ 31 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 32 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 128 */ 33 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 34 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 160 */ 35 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 36 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 192 */ 37 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 38 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 224 */ 39 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 40 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1, /* 256 */ 41 }; 42 /* clang-format on */ 43 44 const int extract_masks[6] = {XMASK1, XMASK2, XMASK3, XMASK4, XMASK5, XMASK6}; 45 46 int 47 u8_rune_to_u32(u32* to, const u8* from, size_t* from_delta) 48 { 49 const u8* pfrom = from; 50 if (!from) 51 return 1; 52 *from_delta = 0; 53 int len = utf_length_table[*from]; 54 *to = *from & extract_masks[len - 1]; 55 for (int i = 1; i < len; i++) 56 { 57 pfrom = &from[i]; 58 if (!*pfrom) 59 { 60 *from_delta = 1; 61 *to = U32_REPL_CHAR; 62 return 1; 63 } 64 *to <<= 6; 65 *to |= *pfrom & XMASKR; 66 } 67 *from_delta = len; 68 69 return 0; 70 } 71 72 int 73 u32_rune_to_u8(u8* to, const u32 from) 74 { 75 u32 cfrom = from; 76 u8 start = 0; 77 int len = 0; 78 if (from >= BOUND6) 79 { 80 start = START6; 81 len = 6; 82 } 83 else if (from >= BOUND5) 84 { 85 start = START5; 86 len = 5; 87 } 88 else if (from >= BOUND4) 89 { 90 start = START4; 91 len = 4; 92 } 93 else if (from >= BOUND3) 94 { 95 start = START3; 96 len = 3; 97 } 98 else if (from >= BOUND2) 99 { 100 start = START2; 101 len = 2; 102 } 103 else 104 { 105 start = START1; 106 len = 1; 107 } 108 for (int i = len - 1; i > 0; i--) 109 { 110 to[i] = STARTR | (cfrom & XMASKR); 111 cfrom >>= 6; 112 } 113 to[0] = start | cfrom; 114 return len; 115 } 116 117 int 118 u8_to_u32(u32* to, const u8* from, const ssize_t max, size_t* from_delta) 119 { 120 const u8* pfrom = from; 121 u32* pto = to; 122 if (!from) 123 return 1; 124 size_t delta = 0; 125 *from_delta = 0; 126 while (pfrom - from < max && *pfrom) 127 { 128 int result = u8_rune_to_u32(pto, pfrom, &delta); 129 if (result) 130 return result; 131 pto++; 132 pfrom += delta; 133 *from_delta += delta; 134 } 135 *pto = 0; 136 return 0; 137 } 138 139 int 140 u32_to_u8(u8* to, const u32* from, const ssize_t max) 141 { 142 const u32* pfrom = from; 143 u8* pto = to; 144 int len = 0; 145 if (!from) 146 return 0; 147 while (pfrom - from < max && *pfrom) 148 { 149 size_t delta = u32_rune_to_u8(pto, *pfrom); 150 pto += delta; 151 len += delta; 152 pfrom++; 153 } 154 return len; 155 } 156 157 size_t 158 u32_strlen(const u32* s, const ssize_t max) 159 { 160 const u32* ps = s; 161 while (ps && ps - s < max && *ps) 162 ps++; 163 return ps - s; 164 }