2.84
xs_unicode.h
1/* copyright (c) 2022 - 2025 grunfink et al. / MIT license */
2
3#ifndef _XS_UNICODE_H
4
5#define _XS_UNICODE_H
6
7 int xs_utf8_enc(char buf[4], unsigned int cpoint);
8 int xs_is_utf8_cont_byte(char c);
9 unsigned int xs_utf8_dec(const char **str);
10 int xs_unicode_width(unsigned int cpoint);
11 int xs_is_surrogate(unsigned int cpoint);
12 int xs_is_diacritic(unsigned int cpoint);
13 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
14 unsigned int xs_surrogate_enc(unsigned int cpoint);
15 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
16 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
17 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
18 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
19 unsigned int xs_unicode_to_upper(unsigned int cpoint);
20 unsigned int xs_unicode_to_lower(unsigned int cpoint);
21 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
22 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
23 int xs_unicode_is_alpha(unsigned int cpoint);
24 int xs_unicode_is_right_to_left(unsigned int cpoint);
25
26#ifdef _XS_H
27 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
28 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
29 xs_str *xs_utf8_to_upper(const char *str);
30 xs_str *xs_utf8_to_lower(const char *str);
31 xs_str *xs_utf8_to_nfd(const char *str);
32 xs_str *xs_utf8_to_nfc(const char *str);
33#endif
34
35#ifdef XS_IMPLEMENTATION
36
37#include <ctype.h>
38
39#ifndef xs_countof
40#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
41#endif
42
43int xs_utf8_enc(char buf[4], unsigned int cpoint)
44/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
45{
46 char *p = buf;
47
48 if (cpoint < 0x80) /* 1 byte char */
49 *p++ = cpoint & 0xff;
50 else {
51 if (cpoint < 0x800) /* 2 byte char */
52 *p++ = 0xc0 | (cpoint >> 6);
53 else {
54 if (cpoint < 0x10000) /* 3 byte char */
55 *p++ = 0xe0 | (cpoint >> 12);
56 else { /* 4 byte char */
57 *p++ = 0xf0 | (cpoint >> 18);
58 *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
59 }
60
61 *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
62 }
63
64 *p++ = 0x80 | (cpoint & 0x3f);
65 }
66
67 return p - buf;
68}
69
70
71int xs_is_utf8_cont_byte(char c)
72/* returns true if c is an utf8 continuation byte */
73{
74 return ((c & 0xc0) == 0x80);
75}
76
77
78unsigned int xs_utf8_dec(const char **str)
79/* decodes an utf-8 char inside str and updates the pointer */
80{
81 const char *p = *str;
82
83 if (!xs_is_string(p))
84 return 0;
85
86 unsigned int cpoint = 0;
87 unsigned char c = *p++;
88 int cb = 0;
89
90 if ((c & 0x80) == 0) { /* 1 byte char */
91 cpoint = c;
92 }
93 else
94 if ((c & 0xe0) == 0xc0) { /* 2 byte char */
95 cpoint = (c & 0x1f) << 6;
96 cb = 1;
97 }
98 else
99 if ((c & 0xf0) == 0xe0) { /* 3 byte char */
100 cpoint = (c & 0x0f) << 12;
101 cb = 2;
102 }
103 else
104 if ((c & 0xf8) == 0xf0) { /* 4 byte char */
105 cpoint = (c & 0x07) << 18;
106 cb = 3;
107 }
108
109 /* process the continuation bytes */
110 while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
111 cpoint |= (*p++ & 0x3f) << (--cb * 6);
112
113 /* incomplete or broken? */
114 if (cb)
115 cpoint = 0xfffd;
116
117 *str = p;
118 return cpoint;
119}
120
121
122/** Unicode character width: intentionally dead simple **/
123
124static unsigned int xs_unicode_width_table[] = {
125 0x300, 0x36f, 0, /* diacritics */
126 0x1100, 0x11ff, 2, /* Hangul */
127 0x2e80, 0xa4cf, 2, /* CJK */
128 0xac00, 0xd7a3, 2, /* more Hangul */
129 0xe000, 0xf8ff, 0, /* private use */
130 0xf900, 0xfaff, 2, /* CJK compatibility */
131 0xff00, 0xff60, 2, /* full width things */
132 0xffdf, 0xffe6, 2, /* full width things */
133 0x1f200, 0x1ffff, 2, /* emojis */
134 0x20000, 0x2fffd, 2 /* more CJK */
135};
136
137int xs_unicode_width(unsigned int cpoint)
138/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
139{
140 int b = 0;
141 int t = xs_countof(xs_unicode_width_table) / 3 - 1;
142
143 while (t >= b) {
144 int n = (b + t) / 2;
145 unsigned int *p = &xs_unicode_width_table[n * 3];
146
147 if (cpoint < p[0])
148 t = n - 1;
149 else
150 if (cpoint > p[1])
151 b = n + 1;
152 else
153 return p[2];
154 }
155
156 return 1;
157}
158
159
160int xs_is_diacritic(unsigned int cpoint)
161{
162 return cpoint >= 0x300 && cpoint <= 0x36f;
163}
164
165
166/** surrogate pairs **/
167
168int xs_is_surrogate(unsigned int cpoint)
169/* checks if cpoint is the first element of a Unicode surrogate pair */
170{
171 return cpoint >= 0xd800 && cpoint <= 0xdfff;
172}
173
174
175unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
176/* "decodes" a surrogate pair into a codepoint */
177{
178 return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
179}
180
181
182unsigned int xs_surrogate_enc(unsigned int cpoint)
183/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
184{
185 unsigned int p1 = 0xd7c0 + (cpoint >> 10);
186 unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
187
188 return (p1 << 16) | p2;
189}
190
191
192#ifdef _XS_H
193
194xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
195/* encodes an Unicode codepoint to utf-8 into str */
196{
197 char tmp[4];
198
199 int c = xs_utf8_enc(tmp, cpoint);
200
201 str = xs_insert_m(str, *offset, tmp, c);
202
203 *offset += c;
204
205 return str;
206}
207
208
209xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
210/* encodes an Unicode codepoint to utf-8 into str */
211{
212 int offset = strlen(str);
213
214 return xs_utf8_insert(str, cpoint, &offset);
215}
216
217#endif /* _XS_H */
218
219
220#ifdef _XS_UNICODE_TBL_H
221
222/* include xs_unicode_tbl.h before this one to use these functions */
223
224unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
225/* searches for an uppercase codepoint in the case fold table */
226{
227 int b = 0;
228 int t = xs_countof(xs_unicode_case_fold_table) / 2 + 1;
229
230 while (t >= b) {
231 int n = (b + t) / 2;
232 unsigned int *p = &xs_unicode_case_fold_table[n * 2];
233
234 if (cpoint < p[0])
235 t = n - 1;
236 else
237 if (cpoint > p[0])
238 b = n + 1;
239 else
240 return p;
241 }
242
243 return NULL;
244}
245
246
247unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
248/* searches for a lowercase codepoint in the case fold table */
249{
250 unsigned int *p = xs_unicode_case_fold_table;
251 unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
252
253 while (p < e) {
254 if (cpoint == p[1])
255 return p;
256
257 p += 2;
258 }
259
260 return NULL;
261}
262
263
264unsigned int xs_unicode_to_lower(unsigned int cpoint)
265/* returns the cpoint to lowercase */
266{
267 if (cpoint < 0x80)
268 return tolower(cpoint);
269
270 unsigned int *p = _xs_unicode_upper_search(cpoint);
271
272 return p == NULL ? cpoint : p[1];
273}
274
275
276unsigned int xs_unicode_to_upper(unsigned int cpoint)
277/* returns the cpoint to uppercase */
278{
279 if (cpoint < 0x80)
280 return toupper(cpoint);
281
282 unsigned int *p = _xs_unicode_lower_search(cpoint);
283
284 return p == NULL ? cpoint : p[0];
285}
286
287
288int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
289/* applies unicode Normalization Form D */
290{
291 int b = 0;
292 int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
293
294 while (t >= b) {
295 int n = (b + t) / 2;
296 unsigned int *p = &xs_unicode_nfd_table[n * 3];
297
298 int c = cpoint - p[0];
299
300 if (c < 0)
301 t = n - 1;
302 else
303 if (c > 0)
304 b = n + 1;
305 else {
306 *base = p[1];
307 *diac = p[2];
308 return 1;
309 }
310 }
311
312 return 0;
313}
314
315
316int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
317/* applies unicode Normalization Form C */
318{
319 unsigned int *p = xs_unicode_nfd_table;
320 unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
321
322 while (p < e) {
323 if (p[1] == base && p[2] == diac) {
324 *cpoint = p[0];
325 return 1;
326 }
327
328 p += 3;
329 }
330
331 return 0;
332}
333
334
335int xs_unicode_is_alpha(unsigned int cpoint)
336/* checks if a codepoint is an alpha (i.e. a letter) */
337{
338 int b = 0;
339 int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
340
341 while (t >= b) {
342 int n = (b + t) / 2;
343 unsigned int *p = &xs_unicode_alpha_table[n * 2];
344
345 if (cpoint < p[0])
346 t = n - 1;
347 else
348 if (cpoint > p[1])
349 b = n + 1;
350 else
351 return 1;
352 }
353
354 return 0;
355}
356
357
358int xs_unicode_is_right_to_left(unsigned int cpoint)
359/* checks if a codepoint is a right-to-left letter */
360{
361 int b = 0;
362 int t = xs_countof(xs_unicode_right_to_left_table) / 2 - 1;
363
364 while (t >= b) {
365 int n = (b + t) / 2;
366 unsigned int *p = &xs_unicode_right_to_left_table[n * 2];
367
368 if (cpoint < p[0])
369 t = n - 1;
370 else
371 if (cpoint > p[1])
372 b = n + 1;
373 else
374 return 1;
375 }
376
377 return 0;
378}
379
380
381#ifdef _XS_H
382
383xs_str *xs_utf8_to_upper(const char *str)
384{
385 xs_str *s = xs_str_new(NULL);
386 unsigned int cpoint;
387 int offset = 0;
388
389 while ((cpoint = xs_utf8_dec(&str))) {
390 cpoint = xs_unicode_to_upper(cpoint);
391 s = xs_utf8_insert(s, cpoint, &offset);
392 }
393
394 return s;
395}
396
397
398xs_str *xs_utf8_to_lower(const char *str)
399{
400 xs_str *s = xs_str_new(NULL);
401 unsigned int cpoint;
402 int offset = 0;
403
404 while ((cpoint = xs_utf8_dec(&str))) {
405 cpoint = xs_unicode_to_lower(cpoint);
406 s = xs_utf8_insert(s, cpoint, &offset);
407 }
408
409 return s;
410}
411
412
413xs_str *xs_utf8_to_nfd(const char *str)
414{
415 xs_str *s = xs_str_new(NULL);
416 unsigned int cpoint;
417 int offset = 0;
418
419 while ((cpoint = xs_utf8_dec(&str))) {
420 unsigned int base;
421 unsigned int diac;
422
423 if (xs_unicode_nfd(cpoint, &base, &diac)) {
424 s = xs_utf8_insert(s, base, &offset);
425 s = xs_utf8_insert(s, diac, &offset);
426 }
427 else
428 s = xs_utf8_insert(s, cpoint, &offset);
429 }
430
431 return s;
432}
433
434
435xs_str *xs_utf8_to_nfc(const char *str)
436{
437 xs_str *s = xs_str_new(NULL);
438 unsigned int cpoint;
439 unsigned int base = 0;
440 int offset = 0;
441
442 while ((cpoint = xs_utf8_dec(&str))) {
443 if (xs_is_diacritic(cpoint)) {
444 if (xs_unicode_nfc(base, cpoint, &base))
445 continue;
446 }
447
448 if (base)
449 s = xs_utf8_insert(s, base, &offset);
450
451 base = cpoint;
452 }
453
454 if (base)
455 s = xs_utf8_insert(s, base, &offset);
456
457 return s;
458}
459
460#endif /* _XS_H */
461
462#endif /* _XS_UNICODE_TBL_H */
463
464#endif /* XS_IMPLEMENTATION */
465
466#endif /* _XS_UNICODE_H */
