2.93
xs_unicode.h
1/* copyright (c) 2022 - 2026 grunfink et al. / MIT license */
2
3#ifndef _XS_UNICODE_H
4
5#define _XS_UNICODE_H
6
7 int xs_utf8_enc(char buf[4], unsigned int cpoint);
8 int xs_is_utf8_cont_byte(char c);
9 unsigned int xs_utf8_dec(const char **str);
10 int xs_unicode_width(unsigned int cpoint);
11 int xs_is_surrogate(unsigned int cpoint);
12 int xs_is_diacritic(unsigned int cpoint);
13 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
14 unsigned int xs_surrogate_enc(unsigned int cpoint);
15 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
16 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
17 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
18 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
19 unsigned int xs_unicode_to_upper(unsigned int cpoint);
20 unsigned int xs_unicode_to_lower(unsigned int cpoint);
21 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
22 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
23 int xs_unicode_is_alpha(unsigned int cpoint);
24 int xs_unicode_is_right_to_left(unsigned int cpoint);
25 int xs_is_emoji(unsigned int cpoint);
26 int xs_utf8_len(const char *str);
27
28#ifdef _XS_H
29 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
30 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
31 xs_str *xs_utf8_crop_i(xs_str *str, int begin, int end);
32 xs_str *xs_utf8_to_upper(const char *str);
33 xs_str *xs_utf8_to_lower(const char *str);
34 xs_str *xs_utf8_to_nfd(const char *str);
35 xs_str *xs_utf8_to_nfc(const char *str);
36#endif
37
38#ifdef XS_IMPLEMENTATION
39
40#include <ctype.h>
41
42#ifndef xs_countof
43#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
44#endif
45
46int xs_utf8_enc(char buf[4], unsigned int cpoint)
47/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
48{
49 char *p = buf;
50
51 if (cpoint < 0x80) /* 1 byte char */
52 *p++ = cpoint & 0xff;
53 else {
54 if (cpoint < 0x800) /* 2 byte char */
55 *p++ = 0xc0 | (cpoint >> 6);
56 else {
57 if (cpoint < 0x10000) /* 3 byte char */
58 *p++ = 0xe0 | (cpoint >> 12);
59 else { /* 4 byte char */
60 *p++ = 0xf0 | (cpoint >> 18);
61 *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
62 }
63
64 *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
65 }
66
67 *p++ = 0x80 | (cpoint & 0x3f);
68 }
69
70 return p - buf;
71}
72
73
74int xs_is_utf8_cont_byte(char c)
75/* returns true if c is an utf8 continuation byte */
76{
77 return ((c & 0xc0) == 0x80);
78}
79
80
81unsigned int xs_utf8_dec(const char **str)
82/* decodes an utf-8 char inside str and updates the pointer */
83{
84 const char *p = *str;
85
86 if (!xs_is_string(p))
87 return 0;
88
89 unsigned int cpoint = 0;
90 unsigned char c = *p++;
91 int cb = 0;
92
93 if ((c & 0x80) == 0) { /* 1 byte char */
94 cpoint = c;
95 }
96 else
97 if ((c & 0xe0) == 0xc0) { /* 2 byte char */
98 cpoint = (c & 0x1f) << 6;
99 cb = 1;
100 }
101 else
102 if ((c & 0xf0) == 0xe0) { /* 3 byte char */
103 cpoint = (c & 0x0f) << 12;
104 cb = 2;
105 }
106 else
107 if ((c & 0xf8) == 0xf0) { /* 4 byte char */
108 cpoint = (c & 0x07) << 18;
109 cb = 3;
110 }
111
112 /* process the continuation bytes */
113 while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
114 cpoint |= (*p++ & 0x3f) << (--cb * 6);
115
116 /* incomplete or broken? */
117 if (cb)
118 cpoint = 0xfffd;
119
120 *str = p;
121 return cpoint;
122}
123
124
125/** Unicode character width: intentionally dead simple **/
126
127static unsigned int xs_unicode_width_table[] = {
128 0x300, 0x36f, 0, /* diacritics */
129 0x1100, 0x11ff, 2, /* Hangul */
130 0x2e80, 0xa4cf, 2, /* CJK */
131 0xac00, 0xd7a3, 2, /* more Hangul */
132 0xe000, 0xf8ff, 0, /* private use */
133 0xf900, 0xfaff, 2, /* CJK compatibility */
134 0xff00, 0xff60, 2, /* full width things */
135 0xffdf, 0xffe6, 2, /* full width things */
136 0x1f200, 0x1ffff, 2, /* emojis */
137 0x20000, 0x2fffd, 2 /* more CJK */
138};
139
140/* magic number from https://en.wikipedia.org/wiki/Emoji#Unicode_blocks */
141int xs_is_emoji(unsigned int cpoint) {
142/* returns wether the input is an utf8 emoji */
143 return cpoint > 0x00A9 && cpoint < 0x1ffff;
144}
145
146int xs_unicode_width(unsigned int cpoint)
147/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
148{
149 int b = 0;
150 int t = xs_countof(xs_unicode_width_table) / 3 - 1;
151
152 while (t >= b) {
153 int n = (b + t) / 2;
154 unsigned int *p = &xs_unicode_width_table[n * 3];
155
156 if (cpoint < p[0])
157 t = n - 1;
158 else
159 if (cpoint > p[1])
160 b = n + 1;
161 else
162 return p[2];
163 }
164
165 return 1;
166}
167
168
169int xs_is_diacritic(unsigned int cpoint)
170{
171 return cpoint >= 0x300 && cpoint <= 0x36f;
172}
173
174
175/** surrogate pairs **/
176
177int xs_is_surrogate(unsigned int cpoint)
178/* checks if cpoint is the first element of a Unicode surrogate pair */
179{
180 return cpoint >= 0xd800 && cpoint <= 0xdfff;
181}
182
183
184unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
185/* "decodes" a surrogate pair into a codepoint */
186{
187 return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
188}
189
190
191unsigned int xs_surrogate_enc(unsigned int cpoint)
192/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
193{
194 unsigned int p1 = 0xd7c0 + (cpoint >> 10);
195 unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
196
197 return (p1 << 16) | p2;
198}
199
200
201int xs_utf8_len(const char *str)
202/* counts the number of characters (codepoints) in str */
203{
204 int len = 0;
205
206 while (xs_utf8_dec(&str))
207 len++;
208
209 return len;
210}
211
212
213#ifdef _XS_H
214
215xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
216/* encodes an Unicode codepoint to utf-8 into str */
217{
218 char tmp[4];
219
220 int c = xs_utf8_enc(tmp, cpoint);
221
222 str = xs_insert_m(str, *offset, tmp, c);
223
224 *offset += c;
225
226 return str;
227}
228
229
230xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
231/* encodes an Unicode codepoint to utf-8 into str */
232{
233 int offset = strlen(str);
234
235 return xs_utf8_insert(str, cpoint, &offset);
236}
237
238
239xs_str *xs_utf8_crop_i(xs_str *str, int begin, int end)
240/* crops str from begin to end by characters (codepoints) */
241{
242 const char *p = str;
243 int sz = xs_utf8_len(str);
244 unsigned int cpoint;
245
246 if (end <= 0)
247 end = sz + end;
248
249 xs_str *ns = xs_str_new(NULL);
250
251 for (int n = 0; n < end && (cpoint = xs_utf8_dec(&p)); n++) {
252 if (n >= begin)
253 ns = xs_utf8_cat(ns, cpoint);
254 }
255
256 xs_free(str);
257
258 return ns;
259}
260
261
262#endif /* _XS_H */
263
264
265#ifdef _XS_UNICODE_TBL_H
266
267/* include xs_unicode_tbl.h before this one to use these functions */
268
269unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
270/* searches for an uppercase codepoint in the case fold table */
271{
272 int b = 0;
273 int t = xs_countof(xs_unicode_case_fold_table) / 2 - 1;
274
275 while (t >= b) {
276 int n = (b + t) / 2;
277 unsigned int *p = &xs_unicode_case_fold_table[n * 2];
278
279 if (cpoint < p[0])
280 t = n - 1;
281 else
282 if (cpoint > p[0])
283 b = n + 1;
284 else
285 return p;
286 }
287
288 return NULL;
289}
290
291
292unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
293/* searches for a lowercase codepoint in the case fold table */
294{
295 unsigned int *p = xs_unicode_case_fold_table;
296 unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
297
298 while (p < e) {
299 if (cpoint == p[1])
300 return p;
301
302 p += 2;
303 }
304
305 return NULL;
306}
307
308
309unsigned int xs_unicode_to_lower(unsigned int cpoint)
310/* returns the cpoint to lowercase */
311{
312 if (cpoint < 0x80)
313 return tolower(cpoint);
314
315 unsigned int *p = _xs_unicode_upper_search(cpoint);
316
317 return p == NULL ? cpoint : p[1];
318}
319
320
321unsigned int xs_unicode_to_upper(unsigned int cpoint)
322/* returns the cpoint to uppercase */
323{
324 if (cpoint < 0x80)
325 return toupper(cpoint);
326
327 unsigned int *p = _xs_unicode_lower_search(cpoint);
328
329 return p == NULL ? cpoint : p[0];
330}
331
332
333int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
334/* applies unicode Normalization Form D */
335{
336 int b = 0;
337 int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
338
339 while (t >= b) {
340 int n = (b + t) / 2;
341 unsigned int *p = &xs_unicode_nfd_table[n * 3];
342
343 int c = cpoint - p[0];
344
345 if (c < 0)
346 t = n - 1;
347 else
348 if (c > 0)
349 b = n + 1;
350 else {
351 *base = p[1];
352 *diac = p[2];
353 return 1;
354 }
355 }
356
357 return 0;
358}
359
360
361int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
362/* applies unicode Normalization Form C */
363{
364 unsigned int *p = xs_unicode_nfd_table;
365 unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
366
367 while (p < e) {
368 if (p[1] == base && p[2] == diac) {
369 *cpoint = p[0];
370 return 1;
371 }
372
373 p += 3;
374 }
375
376 return 0;
377}
378
379
380int xs_unicode_is_alpha(unsigned int cpoint)
381/* checks if a codepoint is an alpha (i.e. a letter) */
382{
383 int b = 0;
384 int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
385
386 while (t >= b) {
387 int n = (b + t) / 2;
388 unsigned int *p = &xs_unicode_alpha_table[n * 2];
389
390 if (cpoint < p[0])
391 t = n - 1;
392 else
393 if (cpoint > p[1])
394 b = n + 1;
395 else
396 return 1;
397 }
398
399 return 0;
400}
401
402
403int xs_unicode_is_right_to_left(unsigned int cpoint)
404/* checks if a codepoint is a right-to-left letter */
405{
406 int b = 0;
407 int t = xs_countof(xs_unicode_right_to_left_table) / 2 - 1;
408
409 while (t >= b) {
410 int n = (b + t) / 2;
411 unsigned int *p = &xs_unicode_right_to_left_table[n * 2];
412
413 if (cpoint < p[0])
414 t = n - 1;
415 else
416 if (cpoint > p[1])
417 b = n + 1;
418 else
419 return 1;
420 }
421
422 return 0;
423}
424
425
426#ifdef _XS_H
427
428xs_str *xs_utf8_to_upper(const char *str)
429{
430 xs_str *s = xs_str_new(NULL);
431 unsigned int cpoint;
432 int offset = 0;
433
434 while ((cpoint = xs_utf8_dec(&str))) {
435 cpoint = xs_unicode_to_upper(cpoint);
436 s = xs_utf8_insert(s, cpoint, &offset);
437 }
438
439 return s;
440}
441
442
443xs_str *xs_utf8_to_lower(const char *str)
444{
445 xs_str *s = xs_str_new(NULL);
446 unsigned int cpoint;
447 int offset = 0;
448
449 while ((cpoint = xs_utf8_dec(&str))) {
450 cpoint = xs_unicode_to_lower(cpoint);
451 s = xs_utf8_insert(s, cpoint, &offset);
452 }
453
454 return s;
455}
456
457
458xs_str *xs_utf8_to_nfd(const char *str)
459{
460 xs_str *s = xs_str_new(NULL);
461 unsigned int cpoint;
462 int offset = 0;
463
464 while ((cpoint = xs_utf8_dec(&str))) {
465 unsigned int base;
466 unsigned int diac;
467
468 if (xs_unicode_nfd(cpoint, &base, &diac)) {
469 s = xs_utf8_insert(s, base, &offset);
470 s = xs_utf8_insert(s, diac, &offset);
471 }
472 else
473 s = xs_utf8_insert(s, cpoint, &offset);
474 }
475
476 return s;
477}
478
479
480xs_str *xs_utf8_to_nfc(const char *str)
481{
482 xs_str *s = xs_str_new(NULL);
483 unsigned int cpoint;
484 unsigned int base = 0;
485 int offset = 0;
486
487 while ((cpoint = xs_utf8_dec(&str))) {
488 if (xs_is_diacritic(cpoint)) {
489 if (xs_unicode_nfc(base, cpoint, &base))
490 continue;
491 }
492
493 if (base)
494 s = xs_utf8_insert(s, base, &offset);
495
496 base = cpoint;
497 }
498
499 if (base)
500 s = xs_utf8_insert(s, base, &offset);
501
502 return s;
503}
504
505#endif /* _XS_H */
506
507#endif /* _XS_UNICODE_TBL_H */
508
509#endif /* XS_IMPLEMENTATION */
510
511#endif /* _XS_UNICODE_H */
