main
xs_unicode.h
1/* copyright (c) 2022 - 2026 grunfink et al. / MIT license */
2
3#ifndef _XS_UNICODE_H
4
5#define _XS_UNICODE_H
6
7#include "xs.h"
8
9 int xs_utf8_enc(char buf[4], unsigned int cpoint);
10 int xs_is_utf8_cont_byte(char c);
11 unsigned int xs_utf8_dec(const char **str);
12 int xs_unicode_width(unsigned int cpoint);
13 int xs_is_surrogate(unsigned int cpoint);
14 int xs_is_diacritic(unsigned int cpoint);
15 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
16 unsigned int xs_surrogate_enc(unsigned int cpoint);
17 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
18 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
19 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
20 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
21 unsigned int xs_unicode_toupper(unsigned int cpoint);
22 unsigned int xs_unicode_tolower(unsigned int cpoint);
23 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
24 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
25 int xs_unicode_is_alpha(unsigned int cpoint);
26 int xs_unicode_is_right_to_left(unsigned int cpoint);
27 int xs_is_emoji(unsigned int cpoint);
28 int xs_utf8_len(const char *str);
29
30#ifdef _XS_H
31 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
32 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
33 xs_str *xs_utf8_crop_i(xs_str *str, int begin, int end);
34 xs_str *xs_utf8_toupper(const char *str);
35 xs_str *xs_utf8_tolower(const char *str);
36 xs_str *xs_utf8_to_nfd(const char *str);
37 xs_str *xs_utf8_to_nfc(const char *str);
38#endif
39
40#ifdef XS_IMPLEMENTATION
41
42#include <ctype.h>
43
44#ifndef xs_countof
45#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
46#endif
47
48int xs_utf8_enc(char buf[4], unsigned int cpoint)
49/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
50{
51 char *p = buf;
52
53 if (cpoint < 0x80) /* 1 byte char */
54 *p++ = cpoint & 0xff;
55 else {
56 if (cpoint < 0x800) /* 2 byte char */
57 *p++ = 0xc0 | (cpoint >> 6);
58 else {
59 if (cpoint < 0x10000) /* 3 byte char */
60 *p++ = 0xe0 | (cpoint >> 12);
61 else { /* 4 byte char */
62 *p++ = 0xf0 | (cpoint >> 18);
63 *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
64 }
65
66 *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
67 }
68
69 *p++ = 0x80 | (cpoint & 0x3f);
70 }
71
72 return p - buf;
73}
74
75
76int xs_is_utf8_cont_byte(char c)
77/* returns true if c is an utf8 continuation byte */
78{
79 return ((c & 0xc0) == 0x80);
80}
81
82
83unsigned int xs_utf8_dec(const char **str)
84/* decodes an utf-8 char inside str and updates the pointer */
85{
86 const char *p = *str;
87
88 if (!xs_is_string(p))
89 return 0;
90
91 unsigned int cpoint = 0;
92 unsigned char c = *p++;
93 int cb = 0;
94
95 if ((c & 0x80) == 0) { /* 1 byte char */
96 cpoint = c;
97 }
98 else
99 if ((c & 0xe0) == 0xc0) { /* 2 byte char */
100 cpoint = (c & 0x1f) << 6;
101 cb = 1;
102 }
103 else
104 if ((c & 0xf0) == 0xe0) { /* 3 byte char */
105 cpoint = (c & 0x0f) << 12;
106 cb = 2;
107 }
108 else
109 if ((c & 0xf8) == 0xf0) { /* 4 byte char */
110 cpoint = (c & 0x07) << 18;
111 cb = 3;
112 }
113
114 /* process the continuation bytes */
115 while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
116 cpoint |= (*p++ & 0x3f) << (--cb * 6);
117
118 /* incomplete or broken? */
119 if (cb)
120 cpoint = 0xfffd;
121
122 *str = p;
123 return cpoint;
124}
125
126
127/** Unicode character width: intentionally dead simple **/
128
129static unsigned int xs_unicode_width_table[] = {
130 0x300, 0x36f, 0, /* diacritics */
131 0x1100, 0x11ff, 2, /* Hangul */
132 0x2e80, 0xa4cf, 2, /* CJK */
133 0xac00, 0xd7a3, 2, /* more Hangul */
134 0xe000, 0xf8ff, 0, /* private use */
135 0xf900, 0xfaff, 2, /* CJK compatibility */
136 0xff00, 0xff60, 2, /* full width things */
137 0xffdf, 0xffe6, 2, /* full width things */
138 0x1f200, 0x1ffff, 2, /* emojis */
139 0x20000, 0x2fffd, 2 /* more CJK */
140};
141
142/* magic number from https://en.wikipedia.org/wiki/Emoji#Unicode_blocks */
143int xs_is_emoji(unsigned int cpoint) {
144/* returns wether the input is an utf8 emoji */
145 return cpoint > 0x00A9 && cpoint < 0x1ffff;
146}
147
148int xs_unicode_width(unsigned int cpoint)
149/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
150{
151 int b = 0;
152 int t = xs_countof(xs_unicode_width_table) / 3 - 1;
153
154 while (t >= b) {
155 int n = (b + t) / 2;
156 unsigned int *p = &xs_unicode_width_table[n * 3];
157
158 if (cpoint < p[0])
159 t = n - 1;
160 else
161 if (cpoint > p[1])
162 b = n + 1;
163 else
164 return p[2];
165 }
166
167 return 1;
168}
169
170
171int xs_is_diacritic(unsigned int cpoint)
172{
173 return cpoint >= 0x300 && cpoint <= 0x36f;
174}
175
176
177/** surrogate pairs **/
178
179int xs_is_surrogate(unsigned int cpoint)
180/* checks if cpoint is the first element of a Unicode surrogate pair */
181{
182 return cpoint >= 0xd800 && cpoint <= 0xdfff;
183}
184
185
186unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
187/* "decodes" a surrogate pair into a codepoint */
188{
189 return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
190}
191
192
193unsigned int xs_surrogate_enc(unsigned int cpoint)
194/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
195{
196 unsigned int p1 = 0xd7c0 + (cpoint >> 10);
197 unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
198
199 return (p1 << 16) | p2;
200}
201
202
203int xs_utf8_len(const char *str)
204/* counts the number of characters (codepoints) in str */
205{
206 int len = 0;
207
208 while (xs_utf8_dec(&str))
209 len++;
210
211 return len;
212}
213
214
215#ifdef _XS_H
216
217xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
218/* encodes an Unicode codepoint to utf-8 into str */
219{
220 char tmp[4];
221
222 int c = xs_utf8_enc(tmp, cpoint);
223
224 str = xs_insert_m(str, *offset, tmp, c);
225
226 *offset += c;
227
228 return str;
229}
230
231
232xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
233/* encodes an Unicode codepoint to utf-8 into str */
234{
235 int offset = strlen(str);
236
237 return xs_utf8_insert(str, cpoint, &offset);
238}
239
240
241xs_str *xs_utf8_crop_i(xs_str *str, int begin, int end)
242/* crops str from begin to end by characters (codepoints) */
243{
244 const char *p = str;
245 int sz = xs_utf8_len(str);
246 unsigned int cpoint;
247
248 if (end <= 0)
249 end = sz + end;
250
251 xs_str *ns = xs_str_new(NULL);
252
253 for (int n = 0; n < end && (cpoint = xs_utf8_dec(&p)); n++) {
254 if (n >= begin)
255 ns = xs_utf8_cat(ns, cpoint);
256 }
257
258 xs_free(str);
259
260 return ns;
261}
262
263
264#endif /* _XS_H */
265
266
267#ifdef _XS_UNICODE_TBL_H
268
269/* include xs_unicode_tbl.h before this one to use these functions */
270
271unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
272/* searches for an uppercase codepoint in the case fold table */
273{
274 int b = 0;
275 int t = xs_countof(xs_unicode_case_fold_table) / 2 - 1;
276
277 while (t >= b) {
278 int n = (b + t) / 2;
279 unsigned int *p = &xs_unicode_case_fold_table[n * 2];
280
281 if (cpoint < p[0])
282 t = n - 1;
283 else
284 if (cpoint > p[0])
285 b = n + 1;
286 else
287 return p;
288 }
289
290 return NULL;
291}
292
293
294unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
295/* searches for a lowercase codepoint in the case fold table */
296{
297 unsigned int *p = xs_unicode_case_fold_table;
298 unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
299
300 while (p < e) {
301 if (cpoint == p[1])
302 return p;
303
304 p += 2;
305 }
306
307 return NULL;
308}
309
310
311unsigned int xs_unicode_tolower(unsigned int cpoint)
312/* returns the cpoint to lowercase */
313{
314 if (cpoint < 0x80)
315 return tolower(cpoint);
316
317 unsigned int *p = _xs_unicode_upper_search(cpoint);
318
319 return p == NULL ? cpoint : p[1];
320}
321
322
323unsigned int xs_unicode_toupper(unsigned int cpoint)
324/* returns the cpoint to uppercase */
325{
326 if (cpoint < 0x80)
327 return toupper(cpoint);
328
329 unsigned int *p = _xs_unicode_lower_search(cpoint);
330
331 return p == NULL ? cpoint : p[0];
332}
333
334
335int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
336/* applies unicode Normalization Form D */
337{
338 int b = 0;
339 int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
340
341 while (t >= b) {
342 int n = (b + t) / 2;
343 unsigned int *p = &xs_unicode_nfd_table[n * 3];
344
345 int c = cpoint - p[0];
346
347 if (c < 0)
348 t = n - 1;
349 else
350 if (c > 0)
351 b = n + 1;
352 else {
353 *base = p[1];
354 *diac = p[2];
355 return 1;
356 }
357 }
358
359 return 0;
360}
361
362
363int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
364/* applies unicode Normalization Form C */
365{
366 unsigned int *p = xs_unicode_nfd_table;
367 unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
368
369 while (p < e) {
370 if (p[1] == base && p[2] == diac) {
371 *cpoint = p[0];
372 return 1;
373 }
374
375 p += 3;
376 }
377
378 return 0;
379}
380
381
382int xs_unicode_is_alpha(unsigned int cpoint)
383/* checks if a codepoint is an alpha (i.e. a letter) */
384{
385 int b = 0;
386 int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
387
388 while (t >= b) {
389 int n = (b + t) / 2;
390 unsigned int *p = &xs_unicode_alpha_table[n * 2];
391
392 if (cpoint < p[0])
393 t = n - 1;
394 else
395 if (cpoint > p[1])
396 b = n + 1;
397 else
398 return 1;
399 }
400
401 return 0;
402}
403
404
405int xs_unicode_is_right_to_left(unsigned int cpoint)
406/* checks if a codepoint is a right-to-left letter */
407{
408 int b = 0;
409 int t = xs_countof(xs_unicode_right_to_left_table) / 2 - 1;
410
411 while (t >= b) {
412 int n = (b + t) / 2;
413 unsigned int *p = &xs_unicode_right_to_left_table[n * 2];
414
415 if (cpoint < p[0])
416 t = n - 1;
417 else
418 if (cpoint > p[1])
419 b = n + 1;
420 else
421 return 1;
422 }
423
424 return 0;
425}
426
427
428#ifdef _XS_H
429
430xs_str *xs_utf8_toupper(const char *str)
431{
432 xs_str *s = xs_str_new(NULL);
433 unsigned int cpoint;
434 int offset = 0;
435
436 while ((cpoint = xs_utf8_dec(&str))) {
437 cpoint = xs_unicode_toupper(cpoint);
438 s = xs_utf8_insert(s, cpoint, &offset);
439 }
440
441 return s;
442}
443
444
445xs_str *xs_utf8_tolower(const char *str)
446{
447 xs_str *s = xs_str_new(NULL);
448 unsigned int cpoint;
449 int offset = 0;
450
451 while ((cpoint = xs_utf8_dec(&str))) {
452 cpoint = xs_unicode_tolower(cpoint);
453 s = xs_utf8_insert(s, cpoint, &offset);
454 }
455
456 return s;
457}
458
459
460xs_str *xs_utf8_to_nfd(const char *str)
461{
462 xs_str *s = xs_str_new(NULL);
463 unsigned int cpoint;
464 int offset = 0;
465
466 while ((cpoint = xs_utf8_dec(&str))) {
467 unsigned int base;
468 unsigned int diac;
469
470 if (xs_unicode_nfd(cpoint, &base, &diac)) {
471 s = xs_utf8_insert(s, base, &offset);
472 s = xs_utf8_insert(s, diac, &offset);
473 }
474 else
475 s = xs_utf8_insert(s, cpoint, &offset);
476 }
477
478 return s;
479}
480
481
482xs_str *xs_utf8_to_nfc(const char *str)
483{
484 xs_str *s = xs_str_new(NULL);
485 unsigned int cpoint;
486 unsigned int base = 0;
487 int offset = 0;
488
489 while ((cpoint = xs_utf8_dec(&str))) {
490 if (xs_is_diacritic(cpoint)) {
491 if (xs_unicode_nfc(base, cpoint, &base))
492 continue;
493 }
494
495 if (base)
496 s = xs_utf8_insert(s, base, &offset);
497
498 base = cpoint;
499 }
500
501 if (base)
502 s = xs_utf8_insert(s, base, &offset);
503
504 return s;
505}
506
507#endif /* _XS_H */
508
509#endif /* _XS_UNICODE_TBL_H */
510
511#endif /* XS_IMPLEMENTATION */
512
513#endif /* _XS_UNICODE_H */
