2.89
xs_unicode.h
1/* copyright (c) 2022 - 2026 grunfink et al. / MIT license */
2
3#ifndef _XS_UNICODE_H
4
5#define _XS_UNICODE_H
6
7 int xs_utf8_enc(char buf[4], unsigned int cpoint);
8 int xs_is_utf8_cont_byte(char c);
9 unsigned int xs_utf8_dec(const char **str);
10 int xs_unicode_width(unsigned int cpoint);
11 int xs_is_surrogate(unsigned int cpoint);
12 int xs_is_diacritic(unsigned int cpoint);
13 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
14 unsigned int xs_surrogate_enc(unsigned int cpoint);
15 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
16 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
17 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
18 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
19 unsigned int xs_unicode_to_upper(unsigned int cpoint);
20 unsigned int xs_unicode_to_lower(unsigned int cpoint);
21 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
22 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
23 int xs_unicode_is_alpha(unsigned int cpoint);
24 int xs_unicode_is_right_to_left(unsigned int cpoint);
25 int xs_is_emoji(unsigned int cpoint);
26
27#ifdef _XS_H
28 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
29 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
30 xs_str *xs_utf8_to_upper(const char *str);
31 xs_str *xs_utf8_to_lower(const char *str);
32 xs_str *xs_utf8_to_nfd(const char *str);
33 xs_str *xs_utf8_to_nfc(const char *str);
34#endif
35
36#ifdef XS_IMPLEMENTATION
37
38#include <ctype.h>
39
40#ifndef xs_countof
41#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
42#endif
43
44int xs_utf8_enc(char buf[4], unsigned int cpoint)
45/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
46{
47 char *p = buf;
48
49 if (cpoint < 0x80) /* 1 byte char */
50 *p++ = cpoint & 0xff;
51 else {
52 if (cpoint < 0x800) /* 2 byte char */
53 *p++ = 0xc0 | (cpoint >> 6);
54 else {
55 if (cpoint < 0x10000) /* 3 byte char */
56 *p++ = 0xe0 | (cpoint >> 12);
57 else { /* 4 byte char */
58 *p++ = 0xf0 | (cpoint >> 18);
59 *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
60 }
61
62 *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
63 }
64
65 *p++ = 0x80 | (cpoint & 0x3f);
66 }
67
68 return p - buf;
69}
70
71
72int xs_is_utf8_cont_byte(char c)
73/* returns true if c is an utf8 continuation byte */
74{
75 return ((c & 0xc0) == 0x80);
76}
77
78
79unsigned int xs_utf8_dec(const char **str)
80/* decodes an utf-8 char inside str and updates the pointer */
81{
82 const char *p = *str;
83
84 if (!xs_is_string(p))
85 return 0;
86
87 unsigned int cpoint = 0;
88 unsigned char c = *p++;
89 int cb = 0;
90
91 if ((c & 0x80) == 0) { /* 1 byte char */
92 cpoint = c;
93 }
94 else
95 if ((c & 0xe0) == 0xc0) { /* 2 byte char */
96 cpoint = (c & 0x1f) << 6;
97 cb = 1;
98 }
99 else
100 if ((c & 0xf0) == 0xe0) { /* 3 byte char */
101 cpoint = (c & 0x0f) << 12;
102 cb = 2;
103 }
104 else
105 if ((c & 0xf8) == 0xf0) { /* 4 byte char */
106 cpoint = (c & 0x07) << 18;
107 cb = 3;
108 }
109
110 /* process the continuation bytes */
111 while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
112 cpoint |= (*p++ & 0x3f) << (--cb * 6);
113
114 /* incomplete or broken? */
115 if (cb)
116 cpoint = 0xfffd;
117
118 *str = p;
119 return cpoint;
120}
121
122
123/** Unicode character width: intentionally dead simple **/
124
125static unsigned int xs_unicode_width_table[] = {
126 0x300, 0x36f, 0, /* diacritics */
127 0x1100, 0x11ff, 2, /* Hangul */
128 0x2e80, 0xa4cf, 2, /* CJK */
129 0xac00, 0xd7a3, 2, /* more Hangul */
130 0xe000, 0xf8ff, 0, /* private use */
131 0xf900, 0xfaff, 2, /* CJK compatibility */
132 0xff00, 0xff60, 2, /* full width things */
133 0xffdf, 0xffe6, 2, /* full width things */
134 0x1f200, 0x1ffff, 2, /* emojis */
135 0x20000, 0x2fffd, 2 /* more CJK */
136};
137
138/* magic number from https://en.wikipedia.org/wiki/Emoji#Unicode_blocks */
139int xs_is_emoji(unsigned int cpoint) {
140/* returns wether the input is an utf8 emoji */
141 return cpoint > 0x00A9 && cpoint < 0x1ffff;
142}
143
144int xs_unicode_width(unsigned int cpoint)
145/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
146{
147 int b = 0;
148 int t = xs_countof(xs_unicode_width_table) / 3 - 1;
149
150 while (t >= b) {
151 int n = (b + t) / 2;
152 unsigned int *p = &xs_unicode_width_table[n * 3];
153
154 if (cpoint < p[0])
155 t = n - 1;
156 else
157 if (cpoint > p[1])
158 b = n + 1;
159 else
160 return p[2];
161 }
162
163 return 1;
164}
165
166
167int xs_is_diacritic(unsigned int cpoint)
168{
169 return cpoint >= 0x300 && cpoint <= 0x36f;
170}
171
172
173/** surrogate pairs **/
174
175int xs_is_surrogate(unsigned int cpoint)
176/* checks if cpoint is the first element of a Unicode surrogate pair */
177{
178 return cpoint >= 0xd800 && cpoint <= 0xdfff;
179}
180
181
182unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
183/* "decodes" a surrogate pair into a codepoint */
184{
185 return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
186}
187
188
189unsigned int xs_surrogate_enc(unsigned int cpoint)
190/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
191{
192 unsigned int p1 = 0xd7c0 + (cpoint >> 10);
193 unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
194
195 return (p1 << 16) | p2;
196}
197
198
199#ifdef _XS_H
200
201xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
202/* encodes an Unicode codepoint to utf-8 into str */
203{
204 char tmp[4];
205
206 int c = xs_utf8_enc(tmp, cpoint);
207
208 str = xs_insert_m(str, *offset, tmp, c);
209
210 *offset += c;
211
212 return str;
213}
214
215
216xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
217/* encodes an Unicode codepoint to utf-8 into str */
218{
219 int offset = strlen(str);
220
221 return xs_utf8_insert(str, cpoint, &offset);
222}
223
224#endif /* _XS_H */
225
226
227#ifdef _XS_UNICODE_TBL_H
228
229/* include xs_unicode_tbl.h before this one to use these functions */
230
231unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
232/* searches for an uppercase codepoint in the case fold table */
233{
234 int b = 0;
235 int t = xs_countof(xs_unicode_case_fold_table) / 2 + 1;
236
237 while (t >= b) {
238 int n = (b + t) / 2;
239 unsigned int *p = &xs_unicode_case_fold_table[n * 2];
240
241 if (cpoint < p[0])
242 t = n - 1;
243 else
244 if (cpoint > p[0])
245 b = n + 1;
246 else
247 return p;
248 }
249
250 return NULL;
251}
252
253
254unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
255/* searches for a lowercase codepoint in the case fold table */
256{
257 unsigned int *p = xs_unicode_case_fold_table;
258 unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
259
260 while (p < e) {
261 if (cpoint == p[1])
262 return p;
263
264 p += 2;
265 }
266
267 return NULL;
268}
269
270
271unsigned int xs_unicode_to_lower(unsigned int cpoint)
272/* returns the cpoint to lowercase */
273{
274 if (cpoint < 0x80)
275 return tolower(cpoint);
276
277 unsigned int *p = _xs_unicode_upper_search(cpoint);
278
279 return p == NULL ? cpoint : p[1];
280}
281
282
283unsigned int xs_unicode_to_upper(unsigned int cpoint)
284/* returns the cpoint to uppercase */
285{
286 if (cpoint < 0x80)
287 return toupper(cpoint);
288
289 unsigned int *p = _xs_unicode_lower_search(cpoint);
290
291 return p == NULL ? cpoint : p[0];
292}
293
294
295int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
296/* applies unicode Normalization Form D */
297{
298 int b = 0;
299 int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
300
301 while (t >= b) {
302 int n = (b + t) / 2;
303 unsigned int *p = &xs_unicode_nfd_table[n * 3];
304
305 int c = cpoint - p[0];
306
307 if (c < 0)
308 t = n - 1;
309 else
310 if (c > 0)
311 b = n + 1;
312 else {
313 *base = p[1];
314 *diac = p[2];
315 return 1;
316 }
317 }
318
319 return 0;
320}
321
322
323int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
324/* applies unicode Normalization Form C */
325{
326 unsigned int *p = xs_unicode_nfd_table;
327 unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
328
329 while (p < e) {
330 if (p[1] == base && p[2] == diac) {
331 *cpoint = p[0];
332 return 1;
333 }
334
335 p += 3;
336 }
337
338 return 0;
339}
340
341
342int xs_unicode_is_alpha(unsigned int cpoint)
343/* checks if a codepoint is an alpha (i.e. a letter) */
344{
345 int b = 0;
346 int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
347
348 while (t >= b) {
349 int n = (b + t) / 2;
350 unsigned int *p = &xs_unicode_alpha_table[n * 2];
351
352 if (cpoint < p[0])
353 t = n - 1;
354 else
355 if (cpoint > p[1])
356 b = n + 1;
357 else
358 return 1;
359 }
360
361 return 0;
362}
363
364
365int xs_unicode_is_right_to_left(unsigned int cpoint)
366/* checks if a codepoint is a right-to-left letter */
367{
368 int b = 0;
369 int t = xs_countof(xs_unicode_right_to_left_table) / 2 - 1;
370
371 while (t >= b) {
372 int n = (b + t) / 2;
373 unsigned int *p = &xs_unicode_right_to_left_table[n * 2];
374
375 if (cpoint < p[0])
376 t = n - 1;
377 else
378 if (cpoint > p[1])
379 b = n + 1;
380 else
381 return 1;
382 }
383
384 return 0;
385}
386
387
388#ifdef _XS_H
389
390xs_str *xs_utf8_to_upper(const char *str)
391{
392 xs_str *s = xs_str_new(NULL);
393 unsigned int cpoint;
394 int offset = 0;
395
396 while ((cpoint = xs_utf8_dec(&str))) {
397 cpoint = xs_unicode_to_upper(cpoint);
398 s = xs_utf8_insert(s, cpoint, &offset);
399 }
400
401 return s;
402}
403
404
405xs_str *xs_utf8_to_lower(const char *str)
406{
407 xs_str *s = xs_str_new(NULL);
408 unsigned int cpoint;
409 int offset = 0;
410
411 while ((cpoint = xs_utf8_dec(&str))) {
412 cpoint = xs_unicode_to_lower(cpoint);
413 s = xs_utf8_insert(s, cpoint, &offset);
414 }
415
416 return s;
417}
418
419
420xs_str *xs_utf8_to_nfd(const char *str)
421{
422 xs_str *s = xs_str_new(NULL);
423 unsigned int cpoint;
424 int offset = 0;
425
426 while ((cpoint = xs_utf8_dec(&str))) {
427 unsigned int base;
428 unsigned int diac;
429
430 if (xs_unicode_nfd(cpoint, &base, &diac)) {
431 s = xs_utf8_insert(s, base, &offset);
432 s = xs_utf8_insert(s, diac, &offset);
433 }
434 else
435 s = xs_utf8_insert(s, cpoint, &offset);
436 }
437
438 return s;
439}
440
441
442xs_str *xs_utf8_to_nfc(const char *str)
443{
444 xs_str *s = xs_str_new(NULL);
445 unsigned int cpoint;
446 unsigned int base = 0;
447 int offset = 0;
448
449 while ((cpoint = xs_utf8_dec(&str))) {
450 if (xs_is_diacritic(cpoint)) {
451 if (xs_unicode_nfc(base, cpoint, &base))
452 continue;
453 }
454
455 if (base)
456 s = xs_utf8_insert(s, base, &offset);
457
458 base = cpoint;
459 }
460
461 if (base)
462 s = xs_utf8_insert(s, base, &offset);
463
464 return s;
465}
466
467#endif /* _XS_H */
468
469#endif /* _XS_UNICODE_TBL_H */
470
471#endif /* XS_IMPLEMENTATION */
472
473#endif /* _XS_UNICODE_H */
