2.80
xs_unicode.h
1/* copyright (c) 2022 - 2025 grunfink et al. / MIT license */
2
3#ifndef _XS_UNICODE_H
4
5#define _XS_UNICODE_H
6
7 int xs_utf8_enc(char buf[4], unsigned int cpoint);
8 int xs_is_utf8_cont_byte(char c);
9 unsigned int xs_utf8_dec(const char **str);
10 int xs_unicode_width(unsigned int cpoint);
11 int xs_is_surrogate(unsigned int cpoint);
12 int xs_is_diacritic(unsigned int cpoint);
13 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
14 unsigned int xs_surrogate_enc(unsigned int cpoint);
15 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
16 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
17 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
18 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
19 unsigned int xs_unicode_to_upper(unsigned int cpoint);
20 unsigned int xs_unicode_to_lower(unsigned int cpoint);
21 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
22 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
23 int xs_unicode_is_alpha(unsigned int cpoint);
24 int xs_unicode_is_right_to_left(unsigned int cpoint);
25
26#ifdef _XS_H
27 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
28 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
29 xs_str *xs_utf8_to_upper(const char *str);
30 xs_str *xs_utf8_to_lower(const char *str);
31 xs_str *xs_utf8_to_nfd(const char *str);
32 xs_str *xs_utf8_to_nfc(const char *str);
33#endif
34
35#ifdef XS_IMPLEMENTATION
36
37#include <ctype.h>
38
39#ifndef xs_countof
40#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
41#endif
42
43int xs_utf8_enc(char buf[4], unsigned int cpoint)
44/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
45{
46 char *p = buf;
47
48 if (cpoint < 0x80) /* 1 byte char */
49 *p++ = cpoint & 0xff;
50 else {
51 if (cpoint < 0x800) /* 2 byte char */
52 *p++ = 0xc0 | (cpoint >> 6);
53 else {
54 if (cpoint < 0x10000) /* 3 byte char */
55 *p++ = 0xe0 | (cpoint >> 12);
56 else { /* 4 byte char */
57 *p++ = 0xf0 | (cpoint >> 18);
58 *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
59 }
60
61 *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
62 }
63
64 *p++ = 0x80 | (cpoint & 0x3f);
65 }
66
67 return p - buf;
68}
69
70
71int xs_is_utf8_cont_byte(char c)
72/* returns true if c is an utf8 continuation byte */
73{
74 return ((c & 0xc0) == 0x80);
75}
76
77
78unsigned int xs_utf8_dec(const char **str)
79/* decodes an utf-8 char inside str and updates the pointer */
80{
81 const char *p = *str;
82 unsigned int cpoint = 0;
83 unsigned char c = *p++;
84 int cb = 0;
85
86 if ((c & 0x80) == 0) { /* 1 byte char */
87 cpoint = c;
88 }
89 else
90 if ((c & 0xe0) == 0xc0) { /* 2 byte char */
91 cpoint = (c & 0x1f) << 6;
92 cb = 1;
93 }
94 else
95 if ((c & 0xf0) == 0xe0) { /* 3 byte char */
96 cpoint = (c & 0x0f) << 12;
97 cb = 2;
98 }
99 else
100 if ((c & 0xf8) == 0xf0) { /* 4 byte char */
101 cpoint = (c & 0x07) << 18;
102 cb = 3;
103 }
104
105 /* process the continuation bytes */
106 while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
107 cpoint |= (*p++ & 0x3f) << (--cb * 6);
108
109 /* incomplete or broken? */
110 if (cb)
111 cpoint = 0xfffd;
112
113 *str = p;
114 return cpoint;
115}
116
117
118/** Unicode character width: intentionally dead simple **/
119
120static unsigned int xs_unicode_width_table[] = {
121 0x300, 0x36f, 0, /* diacritics */
122 0x1100, 0x11ff, 2, /* Hangul */
123 0x2e80, 0xa4cf, 2, /* CJK */
124 0xac00, 0xd7a3, 2, /* more Hangul */
125 0xe000, 0xf8ff, 0, /* private use */
126 0xf900, 0xfaff, 2, /* CJK compatibility */
127 0xff00, 0xff60, 2, /* full width things */
128 0xffdf, 0xffe6, 2, /* full width things */
129 0x1f200, 0x1ffff, 2, /* emojis */
130 0x20000, 0x2fffd, 2 /* more CJK */
131};
132
133int xs_unicode_width(unsigned int cpoint)
134/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
135{
136 int b = 0;
137 int t = xs_countof(xs_unicode_width_table) / 3 - 1;
138
139 while (t >= b) {
140 int n = (b + t) / 2;
141 unsigned int *p = &xs_unicode_width_table[n * 3];
142
143 if (cpoint < p[0])
144 t = n - 1;
145 else
146 if (cpoint > p[1])
147 b = n + 1;
148 else
149 return p[2];
150 }
151
152 return 1;
153}
154
155
156int xs_is_diacritic(unsigned int cpoint)
157{
158 return cpoint >= 0x300 && cpoint <= 0x36f;
159}
160
161
162/** surrogate pairs **/
163
164int xs_is_surrogate(unsigned int cpoint)
165/* checks if cpoint is the first element of a Unicode surrogate pair */
166{
167 return cpoint >= 0xd800 && cpoint <= 0xdfff;
168}
169
170
171unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
172/* "decodes" a surrogate pair into a codepoint */
173{
174 return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
175}
176
177
178unsigned int xs_surrogate_enc(unsigned int cpoint)
179/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
180{
181 unsigned int p1 = 0xd7c0 + (cpoint >> 10);
182 unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
183
184 return (p1 << 16) | p2;
185}
186
187
188#ifdef _XS_H
189
190xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
191/* encodes an Unicode codepoint to utf-8 into str */
192{
193 char tmp[4];
194
195 int c = xs_utf8_enc(tmp, cpoint);
196
197 str = xs_insert_m(str, *offset, tmp, c);
198
199 *offset += c;
200
201 return str;
202}
203
204
205xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
206/* encodes an Unicode codepoint to utf-8 into str */
207{
208 int offset = strlen(str);
209
210 return xs_utf8_insert(str, cpoint, &offset);
211}
212
213#endif /* _XS_H */
214
215
216#ifdef _XS_UNICODE_TBL_H
217
218/* include xs_unicode_tbl.h before this one to use these functions */
219
220unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
221/* searches for an uppercase codepoint in the case fold table */
222{
223 int b = 0;
224 int t = xs_countof(xs_unicode_case_fold_table) / 2 + 1;
225
226 while (t >= b) {
227 int n = (b + t) / 2;
228 unsigned int *p = &xs_unicode_case_fold_table[n * 2];
229
230 if (cpoint < p[0])
231 t = n - 1;
232 else
233 if (cpoint > p[0])
234 b = n + 1;
235 else
236 return p;
237 }
238
239 return NULL;
240}
241
242
243unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
244/* searches for a lowercase codepoint in the case fold table */
245{
246 unsigned int *p = xs_unicode_case_fold_table;
247 unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
248
249 while (p < e) {
250 if (cpoint == p[1])
251 return p;
252
253 p += 2;
254 }
255
256 return NULL;
257}
258
259
260unsigned int xs_unicode_to_lower(unsigned int cpoint)
261/* returns the cpoint to lowercase */
262{
263 if (cpoint < 0x80)
264 return tolower(cpoint);
265
266 unsigned int *p = _xs_unicode_upper_search(cpoint);
267
268 return p == NULL ? cpoint : p[1];
269}
270
271
272unsigned int xs_unicode_to_upper(unsigned int cpoint)
273/* returns the cpoint to uppercase */
274{
275 if (cpoint < 0x80)
276 return toupper(cpoint);
277
278 unsigned int *p = _xs_unicode_lower_search(cpoint);
279
280 return p == NULL ? cpoint : p[0];
281}
282
283
284int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
285/* applies unicode Normalization Form D */
286{
287 int b = 0;
288 int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
289
290 while (t >= b) {
291 int n = (b + t) / 2;
292 unsigned int *p = &xs_unicode_nfd_table[n * 3];
293
294 int c = cpoint - p[0];
295
296 if (c < 0)
297 t = n - 1;
298 else
299 if (c > 0)
300 b = n + 1;
301 else {
302 *base = p[1];
303 *diac = p[2];
304 return 1;
305 }
306 }
307
308 return 0;
309}
310
311
312int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
313/* applies unicode Normalization Form C */
314{
315 unsigned int *p = xs_unicode_nfd_table;
316 unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
317
318 while (p < e) {
319 if (p[1] == base && p[2] == diac) {
320 *cpoint = p[0];
321 return 1;
322 }
323
324 p += 3;
325 }
326
327 return 0;
328}
329
330
331int xs_unicode_is_alpha(unsigned int cpoint)
332/* checks if a codepoint is an alpha (i.e. a letter) */
333{
334 int b = 0;
335 int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
336
337 while (t >= b) {
338 int n = (b + t) / 2;
339 unsigned int *p = &xs_unicode_alpha_table[n * 2];
340
341 if (cpoint < p[0])
342 t = n - 1;
343 else
344 if (cpoint > p[1])
345 b = n + 1;
346 else
347 return 1;
348 }
349
350 return 0;
351}
352
353
354int xs_unicode_is_right_to_left(unsigned int cpoint)
355/* checks if a codepoint is a right-to-left letter */
356{
357 int b = 0;
358 int t = xs_countof(xs_unicode_right_to_left_table) / 2 - 1;
359
360 while (t >= b) {
361 int n = (b + t) / 2;
362 unsigned int *p = &xs_unicode_right_to_left_table[n * 2];
363
364 if (cpoint < p[0])
365 t = n - 1;
366 else
367 if (cpoint > p[1])
368 b = n + 1;
369 else
370 return 1;
371 }
372
373 return 0;
374}
375
376
377#ifdef _XS_H
378
379xs_str *xs_utf8_to_upper(const char *str)
380{
381 xs_str *s = xs_str_new(NULL);
382 unsigned int cpoint;
383 int offset = 0;
384
385 while ((cpoint = xs_utf8_dec(&str))) {
386 cpoint = xs_unicode_to_upper(cpoint);
387 s = xs_utf8_insert(s, cpoint, &offset);
388 }
389
390 return s;
391}
392
393
394xs_str *xs_utf8_to_lower(const char *str)
395{
396 xs_str *s = xs_str_new(NULL);
397 unsigned int cpoint;
398 int offset = 0;
399
400 while ((cpoint = xs_utf8_dec(&str))) {
401 cpoint = xs_unicode_to_lower(cpoint);
402 s = xs_utf8_insert(s, cpoint, &offset);
403 }
404
405 return s;
406}
407
408
409xs_str *xs_utf8_to_nfd(const char *str)
410{
411 xs_str *s = xs_str_new(NULL);
412 unsigned int cpoint;
413 int offset = 0;
414
415 while ((cpoint = xs_utf8_dec(&str))) {
416 unsigned int base;
417 unsigned int diac;
418
419 if (xs_unicode_nfd(cpoint, &base, &diac)) {
420 s = xs_utf8_insert(s, base, &offset);
421 s = xs_utf8_insert(s, diac, &offset);
422 }
423 else
424 s = xs_utf8_insert(s, cpoint, &offset);
425 }
426
427 return s;
428}
429
430
431xs_str *xs_utf8_to_nfc(const char *str)
432{
433 xs_str *s = xs_str_new(NULL);
434 unsigned int cpoint;
435 unsigned int base = 0;
436 int offset = 0;
437
438 while ((cpoint = xs_utf8_dec(&str))) {
439 if (xs_is_diacritic(cpoint)) {
440 if (xs_unicode_nfc(base, cpoint, &base))
441 continue;
442 }
443
444 if (base)
445 s = xs_utf8_insert(s, base, &offset);
446
447 base = cpoint;
448 }
449
450 if (base)
451 s = xs_utf8_insert(s, base, &offset);
452
453 return s;
454}
455
456#endif /* _XS_H */
457
458#endif /* _XS_UNICODE_TBL_H */
459
460#endif /* XS_IMPLEMENTATION */
461
462#endif /* _XS_UNICODE_H */
