2.62
xs_unicode.h
1/* copyright (c) 2022 - 2024 grunfink et al. / MIT license */
2
3#ifndef _XS_UNICODE_H
4
5#define _XS_UNICODE_H
6
7 int xs_utf8_enc(char buf[4], unsigned int cpoint);
8 int xs_is_utf8_cont_byte(char c);
9 unsigned int xs_utf8_dec(const char **str);
10 int xs_unicode_width(unsigned int cpoint);
11 int xs_is_surrogate(unsigned int cpoint);
12 int xs_is_diacritic(unsigned int cpoint);
13 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
14 unsigned int xs_surrogate_enc(unsigned int cpoint);
15 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
16 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
17 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
18 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
19 unsigned int xs_unicode_to_upper(unsigned int cpoint);
20 unsigned int xs_unicode_to_lower(unsigned int cpoint);
21 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
22 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
23 int xs_unicode_is_alpha(unsigned int cpoint);
24
25#ifdef _XS_H
26 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
27 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
28 xs_str *xs_utf8_to_upper(const char *str);
29 xs_str *xs_utf8_to_lower(const char *str);
30 xs_str *xs_utf8_to_nfd(const char *str);
31 xs_str *xs_utf8_to_nfc(const char *str);
32#endif
33
34#ifdef XS_IMPLEMENTATION
35
36#include <ctype.h>
37
38#ifndef xs_countof
39#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
40#endif
41
42int xs_utf8_enc(char buf[4], unsigned int cpoint)
43/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
44{
45 char *p = buf;
46
47 if (cpoint < 0x80) /* 1 byte char */
48 *p++ = cpoint & 0xff;
49 else {
50 if (cpoint < 0x800) /* 2 byte char */
51 *p++ = 0xc0 | (cpoint >> 6);
52 else {
53 if (cpoint < 0x10000) /* 3 byte char */
54 *p++ = 0xe0 | (cpoint >> 12);
55 else { /* 4 byte char */
56 *p++ = 0xf0 | (cpoint >> 18);
57 *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
58 }
59
60 *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
61 }
62
63 *p++ = 0x80 | (cpoint & 0x3f);
64 }
65
66 return p - buf;
67}
68
69
70int xs_is_utf8_cont_byte(char c)
71/* returns true if c is an utf8 continuation byte */
72{
73 return ((c & 0xc0) == 0x80);
74}
75
76
77unsigned int xs_utf8_dec(const char **str)
78/* decodes an utf-8 char inside str and updates the pointer */
79{
80 const char *p = *str;
81 unsigned int cpoint = 0;
82 unsigned char c = *p++;
83 int cb = 0;
84
85 if ((c & 0x80) == 0) { /* 1 byte char */
86 cpoint = c;
87 }
88 else
89 if ((c & 0xe0) == 0xc0) { /* 2 byte char */
90 cpoint = (c & 0x1f) << 6;
91 cb = 1;
92 }
93 else
94 if ((c & 0xf0) == 0xe0) { /* 3 byte char */
95 cpoint = (c & 0x0f) << 12;
96 cb = 2;
97 }
98 else
99 if ((c & 0xf8) == 0xf0) { /* 4 byte char */
100 cpoint = (c & 0x07) << 18;
101 cb = 3;
102 }
103
104 /* process the continuation bytes */
105 while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
106 cpoint |= (*p++ & 0x3f) << (--cb * 6);
107
108 /* incomplete or broken? */
109 if (cb)
110 cpoint = 0xfffd;
111
112 *str = p;
113 return cpoint;
114}
115
116
117/** Unicode character width: intentionally dead simple **/
118
119static unsigned int xs_unicode_width_table[] = {
120 0x300, 0x36f, 0, /* diacritics */
121 0x1100, 0x11ff, 2, /* Hangul */
122 0x2e80, 0xa4cf, 2, /* CJK */
123 0xac00, 0xd7a3, 2, /* more Hangul */
124 0xe000, 0xf8ff, 0, /* private use */
125 0xf900, 0xfaff, 2, /* CJK compatibility */
126 0xff00, 0xff60, 2, /* full width things */
127 0xffdf, 0xffe6, 2, /* full width things */
128 0x1f200, 0x1ffff, 2, /* emojis */
129 0x20000, 0x2fffd, 2 /* more CJK */
130};
131
132int xs_unicode_width(unsigned int cpoint)
133/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
134{
135 int b = 0;
136 int t = xs_countof(xs_unicode_width_table) / 3 - 1;
137
138 while (t >= b) {
139 int n = (b + t) / 2;
140 unsigned int *p = &xs_unicode_width_table[n * 3];
141
142 if (cpoint < p[0])
143 t = n - 1;
144 else
145 if (cpoint > p[1])
146 b = n + 1;
147 else
148 return p[2];
149 }
150
151 return 1;
152}
153
154
155int xs_is_diacritic(unsigned int cpoint)
156{
157 return cpoint >= 0x300 && cpoint <= 0x36f;
158}
159
160
161/** surrogate pairs **/
162
163int xs_is_surrogate(unsigned int cpoint)
164/* checks if cpoint is the first element of a Unicode surrogate pair */
165{
166 return cpoint >= 0xd800 && cpoint <= 0xdfff;
167}
168
169
170unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
171/* "decodes" a surrogate pair into a codepoint */
172{
173 return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
174}
175
176
177unsigned int xs_surrogate_enc(unsigned int cpoint)
178/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
179{
180 unsigned int p1 = 0xd7c0 + (cpoint >> 10);
181 unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
182
183 return (p1 << 16) | p2;
184}
185
186
187#ifdef _XS_H
188
189xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
190/* encodes an Unicode codepoint to utf-8 into str */
191{
192 char tmp[4];
193
194 int c = xs_utf8_enc(tmp, cpoint);
195
196 str = xs_insert_m(str, *offset, tmp, c);
197
198 *offset += c;
199
200 return str;
201}
202
203
204xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
205/* encodes an Unicode codepoint to utf-8 into str */
206{
207 int offset = strlen(str);
208
209 return xs_utf8_insert(str, cpoint, &offset);
210}
211
212#endif /* _XS_H */
213
214
215#ifdef _XS_UNICODE_TBL_H
216
217/* include xs_unicode_tbl.h before this one to use these functions */
218
219unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
220/* searches for an uppercase codepoint in the case fold table */
221{
222 int b = 0;
223 int t = xs_countof(xs_unicode_case_fold_table) / 2 + 1;
224
225 while (t >= b) {
226 int n = (b + t) / 2;
227 unsigned int *p = &xs_unicode_case_fold_table[n * 2];
228
229 if (cpoint < p[0])
230 t = n - 1;
231 else
232 if (cpoint > p[0])
233 b = n + 1;
234 else
235 return p;
236 }
237
238 return NULL;
239}
240
241
242unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
243/* searches for a lowercase codepoint in the case fold table */
244{
245 unsigned int *p = xs_unicode_case_fold_table;
246 unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
247
248 while (p < e) {
249 if (cpoint == p[1])
250 return p;
251
252 p += 2;
253 }
254
255 return NULL;
256}
257
258
259unsigned int xs_unicode_to_lower(unsigned int cpoint)
260/* returns the cpoint to lowercase */
261{
262 if (cpoint < 0x80)
263 return tolower(cpoint);
264
265 unsigned int *p = _xs_unicode_upper_search(cpoint);
266
267 return p == NULL ? cpoint : p[1];
268}
269
270
271unsigned int xs_unicode_to_upper(unsigned int cpoint)
272/* returns the cpoint to uppercase */
273{
274 if (cpoint < 0x80)
275 return toupper(cpoint);
276
277 unsigned int *p = _xs_unicode_lower_search(cpoint);
278
279 return p == NULL ? cpoint : p[0];
280}
281
282
283int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
284/* applies unicode Normalization Form D */
285{
286 int b = 0;
287 int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
288
289 while (t >= b) {
290 int n = (b + t) / 2;
291 unsigned int *p = &xs_unicode_nfd_table[n * 3];
292
293 int c = cpoint - p[0];
294
295 if (c < 0)
296 t = n - 1;
297 else
298 if (c > 0)
299 b = n + 1;
300 else {
301 *base = p[1];
302 *diac = p[2];
303 return 1;
304 }
305 }
306
307 return 0;
308}
309
310
311int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
312/* applies unicode Normalization Form C */
313{
314 unsigned int *p = xs_unicode_nfd_table;
315 unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
316
317 while (p < e) {
318 if (p[1] == base && p[2] == diac) {
319 *cpoint = p[0];
320 return 1;
321 }
322
323 p += 3;
324 }
325
326 return 0;
327}
328
329
330int xs_unicode_is_alpha(unsigned int cpoint)
331/* checks if a codepoint is an alpha (i.e. a letter) */
332{
333 int b = 0;
334 int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
335
336 while (t >= b) {
337 int n = (b + t) / 2;
338 unsigned int *p = &xs_unicode_alpha_table[n * 2];
339
340 if (cpoint < p[0])
341 t = n - 1;
342 else
343 if (cpoint > p[1])
344 b = n + 1;
345 else
346 return 1;
347 }
348
349 return 0;
350}
351
352
353#ifdef _XS_H
354
355xs_str *xs_utf8_to_upper(const char *str)
356{
357 xs_str *s = xs_str_new(NULL);
358 unsigned int cpoint;
359 int offset = 0;
360
361 while ((cpoint = xs_utf8_dec(&str))) {
362 cpoint = xs_unicode_to_upper(cpoint);
363 s = xs_utf8_insert(s, cpoint, &offset);
364 }
365
366 return s;
367}
368
369
370xs_str *xs_utf8_to_lower(const char *str)
371{
372 xs_str *s = xs_str_new(NULL);
373 unsigned int cpoint;
374 int offset = 0;
375
376 while ((cpoint = xs_utf8_dec(&str))) {
377 cpoint = xs_unicode_to_lower(cpoint);
378 s = xs_utf8_insert(s, cpoint, &offset);
379 }
380
381 return s;
382}
383
384
385xs_str *xs_utf8_to_nfd(const char *str)
386{
387 xs_str *s = xs_str_new(NULL);
388 unsigned int cpoint;
389 int offset = 0;
390
391 while ((cpoint = xs_utf8_dec(&str))) {
392 unsigned int base;
393 unsigned int diac;
394
395 if (xs_unicode_nfd(cpoint, &base, &diac)) {
396 s = xs_utf8_insert(s, base, &offset);
397 s = xs_utf8_insert(s, diac, &offset);
398 }
399 else
400 s = xs_utf8_insert(s, cpoint, &offset);
401 }
402
403 return s;
404}
405
406
407xs_str *xs_utf8_to_nfc(const char *str)
408{
409 xs_str *s = xs_str_new(NULL);
410 unsigned int cpoint;
411 unsigned int base = 0;
412 int offset = 0;
413
414 while ((cpoint = xs_utf8_dec(&str))) {
415 if (xs_is_diacritic(cpoint)) {
416 if (xs_unicode_nfc(base, cpoint, &base))
417 continue;
418 }
419
420 if (base)
421 s = xs_utf8_insert(s, base, &offset);
422
423 base = cpoint;
424 }
425
426 if (base)
427 s = xs_utf8_insert(s, base, &offset);
428
429 return s;
430}
431
432#endif /* _XS_H */
433
434#endif /* _XS_UNICODE_TBL_H */
435
436#endif /* XS_IMPLEMENTATION */
437
438#endif /* _XS_UNICODE_H */
