1/* copyright (c) 2022 - 2024 grunfink et al. / MIT license */
  2
  3#ifndef _XS_UNICODE_H
  4
  5#define _XS_UNICODE_H
  6
  7 int xs_utf8_enc(char buf[4], unsigned int cpoint);
  8 int xs_is_utf8_cont_byte(char c);
  9 unsigned int xs_utf8_dec(const char **str);
 10 int xs_unicode_width(unsigned int cpoint);
 11 int xs_is_surrogate(unsigned int cpoint);
 12 int xs_is_diacritic(unsigned int cpoint);
 13 unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2);
 14 unsigned int xs_surrogate_enc(unsigned int cpoint);
 15 unsigned int *_xs_unicode_upper_search(unsigned int cpoint);
 16 unsigned int *_xs_unicode_lower_search(unsigned int cpoint);
 17 #define xs_unicode_is_upper(cpoint) (!!_xs_unicode_upper_search(cpoint))
 18 #define xs_unicode_is_lower(cpoint) (!!_xs_unicode_lower_search(cpoint))
 19 unsigned int xs_unicode_to_upper(unsigned int cpoint);
 20 unsigned int xs_unicode_to_lower(unsigned int cpoint);
 21 int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac);
 22 int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint);
 23 int xs_unicode_is_alpha(unsigned int cpoint);
 24
 25#ifdef _XS_H
 26 xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset);
 27 xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint);
 28 xs_str *xs_utf8_to_upper(const char *str);
 29 xs_str *xs_utf8_to_lower(const char *str);
 30 xs_str *xs_utf8_to_nfd(const char *str);
 31 xs_str *xs_utf8_to_nfc(const char *str);
 32#endif
 33
 34#ifdef XS_IMPLEMENTATION
 35
 36#ifndef xs_countof
 37#define xs_countof(a) (sizeof((a)) / sizeof((*a)))
 38#endif
 39
 40int xs_utf8_enc(char buf[4], unsigned int cpoint)
 41/* encodes an Unicode codepoint to utf-8 into buf and returns the size in bytes */
 42{
 43    char *p = buf;
 44
 45    if (cpoint < 0x80) /* 1 byte char */
 46        *p++ = cpoint & 0xff;
 47    else {
 48        if (cpoint < 0x800) /* 2 byte char */
 49            *p++ = 0xc0 | (cpoint >> 6);
 50        else {
 51            if (cpoint < 0x10000) /* 3 byte char */
 52                *p++ = 0xe0 | (cpoint >> 12);
 53            else { /* 4 byte char */
 54                *p++ = 0xf0 | (cpoint >> 18);
 55                *p++ = 0x80 | ((cpoint >> 12) & 0x3f);
 56            }
 57
 58            *p++ = 0x80 | ((cpoint >> 6) & 0x3f);
 59        }
 60
 61        *p++ = 0x80 | (cpoint & 0x3f);
 62    }
 63
 64    return p - buf;
 65}
 66
 67
 68int xs_is_utf8_cont_byte(char c)
 69/* returns true if c is an utf8 continuation byte */
 70{
 71    return ((c & 0xc0) == 0x80);
 72}
 73
 74
 75unsigned int xs_utf8_dec(const char **str)
 76/* decodes an utf-8 char inside str and updates the pointer */
 77{
 78    const char *p = *str;
 79    unsigned int cpoint = 0;
 80    unsigned char c = *p++;
 81    int cb = 0;
 82
 83    if ((c & 0x80) == 0) { /* 1 byte char */
 84        cpoint = c;
 85    }
 86    else
 87    if ((c & 0xe0) == 0xc0) { /* 2 byte char */
 88        cpoint = (c & 0x1f) << 6;
 89        cb = 1;
 90    }
 91    else
 92    if ((c & 0xf0) == 0xe0) { /* 3 byte char */
 93        cpoint = (c & 0x0f) << 12;
 94        cb = 2;
 95    }
 96    else
 97    if ((c & 0xf8) == 0xf0) { /* 4 byte char */
 98        cpoint = (c & 0x07) << 18;
 99        cb = 3;
100    }
101
102    /* process the continuation bytes */
103    while (cb > 0 && *p && xs_is_utf8_cont_byte(*p))
104        cpoint |= (*p++ & 0x3f) << (--cb * 6);
105
106    /* incomplete or broken? */
107    if (cb)
108        cpoint = 0xfffd;
109
110    *str = p;
111    return cpoint;
112}
113
114
115/** Unicode character width: intentionally dead simple **/
116
117static unsigned int xs_unicode_width_table[] = {
118    0x300,      0x36f,      0,      /* diacritics */
119    0x1100,     0x11ff,     2,      /* Hangul */
120    0x2e80,     0xa4cf,     2,      /* CJK */
121    0xac00,     0xd7a3,     2,      /* more Hangul */
122    0xe000,     0xf8ff,     0,      /* private use */
123    0xf900,     0xfaff,     2,      /* CJK compatibility */
124    0xff00,     0xff60,     2,      /* full width things */
125    0xffdf,     0xffe6,     2,      /* full width things */
126    0x1f200,    0x1ffff,    2,      /* emojis */
127    0x20000,    0x2fffd,    2       /* more CJK */
128};
129
130int xs_unicode_width(unsigned int cpoint)
131/* returns the width in columns of a Unicode codepoint (somewhat simplified) */
132{
133    int b = 0;
134    int t = xs_countof(xs_unicode_width_table) / 3 - 1;
135
136    while (t >= b) {
137        int n = (b + t) / 2;
138        unsigned int *p = &xs_unicode_width_table[n * 3];
139
140        if (cpoint < p[0])
141            t = n - 1;
142        else
143        if (cpoint > p[1])
144            b = n + 1;
145        else
146            return p[2];
147    }
148
149    return 1;
150}
151
152
153int xs_is_diacritic(unsigned int cpoint)
154{
155    return cpoint >= 0x300 && cpoint <= 0x36f;
156}
157
158
159/** surrogate pairs **/
160
161int xs_is_surrogate(unsigned int cpoint)
162/* checks if cpoint is the first element of a Unicode surrogate pair */
163{
164    return cpoint >= 0xd800 && cpoint <= 0xdfff;
165}
166
167
168unsigned int xs_surrogate_dec(unsigned int p1, unsigned int p2)
169/* "decodes" a surrogate pair into a codepoint */
170{
171    return 0x10000 | ((p1 & 0x3ff) << 10) | (p2 & 0x3ff);
172}
173
174
175unsigned int xs_surrogate_enc(unsigned int cpoint)
176/* "encodes" a Unicode into a surrogate pair (p1 in the MSB word) */
177{
178    unsigned int p1 = 0xd7c0 + (cpoint >> 10);
179    unsigned int p2 = 0xdc00 + (cpoint & 0x3ff);
180
181    return (p1 << 16) | p2;
182}
183
184
185#ifdef _XS_H
186
187xs_str *xs_utf8_insert(xs_str *str, unsigned int cpoint, int *offset)
188/* encodes an Unicode codepoint to utf-8 into str */
189{
190    char tmp[4];
191
192    int c = xs_utf8_enc(tmp, cpoint);
193
194    str = xs_insert_m(str, *offset, tmp, c);
195
196    *offset += c;
197
198    return str;
199}
200
201
202xs_str *xs_utf8_cat(xs_str *str, unsigned int cpoint)
203/* encodes an Unicode codepoint to utf-8 into str */
204{
205    int offset = strlen(str);
206
207    return xs_utf8_insert(str, cpoint, &offset);
208}
209
210#endif /* _XS_H */
211
212
213#ifdef _XS_UNICODE_TBL_H
214
215/* include xs_unicode_tbl.h before this one to use these functions */
216
217unsigned int *_xs_unicode_upper_search(unsigned int cpoint)
218/* searches for an uppercase codepoint in the case fold table */
219{
220    int b = 0;
221    int t = xs_countof(xs_unicode_case_fold_table) / 2 + 1;
222
223    while (t >= b) {
224        int n = (b + t) / 2;
225        unsigned int *p = &xs_unicode_case_fold_table[n * 2];
226
227        if (cpoint < p[0])
228            t = n - 1;
229        else
230        if (cpoint > p[0])
231            b = n + 1;
232        else
233            return p;
234    }
235
236    return NULL;
237}
238
239
240unsigned int *_xs_unicode_lower_search(unsigned int cpoint)
241/* searches for a lowercase codepoint in the case fold table */
242{
243    unsigned int *p = xs_unicode_case_fold_table;
244    unsigned int *e = p + xs_countof(xs_unicode_case_fold_table);
245
246    while (p < e) {
247        if (cpoint == p[1])
248            return p;
249
250        p += 2;
251    }
252
253    return NULL;
254}
255
256
257unsigned int xs_unicode_to_lower(unsigned int cpoint)
258/* returns the cpoint to lowercase */
259{
260    if (cpoint < 0x80)
261        return tolower(cpoint);
262
263    unsigned int *p = _xs_unicode_upper_search(cpoint);
264
265    return p == NULL ? cpoint : p[1];
266}
267
268
269unsigned int xs_unicode_to_upper(unsigned int cpoint)
270/* returns the cpoint to uppercase */
271{
272    if (cpoint < 0x80)
273        return toupper(cpoint);
274
275    unsigned int *p = _xs_unicode_lower_search(cpoint);
276
277    return p == NULL ? cpoint : p[0];
278}
279
280
281int xs_unicode_nfd(unsigned int cpoint, unsigned int *base, unsigned int *diac)
282/* applies unicode Normalization Form D */
283{
284    int b = 0;
285    int t = xs_countof(xs_unicode_nfd_table) / 3 - 1;
286
287    while (t >= b) {
288        int n = (b + t) / 2;
289        unsigned int *p = &xs_unicode_nfd_table[n * 3];
290
291        int c = cpoint - p[0];
292
293        if (c < 0)
294            t = n - 1;
295        else
296        if (c > 0)
297            b = n + 1;
298        else {
299            *base = p[1];
300            *diac = p[2];
301            return 1;
302        }
303    }
304
305    return 0;
306}
307
308
309int xs_unicode_nfc(unsigned int base, unsigned int diac, unsigned int *cpoint)
310/* applies unicode Normalization Form C */
311{
312    unsigned int *p = xs_unicode_nfd_table;
313    unsigned int *e = p + xs_countof(xs_unicode_nfd_table);
314
315    while (p < e) {
316        if (p[1] == base && p[2] == diac) {
317            *cpoint = p[0];
318            return 1;
319        }
320
321        p += 3;
322    }
323
324    return 0;
325}
326
327
328int xs_unicode_is_alpha(unsigned int cpoint)
329/* checks if a codepoint is an alpha (i.e. a letter) */
330{
331    int b = 0;
332    int t = xs_countof(xs_unicode_alpha_table) / 2 - 1;
333
334    while (t >= b) {
335        int n = (b + t) / 2;
336        unsigned int *p = &xs_unicode_alpha_table[n * 2];
337
338        if (cpoint < p[0])
339            t = n - 1;
340        else
341        if (cpoint > p[1])
342            b = n + 1;
343        else
344            return 1;
345    }
346
347    return 0;
348}
349
350
351#ifdef _XS_H
352
353xs_str *xs_utf8_to_upper(const char *str)
354{
355    xs_str *s = xs_str_new(NULL);
356    unsigned int cpoint;
357    int offset = 0;
358
359    while ((cpoint = xs_utf8_dec(&str))) {
360        cpoint = xs_unicode_to_upper(cpoint);
361        s = xs_utf8_insert(s, cpoint, &offset);
362    }
363
364    return s;
365}
366
367
368xs_str *xs_utf8_to_lower(const char *str)
369{
370    xs_str *s = xs_str_new(NULL);
371    unsigned int cpoint;
372    int offset = 0;
373
374    while ((cpoint = xs_utf8_dec(&str))) {
375        cpoint = xs_unicode_to_lower(cpoint);
376        s = xs_utf8_insert(s, cpoint, &offset);
377    }
378
379    return s;
380}
381
382
383xs_str *xs_utf8_to_nfd(const char *str)
384{
385    xs_str *s = xs_str_new(NULL);
386    unsigned int cpoint;
387    int offset = 0;
388
389    while ((cpoint = xs_utf8_dec(&str))) {
390        unsigned int base;
391        unsigned int diac;
392
393        if (xs_unicode_nfd(cpoint, &base, &diac)) {
394            s = xs_utf8_insert(s, base, &offset);
395            s = xs_utf8_insert(s, diac, &offset);
396        }
397        else
398            s = xs_utf8_insert(s, cpoint, &offset);
399    }
400
401    return s;
402}
403
404
405xs_str *xs_utf8_to_nfc(const char *str)
406{
407    xs_str *s = xs_str_new(NULL);
408    unsigned int cpoint;
409    unsigned int base = 0;
410    int offset = 0;
411
412    while ((cpoint = xs_utf8_dec(&str))) {
413        if (xs_is_diacritic(cpoint)) {
414            if (xs_unicode_nfc(base, cpoint, &base))
415                continue;
416        }
417
418        if (base)
419            s = xs_utf8_insert(s, base, &offset);
420
421        base = cpoint;
422    }
423
424    if (base)
425        s = xs_utf8_insert(s, base, &offset);
426
427    return s;
428}
429
430#endif /* _XS_H */
431
432#endif /* _XS_UNICODE_TBL_H */
433
434#endif /* XS_IMPLEMENTATION */
435
436#endif /* _XS_UNICODE_H */