commit d688152
uint
·
2026-10-14 18:23:15 +0000 UTC
parent dd4ae93
UTF: Add libgrapheme headers
2 files changed,
+568,
-0
+70,
-0
1@@ -0,0 +1,70 @@
2+/* Copyright 2026 uint (and contributors)
3+ Copyright 2019-2025 Laslo Hunhold <dev@frign.de>
4+
5+Permission to use, copy, modify, and distribute this software for any
6+purpose with or without fee is hereby granted, provided that the above
7+copyright notice and this permission notice appear in all copies.
8+
9+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
16+
17+/* uleutf is based on Laslo Hunhold's libgrapheme which is hosted on
18+ suckless.org */
19+
20+#ifndef ULEUTF_H
21+#define ULEUTF_H
22+
23+#include <stddef.h>
24+
25+#include "uletypes.h"
26+
27+#define GRAPHEME_INVALID_CODEPOINT UINT32_C(0xFFFD)
28+
29+enum uleutf_bidirectional_direction {
30+ ULEUTF_BIDIRECTIONAL_DIRECTION_NEUTRAL,
31+ ULEUTF_BIDIRECTIONAL_DIRECTION_LTR,
32+ ULEUTF_BIDIRECTIONAL_DIRECTION_RTL
33+};
34+
35+size_t uleutf_bidirectional_get_line_embedding_levels(const ule_u32*, size_t, ule_i8*, size_t);
36+size_t uleutf_bidirectional_preprocess_paragraph(const ule_u32*, size_t, enum uleutf_bidirectional_direction, ule_u32*, size_t, enum uleutf_bidirectional_direction*);
37+size_t uleutf_bidirectional_reorder_line(const ule_u32 *, const ule_u32 *, size_t, ule_u32 *, size_t);
38+
39+size_t uleutf_decode_utf8(const char *, size_t, ule_u32 *);
40+size_t uleutf_encode_utf8(ule_u32, char *, size_t);
41+
42+ule_i8 uleutf_is_character_break(ule_u32, ule_u32, ule_u16 *);
43+
44+ule_i8 uleutf_is_lowercase(const ule_u32 *, size_t, size_t *);
45+ule_i8 uleutf_is_titlecase(const ule_u32 *, size_t, size_t *);
46+ule_i8 uleutf_is_uppercase(const ule_u32 *, size_t, size_t *);
47+
48+ule_i8 uleutf_is_lowercase_utf8(const char *, size_t, size_t *);
49+ule_i8 uleutf_is_titlecase_utf8(const char *, size_t, size_t *);
50+ule_i8 uleutf_is_uppercase_utf8(const char *, size_t, size_t *);
51+
52+size_t uleutf_next_character_break(const ule_u32 *, size_t);
53+size_t uleutf_next_line_break(const ule_u32 *, size_t);
54+size_t uleutf_next_sentence_break(const ule_u32 *, size_t);
55+size_t uleutf_next_word_break(const ule_u32 *, size_t);
56+
57+size_t uleutf_next_character_break_utf8(const char *, size_t);
58+size_t uleutf_next_line_break_utf8(const char *, size_t);
59+size_t uleutf_next_sentence_break_utf8(const char *, size_t);
60+size_t uleutf_next_word_break_utf8(const char *, size_t);
61+
62+size_t uleutf_to_lowercase(const ule_u32 *, size_t, ule_u32 *, size_t);
63+size_t uleutf_to_titlecase(const ule_u32 *, size_t, ule_u32 *, size_t);
64+size_t uleutf_to_uppercase(const ule_u32 *, size_t, ule_u32 *, size_t);
65+
66+size_t uleutf_to_lowercase_utf8(const char *, size_t, char *, size_t);
67+size_t uleutf_to_titlecase_utf8(const char *, size_t, char *, size_t);
68+size_t uleutf_to_uppercase_utf8(const char *, size_t, char *, size_t);
69+
70+#endif /* ULEUTF_H */
71+
+498,
-0
1@@ -0,0 +1,498 @@
2+#include <stddef.h>
3+
4+#include "uleutf.h"
5+
6+#define SIZE_MAX (0xFFFFFFFFF)
7+
8+enum herodotus_status {
9+ HERODOTUS_STATUS_SUCCESS,
10+ HERODOTUS_STATUS_END_OF_BUFFER,
11+ HERODOTUS_STATUS_SOFT_LIMIT_REACHED,
12+};
13+
14+enum herodotus_type {
15+ HERODOTUS_TYPE_CODEPOINT,
16+ HERODOTUS_TYPE_UTF8,
17+};
18+
19+typedef struct herodotus_reader {
20+ enum herodotus_type type;
21+ const void *src;
22+ size_t srclen;
23+ size_t off;
24+ ule_i8 terminated_by_null;
25+ size_t soft_limit[10];
26+} HERODOTUS_READER;
27+
28+typedef struct herodotus_writer {
29+ enum herodotus_type type;
30+ void *dest;
31+ size_t destlen;
32+ size_t off;
33+ size_t first_unwritable_offset;
34+} HERODOTUS_WRITER;
35+
36+struct proper {
37+ /*
38+ * prev_prop[1] prev_prop[0] | next_prop[0] next_prop[1]
39+ */
40+ struct {
41+ ule_u8 prev_prop[2];
42+ ule_u8 next_prop[2];
43+ } raw;
44+
45+ struct {
46+ ule_u8 prev_prop[2];
47+ ule_u8 next_prop[2];
48+ } skip;
49+
50+ HERODOTUS_READER mid_reader;
51+ HERODOTUS_READER raw_reader;
52+ HERODOTUS_READER skip_reader;
53+
54+ void *state;
55+ ule_u8 no_prop;
56+ ule_u8 (*get_break_prop)(ule_u32);
57+ ule_i8 (*is_skippable_prop)(ule_u8);
58+ void (*skip_shift_callback)(ule_u8, void *);
59+};
60+
61+void herodotus_reader_init(HERODOTUS_READER *, enum herodotus_type,
62+ const void *, size_t);
63+void herodotus_reader_copy(const HERODOTUS_READER *, HERODOTUS_READER *);
64+void herodotus_reader_push_advance_limit(HERODOTUS_READER *, size_t);
65+void herodotus_reader_pop_limit(HERODOTUS_READER *);
66+size_t herodotus_reader_number_read(const HERODOTUS_READER *);
67+size_t herodotus_reader_next_word_break(const HERODOTUS_READER *);
68+size_t herodotus_reader_next_codepoint_break(const HERODOTUS_READER *);
69+enum herodotus_status herodotus_read_codepoint(HERODOTUS_READER *, ule_i8, ule_u32 *);
70+
71+void herodotus_writer_init(HERODOTUS_WRITER *, enum herodotus_type, void *,
72+ size_t);
73+void herodotus_writer_nul_terminate(HERODOTUS_WRITER *);
74+size_t herodotus_writer_number_written(const HERODOTUS_WRITER *);
75+void herodotus_write_codepoint(HERODOTUS_WRITER *, ule_u32);
76+
77+void proper_init(const HERODOTUS_READER *, void *, ule_u8,
78+ ule_u8 (*get_break_prop)(ule_u32),
79+ ule_i8 (*is_skippable_prop)(ule_u8),
80+ void (*skip_shift_callback)(ule_u8, void *),
81+ struct proper *);
82+int proper_advance(struct proper *);
83+
84+void
85+herodotus_reader_init(HERODOTUS_READER *r, enum herodotus_type type,
86+ const void *src, size_t srclen)
87+{
88+ size_t i;
89+
90+ r->type = type;
91+ r->src = src;
92+ r->srclen = srclen;
93+ r->off = 0;
94+ r->terminated_by_null = 0;
95+
96+ for (i = 0; i < LEN(r->soft_limit); i++) {
97+ r->soft_limit[i] = SIZE_MAX;
98+ }
99+}
100+
101+void
102+herodotus_reader_copy(const HERODOTUS_READER *src, HERODOTUS_READER *dest)
103+{
104+ size_t i;
105+
106+ /*
107+ * we copy such that we have a "fresh" start and build on the
108+ * fact that src->soft_limit[i] for any i and src->srclen are
109+ * always larger or equal to src->off
110+ */
111+ dest->type = src->type;
112+ if (src->type == HERODOTUS_TYPE_CODEPOINT) {
113+ dest->src =
114+ (src->src == NULL) ?
115+ NULL :
116+ ((const ule_u32 *)(src->src)) + src->off;
117+ } else { /* src->type == HERODOTUS_TYPE_UTF8 */
118+ dest->src = (src->src == NULL) ?
119+ NULL :
120+ ((const char *)(src->src)) + src->off;
121+ }
122+ if (src->srclen == SIZE_MAX) {
123+ dest->srclen = SIZE_MAX;
124+ } else {
125+ dest->srclen =
126+ (src->off < src->srclen) ? src->srclen - src->off : 0;
127+ }
128+ dest->off = 0;
129+ dest->terminated_by_null = src->terminated_by_null;
130+
131+ for (i = 0; i < LEN(src->soft_limit); i++) {
132+ if (src->soft_limit[i] == SIZE_MAX) {
133+ dest->soft_limit[i] = SIZE_MAX;
134+ } else {
135+ /*
136+ * if we have a degenerate case where the offset is
137+ * higher than the soft-limit, we simply clamp the
138+ * soft-limit to zero given we can't decide here
139+ * to release the limit and, instead, we just
140+ * prevent any more reads
141+ */
142+ dest->soft_limit[i] =
143+ (src->off < src->soft_limit[i]) ?
144+ src->soft_limit[i] - src->off :
145+ 0;
146+ }
147+ }
148+}
149+
150+void
151+herodotus_reader_push_advance_limit(HERODOTUS_READER *r, size_t count)
152+{
153+ size_t i;
154+
155+ for (i = LEN(r->soft_limit) - 1; i >= 1; i--) {
156+ r->soft_limit[i] = r->soft_limit[i - 1];
157+ }
158+ r->soft_limit[0] = r->off + count;
159+}
160+
161+void
162+herodotus_reader_pop_limit(HERODOTUS_READER *r)
163+{
164+ size_t i;
165+
166+ for (i = 0; i < LEN(r->soft_limit) - 1; i++) {
167+ r->soft_limit[i] = r->soft_limit[i + 1];
168+ }
169+
170+ r->soft_limit[LEN(r->soft_limit) - 1] = SIZE_MAX;/* XXX: uletype SIZE_MAX */
171+}
172+
173+size_t
174+herodotus_reader_next_word_break(const HERODOTUS_READER *r)
175+{
176+ if (r->type == HERODOTUS_TYPE_CODEPOINT) {
177+ return grapheme_next_word_break(
178+ (const ule_u32 *)(r->src) + r->off,
179+ MIN(r->srclen, r->soft_limit[0]) - r->off);
180+ } else { /* r->type == HERODOTUS_TYPE_UTF8 */
181+ return grapheme_next_word_break_utf8(
182+ (const char *)(r->src) + r->off,
183+ MIN(r->srclen, r->soft_limit[0]) - r->off);
184+ }
185+}
186+
187+size_t
188+herodotus_reader_next_codepoint_break(const HERODOTUS_READER *r)
189+{
190+ if (r->type == HERODOTUS_TYPE_CODEPOINT) {
191+ return (r->off < MIN(r->srclen, r->soft_limit[0])) ? 1 : 0;
192+ } else { /* r->type == HERODOTUS_TYPE_UTF8 */
193+ return grapheme_decode_utf8(
194+ (const char *)(r->src) + r->off,
195+ MIN(r->srclen, r->soft_limit[0]) - r->off, NULL);
196+ }
197+}
198+
199+size_t
200+herodotus_reader_number_read(const HERODOTUS_READER *r)
201+{
202+ return r->off;
203+}
204+
205+enum herodotus_status
206+herodotus_read_codepoint(HERODOTUS_READER *r, ule_i8 advance, ule_u32 *cp)
207+{
208+ size_t ret;
209+
210+ if (r->terminated_by_null || r->off >= r->srclen || r->src == NULL) {
211+ *cp = GRAPHEME_INVALID_CODEPOINT;
212+ return HERODOTUS_STATUS_END_OF_BUFFER;
213+ }
214+
215+ if (r->off >= r->soft_limit[0]) {
216+ *cp = GRAPHEME_INVALID_CODEPOINT;
217+ return HERODOTUS_STATUS_SOFT_LIMIT_REACHED;
218+ }
219+
220+ if (r->type == HERODOTUS_TYPE_CODEPOINT) {
221+ *cp = ((const ule_u32 *)(r->src))[r->off];
222+ ret = 1;
223+ } else { /* r->type == HERODOTUS_TYPE_UTF8 */
224+ ret = grapheme_decode_utf8(
225+ (const char *)r->src + r->off,
226+ MIN(r->srclen, r->soft_limit[0]) - r->off, cp);
227+ }
228+
229+ if (unlikely(r->srclen == SIZE_MAX && *cp == 0)) {
230+ /*
231+ * We encountered a null-codepoint. Don't increment
232+ * offset and return as if the buffer had ended here all
233+ * along
234+ */
235+ r->terminated_by_null = 1;
236+ return HERODOTUS_STATUS_END_OF_BUFFER;
237+ }
238+
239+ if (r->off + ret > MIN(r->srclen, r->soft_limit[0])) {
240+ /*
241+ * we want more than we have; instead of returning
242+ * garbage we terminate here.
243+ */
244+ return HERODOTUS_STATUS_END_OF_BUFFER;
245+ }
246+
247+ /*
248+ * Increase offset which we now know won't surpass the limits,
249+ * unless we got told otherwise
250+ */
251+ if (advance) {
252+ r->off += ret;
253+ }
254+
255+ return HERODOTUS_STATUS_SUCCESS;
256+}
257+
258+void
259+herodotus_writer_init(HERODOTUS_WRITER *w, enum herodotus_type type, void *dest,
260+ size_t destlen)
261+{
262+ w->type = type;
263+ w->dest = dest;
264+ w->destlen = destlen;
265+ w->off = 0;
266+ w->first_unwritable_offset = SIZE_MAX;
267+}
268+
269+void
270+herodotus_writer_nul_terminate(HERODOTUS_WRITER *w)
271+{
272+ if (w->dest == NULL) {
273+ return;
274+ }
275+
276+ if (w->off < w->destlen) {
277+ /* We still have space in the buffer. Simply use it */
278+ if (w->type == HERODOTUS_TYPE_CODEPOINT) {
279+ ((ule_u32 *)(w->dest))[w->off] = 0;
280+ } else { /* w->type == HERODOTUS_TYPE_UTF8 */
281+ ((char *)(w->dest))[w->off] = '\0';
282+ }
283+ } else if (w->first_unwritable_offset < w->destlen) {
284+ /*
285+ * There is no more space in the buffer. However,
286+ * we have noted down the first offset we couldn't
287+ * use to write into the buffer and it's smaller than
288+ * destlen. Thus we bailed writing into the
289+ * destination when a multibyte-codepoint couldn't be
290+ * written. So the last "real" byte might be at
291+ * destlen-4, destlen-3, destlen-2 or destlen-1
292+ * (the last case meaning truncation).
293+ */
294+ if (w->type == HERODOTUS_TYPE_CODEPOINT) {
295+ ((ule_u32
296+ *)(w->dest))[w->first_unwritable_offset] = 0;
297+ } else { /* w->type == HERODOTUS_TYPE_UTF8 */
298+ ((char *)(w->dest))[w->first_unwritable_offset] = '\0';
299+ }
300+ } else if (w->destlen > 0) {
301+ /*
302+ * In this case, there is no more space in the buffer and
303+ * the last unwritable offset is larger than
304+ * or equal to the destination buffer length. This means
305+ * that we are forced to simply write into the last
306+ * byte.
307+ */
308+ if (w->type == HERODOTUS_TYPE_CODEPOINT) {
309+ ((ule_u32 *)(w->dest))[w->destlen - 1] = 0;
310+ } else { /* w->type == HERODOTUS_TYPE_UTF8 */
311+ ((char *)(w->dest))[w->destlen - 1] = '\0';
312+ }
313+ }
314+
315+ /* w->off is not incremented in any case */
316+}
317+
318+size_t
319+herodotus_writer_number_written(const HERODOTUS_WRITER *w)
320+{
321+ return w->off;
322+}
323+
324+void
325+herodotus_write_codepoint(HERODOTUS_WRITER *w, ule_u32 cp)
326+{
327+ size_t ret;
328+
329+ /*
330+ * This function will always faithfully say how many codepoints
331+ * were written, even if the buffer ends. This is used to enable
332+ * truncation detection.
333+ */
334+ if (w->type == HERODOTUS_TYPE_CODEPOINT) {
335+ if (w->dest != NULL && w->off < w->destlen) {
336+ ((ule_u32 *)(w->dest))[w->off] = cp;
337+ }
338+
339+ w->off += 1;
340+ } else { /* w->type == HERODOTUS_TYPE_UTF8 */
341+ /*
342+ * First determine how many bytes we need to encode the
343+ * codepoint
344+ */
345+ ret = grapheme_encode_utf8(cp, NULL, 0);
346+
347+ if (w->dest != NULL && w->off + ret < w->destlen) {
348+ /* we still have enough room in the buffer */
349+ grapheme_encode_utf8(cp, (char *)(w->dest) + w->off,
350+ w->destlen - w->off);
351+ } else if (w->first_unwritable_offset == SIZE_MAX) {
352+ /*
353+ * the first unwritable offset has not been
354+ * noted down, so this is the first time we can't
355+ * write (completely) to an offset
356+ */
357+ w->first_unwritable_offset = w->off;
358+ }
359+
360+ w->off += ret;
361+ }
362+}
363+
364+void
365+proper_init(const HERODOTUS_READER *r, void *state, ule_u8 no_prop,
366+ ule_u8 (*get_break_prop)(ule_u32),
367+ ule_i8 (*is_skippable_prop)(ule_u8),
368+ void (*skip_shift_callback)(ule_u8, void *),
369+ struct proper *p)
370+{
371+ ule_u8 prop;
372+ ule_u32 cp;
373+ size_t i;
374+
375+ /* set internal variables */
376+ p->state = state;
377+ p->no_prop = no_prop;
378+ p->get_break_prop = get_break_prop;
379+ p->is_skippable_prop = is_skippable_prop;
380+ p->skip_shift_callback = skip_shift_callback;
381+
382+ /*
383+ * Initialize mid-reader, which is basically just there
384+ * to reflect the current position of the viewing-line
385+ */
386+ herodotus_reader_copy(r, &(p->mid_reader));
387+
388+ /*
389+ * In the initialization, we simply (try to) fill in next_prop.
390+ * If we cannot read in more (due to the buffer ending), we
391+ * fill in the prop as invalid
392+ */
393+
394+ /*
395+ * initialize the previous properties to have no property
396+ * (given we are at the start of the buffer)
397+ */
398+ p->raw.prev_prop[1] = p->raw.prev_prop[0] = p->no_prop;
399+ p->skip.prev_prop[1] = p->skip.prev_prop[0] = p->no_prop;
400+
401+ /*
402+ * initialize the next properties
403+ */
404+
405+ /* initialize the raw reader */
406+ herodotus_reader_copy(r, &(p->raw_reader));
407+
408+ /* fill in the two next raw properties (after no-initialization) */
409+ p->raw.next_prop[0] = p->raw.next_prop[1] = p->no_prop;
410+ for (i = 0;
411+ i < 2 && herodotus_read_codepoint(&(p->raw_reader), 1, &cp) ==
412+ HERODOTUS_STATUS_SUCCESS;) {
413+ p->raw.next_prop[i++] = p->get_break_prop(cp);
414+ }
415+
416+ /* initialize the skip reader */
417+ herodotus_reader_copy(r, &(p->skip_reader));
418+
419+ /* fill in the two next skip properties (after no-initialization) */
420+ p->skip.next_prop[0] = p->skip.next_prop[1] = p->no_prop;
421+ for (i = 0;
422+ i < 2 && herodotus_read_codepoint(&(p->skip_reader), 1, &cp) ==
423+ HERODOTUS_STATUS_SUCCESS;) {
424+ prop = p->get_break_prop(cp);
425+ if (!p->is_skippable_prop(prop)) {
426+ p->skip.next_prop[i++] = prop;
427+ }
428+ }
429+}
430+
431+int
432+proper_advance(struct proper *p)
433+{
434+ ule_u8 prop;
435+ ule_u32 cp;
436+
437+ /* read in next "raw" property */
438+ if (herodotus_read_codepoint(&(p->raw_reader), 1, &cp) ==
439+ HERODOTUS_STATUS_SUCCESS) {
440+ prop = p->get_break_prop(cp);
441+ } else {
442+ prop = p->no_prop;
443+ }
444+
445+ /*
446+ * do a shift-in, unless we find that the property that is to
447+ * be moved past the "raw-viewing-line" (this property is stored
448+ * in p->raw.next_prop[0]) is a no_prop, indicating that
449+ * we are at the end of the buffer.
450+ */
451+ if (p->raw.next_prop[0] == p->no_prop) {
452+ return 1;
453+ }
454+
455+ /* shift in the properties */
456+ p->raw.prev_prop[1] = p->raw.prev_prop[0];
457+ p->raw.prev_prop[0] = p->raw.next_prop[0];
458+ p->raw.next_prop[0] = p->raw.next_prop[1];
459+ p->raw.next_prop[1] = prop;
460+
461+ /* advance the middle reader viewing-line */
462+ (void)herodotus_read_codepoint(&(p->mid_reader), 1, &cp);
463+
464+ /* check skippability-property */
465+ if (!p->is_skippable_prop(p->raw.prev_prop[0])) {
466+ /*
467+ * the property that has moved past the "raw-viewing-line"
468+ * (this property is now (after the raw-shift) stored in
469+ * p->raw.prev_prop[0] and guaranteed not to be a no-prop,
470+ * guaranteeing that we won't shift a no-prop past the
471+ * "viewing-line" in the skip-properties) is not a skippable
472+ * property, thus we need to shift the skip property as well.
473+ */
474+ p->skip.prev_prop[1] = p->skip.prev_prop[0];
475+ p->skip.prev_prop[0] = p->skip.next_prop[0];
476+ p->skip.next_prop[0] = p->skip.next_prop[1];
477+
478+ /*
479+ * call the skip-shift-callback on the property that
480+ * passed the skip-viewing-line (this property is now
481+ * stored in p->skip.prev_prop[0]).
482+ */
483+ p->skip_shift_callback(p->skip.prev_prop[0], p->state);
484+
485+ /* determine the next shift property */
486+ p->skip.next_prop[1] = p->no_prop;
487+ while (herodotus_read_codepoint(&(p->skip_reader), 1, &cp) ==
488+ HERODOTUS_STATUS_SUCCESS) {
489+ prop = p->get_break_prop(cp);
490+ if (!p->is_skippable_prop(prop)) {
491+ p->skip.next_prop[1] = prop;
492+ break;
493+ }
494+ }
495+ }
496+
497+ return 0;
498+}
499+