commit 24fe1a2
uint
·
2026-10-18 21:24:01 +0000 UTC
parent 3980732
UTF: Use utf8.h as a reference instead
2 files changed,
+192,
-545
+192,
-50
1@@ -1,5 +1,4 @@
2 /* Copyright 2026 uint (and contributors)
3- Copyright 2019-2025 Laslo Hunhold <dev@frign.de>
4
5 Permission to use, copy, modify, and distribute this software for any
6 purpose with or without fee is hereby granted, provided that the above
7@@ -13,58 +12,201 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
8 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
9 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
10
11-/* uleutf is based on Laslo Hunhold's libgrapheme which is hosted on
12- suckless.org */
13+/* uleutf is based on utf8.h from */
14
15 #ifndef ULEUTF_H
16 #define ULEUTF_H
17
18-#include <stddef.h>
19-
20-#include "uletypes.h"
21-
22-#define GRAPHEME_INVALID_CODEPOINT UINT32_C(0xFFFD)
23-
24-enum uleutf_bidirectional_direction {
25- ULEUTF_BIDIRECTIONAL_DIRECTION_NEUTRAL,
26- ULEUTF_BIDIRECTIONAL_DIRECTION_LTR,
27- ULEUTF_BIDIRECTIONAL_DIRECTION_RTL
28-};
29-
30-size_t uleutf_bidirectional_get_line_embedding_levels(const ule_u32*, size_t, ule_i8*, size_t);
31-size_t uleutf_bidirectional_preprocess_paragraph(const ule_u32*, size_t, enum uleutf_bidirectional_direction, ule_u32*, size_t, enum uleutf_bidirectional_direction*);
32-size_t uleutf_bidirectional_reorder_line(const ule_u32 *, const ule_u32 *, size_t, ule_u32 *, size_t);
33-
34-size_t uleutf_decode_utf8(const char *, size_t, ule_u32 *);
35-size_t uleutf_encode_utf8(ule_u32, char *, size_t);
36-
37-ule_i8 uleutf_is_character_break(ule_u32, ule_u32, ule_u16 *);
38-
39-ule_i8 uleutf_is_lowercase(const ule_u32 *, size_t, size_t *);
40-ule_i8 uleutf_is_titlecase(const ule_u32 *, size_t, size_t *);
41-ule_i8 uleutf_is_uppercase(const ule_u32 *, size_t, size_t *);
42-
43-ule_i8 uleutf_is_lowercase_utf8(const char *, size_t, size_t *);
44-ule_i8 uleutf_is_titlecase_utf8(const char *, size_t, size_t *);
45-ule_i8 uleutf_is_uppercase_utf8(const char *, size_t, size_t *);
46-
47-size_t uleutf_next_character_break(const ule_u32 *, size_t);
48-size_t uleutf_next_line_break(const ule_u32 *, size_t);
49-size_t uleutf_next_sentence_break(const ule_u32 *, size_t);
50-size_t uleutf_next_word_break(const ule_u32 *, size_t);
51-
52-size_t uleutf_next_character_break_utf8(const char *, size_t);
53-size_t uleutf_next_line_break_utf8(const char *, size_t);
54-size_t uleutf_next_sentence_break_utf8(const char *, size_t);
55-size_t uleutf_next_word_break_utf8(const char *, size_t);
56-
57-size_t uleutf_to_lowercase(const ule_u32 *, size_t, ule_u32 *, size_t);
58-size_t uleutf_to_titlecase(const ule_u32 *, size_t, ule_u32 *, size_t);
59-size_t uleutf_to_uppercase(const ule_u32 *, size_t, ule_u32 *, size_t);
60-
61-size_t uleutf_to_lowercase_utf8(const char *, size_t, char *, size_t);
62-size_t uleutf_to_titlecase_utf8(const char *, size_t, char *, size_t);
63-size_t uleutf_to_uppercase_utf8(const char *, size_t, char *, size_t);
64+/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 >
65+ * src2 respectively, case insensitive. */
66+utf8_constexpr14 utf8_nonnull utf8_pure int
67+utf8casecmp(const utf8_int8_t *src1, const utf8_int8_t *src2);
68+
69+/* Append the utf8 string src onto the utf8 string dst. */
70+utf8_nonnull utf8_weak utf8_int8_t *
71+utf8cat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src);
72+
73+/* Find the first match of the utf8 codepoint chr in the utf8 string src. */
74+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
75+utf8chr(const utf8_int8_t *src, utf8_int32_t chr);
76+
77+/* Return less than 0, 0, greater than 0 if src1 < src2,
78+ * src1 == src2, src1 > src2 respectively. */
79+utf8_constexpr14 utf8_nonnull utf8_pure int utf8cmp(const utf8_int8_t *src1,
80+ const utf8_int8_t *src2);
81+
82+/* Copy the utf8 string src onto the memory allocated in dst. */
83+utf8_nonnull utf8_weak utf8_int8_t *
84+utf8cpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src);
85+
86+/* Number of utf8 codepoints in the utf8 string src that consists entirely
87+ * of utf8 codepoints not from the utf8 string reject. */
88+utf8_constexpr14 utf8_nonnull utf8_pure size_t
89+utf8cspn(const utf8_int8_t *src, const utf8_int8_t *reject);
90+
91+/* Duplicate the utf8 string src by getting its size, malloc'ing a new buffer
92+ * copying over the data, and returning that. Or 0 if malloc failed. */
93+utf8_weak utf8_int8_t *utf8dup(const utf8_int8_t *src);
94+
95+/* Number of utf8 codepoints in the utf8 string str,
96+ * excluding the null terminating byte. */
97+utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8len(const utf8_int8_t *str);
98+
99+/* Similar to utf8len, except that only at most n bytes of src are looked. */
100+utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8nlen(const utf8_int8_t *str,
101+ size_t n);
102+
103+/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 >
104+ * src2 respectively, case insensitive. Checking at most n bytes of each utf8
105+ * string. */
106+utf8_constexpr14 utf8_nonnull utf8_pure int
107+utf8ncasecmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n);
108+
109+/* Append the utf8 string src onto the utf8 string dst,
110+ * writing at most n+1 bytes. Can produce an invalid utf8
111+ * string if n falls partway through a utf8 codepoint. */
112+utf8_nonnull utf8_weak utf8_int8_t *
113+utf8ncat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src,
114+ size_t n);
115+
116+/* Return less than 0, 0, greater than 0 if src1 < src2,
117+ * src1 == src2, src1 > src2 respectively. Checking at most n
118+ * bytes of each utf8 string. */
119+utf8_constexpr14 utf8_nonnull utf8_pure int
120+utf8ncmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n);
121+
122+/* Copy the utf8 string src onto the memory allocated in dst.
123+ * Copies at most n bytes. If n falls partway through a utf8
124+ * codepoint, or if dst doesn't have enough room for a null
125+ * terminator, the final string will be cut short to preserve
126+ * utf8 validity. */
127+
128+utf8_nonnull utf8_weak utf8_int8_t *
129+utf8ncpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src,
130+ size_t n);
131+
132+/* Similar to utf8dup, except that at most n bytes of src are copied. If src is
133+ * longer than n, only n bytes are copied and a null byte is added.
134+ *
135+ * Returns a new string if successful, 0 otherwise */
136+utf8_weak utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n);
137+
138+/* Locates the first occurrence in the utf8 string str of any byte in the
139+ * utf8 string accept, or 0 if no match was found. */
140+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
141+utf8pbrk(const utf8_int8_t *str, const utf8_int8_t *accept);
142+
143+/* Find the last match of the utf8 codepoint chr in the utf8 string src. */
144+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
145+utf8rchr(const utf8_int8_t *src, utf8_int32_t chr);
146+
147+/* Number of bytes in the utf8 string str,
148+ * including the null terminating byte. */
149+utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8size(const utf8_int8_t *str);
150+
151+/* Similar to utf8size, except that the null terminating byte is excluded. */
152+utf8_constexpr14 utf8_nonnull utf8_pure size_t
153+utf8size_lazy(const utf8_int8_t *str);
154+
155+/* Similar to utf8size, except that only at most n bytes of src are looked and
156+ * the null terminating byte is excluded. */
157+utf8_constexpr14 utf8_nonnull utf8_pure size_t
158+utf8nsize_lazy(const utf8_int8_t *str, size_t n);
159+
160+/* Number of utf8 codepoints in the utf8 string src that consists entirely
161+ * of utf8 codepoints from the utf8 string accept. */
162+utf8_constexpr14 utf8_nonnull utf8_pure size_t
163+utf8spn(const utf8_int8_t *src, const utf8_int8_t *accept);
164+
165+/* The position of the utf8 string needle in the utf8 string haystack. */
166+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
167+utf8str(const utf8_int8_t *haystack, const utf8_int8_t *needle);
168+
169+/* The position of the utf8 string needle in the utf8 string haystack, case
170+ * insensitive. */
171+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
172+utf8casestr(const utf8_int8_t *haystack, const utf8_int8_t *needle);
173+
174+/* Return 0 on success, or the position of the invalid
175+ * utf8 codepoint on failure. */
176+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
177+utf8valid(const utf8_int8_t *str);
178+
179+/* Similar to utf8valid, except that only at most n bytes of src are looked. */
180+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
181+utf8nvalid(const utf8_int8_t *str, size_t n);
182+
183+/* Given a null-terminated string, makes the string valid by replacing invalid
184+ * codepoints with a 1-byte replacement. Returns 0 on success. */
185+utf8_nonnull utf8_weak int utf8makevalid(utf8_int8_t *str,
186+ const utf8_int32_t replacement);
187+
188+/* Sets out_codepoint to the current utf8 codepoint in str, and returns the
189+ * address of the next utf8 codepoint after the current one in str. */
190+utf8_constexpr14 utf8_nonnull utf8_int8_t *
191+utf8codepoint(const utf8_int8_t *utf8_restrict str,
192+ utf8_int32_t *utf8_restrict out_codepoint);
193+
194+/* Calculates the size of the next utf8 codepoint in str. */
195+utf8_constexpr14 utf8_nonnull utf8_pure size_t
196+utf8codepointcalcsize(const utf8_int8_t *str);
197+
198+/* Returns the size of the given codepoint in bytes. */
199+utf8_constexpr14 utf8_pure size_t utf8codepointsize(utf8_int32_t chr);
200+
201+/* Returns the number of terminal columns the given codepoint occupies when
202+ * printed with a fixed-width font: 0 for zero-width codepoints (the null
203+ * codepoint and combining marks), 2 for wide (double-width) codepoints, 1 for
204+ * ordinary codepoints, and -1 for non-printable control codepoints. */
205+utf8_constexpr14 utf8_pure int utf8codepointvisualwidth(utf8_int32_t chr);
206+
207+/* Write a codepoint to the given string, and return the address to the next
208+ * place after the written codepoint. Pass how many bytes left in the buffer to
209+ * n. If there is not enough space for the codepoint, this function returns
210+ * null. */
211+utf8_nonnull utf8_weak utf8_int8_t *
212+utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n);
213+
214+/* Returns 1 if the given character is lowercase, or 0 if it is not. */
215+utf8_constexpr14 utf8_pure int utf8islower(utf8_int32_t chr);
216+
217+/* Returns 1 if the given character is uppercase, or 0 if it is not. */
218+utf8_constexpr14 utf8_pure int utf8isupper(utf8_int32_t chr);
219+
220+/* Transform the given string into all lowercase codepoints in place. This
221+ * function does not allocate memory. All currently supported case mappings
222+ * preserve the number of bytes in the encoded string. */
223+utf8_nonnull utf8_weak void utf8lwr(utf8_int8_t *utf8_restrict str);
224+
225+/* Transform the given string into all uppercase codepoints in place. This
226+ * function does not allocate memory. All currently supported case mappings
227+ * preserve the number of bytes in the encoded string. */
228+utf8_nonnull utf8_weak void utf8upr(utf8_int8_t *utf8_restrict str);
229+
230+/* Make a codepoint lower case if possible. */
231+utf8_constexpr14 utf8_pure utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp);
232+
233+/* Make a codepoint upper case if possible. */
234+utf8_constexpr14 utf8_pure utf8_int32_t utf8uprcodepoint(utf8_int32_t cp);
235+
236+/* Sets out_codepoint to the current utf8 codepoint in str, and returns the
237+ * address of the previous utf8 codepoint before the current one in str. */
238+utf8_constexpr14 utf8_nonnull utf8_int8_t *
239+utf8rcodepoint(const utf8_int8_t *utf8_restrict str,
240+ utf8_int32_t *utf8_restrict out_codepoint);
241+
242+/* Duplicate the utf8 string src by getting its size, calling alloc_func_ptr to
243+ * copy over data to a new buffer, and returning that. Or 0 if alloc_func_ptr
244+ * returned null. */
245+utf8_weak utf8_int8_t *utf8dup_ex(const utf8_int8_t *src,
246+ utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *,
247+ size_t),
248+ utf8_int8_t *user_data);
249+
250+/* Similar to utf8dup, except that at most n bytes of src are copied. If src is
251+ * longer than n, only n bytes are copied and a null byte is added.
252+ *
253+ * Returns a new string if successful, 0 otherwise. */
254+utf8_weak utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src, size_t n, utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t), utf8_int8_t *user_data);
255
256 #endif /* ULEUTF_H */
257-
+0,
-495
1@@ -1,498 +1,3 @@
2 #include <stddef.h>
3
4 #include "uleutf.h"
5-
6-#define SIZE_MAX (0xFFFFFFFFF)
7-
8-enum herodotus_status {
9- HERODOTUS_STATUS_SUCCESS,
10- HERODOTUS_STATUS_END_OF_BUFFER,
11- HERODOTUS_STATUS_SOFT_LIMIT_REACHED,
12-};
13-
14-enum herodotus_type {
15- HERODOTUS_TYPE_CODEPOINT,
16- HERODOTUS_TYPE_UTF8,
17-};
18-
19-typedef struct herodotus_reader {
20- enum herodotus_type type;
21- const void *src;
22- size_t srclen;
23- size_t off;
24- ule_i8 terminated_by_null;
25- size_t soft_limit[10];
26-} HERODOTUS_READER;
27-
28-typedef struct herodotus_writer {
29- enum herodotus_type type;
30- void *dest;
31- size_t destlen;
32- size_t off;
33- size_t first_unwritable_offset;
34-} HERODOTUS_WRITER;
35-
36-struct proper {
37- /*
38- * prev_prop[1] prev_prop[0] | next_prop[0] next_prop[1]
39- */
40- struct {
41- ule_u8 prev_prop[2];
42- ule_u8 next_prop[2];
43- } raw;
44-
45- struct {
46- ule_u8 prev_prop[2];
47- ule_u8 next_prop[2];
48- } skip;
49-
50- HERODOTUS_READER mid_reader;
51- HERODOTUS_READER raw_reader;
52- HERODOTUS_READER skip_reader;
53-
54- void *state;
55- ule_u8 no_prop;
56- ule_u8 (*get_break_prop)(ule_u32);
57- ule_i8 (*is_skippable_prop)(ule_u8);
58- void (*skip_shift_callback)(ule_u8, void *);
59-};
60-
61-void herodotus_reader_init(HERODOTUS_READER *, enum herodotus_type,
62- const void *, size_t);
63-void herodotus_reader_copy(const HERODOTUS_READER *, HERODOTUS_READER *);
64-void herodotus_reader_push_advance_limit(HERODOTUS_READER *, size_t);
65-void herodotus_reader_pop_limit(HERODOTUS_READER *);
66-size_t herodotus_reader_number_read(const HERODOTUS_READER *);
67-size_t herodotus_reader_next_word_break(const HERODOTUS_READER *);
68-size_t herodotus_reader_next_codepoint_break(const HERODOTUS_READER *);
69-enum herodotus_status herodotus_read_codepoint(HERODOTUS_READER *, ule_i8, ule_u32 *);
70-
71-void herodotus_writer_init(HERODOTUS_WRITER *, enum herodotus_type, void *,
72- size_t);
73-void herodotus_writer_nul_terminate(HERODOTUS_WRITER *);
74-size_t herodotus_writer_number_written(const HERODOTUS_WRITER *);
75-void herodotus_write_codepoint(HERODOTUS_WRITER *, ule_u32);
76-
77-void proper_init(const HERODOTUS_READER *, void *, ule_u8,
78- ule_u8 (*get_break_prop)(ule_u32),
79- ule_i8 (*is_skippable_prop)(ule_u8),
80- void (*skip_shift_callback)(ule_u8, void *),
81- struct proper *);
82-int proper_advance(struct proper *);
83-
84-void
85-herodotus_reader_init(HERODOTUS_READER *r, enum herodotus_type type,
86- const void *src, size_t srclen)
87-{
88- size_t i;
89-
90- r->type = type;
91- r->src = src;
92- r->srclen = srclen;
93- r->off = 0;
94- r->terminated_by_null = 0;
95-
96- for (i = 0; i < LEN(r->soft_limit); i++) {
97- r->soft_limit[i] = SIZE_MAX;
98- }
99-}
100-
101-void
102-herodotus_reader_copy(const HERODOTUS_READER *src, HERODOTUS_READER *dest)
103-{
104- size_t i;
105-
106- /*
107- * we copy such that we have a "fresh" start and build on the
108- * fact that src->soft_limit[i] for any i and src->srclen are
109- * always larger or equal to src->off
110- */
111- dest->type = src->type;
112- if (src->type == HERODOTUS_TYPE_CODEPOINT) {
113- dest->src =
114- (src->src == NULL) ?
115- NULL :
116- ((const ule_u32 *)(src->src)) + src->off;
117- } else { /* src->type == HERODOTUS_TYPE_UTF8 */
118- dest->src = (src->src == NULL) ?
119- NULL :
120- ((const char *)(src->src)) + src->off;
121- }
122- if (src->srclen == SIZE_MAX) {
123- dest->srclen = SIZE_MAX;
124- } else {
125- dest->srclen =
126- (src->off < src->srclen) ? src->srclen - src->off : 0;
127- }
128- dest->off = 0;
129- dest->terminated_by_null = src->terminated_by_null;
130-
131- for (i = 0; i < LEN(src->soft_limit); i++) {
132- if (src->soft_limit[i] == SIZE_MAX) {
133- dest->soft_limit[i] = SIZE_MAX;
134- } else {
135- /*
136- * if we have a degenerate case where the offset is
137- * higher than the soft-limit, we simply clamp the
138- * soft-limit to zero given we can't decide here
139- * to release the limit and, instead, we just
140- * prevent any more reads
141- */
142- dest->soft_limit[i] =
143- (src->off < src->soft_limit[i]) ?
144- src->soft_limit[i] - src->off :
145- 0;
146- }
147- }
148-}
149-
150-void
151-herodotus_reader_push_advance_limit(HERODOTUS_READER *r, size_t count)
152-{
153- size_t i;
154-
155- for (i = LEN(r->soft_limit) - 1; i >= 1; i--) {
156- r->soft_limit[i] = r->soft_limit[i - 1];
157- }
158- r->soft_limit[0] = r->off + count;
159-}
160-
161-void
162-herodotus_reader_pop_limit(HERODOTUS_READER *r)
163-{
164- size_t i;
165-
166- for (i = 0; i < LEN(r->soft_limit) - 1; i++) {
167- r->soft_limit[i] = r->soft_limit[i + 1];
168- }
169-
170- r->soft_limit[LEN(r->soft_limit) - 1] = SIZE_MAX;/* XXX: uletype SIZE_MAX */
171-}
172-
173-size_t
174-herodotus_reader_next_word_break(const HERODOTUS_READER *r)
175-{
176- if (r->type == HERODOTUS_TYPE_CODEPOINT) {
177- return grapheme_next_word_break(
178- (const ule_u32 *)(r->src) + r->off,
179- MIN(r->srclen, r->soft_limit[0]) - r->off);
180- } else { /* r->type == HERODOTUS_TYPE_UTF8 */
181- return grapheme_next_word_break_utf8(
182- (const char *)(r->src) + r->off,
183- MIN(r->srclen, r->soft_limit[0]) - r->off);
184- }
185-}
186-
187-size_t
188-herodotus_reader_next_codepoint_break(const HERODOTUS_READER *r)
189-{
190- if (r->type == HERODOTUS_TYPE_CODEPOINT) {
191- return (r->off < MIN(r->srclen, r->soft_limit[0])) ? 1 : 0;
192- } else { /* r->type == HERODOTUS_TYPE_UTF8 */
193- return grapheme_decode_utf8(
194- (const char *)(r->src) + r->off,
195- MIN(r->srclen, r->soft_limit[0]) - r->off, NULL);
196- }
197-}
198-
199-size_t
200-herodotus_reader_number_read(const HERODOTUS_READER *r)
201-{
202- return r->off;
203-}
204-
205-enum herodotus_status
206-herodotus_read_codepoint(HERODOTUS_READER *r, ule_i8 advance, ule_u32 *cp)
207-{
208- size_t ret;
209-
210- if (r->terminated_by_null || r->off >= r->srclen || r->src == NULL) {
211- *cp = GRAPHEME_INVALID_CODEPOINT;
212- return HERODOTUS_STATUS_END_OF_BUFFER;
213- }
214-
215- if (r->off >= r->soft_limit[0]) {
216- *cp = GRAPHEME_INVALID_CODEPOINT;
217- return HERODOTUS_STATUS_SOFT_LIMIT_REACHED;
218- }
219-
220- if (r->type == HERODOTUS_TYPE_CODEPOINT) {
221- *cp = ((const ule_u32 *)(r->src))[r->off];
222- ret = 1;
223- } else { /* r->type == HERODOTUS_TYPE_UTF8 */
224- ret = grapheme_decode_utf8(
225- (const char *)r->src + r->off,
226- MIN(r->srclen, r->soft_limit[0]) - r->off, cp);
227- }
228-
229- if (unlikely(r->srclen == SIZE_MAX && *cp == 0)) {
230- /*
231- * We encountered a null-codepoint. Don't increment
232- * offset and return as if the buffer had ended here all
233- * along
234- */
235- r->terminated_by_null = 1;
236- return HERODOTUS_STATUS_END_OF_BUFFER;
237- }
238-
239- if (r->off + ret > MIN(r->srclen, r->soft_limit[0])) {
240- /*
241- * we want more than we have; instead of returning
242- * garbage we terminate here.
243- */
244- return HERODOTUS_STATUS_END_OF_BUFFER;
245- }
246-
247- /*
248- * Increase offset which we now know won't surpass the limits,
249- * unless we got told otherwise
250- */
251- if (advance) {
252- r->off += ret;
253- }
254-
255- return HERODOTUS_STATUS_SUCCESS;
256-}
257-
258-void
259-herodotus_writer_init(HERODOTUS_WRITER *w, enum herodotus_type type, void *dest,
260- size_t destlen)
261-{
262- w->type = type;
263- w->dest = dest;
264- w->destlen = destlen;
265- w->off = 0;
266- w->first_unwritable_offset = SIZE_MAX;
267-}
268-
269-void
270-herodotus_writer_nul_terminate(HERODOTUS_WRITER *w)
271-{
272- if (w->dest == NULL) {
273- return;
274- }
275-
276- if (w->off < w->destlen) {
277- /* We still have space in the buffer. Simply use it */
278- if (w->type == HERODOTUS_TYPE_CODEPOINT) {
279- ((ule_u32 *)(w->dest))[w->off] = 0;
280- } else { /* w->type == HERODOTUS_TYPE_UTF8 */
281- ((char *)(w->dest))[w->off] = '\0';
282- }
283- } else if (w->first_unwritable_offset < w->destlen) {
284- /*
285- * There is no more space in the buffer. However,
286- * we have noted down the first offset we couldn't
287- * use to write into the buffer and it's smaller than
288- * destlen. Thus we bailed writing into the
289- * destination when a multibyte-codepoint couldn't be
290- * written. So the last "real" byte might be at
291- * destlen-4, destlen-3, destlen-2 or destlen-1
292- * (the last case meaning truncation).
293- */
294- if (w->type == HERODOTUS_TYPE_CODEPOINT) {
295- ((ule_u32
296- *)(w->dest))[w->first_unwritable_offset] = 0;
297- } else { /* w->type == HERODOTUS_TYPE_UTF8 */
298- ((char *)(w->dest))[w->first_unwritable_offset] = '\0';
299- }
300- } else if (w->destlen > 0) {
301- /*
302- * In this case, there is no more space in the buffer and
303- * the last unwritable offset is larger than
304- * or equal to the destination buffer length. This means
305- * that we are forced to simply write into the last
306- * byte.
307- */
308- if (w->type == HERODOTUS_TYPE_CODEPOINT) {
309- ((ule_u32 *)(w->dest))[w->destlen - 1] = 0;
310- } else { /* w->type == HERODOTUS_TYPE_UTF8 */
311- ((char *)(w->dest))[w->destlen - 1] = '\0';
312- }
313- }
314-
315- /* w->off is not incremented in any case */
316-}
317-
318-size_t
319-herodotus_writer_number_written(const HERODOTUS_WRITER *w)
320-{
321- return w->off;
322-}
323-
324-void
325-herodotus_write_codepoint(HERODOTUS_WRITER *w, ule_u32 cp)
326-{
327- size_t ret;
328-
329- /*
330- * This function will always faithfully say how many codepoints
331- * were written, even if the buffer ends. This is used to enable
332- * truncation detection.
333- */
334- if (w->type == HERODOTUS_TYPE_CODEPOINT) {
335- if (w->dest != NULL && w->off < w->destlen) {
336- ((ule_u32 *)(w->dest))[w->off] = cp;
337- }
338-
339- w->off += 1;
340- } else { /* w->type == HERODOTUS_TYPE_UTF8 */
341- /*
342- * First determine how many bytes we need to encode the
343- * codepoint
344- */
345- ret = grapheme_encode_utf8(cp, NULL, 0);
346-
347- if (w->dest != NULL && w->off + ret < w->destlen) {
348- /* we still have enough room in the buffer */
349- grapheme_encode_utf8(cp, (char *)(w->dest) + w->off,
350- w->destlen - w->off);
351- } else if (w->first_unwritable_offset == SIZE_MAX) {
352- /*
353- * the first unwritable offset has not been
354- * noted down, so this is the first time we can't
355- * write (completely) to an offset
356- */
357- w->first_unwritable_offset = w->off;
358- }
359-
360- w->off += ret;
361- }
362-}
363-
364-void
365-proper_init(const HERODOTUS_READER *r, void *state, ule_u8 no_prop,
366- ule_u8 (*get_break_prop)(ule_u32),
367- ule_i8 (*is_skippable_prop)(ule_u8),
368- void (*skip_shift_callback)(ule_u8, void *),
369- struct proper *p)
370-{
371- ule_u8 prop;
372- ule_u32 cp;
373- size_t i;
374-
375- /* set internal variables */
376- p->state = state;
377- p->no_prop = no_prop;
378- p->get_break_prop = get_break_prop;
379- p->is_skippable_prop = is_skippable_prop;
380- p->skip_shift_callback = skip_shift_callback;
381-
382- /*
383- * Initialize mid-reader, which is basically just there
384- * to reflect the current position of the viewing-line
385- */
386- herodotus_reader_copy(r, &(p->mid_reader));
387-
388- /*
389- * In the initialization, we simply (try to) fill in next_prop.
390- * If we cannot read in more (due to the buffer ending), we
391- * fill in the prop as invalid
392- */
393-
394- /*
395- * initialize the previous properties to have no property
396- * (given we are at the start of the buffer)
397- */
398- p->raw.prev_prop[1] = p->raw.prev_prop[0] = p->no_prop;
399- p->skip.prev_prop[1] = p->skip.prev_prop[0] = p->no_prop;
400-
401- /*
402- * initialize the next properties
403- */
404-
405- /* initialize the raw reader */
406- herodotus_reader_copy(r, &(p->raw_reader));
407-
408- /* fill in the two next raw properties (after no-initialization) */
409- p->raw.next_prop[0] = p->raw.next_prop[1] = p->no_prop;
410- for (i = 0;
411- i < 2 && herodotus_read_codepoint(&(p->raw_reader), 1, &cp) ==
412- HERODOTUS_STATUS_SUCCESS;) {
413- p->raw.next_prop[i++] = p->get_break_prop(cp);
414- }
415-
416- /* initialize the skip reader */
417- herodotus_reader_copy(r, &(p->skip_reader));
418-
419- /* fill in the two next skip properties (after no-initialization) */
420- p->skip.next_prop[0] = p->skip.next_prop[1] = p->no_prop;
421- for (i = 0;
422- i < 2 && herodotus_read_codepoint(&(p->skip_reader), 1, &cp) ==
423- HERODOTUS_STATUS_SUCCESS;) {
424- prop = p->get_break_prop(cp);
425- if (!p->is_skippable_prop(prop)) {
426- p->skip.next_prop[i++] = prop;
427- }
428- }
429-}
430-
431-int
432-proper_advance(struct proper *p)
433-{
434- ule_u8 prop;
435- ule_u32 cp;
436-
437- /* read in next "raw" property */
438- if (herodotus_read_codepoint(&(p->raw_reader), 1, &cp) ==
439- HERODOTUS_STATUS_SUCCESS) {
440- prop = p->get_break_prop(cp);
441- } else {
442- prop = p->no_prop;
443- }
444-
445- /*
446- * do a shift-in, unless we find that the property that is to
447- * be moved past the "raw-viewing-line" (this property is stored
448- * in p->raw.next_prop[0]) is a no_prop, indicating that
449- * we are at the end of the buffer.
450- */
451- if (p->raw.next_prop[0] == p->no_prop) {
452- return 1;
453- }
454-
455- /* shift in the properties */
456- p->raw.prev_prop[1] = p->raw.prev_prop[0];
457- p->raw.prev_prop[0] = p->raw.next_prop[0];
458- p->raw.next_prop[0] = p->raw.next_prop[1];
459- p->raw.next_prop[1] = prop;
460-
461- /* advance the middle reader viewing-line */
462- (void)herodotus_read_codepoint(&(p->mid_reader), 1, &cp);
463-
464- /* check skippability-property */
465- if (!p->is_skippable_prop(p->raw.prev_prop[0])) {
466- /*
467- * the property that has moved past the "raw-viewing-line"
468- * (this property is now (after the raw-shift) stored in
469- * p->raw.prev_prop[0] and guaranteed not to be a no-prop,
470- * guaranteeing that we won't shift a no-prop past the
471- * "viewing-line" in the skip-properties) is not a skippable
472- * property, thus we need to shift the skip property as well.
473- */
474- p->skip.prev_prop[1] = p->skip.prev_prop[0];
475- p->skip.prev_prop[0] = p->skip.next_prop[0];
476- p->skip.next_prop[0] = p->skip.next_prop[1];
477-
478- /*
479- * call the skip-shift-callback on the property that
480- * passed the skip-viewing-line (this property is now
481- * stored in p->skip.prev_prop[0]).
482- */
483- p->skip_shift_callback(p->skip.prev_prop[0], p->state);
484-
485- /* determine the next shift property */
486- p->skip.next_prop[1] = p->no_prop;
487- while (herodotus_read_codepoint(&(p->skip_reader), 1, &cp) ==
488- HERODOTUS_STATUS_SUCCESS) {
489- prop = p->get_break_prop(cp);
490- if (!p->is_skippable_prop(prop)) {
491- p->skip.next_prop[1] = prop;
492- break;
493- }
494- }
495- }
496-
497- return 0;
498-}
499-