2 * This file is part of DisOrder
3 * Copyright (C) 2007, 2009, 2013 Richard Kettlewell
5 * This program is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 /** @file lib/unicode.c
19 * @brief Unicode support functions
21 * Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
22 * encoding schemes). The primary encoding form is UTF-32 but convenience
23 * wrappers using UTF-8 are provided for a number of functions.
25 * The idea is that all the strings that hit the database will be in a
26 * particular normalization form, and for the search and tags database
27 * in case-folded form, so they can be naively compared within the
30 * As the code stands this guarantee is not well met!
35 * - @ref utf32iterator
47 /** @defgroup utf32props Unicode Code Point Properties */
50 static const struct unidata
*utf32__unidata_hard(uint32_t c
);
52 /** @brief Find definition of code point @p c
54 * @return Pointer to @ref unidata structure for @p c
56 * @p c can be any 32-bit value, a sensible value will be returned regardless.
57 * The returned pointer is NOT guaranteed to be unique to @p c.
59 static inline const struct unidata
*utf32__unidata(uint32_t c
) {
60 /* The bottom half of the table contains almost everything of interest
61 * and we can just return the right thing straight away */
62 if(c
< UNICODE_BREAK_START
)
63 return &unidata
[c
/ UNICODE_MODULUS
][c
% UNICODE_MODULUS
];
65 return utf32__unidata_hard(c
);
68 /** @brief Find definition of code point @p c
70 * @return Pointer to @ref unidata structure for @p c
72 * @p c can be any 32-bit value, a sensible value will be returned regardless.
73 * The returned pointer is NOT guaranteed to be unique to @p c.
75 * Don't use this function (although it will work fine) - use utf32__unidata()
78 static const struct unidata
*utf32__unidata_hard(uint32_t c
) {
79 if(c
< UNICODE_BREAK_START
)
80 return &unidata
[c
/ UNICODE_MODULUS
][c
% UNICODE_MODULUS
];
81 /* Within the break everything is unassigned */
82 if(c
< UNICODE_BREAK_END
)
83 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
84 /* Planes 15 and 16 are (mostly) private use */
85 if((c
>= 0xF0000 && c
<= 0xFFFFD)
86 || (c
>= 0x100000 && c
<= 0x10FFFD))
87 return utf32__unidata(0xE000); /* first Co code point */
88 /* Everything else above the break top is unassigned */
89 if(c
>= UNICODE_BREAK_TOP
)
90 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
91 /* Currently the rest is language tags and variation selectors */
92 c
-= (UNICODE_BREAK_END
- UNICODE_BREAK_START
);
93 return &unidata
[c
/ UNICODE_MODULUS
][c
% UNICODE_MODULUS
];
96 /** @brief Return the combining class of @p c
98 * @return Combining class of @p c
100 * @p c can be any 32-bit value, a sensible value will be returned regardless.
102 static inline int utf32__combining_class(uint32_t c
) {
103 return utf32__unidata(c
)->ccc
;
106 /** @brief Return the combining class of @p c
107 * @param c Code point
108 * @return Combining class of @p c
110 * @p c can be any 32-bit value, a sensible value will be returned regardless.
112 int utf32_combining_class(uint32_t c
) {
113 return utf32__combining_class(c
);
116 /** @brief Return the General_Category value for @p c
117 * @param c Code point
118 * @return General_Category property value
120 * @p c can be any 32-bit value, a sensible value will be returned regardless.
122 static inline enum unicode_General_Category
utf32__general_category(uint32_t c
) {
123 return utf32__unidata(c
)->general_category
;
126 /** @brief Determine Grapheme_Break property
127 * @param c Code point
128 * @return Grapheme_Break property value of @p c
130 * @p c can be any 32-bit value, a sensible value will be returned regardless.
132 static inline enum unicode_Grapheme_Break
utf32__grapheme_break(uint32_t c
) {
133 return utf32__unidata(c
)->grapheme_break
;
136 /** @brief Determine Word_Break property
137 * @param c Code point
138 * @return Word_Break property value of @p c
140 * @p c can be any 32-bit value, a sensible value will be returned regardless.
142 static inline enum unicode_Word_Break
utf32__word_break(uint32_t c
) {
143 return utf32__unidata(c
)->word_break
;
146 /** @brief Determine Sentence_Break property
147 * @param c Code point
148 * @return Word_Break property value of @p c
150 * @p c can be any 32-bit value, a sensible value will be returned regardless.
152 static inline enum unicode_Sentence_Break
utf32__sentence_break(uint32_t c
) {
153 return utf32__unidata(c
)->sentence_break
;
156 /** @brief Return true if @p c is ignorable for boundary specifications
157 * @param wb Word break property value
158 * @return non-0 if @p wb is unicode_Word_Break_Extend or unicode_Word_Break_Format
160 static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb
) {
161 return (wb
== unicode_Word_Break_Extend
162 || wb
== unicode_Word_Break_Format
);
165 /** @brief Return the canonical decomposition of @p c
166 * @param c Code point
167 * @return 0-terminated canonical decomposition, or 0
169 static inline const uint32_t *utf32__decomposition_canon(uint32_t c
) {
170 const struct unidata
*const data
= utf32__unidata(c
);
171 const uint32_t *const decomp
= data
->decomp
;
173 if(decomp
&& !(data
->flags
& unicode_compatibility_decomposition
))
179 /** @brief Return the compatibility decomposition of @p c
180 * @param c Code point
181 * @return 0-terminated decomposition, or 0
183 static inline const uint32_t *utf32__decomposition_compat(uint32_t c
) {
184 return utf32__unidata(c
)->decomp
;
188 /** @defgroup utftransform Functions that transform between different Unicode encoding forms */
191 /** @brief Convert UTF-32 to UTF-8
192 * @param s Source string
193 * @param ns Length of source string in code points
194 * @param ndp Where to store length of destination string (or NULL)
195 * @return Newly allocated destination string or NULL on error
197 * If the UTF-32 is not valid then NULL is returned. A UTF-32 code point is
199 * - it codes for a UTF-16 surrogate
200 * - it codes for a value outside the unicode code space
202 * The return value is always 0-terminated. The value returned via @p *ndp
203 * does not include the terminator.
205 char *utf32_to_utf8(const uint32_t *s
, size_t ns
, size_t *ndp
) {
213 dynstr_append(&d
, c
);
214 else if(c
< 0x0800) {
215 dynstr_append(&d
, 0xC0 | (c
>> 6));
216 dynstr_append(&d
, 0x80 | (c
& 0x3F));
217 } else if(c
< 0x10000) {
218 if(c
>= 0xD800 && c
<= 0xDFFF)
220 dynstr_append(&d
, 0xE0 | (c
>> 12));
221 dynstr_append(&d
, 0x80 | ((c
>> 6) & 0x3F));
222 dynstr_append(&d
, 0x80 | (c
& 0x3F));
223 } else if(c
< 0x110000) {
224 dynstr_append(&d
, 0xF0 | (c
>> 18));
225 dynstr_append(&d
, 0x80 | ((c
>> 12) & 0x3F));
226 dynstr_append(&d
, 0x80 | ((c
>> 6) & 0x3F));
227 dynstr_append(&d
, 0x80 | (c
& 0x3F));
232 dynstr_terminate(&d
);
241 /** @brief Convert UTF-8 to UTF-32
242 * @param s Source string
243 * @param ns Length of source string in code points
244 * @param ndp Where to store length of destination string (or NULL)
245 * @return Newly allocated destination string or NULL on error
247 * The return value is always 0-terminated. The value returned via @p *ndp
248 * does not include the terminator.
250 * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
251 * for a code point is invalid if:
252 * - it is not the shortest possible sequence for the code point
253 * - it codes for a UTF-16 surrogate
254 * - it codes for a value outside the unicode code space
256 uint32_t *utf8_to_utf32(const char *s
, size_t ns
, size_t *ndp
) {
257 struct dynstr_ucs4 d
;
259 const uint8_t *ss
= (const uint8_t *)s
;
262 dynstr_ucs4_init(&d
);
264 const struct unicode_utf8_row
*const r
= &unicode_utf8_valid
[*ss
];
271 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
276 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
281 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
290 for(n
= 1; n
< r
->count
; ++n
) {
291 if(ss
[n
] < 0x80 || ss
[n
] > 0xBF)
293 c32
= (c32
<< 6) | (ss
[n
] & 0x3F);
295 dynstr_ucs4_append(&d
, c32
);
299 dynstr_ucs4_terminate(&d
);
308 /** @brief Convert UTF-16 to UTF-8
309 * @param s Source string
310 * @param ns Length of source string in code points
311 * @param ndp Where to store length of destination string (or NULL)
312 * @return Newly allocated destination string or NULL on error
314 * If the UTF-16 is not valid then NULL is returned. A UTF-16 sequence t is
315 * invalid if it contains an incomplete surrogate.
317 * The return value is always 0-terminated. The value returned via @p *ndp
318 * does not include the terminator.
320 char *utf16_to_utf8(const uint16_t *s
, size_t ns
, size_t *ndp
) {
328 if(c
>= 0xD800 && c
<= 0xDBFF) {
329 if(ns
&& *s
>= 0xDC00 && c
<= 0xDFFF)
330 c
= ((c
- 0xD800) << 10) + (*s
++ - 0xDC00) + 0x10000;
333 } else if(c
>= 0xDC00 && c
<= 0xDFFF)
336 dynstr_append(&d
, c
);
337 else if(c
< 0x0800) {
338 dynstr_append(&d
, 0xC0 | (c
>> 6));
339 dynstr_append(&d
, 0x80 | (c
& 0x3F));
340 } else if(c
< 0x10000) {
341 if(c
>= 0xD800 && c
<= 0xDFFF)
343 dynstr_append(&d
, 0xE0 | (c
>> 12));
344 dynstr_append(&d
, 0x80 | ((c
>> 6) & 0x3F));
345 dynstr_append(&d
, 0x80 | (c
& 0x3F));
346 } else if(c
< 0x110000) {
347 dynstr_append(&d
, 0xF0 | (c
>> 18));
348 dynstr_append(&d
, 0x80 | ((c
>> 12) & 0x3F));
349 dynstr_append(&d
, 0x80 | ((c
>> 6) & 0x3F));
350 dynstr_append(&d
, 0x80 | (c
& 0x3F));
354 dynstr_terminate(&d
);
363 /** @brief Convert UTF-8 to UTF-16
364 * @param s Source string
365 * @param ns Length of source string in code points
366 * @param ndp Where to store length of destination string (or NULL)
367 * @return Newly allocated destination string or NULL on error
369 * The return value is always 0-terminated. The value returned via @p *ndp
370 * does not include the terminator.
372 * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
373 * for a code point is invalid if:
374 * - it is not the shortest possible sequence for the code point
375 * - it codes for a UTF-16 surrogate
376 * - it codes for a value outside the unicode code space
378 uint16_t *utf8_to_utf16(const char *s
, size_t ns
, size_t *ndp
) {
379 struct dynstr_utf16 d
;
381 const uint8_t *ss
= (const uint8_t *)s
;
384 dynstr_utf16_init(&d
);
386 const struct unicode_utf8_row
*const r
= &unicode_utf8_valid
[*ss
];
393 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
398 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
403 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
412 for(n
= 1; n
< r
->count
; ++n
) {
413 if(ss
[n
] < 0x80 || ss
[n
] > 0xBF)
415 c32
= (c32
<< 6) | (ss
[n
] & 0x3F);
419 dynstr_utf16_append(&d
, 0xD800 + (c32
>> 10));
420 dynstr_utf16_append(&d
, 0xDC00 + (c32
& 0x03FF));
422 dynstr_utf16_append(&d
, c32
);
426 dynstr_utf16_terminate(&d
);
435 /** @brief Test whether [s,s+ns) is valid UTF-8
436 * @param s Start of string
437 * @param ns Length of string
438 * @return non-0 if @p s is valid UTF-8, 0 if it is not valid
440 * This function is intended to be much faster than calling utf8_to_utf32() and
441 * throwing away the result.
443 int utf8_valid(const char *s
, size_t ns
) {
444 const uint8_t *ss
= (const uint8_t *)s
;
446 const struct unicode_utf8_row
*const r
= &unicode_utf8_valid
[*ss
];
452 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
456 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
458 if(ss
[2] < 0x80 || ss
[2] > 0xBF)
462 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
464 if(ss
[2] < 0x80 || ss
[2] > 0xBF)
466 if(ss
[3] < 0x80 || ss
[3] > 0xBF)
481 /** @defgroup utf32iterator UTF-32 string iterators */
484 struct utf32_iterator_data
{
485 /** @brief Start of string */
488 /** @brief Length of string */
491 /** @brief Current position */
494 /** @brief Last two non-ignorable characters or (uint32_t)-1
496 * last[1] is the non-Extend/Format character just before position @p n;
497 * last[0] is the one just before that.
499 * Exception 1: if there is no such non-Extend/Format character then an
500 * Extend/Format character is accepted instead.
502 * Exception 2: if there is no such character even taking that into account
503 * the value is (uint32_t)-1.
507 /** @brief Tailoring for Word_Break */
508 unicode_property_tailor
*word_break
;
511 /** @brief Initialize an internal private iterator
513 * @param s Start of string
514 * @param ns Length of string
515 * @param n Absolute position
517 static void utf32__iterator_init(utf32_iterator it
,
518 const uint32_t *s
, size_t ns
, size_t n
) {
522 it
->last
[0] = it
->last
[1] = -1;
524 utf32_iterator_set(it
, n
);
527 /** @brief Create a new iterator pointing at the start of a string
528 * @param s Start of string
529 * @param ns Length of string
530 * @return New iterator
532 utf32_iterator
utf32_iterator_new(const uint32_t *s
, size_t ns
) {
533 utf32_iterator it
= xmalloc(sizeof *it
);
534 utf32__iterator_init(it
, s
, ns
, 0);
538 /** @brief Tailor this iterator's interpretation of the Word_Break property.
540 * @param pt Property tailor function or NULL
542 * After calling this the iterator will call @p pt to determine the Word_Break
543 * property of each code point. If it returns -1 the default value will be
544 * used otherwise the returned value will be used.
546 * @p pt can be NULL to revert to the default value of the property.
548 * It is safe to call this function at any time; the iterator's internal state
549 * will be reset to suit the new tailoring.
551 void utf32_iterator_tailor_word_break(utf32_iterator it
,
552 unicode_property_tailor
*pt
) {
554 utf32_iterator_set(it
, it
->n
);
557 static inline enum unicode_Word_Break
utf32__iterator_word_break(utf32_iterator it
,
560 return utf32__word_break(c
);
562 const int t
= it
->word_break(c
);
565 return utf32__word_break(c
);
571 /** @brief Destroy an iterator
574 void utf32_iterator_destroy(utf32_iterator it
) {
578 /** @brief Find the current position of an interator
581 size_t utf32_iterator_where(utf32_iterator it
) {
585 /** @brief Set an iterator's absolute position
587 * @param n Absolute position
588 * @return 0 on success, non-0 on error
590 * It is an error to position the iterator outside the string (but acceptable
591 * to point it at the hypothetical post-final character). If an invalid value
592 * of @p n is specified then the iterator is not changed.
594 * This function works by backing up and then advancing to reconstruct the
595 * iterator's internal state for position @p n. The worst case will be O(n)
596 * time complexity (with a worse constant factor that utf32_iterator_advance())
597 * but the typical case is essentially constant-time.
599 int utf32_iterator_set(utf32_iterator it
, size_t n
) {
600 /* We can't just jump to position @p n; the @p last[] values will be wrong.
601 * What we need is to jump a bit behind @p n and then advance forward,
602 * updating @p last[] along the way. How far back? We need to cross two
603 * non-ignorable code points as we advance forwards, so we'd better pass two
604 * such characters on the way back (if such are available).
608 if(n
> it
->ns
) /* range check */
610 /* Walk backwards skipping ignorable code points */
613 && (utf32__boundary_ignorable(utf32__iterator_word_break(it
,
616 /* Either m=0 or s[m-1] is not ignorable */
619 /* s[m] is our first non-ignorable code; look for a second in the same
622 && (utf32__boundary_ignorable(utf32__iterator_word_break(it
,
625 /* Either m=0 or s[m-1] is not ignorable */
629 it
->last
[0] = it
->last
[1] = -1;
631 return utf32_iterator_advance(it
, n
- m
);
634 /** @brief Advance an iterator
636 * @param count Number of code points to advance by
637 * @return 0 on success, non-0 on error
639 * It is an error to advance an iterator beyond the hypothetical post-final
640 * character of the string. If an invalid value of @p n is specified then the
641 * iterator is not changed.
643 * This function has O(n) time complexity: it works by advancing naively
644 * forwards through the string.
646 int utf32_iterator_advance(utf32_iterator it
, size_t count
) {
647 if(count
<= it
->ns
- it
->n
) {
649 const uint32_t c
= it
->s
[it
->n
];
650 const enum unicode_Word_Break wb
= utf32__iterator_word_break(it
, c
);
651 if(it
->last
[1] == (uint32_t)-1
652 || !utf32__boundary_ignorable(wb
)) {
653 it
->last
[0] = it
->last
[1];
664 /** @brief Find the current code point
666 * @return Current code point or 0
668 * If the iterator points at the hypothetical post-final character of the
669 * string then 0 is returned. NB that this doesn't mean that there aren't any
670 * 0 code points inside the string!
672 uint32_t utf32_iterator_code(utf32_iterator it
) {
679 /** @brief Test for a grapheme boundary
681 * @return Non-0 if pointing just after a grapheme boundary, otherwise 0
683 * This function identifies default grapheme cluster boundaries as described in
684 * UAX #29 s3. It returns non-0 if @p it points at the code point just after a
685 * grapheme cluster boundary (including the hypothetical code point just after
686 * the end of the string).
688 int utf32_iterator_grapheme_boundary(utf32_iterator it
) {
689 uint32_t before
, after
;
690 enum unicode_Grapheme_Break gbbefore
, gbafter
;
692 if(it
->n
== 0 || it
->n
== it
->ns
)
694 /* Now we know that s[n-1] and s[n] are safe to inspect */
696 before
= it
->s
[it
->n
-1];
697 after
= it
->s
[it
->n
];
698 if(before
== 0x000D && after
== 0x000A)
700 gbbefore
= utf32__grapheme_break(before
);
701 gbafter
= utf32__grapheme_break(after
);
703 if(gbbefore
== unicode_Grapheme_Break_Control
708 if(gbafter
== unicode_Grapheme_Break_Control
713 if(gbbefore
== unicode_Grapheme_Break_L
714 && (gbafter
== unicode_Grapheme_Break_L
715 || gbafter
== unicode_Grapheme_Break_V
716 || gbafter
== unicode_Grapheme_Break_LV
717 || gbafter
== unicode_Grapheme_Break_LVT
))
720 if((gbbefore
== unicode_Grapheme_Break_LV
721 || gbbefore
== unicode_Grapheme_Break_V
)
722 && (gbafter
== unicode_Grapheme_Break_V
723 || gbafter
== unicode_Grapheme_Break_T
))
726 if((gbbefore
== unicode_Grapheme_Break_LVT
727 || gbbefore
== unicode_Grapheme_Break_T
)
728 && gbafter
== unicode_Grapheme_Break_T
)
731 if(gbafter
== unicode_Grapheme_Break_Extend
)
734 if(gbafter
== unicode_Grapheme_Break_SpacingMark
)
737 if(gbbefore
== unicode_Grapheme_Break_Prepend
)
744 /** @brief Test for a word boundary
746 * @return Non-0 if pointing just after a word boundary, otherwise 0
748 * This function identifies default word boundaries as described in UAX #29 s4.
749 * It returns non-0 if @p it points at the code point just after a word
750 * boundary (including the hypothetical code point just after the end of the
751 * string) and 0 otherwise.
753 int utf32_iterator_word_boundary(utf32_iterator it
) {
754 uint32_t before
, after
;
755 enum unicode_Word_Break wbtwobefore
, wbbefore
, wbafter
, wbtwoafter
;
759 if(it
->n
== 0 || it
->n
== it
->ns
)
761 before
= it
->s
[it
->n
-1];
762 after
= it
->s
[it
->n
];
764 if(before
== 0x000D && after
== 0x000A)
767 if(utf32__iterator_word_break(it
, before
) == unicode_Word_Break_Newline
772 if(utf32__iterator_word_break(it
, after
) == unicode_Word_Break_Newline
777 /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
778 if(utf32__sentence_break(before
) != unicode_Sentence_Break_Sep
779 && utf32__boundary_ignorable(utf32__iterator_word_break(it
, after
)))
781 /* Gather the property values we'll need for the rest of the test taking the
782 * s6.2 changes into account */
783 /* First we look at the code points after the proposed boundary */
784 nn
= it
->n
; /* <it->ns */
785 wbafter
= utf32__iterator_word_break(it
, it
->s
[nn
++]);
786 if(!utf32__boundary_ignorable(wbafter
)) {
787 /* X (Extend|Format)* -> X */
789 && utf32__boundary_ignorable(utf32__iterator_word_break(it
,
793 /* It's possible now that nn=ns */
795 wbtwoafter
= utf32__iterator_word_break(it
, it
->s
[nn
]);
797 wbtwoafter
= unicode_Word_Break_Other
;
799 /* We've already recorded the non-ignorable code points before the proposed
801 wbbefore
= utf32__iterator_word_break(it
, it
->last
[1]);
802 wbtwobefore
= utf32__iterator_word_break(it
, it
->last
[0]);
805 if(wbbefore
== unicode_Word_Break_ALetter
806 && wbafter
== unicode_Word_Break_ALetter
)
809 if(wbbefore
== unicode_Word_Break_ALetter
810 && (wbafter
== unicode_Word_Break_MidLetter
811 || wbafter
== unicode_Word_Break_MidNumLet
)
812 && wbtwoafter
== unicode_Word_Break_ALetter
)
815 if(wbtwobefore
== unicode_Word_Break_ALetter
816 && (wbbefore
== unicode_Word_Break_MidLetter
817 || wbbefore
== unicode_Word_Break_MidNumLet
)
818 && wbafter
== unicode_Word_Break_ALetter
)
821 if(wbbefore
== unicode_Word_Break_Numeric
822 && wbafter
== unicode_Word_Break_Numeric
)
825 if(wbbefore
== unicode_Word_Break_ALetter
826 && wbafter
== unicode_Word_Break_Numeric
)
829 if(wbbefore
== unicode_Word_Break_Numeric
830 && wbafter
== unicode_Word_Break_ALetter
)
833 if(wbtwobefore
== unicode_Word_Break_Numeric
834 && (wbbefore
== unicode_Word_Break_MidNum
835 || wbbefore
== unicode_Word_Break_MidNumLet
)
836 && wbafter
== unicode_Word_Break_Numeric
)
839 if(wbbefore
== unicode_Word_Break_Numeric
840 && (wbafter
== unicode_Word_Break_MidNum
841 || wbafter
== unicode_Word_Break_MidNumLet
)
842 && wbtwoafter
== unicode_Word_Break_Numeric
)
845 if(wbbefore
== unicode_Word_Break_Katakana
846 && wbafter
== unicode_Word_Break_Katakana
)
849 if((wbbefore
== unicode_Word_Break_ALetter
850 || wbbefore
== unicode_Word_Break_Numeric
851 || wbbefore
== unicode_Word_Break_Katakana
852 || wbbefore
== unicode_Word_Break_ExtendNumLet
)
853 && wbafter
== unicode_Word_Break_ExtendNumLet
)
856 if(wbbefore
== unicode_Word_Break_ExtendNumLet
857 && (wbafter
== unicode_Word_Break_ALetter
858 || wbafter
== unicode_Word_Break_Numeric
859 || wbafter
== unicode_Word_Break_Katakana
))
866 /** @defgroup utf32 Functions that operate on UTF-32 strings */
869 /** @brief Return the length of a 0-terminated UTF-32 string
870 * @param s Pointer to 0-terminated string
871 * @return Length of string in code points (excluding terminator)
873 * Unlike the conversion functions no validity checking is done on the string.
875 size_t utf32_len(const uint32_t *s
) {
876 const uint32_t *t
= s
;
880 return (size_t)(t
- s
);
883 /** @brief Stably sort [s,s+ns) into descending order of combining class
884 * @param s Start of array
885 * @param ns Number of elements, must be at least 1
886 * @param buffer Buffer of at least @p ns elements
888 static void utf32__sort_ccc(uint32_t *s
, size_t ns
, uint32_t *buffer
) {
889 uint32_t *a
, *b
, *bp
;
893 case 1: /* 1-element array is always sorted */
895 case 2: /* 2-element arrays are trivial to sort */
896 if(utf32__combining_class(s
[0]) > utf32__combining_class(s
[1])) {
903 /* Partition the array */
908 /* Sort the two halves of the array */
909 utf32__sort_ccc(a
, na
, buffer
);
910 utf32__sort_ccc(b
, nb
, buffer
);
911 /* Merge them back into one, via the buffer */
913 while(na
> 0 && nb
> 0) {
914 /* We want ascending order of combining class (hence <)
915 * and we want stability within combining classes (hence <=)
917 if(utf32__combining_class(*a
) <= utf32__combining_class(*b
)) {
933 memcpy(s
, buffer
, ns
* sizeof(uint32_t));
938 /** @brief Put combining characters into canonical order
939 * @param s Pointer to UTF-32 string
940 * @param ns Length of @p s
941 * @return 0 on success, non-0 on error
943 * @p s is modified in-place. See Unicode 5.0 s3.11 for details of the
946 * Currently we only support a maximum of 1024 combining characters after each
947 * base character. If this limit is exceeded then a non-0 value is returned.
949 static int utf32__canonical_ordering(uint32_t *s
, size_t ns
) {
951 uint32_t buffer
[1024];
953 /* The ordering amounts to a stable sort of each contiguous group of
954 * characters with non-0 combining class. */
956 /* Skip non-combining characters */
957 if(utf32__combining_class(*s
) == 0) {
962 /* We must now have at least one combining character; see how many
964 for(nc
= 1; nc
< ns
&& utf32__combining_class(s
[nc
]) != 0; ++nc
)
969 utf32__sort_ccc(s
, nc
, buffer
);
976 /* Magic numbers from UAX #15 s16 */
984 #define NCount (VCount * TCount)
985 #define SCount (LCount * NCount)
987 /** @brief Guts of the decomposition lookup functions */
988 #define utf32__decompose_one_generic(WHICH) do { \
989 const uint32_t *dc = utf32__decomposition_##WHICH(c); \
991 /* Found a canonical decomposition in the table */ \
993 utf32__decompose_one_##WHICH(d, *dc++); \
994 } else if(c >= SBase && c < SBase + SCount) { \
995 /* Mechanically decomposable Hangul syllable (UAX #15 s16) */ \
996 const uint32_t SIndex = c - SBase; \
997 const uint32_t L = LBase + SIndex / NCount; \
998 const uint32_t V = VBase + (SIndex % NCount) / TCount; \
999 const uint32_t T = TBase + SIndex % TCount; \
1000 dynstr_ucs4_append(d, L); \
1001 dynstr_ucs4_append(d, V); \
1003 dynstr_ucs4_append(d, T); \
1005 /* Equal to own canonical decomposition */ \
1006 dynstr_ucs4_append(d, c); \
1009 /** @brief Recursively compute the canonical decomposition of @p c
1010 * @param d Dynamic string to store decomposition in
1011 * @param c Code point to decompose (must be a valid!)
1012 * @return 0 on success, non-0 on error
1014 static void utf32__decompose_one_canon(struct dynstr_ucs4
*d
, uint32_t c
) {
1015 utf32__decompose_one_generic(canon
);
1018 /** @brief Recursively compute the compatibility decomposition of @p c
1019 * @param d Dynamic string to store decomposition in
1020 * @param c Code point to decompose (must be a valid!)
1021 * @return 0 on success, non-0 on error
1023 static void utf32__decompose_one_compat(struct dynstr_ucs4
*d
, uint32_t c
) {
1024 utf32__decompose_one_generic(compat
);
1027 /** @brief Magic utf32__compositions() return value for Hangul Choseong */
1028 static const uint32_t utf32__hangul_L
[1];
1030 /** @brief Return the list of compositions that @p c starts
1031 * @param c Starter code point
1032 * @return Composition list or NULL
1034 * For Hangul leading (Choseong) jamo we return the special value
1035 * utf32__hangul_L. These code points are not listed as the targets of
1036 * canonical decompositions (make-unidata checks) so there is no confusion with
1037 * real decompositions here.
1039 static const uint32_t *utf32__compositions(uint32_t c
) {
1040 const uint32_t *compositions
= utf32__unidata(c
)->composed
;
1043 return compositions
;
1044 /* Special-casing for Hangul */
1045 switch(utf32__grapheme_break(c
)) {
1048 case unicode_Grapheme_Break_L
:
1049 return utf32__hangul_L
;
1053 /** @brief Composition step
1054 * @param s Start of string
1055 * @param ns Length of string
1056 * @return New length of string
1058 * This is called from utf32__decompose_generic() to compose the result string
1061 static size_t utf32__compose(uint32_t *s
, size_t ns
) {
1062 const uint32_t *compositions
;
1063 uint32_t *start
= s
, *t
= s
, *tt
, cc
;
1066 uint32_t starter
= *s
++;
1067 int block_starters
= 0;
1069 /* We don't attempt to compose the following things:
1070 * - final characters whatever kind they are
1071 * - non-starter characters
1072 * - starters that don't take part in a canonical decomposition mapping
1075 || utf32__combining_class(starter
)
1076 || !(compositions
= utf32__compositions(starter
))) {
1080 if(compositions
!= utf32__hangul_L
) {
1081 /* Where we'll put the eventual starter */
1084 /* See if we can find composition of starter+*s */
1085 const uint32_t cchar
= *s
, *cp
= compositions
;
1086 while((cc
= *cp
++)) {
1087 const uint32_t *decomp
= utf32__decomposition_canon(cc
);
1088 /* We know decomp[0] == starter */
1089 if(decomp
[1] == cchar
)
1093 /* Found a composition: cc decomposes to starter,*s */
1095 compositions
= utf32__compositions(starter
);
1099 /* No composition found. */
1100 const int class = utf32__combining_class(*s
);
1102 /* Transfer the uncomposable combining character to the output */
1105 /* All the combining characters of the same class of the
1106 * uncomposable character are blocked by it, but there may be
1107 * others of higher class later. We eat the uncomposable and
1108 * blocked characters and go back round the loop for that higher
1110 while(ns
> 0 && utf32__combining_class(*s
) == class) {
1114 /* Block any subsequent starters */
1117 /* The uncombinable character is itself a starter, so we don't
1118 * transfer it to the output but instead go back round the main
1123 /* Keep going while there are still characters and the starter takes
1124 * part in some composition */
1125 } while(ns
> 0 && compositions
1126 && (!block_starters
|| utf32__combining_class(*s
)));
1127 /* Store any remaining combining characters */
1128 while(ns
> 0 && utf32__combining_class(*s
)) {
1132 /* Store the resulting starter */
1135 /* Special-casing for Hangul
1137 * If there are combining characters between the L and the V then they
1138 * will block the V and so no composition happens. Similarly combining
1139 * characters between V and T will block the T and so we only get as far
1142 if(utf32__grapheme_break(*s
) == unicode_Grapheme_Break_V
) {
1143 const uint32_t V
= *s
++;
1144 const uint32_t LIndex
= starter
- LBase
;
1145 const uint32_t VIndex
= V
- VBase
;
1149 && utf32__grapheme_break(*s
) == unicode_Grapheme_Break_T
) {
1150 /* We have an L V T sequence */
1151 const uint32_t T
= *s
++;
1157 /* Compose to LVT or LV as appropriate */
1158 starter
= (LIndex
* VCount
+ VIndex
) * TCount
+ TIndex
+ SBase
;
1159 } /* else we only have L or LV and no V or T */
1161 /* There could be some combining characters that belong to the V or T.
1162 * These will be treated as non-starter characters at the top of the loop
1163 * and thuss transferred to the output. */
1169 /** @brief Guts of the composition and decomposition functions
1170 * @param WHICH @c canon or @c compat to choose decomposition
1171 * @param COMPOSE @c 0 or @c 1 to compose
1173 #define utf32__decompose_generic(WHICH, COMPOSE) do { \
1174 struct dynstr_ucs4 d; \
1177 dynstr_ucs4_init(&d); \
1180 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
1182 utf32__decompose_one_##WHICH(&d, c); \
1185 if(utf32__canonical_ordering(d.vec, d.nvec)) \
1188 d.nvec = utf32__compose(d.vec, d.nvec); \
1189 dynstr_ucs4_terminate(&d); \
1198 /** @brief Canonically decompose @p [s,s+ns)
1199 * @param s Pointer to string
1200 * @param ns Length of string
1201 * @param ndp Where to store length of result
1202 * @return Pointer to result string, or NULL on error
1204 * Computes NFD (Normalization Form D) of the string at @p s. This implies
1205 * performing all canonical decompositions and then normalizing the order of
1206 * combining characters.
1208 * Returns NULL if the string is not valid for either of the following reasons:
1209 * - it codes for a UTF-16 surrogate
1210 * - it codes for a value outside the unicode code space
1213 * - utf32_decompose_compat()
1214 * - utf32_compose_canon()
1216 uint32_t *utf32_decompose_canon(const uint32_t *s
, size_t ns
, size_t *ndp
) {
1217 utf32__decompose_generic(canon
, 0);
1220 /** @brief Compatibility decompose @p [s,s+ns)
1221 * @param s Pointer to string
1222 * @param ns Length of string
1223 * @param ndp Where to store length of result
1224 * @return Pointer to result string, or NULL on error
1226 * Computes NFKD (Normalization Form KD) of the string at @p s. This implies
1227 * performing all canonical and compatibility decompositions and then
1228 * normalizing the order of combining characters.
1230 * Returns NULL if the string is not valid for either of the following reasons:
1231 * - it codes for a UTF-16 surrogate
1232 * - it codes for a value outside the unicode code space
1235 * - utf32_decompose_canon()
1236 * - utf32_compose_compat()
1238 uint32_t *utf32_decompose_compat(const uint32_t *s
, size_t ns
, size_t *ndp
) {
1239 utf32__decompose_generic(compat
, 0);
1242 /** @brief Canonically compose @p [s,s+ns)
1243 * @param s Pointer to string
1244 * @param ns Length of string
1245 * @param ndp Where to store length of result
1246 * @return Pointer to result string, or NULL on error
1248 * Computes NFC (Normalization Form C) of the string at @p s. This implies
1249 * performing all canonical decompositions, normalizing the order of combining
1250 * characters and then composing all unblocked primary compositables.
1252 * Returns NULL if the string is not valid for either of the following reasons:
1253 * - it codes for a UTF-16 surrogate
1254 * - it codes for a value outside the unicode code space
1257 * - utf32_compose_compat()
1258 * - utf32_decompose_canon()
1260 uint32_t *utf32_compose_canon(const uint32_t *s
, size_t ns
, size_t *ndp
) {
1261 utf32__decompose_generic(canon
, 1);
1264 /** @brief Compatibility compose @p [s,s+ns)
1265 * @param s Pointer to string
1266 * @param ns Length of string
1267 * @param ndp Where to store length of result
1268 * @return Pointer to result string, or NULL on error
1270 * Computes NFKC (Normalization Form KC) of the string at @p s. This implies
1271 * performing all canonical and compatibility decompositions, normalizing the
1272 * order of combining characters and then composing all unblocked primary
1275 * Returns NULL if the string is not valid for either of the following reasons:
1276 * - it codes for a UTF-16 surrogate
1277 * - it codes for a value outside the unicode code space
1280 * - utf32_compose_canon()
1281 * - utf32_decompose_compat()
1283 uint32_t *utf32_compose_compat(const uint32_t *s
, size_t ns
, size_t *ndp
) {
1284 utf32__decompose_generic(compat
, 1);
1287 /** @brief Single-character case-fold and decompose operation */
1288 #define utf32__casefold_one(WHICH) do { \
1289 const uint32_t *cf = utf32__unidata(c)->casefold; \
1291 /* Found a case-fold mapping in the table */ \
1293 utf32__decompose_one_##WHICH(&d, *cf++); \
1295 utf32__decompose_one_##WHICH(&d, c); \
1298 /** @brief Case-fold @p [s,s+ns)
1299 * @param s Pointer to string
1300 * @param ns Length of string
1301 * @param ndp Where to store length of result
1302 * @return Pointer to result string, or NULL on error
1304 * Case-fold the string at @p s according to full default case-folding rules
1305 * (s3.13) for caseless matching. The result will be in NFD.
1307 * Returns NULL if the string is not valid for either of the following reasons:
1308 * - it codes for a UTF-16 surrogate
1309 * - it codes for a value outside the unicode code space
1311 uint32_t *utf32_casefold_canon(const uint32_t *s
, size_t ns
, size_t *ndp
) {
1312 struct dynstr_ucs4 d
;
1317 /* If the canonical decomposition of the string includes any combining
1318 * character that case-folds to a non-combining character then we must
1319 * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING
1320 * GREEK YPOGEGRAMMENI in its decomposition and the various characters that
1321 * canonically decompose to it. */
1322 for(n
= 0; n
< ns
; ++n
)
1323 if(utf32__unidata(s
[n
])->flags
& unicode_normalize_before_casefold
)
1326 /* We need a preliminary decomposition */
1327 if(!(ss
= utf32_decompose_canon(s
, ns
, &ns
)))
1331 dynstr_ucs4_init(&d
);
1334 if((c
>= 0xD800 && c
<= 0xDFFF) || c
> 0x10FFFF)
1336 utf32__casefold_one(canon
);
1339 if(utf32__canonical_ordering(d
.vec
, d
.nvec
))
1341 dynstr_ucs4_terminate(&d
);
1351 /** @brief Compatibility case-fold @p [s,s+ns)
1352 * @param s Pointer to string
1353 * @param ns Length of string
1354 * @param ndp Where to store length of result
1355 * @return Pointer to result string, or NULL on error
1357 * Case-fold the string at @p s according to full default case-folding rules
1358 * (s3.13) for compatibility caseless matching. The result will be in NFKD.
1360 * Returns NULL if the string is not valid for either of the following reasons:
1361 * - it codes for a UTF-16 surrogate
1362 * - it codes for a value outside the unicode code space
1364 uint32_t *utf32_casefold_compat(const uint32_t *s
, size_t ns
, size_t *ndp
) {
1365 struct dynstr_ucs4 d
;
1370 for(n
= 0; n
< ns
; ++n
)
1371 if(utf32__unidata(s
[n
])->flags
& unicode_normalize_before_casefold
)
1374 /* We need a preliminary _canonical_ decomposition */
1375 if(!(ss
= utf32_decompose_canon(s
, ns
, &ns
)))
1379 /* This computes NFKD(toCaseFold(s)) */
1380 #define compat_casefold_middle() do { \
1381 dynstr_ucs4_init(&d); \
1384 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
1386 utf32__casefold_one(compat); \
1389 if(utf32__canonical_ordering(d.vec, d.nvec)) \
1392 /* Do the inner (NFKD o toCaseFold) */
1393 compat_casefold_middle();
1394 /* We can do away with the NFD'd copy of the input now */
1398 /* Do the outer (NFKD o toCaseFold) */
1399 compat_casefold_middle();
1401 dynstr_ucs4_terminate(&d
);
1411 /** @brief Order a pair of UTF-32 strings
1412 * @param a First 0-terminated string
1413 * @param b Second 0-terminated string
1414 * @return -1, 0 or 1 for a less than, equal to or greater than b
1416 * "Comparable to strcmp() at its best."
1418 int utf32_cmp(const uint32_t *a
, const uint32_t *b
) {
1419 while(*a
&& *b
&& *a
== *b
) {
1423 return *a
< *b ?
-1 : (*a
> *b ?
1 : 0);
1426 /** @brief Identify a grapheme cluster boundary
1427 * @param s Start of string (must be NFD)
1428 * @param ns Length of string
1429 * @param n Index within string (in [0,ns].)
1430 * @return 1 at a grapheme cluster boundary, 0 otherwise
1432 * This function identifies default grapheme cluster boundaries as described in
1433 * UAX #29 s3. It returns non-0 if @p n points at the code point just after a
1434 * grapheme cluster boundary (including the hypothetical code point just after
1435 * the end of the string).
1437 * This function uses utf32_iterator_set() internally; see that function for
1438 * remarks on performance.
1440 int utf32_is_grapheme_boundary(const uint32_t *s
, size_t ns
, size_t n
) {
1441 struct utf32_iterator_data it
[1];
1443 utf32__iterator_init(it
, s
, ns
, n
);
1444 return utf32_iterator_grapheme_boundary(it
);
1447 /** @brief Identify a word boundary
1448 * @param s Start of string (must be NFD)
1449 * @param ns Length of string
1450 * @param n Index within string (in [0,ns].)
1451 * @return 1 at a word boundary, 0 otherwise
1453 * This function identifies default word boundaries as described in UAX #29 s4.
1454 * It returns non-0 if @p n points at the code point just after a word boundary
1455 * (including the hypothetical code point just after the end of the string).
1457 * This function uses utf32_iterator_set() internally; see that function for
1458 * remarks on performance.
1460 int utf32_is_word_boundary(const uint32_t *s
, size_t ns
, size_t n
) {
1461 struct utf32_iterator_data it
[1];
1463 utf32__iterator_init(it
, s
, ns
, n
);
1464 return utf32_iterator_word_boundary(it
);
1467 /** @brief Split [s,ns) into multiple words
1468 * @param s Pointer to start of string
1469 * @param ns Length of string
1470 * @param nwp Where to store word count, or NULL
1471 * @param wbreak Word_Break property tailor, or NULL
1472 * @return Pointer to array of pointers to words
1474 * The returned array is terminated by a NULL pointer and individual
1475 * strings are 0-terminated.
1477 uint32_t **utf32_word_split(const uint32_t *s
, size_t ns
, size_t *nwp
,
1478 unicode_property_tailor
*wbreak
) {
1479 struct utf32_iterator_data it
[1];
1480 size_t b1
= 0, b2
= 0 ,i
;
1482 struct vector32 v32
[1];
1486 utf32__iterator_init(it
, s
, ns
, 0);
1487 it
->word_break
= wbreak
;
1488 /* Work our way through the string stopping at each word break. */
1490 if(utf32_iterator_word_boundary(it
)) {
1491 /* We've found a new boundary */
1494 /*fprintf(stderr, "[%zu, %zu) is a candidate word\n", b1, b2);*/
1495 /* Inspect the characters between the boundary and form an opinion as to
1496 * whether they are a word or not */
1498 for(i
= b1
; i
< b2
; ++i
) {
1499 switch(utf32__iterator_word_break(it
, it
->s
[i
])) {
1500 case unicode_Word_Break_ALetter
:
1501 case unicode_Word_Break_Numeric
:
1502 case unicode_Word_Break_Katakana
:
1509 /* If it's a word add it to the list of results */
1511 const size_t len
= b2
- b1
;
1512 w
= xcalloc_noptr(len
+ 1, sizeof(uint32_t));
1513 memcpy(w
, it
->s
+ b1
, len
* sizeof (uint32_t));
1515 vector32_append(v32
, w
);
1518 } while(!utf32_iterator_advance(it
, 1));
1519 vector32_terminate(v32
);
1526 /** @defgroup utf8 Functions that operate on UTF-8 strings */
1529 /** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */
1530 #define utf8__transform(FN) do { \
1531 uint32_t *to32 = 0, *decomp32 = 0; \
1532 size_t nto32, ndecomp32; \
1533 char *decomp8 = 0; \
1535 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error; \
1536 if(!(decomp32 = FN(to32, nto32, &ndecomp32))) goto error; \
1537 decomp8 = utf32_to_utf8(decomp32, ndecomp32, ndp); \
1544 /** @brief Canonically decompose @p [s,s+ns)
1545 * @param s Pointer to string
1546 * @param ns Length of string
1547 * @param ndp Where to store length of result
1548 * @return Pointer to result string, or NULL on error
1550 * Computes NFD (Normalization Form D) of the string at @p s. This implies
1551 * performing all canonical decompositions and then normalizing the order of
1552 * combining characters.
1554 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1558 * - utf32_decompose_canon().
1559 * - utf8_decompose_compat()
1560 * - utf8_compose_canon()
1562 char *utf8_decompose_canon(const char *s
, size_t ns
, size_t *ndp
) {
1563 utf8__transform(utf32_decompose_canon
);
1566 /** @brief Compatibility decompose @p [s,s+ns)
1567 * @param s Pointer to string
1568 * @param ns Length of string
1569 * @param ndp Where to store length of result
1570 * @return Pointer to result string, or NULL on error
1572 * Computes NFKD (Normalization Form KD) of the string at @p s. This implies
1573 * performing all canonical and compatibility decompositions and then
1574 * normalizing the order of combining characters.
1576 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1580 * - utf32_decompose_compat().
1581 * - utf8_decompose_canon()
1582 * - utf8_compose_compat()
1584 char *utf8_decompose_compat(const char *s
, size_t ns
, size_t *ndp
) {
1585 utf8__transform(utf32_decompose_compat
);
1588 /** @brief Canonically compose @p [s,s+ns)
1589 * @param s Pointer to string
1590 * @param ns Length of string
1591 * @param ndp Where to store length of result
1592 * @return Pointer to result string, or NULL on error
1594 * Computes NFC (Normalization Form C) of the string at @p s. This implies
1595 * performing all canonical decompositions, normalizing the order of combining
1596 * characters and then composing all unblocked primary compositables.
1598 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1602 * - utf32_compose_canon()
1603 * - utf8_compose_compat()
1604 * - utf8_decompose_canon()
1606 char *utf8_compose_canon(const char *s
, size_t ns
, size_t *ndp
) {
1607 utf8__transform(utf32_compose_canon
);
1610 /** @brief Compatibility compose @p [s,s+ns)
1611 * @param s Pointer to string
1612 * @param ns Length of string
1613 * @param ndp Where to store length of result
1614 * @return Pointer to result string, or NULL on error
1616 * Computes NFKC (Normalization Form KC) of the string at @p s. This implies
1617 * performing all canonical and compatibility decompositions, normalizing the
1618 * order of combining characters and then composing all unblocked primary
1621 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1625 * - utf32_compose_compat()
1626 * - utf8_compose_canon()
1627 * - utf8_decompose_compat()
1629 char *utf8_compose_compat(const char *s
, size_t ns
, size_t *ndp
) {
1630 utf8__transform(utf32_compose_compat
);
1633 /** @brief Case-fold @p [s,s+ns)
1634 * @param s Pointer to string
1635 * @param ns Length of string
1636 * @param ndp Where to store length of result
1637 * @return Pointer to result string, or NULL on error
1639 * Case-fold the string at @p s according to full default case-folding rules
1640 * (s3.13). The result will be in NFD.
1642 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1645 char *utf8_casefold_canon(const char *s
, size_t ns
, size_t *ndp
) {
1646 utf8__transform(utf32_casefold_canon
);
1649 /** @brief Compatibility case-fold @p [s,s+ns)
1650 * @param s Pointer to string
1651 * @param ns Length of string
1652 * @param ndp Where to store length of result
1653 * @return Pointer to result string, or NULL on error
1655 * Case-fold the string at @p s according to full default case-folding rules
1656 * (s3.13). The result will be in NFKD.
1658 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1661 char *utf8_casefold_compat(const char *s
, size_t ns
, size_t *ndp
) {
1662 utf8__transform(utf32_casefold_compat
);
1665 /** @brief Split [s,ns) into multiple words
1666 * @param s Pointer to start of string
1667 * @param ns Length of string
1668 * @param nwp Where to store word count, or NULL
1669 * @param wbreak Word_Break property tailor, or NULL
1670 * @return Pointer to array of pointers to words
1672 * The returned array is terminated by a NULL pointer and individual
1673 * strings are 0-terminated.
1675 char **utf8_word_split(const char *s
, size_t ns
, size_t *nwp
,
1676 unicode_property_tailor
*wbreak
) {
1677 uint32_t *to32
= 0, **v32
= 0;
1678 size_t nto32
, nv
, n
;
1679 char **v8
= 0, **ret
= 0;
1681 if(!(to32
= utf8_to_utf32(s
, ns
, &nto32
))) goto error
;
1682 if(!(v32
= utf32_word_split(to32
, nto32
, &nv
, wbreak
))) goto error
;
1683 v8
= xcalloc(sizeof (char *), nv
+ 1);
1684 for(n
= 0; n
< nv
; ++n
)
1685 if(!(v8
[n
] = utf32_to_utf8(v32
[n
], utf32_len(v32
[n
]), 0)))
1689 v8
= 0; /* don't free */
1692 for(n
= 0; n
< nv
; ++n
)
1697 for(n
= 0; n
< nv
; ++n
)
1708 /** @brief Return the length of a 0-terminated UTF-16 string
1709 * @param s Pointer to 0-terminated string
1710 * @return Length of string in code points (excluding terminator)
1712 * Unlike the conversion functions no validity checking is done on the string.
1714 size_t utf16_len(const uint16_t *s
) {
1715 const uint16_t *t
= s
;
1719 return (size_t)(t
- s
);
1727 indent-tabs-mode:nil