2 * This file is part of DisOrder
3 * Copyright (C) 2007 Richard Kettlewell
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
20 /** @file lib/unicode.c
21 * @brief Unicode support functions
23 * Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
24 * encoding schemes). The primary encoding form is UTF-32 but convenience
25 * wrappers using UTF-8 are provided for a number of functions.
27 * The idea is that all the strings that hit the database will be in a
28 * particular normalization form, and for the search and tags database
29 * in case-folded form, so they can be naively compared within the
32 * As the code stands this guarantee is not well met!
39 #include <stdio.h> /* TODO */
46 /** @defgroup utf32props Unicode Code Point Properties */
49 static const struct unidata
*utf32__unidata_hard(uint32_t c
);
51 /** @brief Find definition of code point @p c
53 * @return Pointer to @ref unidata structure for @p c
55 * @p c can be any 32-bit value, a sensible value will be returned regardless.
56 * The returned pointer is NOT guaranteed to be unique to @p c.
58 static inline const struct unidata
*utf32__unidata(uint32_t c
) {
59 /* The bottom half of the table contains almost everything of interest
60 * and we can just return the right thing straight away */
61 if(c
< UNICODE_BREAK_START
)
62 return &unidata
[c
/ UNICODE_MODULUS
][c
% UNICODE_MODULUS
];
64 return utf32__unidata_hard(c
);
67 /** @brief Find definition of code point @p c
69 * @return Pointer to @ref unidata structure for @p c
71 * @p c can be any 32-bit value, a sensible value will be returned regardless.
72 * The returned pointer is NOT guaranteed to be unique to @p c.
74 * Don't use this function (although it will work fine) - use utf32__unidata()
77 static const struct unidata
*utf32__unidata_hard(uint32_t c
) {
78 if(c
< UNICODE_BREAK_START
)
79 return &unidata
[c
/ UNICODE_MODULUS
][c
% UNICODE_MODULUS
];
80 /* Within the break everything is unassigned */
81 if(c
< UNICODE_BREAK_END
)
82 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
83 /* Planes 15 and 16 are (mostly) private use */
84 if((c
>= 0xF0000 && c
<= 0xFFFFD)
85 || (c
>= 0x100000 && c
<= 0x10FFFD))
86 return utf32__unidata(0xE000); /* first Co code point */
87 /* Everything else above the break top is unassigned */
88 if(c
>= UNICODE_BREAK_TOP
)
89 return utf32__unidata(0xFFFF); /* guaranteed to be Cn */
90 /* Currently the rest is language tags and variation selectors */
91 c
-= (UNICODE_BREAK_END
- UNICODE_BREAK_START
);
92 return &unidata
[c
/ UNICODE_MODULUS
][c
% UNICODE_MODULUS
];
95 /** @brief Return the combining class of @p c
97 * @return Combining class of @p c
99 * @p c can be any 32-bit value, a sensible value will be returned regardless.
101 static inline int utf32__combining_class(uint32_t c
) {
102 return utf32__unidata(c
)->ccc
;
105 /** @brief Return the General_Category value for @p c
107 * @return General_Category property value
109 * @p c can be any 32-bit value, a sensible value will be returned regardless.
111 static inline enum unicode_General_Category
utf32__general_category(uint32_t c
) {
112 return utf32__unidata(c
)->general_category
;
115 /** @brief Determine Grapheme_Break property
116 * @param c Code point
117 * @return Grapheme_Break property value of @p c
119 * @p c can be any 32-bit value, a sensible value will be returned regardless.
121 static inline enum unicode_Grapheme_Break
utf32__grapheme_break(uint32_t c
) {
122 return utf32__unidata(c
)->grapheme_break
;
125 /** @brief Determine Word_Break property
126 * @param c Code point
127 * @return Word_Break property value of @p c
129 * @p c can be any 32-bit value, a sensible value will be returned regardless.
131 static inline enum unicode_Word_Break
utf32__word_break(uint32_t c
) {
132 return utf32__unidata(c
)->word_break
;
135 /** @brief Determine Sentence_Break property
136 * @param c Code point
137 * @return Word_Break property value of @p c
139 * @p c can be any 32-bit value, a sensible value will be returned regardless.
141 static inline enum unicode_Sentence_Break
utf32__sentence_break(uint32_t c
) {
142 return utf32__unidata(c
)->sentence_break
;
145 /** @brief Return true if @p c is ignorable for boundary specifications
146 * @param wb Word break property value
147 * @return non-0 if @p wb is unicode_Word_Break_Extend or unicode_Word_Break_Format
149 static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb
) {
150 return (wb
== unicode_Word_Break_Extend
151 || wb
== unicode_Word_Break_Format
);
155 /** @defgroup utftransform Functions that transform between different Unicode encoding forms */
158 /** @brief Convert UTF-32 to UTF-8
159 * @param s Source string
160 * @param ns Length of source string in code points
161 * @param ndp Where to store length of destination string (or NULL)
162 * @return Newly allocated destination string or NULL on error
164 * If the UTF-32 is not valid then NULL is returned. A UTF-32 code point is
166 * - it codes for a UTF-16 surrogate
167 * - it codes for a value outside the unicode code space
169 * The return value is always 0-terminated. The value returned via @p *ndp
170 * does not include the terminator.
172 char *utf32_to_utf8(const uint32_t *s
, size_t ns
, size_t *ndp
) {
180 dynstr_append(&d
, c
);
181 else if(c
< 0x0800) {
182 dynstr_append(&d
, 0xC0 | (c
>> 6));
183 dynstr_append(&d
, 0x80 | (c
& 0x3F));
184 } else if(c
< 0x10000) {
185 if(c
>= 0xD800 && c
<= 0xDFFF)
187 dynstr_append(&d
, 0xE0 | (c
>> 12));
188 dynstr_append(&d
, 0x80 | ((c
>> 6) & 0x3F));
189 dynstr_append(&d
, 0x80 | (c
& 0x3F));
190 } else if(c
< 0x110000) {
191 dynstr_append(&d
, 0xF0 | (c
>> 18));
192 dynstr_append(&d
, 0x80 | ((c
>> 12) & 0x3F));
193 dynstr_append(&d
, 0x80 | ((c
>> 6) & 0x3F));
194 dynstr_append(&d
, 0x80 | (c
& 0x3F));
199 dynstr_terminate(&d
);
208 /** @brief Convert UTF-8 to UTF-32
209 * @param s Source string
210 * @param ns Length of source string in code points
211 * @param ndp Where to store length of destination string (or NULL)
212 * @return Newly allocated destination string or NULL
214 * The return value is always 0-terminated. The value returned via @p *ndp
215 * does not include the terminator.
217 * If the UTF-8 is not valid then NULL is returned. A UTF-8 sequence
218 * for a code point is invalid if:
219 * - it is not the shortest possible sequence for the code point
220 * - it codes for a UTF-16 surrogate
221 * - it codes for a value outside the unicode code space
223 uint32_t *utf8_to_utf32(const char *s
, size_t ns
, size_t *ndp
) {
224 struct dynstr_ucs4 d
;
226 const uint8_t *ss
= (const uint8_t *)s
;
229 dynstr_ucs4_init(&d
);
231 const struct unicode_utf8_row
*const r
= &unicode_utf8_valid
[*ss
];
238 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
243 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
248 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
257 for(n
= 1; n
< r
->count
; ++n
) {
258 if(ss
[n
] < 0x80 || ss
[n
] > 0xBF)
260 c32
= (c32
<< 6) | (ss
[n
] & 0x3F);
262 dynstr_ucs4_append(&d
, c32
);
266 dynstr_ucs4_terminate(&d
);
275 /** @brief Test whether [s,s+ns) is valid UTF-8
276 * @param s Start of string
277 * @param ns Length of string
278 * @return non-0 if @p s is valid UTF-8, 0 if it is not valid
280 * This function is intended to be much faster than calling utf8_to_utf32() and
281 * throwing away the result.
283 int utf8_valid(const char *s
, size_t ns
) {
284 const uint8_t *ss
= (const uint8_t *)s
;
286 const struct unicode_utf8_row
*const r
= &unicode_utf8_valid
[*ss
];
292 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
296 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
298 if(ss
[2] < 0x80 || ss
[2] > 0xBF)
302 if(ss
[1] < r
->min2
|| ss
[1] > r
->max2
)
304 if(ss
[2] < 0x80 || ss
[2] > 0xBF)
306 if(ss
[3] < 0x80 || ss
[3] > 0xBF)
321 /** @defgroup utf32iterator UTF-32 string iterators */
324 struct utf32_iterator_data
{
325 /** @brief Start of string */
328 /** @brief Length of string */
331 /** @brief Current position */
334 /** @brief Last two non-ignorable characters or (uint32_t)-1
336 * last[1] is the non-Extend/Format character just before position @p n;
337 * last[0] is the one just before that.
339 * Exception 1: if there is no such non-Extend/Format character then an
340 * Extend/Format character is accepted instead.
342 * Exception 2: if there is no such character even taking that into account
343 * the value is (uint32_t)-1.
348 /** @brief Create a new iterator pointing at the start of a string
349 * @param s Start of string
350 * @param ns Length of string
351 * @return New iterator
353 utf32_iterator
utf32_iterator_new(const uint32_t *s
, size_t ns
) {
354 utf32_iterator it
= xmalloc(sizeof *it
);
358 it
->last
[0] = it
->last
[1] = -1;
362 /** @brief Initialize an internal private iterator
364 * @param s Start of string
365 * @param ns Length of string
366 * @param n Absolute position
368 static void utf32__iterator_init(utf32_iterator it
,
369 const uint32_t *s
, size_t ns
, size_t n
) {
373 it
->last
[0] = it
->last
[1] = -1;
374 utf32_iterator_advance(it
, n
);
377 /** @brief Destroy an iterator
380 void utf32_iterator_destroy(utf32_iterator it
) {
384 /** @brief Find the current position of an interator
387 size_t utf32_iterator_where(utf32_iterator it
) {
391 /** @brief Set an iterator's absolute position
393 * @param n Absolute position
394 * @return 0 on success, non-0 on error
396 * It is an error to position the iterator outside the string (but acceptable
397 * to point it at the hypothetical post-final character). If an invalid value
398 * of @p n is specified then the iterator is not changed.
400 int utf32_iterator_set(utf32_iterator it
, size_t n
) {
401 /* We can't just jump to position @p n; the @p last[] values will be wrong.
402 * What we need is to jump a bit behind @p n and then advance forward,
403 * updating @p last[] along the way. How far back? We need to cross two
404 * non-ignorable code points as we advance forwards, so we'd better pass two
405 * such characters on the way back (if such are available).
410 if(n
> it
->ns
) /* range check */
412 for(i
= 0; i
< 2; ++i
)
414 && utf32__boundary_ignorable(utf32__word_break(it
->s
[m
- 1])))
417 return utf32_iterator_advance(it
, n
- m
);
420 /** @brief Advance an iterator
422 * @param count Number of code points to advance by
423 * @return 0 on success, non-0 on error
425 * It is an error to advance an iterator beyond the hypothetical post-final
426 * character of the string. If an invalid value of @p n is specified then the
427 * iterator is not changed.
429 * This function has O(n) time complexity: it works by advancing naively
430 * forwards through the string.
432 int utf32_iterator_advance(utf32_iterator it
, size_t count
) {
433 if(count
<= it
->ns
- it
->n
) {
435 const uint32_t c
= it
->s
[it
->n
];
436 const enum unicode_Word_Break wb
= utf32__word_break(c
);
437 if(it
->last
[1] == (uint32_t)-1
438 || !utf32__boundary_ignorable(wb
)) {
439 it
->last
[0] = it
->last
[1];
450 /** @brief Find the current code point
452 * @return Current code point or 0
454 * If the iterator points at the hypothetical post-final character of the
455 * string then 0 is returned. NB that this doesn't mean that there aren't any
456 * 0 code points inside the string!
458 uint32_t utf32_iterator_code(utf32_iterator it
) {
465 /** @brief Test for a grapheme boundary
467 * @return Non-0 if pointing just after a grapheme boundary, otherwise 0
469 int utf32_iterator_grapheme_boundary(utf32_iterator it
) {
470 uint32_t before
, after
;
471 enum unicode_Grapheme_Break gbbefore
, gbafter
;
473 if(it
->n
== 0 || it
->n
== it
->ns
)
475 /* Now we know that s[n-1] and s[n] are safe to inspect */
477 before
= it
->s
[it
->n
-1];
478 after
= it
->s
[it
->n
];
479 if(before
== 0x000D && after
== 0x000A)
481 gbbefore
= utf32__grapheme_break(before
);
482 gbafter
= utf32__grapheme_break(after
);
484 if(gbbefore
== unicode_Grapheme_Break_Control
489 if(gbafter
== unicode_Grapheme_Break_Control
494 if(gbbefore
== unicode_Grapheme_Break_L
495 && (gbafter
== unicode_Grapheme_Break_L
496 || gbafter
== unicode_Grapheme_Break_V
497 || gbafter
== unicode_Grapheme_Break_LV
498 || gbafter
== unicode_Grapheme_Break_LVT
))
501 if((gbbefore
== unicode_Grapheme_Break_LV
502 || gbbefore
== unicode_Grapheme_Break_V
)
503 && (gbafter
== unicode_Grapheme_Break_V
504 || gbafter
== unicode_Grapheme_Break_T
))
507 if((gbbefore
== unicode_Grapheme_Break_LVT
508 || gbbefore
== unicode_Grapheme_Break_T
)
509 && gbafter
== unicode_Grapheme_Break_T
)
512 if(gbafter
== unicode_Grapheme_Break_Extend
)
519 /** @brief Test for a word boundary
521 * @return Non-0 if pointing just after a word boundary, otherwise 0
523 int utf32_iterator_word_boundary(utf32_iterator it
) {
524 enum unicode_Word_Break twobefore
, before
, after
, twoafter
;
528 if(it
->n
== 0 || it
->n
== it
->ns
)
531 if(it
->s
[it
->n
-1] == 0x000D && it
->s
[it
->n
] == 0x000A)
534 /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */
535 if(utf32__sentence_break(it
->s
[it
->n
-1]) != unicode_Sentence_Break_Sep
536 && utf32__boundary_ignorable(utf32__word_break(it
->s
[it
->n
])))
538 /* Gather the property values we'll need for the rest of the test taking the
539 * s6.2 changes into account */
540 /* First we look at the code points after the proposed boundary */
541 nn
= it
->n
; /* <it->ns */
542 after
= utf32__word_break(it
->s
[nn
++]);
543 if(!utf32__boundary_ignorable(after
)) {
544 /* X (Extend|Format)* -> X */
546 && utf32__boundary_ignorable(utf32__word_break(it
->s
[nn
])))
549 /* It's possible now that nn=ns */
551 twoafter
= utf32__word_break(it
->s
[nn
]);
553 twoafter
= unicode_Word_Break_Other
;
555 /* We've already recorded the non-ignorable code points before the proposed
557 before
= utf32__word_break(it
->last
[1]);
558 twobefore
= utf32__word_break(it
->last
[0]);
561 if(before
== unicode_Word_Break_ALetter
562 && after
== unicode_Word_Break_ALetter
)
565 if(before
== unicode_Word_Break_ALetter
566 && after
== unicode_Word_Break_MidLetter
567 && twoafter
== unicode_Word_Break_ALetter
)
570 if(twobefore
== unicode_Word_Break_ALetter
571 && before
== unicode_Word_Break_MidLetter
572 && after
== unicode_Word_Break_ALetter
)
575 if(before
== unicode_Word_Break_Numeric
576 && after
== unicode_Word_Break_Numeric
)
579 if(before
== unicode_Word_Break_ALetter
580 && after
== unicode_Word_Break_Numeric
)
583 if(before
== unicode_Word_Break_Numeric
584 && after
== unicode_Word_Break_ALetter
)
587 if(twobefore
== unicode_Word_Break_Numeric
588 && before
== unicode_Word_Break_MidNum
589 && after
== unicode_Word_Break_Numeric
)
592 if(before
== unicode_Word_Break_Numeric
593 && after
== unicode_Word_Break_MidNum
594 && twoafter
== unicode_Word_Break_Numeric
)
597 if(before
== unicode_Word_Break_Katakana
598 && after
== unicode_Word_Break_Katakana
)
601 if((before
== unicode_Word_Break_ALetter
602 || before
== unicode_Word_Break_Numeric
603 || before
== unicode_Word_Break_Katakana
604 || before
== unicode_Word_Break_ExtendNumLet
)
605 && after
== unicode_Word_Break_ExtendNumLet
)
608 if(before
== unicode_Word_Break_ExtendNumLet
609 && (after
== unicode_Word_Break_ALetter
610 || after
== unicode_Word_Break_Numeric
611 || after
== unicode_Word_Break_Katakana
))
618 /** @defgroup utf32 Functions that operate on UTF-32 strings */
621 /** @brief Return the length of a 0-terminated UTF-32 string
622 * @param s Pointer to 0-terminated string
623 * @return Length of string in code points (excluding terminator)
625 * Unlike the conversion functions no validity checking is done on the string.
627 size_t utf32_len(const uint32_t *s
) {
628 const uint32_t *t
= s
;
632 return (size_t)(t
- s
);
635 /** @brief Stably sort [s,s+ns) into descending order of combining class
636 * @param s Start of array
637 * @param ns Number of elements, must be at least 1
638 * @param buffer Buffer of at least @p ns elements
640 static void utf32__sort_ccc(uint32_t *s
, size_t ns
, uint32_t *buffer
) {
641 uint32_t *a
, *b
, *bp
;
645 case 1: /* 1-element array is always sorted */
647 case 2: /* 2-element arrays are trivial to sort */
648 if(utf32__combining_class(s
[0]) > utf32__combining_class(s
[1])) {
655 /* Partition the array */
660 /* Sort the two halves of the array */
661 utf32__sort_ccc(a
, na
, buffer
);
662 utf32__sort_ccc(b
, nb
, buffer
);
663 /* Merge them back into one, via the buffer */
665 while(na
> 0 && nb
> 0) {
666 /* We want descending order of combining class (hence <)
667 * and we want stability within combining classes (hence <=)
669 if(utf32__combining_class(*a
) <= utf32__combining_class(*b
)) {
685 memcpy(s
, buffer
, ns
* sizeof(uint32_t));
690 /** @brief Put combining characters into canonical order
691 * @param s Pointer to UTF-32 string
692 * @param ns Length of @p s
693 * @return 0 on success, -1 on error
695 * @p s is modified in-place. See Unicode 5.0 s3.11 for details of the
698 * Currently we only support a maximum of 1024 combining characters after each
699 * base character. If this limit is exceeded then -1 is returned.
701 static int utf32__canonical_ordering(uint32_t *s
, size_t ns
) {
703 uint32_t buffer
[1024];
705 /* The ordering amounts to a stable sort of each contiguous group of
706 * characters with non-0 combining class. */
708 /* Skip non-combining characters */
709 if(utf32__combining_class(*s
) == 0) {
714 /* We must now have at least one combining character; see how many
716 for(nc
= 1; nc
< ns
&& utf32__combining_class(s
[nc
]) != 0; ++nc
)
721 utf32__sort_ccc(s
, nc
, buffer
);
728 /* Magic numbers from UAX #15 s16 */
736 #define NCount (VCount * TCount)
737 #define SCount (LCount * NCount)
739 /** @brief Guts of the decomposition lookup functions */
740 #define utf32__decompose_one_generic(WHICH) do { \
741 const uint32_t *dc = utf32__unidata(c)->WHICH; \
743 /* Found a canonical decomposition in the table */ \
745 utf32__decompose_one_##WHICH(d, *dc++); \
746 } else if(c >= SBase && c < SBase + SCount) { \
747 /* Mechanically decomposable Hangul syllable (UAX #15 s16) */ \
748 const uint32_t SIndex = c - SBase; \
749 const uint32_t L = LBase + SIndex / NCount; \
750 const uint32_t V = VBase + (SIndex % NCount) / TCount; \
751 const uint32_t T = TBase + SIndex % TCount; \
752 dynstr_ucs4_append(d, L); \
753 dynstr_ucs4_append(d, V); \
755 dynstr_ucs4_append(d, T); \
757 /* Equal to own canonical decomposition */ \
758 dynstr_ucs4_append(d, c); \
761 /** @brief Recursively compute the canonical decomposition of @p c
762 * @param d Dynamic string to store decomposition in
763 * @param c Code point to decompose (must be a valid!)
764 * @return 0 on success, -1 on error
766 static void utf32__decompose_one_canon(struct dynstr_ucs4
*d
, uint32_t c
) {
767 utf32__decompose_one_generic(canon
);
770 /** @brief Recursively compute the compatibility decomposition of @p c
771 * @param d Dynamic string to store decomposition in
772 * @param c Code point to decompose (must be a valid!)
773 * @return 0 on success, -1 on error
775 static void utf32__decompose_one_compat(struct dynstr_ucs4
*d
, uint32_t c
) {
776 utf32__decompose_one_generic(compat
);
779 /** @brief Guts of the decomposition functions */
780 #define utf32__decompose_generic(WHICH) do { \
781 struct dynstr_ucs4 d; \
784 dynstr_ucs4_init(&d); \
787 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
789 utf32__decompose_one_##WHICH(&d, c); \
792 if(utf32__canonical_ordering(d.vec, d.nvec)) \
794 dynstr_ucs4_terminate(&d); \
803 /** @brief Canonically decompose @p [s,s+ns)
804 * @param s Pointer to string
805 * @param ns Length of string
806 * @param ndp Where to store length of result
807 * @return Pointer to result string, or NULL
809 * Computes the canonical decomposition of a string and stably sorts combining
810 * characters into canonical order. The result is in Normalization Form D and
811 * (at the time of writing!) passes the NFD tests defined in Unicode 5.0's
812 * NormalizationTest.txt.
814 * Returns NULL if the string is not valid for either of the following reasons:
815 * - it codes for a UTF-16 surrogate
816 * - it codes for a value outside the unicode code space
818 uint32_t *utf32_decompose_canon(const uint32_t *s
, size_t ns
, size_t *ndp
) {
819 utf32__decompose_generic(canon
);
822 /** @brief Compatibility decompose @p [s,s+ns)
823 * @param s Pointer to string
824 * @param ns Length of string
825 * @param ndp Where to store length of result
826 * @return Pointer to result string, or NULL
828 * Computes the compatibility decomposition of a string and stably sorts
829 * combining characters into canonical order. The result is in Normalization
830 * Form KD and (at the time of writing!) passes the NFKD tests defined in
831 * Unicode 5.0's NormalizationTest.txt.
833 * Returns NULL if the string is not valid for either of the following reasons:
834 * - it codes for a UTF-16 surrogate
835 * - it codes for a value outside the unicode code space
837 uint32_t *utf32_decompose_compat(const uint32_t *s
, size_t ns
, size_t *ndp
) {
838 utf32__decompose_generic(compat
);
841 /** @brief Single-character case-fold and decompose operation */
842 #define utf32__casefold_one(WHICH) do { \
843 const uint32_t *cf = utf32__unidata(c)->casefold; \
845 /* Found a case-fold mapping in the table */ \
847 utf32__decompose_one_##WHICH(&d, *cf++); \
849 utf32__decompose_one_##WHICH(&d, c); \
852 /** @brief Case-fold @p [s,s+ns)
853 * @param s Pointer to string
854 * @param ns Length of string
855 * @param ndp Where to store length of result
856 * @return Pointer to result string, or NULL
858 * Case-fold the string at @p s according to full default case-folding rules
859 * (s3.13) for caseless matching. The result will be in NFD.
861 * Returns NULL if the string is not valid for either of the following reasons:
862 * - it codes for a UTF-16 surrogate
863 * - it codes for a value outside the unicode code space
865 uint32_t *utf32_casefold_canon(const uint32_t *s
, size_t ns
, size_t *ndp
) {
866 struct dynstr_ucs4 d
;
871 /* If the canonical decomposition of the string includes any combining
872 * character that case-folds to a non-combining character then we must
873 * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING
874 * GREEK YPOGEGRAMMENI in its decomposition and the various characters that
875 * canonically decompose to it. */
876 for(n
= 0; n
< ns
; ++n
)
877 if(utf32__unidata(s
[n
])->flags
& unicode_normalize_before_casefold
)
880 /* We need a preliminary decomposition */
881 if(!(ss
= utf32_decompose_canon(s
, ns
, &ns
)))
885 dynstr_ucs4_init(&d
);
888 if((c
>= 0xD800 && c
<= 0xDFFF) || c
> 0x10FFFF)
890 utf32__casefold_one(canon
);
893 if(utf32__canonical_ordering(d
.vec
, d
.nvec
))
895 dynstr_ucs4_terminate(&d
);
905 /** @brief Compatibilit case-fold @p [s,s+ns)
906 * @param s Pointer to string
907 * @param ns Length of string
908 * @param ndp Where to store length of result
909 * @return Pointer to result string, or NULL
911 * Case-fold the string at @p s according to full default case-folding rules
912 * (s3.13) for compatibility caseless matching. The result will be in NFKD.
914 * Returns NULL if the string is not valid for either of the following reasons:
915 * - it codes for a UTF-16 surrogate
916 * - it codes for a value outside the unicode code space
918 uint32_t *utf32_casefold_compat(const uint32_t *s
, size_t ns
, size_t *ndp
) {
919 struct dynstr_ucs4 d
;
924 for(n
= 0; n
< ns
; ++n
)
925 if(utf32__unidata(s
[n
])->flags
& unicode_normalize_before_casefold
)
928 /* We need a preliminary _canonical_ decomposition */
929 if(!(ss
= utf32_decompose_canon(s
, ns
, &ns
)))
933 /* This computes NFKD(toCaseFold(s)) */
934 #define compat_casefold_middle() do { \
935 dynstr_ucs4_init(&d); \
938 if((c >= 0xD800 && c <= 0xDFFF) || c > 0x10FFFF) \
940 utf32__casefold_one(compat); \
943 if(utf32__canonical_ordering(d.vec, d.nvec)) \
946 /* Do the inner (NFKD o toCaseFold) */
947 compat_casefold_middle();
948 /* We can do away with the NFD'd copy of the input now */
952 /* Do the outer (NFKD o toCaseFold) */
953 compat_casefold_middle();
955 dynstr_ucs4_terminate(&d
);
965 /** @brief Order a pair of UTF-32 strings
966 * @param a First 0-terminated string
967 * @param b Second 0-terminated string
968 * @return -1, 0 or 1 for a less than, equal to or greater than b
970 * "Comparable to strcmp() at its best."
972 int utf32_cmp(const uint32_t *a
, const uint32_t *b
) {
973 while(*a
&& *b
&& *a
== *b
) {
977 return *a
< *b ?
-1 : (*a
> *b ?
1 : 0);
980 /** @brief Identify a grapheme cluster boundary
981 * @param s Start of string (must be NFD)
982 * @param ns Length of string
983 * @param n Index within string (in [0,ns].)
984 * @return 1 at a grapheme cluster boundary, 0 otherwise
986 * This function identifies default grapheme cluster boundaries as described in
987 * UAX #29 s3. It returns 1 if @p n points at the code point just after a
988 * grapheme cluster boundary (including the hypothetical code point just after
989 * the end of the string).
991 int utf32_is_grapheme_boundary(const uint32_t *s
, size_t ns
, size_t n
) {
992 struct utf32_iterator_data it
[1];
994 utf32__iterator_init(it
, s
, ns
, n
);
995 return utf32_iterator_grapheme_boundary(it
);
998 /** @brief Identify a word boundary
999 * @param s Start of string (must be NFD)
1000 * @param ns Length of string
1001 * @param n Index within string (in [0,ns].)
1002 * @return 1 at a word boundary, 0 otherwise
1004 * This function identifies default word boundaries as described in UAX #29 s4.
1005 * It returns 1 if @p n points at the code point just after a word boundary
1006 * (including the hypothetical code point just after the end of the string).
1008 int utf32_is_word_boundary(const uint32_t *s
, size_t ns
, size_t n
) {
1009 struct utf32_iterator_data it
[1];
1011 utf32__iterator_init(it
, s
, ns
, n
);
1012 return utf32_iterator_word_boundary(it
);
1016 /** @defgroup utf8 Functions that operate on UTF-8 strings */
1019 /** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */
1020 #define utf8__transform(FN) do { \
1021 uint32_t *to32 = 0, *decomp32 = 0; \
1022 size_t nto32, ndecomp32; \
1023 char *decomp8 = 0; \
1025 if(!(to32 = utf8_to_utf32(s, ns, &nto32))) goto error; \
1026 if(!(decomp32 = FN(to32, nto32, &ndecomp32))) goto error; \
1027 decomp8 = utf32_to_utf8(decomp32, ndecomp32, ndp); \
1034 /** @brief Canonically decompose @p [s,s+ns)
1035 * @param s Pointer to string
1036 * @param ns Length of string
1037 * @param ndp Where to store length of result
1038 * @return Pointer to result string, or NULL
1040 * Computes the canonical decomposition of a string and stably sorts combining
1041 * characters into canonical order. The result is in Normalization Form D and
1042 * (at the time of writing!) passes the NFD tests defined in Unicode 5.0's
1043 * NormalizationTest.txt.
1045 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1048 * See also utf32_decompose_canon().
1050 char *utf8_decompose_canon(const char *s
, size_t ns
, size_t *ndp
) {
1051 utf8__transform(utf32_decompose_canon
);
1054 /** @brief Compatibility decompose @p [s,s+ns)
1055 * @param s Pointer to string
1056 * @param ns Length of string
1057 * @param ndp Where to store length of result
1058 * @return Pointer to result string, or NULL
1060 * Computes the compatibility decomposition of a string and stably sorts
1061 * combining characters into canonical order. The result is in Normalization
1062 * Form KD and (at the time of writing!) passes the NFKD tests defined in
1063 * Unicode 5.0's NormalizationTest.txt.
1065 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1068 * See also utf32_decompose_compat().
1070 char *utf8_decompose_compat(const char *s
, size_t ns
, size_t *ndp
) {
1071 utf8__transform(utf32_decompose_compat
);
1074 /** @brief Case-fold @p [s,s+ns)
1075 * @param s Pointer to string
1076 * @param ns Length of string
1077 * @param ndp Where to store length of result
1078 * @return Pointer to result string, or NULL
1080 * Case-fold the string at @p s according to full default case-folding rules
1081 * (s3.13). The result will be in NFD.
1083 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1086 char *utf8_casefold_canon(const char *s
, size_t ns
, size_t *ndp
) {
1087 utf8__transform(utf32_casefold_canon
);
1090 /** @brief Compatibility case-fold @p [s,s+ns)
1091 * @param s Pointer to string
1092 * @param ns Length of string
1093 * @param ndp Where to store length of result
1094 * @return Pointer to result string, or NULL
1096 * Case-fold the string at @p s according to full default case-folding rules
1097 * (s3.13). The result will be in NFKD.
1099 * Returns NULL if the string is not valid; see utf8_to_utf32() for reasons why
1102 char *utf8_casefold_compat(const char *s
, size_t ns
, size_t *ndp
) {
1103 utf8__transform(utf32_casefold_compat
);
1113 indent-tabs-mode:nil