X-Git-Url: https://git.distorted.org.uk/~mdw/disorder/blobdiff_plain/56fd389cfb7656c00faa213c33ec52bb66c79bd2..bcf9ed7f5b44c177d927d147f87c5c08e377adfa:/lib/unicode.c diff --git a/lib/unicode.c b/lib/unicode.c index 749916a..5e2dacd 100644 --- a/lib/unicode.c +++ b/lib/unicode.c @@ -21,7 +21,8 @@ * @brief Unicode support functions * * Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the - * encoding schemes). + * encoding schemes). The primary encoding form is UTF-32 but convenience + * wrappers using UTF-8 are provided for a number of functions. * * The idea is that all the strings that hit the database will be in a * particular normalization form, and for the search and tags database @@ -203,14 +204,26 @@ size_t utf32_len(const uint32_t *s) { return (size_t)(t - s); } +/** @brief Return the @ref unidata structure for code point @p c + * + * @p c can be any 32-bit value, a sensible value will be returned regardless. + */ +static const struct unidata *utf32__unidata(uint32_t c) { + if(c < UNICODE_NCHARS) + return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS]; + else if((c >= 0xF0000 && c <= 0xFFFFD) + || (c >= 0x100000 && c <= 0x10FFFD)) + return utf32__unidata(0xE000); /* Co */ + else + return utf32__unidata(0xFFFF); /* Cn */ +} + /** @brief Return the combining class of @p c * @param c Code point * @return Combining class of @p c */ static inline int utf32__combining_class(uint32_t c) { - if(c < UNICODE_NCHARS) - return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].ccc; - return 0; + return utf32__unidata(c)->ccc; } /** @brief Stably sort [s,s+ns) into descending order of combining class @@ -319,10 +332,7 @@ static int utf32__canonical_ordering(uint32_t *s, size_t ns) { /** @brief Guts of the decomposition lookup functions */ #define utf32__decompose_one_generic(WHICH) do { \ - const uint32_t *dc = \ - (c < UNICODE_NCHARS \ - ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].WHICH \ - : 0); \ + const uint32_t *dc = utf32__unidata(c)->WHICH; \ if(dc) { \ /* Found a canonical decomposition in the table */ \ while(*dc) \ @@ -424,10 +434,7 @@ uint32_t *utf32_decompose_compat(const uint32_t *s, size_t ns, size_t *ndp) { /** @brief Single-character case-fold and decompose operation */ #define utf32__casefold_one(WHICH) do { \ - const uint32_t *cf = \ - (c < UNICODE_NCHARS \ - ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].casefold \ - : 0); \ + const uint32_t *cf = utf32__unidata(c)->casefold; \ if(cf) { \ /* Found a case-fold mapping in the table */ \ while(*cf) \ @@ -460,13 +467,9 @@ uint32_t *utf32_casefold_canon(const uint32_t *s, size_t ns, size_t *ndp) { * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING * GREEK YPOGEGRAMMENI in its decomposition and the various characters that * canonically decompose to it. */ - for(n = 0; n < ns; ++n) { - c = s[n]; - if(c < UNICODE_NCHARS - && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags - & unicode_normalize_before_casefold)) + for(n = 0; n < ns; ++n) + if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold) break; - } if(n < ns) { /* We need a preliminary decomposition */ if(!(ss = utf32_decompose_canon(s, ns, &ns))) @@ -512,13 +515,9 @@ uint32_t *utf32_casefold_compat(const uint32_t *s, size_t ns, size_t *ndp) { size_t n; uint32_t *ss = 0; - for(n = 0; n < ns; ++n) { - c = s[n]; - if(c < UNICODE_NCHARS - && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags - & unicode_normalize_before_casefold)) + for(n = 0; n < ns; ++n) + if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold) break; - } if(n < ns) { /* We need a preliminary _canonical_ decomposition */ if(!(ss = utf32_decompose_canon(s, ns, &ns))) @@ -572,8 +571,269 @@ int utf32_cmp(const uint32_t *a, const uint32_t *b) { return *a < *b ? -1 : (*a > *b ? 1 : 0); } +/** @brief Return the General_Category value for @p c + * @param Code point + * @return General_Category property value + */ +static inline enum unicode_General_Category utf32__general_category(uint32_t c) { + return utf32__unidata(c)->general_category; +} + +/** @brief Check Grapheme_Cluster_Break property + * @param c Code point + * @return 0 if it is as described, 1 otherwise + */ +static int utf32__is_control_or_cr_or_lf(uint32_t c) { + switch(utf32__general_category(c)) { + default: + return 0; + case unicode_General_Category_Zl: + case unicode_General_Category_Zp: + case unicode_General_Category_Cc: + return 1; + case unicode_General_Category_Cf: + if(c == 0x200C || c == 0x200D) + return 0; + return 1; + } +} + +#define Hangul_Syllable_Type_NA 0 +#define Hangul_Syllable_Type_L 0x1100 +#define Hangul_Syllable_Type_V 0x1160 +#define Hangul_Syllable_Type_T 0x11A8 +#define Hangul_Syllable_Type_LV 0xAC00 +#define Hangul_Syllable_Type_LVT 0xAC01 + +/** @brief Determine Hangul_Syllable_Type of @p c + * @param c Code point + * @return Equivalance class of @p c, or Hangul_Syllable_Type_NA + * + * If this is a Hangul character then a representative member of its + * equivalence class is returned. Otherwise Hangul_Syllable_Type_NA is + * returned. + */ +static uint32_t utf32__hangul_syllable_type(uint32_t c) { + /* Dispose of the bulk of the non-Hangul code points first */ + if(c < 0x1100) return Hangul_Syllable_Type_NA; + if(c > 0x1200 && c < 0xAC00) return Hangul_Syllable_Type_NA; + if(c >= 0xD800) return Hangul_Syllable_Type_NA; + /* Now we pick out the assigned Hangul code points */ + if((c >= 0x1100 && c <= 0x1159) || c == 0x115F) return Hangul_Syllable_Type_L; + if(c >= 0x1160 && c <= 0x11A2) return Hangul_Syllable_Type_V; + if(c >= 0x11A8 && c <= 0x11F9) return Hangul_Syllable_Type_T; + if(c >= 0xAC00 && c <= 0xD7A3) { + if(c % 28 == 16) + return Hangul_Syllable_Type_LV; + else + return Hangul_Syllable_Type_LVT; + } + return Hangul_Syllable_Type_NA; +} + +/** @brief Determine Word_Break property + * @param c Code point + * @return Word_Break property value of @p c + */ +static enum unicode_Word_Break utf32__word_break(uint32_t c) { + if(c < 0xAC00 || c > 0xD7A3) + return utf32__unidata(c)->word_break; + else + return unicode_Word_Break_ALetter; +} + +/** @brief Identify a grapheme cluster boundary + * @param s Start of string (must be NFD) + * @param ns Length of string + * @param n Index within string (in [0,ns].) + * @return 1 at a grapheme cluster boundary, 0 otherwise + * + * This function identifies default grapheme cluster boundaries as described in + * UAX #29 s3. It returns 1 if @p n points at the code point just after a + * grapheme cluster boundary (including the hypothetical code point just after + * the end of the string). + */ +int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) { + uint32_t before, after; + uint32_t hbefore, hafter; + /* GB1 and GB2 */ + if(n == 0 || n == ns) + return 1; + /* Now we know that s[n-1] and s[n] are safe to inspect */ + /* GB3 */ + before = s[n-1]; + after = s[n]; + if(before == 0x000D && after == 0x000A) + return 0; + /* GB4 and GB5 */ + if(utf32__is_control_or_cr_or_lf(before) + || utf32__is_control_or_cr_or_lf(after)) + return 1; + hbefore = utf32__hangul_syllable_type(before); + hafter = utf32__hangul_syllable_type(after); + /* GB6 */ + if(hbefore == Hangul_Syllable_Type_L + && (hafter == Hangul_Syllable_Type_L + || hafter == Hangul_Syllable_Type_V + || hafter == Hangul_Syllable_Type_LV + || hafter == Hangul_Syllable_Type_LVT)) + return 0; + /* GB7 */ + if((hbefore == Hangul_Syllable_Type_LV + || hbefore == Hangul_Syllable_Type_V) + && (hafter == Hangul_Syllable_Type_V + || hafter == Hangul_Syllable_Type_T)) + return 0; + /* GB8 */ + if((hbefore == Hangul_Syllable_Type_LVT + || hbefore == Hangul_Syllable_Type_T) + && hafter == Hangul_Syllable_Type_T) + return 0; + /* GB9 */ + if(utf32__word_break(after) == unicode_Word_Break_Extend) + return 0; + /* GB10 */ + return 1; +} + +/** @brief Return true if @p c is ignorable for boundary specifications */ +static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) { + return (wb == unicode_Word_Break_Extend + || wb == unicode_Word_Break_Format); +} + +/** @brief Identify a word boundary + * @param s Start of string (must be NFD) + * @param ns Length of string + * @param n Index within string (in [0,ns].) + * @return 1 at a word boundary, 0 otherwise + * + * This function identifies default word boundaries as described in UAX #29 s4. + * It returns 1 if @p n points at the code point just after a word boundary + * (including the hypothetical code point just after the end of the string). + */ +int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) { + enum unicode_Word_Break twobefore, before, after, twoafter; + size_t nn; + + /* WB1 and WB2 */ + if(n == 0 || n == ns) + return 1; + /* WB3 */ + if(s[n-1] == 0x000D && s[n] == 0x000A) + return 0; + /* WB4 */ + /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */ + switch(s[n-1]) { /* bit of a bodge */ + case 0x000A: + case 0x000D: + case 0x0085: + case 0x2028: + case 0x2029: + break; + default: + if(utf32__boundary_ignorable(utf32__word_break(s[n]))) + return 0; + break; + } + /* Gather the property values we'll need for the rest of the test taking the + * s6.2 changes into account */ + /* First we look at the code points after the proposed boundary */ + nn = n; /* X */ + while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn]))) + ++nn; + } + /* It's possible now that nn=ns */ + if(nn < ns) + twoafter = utf32__word_break(s[nn]); + else + twoafter = unicode_Word_Break_Other; + + /* Next we look at the code points before the proposed boundary. This is a + * bit fiddlier. */ + nn = n; + while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1]))) + --nn; + if(nn == 0) { + /* s[nn] must be ignorable */ + before = utf32__word_break(s[nn]); + twobefore = unicode_Word_Break_Other; + } else { + /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not + * ignorable. */ + before = utf32__word_break(s[nn - 1]); + --nn; + /* Repeat the exercise */ + while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1]))) + --nn; + if(nn == 0) + twobefore = utf32__word_break(s[nn]); + else + twobefore = utf32__word_break(s[nn - 1]); + } + + /* WB5 */ + if(before == unicode_Word_Break_ALetter + && after == unicode_Word_Break_ALetter) + return 0; + /* WB6 */ + if(before == unicode_Word_Break_ALetter + && after == unicode_Word_Break_MidLetter + && twoafter == unicode_Word_Break_ALetter) + return 0; + /* WB7 */ + if(twobefore == unicode_Word_Break_ALetter + && before == unicode_Word_Break_MidLetter + && after == unicode_Word_Break_ALetter) + return 0; + /* WB8 */ + if(before == unicode_Word_Break_Numeric + && after == unicode_Word_Break_Numeric) + return 0; + /* WB9 */ + if(before == unicode_Word_Break_ALetter + && after == unicode_Word_Break_Numeric) + return 0; + /* WB10 */ + if(before == unicode_Word_Break_Numeric + && after == unicode_Word_Break_ALetter) + return 0; + /* WB11 */ + if(twobefore == unicode_Word_Break_Numeric + && before == unicode_Word_Break_MidNum + && after == unicode_Word_Break_Numeric) + return 0; + /* WB12 */ + if(before == unicode_Word_Break_Numeric + && after == unicode_Word_Break_MidNum + && twoafter == unicode_Word_Break_Numeric) + return 0; + /* WB13 */ + if(before == unicode_Word_Break_Katakana + && after == unicode_Word_Break_Katakana) + return 0; + /* WB13a */ + if((before == unicode_Word_Break_ALetter + || before == unicode_Word_Break_Numeric + || before == unicode_Word_Break_Katakana + || before == unicode_Word_Break_ExtendNumLet) + && after == unicode_Word_Break_ExtendNumLet) + return 0; + /* WB13b */ + if(before == unicode_Word_Break_ExtendNumLet + && (after == unicode_Word_Break_ALetter + || after == unicode_Word_Break_Numeric + || after == unicode_Word_Break_Katakana)) + return 0; + /* WB14 */ + return 1; +} + /*@}*/ -/** @defgroup Functions that operate on UTF-8 strings */ +/** @defgroup utf8 Functions that operate on UTF-8 strings */ /*@{*/ /** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */