X-Git-Url: https://git.distorted.org.uk/~mdw/disorder/blobdiff_plain/0b7052dac24e57edb180f07c0bd3479d397ebb74..c2e01e0ac91c61de69158113d2ab59f6831b2941:/lib/unicode.c diff --git a/lib/unicode.c b/lib/unicode.c index ab2bc2f..02b29fc 100644 --- a/lib/unicode.c +++ b/lib/unicode.c @@ -204,14 +204,36 @@ size_t utf32_len(const uint32_t *s) { return (size_t)(t - s); } +/** @brief Return the @ref unidata structure for code point @p c + * + * @p c can be any 32-bit value, a sensible value will be returned regardless. + */ +static const struct unidata *utf32__unidata(uint32_t c) { + /* The bottom half of the table contains almost everything of interest + * and we can just return the right thing straight away */ + if(c < UNICODE_BREAK_START) + return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS]; + /* Within the break everything is unassigned */ + if(c < UNICODE_BREAK_END) + return utf32__unidata(0xFFFF); /* guaranteed to be Cn */ + /* Planes 15 and 16 are (mostly) private use */ + if((c >= 0xF0000 && c <= 0xFFFFD) + || (c >= 0x100000 && c <= 0x10FFFD)) + return utf32__unidata(0xE000); /* first Co code point */ + /* Everything else above the break top is unassigned */ + if(c >= UNICODE_BREAK_TOP) + return utf32__unidata(0xFFFF); /* guaranteed to be Cn */ + /* Currently the rest is language tags and variation selectors */ + c -= (UNICODE_BREAK_END - UNICODE_BREAK_START); + return &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS]; +} + /** @brief Return the combining class of @p c * @param c Code point * @return Combining class of @p c */ static inline int utf32__combining_class(uint32_t c) { - if(c < UNICODE_NCHARS) - return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].ccc; - return 0; + return utf32__unidata(c)->ccc; } /** @brief Stably sort [s,s+ns) into descending order of combining class @@ -320,10 +342,7 @@ static int utf32__canonical_ordering(uint32_t *s, size_t ns) { /** @brief Guts of the decomposition lookup functions */ #define utf32__decompose_one_generic(WHICH) do { \ - const uint32_t *dc = \ - (c < UNICODE_NCHARS \ - ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].WHICH \ - : 0); \ + const uint32_t *dc = utf32__unidata(c)->WHICH; \ if(dc) { \ /* Found a canonical decomposition in the table */ \ while(*dc) \ @@ -425,10 +444,7 @@ uint32_t *utf32_decompose_compat(const uint32_t *s, size_t ns, size_t *ndp) { /** @brief Single-character case-fold and decompose operation */ #define utf32__casefold_one(WHICH) do { \ - const uint32_t *cf = \ - (c < UNICODE_NCHARS \ - ? unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].casefold \ - : 0); \ + const uint32_t *cf = utf32__unidata(c)->casefold; \ if(cf) { \ /* Found a case-fold mapping in the table */ \ while(*cf) \ @@ -461,13 +477,9 @@ uint32_t *utf32_casefold_canon(const uint32_t *s, size_t ns, size_t *ndp) { * normalize before we fold. In Unicode 5.0.0 this means 0345 COMBINING * GREEK YPOGEGRAMMENI in its decomposition and the various characters that * canonically decompose to it. */ - for(n = 0; n < ns; ++n) { - c = s[n]; - if(c < UNICODE_NCHARS - && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags - & unicode_normalize_before_casefold)) + for(n = 0; n < ns; ++n) + if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold) break; - } if(n < ns) { /* We need a preliminary decomposition */ if(!(ss = utf32_decompose_canon(s, ns, &ns))) @@ -513,13 +525,9 @@ uint32_t *utf32_casefold_compat(const uint32_t *s, size_t ns, size_t *ndp) { size_t n; uint32_t *ss = 0; - for(n = 0; n < ns; ++n) { - c = s[n]; - if(c < UNICODE_NCHARS - && (unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].flags - & unicode_normalize_before_casefold)) + for(n = 0; n < ns; ++n) + if(utf32__unidata(s[n])->flags & unicode_normalize_before_casefold) break; - } if(n < ns) { /* We need a preliminary _canonical_ decomposition */ if(!(ss = utf32_decompose_canon(s, ns, &ns))) @@ -577,78 +585,24 @@ int utf32_cmp(const uint32_t *a, const uint32_t *b) { * @param Code point * @return General_Category property value */ -static inline enum unicode_gc_cat utf32__general_category(uint32_t c) { - if(c < UNICODE_NCHARS) { - const struct unidata *const ud = &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS]; - return ud->gc; - } else - return unicode_gc_Cn; -} - -/** @brief Check Grapheme_Cluster_Break property - * @param c Code point - * @return 0 if it is as described, 1 otherwise - */ -static int utf32__is_control_or_cr_or_lf(uint32_t c) { - switch(utf32__general_category(c)) { - default: - return 0; - case unicode_gc_Zl: - case unicode_gc_Zp: - case unicode_gc_Cc: - return 1; - case unicode_gc_Cf: - if(c == 0x200C || c == 0x200D) - return 0; - return 1; - } +static inline enum unicode_General_Category utf32__general_category(uint32_t c) { + return utf32__unidata(c)->general_category; } -#define Hangul_Syllable_Type_NA 0 -#define Hangul_Syllable_Type_L 0x1100 -#define Hangul_Syllable_Type_V 0x1160 -#define Hangul_Syllable_Type_T 0x11A8 -#define Hangul_Syllable_Type_LV 0xAC00 -#define Hangul_Syllable_Type_LVT 0xAC01 - -/** @brief Determine Hangul_Syllable_Type of @p c +/** @brief Determine Grapheme_Break property * @param c Code point - * @return Equivalance class of @p c, or Hangul_Syllable_Type_NA - * - * If this is a Hangul character then a representative member of its - * equivalence class is returned. Otherwise Hangul_Syllable_Type_NA is - * returned. + * @return Grapheme_Break property value of @p c */ -static uint32_t utf32__hangul_syllable_type(uint32_t c) { - /* Dispose of the bulk of the non-Hangul code points first */ - if(c < 0x1100) return Hangul_Syllable_Type_NA; - if(c > 0x1200 && c < 0xAC00) return Hangul_Syllable_Type_NA; - if(c >= 0xD800) return Hangul_Syllable_Type_NA; - /* Now we pick out the assigned Hangul code points */ - if((c >= 0x1100 && c <= 0x1159) || c == 0x115F) return Hangul_Syllable_Type_L; - if(c >= 0x1160 && c <= 0x11A2) return Hangul_Syllable_Type_V; - if(c >= 0x11A8 && c <= 0x11F9) return Hangul_Syllable_Type_T; - if(c >= 0xAC00 && c <= 0xD7A3) { - if(c % 28 == 16) - return Hangul_Syllable_Type_LV; - else - return Hangul_Syllable_Type_LVT; - } - return Hangul_Syllable_Type_NA; +static inline enum unicode_Grapheme_Break utf32__grapheme_break(uint32_t c) { + return utf32__unidata(c)->grapheme_break; } /** @brief Determine Word_Break property * @param c Code point * @return Word_Break property value of @p c */ -static enum unicode_Word_Break utf32__word_break(uint32_t c) { - if(c < 0xAC00 || c > 0xD7A3) { - if(c < UNICODE_NCHARS) - return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].word_break; - else - return unicode_Word_Break_Other; - } else - return unicode_Word_Break_ALetter; +static inline enum unicode_Word_Break utf32__word_break(uint32_t c) { + return utf32__unidata(c)->word_break; } /** @brief Identify a grapheme cluster boundary @@ -662,9 +616,9 @@ static enum unicode_Word_Break utf32__word_break(uint32_t c) { * grapheme cluster boundary (including the hypothetical code point just after * the end of the string). */ -int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) { +int utf32_is_grapheme_boundary(const uint32_t *s, size_t ns, size_t n) { uint32_t before, after; - uint32_t hbefore, hafter; + enum unicode_Grapheme_Break gbbefore, gbafter; /* GB1 and GB2 */ if(n == 0 || n == ns) return 1; @@ -674,96 +628,47 @@ int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) { after = s[n]; if(before == 0x000D && after == 0x000A) return 0; - /* GB4 and GB5 */ - if(utf32__is_control_or_cr_or_lf(before) - || utf32__is_control_or_cr_or_lf(after)) + gbbefore = utf32__grapheme_break(before); + gbafter = utf32__grapheme_break(after); + /* GB4 */ + if(gbbefore == unicode_Grapheme_Break_Control + || before == 0x000D + || before == 0x000A) + return 1; + /* GB5 */ + if(gbafter == unicode_Grapheme_Break_Control + || after == 0x000D + || after == 0x000A) return 1; - hbefore = utf32__hangul_syllable_type(before); - hafter = utf32__hangul_syllable_type(after); /* GB6 */ - if(hbefore == Hangul_Syllable_Type_L - && (hafter == Hangul_Syllable_Type_L - || hafter == Hangul_Syllable_Type_V - || hafter == Hangul_Syllable_Type_LV - || hafter == Hangul_Syllable_Type_LVT)) + if(gbbefore == unicode_Grapheme_Break_L + && (gbafter == unicode_Grapheme_Break_L + || gbafter == unicode_Grapheme_Break_V + || gbafter == unicode_Grapheme_Break_LV + || gbafter == unicode_Grapheme_Break_LVT)) return 0; /* GB7 */ - if((hbefore == Hangul_Syllable_Type_LV - || hbefore == Hangul_Syllable_Type_V) - && (hafter == Hangul_Syllable_Type_V - || hafter == Hangul_Syllable_Type_T)) + if((gbbefore == unicode_Grapheme_Break_LV + || gbbefore == unicode_Grapheme_Break_V) + && (gbafter == unicode_Grapheme_Break_V + || gbafter == unicode_Grapheme_Break_T)) return 0; /* GB8 */ - if((hbefore == Hangul_Syllable_Type_LVT - || hbefore == Hangul_Syllable_Type_T) - && hafter == Hangul_Syllable_Type_T) + if((gbbefore == unicode_Grapheme_Break_LVT + || gbbefore == unicode_Grapheme_Break_T) + && gbafter == unicode_Grapheme_Break_T) return 0; /* GB9 */ - if(utf32__word_break(after) == unicode_Word_Break_Extend) + if(gbafter == unicode_Grapheme_Break_Extend) return 0; /* GB10 */ return 1; } -/** @brief Return true if code point @p n is part of an initial sequence of Format/Extend - * @param s Start of string - * @param ns Length of string - * @param n Start position - * @return True if it is, false otherwise - * - * This whole stack is not very efficient; we assume we don't see many of the - * problematic characters. - */ -static int utf32__is_initial_sequence(const uint32_t *s, - size_t attribute((unused)) ns, - size_t n) { - enum unicode_Word_Break wb; - - while(n > 0) { - --n; - wb = utf32__word_break(s[n]); - if(wb != unicode_Word_Break_Extend - && wb != unicode_Word_Break_Format) - return 0; - } - return 1; -} - -/** @brief Return the index of the first non-Extend/Format character from n - * @param s Start of string - * @param ns Length of string - * @param n Start position - * @return Index of first suitable character or @p ns - */ -static size_t utf32__first_not_ignorable(const uint32_t *s, size_t ns, - size_t n) { - while(n < ns) { - const enum unicode_Word_Break wb = utf32__word_break(s[n]); - if((wb != unicode_Word_Break_Extend - && wb != unicode_Word_Break_Format) - || utf32__is_initial_sequence(s, ns, n)) - return n; - ++n; - } - return ns; -} - -/** @brief Return the index of the last non-Extend/Format character from n - * @param s Start of string - * @param ns Length of string - * @param n Start position - * @return Index of first suitable character or (size_t)-1 - */ -static size_t utf32__last_not_ignorable(const uint32_t *s, size_t ns, - size_t n) { - do { - const enum unicode_Word_Break wb = utf32__word_break(s[n]); - if((wb != unicode_Word_Break_Extend - && wb != unicode_Word_Break_Format) - || utf32__is_initial_sequence(s, ns, n)) - return n; - } while(n--); - return n; /* will be (size_t)-1 */ +/** @brief Return true if @p c is ignorable for boundary specifications */ +static inline int utf32__boundary_ignorable(enum unicode_Word_Break wb) { + return (wb == unicode_Word_Break_Extend + || wb == unicode_Word_Break_Format); } /** @brief Identify a word boundary @@ -787,28 +692,58 @@ int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) { if(s[n-1] == 0x000D && s[n] == 0x000A) return 0; /* WB4 */ - /* The stated requirement here is to ignore code points with a Word_Break - * value of _Extend or _Format wherever they may appear unless they are part - * of an initial sequence of such characters. */ - twobefore = before = after = twoafter = unicode_Word_Break_Other; - nn = utf32__last_not_ignorable(s, ns, n - 1/* > 0 */); - if(nn != (size_t)-1) { - before = utf32__word_break(s[nn]); - if(nn != 0) { - nn = utf32__last_not_ignorable(s, ns, nn - 1); - if(nn != (size_t)-1) - twobefore = utf32__word_break(s[nn]); - } + /* (!Sep) x (Extend|Format) as in UAX #29 s6.2 */ + switch(s[n-1]) { /* bit of a bodge */ + case 0x000A: + case 0x000D: + case 0x0085: + case 0x2028: + case 0x2029: + break; + default: + if(utf32__boundary_ignorable(utf32__word_break(s[n]))) + return 0; + break; } - nn = utf32__first_not_ignorable(s, ns, n); - if(nn < ns) { - after = utf32__word_break(s[nn]); - if(nn < ns) { - nn = utf32__first_not_ignorable(s, ns, nn + 1); - if(nn < ns) - twoafter = utf32__word_break(s[nn]); - } + /* Gather the property values we'll need for the rest of the test taking the + * s6.2 changes into account */ + /* First we look at the code points after the proposed boundary */ + nn = n; /* X */ + while(nn < ns && utf32__boundary_ignorable(utf32__word_break(s[nn]))) + ++nn; } + /* It's possible now that nn=ns */ + if(nn < ns) + twoafter = utf32__word_break(s[nn]); + else + twoafter = unicode_Word_Break_Other; + + /* Next we look at the code points before the proposed boundary. This is a + * bit fiddlier. */ + nn = n; + while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1]))) + --nn; + if(nn == 0) { + /* s[nn] must be ignorable */ + before = utf32__word_break(s[nn]); + twobefore = unicode_Word_Break_Other; + } else { + /* s[nn] is ignorable or after the proposed boundary; but s[nn-1] is not + * ignorable. */ + before = utf32__word_break(s[nn - 1]); + --nn; + /* Repeat the exercise */ + while(nn > 0 && utf32__boundary_ignorable(utf32__word_break(s[nn - 1]))) + --nn; + if(nn == 0) + twobefore = utf32__word_break(s[nn]); + else + twobefore = utf32__word_break(s[nn - 1]); + } + /* WB5 */ if(before == unicode_Word_Break_ALetter && after == unicode_Word_Break_ALetter) @@ -867,7 +802,7 @@ int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) { } /*@}*/ -/** @defgroup Functions that operate on UTF-8 strings */ +/** @defgroup utf8 Functions that operate on UTF-8 strings */ /*@{*/ /** @brief Wrapper to transform a UTF-8 string using the UTF-32 function */