* @brief Unicode support functions
*
* Here by UTF-8 and UTF-8 we mean the encoding forms of those names (not the
- * encoding schemes).
+ * encoding schemes). The primary encoding form is UTF-32 but convenience
+ * wrappers using UTF-8 are provided for a number of functions.
*
* The idea is that all the strings that hit the database will be in a
* particular normalization form, and for the search and tags database
return *a < *b ? -1 : (*a > *b ? 1 : 0);
}
+/** @brief Return the General_Category value for @p c
+ * @param Code point
+ * @return General_Category property value
+ */
+static inline enum unicode_gc_cat utf32__general_category(uint32_t c) {
+ if(c < UNICODE_NCHARS) {
+ const struct unidata *const ud = &unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS];
+ return ud->gc;
+ } else
+ return unicode_gc_Cn;
+}
+
+/** @brief Check Grapheme_Cluster_Break property
+ * @param c Code point
+ * @return 0 if it is as described, 1 otherwise
+ */
+static int utf32__is_control_or_cr_or_lf(uint32_t c) {
+ switch(utf32__general_category(c)) {
+ default:
+ return 0;
+ case unicode_gc_Zl:
+ case unicode_gc_Zp:
+ case unicode_gc_Cc:
+ return 1;
+ case unicode_gc_Cf:
+ if(c == 0x200C || c == 0x200D)
+ return 0;
+ return 1;
+ }
+}
+
+#define Hangul_Syllable_Type_NA 0
+#define Hangul_Syllable_Type_L 0x1100
+#define Hangul_Syllable_Type_V 0x1160
+#define Hangul_Syllable_Type_T 0x11A8
+#define Hangul_Syllable_Type_LV 0xAC00
+#define Hangul_Syllable_Type_LVT 0xAC01
+
+/** @brief Determine Hangul_Syllable_Type of @p c
+ * @param c Code point
+ * @return Equivalance class of @p c, or Hangul_Syllable_Type_NA
+ *
+ * If this is a Hangul character then a representative member of its
+ * equivalence class is returned. Otherwise Hangul_Syllable_Type_NA is
+ * returned.
+ */
+static uint32_t utf32__hangul_syllable_type(uint32_t c) {
+ /* Dispose of the bulk of the non-Hangul code points first */
+ if(c < 0x1100) return Hangul_Syllable_Type_NA;
+ if(c > 0x1200 && c < 0xAC00) return Hangul_Syllable_Type_NA;
+ if(c >= 0xD800) return Hangul_Syllable_Type_NA;
+ /* Now we pick out the assigned Hangul code points */
+ if((c >= 0x1100 && c <= 0x1159) || c == 0x115F) return Hangul_Syllable_Type_L;
+ if(c >= 0x1160 && c <= 0x11A2) return Hangul_Syllable_Type_V;
+ if(c >= 0x11A8 && c <= 0x11F9) return Hangul_Syllable_Type_T;
+ if(c >= 0xAC00 && c <= 0xD7A3) {
+ if(c % 28 == 16)
+ return Hangul_Syllable_Type_LV;
+ else
+ return Hangul_Syllable_Type_LVT;
+ }
+ return Hangul_Syllable_Type_NA;
+}
+
+/** @brief Determine Word_Break property
+ * @param c Code point
+ * @return Word_Break property value of @p c
+ */
+static enum unicode_Word_Break utf32__word_break(uint32_t c) {
+ if(c < 0xAC00 || c > 0xD7A3) {
+ if(c < UNICODE_NCHARS)
+ return unidata[c / UNICODE_MODULUS][c % UNICODE_MODULUS].word_break;
+ else
+ return unicode_Word_Break_Other;
+ } else
+ return unicode_Word_Break_ALetter;
+}
+
+/** @brief Identify a grapheme cluster boundary
+ * @param s Start of string (must be NFD)
+ * @param ns Length of string
+ * @param n Index within string (in [0,ns].)
+ * @return 1 at a grapheme cluster boundary, 0 otherwise
+ *
+ * This function identifies default grapheme cluster boundaries as described in
+ * UAX #29 s3. It returns 1 if @p n points at the code point just after a
+ * grapheme cluster boundary (including the hypothetical code point just after
+ * the end of the string).
+ */
+int utf32_is_gcb(const uint32_t *s, size_t ns, size_t n) {
+ uint32_t before, after;
+ uint32_t hbefore, hafter;
+ /* GB1 and GB2 */
+ if(n == 0 || n == ns)
+ return 1;
+ /* Now we know that s[n-1] and s[n] are safe to inspect */
+ /* GB3 */
+ before = s[n-1];
+ after = s[n];
+ if(before == 0x000D && after == 0x000A)
+ return 0;
+ /* GB4 and GB5 */
+ if(utf32__is_control_or_cr_or_lf(before)
+ || utf32__is_control_or_cr_or_lf(after))
+ return 1;
+ hbefore = utf32__hangul_syllable_type(before);
+ hafter = utf32__hangul_syllable_type(after);
+ /* GB6 */
+ if(hbefore == Hangul_Syllable_Type_L
+ && (hafter == Hangul_Syllable_Type_L
+ || hafter == Hangul_Syllable_Type_V
+ || hafter == Hangul_Syllable_Type_LV
+ || hafter == Hangul_Syllable_Type_LVT))
+ return 0;
+ /* GB7 */
+ if((hbefore == Hangul_Syllable_Type_LV
+ || hbefore == Hangul_Syllable_Type_V)
+ && (hafter == Hangul_Syllable_Type_V
+ || hafter == Hangul_Syllable_Type_T))
+ return 0;
+ /* GB8 */
+ if((hbefore == Hangul_Syllable_Type_LVT
+ || hbefore == Hangul_Syllable_Type_T)
+ && hafter == Hangul_Syllable_Type_T)
+ return 0;
+ /* GB9 */
+ if(utf32__word_break(after) == unicode_Word_Break_Extend)
+ return 0;
+ /* GB10 */
+ return 1;
+}
+
+/** @brief Return true if code point @p n is part of an initial sequence of Format/Extend
+ * @param s Start of string
+ * @param ns Length of string
+ * @param n Start position
+ * @return True if it is, false otherwise
+ *
+ * This whole stack is not very efficient; we assume we don't see many of the
+ * problematic characters.
+ */
+static int utf32__is_initial_sequence(const uint32_t *s,
+ size_t attribute((unused)) ns,
+ size_t n) {
+ enum unicode_Word_Break wb;
+
+ while(n > 0) {
+ --n;
+ wb = utf32__word_break(s[n]);
+ if(wb != unicode_Word_Break_Extend
+ && wb != unicode_Word_Break_Format)
+ return 0;
+ }
+ return 1;
+}
+
+/** @brief Return the index of the first non-Extend/Format character from n
+ * @param s Start of string
+ * @param ns Length of string
+ * @param n Start position
+ * @return Index of first suitable character or @p ns
+ */
+static size_t utf32__first_not_ignorable(const uint32_t *s, size_t ns,
+ size_t n) {
+ while(n < ns) {
+ const enum unicode_Word_Break wb = utf32__word_break(s[n]);
+ if((wb != unicode_Word_Break_Extend
+ && wb != unicode_Word_Break_Format)
+ || utf32__is_initial_sequence(s, ns, n))
+ return n;
+ ++n;
+ }
+ return ns;
+}
+
+/** @brief Return the index of the last non-Extend/Format character from n
+ * @param s Start of string
+ * @param ns Length of string
+ * @param n Start position
+ * @return Index of first suitable character or (size_t)-1
+ */
+static size_t utf32__last_not_ignorable(const uint32_t *s, size_t ns,
+ size_t n) {
+ do {
+ const enum unicode_Word_Break wb = utf32__word_break(s[n]);
+ if((wb != unicode_Word_Break_Extend
+ && wb != unicode_Word_Break_Format)
+ || utf32__is_initial_sequence(s, ns, n))
+ return n;
+ } while(n--);
+ return n; /* will be (size_t)-1 */
+}
+
+/** @brief Identify a word boundary
+ * @param s Start of string (must be NFD)
+ * @param ns Length of string
+ * @param n Index within string (in [0,ns].)
+ * @return 1 at a word boundary, 0 otherwise
+ *
+ * This function identifies default word boundaries as described in UAX #29 s4.
+ * It returns 1 if @p n points at the code point just after a word boundary
+ * (including the hypothetical code point just after the end of the string).
+ */
+int utf32_is_word_boundary(const uint32_t *s, size_t ns, size_t n) {
+ enum unicode_Word_Break twobefore, before, after, twoafter;
+ size_t nn;
+
+ /* WB1 and WB2 */
+ if(n == 0 || n == ns)
+ return 1;
+ /* WB3 */
+ if(s[n-1] == 0x000D && s[n] == 0x000A)
+ return 0;
+ /* WB4 */
+ /* The stated requirement here is to ignore code points with a Word_Break
+ * value of _Extend or _Format wherever they may appear unless they are part
+ * of an initial sequence of such characters. */
+ twobefore = before = after = twoafter = unicode_Word_Break_Other;
+ nn = utf32__last_not_ignorable(s, ns, n - 1/* > 0 */);
+ if(nn != (size_t)-1) {
+ before = utf32__word_break(s[nn]);
+ if(nn != 0) {
+ nn = utf32__last_not_ignorable(s, ns, nn - 1);
+ if(nn != (size_t)-1)
+ twobefore = utf32__word_break(s[nn]);
+ }
+ }
+ nn = utf32__first_not_ignorable(s, ns, n);
+ if(nn < ns) {
+ after = utf32__word_break(s[nn]);
+ if(nn < ns) {
+ nn = utf32__first_not_ignorable(s, ns, nn + 1);
+ if(nn < ns)
+ twoafter = utf32__word_break(s[nn]);
+ }
+ }
+ /* WB5 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB6 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_MidLetter
+ && twoafter == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB7 */
+ if(twobefore == unicode_Word_Break_ALetter
+ && before == unicode_Word_Break_MidLetter
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB8 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB9 */
+ if(before == unicode_Word_Break_ALetter
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB10 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_ALetter)
+ return 0;
+ /* WB11 */
+ if(twobefore == unicode_Word_Break_Numeric
+ && before == unicode_Word_Break_MidNum
+ && after == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB12 */
+ if(before == unicode_Word_Break_Numeric
+ && after == unicode_Word_Break_MidNum
+ && twoafter == unicode_Word_Break_Numeric)
+ return 0;
+ /* WB13 */
+ if(before == unicode_Word_Break_Katakana
+ && after == unicode_Word_Break_Katakana)
+ return 0;
+ /* WB13a */
+ if((before == unicode_Word_Break_ALetter
+ || before == unicode_Word_Break_Numeric
+ || before == unicode_Word_Break_Katakana
+ || before == unicode_Word_Break_ExtendNumLet)
+ && after == unicode_Word_Break_ExtendNumLet)
+ return 0;
+ /* WB13b */
+ if(before == unicode_Word_Break_ExtendNumLet
+ && (after == unicode_Word_Break_ALetter
+ || after == unicode_Word_Break_Numeric
+ || after == unicode_Word_Break_Katakana))
+ return 0;
+ /* WB14 */
+ return 1;
+}
+
/*@}*/
/** @defgroup Functions that operate on UTF-8 strings */
/*@{*/
~mdw / disorder / blobdiff