[tracker/parser-unicode-libs-review: 80/85] Added first non-tested implementation of the libicu based word breaking and processing



commit 8ae4a0bec0c9d53fe4feddb4e64941ae6caffe95
Author: Aleksander Morgado <aleksander lanedo com>
Date:   Tue May 4 09:42:00 2010 +0200

    Added first non-tested implementation of the libicu based word breaking and processing

 src/libtracker-fts/tracker-parser-libicu.c |  539 +++++++++++++++++-----------
 1 files changed, 334 insertions(+), 205 deletions(-)
---
diff --git a/src/libtracker-fts/tracker-parser-libicu.c b/src/libtracker-fts/tracker-parser-libicu.c
index e29e0ea..9089dca 100644
--- a/src/libtracker-fts/tracker-parser-libicu.c
+++ b/src/libtracker-fts/tracker-parser-libicu.c
@@ -24,13 +24,18 @@
 #include <string.h>
 #include <locale.h>
 
-#include <ubrk.h>
+#include <unicode/utypes.h>
+#include <unicode/ucnv.h>
+#include <unicode/ubrk.h>
+#include <unicode/ustring.h>
+#include <unicode/uchar.h>
+#include <unicode/unorm.h>
 
 #include "tracker-parser.h"
 #include "tracker-parser-utils.h"
 
 /* ASCII-7 is in range [0x00,0x7F] */
-#define IS_ASCII_BYTE(c) ((c) <= 0x7F)
+#define IS_ASCII_UCS4(c) ((c) <= 0x7F)
 
 /* CJK ranges are : [0x3400,0x4DB5], [0x4E00,0x9FA5], [0x20000,0x2A6D6]  */
 #define IS_CJK_UCS4(c)   (((c) >= 0x3400 && (c) <= 0x4DB5)  || \
@@ -41,6 +46,12 @@
 #define WORD_BUFFER_LENGTH 512
 
 
+static gchar *process_word_uchar (TrackerParser *parser,
+                                  const UChar   *word,
+                                  gint           length,
+                                  gboolean       do_strip);
+
+
 struct TrackerParser {
 	const gchar           *txt;
 	gint                   txt_size;
@@ -61,6 +72,8 @@ struct TrackerParser {
 	/* Text as UChars */
 	UChar                 *utxt;
 	gint                   utxt_size;
+	/* Original offset of each UChar in the input txt string */
+	gint32                *offsets;
 
 	/* The word-break iterator */
 	UBreakIterator        *bi;
@@ -69,32 +82,83 @@ struct TrackerParser {
 	gsize                  cursor;
 };
 
-/* Detect if a UTF-8 word is pure ASCII-7, so that there is no need to apply
- *  UNAC stripping.
- * Just check byte per byte, and if any of the bytes is >127, then it's not
- *  ASCII-7 */
+
 static gboolean
-is_ascii_word (const gchar *word,
-               gsize        length)
+get_word_info (const UChar *word,
+               gsize        word_length,
+               gboolean    *p_is_allowed_word_start,
+               gboolean    *p_is_ascii_or_cjk)
 {
-	gsize i;
+	UCharIterator iter;
+	UChar32 unichar;
+	guint8 unichar_gc;
+
+	*p_is_allowed_word_start = FALSE;
+	*p_is_ascii_or_cjk = FALSE;
+
+	/* Get first character of the word as UCS4 */
+	uiter_setString (&iter, word, word_length);
+	unichar = uiter_current32 (&iter);
+	if (unichar == U_SENTINEL) {
+		return FALSE;
+	}
+
+	/* We only want the words where the first character
+	 *  in the word is either a letter, a number or a symbol.
+	 * This is needed because the word break algorithm also
+	 *  considers word breaks after for example commas or other
+	 *  punctuation marks.
+	 * Note that looking at the first character in the string
+	 *  should be compatible with all Unicode normalization
+	 *  methods.
+	 */
+	unichar_gc = u_charType (unichar);
+	if (unichar_gc != U_UPPERCASE_LETTER &&
+	    unichar_gc != U_LOWERCASE_LETTER &&
+	    unichar_gc != U_TITLECASE_LETTER &&
+	    unichar_gc != U_MODIFIER_LETTER &&
+	    unichar_gc != U_OTHER_LETTER &&
+	    unichar_gc != U_DECIMAL_DIGIT_NUMBER &&
+	    unichar_gc != U_LETTER_NUMBER &&
+	    unichar_gc != U_OTHER_NUMBER &&
+	    unichar_gc != U_MATH_SYMBOL &&
+	    unichar_gc != U_CURRENCY_SYMBOL &&
+	    unichar_gc != U_MODIFIER_SYMBOL &&
+	    unichar_gc != U_OTHER_SYMBOL) {
+		*p_is_allowed_word_start = FALSE;
+		return TRUE;
+	}
+
+	/* Word starts with a CJK character? */
+	if (IS_CJK_UCS4 ((guint32)unichar)) {
+		*p_is_ascii_or_cjk = TRUE;
+		return TRUE;
+	}
 
-	for (i = 0; i < length; i++) {
-		if (!IS_ASCII_BYTE ((guchar)word[i])) {
-			return FALSE;
+	/* Is ASCII-only string? */
+	while (unichar != U_SENTINEL)
+	{
+		if (!IS_ASCII_UCS4 ((guint32)unichar)) {
+			*p_is_ascii_or_cjk = TRUE;
+			return TRUE;
 		}
+		unichar = uiter_next32 (&iter);
 	}
+
 	return TRUE;
 }
 
+
 /* libunistring-based parser */
 static gboolean
 parser_next (TrackerParser *parser,
              gint          *byte_offset_start,
              gint          *byte_offset_end)
 {
-	gsize word_length = 0;
+	gsize word_length_uchar = 0;
+	gsize word_length_utf8 = 0;
 	gchar *processed_word = NULL;
+	gsize current_word_offset_utf8;
 
 	*byte_offset_start = 0;
 	*byte_offset_end = 0;
@@ -103,81 +167,72 @@ parser_next (TrackerParser *parser,
 
 	/* Loop to look for next valid word */
 	while (!processed_word &&
-	       parser->cursor < parser->txt_size) {
-		ucs4_t first_unichar;
-		gint first_unichar_len;
-		gsize i;
+	       parser->cursor < parser->utxt_size) {
+		gboolean is_ascii_or_cjk;
+		gboolean is_allowed;
+		gsize next_word_offset_uchar;
+		gsize next_word_offset_utf8;
 		gsize truncated_length;
-		gboolean do_strip;
-
-		/* Get first character of the word as UCS4 */
-		first_unichar_len = u8_strmbtouc (&first_unichar,
-		                                  &(parser->txt[parser->cursor]));
-		if (first_unichar_len <= 0) {
-			/* This should only happen if NIL was passed to u8_strmbtouc,
-			 *  so better just force stop here */
-			parser->cursor = parser->txt_size;
-			break;
-		}
 
-		/* Find next word break */
-		i = parser->cursor + first_unichar_len;
-		while (i < parser->txt_size &&
-		       !parser->word_break_flags [i]) {
-			i++;
+		/* Set current word offset in the original UTF-8 string */
+		current_word_offset_utf8 = parser->offsets[parser->cursor];
+
+		/* Find next word break. */
+		next_word_offset_uchar = ubrk_next (parser->bi);
+		if (next_word_offset_uchar == UBRK_DONE) {
+			/* Last word support... */
+			next_word_offset_uchar = parser->utxt_size;
+			next_word_offset_utf8 = parser->txt_size;
+		}
+		else {
+			next_word_offset_utf8 = parser->offsets[next_word_offset_uchar];
 		}
 
 		/* Word end is the first byte after the word, which is either the
 		 *  start of next word or the end of the string */
-		word_length = i - parser->cursor;
-
-		/* We only want the words where the first character
-		 *  in the word is either a letter, a number or a symbol.
-		 * This is needed because the word break algorithm also
-		 *  considers word breaks after for example commas or other
-		 *  punctuation marks.
-		 * Note that looking at the first character in the string
-		 *  should be compatible with all Unicode normalization
-		 *  methods.
-		 */
-		if (!uc_is_general_category (first_unichar,
-		                             parser->allowed_start)) {
-			/* Skip this word and keep on looping */
-			parser->cursor += word_length;
-			continue;
+		word_length_uchar = next_word_offset_uchar - parser->cursor;
+		word_length_utf8 = next_word_offset_utf8 - current_word_offset_utf8;
+
+		/* Get word info... */
+		if (!get_word_info (&parser->utxt[parser->cursor],
+		                    word_length_uchar,
+		                    &is_allowed,
+		                    &is_ascii_or_cjk)) {
+			/* Quit loop just in case */
+			parser->cursor = parser->utxt_size;
+			break;
 		}
 
-		/* check if word is reserved */
+		/* check if word is reserved (looking at ORIGINAL UTF-8 buffer
+		 *  here! */
 		if (parser->parse_reserved_words &&
-		    word_length == 2 &&
-		    parser->txt[parser->cursor] == 'o' &&
-		    parser->txt[parser->cursor + 1] == 'r') {
+		    word_length_utf8 == 2 &&
+		    parser->txt[current_word_offset_utf8] == 'o' &&
+		    parser->txt[current_word_offset_utf8 + 1] == 'r') {
 			/* Skip this word and keep on looping */
-			parser->cursor += word_length;
+			parser->cursor = next_word_offset_uchar;
 			continue;
 		}
 
-		/* compute truncated word length if needed (to avoid extremely
-		 *  long words)*/
-		truncated_length = (word_length < WORD_BUFFER_LENGTH ?
-		                    word_length :
-		                    WORD_BUFFER_LENGTH - 1);
-
-		/* Enable UNAC stripping only if no ASCII and no CJK */
-		do_strip = (!is_ascii_word (&(parser->txt[parser->cursor]),
-		                            truncated_length) &&
-		            !IS_CJK_UCS4 (first_unichar));
+		/* compute truncated word length (in UChar bytes) if needed (to
+		 * avoid extremely long words) */
+		truncated_length = (word_length_uchar < 2 * WORD_BUFFER_LENGTH ?
+		                    word_length_uchar :
+		                    2 * WORD_BUFFER_LENGTH);
 
 		/* Process the word here. If it fails, we can still go
-		 *  to the next one. Returns newly allocated string
-		 *  always */
-		processed_word = tracker_parser_process_word (parser,
-		                                              &(parser->txt[parser->cursor]),
-		                                              truncated_length,
-		                                              do_strip);
+		 *  to the next one. Returns newly allocated UTF-8
+		 *  string always.
+		 * Enable UNAC stripping only if no ASCII and no CJK
+		 * Note we are passing UChar encoded string here!
+		 */
+		processed_word = process_word_uchar (parser,
+		                                     &(parser->utxt[parser->cursor]),
+		                                     truncated_length,
+		                                     !is_ascii_or_cjk);
 		if (!processed_word) {
 			/* Skip this word and keep on looping */
-			parser->cursor += word_length;
+			parser->cursor = next_word_offset_uchar;
 			continue;
 		}
 	}
@@ -185,11 +240,11 @@ parser_next (TrackerParser *parser,
 	/* If we got a word here, set output */
 	if (processed_word) {
 		/* Set outputs */
-		*byte_offset_start = parser->cursor;
-		*byte_offset_end = parser->cursor + word_length;
+		*byte_offset_start = current_word_offset_utf8;
+		*byte_offset_end = current_word_offset_utf8 + word_length_utf8;
 
 		/* Update cursor */
-		parser->cursor += word_length;
+		parser->cursor += word_length_uchar;
 
 		parser->word_length = strlen (processed_word);
 		parser->word = processed_word;
@@ -218,6 +273,7 @@ tracker_parser_new (TrackerLanguage *language,
 	parser->word_length = 0;
 
 	parser->utxt = NULL;
+	parser->offsets = NULL;
 	parser->utxt_size = 0;
 	parser->bi = NULL;
 	parser->cursor = 0;
@@ -239,6 +295,7 @@ tracker_parser_free (TrackerParser *parser)
 	}
 
 	g_free (parser->utxt);
+	g_free (parser->offsets);
 
 	g_free (parser->word);
 
@@ -254,8 +311,10 @@ tracker_parser_reset (TrackerParser *parser,
                       gboolean       enable_stop_words,
                       gboolean       parse_reserved_words)
 {
-	UErrorCode error;
+	UErrorCode error = U_ZERO_ERROR;
 	UConverter *converter;
+	UChar *last_uchar;
+	const gchar *last_utf8;
 
 	g_return_if_fail (parser != NULL);
 	g_return_if_fail (txt != NULL);
@@ -280,159 +339,229 @@ tracker_parser_reset (TrackerParser *parser,
 	if (!converter) {
 		g_warning ("Cannot open UTF-8 converter: '%s'",
 		           U_FAILURE (error) ? u_errorName (error) : "none");
-		return;
+               return;
 	}
 
-	/* Allocate UChars buffer */
-	parser->utxt_size = txt_size * 2 + 1;
+	/* Allocate UChars and offsets buffers */
+	parser->utxt_size = txt_size * sizeof (UChar) + 1;
 	parser->utxt = g_malloc (parser->utxt_size);
+	parser->offsets = g_malloc (parser->utxt_size);
+
+	/* last_uchar and last_utf8 will be also an output parameter! */
+	last_uchar = parser->utxt;
+	last_utf8 = parser->txt;
+
+	/* Convert to UChars storing offsets */
+	ucnv_toUnicode (converter,
+	                &last_uchar,
+	                &parser->utxt[parser->utxt_size],
+	                &last_utf8,
+	                &parser->txt[parser->txt_size],
+	                parser->offsets,
+	                FALSE,
+	                &error);
+	if (U_SUCCESS (error)) {
+		/* Proper UChar array size is now given by 'last_uchar' */
+		parser->utxt_size = last_uchar - parser->utxt;
+
+		/* Open word-break iterator */
+		parser->bi = ubrk_open(UBRK_WORD,
+		                       setlocale (LC_ALL, NULL),
+		                       parser->utxt,
+		                       parser->utxt_size,
+		                       &error);
+		if (U_SUCCESS (error)) {
+			/* Find FIRST word in the UChar array */
+			parser->cursor = ubrk_first (parser->bi);
+		}
+	}
 
-	/* Convert to UChars */
-	parser->utxt_size = ucnv_toUChars (converter,
-	                                   parser->utxt,
-	                                   parser->utxt_size,
-	                                   parser->txt,
-	                                   parser->txt_size,
-	                                   &error);
+	/* If any error happened, reset buffers */
 	if (U_FAILURE (error)) {
-		g_warning ("Cannot convert from UTF-8 to UChar: '%s'",
+		g_warning ("Error initializing libicu support: '%s'",
 		           u_errorName (error));
-		/* Error converting to UChars... reset buffer */
+		/* Reset buffers */
 		g_free (parser->utxt);
+		g_free (parser->offsets);
 		parser->utxt = NULL;
+		parser->offsets = NULL;
 		parser->utxt_size = 0;
-		ucnv_close (converter);
-		return;
 	}
 
-	/* Open word-break iterator */
-	parser->bi = ubrk_open(UBRK_WORD,
-	                       setlocale (LC_ALL, NULL),
-	                       parser->utxt,
-	                       parser->utxt_size,
-	                       &error);
+	/* Close converter */
+	ucnv_close (converter);
+}
+
+static gchar *
+process_word_uchar (TrackerParser *parser,
+                    const UChar   *word,
+                    gint           length,
+                    gboolean       do_strip)
+{
+	UErrorCode error = U_ZERO_ERROR;
+	UChar casefolded_buffer [WORD_BUFFER_LENGTH];
+	UChar normalized_buffer [WORD_BUFFER_LENGTH];
+	gchar *utf8_str = NULL;
+	gchar *stemmed = NULL;
+	size_t new_word_length;
+
+	/* Casefold... */
+	new_word_length = u_strFoldCase (casefolded_buffer,
+	                                 WORD_BUFFER_LENGTH,
+	                                 word,
+	                                 length,
+	                                 U_FOLD_CASE_DEFAULT,
+	                                 &error);
 	if (U_FAILURE (error)) {
-		g_warning ("Cannot open word-breaker: '%s'",
+		g_warning ("Error casefolding: '%s'",
 		           u_errorName (error));
-		g_free (parser->utxt);
-		parser->utxt = NULL;
-		parser->utxt_size = 0;
+		return NULL;
+	}
+
+	if (new_word_length > WORD_BUFFER_LENGTH)
+		new_word_length = WORD_BUFFER_LENGTH;
+
+	/* NFC normalization... */
+	new_word_length = unorm_normalize (casefolded_buffer,
+	                                   new_word_length,
+	                                   UNORM_NFC,
+	                                   0,
+	                                   normalized_buffer,
+	                                   WORD_BUFFER_LENGTH,
+	                                   &error);
+	if (U_FAILURE (error)) {
+		g_warning ("Error normalizing: '%s'",
+		           u_errorName (error));
+		return NULL;
+	}
+
+	if (new_word_length > WORD_BUFFER_LENGTH)
+		new_word_length = WORD_BUFFER_LENGTH;
+
+	/* UNAC stripping needed? */
+	if (do_strip) {
+		gsize stripped_word_length;
+
+		/* Get unaccented string in UTF-8 */
+		utf8_str = tracker_parser_unaccent_UChar_word (normalized_buffer,
+		                                               new_word_length,
+		                                               &stripped_word_length);
+		if (utf8_str) {
+			new_word_length = stripped_word_length;
+		}
+	}
+
+	/* If stripping failed or not needed, convert to UTF-8 */
+	if (!utf8_str) {
+		UErrorCode icu_error = U_ZERO_ERROR;
+		UConverter *converter;
+		gsize utf8_len;
+
+		/* Open converter UChar to UTF-16BE */
+		converter = ucnv_open ("UTF-8", &icu_error);
+		if (!converter) {
+			g_warning ("Cannot open UTF-8 converter: '%s'",
+			           U_FAILURE (icu_error) ? u_errorName (icu_error) : "none");
+			return NULL;
+		}
+		/* Using same  buffer size as for UTF-16 should always work. */
+		utf8_str = g_malloc (new_word_length + 1);
+
+		/* Convert from UChar to UTF-8 */
+		utf8_len = ucnv_fromUChars (converter,
+		                            utf8_str,
+		                            new_word_length,
+		                            normalized_buffer,
+		                            new_word_length,
+		                            &icu_error);
+		if (U_FAILURE (icu_error)) {
+			g_warning ("Cannot convert from UChar to UTF-8: '%s'",
+			           u_errorName (icu_error));
+			g_free (utf8_str);
+			ucnv_close (converter);
+			return NULL;
+		}
+
+		utf8_str[utf8_len] = '\0';
+		new_word_length = utf8_len;
 		ucnv_close (converter);
-		return;
 	}
 
-	/* Find FIRST word in the UChar array */
-	parser->cursor = ubrk_first (parser->bi);
+	/* Stemming needed? */
+	if (parser->enable_stemmer) {
+		/* Input for stemmer ALWAYS in UTF-8, as well as output */
+		stemmed = tracker_language_stem_word (parser->language,
+		                                      utf8_str,
+		                                      new_word_length);
+
+		/* Log after stemming */
+		tracker_parser_message_hex ("   After stemming",
+		                            stemmed, strlen (stemmed));
+	}
+
+	/* If stemmed wanted and succeeded, free previous and return it */
+	if (stemmed) {
+		g_free (utf8_str);
+		return stemmed;
+	}
+
+	return utf8_str;
 }
 
-/* libunistring version of the word processor. */
+
+/* Both Input and Output are always UTF-8 */
 gchar *
 tracker_parser_process_word (TrackerParser *parser,
-                             const gchar    *word,
+                             const gchar   *word,
                              gint           length,
                              gboolean       do_strip)
 {
-	/* gchar word_buffer [WORD_BUFFER_LENGTH]; */
-	/* gchar *normalized = NULL; */
-	/* gchar *stripped = NULL; */
-	/* gchar *stemmed = NULL; */
-	/* size_t new_word_length; */
-
-	/* g_return_val_if_fail (parser != NULL, NULL); */
-	/* g_return_val_if_fail (word != NULL, NULL); */
-
-
-	/* /\* If length is set as -1, the input word MUST be NIL-terminated. */
-	/*  * Otherwise, this restriction is not needed as the length to process */
-	/*  *  is given as input argument *\/ */
-	/* if (length < 0) { */
-	/* 	length = strlen (word); */
-	/* } */
-
-	/* /\* Log original word *\/ */
-	/* tracker_parser_message_hex ("ORIGINAL word", */
-	/*                             word, length); */
-
-	/* /\* Leave space for last NIL *\/ */
-	/* new_word_length = WORD_BUFFER_LENGTH - 1; */
-
-	/* /\* Casefold and NFC normalization in output. */
-	/*  *  NOTE: if the output buffer is not big enough, u8_casefold will */
-	/*  *  return a newly-allocated buffer. *\/ */
-	/* normalized = u8_casefold ((const uint8_t *)word, */
-	/*                           length, */
-	/*                           uc_locale_language (), */
-	/*                           UNINORM_NFC, */
-	/*                           word_buffer, */
-	/*                           &new_word_length); */
-
-	/* /\* Case folding + Normalization failed, skip this word *\/ */
-	/* g_return_val_if_fail (normalized != NULL, NULL); */
-
-	/* /\* If output buffer is not the same as the one passed to */
-	/*  *  u8_casefold, we know it was newly-allocated, so need */
-	/*  *  to resize it in 1 byte to add last NIL *\/ */
-	/* if (normalized != word_buffer) { */
-	/* 	normalized = g_realloc (normalized, new_word_length + 1); */
-	/* } */
-
-	/* /\* Set output NIL *\/ */
-	/* normalized[new_word_length] = '\0'; */
-
-	/* /\* Log after Normalization *\/ */
-	/* tracker_parser_message_hex (" After Casefolding and NFC normalization", */
-	/*                             normalized, new_word_length); */
-
-	/* /\* UNAC stripping needed? *\/ */
-	/* if (do_strip) { */
-	/* 	gsize stripped_word_length; */
-
-	/* 	stripped = tracker_parser_unaccent_string (normalized, */
-	/* 	                                           new_word_length, */
-	/* 	                                           &stripped_word_length); */
-
-	/* 	if (stripped) { */
-	/* 		/\* Log after UNAC stripping *\/ */
-	/* 		tracker_parser_message_hex ("  After UNAC stripping", */
-	/* 		                            stripped, stripped_word_length); */
-	/* 		new_word_length = stripped_word_length; */
-	/* 	} */
-	/* } */
-
-
-	/* /\* Stemming needed? *\/ */
-	/* if (parser->enable_stemmer) { */
-	/* 	stemmed = tracker_language_stem_word (parser->language, */
-	/* 	                                      stripped ? stripped : normalized, */
-	/* 	                                      new_word_length); */
-
-	/* 	/\* Log after stemming *\/ */
-	/* 	tracker_parser_message_hex ("   After stemming", */
-	/* 	                            stemmed, strlen (stemmed)); */
-	/* } */
-
-	/* /\* If stemmed wanted and succeeded, free previous and return it *\/ */
-	/* if (stemmed) { */
-	/* 	g_free (stripped); */
-	/* 	if (normalized != word_buffer) { */
-	/* 		g_free (normalized); */
-	/* 	} */
-	/* 	return stemmed; */
-	/* } */
-
-	/* /\* If stripped wanted and succeeded, free previous and return it *\/ */
-	/* if (stripped) { */
-	/* 	if (normalized != word_buffer) { */
-	/* 		g_free (normalized); */
-	/* 	} */
-	/* 	return stripped; */
-	/* } */
-
-	/* /\* It may be the case that no stripping and no stemming was needed, and */
-	/*  * that the output buffer in stack was enough for case-folding and */
-	/*  * normalization. In this case, need to strdup() the string to return it *\/ */
-	/* return normalized == word_buffer ? g_strdup (word_buffer) : normalized; */
-	return NULL;
+	UErrorCode icu_error = U_ZERO_ERROR;
+	UConverter *converter;
+	UChar *uchar_word;
+	gsize uchar_len;
+	gchar *processed;
+
+	/* Open converter UTF-8 to UChar */
+	converter = ucnv_open ("UTF-8", &icu_error);
+	if (!converter) {
+		g_warning ("Cannot open UTF-8 converter: '%s'",
+		           U_FAILURE (icu_error) ? u_errorName (icu_error) : "none");
+		return NULL;
+	}
+
+	/* Compute length if not already as input */
+	if (length < 0) {
+		length = strlen (word);
+	}
+
+	/* Twice the size of the UTF-8 string for UChars */
+	uchar_word = g_malloc (2 * length);
+
+	/* Convert from UTF-8 to UChars*/
+	uchar_len = ucnv_toUChars (converter,
+	                           uchar_word,
+	                           2 * length,
+	                           word,
+	                           length,
+	                           &icu_error);
+	if (U_FAILURE (icu_error)) {
+		g_warning ("Cannot convert from UTF-8 to UChar: '%s'",
+		           u_errorName (icu_error));
+		g_free (uchar_word);
+		ucnv_close (converter);
+		return NULL;
+	}
+
+	ucnv_close (converter);
+
+	/* Process UChar based word */
+	processed = process_word_uchar (parser,
+	                                uchar_word,
+	                                uchar_len,
+	                                do_strip);
+	g_free (uchar_word);
+	return processed;
 }
 
 const gchar *



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]