Some UTF-8 mini-additions

[Owen Taylor <otaylor redhat com>]

The following patch adds a couple of missing UTF-8 handling facilities:

 1) g_utf8_get_char_validated(). This is needed if you want to
    iterate over possibly partial or invalid UTF-8 buffers.
 
    It is needed for outstanding IOChannel changes and is also
    open as a separate request as #56958.

 2) g_string_append/prepend/insert_unichar. I've found myself 
    writing these convenience functiosn a bunch of times in terms 
    of g_unichar_to_utf8(). Not introducing these causes no
    loss of functionality, but they are simple enough and handy
    enough I think they are worth adding.

This patch introduces no source or binary incompatibilities.

Regards,
                                        Owen

Index: ChangeLog
===================================================================
RCS file: /cvs/gnome/glib/ChangeLog,v
retrieving revision 1.755
diff -u -r1.755 ChangeLog
--- ChangeLog	2001/07/17 19:01:10	1.755
+++ ChangeLog	2001/07/18 19:06:25
@@ -1,3 +1,16 @@
+Fri Jul 13 19:20:06 2001  Owen Taylor  <otaylor redhat com>
+
+	* glib/gstring.c (g_string_insert/append/prepend_unichar):
+	Add functions to insert a unichar as UTF-8, since this
+	is reasonably common.
+
+	* glib/gutf8.c glib/gunicode.h (g_utf8_get_char_validated):
+	New function exposing iterating through possibly invalid/incomplete
+	UTF-8 to unicode to the outside world.
+
+	* glib/gutf8.c (g_utf8_get_char_extended): Fix max_len argument
+	to be gssize, not gsize.
+
 2001-07-17  Kjartan Maraas  <kmaraas gnome org>
 
 	* configure.in: Added "nn" to ALL_LINGUAS.
Index: glib/gstring.c
===================================================================
RCS file: /cvs/gnome/glib/glib/gstring.c,v
retrieving revision 1.25
diff -u -r1.25 gstring.c
--- glib/gstring.c	2001/06/30 16:54:32	1.25
+++ glib/gstring.c	2001/07/18 19:06:25
@@ -465,7 +465,26 @@
   return g_string_insert_c (fstring, -1, c);
 }
 
+/**
+ * g_string_append_unichar:
+ * @string: a #GString
+ * @wc: a Unicode character
+ * 
+ * Converts a Unicode character into UTF-8, and appends it
+ * to the string.
+ * 
+ * Return value: @string
+ **/
 GString*
+g_string_append_unichar (GString  *string,
+			 gunichar  wc)
+{  
+  g_return_val_if_fail (string != NULL, NULL);
+  
+  return g_string_insert_unichar (string, -1, wc);
+}
+
+GString*
 g_string_prepend (GString     *fstring,
 		  const gchar *val)
 {
@@ -495,7 +514,26 @@
   return g_string_insert_c (fstring, 0, c);
 }
 
+/**
+ * g_string_append_unichar:
+ * @string: a #GString
+ * @wc: a Unicode character
+ * 
+ * Converts a Unicode character into UTF-8, and prepends it
+ * to the string.
+ * 
+ * Return value: @string
+ **/
 GString*
+g_string_prepend_unichar (GString  *string,
+			  gunichar  wc)
+{  
+  g_return_val_if_fail (string != NULL, NULL);
+  
+  return g_string_insert_unichar (string, 0, wc);
+}
+
+GString*
 g_string_insert (GString     *fstring,
 		 gssize       pos,    
 		 const gchar *val)
@@ -535,6 +573,36 @@
   string->str[string->len] = 0;
 
   return fstring;
+}
+
+/**
+ * g_string_insert_unichar:
+ * @string: a #Gstring
+ * @pos: the position at which to insert character, or -1 to
+ *       append at the end of the string.
+ * @wc: a Unicode character
+ * 
+ * Converts a Unicode character into UTF-8, and insert it
+ * into the string at the given position.
+ * 
+ * Return value: @string
+ **/
+GString*
+g_string_insert_unichar (GString *string,
+			 gssize   pos,    
+			 gunichar wc)
+{  
+  gchar buf[6];
+  gint charlen;
+
+  /* We could be somewhat more efficient here by computing
+   * the length, adding the space, then converting into that
+   * space, by cut-and-pasting the internals of g_unichar_to_utf8.
+   */
+  g_return_val_if_fail (string != NULL, NULL);
+
+  charlen = g_unichar_to_utf8 (wc, buf);
+  return g_string_insert_len (string, pos, buf, charlen);
 }
 
 GString*
Index: glib/gstring.h
===================================================================
RCS file: /cvs/gnome/glib/glib/gstring.h,v
retrieving revision 1.8
diff -u -r1.8 gstring.h
--- glib/gstring.h	2001/06/30 17:19:20	1.8
+++ glib/gstring.h	2001/07/18 19:06:25
@@ -28,6 +28,7 @@
 #define __G_STRING_H__
 
 #include <glib/gtypes.h>
+#include <glib/gunicode.h>
 
 G_BEGIN_DECLS
 
@@ -79,10 +80,14 @@
                                          gssize           len);  
 GString*     g_string_append_c          (GString	 *string,
 					 gchar		  c);
+GString*     g_string_append_unichar    (GString	 *string,
+					 gunichar	  wc);
 GString*     g_string_prepend           (GString	 *string,
 					 const gchar	 *val);
 GString*     g_string_prepend_c         (GString	 *string,
 					 gchar		  c);
+GString*     g_string_prepend_unichar   (GString	 *string,
+					 gunichar	  wc);
 GString*     g_string_prepend_len       (GString	 *string,
 			                 const gchar	 *val,
                                          gssize           len);  
@@ -92,6 +97,9 @@
 GString*     g_string_insert_c          (GString	 *string,
 					 gssize		  pos,    
 					 gchar		  c);
+GString*     g_string_insert_unichar    (GString	 *string,
+					 gssize		  pos,    
+					 gunichar	  wc);
 GString*     g_string_erase	        (GString	 *string,
 					 gsize		  pos,    
 					 gsize		  len);   
Index: glib/gunicode.h
===================================================================
RCS file: /cvs/gnome/glib/glib/gunicode.h,v
retrieving revision 1.23
diff -u -r1.23 gunicode.h
--- glib/gunicode.h	2001/07/07 02:42:47	1.23
+++ glib/gunicode.h	2001/07/18 19:06:25
@@ -167,7 +167,10 @@
 
 #define g_utf8_next_char(p) (char *)((p) + g_utf8_skip[*(guchar *)(p)])
 
-gunichar g_utf8_get_char          (const gchar *p);
+gunichar g_utf8_get_char           (const gchar  *p);
+gunichar g_utf8_get_char_validated (const  gchar *p,
+				    gssize        max_len);
+
 gchar*   g_utf8_offset_to_pointer (const gchar *str,
                                    glong        offset);  
 glong    g_utf8_pointer_to_offset (const gchar *str,      
Index: glib/guniprop.c
===================================================================
RCS file: /cvs/gnome/glib/glib/guniprop.c,v
retrieving revision 1.10
diff -u -r1.10 guniprop.c
--- glib/guniprop.c	2001/07/11 15:28:35	1.10
+++ glib/guniprop.c	2001/07/18 19:06:25
@@ -872,8 +872,6 @@
 {
   GString *result = g_string_new (NULL);
   const char *p;
-  gchar buf[6];
-  int charlen;
 
   p = str;
   while ((len < 0 || p < str + len) && *p)
@@ -903,9 +901,7 @@
 	    }
 	}
 
-      ch = g_unichar_tolower (ch);
-      charlen = g_unichar_to_utf8 (ch, buf);
-      g_string_append_len (result, buf, charlen);
+      g_string_append_unichar (result, g_unichar_tolower (ch));
       
     next:
       p = g_utf8_next_char (p);
Index: glib/gutf8.c
===================================================================
RCS file: /cvs/gnome/glib/glib/gutf8.c,v
retrieving revision 1.21
diff -u -r1.21 gutf8.c
--- glib/gutf8.c	2001/06/23 13:55:07	1.21
+++ glib/gutf8.c	2001/07/18 19:06:25
@@ -245,7 +245,9 @@
  * 
  * Convert a sequence of bytes encoded as UTF-8 to a unicode character.
  * If @p does not point to a valid UTF-8 encoded character, results are
- * undefined.
+ * undefined. If you are not sure that the bytes are complete
+ * valid unicode characters, you should use g_utf8_get_char_validated()
+ * instead.
  * 
  * Return value: the resulting character
  **/
@@ -550,7 +552,8 @@
  * and return (gunichar)-2 on incomplete trailing character
  */
 static inline gunichar
-g_utf8_get_char_extended (const gchar *p, gsize max_len)  
+g_utf8_get_char_extended (const  gchar *p,
+			  gssize max_len)  
 {
   guint i, len;
   gunichar wc = (guchar) *p;
@@ -623,6 +626,35 @@
     return (gunichar)-1;
   
   return wc;
+}
+
+/**
+ * g_utf8_get_char_validated:
+ * @p: a pointer to unicode character encoded as UTF-8
+ * @max_len: the maximum number of bytes to read, or -1, for no maximum.
+ * 
+ * Convert a sequence of bytes encoded as UTF-8 to a unicode character.
+ * This function checks for incomplete characters, for invalid characters
+ * such as characters that are out of the range of Unicode, and for
+ * overlong encodings of valid characters.
+ * 
+ * Return value: the resulting character. If @p points to a partial
+ *    sequence at the end of a string that could begin a valid character,
+ *    returns (gunichar)-2; otherwise, if @p does not point to a valid
+ *    UTF-8 encoded unicode character, returns (gunichar)-1.
+ **/
+gunichar
+g_utf8_get_char_validated (const  gchar *p,
+			   gssize max_len)
+{
+  gunichar result = g_utf8_get_char_extended (p, max_len);
+
+  if (result & 0x80000000)
+    return result;
+  else if (!UNICODE_VALID (result))
+    return (gunichar)-1;
+  else
+    return result;
 }
 
 /**
Index: tests/unicode-normalize.c
===================================================================
RCS file: /cvs/gnome/glib/tests/unicode-normalize.c,v
retrieving revision 1.4
diff -u -r1.4 unicode-normalize.c
--- tests/unicode-normalize.c	2001/07/08 13:01:19	1.4
+++ tests/unicode-normalize.c	2001/07/18 19:06:25
@@ -11,8 +11,6 @@
   unsigned ch;
   int offset = 0;
   GString *result = g_string_new (NULL);
-  int len;
-  char buf[6];
   
   do 
     {
@@ -30,8 +28,7 @@
 	  return NULL;
 	}
 
-      len = g_unichar_to_utf8 (ch, buf);
-      g_string_append_len (result, buf, len);
+      g_string_append_unichar (result, ch);
       
       while (input[offset] && input[offset] != ' ')
 	offset++;