[gnumeric] Functions: Fix DEC2HEX issues with large numbers.

[Morten Welinder <mortenw src gnome org>]

commit 0cec576305ed6e9caa835835b70e081685fbcd6b
Author: Morten Welinder <terra gnome org>
Date:   Mon Jul 27 13:08:37 2009 -0400

    Functions: Fix DEC2HEX issues with large numbers.

 NEWS                       |    1 +
 plugins/fn-eng/ChangeLog   |    8 ++++++++
 plugins/fn-eng/functions.c |   22 ++++++++++++++--------
 3 files changed, 23 insertions(+), 8 deletions(-)
---
diff --git a/NEWS b/NEWS
index 667c51f..97db51a 100644
--- a/NEWS
+++ b/NEWS
@@ -33,6 +33,7 @@ Morten:
 	* Fix problems with in-sheet combos.  [#587992]
 	* Fix translation textdomain confusion.  [#588110]
 	* Avoid using gtk_tree_view_column_get_cell_renderers.  [#589105]
+	* Fix DEC2HEX and friends for large values.  [#588997]
 
 --------------------------------------------------------------------------
 Gnumeric 1.9.9
diff --git a/plugins/fn-eng/ChangeLog b/plugins/fn-eng/ChangeLog
index c10c953..c4fd25d 100644
--- a/plugins/fn-eng/ChangeLog
+++ b/plugins/fn-eng/ChangeLog
@@ -1,3 +1,11 @@
+2009-07-27  Morten Welinder  <terra gnome org>
+
+	* functions.c (gnumeric_hex2dec): Fix maximum.
+	(val_to_base): Check range even in the string case.  Only do
+	two-complement thingy when src_base is not 10.  Use
+	g_ascii_strtoll, not strtol.  Use value_new_float, not
+	value_new_int.  Fixes #588997.
+
 2009-07-01  Andreas J. Guelzow <aguelzow pyrshep ca>
 
 	* functions.c: conversion to new function description
diff --git a/plugins/fn-eng/functions.c b/plugins/fn-eng/functions.c
index e81101d..e824e62 100644
--- a/plugins/fn-eng/functions.c
+++ b/plugins/fn-eng/functions.c
@@ -71,7 +71,7 @@ val_to_base (GnmFuncEvalInfo *ei,
 	     Val2BaseFlags flags)
 {
 	int digit, min, max, places;
-	gnm_float v, b10;
+	gnm_float v;
 	GString *buffer;
 	GnmValue *vstring = NULL;
 
@@ -126,10 +126,13 @@ val_to_base (GnmFuncEvalInfo *ei,
 					hsuffix = TRUE;
 			}
 
-			v = strtol (str, &err, src_base);
+			v = g_ascii_strtoll (str, &err, src_base);
 			if (err == str || err[hsuffix] != 0)
 				return value_new_error_NUM (ei->pos);
 
+			if (v < min_value || v > max_value)
+				return value_new_error_NUM (ei->pos);
+
 			break;
 		}
 		/* Fall through.  */
@@ -149,19 +152,21 @@ val_to_base (GnmFuncEvalInfo *ei,
 				 "%.0" GNM_FORMAT_f,
 				 val);
 
-		v = strtol (buf, &err, src_base);
+		v = g_ascii_strtoll (buf, &err, src_base);
 		if (*err != 0)
 			return value_new_error_NUM (ei->pos);
 		break;
 	}
 	}
 
-	b10 = gnm_pow (src_base, 10);
-	if (v >= b10 / 2) /* N's complement */
-		v = v - b10;
+	if (src_base != 10) {
+		gnm_float b10 = gnm_pow (src_base, 10);
+		if (v >= b10 / 2) /* N's complement */
+			v = v - b10;
+	}
 
 	if (flags & V2B_NUMBER)
-		return value_new_int (v);
+		return value_new_float (v);
 
 	if (v < 0) {
 		min = 1;
@@ -525,9 +530,10 @@ static GnmFuncHelp const help_hex2dec[] = {
 static GnmValue *
 gnumeric_hex2dec (GnmFuncEvalInfo *ei, GnmValue const * const *argv)
 {
+	static gnm_float pow_2_40 = GNM_const(1099511627776.0);
 	return val_to_base (ei, argv[0], NULL,
 			    16, 10,
-			    0, GNM_const(9999999999.0),
+			    0, pow_2_40 - 1,
 			    V2B_STRINGS_MAXLEN |
 			    V2B_STRINGS_BLANK_ZERO |
 			    V2B_NUMBER);