[gnumeric] CORREL: Clamp to [-1,+1]



commit 4fe908e7942169cc1e13611835e543d892d9edb0
Author: Morten Welinder <terra gnome org>
Date:   Mon Aug 14 11:35:40 2017 -0400

    CORREL: Clamp to [-1,+1]
    
    Intermediate rounding errors can push us beyond.  Clamping will avoid
    further trouble down the road.

 NEWS            |    1 +
 src/rangefunc.c |   11 +++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)
---
diff --git a/NEWS b/NEWS
index 126dd69..4209bb1 100644
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,7 @@ Gnumeric 1.12.36
 Morten:
        * Convert all xpm files to png.
        * Avoid to-pixdata options for resources.  It's going away.
+       * Minor CORREL improvement.
 
 --------------------------------------------------------------------------
 Gnumeric 1.12.35
diff --git a/src/rangefunc.c b/src/rangefunc.c
index be46a83..28a2e04 100644
--- a/src/rangefunc.c
+++ b/src/rangefunc.c
@@ -383,14 +383,21 @@ gnm_range_covar_est (gnm_float const *xs, const gnm_float *ys, int n, gnm_float
 int
 gnm_range_correl_pop (gnm_float const *xs, const gnm_float *ys, int n, gnm_float *res)
 {
-       gnm_float sx, sy, vxy;
+       gnm_float sx, sy, vxy, c;
 
        if (gnm_range_stddev_pop (xs, n, &sx) || sx == 0 ||
            gnm_range_stddev_pop (ys, n, &sy) || sy == 0 ||
            gnm_range_covar_pop (xs, ys, n, &vxy))
                return 1;
 
-       *res = vxy / (sx * sy);
+       c = vxy / (sx * sy);
+
+       // Rounding errors can push us beyond [-1,+1].  Avoid that.
+       // This isn't a great solution, but it'll have to do until
+       // someone comes up with a better approach.
+       c = CLAMP (c, -1.0, +1.0);
+
+       *res = c;
        return 0;
 }
 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]