[gimp/gimp-2-8] Bug 740193: Foreground selection causes segfault

[Massimo Valentini <mvalentini src gnome org>]

commit 350300c55a8df81e9be8c9ecdf6aa1e10e2db3c6
Author: Massimo Valentini <mvalentini src gnome org>
Date:   Thu Dec 4 18:33:54 2014 +0100

    Bug 740193: Foreground selection causes segfault
    
    Create leaves if max-min <= limits or pivot == max
    otherwise it is possible to start an infinite recursion
    or to access a buffer beyond its end.

 app/base/siox.c |  165 ++++++++++++++++++++++++++++---------------------------
 1 files changed, 85 insertions(+), 80 deletions(-)
---
diff --git a/app/base/siox.c b/app/base/siox.c
index bccf48c..7846128 100644
--- a/app/base/siox.c
+++ b/app/base/siox.c
@@ -190,6 +190,9 @@ stageone (lab          *points,
   const gint curdim = depth % dims;
   gfloat     min, max;
   gfloat     curval;
+  gfloat     l = 0;
+  gfloat     a = 0;
+  gfloat     b = 0;
   gint       i;
 
   min = CURRENT_VALUE (points, left, curdim);
@@ -209,54 +212,54 @@ stageone (lab          *points,
   if (max - min > limits[curdim])
     {
       const gfloat pivot = (min + max) / 2.0;
-      gint         l     = left;
-      gint         r     = right - 1;
-      lab          tmp;
 
-      while (TRUE)
+      if (pivot != max)
         {
-          while (CURRENT_VALUE (points, l, curdim) <= pivot)
-            ++l;
+          gint l = left;
+          gint r = right - 1;
+          lab  tmp;
 
-          while (CURRENT_VALUE (points, r, curdim) > pivot)
-            --r;
+          while (TRUE)
+            {
+              while (CURRENT_VALUE (points, l, curdim) <= pivot)
+                ++l;
 
-          if (l > r)
-            break;
+              while (CURRENT_VALUE (points, r, curdim) > pivot)
+                --r;
 
-          tmp = points[l];
-          points[l] = points[r];
-          points[r] = tmp;
+              if (l > r)
+                break;
 
-          ++l;
-          --r;
-        }
+              tmp = points[l];
+              points[l] = points[r];
+              points[r] = tmp;
 
-      /* create subtrees */
-      stageone (points, left, l, depth + 1, clusters, limits, dims);
-      stageone (points, l, right, depth + 1, clusters, limits, dims);
-    }
-  else
-    {
-      /* create leave */
-      gfloat  l = 0;
-      gfloat  a = 0;
-      gfloat  b = 0;
+              ++l;
+              --r;
+            }
 
-      points[*clusters].cardinality = right - left;
+          /* create subtrees */
+          stageone (points, left, l, depth + 1, clusters, limits, dims);
+          stageone (points, l, right, depth + 1, clusters, limits, dims);
 
-      for (; left < right; ++left)
-        {
-          l += points[left].l;
-          a += points[left].a;
-          b += points[left].b;
+          return;
         }
+    }
 
-      points[*clusters].l = l / points[*clusters].cardinality;
-      points[*clusters].a = a / points[*clusters].cardinality;
-      points[*clusters].b = b / points[*clusters].cardinality;
-      ++(*clusters);
+  /* create leave */
+  points[*clusters].cardinality = right - left;
+
+  for (; left < right; ++left)
+    {
+      l += points[left].l;
+      a += points[left].a;
+      b += points[left].b;
     }
+
+  points[*clusters].l = l / points[*clusters].cardinality;
+  points[*clusters].a = a / points[*clusters].cardinality;
+  points[*clusters].b = b / points[*clusters].cardinality;
+  ++(*clusters);
 }
 
 
@@ -278,7 +281,7 @@ stagetwo (lab           *points,
   const gint curdim = depth % dims;
   gfloat     min, max;
   gfloat     curval;
-  gint       i;
+  gint       i, sum = 0;
 
   min = CURRENT_VALUE (points, left, curdim);
   max = min;
@@ -297,64 +300,66 @@ stagetwo (lab           *points,
   if (max - min > limits[curdim])
     {
       const gfloat pivot = (min + max) / 2.0;
-      gint         l     = left;
-      gint         r     = right - 1;
-      lab          tmp;
 
-      while (TRUE)
+      if (pivot != max)
         {
-          while (CURRENT_VALUE (points, l, curdim) <= pivot)
-            ++l;
+          gint l = left;
+          gint r = right - 1;
+          lab  tmp;
 
-          while (CURRENT_VALUE (points, r, curdim) > pivot)
-            --r;
+          while (TRUE)
+            {
+              while (CURRENT_VALUE (points, l, curdim) <= pivot)
+                ++l;
 
-          if (l > r)
-            break;
+              while (CURRENT_VALUE (points, r, curdim) > pivot)
+                --r;
 
-          tmp = points[l];
-          points[l] = points[r];
-          points[r] = tmp;
+              if (l > r)
+                break;
 
-          ++l;
-          --r;
-        }
+              tmp = points[l];
+              points[l] = points[r];
+              points[r] = tmp;
+
+              ++l;
+              --r;
+            }
 
-      /* create subtrees */
-      stagetwo (points, left, l, depth + 1, clusters, limits, threshold, dims);
-      stagetwo (points, l, right, depth + 1, clusters, limits, threshold, dims);
+          /* create subtrees */
+          stagetwo (points, left, l, depth + 1, clusters, limits, threshold, dims);
+          stagetwo (points, l, right, depth + 1, clusters, limits, threshold, dims);
+
+          return;
+        }
     }
-  else /* create leave */
-    {
-      gint sum = 0;
 
-      for (i = left; i < right; i++)
-        sum += points[i].cardinality;
+  for (i = left; i < right; i++)
+    sum += points[i].cardinality;
 
-      if (sum >= threshold)
-        {
-          const gint c = right - left;
-          gfloat     l = 0;
-          gfloat     a = 0;
-          gfloat     b = 0;
+  if (sum >= threshold)
+    {
+      const gint c = right - left;
+      gfloat     l = 0;
+      gfloat     a = 0;
+      gfloat     b = 0;
 
-          for (; left < right; ++left)
-            {
-              l += points[left].l;
-              a += points[left].a;
-              b += points[left].b;
-            }
+      for (; left < right; ++left)
+        {
+          l += points[left].l;
+          a += points[left].a;
+          b += points[left].b;
+        }
 
-          points[*clusters].l = l / c;
-          points[*clusters].a = a / c;
-          points[*clusters].b = b / c;
-          ++(*clusters);
+      points[*clusters].l = l / c;
+      points[*clusters].a = a / c;
+      points[*clusters].b = b / c;
+      ++(*clusters);
 
 #ifdef SIOX_DEBUG
-          g_printerr ("siox.c: cluster=%f, %f, %f sum=%d\n",
-                     l, a, b, sum);
+      g_printerr ("siox.c: cluster=%f, %f, %f sum=%d\n",
+                  l, a, b, sum);
 #endif
-        }
     }
 }