[clutter/wip/cogl-winsys-glx: 1/20] paint-volume: Fix culling bug to handle partial culls

[Robert Bragg <rbragg src gnome org>]

commit 114133c98c417794c315fec81579fef1ffccf98b
Author: Robert Bragg <robert linux intel com>
Date:   Tue Mar 15 10:23:41 2011 +0000

    paint-volume: Fix culling bug to handle partial culls
    
    This updates the inner loops of the cull function so now the vertices of
    the polygon being culled are iterated in the inner loop instead of the
    clip planes and we count how many vertices are outside the current
    plane so we can bail out immediately if all the vertices are outside of
    any plane and so we can correctly track partial intersections with the
    clip region.
    
    The previous approach could catch some partial intersections but for
    example a rectangle that was larger than the clip region centred over
    the clip region with all corners outside would be reported as outside,
    not as a partial intersection.

 clutter/clutter-paint-volume.c |   38 ++++++++++++++++----------------------
 1 files changed, 16 insertions(+), 22 deletions(-)
---
diff --git a/clutter/clutter-paint-volume.c b/clutter/clutter-paint-volume.c
index 1b48057..43e4562 100644
--- a/clutter/clutter-paint-volume.c
+++ b/clutter/clutter-paint-volume.c
@@ -922,8 +922,7 @@ _clutter_paint_volume_cull (ClutterPaintVolume *pv,
 {
   int vertex_count;
   ClutterVertex *vertices = pv->vertices;
-  gboolean in = TRUE;
-  gboolean out = TRUE;
+  gboolean partial = FALSE;
   int i;
   int j;
 
@@ -942,10 +941,10 @@ _clutter_paint_volume_cull (ClutterPaintVolume *pv,
   else
     vertex_count = 8;
 
-  for (i = 0; i < vertex_count; i++)
+  for (i = 0; i < 4; i++)
     {
-      gboolean point_in = TRUE;
-      for (j = 0; j < 4; j++)
+      int out = 0;
+      for (j = 0; j < vertex_count; j++)
         {
           ClutterVertex p;
           float distance;
@@ -953,32 +952,27 @@ _clutter_paint_volume_cull (ClutterPaintVolume *pv,
           /* XXX: for perspective projections this can be optimized
            * out because all the planes should pass through the origin
            * so (0,0,0) is a valid v0. */
-          p.x = vertices[i].x - planes[j].v0.x;
-          p.y = vertices[i].y - planes[j].v0.y;
-          p.z = vertices[i].z - planes[j].v0.z;
+          p.x = vertices[j].x - planes[i].v0.x;
+          p.y = vertices[j].y - planes[i].v0.y;
+          p.z = vertices[j].z - planes[i].v0.z;
 
           distance =
-            planes[j].n.x * p.x + planes[j].n.y * p.y + planes[j].n.z * p.z;
+            planes[i].n.x * p.x + planes[i].n.y * p.y + planes[i].n.z * p.z;
 
           if (distance < 0)
-            {
-              point_in = FALSE;
-              break;
-            }
+            out++;
         }
 
-      if (!point_in)
-        in = FALSE;
-      else
-        out = FALSE;
+      if (out == 4)
+        return CLUTTER_CULL_RESULT_OUT;
+      else if (out != 0)
+        partial = TRUE;
     }
 
-  if (in)
-    return CLUTTER_CULL_RESULT_IN;
-  else if (out)
-    return CLUTTER_CULL_RESULT_OUT;
-  else
+  if (partial)
     return CLUTTER_CULL_RESULT_PARTIAL;
+  else
+    return CLUTTER_CULL_RESULT_IN;
 }
 
 void