[gimp] app: make sure undrawing the selection doesn't leave right/bottom pixel garbage

[Michael Natterer <mitch src gnome org>]

commit 71e8fc8c3a863c43c00a649262726c802abb195e
Author: Michael Natterer <mitch gimp org>
Date:   Tue Feb 21 16:01:53 2012 +0100

    app: make sure undrawing the selection doesn't leave right/bottom pixel garbage
    
    gimp_display_shell_mask_bounds(): transform the selection bounds with
    sub-pixel precision and then apply floor()/ceil() accordingly.

 app/display/gimpdisplayshell.c |   14 ++++++++------
 1 files changed, 8 insertions(+), 6 deletions(-)
---
diff --git a/app/display/gimpdisplayshell.c b/app/display/gimpdisplayshell.c
index a39957f..81fa602 100644
--- a/app/display/gimpdisplayshell.c
+++ b/app/display/gimpdisplayshell.c
@@ -1597,6 +1597,8 @@ gimp_display_shell_mask_bounds (GimpDisplayShell *shell,
 {
   GimpImage *image;
   GimpLayer *layer;
+  gdouble    x1_f, y1_f;
+  gdouble    x2_f, y2_f;
 
   g_return_val_if_fail (GIMP_IS_DISPLAY_SHELL (shell), FALSE);
   g_return_val_if_fail (x1 != NULL, FALSE);
@@ -1636,14 +1638,14 @@ gimp_display_shell_mask_bounds (GimpDisplayShell *shell,
       return FALSE;
     }
 
-  gimp_display_shell_transform_xy (shell, *x1, *y1, x1, y1);
-  gimp_display_shell_transform_xy (shell, *x2, *y2, x2, y2);
+  gimp_display_shell_transform_xy_f (shell, *x1, *y1, &x1_f, &y1_f);
+  gimp_display_shell_transform_xy_f (shell, *x2, *y2, &x2_f, &y2_f);
 
   /*  Make sure the extents are within bounds  */
-  *x1 = CLAMP (*x1, 0, shell->disp_width);
-  *y1 = CLAMP (*y1, 0, shell->disp_height);
-  *x2 = CLAMP (*x2, 0, shell->disp_width);
-  *y2 = CLAMP (*y2, 0, shell->disp_height);
+  *x1 = CLAMP (floor (x1_f), 0, shell->disp_width);
+  *y1 = CLAMP (floor (y1_f), 0, shell->disp_height);
+  *x2 = CLAMP (ceil (x2_f),  0, shell->disp_width);
+  *y2 = CLAMP (ceil (y2_f),  0, shell->disp_height);
 
   return ((*x2 - *x1) > 0) && ((*y2 - *y1) > 0);
 }