[vte/wip/sixels: 58/82] sixel: Fix out-of-bounds write caused by bad resize logic

[Hans Petter Jansson <hansp src gnome org>]

commit c64f0f56381b731e5444a519278c07ca524f1f0f
Author: Hans Petter Jansson <hpj cl no>
Date:   Sun Jun 14 15:01:59 2020 +0200

    sixel: Fix out-of-bounds write caused by bad resize logic
    
    The finalization code shrinks the image extents if possible, but in
    doing so it was only checking if one or the other dimension would be
    reduced, leaving open the possibility that the other dimension could
    be greater. When this happened, the result could exceed the buffer
    provided by the caller.

 src/sixel.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)
---
diff --git a/src/sixel.cc b/src/sixel.cc
index 2ef26901..d5b0c9db 100644
--- a/src/sixel.cc
+++ b/src/sixel.cc
@@ -195,6 +195,9 @@ image_buffer_resize(
        int n;
        int min_height;
 
+        if (width == image->width && height == image->height)
+                return 0;
+
        size = (size_t)(width * height) * sizeof(sixel_color_no_t);
        alt_buffer = (sixel_color_no_t *)g_malloc(size);
        if (alt_buffer == NULL) {
@@ -312,11 +315,11 @@ sixel_parser_finalize(sixel_state_t *st, unsigned char *pixels)
        sx = st->max_x;
        sy = st->max_y;
 
-       if (image->width > sx || image->height > sy) {
-               status = image_buffer_resize(image, sx, sy);
-               if (status < 0)
-                       goto end;
-       }
+        status = image_buffer_resize(image,
+                                     MIN (image->width, sx),
+                                     MIN (image->height, sy));
+        if (status < 0)
+                goto end;
 
        if (image->use_private_register && image->ncolors > 2 && !image->palette_modified) {
                status = set_default_color(image);