[vte/wip/sixels: 68/111] sixel: Fix out-of-bounds write caused by bad resize logic

[Christian Persch <chpe src gnome org>]

commit 0c2fdd639d4ecb80db9a1268c8f02a483a37ddb9
Author: Hans Petter Jansson <hpj cl no>
Date:   Sat Aug 8 20:42:49 2020 +0200

    sixel: Fix out-of-bounds write caused by bad resize logic
    
    The finalization code shrinks the image extents if possible, but in
    doing so it was only checking if one or the other dimension would be
    reduced, leaving open the possibility that the other dimension could
    be greater. When this happened, the result could exceed the buffer
    provided by the caller.

 src/sixel.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)
---
diff --git a/src/sixel.cc b/src/sixel.cc
index 2ef26901..d5b0c9db 100644
--- a/src/sixel.cc
+++ b/src/sixel.cc
@@ -195,6 +195,9 @@ image_buffer_resize(
        int n;
        int min_height;
 
+        if (width == image->width && height == image->height)
+                return 0;
+
        size = (size_t)(width * height) * sizeof(sixel_color_no_t);
        alt_buffer = (sixel_color_no_t *)g_malloc(size);
        if (alt_buffer == NULL) {
@@ -312,11 +315,11 @@ sixel_parser_finalize(sixel_state_t *st, unsigned char *pixels)
        sx = st->max_x;
        sy = st->max_y;
 
-       if (image->width > sx || image->height > sy) {
-               status = image_buffer_resize(image, sx, sy);
-               if (status < 0)
-                       goto end;
-       }
+        status = image_buffer_resize(image,
+                                     MIN (image->width, sx),
+                                     MIN (image->height, sy));
+        if (status < 0)
+                goto end;
 
        if (image->use_private_register && image->ncolors > 2 && !image->palette_modified) {
                status = set_default_color(image);