[gimp] app: fix subtractive src-over compositing

[N/A <ell src gnome org>]

commit 91ef42d1d07bf85fb6d92b3d4528dfb8b6185ccf
Author: Ell <ell_se yahoo com>
Date:   Fri Mar 10 15:10:35 2017 -0500

    app: fix subtractive src-over compositing

 .../layer-modes/gimpoperationlayermode.c           |    7 ++++---
 1 files changed, 4 insertions(+), 3 deletions(-)
---
diff --git a/app/operations/layer-modes/gimpoperationlayermode.c 
b/app/operations/layer-modes/gimpoperationlayermode.c
index 683ab59..ee2bf8a 100644
--- a/app/operations/layer-modes/gimpoperationlayermode.c
+++ b/app/operations/layer-modes/gimpoperationlayermode.c
@@ -814,7 +814,7 @@ composite_func_src_over_sub_core (gfloat *in,
           out[GREEN] = in[GREEN];
           out[BLUE]  = in[BLUE];
         }
-      else if (in[ALPHA] == 0.0f)
+      else if (in_alpha == 0.0f)
         {
           out[RED]   = layer[RED];
           out[GREEN] = layer[GREEN];
@@ -822,11 +822,12 @@ composite_func_src_over_sub_core (gfloat *in,
         }
       else
         {
-          gfloat recip = 1.0f / new_alpha;
+          gfloat ratio       = in_alpha / new_alpha;
+          gfloat layer_coeff = 1.0f / in_alpha - 1.0f;
           gint   b;
 
           for (b = RED; b < ALPHA; b++)
-            out[b] = (layer_alpha * (in_alpha * (comp_alpha * comp[b] - layer[b]) + layer[b] - in[b]) + 
in[b]) * recip;
+            out[b] = ratio * (layer_alpha * (comp_alpha * comp[b] + layer_coeff * layer[b] - in[b]) + in[b]);
         }
 
       out[ALPHA] = new_alpha;