[shotwell/wip/phako/enhanced-faces: 28/136] Added DNN load and face-to-vec convertor function

[Jens Georg <jensgeorg src gnome org>]

commit f8b7df3089bbffb1c8deb70a9a2e5262c4ff9fde
Author: NarendraMA <narendra_m_a yahoo com>
Date:   Sun Jul 15 16:18:22 2018 +0530

    Added DNN load and face-to-vec convertor function

 facedetect/facedetect-opencv.cpp        | 79 +++++++++++++++++++++++++--------
 facedetect/org.gnome.ShotwellFaces1.xml | 28 +++++-------
 facedetect/shotwell-facedetect.cpp      | 47 ++++++++++++--------
 facedetect/shotwell-facedetect.hpp      | 18 ++++----
 4 files changed, 110 insertions(+), 62 deletions(-)
---
diff --git a/facedetect/facedetect-opencv.cpp b/facedetect/facedetect-opencv.cpp
index fd6394a0..9bf151c0 100644
--- a/facedetect/facedetect-opencv.cpp
+++ b/facedetect/facedetect-opencv.cpp
@@ -1,37 +1,35 @@
 #include "shotwell-facedetect.hpp"
 
-using namespace std;
-using namespace cv;
-
-vector<FaceRect> detectFaces(String inputName, String cascadeName, double scale) {
-       CascadeClassifier cascade;
+// Detect faces in a photo
+std::vector<FaceRect> detectFaces(cv::String inputName, cv::String cascadeName, double scale) {
+    cv::CascadeClassifier cascade;
        if (!cascade.load(cascadeName)) {
-               cout << "error;Could not load classifier cascade. Filename: \"" << cascadeName << "\"" << 
endl;
+        std::cout << "error;Could not load classifier cascade. Filename: \"" << cascadeName << "\"" << 
std::endl;
        }
 
        if (inputName.empty()) {
-               cout << "error;You must specify the file to process." << endl;
+        std::cout << "error;You must specify the file to process." << std::endl;
        }
 
-       Mat img = imread(inputName, 1);
+    cv::Mat img = cv::imread(inputName, 1);
        if (img.empty()) {
-               cout << "error;Could not load the file to process. Filename: \"" << inputName << "\"" << endl;
+        std::cout << "error;Could not load the file to process. Filename: \"" << inputName << "\"" << 
std::endl;
        }
     
-    Mat gray;
+    cv::Mat gray;
     cvtColor(img, gray, CV_BGR2GRAY);
 
-    Mat smallImg(cvRound(img.rows / scale), cvRound(img.cols / scale), CV_8UC1);
-    Size smallImgSize = smallImg.size();
+    cv::Mat smallImg(cvRound(img.rows / scale), cvRound(img.cols / scale), CV_8UC1);
+    cv::Size smallImgSize = smallImg.size();
 
-    resize(gray, smallImg, smallImgSize, 0, 0, INTER_LINEAR);
-    equalizeHist(smallImg, smallImg);
+    cv::resize(gray, smallImg, smallImgSize, 0, 0, cv::INTER_LINEAR);
+    cv::equalizeHist(smallImg, smallImg);
 
-    vector<Rect> faces;
-    cascade.detectMultiScale(smallImg, faces, 1.1, 2, CV_HAAR_SCALE_IMAGE, Size(30, 30));
+    std::vector<cv::Rect> faces;
+    cascade.detectMultiScale(smallImg, faces, 1.1, 2, CV_HAAR_SCALE_IMAGE, cv::Size(30, 30));
 
-    vector<FaceRect> scaled;
-    for (vector<Rect>::const_iterator r = faces.begin(); r != faces.end(); r++) {
+    std::vector<FaceRect> scaled;
+    for (std::vector<cv::Rect>::const_iterator r = faces.begin(); r != faces.end(); r++) {
         FaceRect i;
         i.x = (float) r->x / smallImgSize.width;
         i.y = (float) r->y / smallImgSize.height;
@@ -42,3 +40,48 @@ vector<FaceRect> detectFaces(String inputName, String cascadeName, double scale)
 
     return scaled;
 }
+
+// Load network into global var
+bool loadNet(cv::String netFile) {
+    try {
+        faceRecogNet = cv::dnn::readNetFromTorch(netFile);
+    } catch(cv::Exception e) {
+        std::cout << "File load failed: " << e.msg << std::endl;
+        return false;
+    }
+    if (faceRecogNet.empty()) {
+        std::cout << "Loading net " << netFile << " failed!" << std::endl;
+        return false;
+    } else {
+        std::cout << "Loaded " << netFile << std::endl;
+        return true;
+    }
+}
+
+// Face to vector convertor
+// Adapted from OpenCV example:
+// https://github.com/opencv/opencv/blob/master/samples/dnn/js_face_recognition.html
+std::vector<double> faceToVec(cv::String inputName) {
+    std::vector<double> ret;
+    cv::Mat img = imread(inputName, 1);
+       if (img.empty()) {
+        std::cout << "error;Could not load the file to process. Filename: \"" << inputName << "\"" << 
std::endl;
+        return ret;
+       }
+
+    cv::Mat smallImg(96, 96, CV_8UC1);
+    cv::Size smallImgSize = smallImg.size();
+    cv::resize(img, smallImg, smallImgSize, 0, 0, cv::INTER_LINEAR);
+
+    // Generate 128 element face vector using DNN
+    cv::Mat blob = cv::dnn::blobFromImage(smallImg, 1.0 / 255, smallImgSize,
+                                          cv::Scalar(), true, false);
+    faceRecogNet.setInput(blob);
+    std::cout << "Starting recognition on " << inputName << " blob height " <<
+        blob.size().height << " width " << blob.size().width << std::endl;
+    cv::Mat vec = faceRecogNet.forward();
+    std::cout << "Recognition done!" << std::endl;
+    // Return vector
+    ret.assign((double*)vec.datastart, (double*)vec.dataend);
+    return ret;
+}
diff --git a/facedetect/org.gnome.ShotwellFaces1.xml b/facedetect/org.gnome.ShotwellFaces1.xml
index 17d1ba1b..26bb321b 100644
--- a/facedetect/org.gnome.ShotwellFaces1.xml
+++ b/facedetect/org.gnome.ShotwellFaces1.xml
@@ -22,31 +22,23 @@
     </method>
 
     <!--
-        TrainFaces
-        @images: List of image files with a face in it
-        @labels: List of labels for each face
-        @model: File name to save model in
+        LoadNet
+        @net: Torch t7 net file
         Returns non-zero on any error
     -->
-    <method name="TrainFaces">
-      <arg type="as" name="images" direction="in" />
-      <arg type="as" name="labels" direction="in" />
-      <arg type="s" name="model" direction="in" />
-      <arg type="y" name="status" direction="out" />
+    <method name="LoadNet">
+      <arg type="s" name="net" direction="in" />
+      <arg type="b" name="ret" direction="out" />
     </method>
 
     <!--
-        RecogniseFace
-        @image: Image of face to recognise
-        @model: File name to read model from
-        @threshold: Probability threshold
-        Returns array of (label, probability) pairs 
+        FaceToVec
+        @image: Image of face to convert
+        Returns 128 element vector
     -->
-    <method name="RecogniseFace">
+    <method name="FaceToVec">
       <arg type="s" name="image" direction="in" />
-      <arg type="s" name="model" direction="in" />
-      <arg type="d" name="threshold" direction="in" />
-      <arg type="a(sd)" name="labels" direction="out" />
+      <arg type="ad" name="vec" direction="out" />
     </method>
 
     <!--
diff --git a/facedetect/shotwell-facedetect.cpp b/facedetect/shotwell-facedetect.cpp
index a1f75f60..8061a844 100644
--- a/facedetect/shotwell-facedetect.cpp
+++ b/facedetect/shotwell-facedetect.cpp
@@ -11,9 +11,6 @@
 #include "shotwell-facedetect.hpp"
 #include "dbus-interface.h"
 
-using namespace std;
-using namespace cv;
-
 // DBus binding functions
 static gboolean on_handle_detect_faces(ShotwellFaces1 *object,
                                        GDBusMethodInvocation *invocation,
@@ -22,11 +19,11 @@ static gboolean on_handle_detect_faces(ShotwellFaces1 *object,
                                        gdouble arg_scale) {
     GVariantBuilder *builder;
     GVariant *faces;
-    vector<FaceRect> rects = 
+    std::vector<FaceRect> rects = 
         detectFaces(arg_image, arg_cascade, arg_scale);
     // Construct return value
     builder = g_variant_builder_new(G_VARIANT_TYPE ("a(dddd)"));
-    for (vector<FaceRect>::const_iterator r = rects.begin(); r != rects.end(); r++) {
+    for (std::vector<FaceRect>::const_iterator r = rects.begin(); r != rects.end(); r++) {
         GVariant *rect = g_variant_new("(dddd)", r->x, r->y, r->width, r->height);
         g_variant_builder_add(builder, "(dddd)", rect);
         g_debug("Returning %f,%f", r->x, r->y);
@@ -39,24 +36,36 @@ static gboolean on_handle_detect_faces(ShotwellFaces1 *object,
     return TRUE;
 }
 
-gboolean on_handle_train_faces(ShotwellFaces1 *object,
-                               GDBusMethodInvocation *invocation,
-                               const gchar *const *arg_images,
-                               const gchar *const *arg_labels,
-                               const gchar *arg_model) {
+static gboolean on_handle_load_net(ShotwellFaces1 *object,
+                                   GDBusMethodInvocation *invocation,
+                                   const gchar *arg_net) {
+    bool ret = loadNet(arg_net);
+    // Call return
+    shotwell_faces1_complete_load_net(object, invocation,
+                                      ret);
     return TRUE;
 }
 
-gboolean on_handle_recognise_face(ShotwellFaces1 *object,
-                                  GDBusMethodInvocation *invocation,
-                                  const gchar *arg_image,
-                                  const gchar *arg_model,
-                                  gdouble arg_threshold) {
+static gboolean on_handle_face_to_vec(ShotwellFaces1 *object,
+                                      GDBusMethodInvocation *invocation,
+                                      const gchar *arg_image) {
+    GVariantBuilder *builder;
+    GVariant *ret;
+    std::vector<double> vec = faceToVec(arg_image);
+    builder = g_variant_builder_new(G_VARIANT_TYPE ("ad"));
+    for (std::vector<double>::const_iterator r = vec.begin(); r != vec.end(); r++) {
+        GVariant *v = g_variant_new("d", *r);
+        g_variant_builder_add(builder, "d", v);
+    }
+    ret = g_variant_new("ad", builder);
+    g_variant_builder_unref(builder);
+    shotwell_faces1_complete_face_to_vec(object, invocation,
+                                         ret);
     return TRUE;
 }
 
-gboolean on_handle_terminate(ShotwellFaces1 *object,
-                             GDBusMethodInvocation *invocation) {
+static gboolean on_handle_terminate(ShotwellFaces1 *object,
+                                    GDBusMethodInvocation *invocation) {
     g_debug("Exiting...");
     exit(0);
     return TRUE;
@@ -70,11 +79,13 @@ static void on_name_acquired(GDBusConnection *connection,
     g_debug("Got name %s", name);
     g_signal_connect(interface, "handle-detect-faces", G_CALLBACK (on_handle_detect_faces), NULL);
     g_signal_connect(interface, "handle-terminate", G_CALLBACK (on_handle_terminate), NULL);
+    g_signal_connect(interface, "handle-load-net", G_CALLBACK (on_handle_load_net), NULL);
+    g_signal_connect(interface, "handle-face-to-vec", G_CALLBACK (on_handle_face_to_vec), NULL);
     error = NULL;
     !g_dbus_interface_skeleton_export(G_DBUS_INTERFACE_SKELETON(interface), connection, 
"/org/gnome/shotwell/faces", &error);
 }
 
-int main() {
+int main(int argc, char **argv) {
        GMainLoop *loop;
        loop = g_main_loop_new (NULL, FALSE);
        g_bus_own_name(G_BUS_TYPE_SESSION, "org.gnome.shotwell.faces", G_BUS_NAME_OWNER_FLAGS_NONE, NULL,
diff --git a/facedetect/shotwell-facedetect.hpp b/facedetect/shotwell-facedetect.hpp
index d861eded..0fdff2b7 100644
--- a/facedetect/shotwell-facedetect.hpp
+++ b/facedetect/shotwell-facedetect.hpp
@@ -7,9 +7,11 @@
  * Header file for facedetect/recognition routines
  */
 
-#include "opencv2/objdetect/objdetect.hpp"
-#include "opencv2/highgui/highgui.hpp"
-#include "opencv2/imgproc/imgproc.hpp"
+#include <opencv2/core/core.hpp>
+#include <opencv2/objdetect/objdetect.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include <opencv2/dnn.hpp>
 
 #include <iostream>
 #include <stdio.h>
@@ -18,9 +20,9 @@ typedef struct {
     float x, y, width, height;
 } FaceRect;
 
-using namespace std;
-using namespace cv;
+// Global variable for DNN to generate vector out of face
+static cv::dnn::Net faceRecogNet;
 
-vector<FaceRect> detectFaces(String inputName, String cascadeName, double scale);
-int trainFaces(vector<String> images, vector<String> labels, String modelFile);
-vector<pair<String, double>> recogniseFace(String image, String modelFile);
+bool loadNet(cv::String netFile);
+std::vector<FaceRect> detectFaces(cv::String inputName, cv::String cascadeName, double scale);
+std::vector<double> faceToVec(cv::String inputName);