[gnome-builder/wip/file-loader: 1/5] context: do not rely on Vcs to discover project files

[Christian Hergert <chergert src gnome org>]

commit 85dc2ab9dc6557c49a76504ba3791bfbaf6c8c3d
Author: Christian Hergert <christian hergert me>
Date:   Mon Apr 13 01:03:54 2015 -0700

    context: do not rely on Vcs to discover project files
    
    This changes how project files are loaded, which is far more convenient
    now that we have Vcs plumbing in place.
    
     1) Discover the build system (configure.ac, etc)
     2) Discover the VCS root directory (startying from configure.ac)
     3) Load files starting from the Vcs:working-directory, but ignoring
        files as suggested by the IdeVcs.
    
    This requires an in-flight patch to libgit2-glib.

 libide/directory/ide-directory-vcs.c   |   50 ++++++++++++++----
 libide/git/ide-git-vcs.c               |   36 +++++++++++--
 libide/ide-context.c                   |   40 ++++++++++++++
 libide/ide-vcs.c                       |   13 +++++
 libide/ide-vcs.h                       |   12 +++-
 libide/tasks/ide-load-directory-task.c |   90 +++++---------------------------
 6 files changed, 146 insertions(+), 95 deletions(-)
---
diff --git a/libide/directory/ide-directory-vcs.c b/libide/directory/ide-directory-vcs.c
index ec54ac7..e063864 100644
--- a/libide/directory/ide-directory-vcs.c
+++ b/libide/directory/ide-directory-vcs.c
@@ -22,20 +22,13 @@
 #include "ide-directory-vcs.h"
 #include "ide-project.h"
 #include "ide-project-files.h"
-#include "tasks/ide-load-directory-task.h"
-
-/*
- * TODO: This all needs to be written synchronously on a thread so that we
- *       cant hit the case of too many open files. Right now, child directories
- *       are done async, and that can fail.
- */
 
 typedef struct
 {
   GFile *working_directory;
 } IdeDirectoryVcsPrivate;
 
-#define LOAD_MAX_FILES 2000
+#define LOAD_MAX_FILES 5000
 
 static void async_initable_iface_init (GAsyncInitableIface *iface);
 
@@ -55,6 +48,41 @@ ide_directory_vcs_get_working_directory (IdeVcs *vcs)
   return priv->working_directory;
 }
 
+static gboolean
+ide_directory_vcs_is_ignored (IdeVcs  *vcs,
+                              GFile   *file,
+                              GError **error)
+{
+  g_autofree gchar *reversed = NULL;
+  g_autofree gchar *name = NULL;
+
+  g_assert (IDE_IS_VCS (vcs));
+  g_assert (G_IS_FILE (file));
+
+  name = g_file_get_basename (file);
+  reversed = g_strreverse (name);
+
+  /* check suffixes, in reverse */
+  if ((reversed [0] == '~') ||
+      (strncmp (reversed, "al.", 3) == 0) ||        /* .la */
+      (strncmp (reversed, "ol.", 3) == 0) ||        /* .lo */
+      (strncmp (reversed, "o.", 2) == 0) ||         /* .o */
+      (strncmp (reversed, "pws.", 4) == 0) ||       /* .swp */
+      (strncmp (reversed, "sped.", 5) == 0) ||      /* .deps */
+      (strncmp (reversed, "sbil.", 5) == 0) ||      /* .libs */
+      (strncmp (reversed, "cyp.", 4) == 0) ||       /* .pyc */
+      (strncmp (reversed, "oyp.", 4) == 0) ||       /* .pyo */
+      (strncmp (reversed, "omg.", 4) == 0) ||       /* .gmo */
+      (strncmp (reversed, "tig.", 4) == 0) ||       /* .git */
+      (strncmp (reversed, "rzb.", 4) == 0) ||       /* .bzr */
+      (strncmp (reversed, "nvs.", 4) == 0) ||       /* .svn */
+      (strncmp (reversed, "pmatsrid.", 9) == 0) ||  /* .dirstamp */
+      (strncmp (reversed, "hcg.", 4) == 0))         /* .gch */
+    return TRUE;
+
+  return FALSE;
+}
+
 static void
 ide_directory_vcs_dispose (GObject *object)
 {
@@ -73,6 +101,7 @@ ide_directory_vcs_class_init (IdeDirectoryVcsClass *klass)
   GObjectClass *object_class = G_OBJECT_CLASS (klass);
 
   vcs_class->get_working_directory = ide_directory_vcs_get_working_directory;
+  vcs_class->is_ignored = ide_directory_vcs_is_ignored;
 
   object_class->dispose = ide_directory_vcs_dispose;
 }
@@ -114,9 +143,8 @@ ide_directory_vcs_init_async (GAsyncInitable      *initable,
                         NULL);
   ide_project_item_append (root, files);
 
-  task = ide_load_directory_task_new (self, directory, files, LOAD_MAX_FILES,
-                                      io_priority, cancellable, callback,
-                                      user_data);
+  task = g_task_new (self, cancellable, callback, user_data);
+  g_task_return_boolean (task, TRUE);
 
   g_object_unref (files);
   g_object_unref (task);
diff --git a/libide/git/ide-git-vcs.c b/libide/git/ide-git-vcs.c
index 989feff..d18d139 100644
--- a/libide/git/ide-git-vcs.c
+++ b/libide/git/ide-git-vcs.c
@@ -519,11 +519,14 @@ ide_git_vcs_reload__build_tree_cb (GObject      *object,
       context = ide_object_get_context (IDE_OBJECT (self));
       project = ide_context_get_project (context);
 
-      ide_project_writer_lock (project);
-      root = ide_project_get_root (project);
-      /* TODO: Replace existing item!!! */
-      ide_project_item_append (root, IDE_PROJECT_ITEM (files));
-      ide_project_writer_unlock (project);
+      if (FALSE)
+        {
+          ide_project_writer_lock (project);
+          root = ide_project_get_root (project);
+          /* TODO: Replace existing item!!! */
+          ide_project_item_append (root, IDE_PROJECT_ITEM (files));
+          ide_project_writer_unlock (project);
+        }
 
       self->loaded_files = TRUE;
     }
@@ -651,6 +654,28 @@ ide_git_vcs_reload_finish (IdeGitVcs     *self,
   return g_task_propagate_boolean (task, error);
 }
 
+static gboolean
+ide_git_vcs_is_ignored (IdeVcs  *vcs,
+                        GFile   *file,
+                        GError **error)
+{
+  g_autofree gchar *name = NULL;
+  IdeGitVcs *self = (IdeGitVcs *)vcs;
+
+  g_assert (IDE_IS_GIT_VCS (self));
+  g_assert (G_IS_FILE (file));
+
+  name = g_file_get_relative_path (self->working_directory, file);
+
+  if (g_strcmp0 (name, ".git") == 0)
+    return TRUE;
+
+  if (name != NULL)
+    return ggit_repository_path_is_ignored (self->repository, name);
+
+  return FALSE;
+}
+
 static void
 ide_git_vcs_dispose (GObject *object)
 {
@@ -706,6 +731,7 @@ ide_git_vcs_class_init (IdeGitVcsClass *klass)
 
   vcs_class->get_working_directory = ide_git_vcs_get_working_directory;
   vcs_class->get_buffer_change_monitor = ide_git_vcs_get_buffer_change_monitor;
+  vcs_class->is_ignored = ide_git_vcs_is_ignored;
 
   /**
    * IdeGitVcs:repository:
diff --git a/libide/ide-context.c b/libide/ide-context.c
index 4ff350c..e652e61 100644
--- a/libide/ide-context.c
+++ b/libide/ide-context.c
@@ -30,7 +30,10 @@
 #include "ide-device-manager.h"
 #include "ide-global.h"
 #include "ide-internal.h"
+#include "ide-load-directory-task.h"
 #include "ide-project.h"
+#include "ide-project-item.h"
+#include "ide-project-files.h"
 #include "ide-script-manager.h"
 #include "ide-search-engine.h"
 #include "ide-search-provider.h"
@@ -846,6 +849,42 @@ ide_context_init_project_name (gpointer             source_object,
 }
 
 static void
+ide_context_init_files (gpointer             source_object,
+                        GCancellable        *cancellable,
+                        GAsyncReadyCallback  callback,
+                        gpointer             user_data)
+{
+  g_autoptr(GTask) task = NULL;
+  g_autoptr(IdeProjectItem) files = NULL;
+  IdeContext *context = source_object;
+  IdeProject *project;
+  IdeProjectItem *root;
+  IdeVcs *vcs;
+  GFile *workdir;
+
+  g_assert (IDE_IS_CONTEXT (context));
+
+  vcs = ide_context_get_vcs (context);
+  workdir = ide_vcs_get_working_directory (vcs);
+  project = ide_context_get_project (context);
+  root = ide_project_get_root (project);
+  files = g_object_new (IDE_TYPE_PROJECT_FILES,
+                        "context", context,
+                        "parent", root,
+                        NULL);
+  ide_project_item_append (root, files);
+
+  task = ide_load_directory_task_new (context,
+                                      workdir,
+                                      files,
+                                      0,
+                                      G_PRIORITY_DEFAULT,
+                                      cancellable,
+                                      callback,
+                                      user_data);
+}
+
+static void
 ide_context_init_vcs_cb (GObject      *object,
                          GAsyncResult *result,
                          gpointer      user_data)
@@ -1282,6 +1321,7 @@ ide_context_init_async (GAsyncInitable      *initable,
                         ide_context_init_services,
                         ide_context_init_build_system,
                         ide_context_init_vcs,
+                        ide_context_init_files,
                         ide_context_init_project_name,
                         ide_context_init_back_forward_list,
                         ide_context_init_unsaved_files,
diff --git a/libide/ide-vcs.c b/libide/ide-vcs.c
index 6485f9b..0648c69 100644
--- a/libide/ide-vcs.c
+++ b/libide/ide-vcs.c
@@ -32,6 +32,19 @@ ide_vcs_init (IdeVcs *self)
 {
 }
 
+gboolean
+ide_vcs_is_ignored (IdeVcs  *self,
+                    GFile   *file,
+                    GError **error)
+{
+  g_return_val_if_fail (IDE_IS_VCS (self), FALSE);
+
+  if (IDE_VCS_GET_CLASS (self)->is_ignored)
+    return IDE_VCS_GET_CLASS (self)->is_ignored (self, file, error);
+
+  return FALSE;
+}
+
 /**
  * ide_vcs_get_working_directory:
  * @self: An #IdeVcs.
diff --git a/libide/ide-vcs.h b/libide/ide-vcs.h
index 74897cc..48f0181 100644
--- a/libide/ide-vcs.h
+++ b/libide/ide-vcs.h
@@ -34,9 +34,12 @@ struct _IdeVcsClass
 {
   IdeObjectClass parent;
 
-  GFile                  *(*get_working_directory)     (IdeVcs    *self);
-  IdeBufferChangeMonitor *(*get_buffer_change_monitor) (IdeVcs    *self,
-                                                        IdeBuffer *buffer);
+  GFile                  *(*get_working_directory)     (IdeVcs     *self);
+  IdeBufferChangeMonitor *(*get_buffer_change_monitor) (IdeVcs     *self,
+                                                        IdeBuffer  *buffer);
+  gboolean                (*is_ignored)                (IdeVcs     *self,
+                                                        GFile      *file,
+                                                        GError    **error);
 };
 
 IdeBufferChangeMonitor *ide_vcs_get_buffer_change_monitor (IdeVcs               *self,
@@ -49,6 +52,9 @@ void                    ide_vcs_new_async                 (IdeContext
                                                            gpointer              user_data);
 IdeVcs                 *ide_vcs_new_finish                (GAsyncResult         *result,
                                                            GError              **error);
+gboolean                ide_vcs_is_ignored                (IdeVcs               *self,
+                                                           GFile                *file,
+                                                           GError              **error);
 
 G_END_DECLS
 
diff --git a/libide/tasks/ide-load-directory-task.c b/libide/tasks/ide-load-directory-task.c
index 0cc04e9..0873f39 100644
--- a/libide/tasks/ide-load-directory-task.c
+++ b/libide/tasks/ide-load-directory-task.c
@@ -23,6 +23,7 @@
 #include "ide-project.h"
 #include "ide-project-item.h"
 #include "ide-project-file.h"
+#include "ide-vcs.h"
 
 #define DEFAULT_MAX_FILES 10000
 
@@ -73,33 +74,6 @@ ide_load_directory_task_free (gpointer data)
 }
 
 static gboolean
-is_special_directory (GFile *directory)
-{
-  g_autofree gchar *path = NULL;
-  g_autofree gchar *name = NULL;
-  guint i;
-
-  /* ignore dot directories */
-  name = g_file_get_basename (directory);
-  if (name [0] == '.')
-    return TRUE;
-
-  /* if this is a remove uri, then its not special */
-  path = g_file_get_path (directory);
-  if (!path)
-    return FALSE;
-
-  /* check for various xdg special dirs */
-  for (i = 0; i < G_N_ELEMENTS (gSpecialDirs); i++)
-    {
-      if (0 == g_strcmp0 (path, gSpecialDirs [i]))
-        return TRUE;
-    }
-
-  return FALSE;
-}
-
-static gboolean
 is_home_directory (GFile *directory)
 {
   g_autofree gchar *path = NULL;
@@ -111,35 +85,6 @@ is_home_directory (GFile *directory)
 }
 
 static gboolean
-is_ignored_file (const gchar *display_name)
-{
-  g_autofree gchar *reversed = g_strreverse (g_strdup (display_name));
-
-  if (!reversed)
-    return TRUE;
-
-  /* check suffixes, in reverse */
-  if ((reversed [0] == '~') ||
-      (strncmp (reversed, "al.", 3) == 0) ||   /* .la */
-      (strncmp (reversed, "ol.", 3) == 0) ||   /* .lo */
-      (strncmp (reversed, "o.", 2) == 0) ||    /* .o */
-      (strncmp (reversed, "pws.", 4) == 0) ||  /* .swp */
-      (strncmp (reversed, "sped.", 5) == 0) || /* .deps */
-      (strncmp (reversed, "sbil.", 5) == 0) || /* .libs */
-      (strncmp (reversed, "cyp.", 4) == 0) ||  /* .pyc */
-      (strncmp (reversed, "oyp.", 4) == 0) ||  /* .pyo */
-      (strncmp (reversed, "omg.", 4) == 0) ||  /* .gmo */
-      (strncmp (reversed, "tig.", 4) == 0) ||  /* .git */
-      (strncmp (reversed, "rzb.", 4) == 0) ||  /* .bzr */
-      (strncmp (reversed, "nvs.", 4) == 0) ||  /* .svn */
-      (strncmp (reversed, "pmatsrid.", 9) == 0) ||  /* .dirstamp */
-      (strncmp (reversed, "hcg.", 4) == 0))    /* .gch */
-    return TRUE;
-
-  return FALSE;
-}
-
-static gboolean
 ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
                                         GFile                 *directory,
                                         GError               **error)
@@ -148,6 +93,7 @@ ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
   IdeProjectItem *parent = NULL;
   g_autoptr(GPtrArray) directories = NULL;
   GFileInfo *child_info;
+  IdeVcs *vcs;
   GFileType file_type;
   GError *local_error = NULL;
   gsize i;
@@ -187,12 +133,6 @@ ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
     return TRUE;
 
   /*
-   * If this is a special directory (.git, Music, Pictures, etc), ignore it.
-   */
-  if (is_special_directory (directory))
-    return TRUE;
-
-  /*
    * Get an enumerator for children in this directory.
    */
   children = g_file_enumerate_children (directory,
@@ -212,6 +152,11 @@ ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
   parent = g_hash_table_lookup (self->directories, directory) ?: self->parent;
 
   /*
+   * Get a handle to our vcs, which is used to check for ignored files.
+   */
+  vcs = ide_context_get_vcs (self->context);
+
+  /*
    * Walk the children to inflate their IdeProjectFile instances.
    */
   while ((child_info = g_file_enumerator_next_file (children, self->cancellable, &local_error)))
@@ -219,27 +164,22 @@ ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
       g_autoptr(IdeProjectItem) item = NULL;
       g_autoptr(GFile) file = NULL;
       g_autofree gchar *path = NULL;
-      gboolean can_execute;
       GFileType file_type;
-      const gchar *display_name;
       const gchar *name;
 
       name = g_file_info_get_attribute_byte_string (child_info, G_FILE_ATTRIBUTE_STANDARD_NAME);
-      display_name = g_file_info_get_attribute_string (child_info,
-                                                       G_FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME);
       file_type = g_file_info_get_attribute_uint32 (child_info, G_FILE_ATTRIBUTE_STANDARD_TYPE);
-      can_execute = g_file_info_get_attribute_boolean (child_info,
-                                                       G_FILE_ATTRIBUTE_ACCESS_CAN_EXECUTE);
 
       switch (file_type)
         {
         case G_FILE_TYPE_DIRECTORY:
+          file = g_file_get_child (directory, name);
+
           /* check for known ignored files */
-          if (is_ignored_file (display_name))
+          if (ide_vcs_is_ignored (vcs, file, NULL))
             break;
 
           /* add the file item to the project tree */
-          file = g_file_get_child (directory, name);
           path = g_file_get_relative_path (self->directory, file);
           item = g_object_new (IDE_TYPE_PROJECT_FILE,
                                "context", self->context,
@@ -260,12 +200,13 @@ ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
           break;
 
         case G_FILE_TYPE_REGULAR:
-          /* ignore executables and known ignored files */
-          if (can_execute || is_ignored_file (display_name))
+          file = g_file_get_child (directory, name);
+
+          /* check for known ignored files */
+          if (ide_vcs_is_ignored (vcs, file, NULL))
             break;
 
           /* add the file item to the project tree */
-          file = g_file_get_child (directory, name);
           path = g_file_get_relative_path (self->directory, file);
           item = g_object_new (IDE_TYPE_PROJECT_FILE,
                                "context", self->context,
@@ -317,9 +258,6 @@ ide_load_directory_task_load_directory (IdeLoadDirectoryTask  *self,
         {
           GFile *file = g_ptr_array_index (directories, i);
 
-          if (is_special_directory (file))
-            continue;
-
           if (!ide_load_directory_task_load_directory (self, file, error))
             return FALSE;
         }