[gtk+/wip/otte/shader: 317/367] gsksltokenizer: Detect preprocessor directives

[Benjamin Otte <otte src gnome org>]

commit c940fa141d1d0efd9903e2e0660dd2a692725c81
Author: Benjamin Otte <otte redhat com>
Date:   Sat Sep 23 03:24:19 2017 +0200

    gsksltokenizer: Detect preprocessor directives
    
    ... and add code to skip and error about them.

 gsk/gskslpreprocessor.c     |  121 +++++++++++++++++++++++++++++++++++++++---
 gsk/gsksltokenizer.c        |   30 +++++++++--
 gsk/gsksltokenizerprivate.h |    2 +
 3 files changed, 138 insertions(+), 15 deletions(-)
---
diff --git a/gsk/gskslpreprocessor.c b/gsk/gskslpreprocessor.c
index 7f90ff6..c633a7f 100644
--- a/gsk/gskslpreprocessor.c
+++ b/gsk/gskslpreprocessor.c
@@ -93,29 +93,132 @@ static gboolean
 gsk_sl_token_is_skipped (const GskSlToken *token)
 {
   return gsk_sl_token_is (token, GSK_SL_TOKEN_ERROR)
+      || gsk_sl_token_is (token, GSK_SL_TOKEN_NEWLINE)
       || gsk_sl_token_is (token, GSK_SL_TOKEN_WHITESPACE)
       || gsk_sl_token_is (token, GSK_SL_TOKEN_COMMENT)
       || gsk_sl_token_is (token, GSK_SL_TOKEN_SINGLE_LINE_COMMENT);
 }
 
+static gboolean
+gsk_sl_preprocessor_next_token (GskSlPreprocessor *preproc)
+{
+  gboolean was_newline;
+  
+  do 
+    {
+      preproc->location = *gsk_sl_tokenizer_get_location (preproc->tokenizer);
+      was_newline = gsk_sl_token_is (&preproc->token, GSK_SL_TOKEN_NEWLINE);
+      gsk_sl_tokenizer_read_token (preproc->tokenizer, &preproc->token);
+    }
+  while (gsk_sl_token_is_skipped (&preproc->token));
+
+  return was_newline;
+}
+
 static void
-gsk_sl_token_ensure (GskSlPreprocessor *preproc)
+gsk_sl_preprocessor_handle_preprocessor_directive (GskSlPreprocessor *preproc)
 {
+  gboolean was_newline = gsk_sl_preprocessor_next_token (preproc);
+
+  /* empty # line */
+  if (was_newline)
+    return;
+
+  if (gsk_sl_token_is (&preproc->token, GSK_SL_TOKEN_IDENTIFIER))
+    {
+      if (g_str_equal (preproc->token.str, "define"))
+        {
+          gsk_sl_preprocessor_error (preproc, "Unknown preprocessor directive #define.");
+        }
+#if 0
+      else if (g_str_equal (preproc->token.str, "else"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "elif"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "endif"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "error"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "extension"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "if"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "ifdef"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "ifndef"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "line"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "pragma"))
+        {
+        }
+      else if (g_str_equal (preproc->token.str, "version"))
+        {
+        }
+#endif
+      else
+        {
+          gsk_sl_preprocessor_error (preproc, "Unknown preprocessor directive #%s.", preproc->token.str);
+        }
+    }
+  else if (gsk_sl_token_is (&preproc->token, GSK_SL_TOKEN_ELSE))
+    {
+      gsk_sl_preprocessor_error (preproc, "Unknown preprocessor directive #else.");
+    }
+  else
+    {
+      gsk_sl_preprocessor_error (preproc, "Missing identifier for preprocessor directive.");
+    }
+
+  while (!gsk_sl_preprocessor_next_token (preproc));
+}
+
+static void
+gsk_sl_preprocessor_ensure (GskSlPreprocessor *preproc)
+{
+  gboolean was_newline = FALSE;
+
   if (!gsk_sl_token_is (&preproc->token, GSK_SL_TOKEN_EOF))
     return;
 
-  do 
+  was_newline = gsk_sl_preprocessor_next_token (preproc);
+
+  while (TRUE)
     {
-      preproc->location = *gsk_sl_tokenizer_get_location (stream->tokenizer);
-      gsk_sl_tokenizer_read_token (preproc->tokenizer, &stream->token);
+      if (gsk_sl_token_is (&preproc->token, GSK_SL_TOKEN_HASH))
+        {
+          if (!was_newline &&
+              preproc->location.bytes != 0)
+            {
+              gsk_sl_preprocessor_error (preproc, "Unexpected \"#\" - preprocessor directives must be at 
start of line.");
+              was_newline = gsk_sl_preprocessor_next_token (preproc);
+            }
+          else
+            {
+              gsk_sl_preprocessor_handle_preprocessor_directive (preproc);
+              was_newline = TRUE;
+            }
+        }
+      else
+        {
+          break;
+        }
     }
-  while (gsk_sl_token_is_skipped (&preproc->token));
 }
 
 const GskSlToken *
 gsk_sl_preprocessor_get (GskSlPreprocessor *preproc)
 {
-  gsk_sl_token_ensure (preproc);
+  gsk_sl_preprocessor_ensure (preproc);
 
   return &preproc->token;
 }
@@ -123,7 +226,7 @@ gsk_sl_preprocessor_get (GskSlPreprocessor *preproc)
 const GskCodeLocation *
 gsk_sl_preprocessor_get_location (GskSlPreprocessor *preproc)
 {
-  gsk_sl_token_ensure (preproc);
+  gsk_sl_preprocessor_ensure (preproc);
 
   return &preproc->location;
 }
@@ -132,7 +235,7 @@ void
 gsk_sl_preprocessor_consume (GskSlPreprocessor *preproc,
                              GskSlNode         *consumer)
 {
-  gsk_sl_token_ensure (preproc);
+  gsk_sl_preprocessor_ensure (preproc);
 
   gsk_sl_token_clear (&preproc->token);
 }
@@ -152,7 +255,7 @@ gsk_sl_preprocessor_error (GskSlPreprocessor *preproc,
                               args);
   va_end (args);
 
-  gsk_sl_token_ensure (preproc);
+  gsk_sl_preprocessor_ensure (preproc);
   gsk_sl_preprocessor_error_func (preproc->tokenizer,
                                   TRUE,
                                   &preproc->location,
diff --git a/gsk/gsksltokenizer.c b/gsk/gsksltokenizer.c
index 1f8197e..f734d97 100644
--- a/gsk/gsksltokenizer.c
+++ b/gsk/gsksltokenizer.c
@@ -90,6 +90,7 @@ gsk_sl_token_clear (GskSlToken *token)
 
     case GSK_SL_TOKEN_EOF:
     case GSK_SL_TOKEN_ERROR:
+    case GSK_SL_TOKEN_NEWLINE:
     case GSK_SL_TOKEN_WHITESPACE:
     case GSK_SL_TOKEN_COMMENT:
     case GSK_SL_TOKEN_SINGLE_LINE_COMMENT:
@@ -296,6 +297,7 @@ gsk_sl_token_clear (GskSlToken *token)
     case GSK_SL_TOKEN_CARET:
     case GSK_SL_TOKEN_AMPERSAND:
     case GSK_SL_TOKEN_QUESTION:
+    case GSK_SL_TOKEN_HASH:
     case GSK_SL_TOKEN_INVARIANT:
     case GSK_SL_TOKEN_PRECISE:
     case GSK_SL_TOKEN_HIGH_PRECISION:
@@ -483,6 +485,7 @@ gsk_sl_token_print (const GskSlToken *token,
     case GSK_SL_TOKEN_SINGLE_LINE_COMMENT:
       break;
 
+    case GSK_SL_TOKEN_NEWLINE:
     case GSK_SL_TOKEN_WHITESPACE:
       g_string_append (string, " ");
       break;
@@ -860,6 +863,10 @@ gsk_sl_token_print (const GskSlToken *token,
       g_string_append_c (string, '?');
       break;
 
+    case GSK_SL_TOKEN_HASH:
+      g_string_append_c (string, '#');
+      break;
+
     default:
       g_assert_not_reached ();
       break;
@@ -975,8 +982,7 @@ is_whitespace (char c)
   return c == ' '
       || c == '\t'
       || c == '\f'
-      || c == '\n'
-      || c == '\r';
+      || is_newline (c);
 }
 
 static inline gsize
@@ -1183,11 +1189,18 @@ static void
 gsk_sl_token_reader_read_whitespace (GskSlTokenReader  *reader,
                                      GskSlToken        *token)
 {
-  do {
-    gsk_sl_token_reader_consume (reader, 1);
-  } while (is_whitespace (gsk_sl_token_reader_get (reader, 0)));
+  gboolean has_newline = FALSE;
+  char c;
 
-  gsk_sl_token_init (token, GSK_SL_TOKEN_WHITESPACE);
+  for (c = gsk_sl_token_reader_get (reader, 0);
+       is_whitespace (c);
+       c = gsk_sl_token_reader_get (reader, 0))
+    {
+      has_newline |= is_newline (c);
+      gsk_sl_token_reader_consume (reader, 1);
+    }
+
+  gsk_sl_token_init (token, has_newline ? GSK_SL_TOKEN_NEWLINE : GSK_SL_TOKEN_WHITESPACE);
 }
 
 static gboolean
@@ -1794,6 +1807,11 @@ gsk_sl_tokenizer_read_token (GskSlTokenizer *tokenizer,
       gsk_sl_token_reader_consume (&reader, 1);
       break;
 
+    case '#':
+      gsk_sl_token_init (token, GSK_SL_TOKEN_HASH);
+      gsk_sl_token_reader_consume (&reader, 1);
+      break;
+
     case ',':
       gsk_sl_token_init (token, GSK_SL_TOKEN_COMMA);
       gsk_sl_token_reader_consume (&reader, 1);
diff --git a/gsk/gsksltokenizerprivate.h b/gsk/gsksltokenizerprivate.h
index ea769fc..ec2fc92 100644
--- a/gsk/gsksltokenizerprivate.h
+++ b/gsk/gsksltokenizerprivate.h
@@ -25,6 +25,7 @@ G_BEGIN_DECLS
 typedef enum {
   GSK_SL_TOKEN_EOF = 0,
   GSK_SL_TOKEN_ERROR,
+  GSK_SL_TOKEN_NEWLINE,
   GSK_SL_TOKEN_WHITESPACE,
   GSK_SL_TOKEN_COMMENT,
   GSK_SL_TOKEN_SINGLE_LINE_COMMENT,
@@ -232,6 +233,7 @@ typedef enum {
   GSK_SL_TOKEN_CARET,
   GSK_SL_TOKEN_AMPERSAND,
   GSK_SL_TOKEN_QUESTION,
+  GSK_SL_TOKEN_HASH,
   GSK_SL_TOKEN_INVARIANT,
   GSK_SL_TOKEN_PRECISE,
   GSK_SL_TOKEN_HIGH_PRECISION,