From 4c2f37b00099ee471c884d7845ba259eda7e394f Mon Sep 17 00:00:00 2001
From: Alexander Luzgarev <mahalex@gmail.com>
Date: Wed, 4 Apr 2018 20:21:50 +0200
Subject: [PATCH] Separate new lines trivia

---
 Lexer.Tests/MLexerShould.cs   |  2 +-
 Lexer/MLexer.cs               | 21 +++++++++++++++++----
 Lexer/PureTokenFactory.cs     | 12 +++++++-----
 Lexer/TokenKind.cs            |  1 +
 Lexer/TriviaType.cs           |  1 +
 Parser.Tests/MParserShould.cs |  1 +
 Parser/SyntaxNode.cs          | 18 ++++++++++++++++++
 7 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/Lexer.Tests/MLexerShould.cs b/Lexer.Tests/MLexerShould.cs
index 64bf718..dc44621 100644
--- a/Lexer.Tests/MLexerShould.cs
+++ b/Lexer.Tests/MLexerShould.cs
@@ -15,7 +15,7 @@ namespace Parser.Tests
         [Test]
         public void ParseSequenceOfIdentifiers()
         {
-            var sut = CreateLexer("undefined is not a function");
+            var sut = CreateLexer("undefined is not\n a function");
             var tokens = sut.ParseAll();
             Assert.AreEqual(6, tokens.Count);
             CollectionAssert.AreEqual(
diff --git a/Lexer/MLexer.cs b/Lexer/MLexer.cs
index f0a3e0f..7683a91 100644
--- a/Lexer/MLexer.cs
+++ b/Lexer/MLexer.cs
@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 
 namespace Lexer
@@ -7,6 +8,7 @@ namespace Lexer
     {
         private ITextWindow Window { get; }
         private Token LastToken { get; set; }
+        private int TokensSinceNewLine { get; set; }
         private PureTokenFactory PureTokenFactory { get; }
 
         public MLexer(ITextWindow window, PureTokenFactory pureTokenFactory)
@@ -67,16 +69,19 @@ namespace Lexer
                         break;
                     case '\r':
                     case '\n':
+                        if (whiteSpaceCache.Length > 0)
+                        {
+                            triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
+                        }
+
+                        whiteSpaceCache.Clear();                 
                         Window.ConsumeChar();
-                        whiteSpaceCache.Append(character);
-                        var whiteSpace = whiteSpaceCache.ToString();
-                        triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpace));
+                        triviaList.Add(new Trivia(TriviaType.NewLine, character.ToString()));
                         if (isTrailing)
                         {
                             return triviaList;
                         }
 
-                        whiteSpaceCache.Clear();
                         break;
                     case '%':
                         if (whiteSpaceCache.Length > 0)
@@ -595,6 +600,14 @@ namespace Lexer
             var leadingTrivia = LexTrivia(false);
             var token = LexTokenWithoutTrivia(leadingTrivia);
             var trailingTrivia = LexTrivia(true);
+            if (trailingTrivia.Where(t => t.Type == TriviaType.NewLine).Any())
+            {
+                TokensSinceNewLine = 0;
+            }
+            else
+            {
+                TokensSinceNewLine++;
+            }
 
             var result = new Token(token, leadingTrivia, trailingTrivia);
             LastToken = result;
diff --git a/Lexer/PureTokenFactory.cs b/Lexer/PureTokenFactory.cs
index 82e8fd4..ebb3727 100644
--- a/Lexer/PureTokenFactory.cs
+++ b/Lexer/PureTokenFactory.cs
@@ -12,11 +12,13 @@
         private static readonly string[] PureTokenOfKind =
         {
             null, // None = 0,
-            null, // Identifier = 1,
-            null, // NumberLiteral = 2,
-            null, // StringLiteral = 3,
-            null, // DoubleQuotedStringLiteral = 4,
-            null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
+            null, // EndOfFile = 1,
+            null, // Identifier = 2,
+            null, // NumberLiteral = 3,
+            null, // StringLiteral = 4,
+            null, // DoubleQuotedStringLiteral = 5,
+            null, // UnquotedStringLiteral = 6
+            null, null, null, null, null, null, null, null, null, null, null, null, null,
             "=", // Assignment = 20,
             "==", // Equality = 21,
             "~=", // Inequality = 22,
diff --git a/Lexer/TokenKind.cs b/Lexer/TokenKind.cs
index ea77db8..b842016 100644
--- a/Lexer/TokenKind.cs
+++ b/Lexer/TokenKind.cs
@@ -8,6 +8,7 @@
         NumberLiteral = 3,
         StringLiteral = 4,
         DoubleQuotedStringLiteral = 5,
+        UnquotedStringLiteral = 6,
         
         Assignment = 20,
         Equality = 21,
diff --git a/Lexer/TriviaType.cs b/Lexer/TriviaType.cs
index 91b4287..d7cba4a 100644
--- a/Lexer/TriviaType.cs
+++ b/Lexer/TriviaType.cs
@@ -3,6 +3,7 @@
     public enum TriviaType
     {
         Whitespace,
+        NewLine,
         Comment
     }
 }
\ No newline at end of file
diff --git a/Parser.Tests/MParserShould.cs b/Parser.Tests/MParserShould.cs
index 409fce2..736ae11 100644
--- a/Parser.Tests/MParserShould.cs
+++ b/Parser.Tests/MParserShould.cs
@@ -449,5 +449,6 @@ namespace Parser.Tests
             Assert.IsInstanceOf<TryCatchStatementNode>(actual);
             Assert.AreEqual(text, actual.FullText);
         }
+        
     }
 }
\ No newline at end of file
diff --git a/Parser/SyntaxNode.cs b/Parser/SyntaxNode.cs
index 8c188bc..1a3810c 100644
--- a/Parser/SyntaxNode.cs
+++ b/Parser/SyntaxNode.cs
@@ -335,6 +335,24 @@ namespace Parser
         }
     }
 
+    public class UnquotedStringLiteralNode : ExpressionNode
+    {
+        public Token Token { get; }
+
+        public UnquotedStringLiteralNode(Token token) : base(null)
+        {
+            Token = token;
+        }
+        
+        public override string FullText => Token.FullText;
+        
+        public override IEnumerable<Token> ChildTokens
+        {
+            get { yield return Token; }
+        }
+
+    }
+
     public class StatementNode : SyntaxNode
     {
         public TokenNode SemicolonOrComma { get; set; }