Separate new lines trivia

Lexer.Tests/MLexerShould.cs

@@ -15,7 +15,7 @@ namespace Parser.Tests
         [Test]
         public void ParseSequenceOfIdentifiers()
         {
-            var sut = CreateLexer("undefined is not a function");
+            var sut = CreateLexer("undefined is not\n a function");
             var tokens = sut.ParseAll();
             Assert.AreEqual(6, tokens.Count);
             CollectionAssert.AreEqual(

Lexer/MLexer.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 
 namespace Lexer
@@ -7,6 +8,7 @@ namespace Lexer
     {
         private ITextWindow Window { get; }
         private Token LastToken { get; set; }
+        private int TokensSinceNewLine { get; set; }
         private PureTokenFactory PureTokenFactory { get; }
 
         public MLexer(ITextWindow window, PureTokenFactory pureTokenFactory)
@@ -67,16 +69,19 @@ namespace Lexer
                         break;
                     case '\r':
                     case '\n':
+                        if (whiteSpaceCache.Length > 0)
+                        {
+                            triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
+                        }
+
+                        whiteSpaceCache.Clear();                 
                         Window.ConsumeChar();
-                        whiteSpaceCache.Append(character);
+                        triviaList.Add(new Trivia(TriviaType.NewLine, character.ToString()));
-                        var whiteSpace = whiteSpaceCache.ToString();
-                        triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpace));
                         if (isTrailing)
                         {
                             return triviaList;
                         }
 
-                        whiteSpaceCache.Clear();
                         break;
                     case '%':
                         if (whiteSpaceCache.Length > 0)
@@ -595,6 +600,14 @@ namespace Lexer
             var leadingTrivia = LexTrivia(false);
             var token = LexTokenWithoutTrivia(leadingTrivia);
             var trailingTrivia = LexTrivia(true);
+            if (trailingTrivia.Where(t => t.Type == TriviaType.NewLine).Any())
+            {
+                TokensSinceNewLine = 0;
+            }
+            else
+            {
+                TokensSinceNewLine++;
+            }
 
             var result = new Token(token, leadingTrivia, trailingTrivia);
             LastToken = result;

Lexer/PureTokenFactory.cs

@@ -12,11 +12,13 @@
         private static readonly string[] PureTokenOfKind =
         {
             null, // None = 0,
-            null, // Identifier = 1,
+            null, // EndOfFile = 1,
-            null, // NumberLiteral = 2,
+            null, // Identifier = 2,
-            null, // StringLiteral = 3,
+            null, // NumberLiteral = 3,
-            null, // DoubleQuotedStringLiteral = 4,
+            null, // StringLiteral = 4,
-            null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
+            null, // DoubleQuotedStringLiteral = 5,
+            null, // UnquotedStringLiteral = 6
+            null, null, null, null, null, null, null, null, null, null, null, null, null,
             "=", // Assignment = 20,
             "==", // Equality = 21,
             "~=", // Inequality = 22,

Lexer/TokenKind.cs

@@ -8,6 +8,7 @@
         NumberLiteral = 3,
         StringLiteral = 4,
         DoubleQuotedStringLiteral = 5,
+        UnquotedStringLiteral = 6,
         
         Assignment = 20,
         Equality = 21,

Lexer/TriviaType.cs

@@ -3,6 +3,7 @@
     public enum TriviaType
     {
         Whitespace,
+        NewLine,
         Comment
     }
 }

Parser.Tests/MParserShould.cs

@@ -449,5 +449,6 @@ namespace Parser.Tests
             Assert.IsInstanceOf<TryCatchStatementNode>(actual);
             Assert.AreEqual(text, actual.FullText);
         }
+        
     }
 }

Parser/SyntaxNode.cs

@@ -335,6 +335,24 @@ namespace Parser
         }
     }
 
+    public class UnquotedStringLiteralNode : ExpressionNode
+    {
+        public Token Token { get; }
+
+        public UnquotedStringLiteralNode(Token token) : base(null)
+        {
+            Token = token;
+        }
+        
+        public override string FullText => Token.FullText;
+        
+        public override IEnumerable<Token> ChildTokens
+        {
+            get { yield return Token; }
+        }
+
+    }
+
     public class StatementNode : SyntaxNode
     {
         public TokenNode SemicolonOrComma { get; set; }