diff --git a/Lexer.Tests/MLexerShould.cs b/Lexer.Tests/MLexerShould.cs index dc44621..0b1b518 100644 --- a/Lexer.Tests/MLexerShould.cs +++ b/Lexer.Tests/MLexerShould.cs @@ -1,8 +1,7 @@ using System.Linq; -using Lexer; using NUnit.Framework; -namespace Parser.Tests +namespace Lexer.Tests { public class MLexerShould { @@ -22,8 +21,9 @@ namespace Parser.Tests new[] {"undefined", "is", "not", "a", "function"}, tokens.Take(5).Select(token => token.PureToken.LiteralText)); CollectionAssert.AreEqual( - Enumerable.Repeat(TokenKind.Identifier, 5), - tokens.Take(5).Select(token => token.PureToken.Kind)); + new[] { TokenKind.Identifier, TokenKind.UnquotedStringLiteral, TokenKind.UnquotedStringLiteral, + TokenKind.Identifier, TokenKind.UnquotedStringLiteral }, + tokens.Take(5).Select(token => token.Kind)); } [Test] diff --git a/Lexer.Tests/TextWindowShould.cs b/Lexer.Tests/TextWindowShould.cs index 753bd70..27c3a50 100644 --- a/Lexer.Tests/TextWindowShould.cs +++ b/Lexer.Tests/TextWindowShould.cs @@ -1,7 +1,6 @@ -using Lexer; using NUnit.Framework; -namespace Parser.Tests +namespace Lexer.Tests { [TestFixture] public class TestWindowShould diff --git a/Lexer.Tests/TextWindowWithNullShould.cs b/Lexer.Tests/TextWindowWithNullShould.cs index c935ecc..17dbafe 100644 --- a/Lexer.Tests/TextWindowWithNullShould.cs +++ b/Lexer.Tests/TextWindowWithNullShould.cs @@ -1,7 +1,6 @@ -using Lexer; using NUnit.Framework; -namespace Parser.Tests +namespace Lexer.Tests { [TestFixture] public class TestWindowWithNullShould diff --git a/Lexer/MLexer.cs b/Lexer/MLexer.cs index 7683a91..cb0e559 100644 --- a/Lexer/MLexer.cs +++ b/Lexer/MLexer.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Text; @@ -363,9 +364,55 @@ namespace Lexer return PureTokenFactory.CreateDoubleQuotedStringLiteral(literal); } + private PureToken ContinueParsingUnquotedStringLiteral() + { + var n = 0; + while (true) + { + var c = Window.PeekChar(n); + if (c == ' ' || c == '\n' || c == '\0') + { + var literal = Window.GetAndConsumeChars(n); + return PureTokenFactory.CreateUnquotedStringLiteral(literal); + } + + n++; + } + } + + private static readonly HashSet Keywords; + + static MLexer() + { + Keywords = new HashSet + { + "for", "if", "function", "while", "case", "try", "catch", "end", + "switch", "classdef", "elseif", "persistent", + }; + } + private PureToken LexTokenWithoutTrivia(List leadingTrivia) { var character = Window.PeekChar(); + if (character == '\0') + { + return PureTokenFactory.CreateEndOfFileToken(); + } + + if (TokensSinceNewLine == 1 + && LastToken.Kind == TokenKind.Identifier + && LastToken.TrailingTrivia.Any() + && character != '=' + && character != '(' + && !Keywords.Contains(LastToken.PureToken.LiteralText)) + { + return ContinueParsingUnquotedStringLiteral(); + } + if (LastToken?.Kind == TokenKind.UnquotedStringLiteral && + TokensSinceNewLine > 0) + { + return ContinueParsingUnquotedStringLiteral(); + } switch (character) { case 'a': diff --git a/Lexer/PureTokenFactory.cs b/Lexer/PureTokenFactory.cs index ebb3727..73373b5 100644 --- a/Lexer/PureTokenFactory.cs +++ b/Lexer/PureTokenFactory.cs @@ -93,6 +93,11 @@ return new PureToken(TokenKind.DoubleQuotedStringLiteral, "\"" + s + "\"", s, Window.Position); } + public PureToken CreateUnquotedStringLiteral(string s) + { + return new PureToken(TokenKind.UnquotedStringLiteral, s, s, Window.Position); + } + public PureToken CreateEndOfFileToken() { return new PureToken(TokenKind.EndOfFile, "", null, Window.Position);