diff --git a/Parser.Tests/MLexerGreenTests.cs b/Parser.Tests/MLexerGreenTests.cs new file mode 100644 index 0000000..bf8a54f --- /dev/null +++ b/Parser.Tests/MLexerGreenTests.cs @@ -0,0 +1,82 @@ +using Parser.Internal; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Parser.Tests +{ + public class MLexerGreenTests + { + private IEnumerable ParseText(string text) + { + var lexer = new MLexerGreen(new TextWindowWithNull(text)); + return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFile); + } + + [Theory] + [MemberData(nameof(SingleTokensData))] + public void MLexerGreen_Parses_Token(TokenKind kind, string text) + { + var tokens = ParseText(text); + var token = Assert.Single(tokens); + Assert.Equal(kind, token.Kind); + } + + public static IEnumerable SingleTokensData() + { + return SingleTokens().Select(pair => new object[] { pair.kind, pair.text }); + } + + public static IEnumerable<(TokenKind kind, string text)> SingleTokens() + { + return new[] + { + (TokenKind.Identifier, "a"), + (TokenKind.Identifier, "abc"), + (TokenKind.NumberLiteral, "1"), + (TokenKind.NumberLiteral, "123"), + (TokenKind.NumberLiteral, "145.67"), + (TokenKind.NumberLiteral, "14.5e-3"), + (TokenKind.NumberLiteral, "3.14e8"), + (TokenKind.StringLiteral, "'what is that'"), + (TokenKind.DoubleQuotedStringLiteral, "\"Another ' string\""), + + (TokenKind.Assignment, "="), + (TokenKind.Equality, "=="), + (TokenKind.Inequality, "~="), + (TokenKind.LogicalAnd, "&&"), + (TokenKind.LogicalOr, "||"), + (TokenKind.BitwiseAnd, "&"), + (TokenKind.BitwiseOr, "|"), + (TokenKind.Less, "<"), + (TokenKind.LessOrEqual, "<="), + (TokenKind.Greater, ">"), + (TokenKind.GreaterOrEqual, ">="), + (TokenKind.Not, "~"), + (TokenKind.Plus, "+"), + (TokenKind.Minus, "-"), + (TokenKind.Multiply, "*"), + (TokenKind.Divide, "/"), + (TokenKind.Power, "^"), + (TokenKind.Backslash, "\\"), + (TokenKind.DotMultiply, ".*"), + (TokenKind.DotDivide, "./"), + (TokenKind.DotPower, ".^"), + (TokenKind.DotBackslash, ".\\"), + (TokenKind.DotTranspose, ".'"), + (TokenKind.At, "@"), + (TokenKind.Colon, ":"), + (TokenKind.QuestionMark, "?"), + (TokenKind.Comma, ","), + (TokenKind.Semicolon, ";"), + //(TokenKind.OpeningBrace, "{"), + //(TokenKind.ClosingBrace, "}"), + //(TokenKind.OpeningSquareBracket, "["), + //(TokenKind.ClosingSquareBracket, "]"), + //(TokenKind.OpeningBracket, "("), + //(TokenKind.ClosingBracket, ")"), + (TokenKind.Dot, "."), + }; + } + } +} diff --git a/Parser/AssemblyInfo.cs b/Parser/AssemblyInfo.cs new file mode 100644 index 0000000..02a04f1 --- /dev/null +++ b/Parser/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly:InternalsVisibleTo("Parser.Tests")] diff --git a/Parser/Internal/MLexerGreen.cs b/Parser/Internal/MLexerGreen.cs index 2156128..6f53ef0 100644 --- a/Parser/Internal/MLexerGreen.cs +++ b/Parser/Internal/MLexerGreen.cs @@ -445,7 +445,7 @@ namespace Parser.Internal private bool ContinueLexingDoubleQuotedStringLiteral(ref TokenInfo tokenInfo) { ContinueLexingGeneralStringLiteral(ref tokenInfo, '"'); - tokenInfo.Kind = TokenKind.StringLiteral; + tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteral; return true; } @@ -851,6 +851,13 @@ namespace Parser.Internal tokenInfo.StringValue, leadingTrivia, trailingTrivia); + case TokenKind.DoubleQuotedStringLiteral: + return TokenFactory.CreateTokenWithValueAndTrivia( + tokenInfo.Kind, + tokenInfo.Text, + tokenInfo.StringValue, + leadingTrivia, + trailingTrivia); default: return TokenFactory.CreateTokenWithTrivia( tokenInfo.Kind, diff --git a/Parser/MParser.cs b/Parser/MParser.cs index 9a3ffd4..e9e5db0 100644 --- a/Parser/MParser.cs +++ b/Parser/MParser.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using Parser.Internal; namespace Parser