diff --git a/Parser.Tests/MLexerGreenTests.cs b/Parser.Tests/MLexerGreenTests.cs index bf8a54f..a912d0b 100644 --- a/Parser.Tests/MLexerGreenTests.cs +++ b/Parser.Tests/MLexerGreenTests.cs @@ -1,4 +1,5 @@ -using Parser.Internal; +using System; +using Parser.Internal; using System.Collections.Generic; using System.Linq; using Xunit; @@ -24,12 +25,22 @@ namespace Parser.Tests public static IEnumerable SingleTokensData() { - return SingleTokens().Select(pair => new object[] { pair.kind, pair.text }); + return GetTokens().Select(pair => new object[] { pair.kind, pair.text }); } - public static IEnumerable<(TokenKind kind, string text)> SingleTokens() + public static IEnumerable<(TokenKind kind, string text)> GetTokens() { - return new[] + var fixedTokens = Enum.GetValues(typeof(TokenKind)) + .Cast() + .Select(k => (kind: k, text: SyntaxFacts.GetText(k))) + .Where(t => !(t.text is null)) + .Where(t => !(SyntaxFacts.IsUnaryTokenKind(t.kind) + || SyntaxFacts.IsOpeningToken(t.kind) + || SyntaxFacts.IsClosingToken(t.kind) + || t.kind == TokenKind.Transpose)); + + + var dynamicTokens = new[] { (TokenKind.Identifier, "a"), (TokenKind.Identifier, "abc"), @@ -39,44 +50,9 @@ namespace Parser.Tests (TokenKind.NumberLiteral, "14.5e-3"), (TokenKind.NumberLiteral, "3.14e8"), (TokenKind.StringLiteral, "'what is that'"), - (TokenKind.DoubleQuotedStringLiteral, "\"Another ' string\""), - - (TokenKind.Assignment, "="), - (TokenKind.Equality, "=="), - (TokenKind.Inequality, "~="), - (TokenKind.LogicalAnd, "&&"), - (TokenKind.LogicalOr, "||"), - (TokenKind.BitwiseAnd, "&"), - (TokenKind.BitwiseOr, "|"), - (TokenKind.Less, "<"), - (TokenKind.LessOrEqual, "<="), - (TokenKind.Greater, ">"), - (TokenKind.GreaterOrEqual, ">="), - (TokenKind.Not, "~"), - (TokenKind.Plus, "+"), - (TokenKind.Minus, "-"), - (TokenKind.Multiply, "*"), - (TokenKind.Divide, "/"), - (TokenKind.Power, "^"), - (TokenKind.Backslash, "\\"), - (TokenKind.DotMultiply, ".*"), - (TokenKind.DotDivide, "./"), - (TokenKind.DotPower, ".^"), - (TokenKind.DotBackslash, ".\\"), - (TokenKind.DotTranspose, ".'"), - (TokenKind.At, "@"), - (TokenKind.Colon, ":"), - (TokenKind.QuestionMark, "?"), - (TokenKind.Comma, ","), - (TokenKind.Semicolon, ";"), - //(TokenKind.OpeningBrace, "{"), - //(TokenKind.ClosingBrace, "}"), - //(TokenKind.OpeningSquareBracket, "["), - //(TokenKind.ClosingSquareBracket, "]"), - //(TokenKind.OpeningBracket, "("), - //(TokenKind.ClosingBracket, ")"), - (TokenKind.Dot, "."), + (TokenKind.DoubleQuotedStringLiteral, "\"Another ' string\"") }; + return fixedTokens.Concat(dynamicTokens); } } } diff --git a/Parser/Internal/SyntaxFacts.cs b/Parser/Internal/SyntaxFacts.cs index 39eaeff..a58cae5 100644 --- a/Parser/Internal/SyntaxFacts.cs +++ b/Parser/Internal/SyntaxFacts.cs @@ -158,13 +158,14 @@ namespace Parser.Internal private static readonly string[] StringFromKind = { null, // None = 0, - "", // EndOfFile = 1, - null, // Identifier = 2, - null, // NumberLiteral = 3, - null, // StringLiteral = 4, - null, // DoubleQuotedStringLiteral = 5, - null, // UnquotedStringLiteral = 6 - null, null, null, null, null, null, null, null, null, null, null, null, null, + null, // BadToken = 1, + null, // EndOfFile = 2, + null, // Identifier = 3, + null, // NumberLiteral = 4, + null, // StringLiteral = 5, + null, // DoubleQuotedStringLiteral = 6, + null, // UnquotedStringLiteral = 7 + null, null, null, null, null, null, null, null, null, null, null, null, "=", // Assignment = 20, "==", // Equality = 21, "~=", // Inequality = 22, @@ -201,7 +202,7 @@ namespace Parser.Internal "(", // OpeningBracket = 53, ")", // ClosingBracket = 54, ".", // Dot = 55, - "...", // DotDotDot = 56, + null, // DotDotDot = 56, // This is not used at the moment. "+", // UnaryPlus = 57, "-", // UnaryMinus = 58, @@ -211,7 +212,12 @@ namespace Parser.Internal public static string GetText(TokenKind kind) { - return StringFromKind[(int) kind]; + if ((int) kind < (int) TokenKind.File) + { + return StringFromKind[(int) kind]; + } + + return null; } public static bool IsUnaryOperator(TokenKind kind) @@ -228,6 +234,20 @@ namespace Parser.Internal } } + public static bool IsUnaryTokenKind(TokenKind kind) + { + switch (kind) + { + case TokenKind.UnaryPlus: + case TokenKind.UnaryMinus: + case TokenKind.UnaryNot: + case TokenKind.UnaryQuestionMark: + return true; + default: + return false; + } + } + public static bool IsBinaryOperator(TokenKind kind) { switch (kind)