diff --git a/ConsoleDemo/DumbWalker.cs b/ConsoleDemo/DumbWalker.cs index 909db41..597e084 100644 --- a/ConsoleDemo/DumbWalker.cs +++ b/ConsoleDemo/DumbWalker.cs @@ -53,7 +53,7 @@ namespace ConsoleDemo { switch (token.Kind) { - case TokenKind.Comma: + case TokenKind.CommaToken: return true; default: break; diff --git a/ConsoleDemo/PrettyPrinter.cs b/ConsoleDemo/PrettyPrinter.cs index 143aa82..585a1c8 100644 --- a/ConsoleDemo/PrettyPrinter.cs +++ b/ConsoleDemo/PrettyPrinter.cs @@ -150,7 +150,7 @@ namespace ProjectConsole if (nodeOrToken.IsToken) { var token = nodeOrToken.AsToken(); - if (token.Kind == TokenKind.Identifier) + if (token.Kind == TokenKind.IdentifierToken) { OutputIdentifier(token); } diff --git a/Parser.Tests/MLexerGreenTests.cs b/Parser.Tests/MLexerGreenTests.cs index 36778c1..21832b7 100644 --- a/Parser.Tests/MLexerGreenTests.cs +++ b/Parser.Tests/MLexerGreenTests.cs @@ -11,7 +11,7 @@ namespace Parser.Tests private IEnumerable ParseText(string text) { var lexer = new MLexerGreen(new TextWindowWithNull(text)); - return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFile); + return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFileToken); } [Theory] @@ -40,12 +40,12 @@ namespace Parser.Tests { var text = text1 + separatorText + text2; var tokens = ParseText(text).ToArray(); - if (kind1 == TokenKind.Identifier && !ContainsNewLine(separatorText)) + if (kind1 == TokenKind.IdentifierToken && !ContainsNewLine(separatorText)) { - Assert.Equal(TokenKind.Identifier, tokens[0].Kind); + Assert.Equal(TokenKind.IdentifierToken, tokens[0].Kind); foreach (var token in tokens.Skip(1)) { - Assert.Equal(TokenKind.UnquotedStringLiteral, token.Kind); + Assert.Equal(TokenKind.UnquotedStringLiteralToken, token.Kind); } } else @@ -85,20 +85,20 @@ namespace Parser.Tests .Where(t => !(SyntaxFacts.IsUnaryTokenKind(t.kind) || SyntaxFacts.IsOpeningToken(t.kind) || SyntaxFacts.IsClosingToken(t.kind) - || t.kind == TokenKind.Transpose)); + || t.kind == TokenKind.ApostropheToken)); var dynamicTokens = new[] { - (TokenKind.Identifier, "a"), - (TokenKind.Identifier, "abc"), - (TokenKind.NumberLiteral, "1"), - (TokenKind.NumberLiteral, "123"), - (TokenKind.NumberLiteral, "145.67"), - (TokenKind.NumberLiteral, "14.5e-3"), - (TokenKind.NumberLiteral, "3.14e8"), - (TokenKind.StringLiteral, "'what is that'"), - (TokenKind.DoubleQuotedStringLiteral, "\"Another ' string\"") + (TokenKind.IdentifierToken, "a"), + (TokenKind.IdentifierToken, "abc"), + (TokenKind.NumberLiteralToken, "1"), + (TokenKind.NumberLiteralToken, "123"), + (TokenKind.NumberLiteralToken, "145.67"), + (TokenKind.NumberLiteralToken, "14.5e-3"), + (TokenKind.NumberLiteralToken, "3.14e8"), + (TokenKind.StringLiteralToken, "'what is that'"), + (TokenKind.DoubleQuotedStringLiteralToken, "\"Another ' string\"") }; return fixedTokens.Concat(dynamicTokens); } @@ -136,127 +136,127 @@ namespace Parser.Tests private static bool RequiresSeparator(TokenKind kind1, TokenKind kind2) { - if (kind1 == TokenKind.Less && kind2 == TokenKind.Equality) + if (kind1 == TokenKind.LessToken && kind2 == TokenKind.EqualsEqualsToken) { return true; } - if (kind1 == TokenKind.Less && kind2 == TokenKind.Assignment) + if (kind1 == TokenKind.LessToken && kind2 == TokenKind.EqualsToken) { return true; } - if (kind1 == TokenKind.Greater && kind2 == TokenKind.Equality) + if (kind1 == TokenKind.GreaterToken && kind2 == TokenKind.EqualsEqualsToken) { return true; } - if (kind1 == TokenKind.Greater && kind2 == TokenKind.Assignment) + if (kind1 == TokenKind.GreaterToken && kind2 == TokenKind.EqualsToken) { return true; } - if (kind1 == TokenKind.Not && kind2 == TokenKind.Assignment) + if (kind1 == TokenKind.TildeToken && kind2 == TokenKind.EqualsToken) { return true; } - if (kind1 == TokenKind.Not && kind2 == TokenKind.Equality) + if (kind1 == TokenKind.TildeToken && kind2 == TokenKind.EqualsEqualsToken) { return true; } - if (kind1 == TokenKind.Assignment && kind2 == TokenKind.Assignment) + if (kind1 == TokenKind.EqualsToken && kind2 == TokenKind.EqualsToken) { return true; } - if (kind1 == TokenKind.Assignment && kind2 == TokenKind.Equality) + if (kind1 == TokenKind.EqualsToken && kind2 == TokenKind.EqualsEqualsToken) { return true; } - if (kind1 == TokenKind.BitwiseAnd && kind2 == TokenKind.LogicalAnd) + if (kind1 == TokenKind.AmpersandToken && kind2 == TokenKind.AmpersandAmpersandToken) { return true; } - if (kind1 == TokenKind.BitwiseAnd && kind2 == TokenKind.BitwiseAnd) + if (kind1 == TokenKind.AmpersandToken && kind2 == TokenKind.AmpersandToken) { return true; } - if (kind1 == TokenKind.BitwiseOr && kind2 == TokenKind.LogicalOr) + if (kind1 == TokenKind.PipeToken && kind2 == TokenKind.PipePipeToken) { return true; } - if (kind1 == TokenKind.BitwiseOr && kind2 == TokenKind.BitwiseOr) + if (kind1 == TokenKind.PipeToken && kind2 == TokenKind.PipeToken) { return true; } - if (kind1 == TokenKind.Dot && kind2 == TokenKind.Multiply) + if (kind1 == TokenKind.DotToken && kind2 == TokenKind.StarToken) { return true; } - if (kind1 == TokenKind.Dot && kind2 == TokenKind.Divide) + if (kind1 == TokenKind.DotToken && kind2 == TokenKind.SlashToken) { return true; } - if (kind1 == TokenKind.Dot && kind2 == TokenKind.Backslash) + if (kind1 == TokenKind.DotToken && kind2 == TokenKind.BackslashToken) { return true; } - if (kind1 == TokenKind.Dot && kind2 == TokenKind.Power) + if (kind1 == TokenKind.DotToken && kind2 == TokenKind.CaretToken) { return true; } - if (kind1 == TokenKind.Dot && kind2 == TokenKind.NumberLiteral) + if (kind1 == TokenKind.DotToken && kind2 == TokenKind.NumberLiteralToken) { return true; } - if (kind1 == TokenKind.NumberLiteral && kind2 == TokenKind.Dot) + if (kind1 == TokenKind.NumberLiteralToken && kind2 == TokenKind.DotToken) { return true; } - if (kind1 == TokenKind.Dot && kind2 == TokenKind.StringLiteral) + if (kind1 == TokenKind.DotToken && kind2 == TokenKind.StringLiteralToken) { return true; } - if (kind1 == TokenKind.NumberLiteral && kind2 == TokenKind.NumberLiteral) + if (kind1 == TokenKind.NumberLiteralToken && kind2 == TokenKind.NumberLiteralToken) { return true; } - if (kind1 == TokenKind.StringLiteral && kind2 == TokenKind.StringLiteral) + if (kind1 == TokenKind.StringLiteralToken && kind2 == TokenKind.StringLiteralToken) { return true; } - if (kind1 == TokenKind.DoubleQuotedStringLiteral && kind2 == TokenKind.DoubleQuotedStringLiteral) + if (kind1 == TokenKind.DoubleQuotedStringLiteralToken && kind2 == TokenKind.DoubleQuotedStringLiteralToken) { return true; } - if (kind1 == TokenKind.Identifier && kind2 == TokenKind.Identifier) + if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.IdentifierToken) { return true; } - if (kind1 == TokenKind.Identifier && kind2 == TokenKind.NumberLiteral) + if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.NumberLiteralToken) { return true; } - if (kind1 == TokenKind.Identifier && kind2 == TokenKind.StringLiteral) + if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.StringLiteralToken) { return true; } diff --git a/Parser/Internal/MLexerGreen.cs b/Parser/Internal/MLexerGreen.cs index 92dcb9b..e896b0f 100644 --- a/Parser/Internal/MLexerGreen.cs +++ b/Parser/Internal/MLexerGreen.cs @@ -40,7 +40,7 @@ namespace Parser.Internal n++; } - return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); + return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n)); } private SyntaxTrivia LexMultilineComment() @@ -54,7 +54,7 @@ namespace Parser.Internal if (c == '\0') { Diagnostics.ReportUnexpectedEndOfFile(new TextSpan(Window.Position.Offset, 0)); - return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); + return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n)); } if (c == '\n' || (c == '\r' && Window.PeekChar(n + 1) == '\n')) @@ -69,12 +69,12 @@ namespace Parser.Internal n++; } - return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); + return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n)); } if (metPercentSign && c == '}') { - return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n+1)); + return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n+1)); } metPercentSign = c == '%'; @@ -91,13 +91,13 @@ namespace Parser.Internal n++; } - var comment = TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); + var comment = TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n)); var result = new List { comment }; var character = Window.PeekChar(); if (character == '\n' || character == '\r') { Window.ConsumeChar(); - result.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, character.ToString())); + result.Add(TokenFactory.CreateTrivia(TokenKind.WhitespaceToken, character.ToString())); } return result; @@ -112,7 +112,7 @@ namespace Parser.Internal { if (whitespaceCache.Length > 0) { - triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, whitespaceCache.ToString())); + triviaList.Add(TokenFactory.CreateTrivia(TokenKind.WhitespaceToken, whitespaceCache.ToString())); } whitespaceCache.Clear(); @@ -132,7 +132,7 @@ namespace Parser.Internal case '\n': FlushWhitespaceCache(); Window.ConsumeChar(); - triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Newline, character.ToString())); + triviaList.Add(TokenFactory.CreateTrivia(TokenKind.NewlineToken, character.ToString())); if (isTrailing) { return triviaList; @@ -177,7 +177,7 @@ namespace Parser.Internal } var identifier = Window.GetAndConsumeChars(n); - tokenInfo.Kind = TokenKind.Identifier; + tokenInfo.Kind = TokenKind.IdentifierToken; tokenInfo.Text = identifier; return true; } @@ -191,7 +191,7 @@ namespace Parser.Internal if (c == ' ' || c == '\n' || c == '\0') { var literal = Window.GetAndConsumeChars(n); - tokenInfo.Kind = TokenKind.UnquotedStringLiteral; + tokenInfo.Kind = TokenKind.UnquotedStringLiteralToken; tokenInfo.Text = literal; tokenInfo.StringValue = literal; return true; @@ -330,7 +330,7 @@ namespace Parser.Internal if (fail) { var s = Window.GetAndConsumeChars(n); - tokenInfo.Kind = TokenKind.NumberLiteral; + tokenInfo.Kind = TokenKind.NumberLiteralToken; tokenInfo.Text = s; return false; } @@ -363,7 +363,7 @@ namespace Parser.Internal } var s = Window.GetAndConsumeChars(n); - tokenInfo.Kind = TokenKind.NumberLiteral; + tokenInfo.Kind = TokenKind.NumberLiteralToken; tokenInfo.Text = s; return true; } @@ -439,14 +439,14 @@ namespace Parser.Internal private bool ContinueLexingStringLiteral(ref TokenInfo tokenInfo) { ContinueLexingGeneralStringLiteral(ref tokenInfo, '\''); - tokenInfo.Kind = TokenKind.StringLiteral; + tokenInfo.Kind = TokenKind.StringLiteralToken; return true; } private bool ContinueLexingDoubleQuotedStringLiteral(ref TokenInfo tokenInfo) { ContinueLexingGeneralStringLiteral(ref tokenInfo, '"'); - tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteral; + tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteralToken; return true; } @@ -455,14 +455,14 @@ namespace Parser.Internal var character = Window.PeekChar(); if (character == '\0') { - tokenInfo.Kind = TokenKind.EndOfFile; + tokenInfo.Kind = TokenKind.EndOfFileToken; tokenInfo.Text = ""; return true; } if (TokensSinceNewLine == 1 && !TokenStack.Any() - && LastToken.Kind == TokenKind.Identifier + && LastToken.Kind == TokenKind.IdentifierToken && LastToken.TrailingTrivia.Any() && character != '=' && character != '(' @@ -470,7 +470,7 @@ namespace Parser.Internal { return ContinueParsingUnquotedStringLiteral(ref tokenInfo); } - if (LastToken?.Kind == TokenKind.UnquotedStringLiteral + if (LastToken?.Kind == TokenKind.UnquotedStringLiteralToken && !TokenStack.Any() && TokensSinceNewLine > 0) { @@ -553,11 +553,11 @@ namespace Parser.Internal if (Window.PeekChar() == '=') { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Equality; + tokenInfo.Kind = TokenKind.EqualsEqualsToken; } else { - tokenInfo.Kind = TokenKind.Assignment; + tokenInfo.Kind = TokenKind.EqualsToken; } return true; @@ -578,72 +578,72 @@ namespace Parser.Internal { case '*': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.DotMultiply; + tokenInfo.Kind = TokenKind.DotStarToken; break; case '/': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.DotDivide; + tokenInfo.Kind = TokenKind.DotSlashToken; break; case '^': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.DotPower; + tokenInfo.Kind = TokenKind.DotCaretToken; break; case '\\': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.DotBackslash; + tokenInfo.Kind = TokenKind.DotBackslashToken; break; case '\'': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.DotTranspose; + tokenInfo.Kind = TokenKind.DotApostropheToken; break; default: - tokenInfo.Kind = TokenKind.Dot; + tokenInfo.Kind = TokenKind.DotToken; break; } return true; case '(': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.OpeningBracket; + tokenInfo.Kind = TokenKind.OpenParenthesisToken; return true; case ')': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.ClosingBracket; + tokenInfo.Kind = TokenKind.CloseParenthesisToken; return true; case '[': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.OpeningSquareBracket; + tokenInfo.Kind = TokenKind.OpenSquareBracketToken; return true; case ']': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.ClosingSquareBracket; + tokenInfo.Kind = TokenKind.CloseSquareBracketToken; return true; case '{': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.OpeningBrace; + tokenInfo.Kind = TokenKind.OpenBraceToken; return true; case '}': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.ClosingBrace; + tokenInfo.Kind = TokenKind.CloseBraceToken; return true; case ',': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Comma; + tokenInfo.Kind = TokenKind.CommaToken; return true; case ';': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Semicolon; + tokenInfo.Kind = TokenKind.SemicolonToken; return true; case '&': Window.ConsumeChar(); if (Window.PeekChar() == '&') { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.LogicalAnd; + tokenInfo.Kind = TokenKind.AmpersandAmpersandToken; } else { - tokenInfo.Kind = TokenKind.BitwiseAnd; + tokenInfo.Kind = TokenKind.AmpersandToken; } return true; @@ -652,11 +652,11 @@ namespace Parser.Internal if (Window.PeekChar() == '|') { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.LogicalOr; + tokenInfo.Kind = TokenKind.PipePipeToken; } else { - tokenInfo.Kind = TokenKind.BitwiseOr; + tokenInfo.Kind = TokenKind.PipeToken; } @@ -666,11 +666,11 @@ namespace Parser.Internal if (Window.PeekChar() == '=') { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.LessOrEqual; + tokenInfo.Kind = TokenKind.LessOrEqualsToken; } else { - tokenInfo.Kind = TokenKind.Less; + tokenInfo.Kind = TokenKind.LessToken; } return true; @@ -679,11 +679,11 @@ namespace Parser.Internal if (Window.PeekChar() == '=') { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.GreaterOrEqual; + tokenInfo.Kind = TokenKind.GreaterOrEqualsToken; } else { - tokenInfo.Kind = TokenKind.Greater; + tokenInfo.Kind = TokenKind.GreaterToken; } return true; @@ -692,61 +692,61 @@ namespace Parser.Internal if (Window.PeekChar() == '=') { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Inequality; + tokenInfo.Kind = TokenKind.TildeEqualsToken; } else { - tokenInfo.Kind = TokenKind.Not; + tokenInfo.Kind = TokenKind.TildeToken; } return true; case '+': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Plus; + tokenInfo.Kind = TokenKind.PlusToken; return true; case '-': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Minus; + tokenInfo.Kind = TokenKind.MinusToken; return true; case '*': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Multiply; + tokenInfo.Kind = TokenKind.StarToken; return true; case '/': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Divide; + tokenInfo.Kind = TokenKind.SlashToken; return true; case '\\': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Backslash; + tokenInfo.Kind = TokenKind.BackslashToken; return true; case '^': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Power; + tokenInfo.Kind = TokenKind.CaretToken; return true; case '@': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.At; + tokenInfo.Kind = TokenKind.AtToken; return true; case ':': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Colon; + tokenInfo.Kind = TokenKind.ColonToken; return true; case '?': Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.QuestionMark; + tokenInfo.Kind = TokenKind.QuestionToken; return true; case '\'': if (LastToken != null && - (LastToken.Kind == TokenKind.ClosingBrace - || LastToken.Kind == TokenKind.ClosingBracket - || LastToken.Kind == TokenKind.ClosingSquareBracket - || LastToken.Kind == TokenKind.Identifier)) + (LastToken.Kind == TokenKind.CloseBraceToken + || LastToken.Kind == TokenKind.CloseParenthesisToken + || LastToken.Kind == TokenKind.CloseSquareBracketToken + || LastToken.Kind == TokenKind.IdentifierToken)) { if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0) { Window.ConsumeChar(); - tokenInfo.Kind = TokenKind.Transpose; + tokenInfo.Kind = TokenKind.ApostropheToken; return true; } } @@ -754,7 +754,7 @@ namespace Parser.Internal case '"': return ContinueLexingDoubleQuotedStringLiteral(ref tokenInfo); case '\0': - tokenInfo.Kind = TokenKind.EndOfFile; + tokenInfo.Kind = TokenKind.EndOfFileToken; return true; default: Diagnostics.ReportUnknownSymbol(new TextSpan(Window.Position.Offset, 1), character); @@ -772,7 +772,7 @@ namespace Parser.Internal var tokenInfo = new TokenInfo(); LexTokenWithoutTrivia(leadingTrivia, ref tokenInfo); var trailingTrivia = LexTrivia(true); - if (trailingTrivia.Any(t => t.Kind == TokenKind.Newline)) + if (trailingTrivia.Any(t => t.Kind == TokenKind.NewlineToken)) { TokensSinceNewLine = 0; } @@ -806,7 +806,7 @@ namespace Parser.Internal } } - if (tokenInfo.Kind == TokenKind.EndOfFile + if (tokenInfo.Kind == TokenKind.EndOfFileToken && TokenStack.Any()) { throw new ParsingException($"Unmatched \"{TokenStack.Pop()}\" by the end of file."); @@ -827,32 +827,32 @@ namespace Parser.Internal { switch (tokenInfo.Kind) { - case TokenKind.Identifier: + case TokenKind.IdentifierToken: return TokenFactory.CreateIdentifier( tokenInfo.Text, leadingTrivia, trailingTrivia); - case TokenKind.UnquotedStringLiteral: + case TokenKind.UnquotedStringLiteralToken: return TokenFactory.CreateUnquotedStringLiteral( tokenInfo.Text, tokenInfo.StringValue, leadingTrivia, trailingTrivia); - case TokenKind.NumberLiteral: + case TokenKind.NumberLiteralToken: return TokenFactory.CreateTokenWithValueAndTrivia( tokenInfo.Kind, tokenInfo.Text, tokenInfo.DoubleValue, leadingTrivia, trailingTrivia); - case TokenKind.StringLiteral: + case TokenKind.StringLiteralToken: return TokenFactory.CreateTokenWithValueAndTrivia( tokenInfo.Kind, tokenInfo.Text, tokenInfo.StringValue, leadingTrivia, trailingTrivia); - case TokenKind.DoubleQuotedStringLiteral: + case TokenKind.DoubleQuotedStringLiteralToken: return TokenFactory.CreateTokenWithValueAndTrivia( tokenInfo.Kind, tokenInfo.Text, @@ -879,7 +879,7 @@ namespace Parser.Internal throw new ParsingException($"Unexpected character: '{Window.PeekChar()}' at {Window.Position}."); } result.Add(pair); - if (token.Kind == TokenKind.EndOfFile) + if (token.Kind == TokenKind.EndOfFileToken) { return result; } diff --git a/Parser/Internal/MParserGreen.cs b/Parser/Internal/MParserGreen.cs index 0f55b55..a578042 100644 --- a/Parser/Internal/MParserGreen.cs +++ b/Parser/Internal/MParserGreen.cs @@ -41,27 +41,27 @@ namespace Parser.Internal private SyntaxToken EatIdentifier() { - return EatToken(TokenKind.Identifier); + return EatToken(TokenKind.IdentifierToken); } private bool IsIdentifier(SyntaxToken token, string s) { - return token.Kind == TokenKind.Identifier && token.Text == s; + return token.Kind == TokenKind.IdentifierToken && token.Text == s; } private SyntaxToken EatIdentifier(string s) { var token = CurrentToken; - if (token.Kind != TokenKind.Identifier) + if (token.Kind != TokenKind.IdentifierToken) { - Errors.Add($"Unexpected token \"{token.Text}\" instead of {TokenKind.Identifier} at {CurrentPosition}."); - return TokenFactory.CreateMissing(TokenKind.Identifier, null, null); + Errors.Add($"Unexpected token \"{token.Text}\" instead of {TokenKind.IdentifierToken} at {CurrentPosition}."); + return TokenFactory.CreateMissing(TokenKind.IdentifierToken, null, null); } if (token.Text != s) { Errors.Add($"Unexpected token \"{token.Text}\" instead of identifier {s} at {CurrentPosition}."); - return TokenFactory.CreateMissing(TokenKind.Identifier, null, null); + return TokenFactory.CreateMissing(TokenKind.IdentifierToken, null, null); } _index++; @@ -71,7 +71,7 @@ namespace Parser.Internal private SyntaxToken PossiblyEatIdentifier(string s) { var token = CurrentToken; - if (token.Kind == TokenKind.Identifier && token.Text == s) + if (token.Kind == TokenKind.IdentifierToken && token.Text == s) { return EatToken(); } @@ -82,13 +82,13 @@ namespace Parser.Internal private SyntaxToken EatPossiblyMissingIdentifier(string s) { var token = CurrentToken; - if (token.Kind == TokenKind.Identifier && token.Text == s) + if (token.Kind == TokenKind.IdentifierToken && token.Text == s) { return EatToken(); } return TokenFactory.CreateMissing( - TokenKind.Identifier, + TokenKind.IdentifierToken, new List(), new List()); } @@ -97,15 +97,15 @@ namespace Parser.Internal { var outputs = new SyntaxListBuilder(); var firstToken = true; - while (CurrentToken.Kind != TokenKind.ClosingSquareBracket) + while (CurrentToken.Kind != TokenKind.CloseSquareBracketToken) { - if (!firstToken && CurrentToken.Kind == TokenKind.Comma) + if (!firstToken && CurrentToken.Kind == TokenKind.CommaToken) { outputs.Add(EatToken()); } firstToken = false; - outputs.Add(Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier))); + outputs.Add(Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken))); } return outputs.ToList(); @@ -116,19 +116,19 @@ namespace Parser.Internal SyntaxToken assignmentSign; var builder = new SyntaxListBuilder(); - if (CurrentToken.Kind == TokenKind.Identifier) + if (CurrentToken.Kind == TokenKind.IdentifierToken) { - if (PeekToken(1).Kind == TokenKind.Assignment) + if (PeekToken(1).Kind == TokenKind.EqualsToken) { builder.Add(Factory.IdentifierNameSyntax(EatToken())); - assignmentSign = EatToken(TokenKind.Assignment); + assignmentSign = EatToken(TokenKind.EqualsToken); } else { return null; } } - else if (CurrentToken.Kind == TokenKind.OpeningSquareBracket) + else if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken) { builder.Add(EatToken()); var outputs = ParseFunctionOutputList(); @@ -137,8 +137,8 @@ namespace Parser.Internal builder.AddRange(outputs); } - builder.Add(EatToken(TokenKind.ClosingSquareBracket)); - assignmentSign = EatToken(TokenKind.Assignment); + builder.Add(EatToken(TokenKind.CloseSquareBracketToken)); + assignmentSign = EatToken(TokenKind.EqualsToken); } else { @@ -154,25 +154,25 @@ namespace Parser.Internal { var builder = new SyntaxListBuilder(); var firstToken = true; - while (CurrentToken.Kind != TokenKind.ClosingBracket) + while (CurrentToken.Kind != TokenKind.CloseParenthesisToken) { if (!firstToken) { - builder.Add(EatToken(TokenKind.Comma)); + builder.Add(EatToken(TokenKind.CommaToken)); } else { firstToken = false; } - if (CurrentToken.Kind == TokenKind.Not) + if (CurrentToken.Kind == TokenKind.TildeToken) { var notToken = EatToken(); builder.Add(notToken); } else { - var identifierToken = EatToken(TokenKind.Identifier); + var identifierToken = EatToken(TokenKind.IdentifierToken); builder.Add(Factory.IdentifierNameSyntax(identifierToken)); } } @@ -182,11 +182,11 @@ namespace Parser.Internal private FunctionInputDescriptionSyntaxNode ParseFunctionInputDescription() { - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { - var openingBracket = EatToken(TokenKind.OpeningBracket); + var openingBracket = EatToken(TokenKind.OpenParenthesisToken); var parameterList = ParseParameterList(); - var closingBracket = EatToken(TokenKind.ClosingBracket); + var closingBracket = EatToken(TokenKind.CloseParenthesisToken); return Factory.FunctionInputDescriptionSyntax( openingBracket, parameterList, @@ -202,7 +202,7 @@ namespace Parser.Internal { var functionKeyword = EatIdentifier("function"); var outputDescription = ParseFunctionOutputDescription(); - var name = EatToken(TokenKind.Identifier); + var name = EatToken(TokenKind.IdentifierToken); var inputDescription = ParseFunctionInputDescription(); var commas = ParseOptionalCommas(); var body = ParseStatementList(); @@ -236,9 +236,9 @@ namespace Parser.Internal private ArrayLiteralExpressionSyntaxNode ParseArrayLiteral() { - var openingSquareBracket = EatToken(TokenKind.OpeningSquareBracket); - var elements = ParseArrayElementList(TokenKind.ClosingSquareBracket); - var closingSquareBracket = EatToken(TokenKind.ClosingSquareBracket); + var openingSquareBracket = EatToken(TokenKind.OpenSquareBracketToken); + var elements = ParseArrayElementList(TokenKind.CloseSquareBracketToken); + var closingSquareBracket = EatToken(TokenKind.CloseSquareBracketToken); return Factory.ArrayLiteralExpressionSyntax( openingSquareBracket, elements, @@ -247,9 +247,9 @@ namespace Parser.Internal private CellArrayLiteralExpressionSyntaxNode ParseCellArrayLiteral() { - var openingBrace = EatToken(TokenKind.OpeningBrace); - var elements = ParseArrayElementList(TokenKind.ClosingBrace); - var closingBrace = EatToken(TokenKind.ClosingBrace); + var openingBrace = EatToken(TokenKind.OpenBraceToken); + var elements = ParseArrayElementList(TokenKind.CloseBraceToken); + var closingBrace = EatToken(TokenKind.CloseBraceToken); return Factory.CellArrayLiteralExpressionSyntax( openingBrace, elements, @@ -265,8 +265,8 @@ namespace Parser.Internal { if (!firstToken) { - if (CurrentToken.Kind == TokenKind.Comma - || CurrentToken.Kind == TokenKind.Semicolon) + if (CurrentToken.Kind == TokenKind.CommaToken + || CurrentToken.Kind == TokenKind.SemicolonToken) { builder.Add(EatToken()); } @@ -292,28 +292,28 @@ namespace Parser.Internal ExpressionSyntaxNode expression = null; switch (token.Kind) { - case TokenKind.Identifier: + case TokenKind.IdentifierToken: expression = Factory.IdentifierNameSyntax(EatToken()); break; - case TokenKind.NumberLiteral: + case TokenKind.NumberLiteralToken: expression = Factory.NumberLiteralSyntax(EatToken()); break; - case TokenKind.StringLiteral: + case TokenKind.StringLiteralToken: expression = Factory.StringLiteralSyntax(EatToken()); break; - case TokenKind.DoubleQuotedStringLiteral: + case TokenKind.DoubleQuotedStringLiteralToken: expression = Factory.DoubleQuotedStringLiteralSyntax(EatToken()); break; - case TokenKind.OpeningSquareBracket: + case TokenKind.OpenSquareBracketToken: expression = ParseArrayLiteral(); break; - case TokenKind.OpeningBrace: + case TokenKind.OpenBraceToken: expression = ParseCellArrayLiteral(); break; - case TokenKind.Colon: + case TokenKind.ColonToken: expression = Factory.EmptyExpressionSyntax(); break; - case TokenKind.OpeningBracket: + case TokenKind.OpenParenthesisToken: expression = ParseParenthesizedExpression(); break; } @@ -332,14 +332,14 @@ namespace Parser.Internal var token = CurrentToken; switch (token.Kind) { - case TokenKind.OpeningBrace: // cell array element access + case TokenKind.OpenBraceToken: // cell array element access if (options.ParsingArrayElements && expression.TrailingTrivia.Any()) { return expression; } var openingBrace = EatToken(); - var indices = ParseArrayElementList(TokenKind.ClosingBrace); - var closingBrace = EatToken(TokenKind.ClosingBrace); + var indices = ParseArrayElementList(TokenKind.CloseBraceToken); + var closingBrace = EatToken(TokenKind.CloseBraceToken); expression = Factory.CellArrayElementAccessExpressionSyntax( expression, openingBrace, @@ -347,21 +347,21 @@ namespace Parser.Internal closingBrace ); break; - case TokenKind.OpeningBracket: // function call + case TokenKind.OpenParenthesisToken: // function call if (options.ParsingArrayElements && expression.TrailingTrivia.Any()) { return expression; } var openingBracket = EatToken(); var parameters = ParseFunctionCallParameterList(); - var closingBracket = EatToken(TokenKind.ClosingBracket); + var closingBracket = EatToken(TokenKind.CloseParenthesisToken); expression = Factory.FunctionCallExpressionSyntax( expression, openingBracket, parameters, closingBracket); break; - case TokenKind.Dot: // member access + case TokenKind.DotToken: // member access if (expression is IdentifierNameSyntaxNode || expression is MemberAccessSyntaxNode || expression is FunctionCallExpressionSyntaxNode @@ -378,14 +378,14 @@ namespace Parser.Internal } break; - case TokenKind.Transpose: - case TokenKind.DotTranspose: + case TokenKind.ApostropheToken: + case TokenKind.DotApostropheToken: var operation = EatToken(); expression = Factory.UnaryPostixOperationExpressionSyntax(expression, operation); break; - case TokenKind.UnquotedStringLiteral: + case TokenKind.UnquotedStringLiteralToken: return ParseCommandExpression(expression); - case TokenKind.At: + case TokenKind.AtToken: if (expression.TrailingTrivia.Any()) { return expression; @@ -402,7 +402,7 @@ namespace Parser.Internal if (expression is IdentifierNameSyntaxNode idNameNode) { var builder = new SyntaxListBuilder(); - while (CurrentToken.Kind == TokenKind.UnquotedStringLiteral) + while (CurrentToken.Kind == TokenKind.UnquotedStringLiteralToken) { builder.Add(Factory.UnquotedStringLiteralSyntax(EatToken())); } @@ -433,15 +433,15 @@ namespace Parser.Internal private ExpressionSyntaxNode ParseMemberAccess() { - if (CurrentToken.Kind == TokenKind.Identifier) + if (CurrentToken.Kind == TokenKind.IdentifierToken) { return Factory.IdentifierNameSyntax(EatToken()); } - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { var openingBracket = EatToken(); var indirectMember = ParseExpression(); - var closingBracket = EatToken(TokenKind.ClosingBracket); + var closingBracket = EatToken(TokenKind.CloseParenthesisToken); return Factory.IndirectMemberAccessSyntax( openingBracket, indirectMember, @@ -454,11 +454,11 @@ namespace Parser.Internal { var builder = new SyntaxListBuilder(); var firstToken = true; - while (CurrentToken.Kind != TokenKind.ClosingBracket) + while (CurrentToken.Kind != TokenKind.CloseParenthesisToken) { if (!firstToken) { - builder.Add(EatToken(TokenKind.Comma)); + builder.Add(EatToken(TokenKind.CommaToken)); } else { @@ -473,9 +473,9 @@ namespace Parser.Internal private ParenthesizedExpressionSyntaxNode ParseParenthesizedExpression() { - var openParen = EatToken(TokenKind.OpeningBracket); + var openParen = EatToken(TokenKind.OpenParenthesisToken); var expression = ParseExpression(); - var closeParen = EatToken(TokenKind.ClosingBracket); + var closeParen = EatToken(TokenKind.CloseParenthesisToken); return Factory.ParenthesizedExpressionSyntax( openParen, expression, @@ -484,16 +484,16 @@ namespace Parser.Internal private CompoundNameSyntaxNode ParseCompoundName() { - var lastToken = EatToken(TokenKind.Identifier); + var lastToken = EatToken(TokenKind.IdentifierToken); var firstName = lastToken; var builder = new SyntaxListBuilder(); builder.Add(firstName); - while (CurrentToken.Kind == TokenKind.Dot + while (CurrentToken.Kind == TokenKind.DotToken && !lastToken.TrailingTrivia.Any()) { var dot = EatToken(); builder.Add(dot); - lastToken = EatToken(TokenKind.Identifier); + lastToken = EatToken(TokenKind.IdentifierToken); builder.Add(lastToken); } @@ -503,14 +503,14 @@ namespace Parser.Internal private FunctionHandleSyntaxNode ParseFunctionHandle() { var atSign = EatToken(); - if (CurrentToken.Kind == TokenKind.Identifier) + if (CurrentToken.Kind == TokenKind.IdentifierToken) { var compoundName = ParseCompoundName(); return Factory.NamedFunctionHandleSyntax( atSign, compoundName); } - else if (CurrentToken.Kind == TokenKind.OpeningBracket) + else if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { var inputs = ParseFunctionInputDescription(); var body = ParseExpression(); @@ -532,7 +532,7 @@ namespace Parser.Internal var operand = ParseSubExpression(options, newPrecedence); if (operand == null) { - if (options.ParsingArrayElements && operation.Kind == TokenKind.Not) + if (options.ParsingArrayElements && operation.Kind == TokenKind.TildeToken) { operand = Factory.EmptyExpressionSyntax(); } @@ -543,7 +543,7 @@ namespace Parser.Internal } lhs = Factory.UnaryPrefixOperationExpressionSyntax(operation, operand); } - else if (CurrentToken.Kind == TokenKind.At) + else if (CurrentToken.Kind == TokenKind.AtToken) { return ParseFunctionHandle(); } @@ -570,11 +570,11 @@ namespace Parser.Internal EatToken(); var rhs = ParseSubExpression(options, newPrecedence); - if (rhs == null && token.Kind == TokenKind.Colon) // for parsing things like a{:} + if (rhs == null && token.Kind == TokenKind.ColonToken) // for parsing things like a{:} { rhs = Factory.EmptyExpressionSyntax(); } - if (token.Kind == TokenKind.Assignment) + if (token.Kind == TokenKind.EqualsToken) { lhs = Factory.AssignmentExpressionSyntax(lhs, token, rhs); } @@ -595,7 +595,7 @@ namespace Parser.Internal private SyntaxList ParseOptionalCommas() { var builder = new SyntaxListBuilder(); - while (CurrentToken.Kind == TokenKind.Comma) + while (CurrentToken.Kind == TokenKind.CommaToken) { builder.Add(EatToken()); } @@ -606,8 +606,8 @@ namespace Parser.Internal private SyntaxList ParseOptionalSemicolonsOrCommas() { var builder = new SyntaxListBuilder(); - while (CurrentToken.Kind == TokenKind.Comma - || CurrentToken.Kind == TokenKind.Semicolon) + while (CurrentToken.Kind == TokenKind.CommaToken + || CurrentToken.Kind == TokenKind.SemicolonToken) { builder.Add(EatToken()); } @@ -763,7 +763,7 @@ namespace Parser.Internal private AttributeAssignmentSyntaxNode ParseAttributeAssignment() { - if (CurrentToken.Kind == TokenKind.Assignment) + if (CurrentToken.Kind == TokenKind.EqualsToken) { var assignmentSign = EatToken(); var value = ParseExpression(); @@ -775,7 +775,7 @@ namespace Parser.Internal private AttributeSyntaxNode ParseAttribute() { - var name = Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier)); + var name = Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken)); var assignment = ParseAttributeAssignment(); return Factory.AttributeSyntax(name, assignment); } @@ -785,11 +785,11 @@ namespace Parser.Internal var openingBracket = EatToken(); var first = true; var builder = new SyntaxListBuilder(); - while (CurrentToken.Kind != TokenKind.ClosingBracket) + while (CurrentToken.Kind != TokenKind.CloseParenthesisToken) { if (!first) { - var comma = EatToken(TokenKind.Comma); + var comma = EatToken(TokenKind.CommaToken); builder.Add(comma); } @@ -816,13 +816,13 @@ namespace Parser.Internal return ParseMethodDefinition(); } - if (CurrentToken.Kind == TokenKind.OpeningSquareBracket - || CurrentToken.Kind == TokenKind.Identifier) + if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken + || CurrentToken.Kind == TokenKind.IdentifierToken) { return ParseAbstractMethodDeclaration(); } - if (CurrentToken.Kind == TokenKind.Semicolon) + if (CurrentToken.Kind == TokenKind.SemicolonToken) { return Factory.EmptyStatementSyntax(EatToken()); } @@ -852,7 +852,7 @@ namespace Parser.Internal { var methodsKeyword = EatToken(); AttributeListSyntaxNode attributes = null; - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { attributes = ParseAttributesList(); } @@ -870,7 +870,7 @@ namespace Parser.Internal private GreenNode ParsePropertyDeclaration() { - if (CurrentToken.Kind == TokenKind.Comma) + if (CurrentToken.Kind == TokenKind.CommaToken) { return EatToken(); } @@ -886,7 +886,7 @@ namespace Parser.Internal { var propertiesKeyword = EatToken(); AttributeListSyntaxNode attributes = null; - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { attributes = ParseAttributesList(); } @@ -903,18 +903,18 @@ namespace Parser.Internal private EnumerationItemValueSyntaxNode ParseEnumerationValue() { - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { var builder = new SyntaxListBuilder(); - var openingBracket = EatToken(TokenKind.OpeningBracket); + var openingBracket = EatToken(TokenKind.OpenParenthesisToken); var expression = ParseExpression() ?? Factory.EmptyExpressionSyntax(); builder.Add(expression); - while (CurrentToken.Kind == TokenKind.Comma) + while (CurrentToken.Kind == TokenKind.CommaToken) { builder.Add(EatToken()); builder.Add(ParseExpression()); } - var closingBracket = EatToken(TokenKind.ClosingBracket); + var closingBracket = EatToken(TokenKind.CloseParenthesisToken); return Factory.EnumerationItemValueSyntax(openingBracket, builder.ToList(), closingBracket); } return null; @@ -933,7 +933,7 @@ namespace Parser.Internal var enumerationKeyword = EatToken(); var builder = new SyntaxListBuilder(); AttributeListSyntaxNode attributes = null; - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { attributes = ParseAttributesList(); } @@ -955,7 +955,7 @@ namespace Parser.Internal { var eventsKeyword = EatToken(); AttributeListSyntaxNode attributes = null; - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { attributes = ParseAttributesList(); } @@ -974,7 +974,7 @@ namespace Parser.Internal { var builder = new SyntaxListBuilder(); builder.Add(ParseCompoundName()); - while (CurrentToken.Kind == TokenKind.BitwiseAnd) + while (CurrentToken.Kind == TokenKind.AmpersandToken) { builder.Add(EatToken()); builder.Add(ParseCompoundName()); @@ -994,13 +994,13 @@ namespace Parser.Internal { var classdefKeyword = EatToken(); AttributeListSyntaxNode attributes = null; - if (CurrentToken.Kind == TokenKind.OpeningBracket) + if (CurrentToken.Kind == TokenKind.OpenParenthesisToken) { attributes = ParseAttributesList(); } - var className = Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier)); + var className = Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken)); BaseClassListSyntaxNode baseClassList = null; - if (CurrentToken.Kind == TokenKind.Less) + if (CurrentToken.Kind == TokenKind.LessToken) { baseClassList = ParseBaseClassList(); } @@ -1046,7 +1046,7 @@ namespace Parser.Internal private StatementSyntaxNode ParseStatement() { - if (CurrentToken.Kind == TokenKind.Identifier) + if (CurrentToken.Kind == TokenKind.IdentifierToken) { switch (CurrentToken.Text) { @@ -1075,12 +1075,12 @@ namespace Parser.Internal } } - if (CurrentToken.Kind == TokenKind.OpeningSquareBracket) + if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken) { return ParseExpressionStatement(); } - if (CurrentToken.Kind == TokenKind.Semicolon) + if (CurrentToken.Kind == TokenKind.SemicolonToken) { return Factory.EmptyStatementSyntax(EatToken()); } @@ -1090,7 +1090,7 @@ namespace Parser.Internal private SyntaxList ParseStatementList() { var builder = new SyntaxListBuilder(); - while (CurrentToken.Kind != TokenKind.EndOfFile) + while (CurrentToken.Kind != TokenKind.EndOfFileToken) { var node = ParseStatement(); if (node == null) @@ -1098,8 +1098,8 @@ namespace Parser.Internal break; } builder.Add(node); - while (CurrentToken.Kind == TokenKind.Comma - || CurrentToken.Kind == TokenKind.Semicolon) + while (CurrentToken.Kind == TokenKind.CommaToken + || CurrentToken.Kind == TokenKind.SemicolonToken) { builder.Add(EatToken()); } diff --git a/Parser/Internal/SyntaxFacts.cs b/Parser/Internal/SyntaxFacts.cs index a58cae5..b2f6c1d 100644 --- a/Parser/Internal/SyntaxFacts.cs +++ b/Parser/Internal/SyntaxFacts.cs @@ -38,41 +38,41 @@ namespace Parser.Internal { switch (kind) { - case TokenKind.Assignment: + case TokenKind.EqualsToken: return Precedence.Assignment; - case TokenKind.LogicalOr: + case TokenKind.PipePipeToken: return Precedence.LogicalOr; - case TokenKind.LogicalAnd: + case TokenKind.AmpersandAmpersandToken: return Precedence.LogicalAnd; - case TokenKind.BitwiseOr: + case TokenKind.PipeToken: return Precedence.BitwiseOr; - case TokenKind.BitwiseAnd: + case TokenKind.AmpersandToken: return Precedence.BitwiseAnd; - case TokenKind.Less: - case TokenKind.LessOrEqual: - case TokenKind.Greater: - case TokenKind.GreaterOrEqual: - case TokenKind.Equality: - case TokenKind.Inequality: + case TokenKind.LessToken: + case TokenKind.LessOrEqualsToken: + case TokenKind.GreaterToken: + case TokenKind.GreaterOrEqualsToken: + case TokenKind.EqualsEqualsToken: + case TokenKind.TildeEqualsToken: return Precedence.Relational; - case TokenKind.Colon: + case TokenKind.ColonToken: return Precedence.Colon; - case TokenKind.Plus: - case TokenKind.Minus: + case TokenKind.PlusToken: + case TokenKind.MinusToken: return Precedence.Additive; - case TokenKind.Multiply: - case TokenKind.DotMultiply: - case TokenKind.Divide: - case TokenKind.DotDivide: - case TokenKind.Backslash: - case TokenKind.DotBackslash: + case TokenKind.StarToken: + case TokenKind.DotStarToken: + case TokenKind.SlashToken: + case TokenKind.DotSlashToken: + case TokenKind.BackslashToken: + case TokenKind.DotBackslashToken: return Precedence.Multiplicative; - case TokenKind.Not: + case TokenKind.TildeToken: return Precedence.Unary; - case TokenKind.Power: - case TokenKind.DotPower: - case TokenKind.Transpose: - case TokenKind.DotTranspose: + case TokenKind.CaretToken: + case TokenKind.DotCaretToken: + case TokenKind.ApostropheToken: + case TokenKind.DotApostropheToken: return Precedence.Power; default: return Precedence.Expression; @@ -113,9 +113,9 @@ namespace Parser.Internal { switch (tokenKind) { - case TokenKind.OpeningBrace: - case TokenKind.OpeningBracket: - case TokenKind.OpeningSquareBracket: + case TokenKind.OpenBraceToken: + case TokenKind.OpenParenthesisToken: + case TokenKind.OpenSquareBracketToken: return true; default: return false; @@ -126,9 +126,9 @@ namespace Parser.Internal { switch (tokenKind) { - case TokenKind.ClosingBrace: - case TokenKind.ClosingBracket: - case TokenKind.ClosingSquareBracket: + case TokenKind.CloseBraceToken: + case TokenKind.CloseParenthesisToken: + case TokenKind.CloseSquareBracketToken: return true; default: return false; @@ -144,12 +144,12 @@ namespace Parser.Internal { switch (tokenKind) { - case TokenKind.ClosingBrace: - return TokenKind.OpeningBrace; - case TokenKind.ClosingBracket: - return TokenKind.OpeningBracket; - case TokenKind.ClosingSquareBracket: - return TokenKind.OpeningSquareBracket; + case TokenKind.CloseBraceToken: + return TokenKind.OpenBraceToken; + case TokenKind.CloseParenthesisToken: + return TokenKind.OpenParenthesisToken; + case TokenKind.CloseSquareBracketToken: + return TokenKind.OpenSquareBracketToken; default: return null; } @@ -224,10 +224,10 @@ namespace Parser.Internal { switch (kind) { - case TokenKind.Plus: - case TokenKind.Minus: - case TokenKind.Not: - case TokenKind.QuestionMark: + case TokenKind.PlusToken: + case TokenKind.MinusToken: + case TokenKind.TildeToken: + case TokenKind.QuestionToken: return true; default: return false; @@ -252,29 +252,29 @@ namespace Parser.Internal { switch (kind) { - case TokenKind.Assignment: - case TokenKind.LogicalOr: - case TokenKind.LogicalAnd: - case TokenKind.BitwiseOr: - case TokenKind.BitwiseAnd: - case TokenKind.Less: - case TokenKind.LessOrEqual: - case TokenKind.Greater: - case TokenKind.GreaterOrEqual: - case TokenKind.Equality: - case TokenKind.Inequality: - case TokenKind.Colon: - case TokenKind.Plus: - case TokenKind.Minus: - case TokenKind.Multiply: - case TokenKind.DotMultiply: - case TokenKind.Divide: - case TokenKind.DotDivide: - case TokenKind.Backslash: - case TokenKind.DotBackslash: - case TokenKind.Not: - case TokenKind.Power: - case TokenKind.DotPower: + case TokenKind.EqualsToken: + case TokenKind.PipePipeToken: + case TokenKind.AmpersandAmpersandToken: + case TokenKind.PipeToken: + case TokenKind.AmpersandToken: + case TokenKind.LessToken: + case TokenKind.LessOrEqualsToken: + case TokenKind.GreaterToken: + case TokenKind.GreaterOrEqualsToken: + case TokenKind.EqualsEqualsToken: + case TokenKind.TildeEqualsToken: + case TokenKind.ColonToken: + case TokenKind.PlusToken: + case TokenKind.MinusToken: + case TokenKind.StarToken: + case TokenKind.DotStarToken: + case TokenKind.SlashToken: + case TokenKind.DotSlashToken: + case TokenKind.BackslashToken: + case TokenKind.DotBackslashToken: + case TokenKind.TildeToken: + case TokenKind.CaretToken: + case TokenKind.DotCaretToken: return true; default: return false; @@ -290,13 +290,13 @@ namespace Parser.Internal { switch (kind) { - case TokenKind.Plus: + case TokenKind.PlusToken: return TokenKind.UnaryPlus; - case TokenKind.Minus: + case TokenKind.MinusToken: return TokenKind.UnaryMinus; - case TokenKind.Not: + case TokenKind.TildeToken: return TokenKind.UnaryNot; - case TokenKind.QuestionMark: + case TokenKind.QuestionToken: return TokenKind.UnaryQuestionMark; default: throw new ArgumentException(nameof(kind)); diff --git a/Parser/Internal/SyntaxToken.cs b/Parser/Internal/SyntaxToken.cs index b82a020..beafab0 100644 --- a/Parser/Internal/SyntaxToken.cs +++ b/Parser/Internal/SyntaxToken.cs @@ -124,7 +124,7 @@ namespace Parser.Internal public SyntaxIdentifier( string text - ) : base(TokenKind.Identifier) + ) : base(TokenKind.IdentifierToken) { _text = text; } diff --git a/Parser/Internal/TokenFactory.cs b/Parser/Internal/TokenFactory.cs index 9ff1d70..221d734 100644 --- a/Parser/Internal/TokenFactory.cs +++ b/Parser/Internal/TokenFactory.cs @@ -42,7 +42,7 @@ namespace Parser.Internal IReadOnlyList trailingTrivia) { return new SyntaxToken.SyntaxTokenWithValueAndTrivia( - TokenKind.UnquotedStringLiteral, + TokenKind.UnquotedStringLiteralToken, text, value, leadingTrivia, diff --git a/Parser/TokenKind.cs b/Parser/TokenKind.cs index 735ca0f..e614836 100644 --- a/Parser/TokenKind.cs +++ b/Parser/TokenKind.cs @@ -8,114 +8,115 @@ None = 0, BadToken = 1, // The lexer puts a virtual "end of file" token at the end of the parsed file. - EndOfFile = 2, + EndOfFileToken = 2, // Identifier: could be a reserved word, a variable name, a class name, etc. - Identifier = 3, + IdentifierToken = 3, // Number literal: 123, 45.678, 2e-5, etc. - NumberLiteral = 4, + NumberLiteralToken = 4, // String literal: 'abc', '123', etc. The "usual" string literals are single-quoted and are just char arrays. - StringLiteral = 5, + StringLiteralToken = 5, // Double-quoted string literal: "abc", "123", etc. These are the "new" string literal that are more like strings // and less like char arrays (for example, char arrays could be columns instead of rows, or even multi-dimensional). - DoubleQuotedStringLiteral = 6, + DoubleQuotedStringLiteralToken = 6, // This is for supporting "command statements" like // > cd some/+folder/ // In this example, "some/folder" should be treated as a string literal (for example, "+' there should be a part // of it, and not parsed as a binary operator). - UnquotedStringLiteral = 7, + UnquotedStringLiteralToken = 7, // trivia // Spaces, tabs, etc. - Whitespace = 10, + WhitespaceToken = 10, // New line characters. - Newline = 11, + NewlineToken = 11, // There are three types of comments: // * comments starting with % and lasting till the end of the line; // * comments starting with %{ and ending with %}, possibly spanning several lines; // * comments starting with ... and lasting till the end of the line. // At the moment, this token is used to accomodate all of them, so the other two // (MultilineComment = 13 and DotDotDot = 56 are not used). - Comment = 12, + CommentToken = 12, // Comments starting with %{ and ending with %}, possibly spanning several lines. Not used, see Comment = 12. - MultilineComment = 13, + MultilineCommentToken = 13, // operators // = - Assignment = 20, + EqualsToken = 20, // == - Equality = 21, + EqualsEqualsToken = 21, // ~= - Inequality = 22, + TildeEqualsToken = 22, // && - LogicalAnd = 23, + AmpersandAmpersandToken = 23, // || - LogicalOr = 24, + PipePipeToken = 24, // & - BitwiseAnd = 25, + AmpersandToken = 25, // | - BitwiseOr = 26, + PipeToken = 26, // < - Less = 27, + LessToken = 27, // <= - LessOrEqual = 28, + LessOrEqualsToken = 28, // > - Greater = 29, + GreaterToken = 29, // >= - GreaterOrEqual = 30, + GreaterOrEqualsToken = 30, // ~ - Not = 31, + TildeToken = 31, // + - Plus = 32, + PlusToken = 32, // - - Minus = 33, + MinusToken = 33, // * - Multiply = 34, + StarToken = 34, // / - Divide = 35, + SlashToken = 35, // ^ - Power = 36, + CaretToken = 36, // \ - Backslash = 37, + BackslashToken = 37, // ' (this is the same as starting string literal; we'll have some fun distinguishing those). - Transpose = 38, + ApostropheToken = 38, // .* - DotMultiply = 39, + DotStarToken = 39, // ./ - DotDivide = 40, + DotSlashToken = 40, // .^ - DotPower = 41, + DotCaretToken = 41, // .\ - DotBackslash = 42, + DotBackslashToken = 42, // .' - DotTranspose = 43, + DotApostropheToken = 43, // @ - At = 44, + AtToken = 44, // : - Colon = 45, + ColonToken = 45, // ? - QuestionMark = 46, + QuestionToken = 46, // , - Comma = 47, + CommaToken = 47, // ; - Semicolon = 48, + SemicolonToken = 48, // { - OpeningBrace = 49, + OpenBraceToken = 49, // } - ClosingBrace = 50, + CloseBraceToken = 50, // [ - OpeningSquareBracket = 51, + OpenSquareBracketToken = 51, // ] - ClosingSquareBracket = 52, + CloseSquareBracketToken = 52, // ( - OpeningBracket = 53, + OpenParenthesisToken = 53, // ) - ClosingBracket = 54, + CloseParenthesisToken = 54, // . - Dot = 55, + DotToken = 55, // Comments starting with ... and lasting till the end of the line. Not used, see Comment = 12. - DotDotDot = 56, + DotDotDotToken = 56, + // Unary tokens are not recognized during lexing; they are contextually recognized while parsing. UnaryPlus = 57, UnaryMinus = 58,