mahalex/MParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename TokenKinds for consistency

Browse Source
This commit is contained in:
Alexander Luzgarev 2018-11-19 21:39:36 +01:00
parent 1c37559d9e
commit 5eb63a00e0
9 changed files with 323 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ConsoleDemo/DumbWalker.cs
View File

@ -53,7 +53,7 @@ namespace ConsoleDemo
{ {
switch (token.Kind) switch (token.Kind)
{ {
case TokenKind.Comma: case TokenKind.CommaToken:
return true; return true;
default: default:
break; break;

2
ConsoleDemo/PrettyPrinter.cs
View File

@ -150,7 +150,7 @@ namespace ProjectConsole
if (nodeOrToken.IsToken) if (nodeOrToken.IsToken)
{ {
var token = nodeOrToken.AsToken(); var token = nodeOrToken.AsToken();
if (token.Kind == TokenKind.Identifier) if (token.Kind == TokenKind.IdentifierToken)
{ {
OutputIdentifier(token); OutputIdentifier(token);
} }

78
Parser.Tests/MLexerGreenTests.cs
View File

@ -11,7 +11,7 @@ namespace Parser.Tests
private IEnumerable<Internal.SyntaxToken> ParseText(string text) private IEnumerable<Internal.SyntaxToken> ParseText(string text)
{ {
var lexer = new MLexerGreen(new TextWindowWithNull(text)); var lexer = new MLexerGreen(new TextWindowWithNull(text));
return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFile); return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFileToken);
} }
[Theory] [Theory]
@ -40,12 +40,12 @@ namespace Parser.Tests
{ {
var text = text1 + separatorText + text2; var text = text1 + separatorText + text2;
var tokens = ParseText(text).ToArray(); var tokens = ParseText(text).ToArray();
if (kind1 == TokenKind.Identifier && !ContainsNewLine(separatorText)) if (kind1 == TokenKind.IdentifierToken && !ContainsNewLine(separatorText))
{ {
Assert.Equal(TokenKind.Identifier, tokens[0].Kind); Assert.Equal(TokenKind.IdentifierToken, tokens[0].Kind);
foreach (var token in tokens.Skip(1)) foreach (var token in tokens.Skip(1))
{ {
Assert.Equal(TokenKind.UnquotedStringLiteral, token.Kind); Assert.Equal(TokenKind.UnquotedStringLiteralToken, token.Kind);
} }
} }
else else
@ -85,20 +85,20 @@ namespace Parser.Tests
.Where(t => !(SyntaxFacts.IsUnaryTokenKind(t.kind) .Where(t => !(SyntaxFacts.IsUnaryTokenKind(t.kind)
|| SyntaxFacts.IsOpeningToken(t.kind) || SyntaxFacts.IsOpeningToken(t.kind)
|| SyntaxFacts.IsClosingToken(t.kind) || SyntaxFacts.IsClosingToken(t.kind)
|| t.kind == TokenKind.Transpose)); || t.kind == TokenKind.ApostropheToken));
var dynamicTokens = new[] var dynamicTokens = new[]
{ {
(TokenKind.Identifier, "a"), (TokenKind.IdentifierToken, "a"),
(TokenKind.Identifier, "abc"), (TokenKind.IdentifierToken, "abc"),
(TokenKind.NumberLiteral, "1"), (TokenKind.NumberLiteralToken, "1"),
(TokenKind.NumberLiteral, "123"), (TokenKind.NumberLiteralToken, "123"),
(TokenKind.NumberLiteral, "145.67"), (TokenKind.NumberLiteralToken, "145.67"),
(TokenKind.NumberLiteral, "14.5e-3"), (TokenKind.NumberLiteralToken, "14.5e-3"),
(TokenKind.NumberLiteral, "3.14e8"), (TokenKind.NumberLiteralToken, "3.14e8"),
(TokenKind.StringLiteral, "'what is that'"), (TokenKind.StringLiteralToken, "'what is that'"),
(TokenKind.DoubleQuotedStringLiteral, "\"Another ' string\"") (TokenKind.DoubleQuotedStringLiteralToken, "\"Another ' string\"")
}; };
return fixedTokens.Concat(dynamicTokens); return fixedTokens.Concat(dynamicTokens);
} }
@ -136,127 +136,127 @@ namespace Parser.Tests
private static bool RequiresSeparator(TokenKind kind1, TokenKind kind2) private static bool RequiresSeparator(TokenKind kind1, TokenKind kind2)
{ {
if (kind1 == TokenKind.Less && kind2 == TokenKind.Equality) if (kind1 == TokenKind.LessToken && kind2 == TokenKind.EqualsEqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Less && kind2 == TokenKind.Assignment) if (kind1 == TokenKind.LessToken && kind2 == TokenKind.EqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Greater && kind2 == TokenKind.Equality) if (kind1 == TokenKind.GreaterToken && kind2 == TokenKind.EqualsEqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Greater && kind2 == TokenKind.Assignment) if (kind1 == TokenKind.GreaterToken && kind2 == TokenKind.EqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Not && kind2 == TokenKind.Assignment) if (kind1 == TokenKind.TildeToken && kind2 == TokenKind.EqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Not && kind2 == TokenKind.Equality) if (kind1 == TokenKind.TildeToken && kind2 == TokenKind.EqualsEqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Assignment && kind2 == TokenKind.Assignment) if (kind1 == TokenKind.EqualsToken && kind2 == TokenKind.EqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Assignment && kind2 == TokenKind.Equality) if (kind1 == TokenKind.EqualsToken && kind2 == TokenKind.EqualsEqualsToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.BitwiseAnd && kind2 == TokenKind.LogicalAnd) if (kind1 == TokenKind.AmpersandToken && kind2 == TokenKind.AmpersandAmpersandToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.BitwiseAnd && kind2 == TokenKind.BitwiseAnd) if (kind1 == TokenKind.AmpersandToken && kind2 == TokenKind.AmpersandToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.BitwiseOr && kind2 == TokenKind.LogicalOr) if (kind1 == TokenKind.PipeToken && kind2 == TokenKind.PipePipeToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.BitwiseOr && kind2 == TokenKind.BitwiseOr) if (kind1 == TokenKind.PipeToken && kind2 == TokenKind.PipeToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Multiply) if (kind1 == TokenKind.DotToken && kind2 == TokenKind.StarToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Divide) if (kind1 == TokenKind.DotToken && kind2 == TokenKind.SlashToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Backslash) if (kind1 == TokenKind.DotToken && kind2 == TokenKind.BackslashToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Power) if (kind1 == TokenKind.DotToken && kind2 == TokenKind.CaretToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Dot && kind2 == TokenKind.NumberLiteral) if (kind1 == TokenKind.DotToken && kind2 == TokenKind.NumberLiteralToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.NumberLiteral && kind2 == TokenKind.Dot) if (kind1 == TokenKind.NumberLiteralToken && kind2 == TokenKind.DotToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Dot && kind2 == TokenKind.StringLiteral) if (kind1 == TokenKind.DotToken && kind2 == TokenKind.StringLiteralToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.NumberLiteral && kind2 == TokenKind.NumberLiteral) if (kind1 == TokenKind.NumberLiteralToken && kind2 == TokenKind.NumberLiteralToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.StringLiteral && kind2 == TokenKind.StringLiteral) if (kind1 == TokenKind.StringLiteralToken && kind2 == TokenKind.StringLiteralToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.DoubleQuotedStringLiteral && kind2 == TokenKind.DoubleQuotedStringLiteral) if (kind1 == TokenKind.DoubleQuotedStringLiteralToken && kind2 == TokenKind.DoubleQuotedStringLiteralToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Identifier && kind2 == TokenKind.Identifier) if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.IdentifierToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Identifier && kind2 == TokenKind.NumberLiteral) if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.NumberLiteralToken)
{ {
return true; return true;
} }
if (kind1 == TokenKind.Identifier && kind2 == TokenKind.StringLiteral) if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.StringLiteralToken)
{ {
return true; return true;
} }

132
Parser/Internal/MLexerGreen.cs
View File

@ -40,7 +40,7 @@ namespace Parser.Internal
n++; n++;
} }
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
} }
private SyntaxTrivia LexMultilineComment() private SyntaxTrivia LexMultilineComment()
@ -54,7 +54,7 @@ namespace Parser.Internal
if (c == '\0') if (c == '\0')
{ {
Diagnostics.ReportUnexpectedEndOfFile(new TextSpan(Window.Position.Offset, 0)); Diagnostics.ReportUnexpectedEndOfFile(new TextSpan(Window.Position.Offset, 0));
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
} }
if (c == '\n' || (c == '\r' && Window.PeekChar(n + 1) == '\n')) if (c == '\n' || (c == '\r' && Window.PeekChar(n + 1) == '\n'))
@ -69,12 +69,12 @@ namespace Parser.Internal
n++; n++;
} }
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
} }
if (metPercentSign && c == '}') if (metPercentSign && c == '}')
{ {
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n+1)); return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n+1));
} }
metPercentSign = c == '%'; metPercentSign = c == '%';
@ -91,13 +91,13 @@ namespace Parser.Internal
n++; n++;
} }
var comment = TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n)); var comment = TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
var result = new List<SyntaxTrivia> { comment }; var result = new List<SyntaxTrivia> { comment };
var character = Window.PeekChar(); var character = Window.PeekChar();
if (character == '\n' || character == '\r') if (character == '\n' || character == '\r')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
result.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, character.ToString())); result.Add(TokenFactory.CreateTrivia(TokenKind.WhitespaceToken, character.ToString()));
} }
return result; return result;
@ -112,7 +112,7 @@ namespace Parser.Internal
{ {
if (whitespaceCache.Length > 0) if (whitespaceCache.Length > 0)
{ {
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, whitespaceCache.ToString())); triviaList.Add(TokenFactory.CreateTrivia(TokenKind.WhitespaceToken, whitespaceCache.ToString()));
} }
whitespaceCache.Clear(); whitespaceCache.Clear();
@ -132,7 +132,7 @@ namespace Parser.Internal
case '\n': case '\n':
FlushWhitespaceCache(); FlushWhitespaceCache();
Window.ConsumeChar(); Window.ConsumeChar();
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Newline, character.ToString())); triviaList.Add(TokenFactory.CreateTrivia(TokenKind.NewlineToken, character.ToString()));
if (isTrailing) if (isTrailing)
{ {
return triviaList; return triviaList;
@ -177,7 +177,7 @@ namespace Parser.Internal
} }
var identifier = Window.GetAndConsumeChars(n); var identifier = Window.GetAndConsumeChars(n);
tokenInfo.Kind = TokenKind.Identifier; tokenInfo.Kind = TokenKind.IdentifierToken;
tokenInfo.Text = identifier; tokenInfo.Text = identifier;
return true; return true;
} }
@ -191,7 +191,7 @@ namespace Parser.Internal
if (c == ' ' || c == '\n' || c == '\0') if (c == ' ' || c == '\n' || c == '\0')
{ {
var literal = Window.GetAndConsumeChars(n); var literal = Window.GetAndConsumeChars(n);
tokenInfo.Kind = TokenKind.UnquotedStringLiteral; tokenInfo.Kind = TokenKind.UnquotedStringLiteralToken;
tokenInfo.Text = literal; tokenInfo.Text = literal;
tokenInfo.StringValue = literal; tokenInfo.StringValue = literal;
return true; return true;
@ -330,7 +330,7 @@ namespace Parser.Internal
if (fail) if (fail)
{ {
var s = Window.GetAndConsumeChars(n); var s = Window.GetAndConsumeChars(n);
tokenInfo.Kind = TokenKind.NumberLiteral; tokenInfo.Kind = TokenKind.NumberLiteralToken;
tokenInfo.Text = s; tokenInfo.Text = s;
return false; return false;
} }
@ -363,7 +363,7 @@ namespace Parser.Internal
} }
var s = Window.GetAndConsumeChars(n); var s = Window.GetAndConsumeChars(n);
tokenInfo.Kind = TokenKind.NumberLiteral; tokenInfo.Kind = TokenKind.NumberLiteralToken;
tokenInfo.Text = s; tokenInfo.Text = s;
return true; return true;
} }
@ -439,14 +439,14 @@ namespace Parser.Internal
private bool ContinueLexingStringLiteral(ref TokenInfo tokenInfo) private bool ContinueLexingStringLiteral(ref TokenInfo tokenInfo)
{ {
ContinueLexingGeneralStringLiteral(ref tokenInfo, '\''); ContinueLexingGeneralStringLiteral(ref tokenInfo, '\'');
tokenInfo.Kind = TokenKind.StringLiteral; tokenInfo.Kind = TokenKind.StringLiteralToken;
return true; return true;
} }
private bool ContinueLexingDoubleQuotedStringLiteral(ref TokenInfo tokenInfo) private bool ContinueLexingDoubleQuotedStringLiteral(ref TokenInfo tokenInfo)
{ {
ContinueLexingGeneralStringLiteral(ref tokenInfo, '"'); ContinueLexingGeneralStringLiteral(ref tokenInfo, '"');
tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteral; tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteralToken;
return true; return true;
} }
@ -455,14 +455,14 @@ namespace Parser.Internal
var character = Window.PeekChar(); var character = Window.PeekChar();
if (character == '\0') if (character == '\0')
{ {
tokenInfo.Kind = TokenKind.EndOfFile; tokenInfo.Kind = TokenKind.EndOfFileToken;
tokenInfo.Text = ""; tokenInfo.Text = "";
return true; return true;
} }
if (TokensSinceNewLine == 1 if (TokensSinceNewLine == 1
&& !TokenStack.Any() && !TokenStack.Any()
&& LastToken.Kind == TokenKind.Identifier && LastToken.Kind == TokenKind.IdentifierToken
&& LastToken.TrailingTrivia.Any() && LastToken.TrailingTrivia.Any()
&& character != '=' && character != '='
&& character != '(' && character != '('
@ -470,7 +470,7 @@ namespace Parser.Internal
{ {
return ContinueParsingUnquotedStringLiteral(ref tokenInfo); return ContinueParsingUnquotedStringLiteral(ref tokenInfo);
} }
if (LastToken?.Kind == TokenKind.UnquotedStringLiteral if (LastToken?.Kind == TokenKind.UnquotedStringLiteralToken
&& !TokenStack.Any() && !TokenStack.Any()
&& TokensSinceNewLine > 0) && TokensSinceNewLine > 0)
{ {
@ -553,11 +553,11 @@ namespace Parser.Internal
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Equality; tokenInfo.Kind = TokenKind.EqualsEqualsToken;
} }
else else
{ {
tokenInfo.Kind = TokenKind.Assignment; tokenInfo.Kind = TokenKind.EqualsToken;
} }
return true; return true;
@ -578,72 +578,72 @@ namespace Parser.Internal
{ {
case '*': case '*':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.DotMultiply; tokenInfo.Kind = TokenKind.DotStarToken;
break; break;
case '/': case '/':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.DotDivide; tokenInfo.Kind = TokenKind.DotSlashToken;
break; break;
case '^': case '^':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.DotPower; tokenInfo.Kind = TokenKind.DotCaretToken;
break; break;
case '\\': case '\\':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.DotBackslash; tokenInfo.Kind = TokenKind.DotBackslashToken;
break; break;
case '\'': case '\'':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.DotTranspose; tokenInfo.Kind = TokenKind.DotApostropheToken;
break; break;
default: default:
tokenInfo.Kind = TokenKind.Dot; tokenInfo.Kind = TokenKind.DotToken;
break; break;
} }
return true; return true;
case '(': case '(':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.OpeningBracket; tokenInfo.Kind = TokenKind.OpenParenthesisToken;
return true; return true;
case ')': case ')':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.ClosingBracket; tokenInfo.Kind = TokenKind.CloseParenthesisToken;
return true; return true;
case '[': case '[':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.OpeningSquareBracket; tokenInfo.Kind = TokenKind.OpenSquareBracketToken;
return true; return true;
case ']': case ']':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.ClosingSquareBracket; tokenInfo.Kind = TokenKind.CloseSquareBracketToken;
return true; return true;
case '{': case '{':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.OpeningBrace; tokenInfo.Kind = TokenKind.OpenBraceToken;
return true; return true;
case '}': case '}':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.ClosingBrace; tokenInfo.Kind = TokenKind.CloseBraceToken;
return true; return true;
case ',': case ',':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Comma; tokenInfo.Kind = TokenKind.CommaToken;
return true; return true;
case ';': case ';':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Semicolon; tokenInfo.Kind = TokenKind.SemicolonToken;
return true; return true;
case '&': case '&':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '&') if (Window.PeekChar() == '&')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.LogicalAnd; tokenInfo.Kind = TokenKind.AmpersandAmpersandToken;
} }
else else
{ {
tokenInfo.Kind = TokenKind.BitwiseAnd; tokenInfo.Kind = TokenKind.AmpersandToken;
} }
return true; return true;
@ -652,11 +652,11 @@ namespace Parser.Internal
if (Window.PeekChar() == '|') if (Window.PeekChar() == '|')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.LogicalOr; tokenInfo.Kind = TokenKind.PipePipeToken;
} }
else else
{ {
tokenInfo.Kind = TokenKind.BitwiseOr; tokenInfo.Kind = TokenKind.PipeToken;
} }
@ -666,11 +666,11 @@ namespace Parser.Internal
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.LessOrEqual; tokenInfo.Kind = TokenKind.LessOrEqualsToken;
} }
else else
{ {
tokenInfo.Kind = TokenKind.Less; tokenInfo.Kind = TokenKind.LessToken;
} }
return true; return true;
@ -679,11 +679,11 @@ namespace Parser.Internal
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.GreaterOrEqual; tokenInfo.Kind = TokenKind.GreaterOrEqualsToken;
} }
else else
{ {
tokenInfo.Kind = TokenKind.Greater; tokenInfo.Kind = TokenKind.GreaterToken;
} }
return true; return true;
@ -692,61 +692,61 @@ namespace Parser.Internal
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Inequality; tokenInfo.Kind = TokenKind.TildeEqualsToken;
} }
else else
{ {
tokenInfo.Kind = TokenKind.Not; tokenInfo.Kind = TokenKind.TildeToken;
} }
return true; return true;
case '+': case '+':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Plus; tokenInfo.Kind = TokenKind.PlusToken;
return true; return true;
case '-': case '-':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Minus; tokenInfo.Kind = TokenKind.MinusToken;
return true; return true;
case '*': case '*':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Multiply; tokenInfo.Kind = TokenKind.StarToken;
return true; return true;
case '/': case '/':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Divide; tokenInfo.Kind = TokenKind.SlashToken;
return true; return true;
case '\\': case '\\':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Backslash; tokenInfo.Kind = TokenKind.BackslashToken;
return true; return true;
case '^': case '^':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Power; tokenInfo.Kind = TokenKind.CaretToken;
return true; return true;
case '@': case '@':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.At; tokenInfo.Kind = TokenKind.AtToken;
return true; return true;
case ':': case ':':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Colon; tokenInfo.Kind = TokenKind.ColonToken;
return true; return true;
case '?': case '?':
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.QuestionMark; tokenInfo.Kind = TokenKind.QuestionToken;
return true; return true;
case '\'': case '\'':
if (LastToken != null && if (LastToken != null &&
(LastToken.Kind == TokenKind.ClosingBrace (LastToken.Kind == TokenKind.CloseBraceToken
|| LastToken.Kind == TokenKind.ClosingBracket || LastToken.Kind == TokenKind.CloseParenthesisToken
|| LastToken.Kind == TokenKind.ClosingSquareBracket || LastToken.Kind == TokenKind.CloseSquareBracketToken
|| LastToken.Kind == TokenKind.Identifier)) || LastToken.Kind == TokenKind.IdentifierToken))
{ {
if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0) if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0)
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.Transpose; tokenInfo.Kind = TokenKind.ApostropheToken;
return true; return true;
} }
} }
@ -754,7 +754,7 @@ namespace Parser.Internal
case '"': case '"':
return ContinueLexingDoubleQuotedStringLiteral(ref tokenInfo); return ContinueLexingDoubleQuotedStringLiteral(ref tokenInfo);
case '\0': case '\0':
tokenInfo.Kind = TokenKind.EndOfFile; tokenInfo.Kind = TokenKind.EndOfFileToken;
return true; return true;
default: default:
Diagnostics.ReportUnknownSymbol(new TextSpan(Window.Position.Offset, 1), character); Diagnostics.ReportUnknownSymbol(new TextSpan(Window.Position.Offset, 1), character);
@ -772,7 +772,7 @@ namespace Parser.Internal
var tokenInfo = new TokenInfo(); var tokenInfo = new TokenInfo();
LexTokenWithoutTrivia(leadingTrivia, ref tokenInfo); LexTokenWithoutTrivia(leadingTrivia, ref tokenInfo);
var trailingTrivia = LexTrivia(true); var trailingTrivia = LexTrivia(true);
if (trailingTrivia.Any(t => t.Kind == TokenKind.Newline)) if (trailingTrivia.Any(t => t.Kind == TokenKind.NewlineToken))
{ {
TokensSinceNewLine = 0; TokensSinceNewLine = 0;
} }
@ -806,7 +806,7 @@ namespace Parser.Internal
} }
} }
if (tokenInfo.Kind == TokenKind.EndOfFile if (tokenInfo.Kind == TokenKind.EndOfFileToken
&& TokenStack.Any()) && TokenStack.Any())
{ {
throw new ParsingException($"Unmatched \"{TokenStack.Pop()}\" by the end of file."); throw new ParsingException($"Unmatched \"{TokenStack.Pop()}\" by the end of file.");
@ -827,32 +827,32 @@ namespace Parser.Internal
{ {
switch (tokenInfo.Kind) switch (tokenInfo.Kind)
{ {
case TokenKind.Identifier: case TokenKind.IdentifierToken:
return TokenFactory.CreateIdentifier( return TokenFactory.CreateIdentifier(
tokenInfo.Text, tokenInfo.Text,
leadingTrivia, leadingTrivia,
trailingTrivia); trailingTrivia);
case TokenKind.UnquotedStringLiteral: case TokenKind.UnquotedStringLiteralToken:
return TokenFactory.CreateUnquotedStringLiteral( return TokenFactory.CreateUnquotedStringLiteral(
tokenInfo.Text, tokenInfo.Text,
tokenInfo.StringValue, tokenInfo.StringValue,
leadingTrivia, leadingTrivia,
trailingTrivia); trailingTrivia);
case TokenKind.NumberLiteral: case TokenKind.NumberLiteralToken:
return TokenFactory.CreateTokenWithValueAndTrivia<double>( return TokenFactory.CreateTokenWithValueAndTrivia<double>(
tokenInfo.Kind, tokenInfo.Kind,
tokenInfo.Text, tokenInfo.Text,
tokenInfo.DoubleValue, tokenInfo.DoubleValue,
leadingTrivia, leadingTrivia,
trailingTrivia); trailingTrivia);
case TokenKind.StringLiteral: case TokenKind.StringLiteralToken:
return TokenFactory.CreateTokenWithValueAndTrivia<string>( return TokenFactory.CreateTokenWithValueAndTrivia<string>(
tokenInfo.Kind, tokenInfo.Kind,
tokenInfo.Text, tokenInfo.Text,
tokenInfo.StringValue, tokenInfo.StringValue,
leadingTrivia, leadingTrivia,
trailingTrivia); trailingTrivia);
case TokenKind.DoubleQuotedStringLiteral: case TokenKind.DoubleQuotedStringLiteralToken:
return TokenFactory.CreateTokenWithValueAndTrivia<string>( return TokenFactory.CreateTokenWithValueAndTrivia<string>(
tokenInfo.Kind, tokenInfo.Kind,
tokenInfo.Text, tokenInfo.Text,
@ -879,7 +879,7 @@ namespace Parser.Internal
throw new ParsingException($"Unexpected character: '{Window.PeekChar()}' at {Window.Position}."); throw new ParsingException($"Unexpected character: '{Window.PeekChar()}' at {Window.Position}.");
} }
result.Add(pair); result.Add(pair);
if (token.Kind == TokenKind.EndOfFile) if (token.Kind == TokenKind.EndOfFileToken)
{ {
return result; return result;
} }

196
Parser/Internal/MParserGreen.cs
View File

@ -41,27 +41,27 @@ namespace Parser.Internal
private SyntaxToken EatIdentifier() private SyntaxToken EatIdentifier()
{ {
return EatToken(TokenKind.Identifier); return EatToken(TokenKind.IdentifierToken);
} }
private bool IsIdentifier(SyntaxToken token, string s) private bool IsIdentifier(SyntaxToken token, string s)
{ {
return token.Kind == TokenKind.Identifier && token.Text == s; return token.Kind == TokenKind.IdentifierToken && token.Text == s;
} }
private SyntaxToken EatIdentifier(string s) private SyntaxToken EatIdentifier(string s)
{ {
var token = CurrentToken; var token = CurrentToken;
if (token.Kind != TokenKind.Identifier) if (token.Kind != TokenKind.IdentifierToken)
{ {
Errors.Add($"Unexpected token \"{token.Text}\" instead of {TokenKind.Identifier} at {CurrentPosition}."); Errors.Add($"Unexpected token \"{token.Text}\" instead of {TokenKind.IdentifierToken} at {CurrentPosition}.");
return TokenFactory.CreateMissing(TokenKind.Identifier, null, null); return TokenFactory.CreateMissing(TokenKind.IdentifierToken, null, null);
} }
if (token.Text != s) if (token.Text != s)
{ {
Errors.Add($"Unexpected token \"{token.Text}\" instead of identifier {s} at {CurrentPosition}."); Errors.Add($"Unexpected token \"{token.Text}\" instead of identifier {s} at {CurrentPosition}.");
return TokenFactory.CreateMissing(TokenKind.Identifier, null, null); return TokenFactory.CreateMissing(TokenKind.IdentifierToken, null, null);
} }
_index++; _index++;
@ -71,7 +71,7 @@ namespace Parser.Internal
private SyntaxToken PossiblyEatIdentifier(string s) private SyntaxToken PossiblyEatIdentifier(string s)
{ {
var token = CurrentToken; var token = CurrentToken;
if (token.Kind == TokenKind.Identifier && token.Text == s) if (token.Kind == TokenKind.IdentifierToken && token.Text == s)
{ {
return EatToken(); return EatToken();
} }
@ -82,13 +82,13 @@ namespace Parser.Internal
private SyntaxToken EatPossiblyMissingIdentifier(string s) private SyntaxToken EatPossiblyMissingIdentifier(string s)
{ {
var token = CurrentToken; var token = CurrentToken;
if (token.Kind == TokenKind.Identifier && token.Text == s) if (token.Kind == TokenKind.IdentifierToken && token.Text == s)
{ {
return EatToken(); return EatToken();
} }
return TokenFactory.CreateMissing( return TokenFactory.CreateMissing(
TokenKind.Identifier, TokenKind.IdentifierToken,
new List<SyntaxTrivia>(), new List<SyntaxTrivia>(),
new List<SyntaxTrivia>()); new List<SyntaxTrivia>());
} }
@ -97,15 +97,15 @@ namespace Parser.Internal
{ {
var outputs = new SyntaxListBuilder(); var outputs = new SyntaxListBuilder();
var firstToken = true; var firstToken = true;
while (CurrentToken.Kind != TokenKind.ClosingSquareBracket) while (CurrentToken.Kind != TokenKind.CloseSquareBracketToken)
{ {
if (!firstToken && CurrentToken.Kind == TokenKind.Comma) if (!firstToken && CurrentToken.Kind == TokenKind.CommaToken)
{ {
outputs.Add(EatToken()); outputs.Add(EatToken());
} }
firstToken = false; firstToken = false;
outputs.Add(Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier))); outputs.Add(Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken)));
} }
return outputs.ToList(); return outputs.ToList();
@ -116,19 +116,19 @@ namespace Parser.Internal
SyntaxToken assignmentSign; SyntaxToken assignmentSign;
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
if (CurrentToken.Kind == TokenKind.Identifier) if (CurrentToken.Kind == TokenKind.IdentifierToken)
{ {
if (PeekToken(1).Kind == TokenKind.Assignment) if (PeekToken(1).Kind == TokenKind.EqualsToken)
{ {
builder.Add(Factory.IdentifierNameSyntax(EatToken())); builder.Add(Factory.IdentifierNameSyntax(EatToken()));
assignmentSign = EatToken(TokenKind.Assignment); assignmentSign = EatToken(TokenKind.EqualsToken);
} }
else else
{ {
return null; return null;
} }
} }
else if (CurrentToken.Kind == TokenKind.OpeningSquareBracket) else if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
var outputs = ParseFunctionOutputList(); var outputs = ParseFunctionOutputList();
@ -137,8 +137,8 @@ namespace Parser.Internal
builder.AddRange(outputs); builder.AddRange(outputs);
} }
builder.Add(EatToken(TokenKind.ClosingSquareBracket)); builder.Add(EatToken(TokenKind.CloseSquareBracketToken));
assignmentSign = EatToken(TokenKind.Assignment); assignmentSign = EatToken(TokenKind.EqualsToken);
} }
else else
{ {
@ -154,25 +154,25 @@ namespace Parser.Internal
{ {
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
var firstToken = true; var firstToken = true;
while (CurrentToken.Kind != TokenKind.ClosingBracket) while (CurrentToken.Kind != TokenKind.CloseParenthesisToken)
{ {
if (!firstToken) if (!firstToken)
{ {
builder.Add(EatToken(TokenKind.Comma)); builder.Add(EatToken(TokenKind.CommaToken));
} }
else else
{ {
firstToken = false; firstToken = false;
} }
if (CurrentToken.Kind == TokenKind.Not) if (CurrentToken.Kind == TokenKind.TildeToken)
{ {
var notToken = EatToken(); var notToken = EatToken();
builder.Add(notToken); builder.Add(notToken);
} }
else else
{ {
var identifierToken = EatToken(TokenKind.Identifier); var identifierToken = EatToken(TokenKind.IdentifierToken);
builder.Add(Factory.IdentifierNameSyntax(identifierToken)); builder.Add(Factory.IdentifierNameSyntax(identifierToken));
} }
} }
@ -182,11 +182,11 @@ namespace Parser.Internal
private FunctionInputDescriptionSyntaxNode ParseFunctionInputDescription() private FunctionInputDescriptionSyntaxNode ParseFunctionInputDescription()
{ {
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
var openingBracket = EatToken(TokenKind.OpeningBracket); var openingBracket = EatToken(TokenKind.OpenParenthesisToken);
var parameterList = ParseParameterList(); var parameterList = ParseParameterList();
var closingBracket = EatToken(TokenKind.ClosingBracket); var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
return Factory.FunctionInputDescriptionSyntax( return Factory.FunctionInputDescriptionSyntax(
openingBracket, openingBracket,
parameterList, parameterList,
@ -202,7 +202,7 @@ namespace Parser.Internal
{ {
var functionKeyword = EatIdentifier("function"); var functionKeyword = EatIdentifier("function");
var outputDescription = ParseFunctionOutputDescription(); var outputDescription = ParseFunctionOutputDescription();
var name = EatToken(TokenKind.Identifier); var name = EatToken(TokenKind.IdentifierToken);
var inputDescription = ParseFunctionInputDescription(); var inputDescription = ParseFunctionInputDescription();
var commas = ParseOptionalCommas(); var commas = ParseOptionalCommas();
var body = ParseStatementList(); var body = ParseStatementList();
@ -236,9 +236,9 @@ namespace Parser.Internal
private ArrayLiteralExpressionSyntaxNode ParseArrayLiteral() private ArrayLiteralExpressionSyntaxNode ParseArrayLiteral()
{ {
var openingSquareBracket = EatToken(TokenKind.OpeningSquareBracket); var openingSquareBracket = EatToken(TokenKind.OpenSquareBracketToken);
var elements = ParseArrayElementList(TokenKind.ClosingSquareBracket); var elements = ParseArrayElementList(TokenKind.CloseSquareBracketToken);
var closingSquareBracket = EatToken(TokenKind.ClosingSquareBracket); var closingSquareBracket = EatToken(TokenKind.CloseSquareBracketToken);
return Factory.ArrayLiteralExpressionSyntax( return Factory.ArrayLiteralExpressionSyntax(
openingSquareBracket, openingSquareBracket,
elements, elements,
@ -247,9 +247,9 @@ namespace Parser.Internal
private CellArrayLiteralExpressionSyntaxNode ParseCellArrayLiteral() private CellArrayLiteralExpressionSyntaxNode ParseCellArrayLiteral()
{ {
var openingBrace = EatToken(TokenKind.OpeningBrace); var openingBrace = EatToken(TokenKind.OpenBraceToken);
var elements = ParseArrayElementList(TokenKind.ClosingBrace); var elements = ParseArrayElementList(TokenKind.CloseBraceToken);
var closingBrace = EatToken(TokenKind.ClosingBrace); var closingBrace = EatToken(TokenKind.CloseBraceToken);
return Factory.CellArrayLiteralExpressionSyntax( return Factory.CellArrayLiteralExpressionSyntax(
openingBrace, openingBrace,
elements, elements,
@ -265,8 +265,8 @@ namespace Parser.Internal
{ {
if (!firstToken) if (!firstToken)
{ {
if (CurrentToken.Kind == TokenKind.Comma if (CurrentToken.Kind == TokenKind.CommaToken
|| CurrentToken.Kind == TokenKind.Semicolon) || CurrentToken.Kind == TokenKind.SemicolonToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
} }
@ -292,28 +292,28 @@ namespace Parser.Internal
ExpressionSyntaxNode expression = null; ExpressionSyntaxNode expression = null;
switch (token.Kind) switch (token.Kind)
{ {
case TokenKind.Identifier: case TokenKind.IdentifierToken:
expression = Factory.IdentifierNameSyntax(EatToken()); expression = Factory.IdentifierNameSyntax(EatToken());
break; break;
case TokenKind.NumberLiteral: case TokenKind.NumberLiteralToken:
expression = Factory.NumberLiteralSyntax(EatToken()); expression = Factory.NumberLiteralSyntax(EatToken());
break; break;
case TokenKind.StringLiteral: case TokenKind.StringLiteralToken:
expression = Factory.StringLiteralSyntax(EatToken()); expression = Factory.StringLiteralSyntax(EatToken());
break; break;
case TokenKind.DoubleQuotedStringLiteral: case TokenKind.DoubleQuotedStringLiteralToken:
expression = Factory.DoubleQuotedStringLiteralSyntax(EatToken()); expression = Factory.DoubleQuotedStringLiteralSyntax(EatToken());
break; break;
case TokenKind.OpeningSquareBracket: case TokenKind.OpenSquareBracketToken:
expression = ParseArrayLiteral(); expression = ParseArrayLiteral();
break; break;
case TokenKind.OpeningBrace: case TokenKind.OpenBraceToken:
expression = ParseCellArrayLiteral(); expression = ParseCellArrayLiteral();
break; break;
case TokenKind.Colon: case TokenKind.ColonToken:
expression = Factory.EmptyExpressionSyntax(); expression = Factory.EmptyExpressionSyntax();
break; break;
case TokenKind.OpeningBracket: case TokenKind.OpenParenthesisToken:
expression = ParseParenthesizedExpression(); expression = ParseParenthesizedExpression();
break; break;
} }
@ -332,14 +332,14 @@ namespace Parser.Internal
var token = CurrentToken; var token = CurrentToken;
switch (token.Kind) switch (token.Kind)
{ {
case TokenKind.OpeningBrace: // cell array element access case TokenKind.OpenBraceToken: // cell array element access
if (options.ParsingArrayElements && expression.TrailingTrivia.Any()) if (options.ParsingArrayElements && expression.TrailingTrivia.Any())
{ {
return expression; return expression;
} }
var openingBrace = EatToken(); var openingBrace = EatToken();
var indices = ParseArrayElementList(TokenKind.ClosingBrace); var indices = ParseArrayElementList(TokenKind.CloseBraceToken);
var closingBrace = EatToken(TokenKind.ClosingBrace); var closingBrace = EatToken(TokenKind.CloseBraceToken);
expression = Factory.CellArrayElementAccessExpressionSyntax( expression = Factory.CellArrayElementAccessExpressionSyntax(
expression, expression,
openingBrace, openingBrace,
@ -347,21 +347,21 @@ namespace Parser.Internal
closingBrace closingBrace
); );
break; break;
case TokenKind.OpeningBracket: // function call case TokenKind.OpenParenthesisToken: // function call
if (options.ParsingArrayElements && expression.TrailingTrivia.Any()) if (options.ParsingArrayElements && expression.TrailingTrivia.Any())
{ {
return expression; return expression;
} }
var openingBracket = EatToken(); var openingBracket = EatToken();
var parameters = ParseFunctionCallParameterList(); var parameters = ParseFunctionCallParameterList();
var closingBracket = EatToken(TokenKind.ClosingBracket); var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
expression = Factory.FunctionCallExpressionSyntax( expression = Factory.FunctionCallExpressionSyntax(
expression, expression,
openingBracket, openingBracket,
parameters, parameters,
closingBracket); closingBracket);
break; break;
case TokenKind.Dot: // member access case TokenKind.DotToken: // member access
if (expression is IdentifierNameSyntaxNode if (expression is IdentifierNameSyntaxNode
|| expression is MemberAccessSyntaxNode || expression is MemberAccessSyntaxNode
|| expression is FunctionCallExpressionSyntaxNode || expression is FunctionCallExpressionSyntaxNode
@ -378,14 +378,14 @@ namespace Parser.Internal
} }
break; break;
case TokenKind.Transpose: case TokenKind.ApostropheToken:
case TokenKind.DotTranspose: case TokenKind.DotApostropheToken:
var operation = EatToken(); var operation = EatToken();
expression = Factory.UnaryPostixOperationExpressionSyntax(expression, operation); expression = Factory.UnaryPostixOperationExpressionSyntax(expression, operation);
break; break;
case TokenKind.UnquotedStringLiteral: case TokenKind.UnquotedStringLiteralToken:
return ParseCommandExpression(expression); return ParseCommandExpression(expression);
case TokenKind.At: case TokenKind.AtToken:
if (expression.TrailingTrivia.Any()) if (expression.TrailingTrivia.Any())
{ {
return expression; return expression;
@ -402,7 +402,7 @@ namespace Parser.Internal
if (expression is IdentifierNameSyntaxNode idNameNode) if (expression is IdentifierNameSyntaxNode idNameNode)
{ {
var builder = new SyntaxListBuilder<UnquotedStringLiteralSyntaxNode>(); var builder = new SyntaxListBuilder<UnquotedStringLiteralSyntaxNode>();
while (CurrentToken.Kind == TokenKind.UnquotedStringLiteral) while (CurrentToken.Kind == TokenKind.UnquotedStringLiteralToken)
{ {
builder.Add(Factory.UnquotedStringLiteralSyntax(EatToken())); builder.Add(Factory.UnquotedStringLiteralSyntax(EatToken()));
} }
@ -433,15 +433,15 @@ namespace Parser.Internal
private ExpressionSyntaxNode ParseMemberAccess() private ExpressionSyntaxNode ParseMemberAccess()
{ {
if (CurrentToken.Kind == TokenKind.Identifier) if (CurrentToken.Kind == TokenKind.IdentifierToken)
{ {
return Factory.IdentifierNameSyntax(EatToken()); return Factory.IdentifierNameSyntax(EatToken());
} }
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
var openingBracket = EatToken(); var openingBracket = EatToken();
var indirectMember = ParseExpression(); var indirectMember = ParseExpression();
var closingBracket = EatToken(TokenKind.ClosingBracket); var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
return Factory.IndirectMemberAccessSyntax( return Factory.IndirectMemberAccessSyntax(
openingBracket, openingBracket,
indirectMember, indirectMember,
@ -454,11 +454,11 @@ namespace Parser.Internal
{ {
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
var firstToken = true; var firstToken = true;
while (CurrentToken.Kind != TokenKind.ClosingBracket) while (CurrentToken.Kind != TokenKind.CloseParenthesisToken)
{ {
if (!firstToken) if (!firstToken)
{ {
builder.Add(EatToken(TokenKind.Comma)); builder.Add(EatToken(TokenKind.CommaToken));
} }
else else
{ {
@ -473,9 +473,9 @@ namespace Parser.Internal
private ParenthesizedExpressionSyntaxNode ParseParenthesizedExpression() private ParenthesizedExpressionSyntaxNode ParseParenthesizedExpression()
{ {
var openParen = EatToken(TokenKind.OpeningBracket); var openParen = EatToken(TokenKind.OpenParenthesisToken);
var expression = ParseExpression(); var expression = ParseExpression();
var closeParen = EatToken(TokenKind.ClosingBracket); var closeParen = EatToken(TokenKind.CloseParenthesisToken);
return Factory.ParenthesizedExpressionSyntax( return Factory.ParenthesizedExpressionSyntax(
openParen, openParen,
expression, expression,
@ -484,16 +484,16 @@ namespace Parser.Internal
private CompoundNameSyntaxNode ParseCompoundName() private CompoundNameSyntaxNode ParseCompoundName()
{ {
var lastToken = EatToken(TokenKind.Identifier); var lastToken = EatToken(TokenKind.IdentifierToken);
var firstName = lastToken; var firstName = lastToken;
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
builder.Add(firstName); builder.Add(firstName);
while (CurrentToken.Kind == TokenKind.Dot while (CurrentToken.Kind == TokenKind.DotToken
&& !lastToken.TrailingTrivia.Any()) && !lastToken.TrailingTrivia.Any())
{ {
var dot = EatToken(); var dot = EatToken();
builder.Add(dot); builder.Add(dot);
lastToken = EatToken(TokenKind.Identifier); lastToken = EatToken(TokenKind.IdentifierToken);
builder.Add(lastToken); builder.Add(lastToken);
} }
@ -503,14 +503,14 @@ namespace Parser.Internal
private FunctionHandleSyntaxNode ParseFunctionHandle() private FunctionHandleSyntaxNode ParseFunctionHandle()
{ {
var atSign = EatToken(); var atSign = EatToken();
if (CurrentToken.Kind == TokenKind.Identifier) if (CurrentToken.Kind == TokenKind.IdentifierToken)
{ {
var compoundName = ParseCompoundName(); var compoundName = ParseCompoundName();
return Factory.NamedFunctionHandleSyntax( return Factory.NamedFunctionHandleSyntax(
atSign, atSign,
compoundName); compoundName);
} }
else if (CurrentToken.Kind == TokenKind.OpeningBracket) else if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
var inputs = ParseFunctionInputDescription(); var inputs = ParseFunctionInputDescription();
var body = ParseExpression(); var body = ParseExpression();
@ -532,7 +532,7 @@ namespace Parser.Internal
var operand = ParseSubExpression(options, newPrecedence); var operand = ParseSubExpression(options, newPrecedence);
if (operand == null) if (operand == null)
{ {
if (options.ParsingArrayElements && operation.Kind == TokenKind.Not) if (options.ParsingArrayElements && operation.Kind == TokenKind.TildeToken)
{ {
operand = Factory.EmptyExpressionSyntax(); operand = Factory.EmptyExpressionSyntax();
} }
@ -543,7 +543,7 @@ namespace Parser.Internal
} }
lhs = Factory.UnaryPrefixOperationExpressionSyntax(operation, operand); lhs = Factory.UnaryPrefixOperationExpressionSyntax(operation, operand);
} }
else if (CurrentToken.Kind == TokenKind.At) else if (CurrentToken.Kind == TokenKind.AtToken)
{ {
return ParseFunctionHandle(); return ParseFunctionHandle();
} }
@ -570,11 +570,11 @@ namespace Parser.Internal
EatToken(); EatToken();
var rhs = ParseSubExpression(options, newPrecedence); var rhs = ParseSubExpression(options, newPrecedence);
if (rhs == null && token.Kind == TokenKind.Colon) // for parsing things like a{:} if (rhs == null && token.Kind == TokenKind.ColonToken) // for parsing things like a{:}
{ {
rhs = Factory.EmptyExpressionSyntax(); rhs = Factory.EmptyExpressionSyntax();
} }
if (token.Kind == TokenKind.Assignment) if (token.Kind == TokenKind.EqualsToken)
{ {
lhs = Factory.AssignmentExpressionSyntax(lhs, token, rhs); lhs = Factory.AssignmentExpressionSyntax(lhs, token, rhs);
} }
@ -595,7 +595,7 @@ namespace Parser.Internal
private SyntaxList<SyntaxToken> ParseOptionalCommas() private SyntaxList<SyntaxToken> ParseOptionalCommas()
{ {
var builder = new SyntaxListBuilder<SyntaxToken>(); var builder = new SyntaxListBuilder<SyntaxToken>();
while (CurrentToken.Kind == TokenKind.Comma) while (CurrentToken.Kind == TokenKind.CommaToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
} }
@ -606,8 +606,8 @@ namespace Parser.Internal
private SyntaxList<SyntaxToken> ParseOptionalSemicolonsOrCommas() private SyntaxList<SyntaxToken> ParseOptionalSemicolonsOrCommas()
{ {
var builder = new SyntaxListBuilder<SyntaxToken>(); var builder = new SyntaxListBuilder<SyntaxToken>();
while (CurrentToken.Kind == TokenKind.Comma while (CurrentToken.Kind == TokenKind.CommaToken
|| CurrentToken.Kind == TokenKind.Semicolon) || CurrentToken.Kind == TokenKind.SemicolonToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
} }
@ -763,7 +763,7 @@ namespace Parser.Internal
private AttributeAssignmentSyntaxNode ParseAttributeAssignment() private AttributeAssignmentSyntaxNode ParseAttributeAssignment()
{ {
if (CurrentToken.Kind == TokenKind.Assignment) if (CurrentToken.Kind == TokenKind.EqualsToken)
{ {
var assignmentSign = EatToken(); var assignmentSign = EatToken();
var value = ParseExpression(); var value = ParseExpression();
@ -775,7 +775,7 @@ namespace Parser.Internal
private AttributeSyntaxNode ParseAttribute() private AttributeSyntaxNode ParseAttribute()
{ {
var name = Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier)); var name = Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken));
var assignment = ParseAttributeAssignment(); var assignment = ParseAttributeAssignment();
return Factory.AttributeSyntax(name, assignment); return Factory.AttributeSyntax(name, assignment);
} }
@ -785,11 +785,11 @@ namespace Parser.Internal
var openingBracket = EatToken(); var openingBracket = EatToken();
var first = true; var first = true;
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
while (CurrentToken.Kind != TokenKind.ClosingBracket) while (CurrentToken.Kind != TokenKind.CloseParenthesisToken)
{ {
if (!first) if (!first)
{ {
var comma = EatToken(TokenKind.Comma); var comma = EatToken(TokenKind.CommaToken);
builder.Add(comma); builder.Add(comma);
} }
@ -816,13 +816,13 @@ namespace Parser.Internal
return ParseMethodDefinition(); return ParseMethodDefinition();
} }
if (CurrentToken.Kind == TokenKind.OpeningSquareBracket if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken
|| CurrentToken.Kind == TokenKind.Identifier) || CurrentToken.Kind == TokenKind.IdentifierToken)
{ {
return ParseAbstractMethodDeclaration(); return ParseAbstractMethodDeclaration();
} }
if (CurrentToken.Kind == TokenKind.Semicolon) if (CurrentToken.Kind == TokenKind.SemicolonToken)
{ {
return Factory.EmptyStatementSyntax(EatToken()); return Factory.EmptyStatementSyntax(EatToken());
} }
@ -852,7 +852,7 @@ namespace Parser.Internal
{ {
var methodsKeyword = EatToken(); var methodsKeyword = EatToken();
AttributeListSyntaxNode attributes = null; AttributeListSyntaxNode attributes = null;
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
attributes = ParseAttributesList(); attributes = ParseAttributesList();
} }
@ -870,7 +870,7 @@ namespace Parser.Internal
private GreenNode ParsePropertyDeclaration() private GreenNode ParsePropertyDeclaration()
{ {
if (CurrentToken.Kind == TokenKind.Comma) if (CurrentToken.Kind == TokenKind.CommaToken)
{ {
return EatToken(); return EatToken();
} }
@ -886,7 +886,7 @@ namespace Parser.Internal
{ {
var propertiesKeyword = EatToken(); var propertiesKeyword = EatToken();
AttributeListSyntaxNode attributes = null; AttributeListSyntaxNode attributes = null;
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
attributes = ParseAttributesList(); attributes = ParseAttributesList();
} }
@ -903,18 +903,18 @@ namespace Parser.Internal
private EnumerationItemValueSyntaxNode ParseEnumerationValue() private EnumerationItemValueSyntaxNode ParseEnumerationValue()
{ {
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
var openingBracket = EatToken(TokenKind.OpeningBracket); var openingBracket = EatToken(TokenKind.OpenParenthesisToken);
var expression = ParseExpression() ?? Factory.EmptyExpressionSyntax(); var expression = ParseExpression() ?? Factory.EmptyExpressionSyntax();
builder.Add(expression); builder.Add(expression);
while (CurrentToken.Kind == TokenKind.Comma) while (CurrentToken.Kind == TokenKind.CommaToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
builder.Add(ParseExpression()); builder.Add(ParseExpression());
} }
var closingBracket = EatToken(TokenKind.ClosingBracket); var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
return Factory.EnumerationItemValueSyntax(openingBracket, builder.ToList(), closingBracket); return Factory.EnumerationItemValueSyntax(openingBracket, builder.ToList(), closingBracket);
} }
return null; return null;
@ -933,7 +933,7 @@ namespace Parser.Internal
var enumerationKeyword = EatToken(); var enumerationKeyword = EatToken();
var builder = new SyntaxListBuilder<EnumerationItemSyntaxNode>(); var builder = new SyntaxListBuilder<EnumerationItemSyntaxNode>();
AttributeListSyntaxNode attributes = null; AttributeListSyntaxNode attributes = null;
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
attributes = ParseAttributesList(); attributes = ParseAttributesList();
} }
@ -955,7 +955,7 @@ namespace Parser.Internal
{ {
var eventsKeyword = EatToken(); var eventsKeyword = EatToken();
AttributeListSyntaxNode attributes = null; AttributeListSyntaxNode attributes = null;
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
attributes = ParseAttributesList(); attributes = ParseAttributesList();
} }
@ -974,7 +974,7 @@ namespace Parser.Internal
{ {
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
builder.Add(ParseCompoundName()); builder.Add(ParseCompoundName());
while (CurrentToken.Kind == TokenKind.BitwiseAnd) while (CurrentToken.Kind == TokenKind.AmpersandToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
builder.Add(ParseCompoundName()); builder.Add(ParseCompoundName());
@ -994,13 +994,13 @@ namespace Parser.Internal
{ {
var classdefKeyword = EatToken(); var classdefKeyword = EatToken();
AttributeListSyntaxNode attributes = null; AttributeListSyntaxNode attributes = null;
if (CurrentToken.Kind == TokenKind.OpeningBracket) if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
{ {
attributes = ParseAttributesList(); attributes = ParseAttributesList();
} }
var className = Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier)); var className = Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken));
BaseClassListSyntaxNode baseClassList = null; BaseClassListSyntaxNode baseClassList = null;
if (CurrentToken.Kind == TokenKind.Less) if (CurrentToken.Kind == TokenKind.LessToken)
{ {
baseClassList = ParseBaseClassList(); baseClassList = ParseBaseClassList();
} }
@ -1046,7 +1046,7 @@ namespace Parser.Internal
private StatementSyntaxNode ParseStatement() private StatementSyntaxNode ParseStatement()
{ {
if (CurrentToken.Kind == TokenKind.Identifier) if (CurrentToken.Kind == TokenKind.IdentifierToken)
{ {
switch (CurrentToken.Text) switch (CurrentToken.Text)
{ {
@ -1075,12 +1075,12 @@ namespace Parser.Internal
} }
} }
if (CurrentToken.Kind == TokenKind.OpeningSquareBracket) if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken)
{ {
return ParseExpressionStatement(); return ParseExpressionStatement();
} }
if (CurrentToken.Kind == TokenKind.Semicolon) if (CurrentToken.Kind == TokenKind.SemicolonToken)
{ {
return Factory.EmptyStatementSyntax(EatToken()); return Factory.EmptyStatementSyntax(EatToken());
} }
@ -1090,7 +1090,7 @@ namespace Parser.Internal
private SyntaxList ParseStatementList() private SyntaxList ParseStatementList()
{ {
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
while (CurrentToken.Kind != TokenKind.EndOfFile) while (CurrentToken.Kind != TokenKind.EndOfFileToken)
{ {
var node = ParseStatement(); var node = ParseStatement();
if (node == null) if (node == null)
@ -1098,8 +1098,8 @@ namespace Parser.Internal
break; break;
} }
builder.Add(node); builder.Add(node);
while (CurrentToken.Kind == TokenKind.Comma while (CurrentToken.Kind == TokenKind.CommaToken
|| CurrentToken.Kind == TokenKind.Semicolon) || CurrentToken.Kind == TokenKind.SemicolonToken)
{ {
builder.Add(EatToken()); builder.Add(EatToken());
} }

136
Parser/Internal/SyntaxFacts.cs
View File

@ -38,41 +38,41 @@ namespace Parser.Internal
{ {
switch (kind) switch (kind)
{ {
case TokenKind.Assignment: case TokenKind.EqualsToken:
return Precedence.Assignment; return Precedence.Assignment;
case TokenKind.LogicalOr: case TokenKind.PipePipeToken:
return Precedence.LogicalOr; return Precedence.LogicalOr;
case TokenKind.LogicalAnd: case TokenKind.AmpersandAmpersandToken:
return Precedence.LogicalAnd; return Precedence.LogicalAnd;
case TokenKind.BitwiseOr: case TokenKind.PipeToken:
return Precedence.BitwiseOr; return Precedence.BitwiseOr;
case TokenKind.BitwiseAnd: case TokenKind.AmpersandToken:
return Precedence.BitwiseAnd; return Precedence.BitwiseAnd;
case TokenKind.Less: case TokenKind.LessToken:
case TokenKind.LessOrEqual: case TokenKind.LessOrEqualsToken:
case TokenKind.Greater: case TokenKind.GreaterToken:
case TokenKind.GreaterOrEqual: case TokenKind.GreaterOrEqualsToken:
case TokenKind.Equality: case TokenKind.EqualsEqualsToken:
case TokenKind.Inequality: case TokenKind.TildeEqualsToken:
return Precedence.Relational; return Precedence.Relational;
case TokenKind.Colon: case TokenKind.ColonToken:
return Precedence.Colon; return Precedence.Colon;
case TokenKind.Plus: case TokenKind.PlusToken:
case TokenKind.Minus: case TokenKind.MinusToken:
return Precedence.Additive; return Precedence.Additive;
case TokenKind.Multiply: case TokenKind.StarToken:
case TokenKind.DotMultiply: case TokenKind.DotStarToken:
case TokenKind.Divide: case TokenKind.SlashToken:
case TokenKind.DotDivide: case TokenKind.DotSlashToken:
case TokenKind.Backslash: case TokenKind.BackslashToken:
case TokenKind.DotBackslash: case TokenKind.DotBackslashToken:
return Precedence.Multiplicative; return Precedence.Multiplicative;
case TokenKind.Not: case TokenKind.TildeToken:
return Precedence.Unary; return Precedence.Unary;
case TokenKind.Power: case TokenKind.CaretToken:
case TokenKind.DotPower: case TokenKind.DotCaretToken:
case TokenKind.Transpose: case TokenKind.ApostropheToken:
case TokenKind.DotTranspose: case TokenKind.DotApostropheToken:
return Precedence.Power; return Precedence.Power;
default: default:
return Precedence.Expression; return Precedence.Expression;
@ -113,9 +113,9 @@ namespace Parser.Internal
{ {
switch (tokenKind) switch (tokenKind)
{ {
case TokenKind.OpeningBrace: case TokenKind.OpenBraceToken:
case TokenKind.OpeningBracket: case TokenKind.OpenParenthesisToken:
case TokenKind.OpeningSquareBracket: case TokenKind.OpenSquareBracketToken:
return true; return true;
default: default:
return false; return false;
@ -126,9 +126,9 @@ namespace Parser.Internal
{ {
switch (tokenKind) switch (tokenKind)
{ {
case TokenKind.ClosingBrace: case TokenKind.CloseBraceToken:
case TokenKind.ClosingBracket: case TokenKind.CloseParenthesisToken:
case TokenKind.ClosingSquareBracket: case TokenKind.CloseSquareBracketToken:
return true; return true;
default: default:
return false; return false;
@ -144,12 +144,12 @@ namespace Parser.Internal
{ {
switch (tokenKind) switch (tokenKind)
{ {
case TokenKind.ClosingBrace: case TokenKind.CloseBraceToken:
return TokenKind.OpeningBrace; return TokenKind.OpenBraceToken;
case TokenKind.ClosingBracket: case TokenKind.CloseParenthesisToken:
return TokenKind.OpeningBracket; return TokenKind.OpenParenthesisToken;
case TokenKind.ClosingSquareBracket: case TokenKind.CloseSquareBracketToken:
return TokenKind.OpeningSquareBracket; return TokenKind.OpenSquareBracketToken;
default: default:
return null; return null;
} }
@ -224,10 +224,10 @@ namespace Parser.Internal
{ {
switch (kind) switch (kind)
{ {
case TokenKind.Plus: case TokenKind.PlusToken:
case TokenKind.Minus: case TokenKind.MinusToken:
case TokenKind.Not: case TokenKind.TildeToken:
case TokenKind.QuestionMark: case TokenKind.QuestionToken:
return true; return true;
default: default:
return false; return false;
@ -252,29 +252,29 @@ namespace Parser.Internal
{ {
switch (kind) switch (kind)
{ {
case TokenKind.Assignment: case TokenKind.EqualsToken:
case TokenKind.LogicalOr: case TokenKind.PipePipeToken:
case TokenKind.LogicalAnd: case TokenKind.AmpersandAmpersandToken:
case TokenKind.BitwiseOr: case TokenKind.PipeToken:
case TokenKind.BitwiseAnd: case TokenKind.AmpersandToken:
case TokenKind.Less: case TokenKind.LessToken:
case TokenKind.LessOrEqual: case TokenKind.LessOrEqualsToken:
case TokenKind.Greater: case TokenKind.GreaterToken:
case TokenKind.GreaterOrEqual: case TokenKind.GreaterOrEqualsToken:
case TokenKind.Equality: case TokenKind.EqualsEqualsToken:
case TokenKind.Inequality: case TokenKind.TildeEqualsToken:
case TokenKind.Colon: case TokenKind.ColonToken:
case TokenKind.Plus: case TokenKind.PlusToken:
case TokenKind.Minus: case TokenKind.MinusToken:
case TokenKind.Multiply: case TokenKind.StarToken:
case TokenKind.DotMultiply: case TokenKind.DotStarToken:
case TokenKind.Divide: case TokenKind.SlashToken:
case TokenKind.DotDivide: case TokenKind.DotSlashToken:
case TokenKind.Backslash: case TokenKind.BackslashToken:
case TokenKind.DotBackslash: case TokenKind.DotBackslashToken:
case TokenKind.Not: case TokenKind.TildeToken:
case TokenKind.Power: case TokenKind.CaretToken:
case TokenKind.DotPower: case TokenKind.DotCaretToken:
return true; return true;
default: default:
return false; return false;
@ -290,13 +290,13 @@ namespace Parser.Internal
{ {
switch (kind) switch (kind)
{ {
case TokenKind.Plus: case TokenKind.PlusToken:
return TokenKind.UnaryPlus; return TokenKind.UnaryPlus;
case TokenKind.Minus: case TokenKind.MinusToken:
return TokenKind.UnaryMinus; return TokenKind.UnaryMinus;
case TokenKind.Not: case TokenKind.TildeToken:
return TokenKind.UnaryNot; return TokenKind.UnaryNot;
case TokenKind.QuestionMark: case TokenKind.QuestionToken:
return TokenKind.UnaryQuestionMark; return TokenKind.UnaryQuestionMark;
default: default:
throw new ArgumentException(nameof(kind)); throw new ArgumentException(nameof(kind));

2
Parser/Internal/SyntaxToken.cs
View File

@ -124,7 +124,7 @@ namespace Parser.Internal
public SyntaxIdentifier( public SyntaxIdentifier(
string text string text
) : base(TokenKind.Identifier) ) : base(TokenKind.IdentifierToken)
{ {
_text = text; _text = text;
} }

2
Parser/Internal/TokenFactory.cs
View File

@ -42,7 +42,7 @@ namespace Parser.Internal
IReadOnlyList<SyntaxTrivia> trailingTrivia) IReadOnlyList<SyntaxTrivia> trailingTrivia)
{ {
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>( return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>(
TokenKind.UnquotedStringLiteral, TokenKind.UnquotedStringLiteralToken,
text, text,
value, value,
leadingTrivia, leadingTrivia,

95
Parser/TokenKind.cs
View File

@ -8,114 +8,115 @@
None = 0, None = 0,
BadToken = 1, BadToken = 1,
// The lexer puts a virtual "end of file" token at the end of the parsed file. // The lexer puts a virtual "end of file" token at the end of the parsed file.
EndOfFile = 2, EndOfFileToken = 2,
// Identifier: could be a reserved word, a variable name, a class name, etc. // Identifier: could be a reserved word, a variable name, a class name, etc.
Identifier = 3, IdentifierToken = 3,
// Number literal: 123, 45.678, 2e-5, etc. // Number literal: 123, 45.678, 2e-5, etc.
NumberLiteral = 4, NumberLiteralToken = 4,
// String literal: 'abc', '123', etc. The "usual" string literals are single-quoted and are just char arrays. // String literal: 'abc', '123', etc. The "usual" string literals are single-quoted and are just char arrays.
StringLiteral = 5, StringLiteralToken = 5,
// Double-quoted string literal: "abc", "123", etc. These are the "new" string literal that are more like strings // Double-quoted string literal: "abc", "123", etc. These are the "new" string literal that are more like strings
// and less like char arrays (for example, char arrays could be columns instead of rows, or even multi-dimensional). // and less like char arrays (for example, char arrays could be columns instead of rows, or even multi-dimensional).
DoubleQuotedStringLiteral = 6, DoubleQuotedStringLiteralToken = 6,
// This is for supporting "command statements" like // This is for supporting "command statements" like
// > cd some/+folder/ // > cd some/+folder/
// In this example, "some/folder" should be treated as a string literal (for example, "+' there should be a part // In this example, "some/folder" should be treated as a string literal (for example, "+' there should be a part
// of it, and not parsed as a binary operator). // of it, and not parsed as a binary operator).
UnquotedStringLiteral = 7, UnquotedStringLiteralToken = 7,
// trivia // trivia
// Spaces, tabs, etc. // Spaces, tabs, etc.
Whitespace = 10, WhitespaceToken = 10,
// New line characters. // New line characters.
Newline = 11, NewlineToken = 11,
// There are three types of comments: // There are three types of comments:
// * comments starting with % and lasting till the end of the line; // * comments starting with % and lasting till the end of the line;
// * comments starting with %{ and ending with %}, possibly spanning several lines; // * comments starting with %{ and ending with %}, possibly spanning several lines;
// * comments starting with ... and lasting till the end of the line. // * comments starting with ... and lasting till the end of the line.
// At the moment, this token is used to accomodate all of them, so the other two // At the moment, this token is used to accomodate all of them, so the other two
// (MultilineComment = 13 and DotDotDot = 56 are not used). // (MultilineComment = 13 and DotDotDot = 56 are not used).
Comment = 12, CommentToken = 12,
// Comments starting with %{ and ending with %}, possibly spanning several lines. Not used, see Comment = 12. // Comments starting with %{ and ending with %}, possibly spanning several lines. Not used, see Comment = 12.
MultilineComment = 13, MultilineCommentToken = 13,
// operators // operators
// = // =
Assignment = 20, EqualsToken = 20,
// == // ==
Equality = 21, EqualsEqualsToken = 21,
// ~= // ~=
Inequality = 22, TildeEqualsToken = 22,
// && // &&
LogicalAnd = 23, AmpersandAmpersandToken = 23,
// || // ||
LogicalOr = 24, PipePipeToken = 24,
// & // &
BitwiseAnd = 25, AmpersandToken = 25,
// | // |
BitwiseOr = 26, PipeToken = 26,
// < // <
Less = 27, LessToken = 27,
// <= // <=
LessOrEqual = 28, LessOrEqualsToken = 28,
// > // >
Greater = 29, GreaterToken = 29,
// >= // >=
GreaterOrEqual = 30, GreaterOrEqualsToken = 30,
// ~ // ~
Not = 31, TildeToken = 31,
// + // +
Plus = 32, PlusToken = 32,
// - // -
Minus = 33, MinusToken = 33,
// * // *
Multiply = 34, StarToken = 34,
// / // /
Divide = 35, SlashToken = 35,
// ^ // ^
Power = 36, CaretToken = 36,
// \ // \
Backslash = 37, BackslashToken = 37,
// ' (this is the same as starting string literal; we'll have some fun distinguishing those). // ' (this is the same as starting string literal; we'll have some fun distinguishing those).
Transpose = 38, ApostropheToken = 38,
// .* // .*
DotMultiply = 39, DotStarToken = 39,
// ./ // ./
DotDivide = 40, DotSlashToken = 40,
// .^ // .^
DotPower = 41, DotCaretToken = 41,
// .\ // .\
DotBackslash = 42, DotBackslashToken = 42,
// .' // .'
DotTranspose = 43, DotApostropheToken = 43,
// @ // @
At = 44, AtToken = 44,
// : // :
Colon = 45, ColonToken = 45,
// ? // ?
QuestionMark = 46, QuestionToken = 46,
// , // ,
Comma = 47, CommaToken = 47,
// ; // ;
Semicolon = 48, SemicolonToken = 48,
// { // {
OpeningBrace = 49, OpenBraceToken = 49,
// } // }
ClosingBrace = 50, CloseBraceToken = 50,
// [ // [
OpeningSquareBracket = 51, OpenSquareBracketToken = 51,
// ] // ]
ClosingSquareBracket = 52, CloseSquareBracketToken = 52,
// ( // (
OpeningBracket = 53, OpenParenthesisToken = 53,
// ) // )
ClosingBracket = 54, CloseParenthesisToken = 54,
// . // .
Dot = 55, DotToken = 55,
// Comments starting with ... and lasting till the end of the line. Not used, see Comment = 12. // Comments starting with ... and lasting till the end of the line. Not used, see Comment = 12.
DotDotDot = 56, DotDotDotToken = 56,
// Unary tokens are not recognized during lexing; they are contextually recognized while parsing. // Unary tokens are not recognized during lexing; they are contextually recognized while parsing.
UnaryPlus = 57, UnaryPlus = 57,
UnaryMinus = 58, UnaryMinus = 58,