Rename TokenKinds for consistency
This commit is contained in:
parent
1c37559d9e
commit
5eb63a00e0
@ -53,7 +53,7 @@ namespace ConsoleDemo
|
||||
{
|
||||
switch (token.Kind)
|
||||
{
|
||||
case TokenKind.Comma:
|
||||
case TokenKind.CommaToken:
|
||||
return true;
|
||||
default:
|
||||
break;
|
||||
|
||||
@ -150,7 +150,7 @@ namespace ProjectConsole
|
||||
if (nodeOrToken.IsToken)
|
||||
{
|
||||
var token = nodeOrToken.AsToken();
|
||||
if (token.Kind == TokenKind.Identifier)
|
||||
if (token.Kind == TokenKind.IdentifierToken)
|
||||
{
|
||||
OutputIdentifier(token);
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@ namespace Parser.Tests
|
||||
private IEnumerable<Internal.SyntaxToken> ParseText(string text)
|
||||
{
|
||||
var lexer = new MLexerGreen(new TextWindowWithNull(text));
|
||||
return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFile);
|
||||
return lexer.ParseAll().Select(x => x.Item1).Where(x => x.Kind != TokenKind.EndOfFileToken);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@ -40,12 +40,12 @@ namespace Parser.Tests
|
||||
{
|
||||
var text = text1 + separatorText + text2;
|
||||
var tokens = ParseText(text).ToArray();
|
||||
if (kind1 == TokenKind.Identifier && !ContainsNewLine(separatorText))
|
||||
if (kind1 == TokenKind.IdentifierToken && !ContainsNewLine(separatorText))
|
||||
{
|
||||
Assert.Equal(TokenKind.Identifier, tokens[0].Kind);
|
||||
Assert.Equal(TokenKind.IdentifierToken, tokens[0].Kind);
|
||||
foreach (var token in tokens.Skip(1))
|
||||
{
|
||||
Assert.Equal(TokenKind.UnquotedStringLiteral, token.Kind);
|
||||
Assert.Equal(TokenKind.UnquotedStringLiteralToken, token.Kind);
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -85,20 +85,20 @@ namespace Parser.Tests
|
||||
.Where(t => !(SyntaxFacts.IsUnaryTokenKind(t.kind)
|
||||
|| SyntaxFacts.IsOpeningToken(t.kind)
|
||||
|| SyntaxFacts.IsClosingToken(t.kind)
|
||||
|| t.kind == TokenKind.Transpose));
|
||||
|| t.kind == TokenKind.ApostropheToken));
|
||||
|
||||
|
||||
var dynamicTokens = new[]
|
||||
{
|
||||
(TokenKind.Identifier, "a"),
|
||||
(TokenKind.Identifier, "abc"),
|
||||
(TokenKind.NumberLiteral, "1"),
|
||||
(TokenKind.NumberLiteral, "123"),
|
||||
(TokenKind.NumberLiteral, "145.67"),
|
||||
(TokenKind.NumberLiteral, "14.5e-3"),
|
||||
(TokenKind.NumberLiteral, "3.14e8"),
|
||||
(TokenKind.StringLiteral, "'what is that'"),
|
||||
(TokenKind.DoubleQuotedStringLiteral, "\"Another ' string\"")
|
||||
(TokenKind.IdentifierToken, "a"),
|
||||
(TokenKind.IdentifierToken, "abc"),
|
||||
(TokenKind.NumberLiteralToken, "1"),
|
||||
(TokenKind.NumberLiteralToken, "123"),
|
||||
(TokenKind.NumberLiteralToken, "145.67"),
|
||||
(TokenKind.NumberLiteralToken, "14.5e-3"),
|
||||
(TokenKind.NumberLiteralToken, "3.14e8"),
|
||||
(TokenKind.StringLiteralToken, "'what is that'"),
|
||||
(TokenKind.DoubleQuotedStringLiteralToken, "\"Another ' string\"")
|
||||
};
|
||||
return fixedTokens.Concat(dynamicTokens);
|
||||
}
|
||||
@ -136,127 +136,127 @@ namespace Parser.Tests
|
||||
|
||||
private static bool RequiresSeparator(TokenKind kind1, TokenKind kind2)
|
||||
{
|
||||
if (kind1 == TokenKind.Less && kind2 == TokenKind.Equality)
|
||||
if (kind1 == TokenKind.LessToken && kind2 == TokenKind.EqualsEqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Less && kind2 == TokenKind.Assignment)
|
||||
if (kind1 == TokenKind.LessToken && kind2 == TokenKind.EqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Greater && kind2 == TokenKind.Equality)
|
||||
if (kind1 == TokenKind.GreaterToken && kind2 == TokenKind.EqualsEqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Greater && kind2 == TokenKind.Assignment)
|
||||
if (kind1 == TokenKind.GreaterToken && kind2 == TokenKind.EqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Not && kind2 == TokenKind.Assignment)
|
||||
if (kind1 == TokenKind.TildeToken && kind2 == TokenKind.EqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Not && kind2 == TokenKind.Equality)
|
||||
if (kind1 == TokenKind.TildeToken && kind2 == TokenKind.EqualsEqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Assignment && kind2 == TokenKind.Assignment)
|
||||
if (kind1 == TokenKind.EqualsToken && kind2 == TokenKind.EqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Assignment && kind2 == TokenKind.Equality)
|
||||
if (kind1 == TokenKind.EqualsToken && kind2 == TokenKind.EqualsEqualsToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.BitwiseAnd && kind2 == TokenKind.LogicalAnd)
|
||||
if (kind1 == TokenKind.AmpersandToken && kind2 == TokenKind.AmpersandAmpersandToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.BitwiseAnd && kind2 == TokenKind.BitwiseAnd)
|
||||
if (kind1 == TokenKind.AmpersandToken && kind2 == TokenKind.AmpersandToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.BitwiseOr && kind2 == TokenKind.LogicalOr)
|
||||
if (kind1 == TokenKind.PipeToken && kind2 == TokenKind.PipePipeToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.BitwiseOr && kind2 == TokenKind.BitwiseOr)
|
||||
if (kind1 == TokenKind.PipeToken && kind2 == TokenKind.PipeToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Multiply)
|
||||
if (kind1 == TokenKind.DotToken && kind2 == TokenKind.StarToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Divide)
|
||||
if (kind1 == TokenKind.DotToken && kind2 == TokenKind.SlashToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Backslash)
|
||||
if (kind1 == TokenKind.DotToken && kind2 == TokenKind.BackslashToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Dot && kind2 == TokenKind.Power)
|
||||
if (kind1 == TokenKind.DotToken && kind2 == TokenKind.CaretToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Dot && kind2 == TokenKind.NumberLiteral)
|
||||
if (kind1 == TokenKind.DotToken && kind2 == TokenKind.NumberLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.NumberLiteral && kind2 == TokenKind.Dot)
|
||||
if (kind1 == TokenKind.NumberLiteralToken && kind2 == TokenKind.DotToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Dot && kind2 == TokenKind.StringLiteral)
|
||||
if (kind1 == TokenKind.DotToken && kind2 == TokenKind.StringLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.NumberLiteral && kind2 == TokenKind.NumberLiteral)
|
||||
if (kind1 == TokenKind.NumberLiteralToken && kind2 == TokenKind.NumberLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.StringLiteral && kind2 == TokenKind.StringLiteral)
|
||||
if (kind1 == TokenKind.StringLiteralToken && kind2 == TokenKind.StringLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.DoubleQuotedStringLiteral && kind2 == TokenKind.DoubleQuotedStringLiteral)
|
||||
if (kind1 == TokenKind.DoubleQuotedStringLiteralToken && kind2 == TokenKind.DoubleQuotedStringLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Identifier && kind2 == TokenKind.Identifier)
|
||||
if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.IdentifierToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Identifier && kind2 == TokenKind.NumberLiteral)
|
||||
if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.NumberLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (kind1 == TokenKind.Identifier && kind2 == TokenKind.StringLiteral)
|
||||
if (kind1 == TokenKind.IdentifierToken && kind2 == TokenKind.StringLiteralToken)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -40,7 +40,7 @@ namespace Parser.Internal
|
||||
n++;
|
||||
}
|
||||
|
||||
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
|
||||
return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
|
||||
}
|
||||
|
||||
private SyntaxTrivia LexMultilineComment()
|
||||
@ -54,7 +54,7 @@ namespace Parser.Internal
|
||||
if (c == '\0')
|
||||
{
|
||||
Diagnostics.ReportUnexpectedEndOfFile(new TextSpan(Window.Position.Offset, 0));
|
||||
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
|
||||
return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
|
||||
}
|
||||
|
||||
if (c == '\n' || (c == '\r' && Window.PeekChar(n + 1) == '\n'))
|
||||
@ -69,12 +69,12 @@ namespace Parser.Internal
|
||||
n++;
|
||||
}
|
||||
|
||||
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
|
||||
return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
|
||||
}
|
||||
|
||||
if (metPercentSign && c == '}')
|
||||
{
|
||||
return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n+1));
|
||||
return TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n+1));
|
||||
}
|
||||
|
||||
metPercentSign = c == '%';
|
||||
@ -91,13 +91,13 @@ namespace Parser.Internal
|
||||
n++;
|
||||
}
|
||||
|
||||
var comment = TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
|
||||
var comment = TokenFactory.CreateTrivia(TokenKind.CommentToken, Window.GetAndConsumeChars(n));
|
||||
var result = new List<SyntaxTrivia> { comment };
|
||||
var character = Window.PeekChar();
|
||||
if (character == '\n' || character == '\r')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
result.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, character.ToString()));
|
||||
result.Add(TokenFactory.CreateTrivia(TokenKind.WhitespaceToken, character.ToString()));
|
||||
}
|
||||
|
||||
return result;
|
||||
@ -112,7 +112,7 @@ namespace Parser.Internal
|
||||
{
|
||||
if (whitespaceCache.Length > 0)
|
||||
{
|
||||
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, whitespaceCache.ToString()));
|
||||
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.WhitespaceToken, whitespaceCache.ToString()));
|
||||
}
|
||||
|
||||
whitespaceCache.Clear();
|
||||
@ -132,7 +132,7 @@ namespace Parser.Internal
|
||||
case '\n':
|
||||
FlushWhitespaceCache();
|
||||
Window.ConsumeChar();
|
||||
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Newline, character.ToString()));
|
||||
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.NewlineToken, character.ToString()));
|
||||
if (isTrailing)
|
||||
{
|
||||
return triviaList;
|
||||
@ -177,7 +177,7 @@ namespace Parser.Internal
|
||||
}
|
||||
|
||||
var identifier = Window.GetAndConsumeChars(n);
|
||||
tokenInfo.Kind = TokenKind.Identifier;
|
||||
tokenInfo.Kind = TokenKind.IdentifierToken;
|
||||
tokenInfo.Text = identifier;
|
||||
return true;
|
||||
}
|
||||
@ -191,7 +191,7 @@ namespace Parser.Internal
|
||||
if (c == ' ' || c == '\n' || c == '\0')
|
||||
{
|
||||
var literal = Window.GetAndConsumeChars(n);
|
||||
tokenInfo.Kind = TokenKind.UnquotedStringLiteral;
|
||||
tokenInfo.Kind = TokenKind.UnquotedStringLiteralToken;
|
||||
tokenInfo.Text = literal;
|
||||
tokenInfo.StringValue = literal;
|
||||
return true;
|
||||
@ -330,7 +330,7 @@ namespace Parser.Internal
|
||||
if (fail)
|
||||
{
|
||||
var s = Window.GetAndConsumeChars(n);
|
||||
tokenInfo.Kind = TokenKind.NumberLiteral;
|
||||
tokenInfo.Kind = TokenKind.NumberLiteralToken;
|
||||
tokenInfo.Text = s;
|
||||
return false;
|
||||
}
|
||||
@ -363,7 +363,7 @@ namespace Parser.Internal
|
||||
}
|
||||
var s = Window.GetAndConsumeChars(n);
|
||||
|
||||
tokenInfo.Kind = TokenKind.NumberLiteral;
|
||||
tokenInfo.Kind = TokenKind.NumberLiteralToken;
|
||||
tokenInfo.Text = s;
|
||||
return true;
|
||||
}
|
||||
@ -439,14 +439,14 @@ namespace Parser.Internal
|
||||
private bool ContinueLexingStringLiteral(ref TokenInfo tokenInfo)
|
||||
{
|
||||
ContinueLexingGeneralStringLiteral(ref tokenInfo, '\'');
|
||||
tokenInfo.Kind = TokenKind.StringLiteral;
|
||||
tokenInfo.Kind = TokenKind.StringLiteralToken;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool ContinueLexingDoubleQuotedStringLiteral(ref TokenInfo tokenInfo)
|
||||
{
|
||||
ContinueLexingGeneralStringLiteral(ref tokenInfo, '"');
|
||||
tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteral;
|
||||
tokenInfo.Kind = TokenKind.DoubleQuotedStringLiteralToken;
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -455,14 +455,14 @@ namespace Parser.Internal
|
||||
var character = Window.PeekChar();
|
||||
if (character == '\0')
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.EndOfFile;
|
||||
tokenInfo.Kind = TokenKind.EndOfFileToken;
|
||||
tokenInfo.Text = "";
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TokensSinceNewLine == 1
|
||||
&& !TokenStack.Any()
|
||||
&& LastToken.Kind == TokenKind.Identifier
|
||||
&& LastToken.Kind == TokenKind.IdentifierToken
|
||||
&& LastToken.TrailingTrivia.Any()
|
||||
&& character != '='
|
||||
&& character != '('
|
||||
@ -470,7 +470,7 @@ namespace Parser.Internal
|
||||
{
|
||||
return ContinueParsingUnquotedStringLiteral(ref tokenInfo);
|
||||
}
|
||||
if (LastToken?.Kind == TokenKind.UnquotedStringLiteral
|
||||
if (LastToken?.Kind == TokenKind.UnquotedStringLiteralToken
|
||||
&& !TokenStack.Any()
|
||||
&& TokensSinceNewLine > 0)
|
||||
{
|
||||
@ -553,11 +553,11 @@ namespace Parser.Internal
|
||||
if (Window.PeekChar() == '=')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Equality;
|
||||
tokenInfo.Kind = TokenKind.EqualsEqualsToken;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.Assignment;
|
||||
tokenInfo.Kind = TokenKind.EqualsToken;
|
||||
}
|
||||
|
||||
return true;
|
||||
@ -578,72 +578,72 @@ namespace Parser.Internal
|
||||
{
|
||||
case '*':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.DotMultiply;
|
||||
tokenInfo.Kind = TokenKind.DotStarToken;
|
||||
break;
|
||||
case '/':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.DotDivide;
|
||||
tokenInfo.Kind = TokenKind.DotSlashToken;
|
||||
break;
|
||||
case '^':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.DotPower;
|
||||
tokenInfo.Kind = TokenKind.DotCaretToken;
|
||||
break;
|
||||
case '\\':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.DotBackslash;
|
||||
tokenInfo.Kind = TokenKind.DotBackslashToken;
|
||||
break;
|
||||
case '\'':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.DotTranspose;
|
||||
tokenInfo.Kind = TokenKind.DotApostropheToken;
|
||||
break;
|
||||
default:
|
||||
tokenInfo.Kind = TokenKind.Dot;
|
||||
tokenInfo.Kind = TokenKind.DotToken;
|
||||
break;
|
||||
}
|
||||
|
||||
return true;
|
||||
case '(':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.OpeningBracket;
|
||||
tokenInfo.Kind = TokenKind.OpenParenthesisToken;
|
||||
return true;
|
||||
case ')':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.ClosingBracket;
|
||||
tokenInfo.Kind = TokenKind.CloseParenthesisToken;
|
||||
return true;
|
||||
case '[':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.OpeningSquareBracket;
|
||||
tokenInfo.Kind = TokenKind.OpenSquareBracketToken;
|
||||
return true;
|
||||
case ']':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.ClosingSquareBracket;
|
||||
tokenInfo.Kind = TokenKind.CloseSquareBracketToken;
|
||||
return true;
|
||||
case '{':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.OpeningBrace;
|
||||
tokenInfo.Kind = TokenKind.OpenBraceToken;
|
||||
return true;
|
||||
case '}':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.ClosingBrace;
|
||||
tokenInfo.Kind = TokenKind.CloseBraceToken;
|
||||
return true;
|
||||
case ',':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Comma;
|
||||
tokenInfo.Kind = TokenKind.CommaToken;
|
||||
return true;
|
||||
case ';':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Semicolon;
|
||||
tokenInfo.Kind = TokenKind.SemicolonToken;
|
||||
return true;
|
||||
case '&':
|
||||
Window.ConsumeChar();
|
||||
if (Window.PeekChar() == '&')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.LogicalAnd;
|
||||
tokenInfo.Kind = TokenKind.AmpersandAmpersandToken;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.BitwiseAnd;
|
||||
tokenInfo.Kind = TokenKind.AmpersandToken;
|
||||
}
|
||||
|
||||
return true;
|
||||
@ -652,11 +652,11 @@ namespace Parser.Internal
|
||||
if (Window.PeekChar() == '|')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.LogicalOr;
|
||||
tokenInfo.Kind = TokenKind.PipePipeToken;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.BitwiseOr;
|
||||
tokenInfo.Kind = TokenKind.PipeToken;
|
||||
|
||||
}
|
||||
|
||||
@ -666,11 +666,11 @@ namespace Parser.Internal
|
||||
if (Window.PeekChar() == '=')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.LessOrEqual;
|
||||
tokenInfo.Kind = TokenKind.LessOrEqualsToken;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.Less;
|
||||
tokenInfo.Kind = TokenKind.LessToken;
|
||||
}
|
||||
|
||||
return true;
|
||||
@ -679,11 +679,11 @@ namespace Parser.Internal
|
||||
if (Window.PeekChar() == '=')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.GreaterOrEqual;
|
||||
tokenInfo.Kind = TokenKind.GreaterOrEqualsToken;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.Greater;
|
||||
tokenInfo.Kind = TokenKind.GreaterToken;
|
||||
}
|
||||
|
||||
return true;
|
||||
@ -692,61 +692,61 @@ namespace Parser.Internal
|
||||
if (Window.PeekChar() == '=')
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Inequality;
|
||||
tokenInfo.Kind = TokenKind.TildeEqualsToken;
|
||||
}
|
||||
else
|
||||
{
|
||||
tokenInfo.Kind = TokenKind.Not;
|
||||
tokenInfo.Kind = TokenKind.TildeToken;
|
||||
}
|
||||
|
||||
return true;
|
||||
case '+':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Plus;
|
||||
tokenInfo.Kind = TokenKind.PlusToken;
|
||||
return true;
|
||||
case '-':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Minus;
|
||||
tokenInfo.Kind = TokenKind.MinusToken;
|
||||
return true;
|
||||
case '*':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Multiply;
|
||||
tokenInfo.Kind = TokenKind.StarToken;
|
||||
return true;
|
||||
case '/':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Divide;
|
||||
tokenInfo.Kind = TokenKind.SlashToken;
|
||||
return true;
|
||||
case '\\':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Backslash;
|
||||
tokenInfo.Kind = TokenKind.BackslashToken;
|
||||
return true;
|
||||
case '^':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Power;
|
||||
tokenInfo.Kind = TokenKind.CaretToken;
|
||||
return true;
|
||||
case '@':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.At;
|
||||
tokenInfo.Kind = TokenKind.AtToken;
|
||||
return true;
|
||||
case ':':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Colon;
|
||||
tokenInfo.Kind = TokenKind.ColonToken;
|
||||
return true;
|
||||
case '?':
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.QuestionMark;
|
||||
tokenInfo.Kind = TokenKind.QuestionToken;
|
||||
return true;
|
||||
case '\'':
|
||||
if (LastToken != null &&
|
||||
(LastToken.Kind == TokenKind.ClosingBrace
|
||||
|| LastToken.Kind == TokenKind.ClosingBracket
|
||||
|| LastToken.Kind == TokenKind.ClosingSquareBracket
|
||||
|| LastToken.Kind == TokenKind.Identifier))
|
||||
(LastToken.Kind == TokenKind.CloseBraceToken
|
||||
|| LastToken.Kind == TokenKind.CloseParenthesisToken
|
||||
|| LastToken.Kind == TokenKind.CloseSquareBracketToken
|
||||
|| LastToken.Kind == TokenKind.IdentifierToken))
|
||||
{
|
||||
if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0)
|
||||
{
|
||||
Window.ConsumeChar();
|
||||
tokenInfo.Kind = TokenKind.Transpose;
|
||||
tokenInfo.Kind = TokenKind.ApostropheToken;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -754,7 +754,7 @@ namespace Parser.Internal
|
||||
case '"':
|
||||
return ContinueLexingDoubleQuotedStringLiteral(ref tokenInfo);
|
||||
case '\0':
|
||||
tokenInfo.Kind = TokenKind.EndOfFile;
|
||||
tokenInfo.Kind = TokenKind.EndOfFileToken;
|
||||
return true;
|
||||
default:
|
||||
Diagnostics.ReportUnknownSymbol(new TextSpan(Window.Position.Offset, 1), character);
|
||||
@ -772,7 +772,7 @@ namespace Parser.Internal
|
||||
var tokenInfo = new TokenInfo();
|
||||
LexTokenWithoutTrivia(leadingTrivia, ref tokenInfo);
|
||||
var trailingTrivia = LexTrivia(true);
|
||||
if (trailingTrivia.Any(t => t.Kind == TokenKind.Newline))
|
||||
if (trailingTrivia.Any(t => t.Kind == TokenKind.NewlineToken))
|
||||
{
|
||||
TokensSinceNewLine = 0;
|
||||
}
|
||||
@ -806,7 +806,7 @@ namespace Parser.Internal
|
||||
}
|
||||
}
|
||||
|
||||
if (tokenInfo.Kind == TokenKind.EndOfFile
|
||||
if (tokenInfo.Kind == TokenKind.EndOfFileToken
|
||||
&& TokenStack.Any())
|
||||
{
|
||||
throw new ParsingException($"Unmatched \"{TokenStack.Pop()}\" by the end of file.");
|
||||
@ -827,32 +827,32 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (tokenInfo.Kind)
|
||||
{
|
||||
case TokenKind.Identifier:
|
||||
case TokenKind.IdentifierToken:
|
||||
return TokenFactory.CreateIdentifier(
|
||||
tokenInfo.Text,
|
||||
leadingTrivia,
|
||||
trailingTrivia);
|
||||
case TokenKind.UnquotedStringLiteral:
|
||||
case TokenKind.UnquotedStringLiteralToken:
|
||||
return TokenFactory.CreateUnquotedStringLiteral(
|
||||
tokenInfo.Text,
|
||||
tokenInfo.StringValue,
|
||||
leadingTrivia,
|
||||
trailingTrivia);
|
||||
case TokenKind.NumberLiteral:
|
||||
case TokenKind.NumberLiteralToken:
|
||||
return TokenFactory.CreateTokenWithValueAndTrivia<double>(
|
||||
tokenInfo.Kind,
|
||||
tokenInfo.Text,
|
||||
tokenInfo.DoubleValue,
|
||||
leadingTrivia,
|
||||
trailingTrivia);
|
||||
case TokenKind.StringLiteral:
|
||||
case TokenKind.StringLiteralToken:
|
||||
return TokenFactory.CreateTokenWithValueAndTrivia<string>(
|
||||
tokenInfo.Kind,
|
||||
tokenInfo.Text,
|
||||
tokenInfo.StringValue,
|
||||
leadingTrivia,
|
||||
trailingTrivia);
|
||||
case TokenKind.DoubleQuotedStringLiteral:
|
||||
case TokenKind.DoubleQuotedStringLiteralToken:
|
||||
return TokenFactory.CreateTokenWithValueAndTrivia<string>(
|
||||
tokenInfo.Kind,
|
||||
tokenInfo.Text,
|
||||
@ -879,7 +879,7 @@ namespace Parser.Internal
|
||||
throw new ParsingException($"Unexpected character: '{Window.PeekChar()}' at {Window.Position}.");
|
||||
}
|
||||
result.Add(pair);
|
||||
if (token.Kind == TokenKind.EndOfFile)
|
||||
if (token.Kind == TokenKind.EndOfFileToken)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -41,27 +41,27 @@ namespace Parser.Internal
|
||||
|
||||
private SyntaxToken EatIdentifier()
|
||||
{
|
||||
return EatToken(TokenKind.Identifier);
|
||||
return EatToken(TokenKind.IdentifierToken);
|
||||
}
|
||||
|
||||
private bool IsIdentifier(SyntaxToken token, string s)
|
||||
{
|
||||
return token.Kind == TokenKind.Identifier && token.Text == s;
|
||||
return token.Kind == TokenKind.IdentifierToken && token.Text == s;
|
||||
}
|
||||
|
||||
private SyntaxToken EatIdentifier(string s)
|
||||
{
|
||||
var token = CurrentToken;
|
||||
if (token.Kind != TokenKind.Identifier)
|
||||
if (token.Kind != TokenKind.IdentifierToken)
|
||||
{
|
||||
Errors.Add($"Unexpected token \"{token.Text}\" instead of {TokenKind.Identifier} at {CurrentPosition}.");
|
||||
return TokenFactory.CreateMissing(TokenKind.Identifier, null, null);
|
||||
Errors.Add($"Unexpected token \"{token.Text}\" instead of {TokenKind.IdentifierToken} at {CurrentPosition}.");
|
||||
return TokenFactory.CreateMissing(TokenKind.IdentifierToken, null, null);
|
||||
}
|
||||
|
||||
if (token.Text != s)
|
||||
{
|
||||
Errors.Add($"Unexpected token \"{token.Text}\" instead of identifier {s} at {CurrentPosition}.");
|
||||
return TokenFactory.CreateMissing(TokenKind.Identifier, null, null);
|
||||
return TokenFactory.CreateMissing(TokenKind.IdentifierToken, null, null);
|
||||
}
|
||||
|
||||
_index++;
|
||||
@ -71,7 +71,7 @@ namespace Parser.Internal
|
||||
private SyntaxToken PossiblyEatIdentifier(string s)
|
||||
{
|
||||
var token = CurrentToken;
|
||||
if (token.Kind == TokenKind.Identifier && token.Text == s)
|
||||
if (token.Kind == TokenKind.IdentifierToken && token.Text == s)
|
||||
{
|
||||
return EatToken();
|
||||
}
|
||||
@ -82,13 +82,13 @@ namespace Parser.Internal
|
||||
private SyntaxToken EatPossiblyMissingIdentifier(string s)
|
||||
{
|
||||
var token = CurrentToken;
|
||||
if (token.Kind == TokenKind.Identifier && token.Text == s)
|
||||
if (token.Kind == TokenKind.IdentifierToken && token.Text == s)
|
||||
{
|
||||
return EatToken();
|
||||
}
|
||||
|
||||
return TokenFactory.CreateMissing(
|
||||
TokenKind.Identifier,
|
||||
TokenKind.IdentifierToken,
|
||||
new List<SyntaxTrivia>(),
|
||||
new List<SyntaxTrivia>());
|
||||
}
|
||||
@ -97,15 +97,15 @@ namespace Parser.Internal
|
||||
{
|
||||
var outputs = new SyntaxListBuilder();
|
||||
var firstToken = true;
|
||||
while (CurrentToken.Kind != TokenKind.ClosingSquareBracket)
|
||||
while (CurrentToken.Kind != TokenKind.CloseSquareBracketToken)
|
||||
{
|
||||
if (!firstToken && CurrentToken.Kind == TokenKind.Comma)
|
||||
if (!firstToken && CurrentToken.Kind == TokenKind.CommaToken)
|
||||
{
|
||||
outputs.Add(EatToken());
|
||||
}
|
||||
|
||||
firstToken = false;
|
||||
outputs.Add(Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier)));
|
||||
outputs.Add(Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken)));
|
||||
}
|
||||
|
||||
return outputs.ToList();
|
||||
@ -116,19 +116,19 @@ namespace Parser.Internal
|
||||
SyntaxToken assignmentSign;
|
||||
var builder = new SyntaxListBuilder();
|
||||
|
||||
if (CurrentToken.Kind == TokenKind.Identifier)
|
||||
if (CurrentToken.Kind == TokenKind.IdentifierToken)
|
||||
{
|
||||
if (PeekToken(1).Kind == TokenKind.Assignment)
|
||||
if (PeekToken(1).Kind == TokenKind.EqualsToken)
|
||||
{
|
||||
builder.Add(Factory.IdentifierNameSyntax(EatToken()));
|
||||
assignmentSign = EatToken(TokenKind.Assignment);
|
||||
assignmentSign = EatToken(TokenKind.EqualsToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
else if (CurrentToken.Kind == TokenKind.OpeningSquareBracket)
|
||||
else if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
var outputs = ParseFunctionOutputList();
|
||||
@ -137,8 +137,8 @@ namespace Parser.Internal
|
||||
builder.AddRange(outputs);
|
||||
}
|
||||
|
||||
builder.Add(EatToken(TokenKind.ClosingSquareBracket));
|
||||
assignmentSign = EatToken(TokenKind.Assignment);
|
||||
builder.Add(EatToken(TokenKind.CloseSquareBracketToken));
|
||||
assignmentSign = EatToken(TokenKind.EqualsToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -154,25 +154,25 @@ namespace Parser.Internal
|
||||
{
|
||||
var builder = new SyntaxListBuilder();
|
||||
var firstToken = true;
|
||||
while (CurrentToken.Kind != TokenKind.ClosingBracket)
|
||||
while (CurrentToken.Kind != TokenKind.CloseParenthesisToken)
|
||||
{
|
||||
if (!firstToken)
|
||||
{
|
||||
builder.Add(EatToken(TokenKind.Comma));
|
||||
builder.Add(EatToken(TokenKind.CommaToken));
|
||||
}
|
||||
else
|
||||
{
|
||||
firstToken = false;
|
||||
}
|
||||
|
||||
if (CurrentToken.Kind == TokenKind.Not)
|
||||
if (CurrentToken.Kind == TokenKind.TildeToken)
|
||||
{
|
||||
var notToken = EatToken();
|
||||
builder.Add(notToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
var identifierToken = EatToken(TokenKind.Identifier);
|
||||
var identifierToken = EatToken(TokenKind.IdentifierToken);
|
||||
builder.Add(Factory.IdentifierNameSyntax(identifierToken));
|
||||
}
|
||||
}
|
||||
@ -182,11 +182,11 @@ namespace Parser.Internal
|
||||
|
||||
private FunctionInputDescriptionSyntaxNode ParseFunctionInputDescription()
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
var openingBracket = EatToken(TokenKind.OpeningBracket);
|
||||
var openingBracket = EatToken(TokenKind.OpenParenthesisToken);
|
||||
var parameterList = ParseParameterList();
|
||||
var closingBracket = EatToken(TokenKind.ClosingBracket);
|
||||
var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
|
||||
return Factory.FunctionInputDescriptionSyntax(
|
||||
openingBracket,
|
||||
parameterList,
|
||||
@ -202,7 +202,7 @@ namespace Parser.Internal
|
||||
{
|
||||
var functionKeyword = EatIdentifier("function");
|
||||
var outputDescription = ParseFunctionOutputDescription();
|
||||
var name = EatToken(TokenKind.Identifier);
|
||||
var name = EatToken(TokenKind.IdentifierToken);
|
||||
var inputDescription = ParseFunctionInputDescription();
|
||||
var commas = ParseOptionalCommas();
|
||||
var body = ParseStatementList();
|
||||
@ -236,9 +236,9 @@ namespace Parser.Internal
|
||||
|
||||
private ArrayLiteralExpressionSyntaxNode ParseArrayLiteral()
|
||||
{
|
||||
var openingSquareBracket = EatToken(TokenKind.OpeningSquareBracket);
|
||||
var elements = ParseArrayElementList(TokenKind.ClosingSquareBracket);
|
||||
var closingSquareBracket = EatToken(TokenKind.ClosingSquareBracket);
|
||||
var openingSquareBracket = EatToken(TokenKind.OpenSquareBracketToken);
|
||||
var elements = ParseArrayElementList(TokenKind.CloseSquareBracketToken);
|
||||
var closingSquareBracket = EatToken(TokenKind.CloseSquareBracketToken);
|
||||
return Factory.ArrayLiteralExpressionSyntax(
|
||||
openingSquareBracket,
|
||||
elements,
|
||||
@ -247,9 +247,9 @@ namespace Parser.Internal
|
||||
|
||||
private CellArrayLiteralExpressionSyntaxNode ParseCellArrayLiteral()
|
||||
{
|
||||
var openingBrace = EatToken(TokenKind.OpeningBrace);
|
||||
var elements = ParseArrayElementList(TokenKind.ClosingBrace);
|
||||
var closingBrace = EatToken(TokenKind.ClosingBrace);
|
||||
var openingBrace = EatToken(TokenKind.OpenBraceToken);
|
||||
var elements = ParseArrayElementList(TokenKind.CloseBraceToken);
|
||||
var closingBrace = EatToken(TokenKind.CloseBraceToken);
|
||||
return Factory.CellArrayLiteralExpressionSyntax(
|
||||
openingBrace,
|
||||
elements,
|
||||
@ -265,8 +265,8 @@ namespace Parser.Internal
|
||||
{
|
||||
if (!firstToken)
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.Comma
|
||||
|| CurrentToken.Kind == TokenKind.Semicolon)
|
||||
if (CurrentToken.Kind == TokenKind.CommaToken
|
||||
|| CurrentToken.Kind == TokenKind.SemicolonToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
}
|
||||
@ -292,28 +292,28 @@ namespace Parser.Internal
|
||||
ExpressionSyntaxNode expression = null;
|
||||
switch (token.Kind)
|
||||
{
|
||||
case TokenKind.Identifier:
|
||||
case TokenKind.IdentifierToken:
|
||||
expression = Factory.IdentifierNameSyntax(EatToken());
|
||||
break;
|
||||
case TokenKind.NumberLiteral:
|
||||
case TokenKind.NumberLiteralToken:
|
||||
expression = Factory.NumberLiteralSyntax(EatToken());
|
||||
break;
|
||||
case TokenKind.StringLiteral:
|
||||
case TokenKind.StringLiteralToken:
|
||||
expression = Factory.StringLiteralSyntax(EatToken());
|
||||
break;
|
||||
case TokenKind.DoubleQuotedStringLiteral:
|
||||
case TokenKind.DoubleQuotedStringLiteralToken:
|
||||
expression = Factory.DoubleQuotedStringLiteralSyntax(EatToken());
|
||||
break;
|
||||
case TokenKind.OpeningSquareBracket:
|
||||
case TokenKind.OpenSquareBracketToken:
|
||||
expression = ParseArrayLiteral();
|
||||
break;
|
||||
case TokenKind.OpeningBrace:
|
||||
case TokenKind.OpenBraceToken:
|
||||
expression = ParseCellArrayLiteral();
|
||||
break;
|
||||
case TokenKind.Colon:
|
||||
case TokenKind.ColonToken:
|
||||
expression = Factory.EmptyExpressionSyntax();
|
||||
break;
|
||||
case TokenKind.OpeningBracket:
|
||||
case TokenKind.OpenParenthesisToken:
|
||||
expression = ParseParenthesizedExpression();
|
||||
break;
|
||||
}
|
||||
@ -332,14 +332,14 @@ namespace Parser.Internal
|
||||
var token = CurrentToken;
|
||||
switch (token.Kind)
|
||||
{
|
||||
case TokenKind.OpeningBrace: // cell array element access
|
||||
case TokenKind.OpenBraceToken: // cell array element access
|
||||
if (options.ParsingArrayElements && expression.TrailingTrivia.Any())
|
||||
{
|
||||
return expression;
|
||||
}
|
||||
var openingBrace = EatToken();
|
||||
var indices = ParseArrayElementList(TokenKind.ClosingBrace);
|
||||
var closingBrace = EatToken(TokenKind.ClosingBrace);
|
||||
var indices = ParseArrayElementList(TokenKind.CloseBraceToken);
|
||||
var closingBrace = EatToken(TokenKind.CloseBraceToken);
|
||||
expression = Factory.CellArrayElementAccessExpressionSyntax(
|
||||
expression,
|
||||
openingBrace,
|
||||
@ -347,21 +347,21 @@ namespace Parser.Internal
|
||||
closingBrace
|
||||
);
|
||||
break;
|
||||
case TokenKind.OpeningBracket: // function call
|
||||
case TokenKind.OpenParenthesisToken: // function call
|
||||
if (options.ParsingArrayElements && expression.TrailingTrivia.Any())
|
||||
{
|
||||
return expression;
|
||||
}
|
||||
var openingBracket = EatToken();
|
||||
var parameters = ParseFunctionCallParameterList();
|
||||
var closingBracket = EatToken(TokenKind.ClosingBracket);
|
||||
var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
|
||||
expression = Factory.FunctionCallExpressionSyntax(
|
||||
expression,
|
||||
openingBracket,
|
||||
parameters,
|
||||
closingBracket);
|
||||
break;
|
||||
case TokenKind.Dot: // member access
|
||||
case TokenKind.DotToken: // member access
|
||||
if (expression is IdentifierNameSyntaxNode
|
||||
|| expression is MemberAccessSyntaxNode
|
||||
|| expression is FunctionCallExpressionSyntaxNode
|
||||
@ -378,14 +378,14 @@ namespace Parser.Internal
|
||||
}
|
||||
|
||||
break;
|
||||
case TokenKind.Transpose:
|
||||
case TokenKind.DotTranspose:
|
||||
case TokenKind.ApostropheToken:
|
||||
case TokenKind.DotApostropheToken:
|
||||
var operation = EatToken();
|
||||
expression = Factory.UnaryPostixOperationExpressionSyntax(expression, operation);
|
||||
break;
|
||||
case TokenKind.UnquotedStringLiteral:
|
||||
case TokenKind.UnquotedStringLiteralToken:
|
||||
return ParseCommandExpression(expression);
|
||||
case TokenKind.At:
|
||||
case TokenKind.AtToken:
|
||||
if (expression.TrailingTrivia.Any())
|
||||
{
|
||||
return expression;
|
||||
@ -402,7 +402,7 @@ namespace Parser.Internal
|
||||
if (expression is IdentifierNameSyntaxNode idNameNode)
|
||||
{
|
||||
var builder = new SyntaxListBuilder<UnquotedStringLiteralSyntaxNode>();
|
||||
while (CurrentToken.Kind == TokenKind.UnquotedStringLiteral)
|
||||
while (CurrentToken.Kind == TokenKind.UnquotedStringLiteralToken)
|
||||
{
|
||||
builder.Add(Factory.UnquotedStringLiteralSyntax(EatToken()));
|
||||
}
|
||||
@ -433,15 +433,15 @@ namespace Parser.Internal
|
||||
|
||||
private ExpressionSyntaxNode ParseMemberAccess()
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.Identifier)
|
||||
if (CurrentToken.Kind == TokenKind.IdentifierToken)
|
||||
{
|
||||
return Factory.IdentifierNameSyntax(EatToken());
|
||||
}
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
var openingBracket = EatToken();
|
||||
var indirectMember = ParseExpression();
|
||||
var closingBracket = EatToken(TokenKind.ClosingBracket);
|
||||
var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
|
||||
return Factory.IndirectMemberAccessSyntax(
|
||||
openingBracket,
|
||||
indirectMember,
|
||||
@ -454,11 +454,11 @@ namespace Parser.Internal
|
||||
{
|
||||
var builder = new SyntaxListBuilder();
|
||||
var firstToken = true;
|
||||
while (CurrentToken.Kind != TokenKind.ClosingBracket)
|
||||
while (CurrentToken.Kind != TokenKind.CloseParenthesisToken)
|
||||
{
|
||||
if (!firstToken)
|
||||
{
|
||||
builder.Add(EatToken(TokenKind.Comma));
|
||||
builder.Add(EatToken(TokenKind.CommaToken));
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -473,9 +473,9 @@ namespace Parser.Internal
|
||||
|
||||
private ParenthesizedExpressionSyntaxNode ParseParenthesizedExpression()
|
||||
{
|
||||
var openParen = EatToken(TokenKind.OpeningBracket);
|
||||
var openParen = EatToken(TokenKind.OpenParenthesisToken);
|
||||
var expression = ParseExpression();
|
||||
var closeParen = EatToken(TokenKind.ClosingBracket);
|
||||
var closeParen = EatToken(TokenKind.CloseParenthesisToken);
|
||||
return Factory.ParenthesizedExpressionSyntax(
|
||||
openParen,
|
||||
expression,
|
||||
@ -484,16 +484,16 @@ namespace Parser.Internal
|
||||
|
||||
private CompoundNameSyntaxNode ParseCompoundName()
|
||||
{
|
||||
var lastToken = EatToken(TokenKind.Identifier);
|
||||
var lastToken = EatToken(TokenKind.IdentifierToken);
|
||||
var firstName = lastToken;
|
||||
var builder = new SyntaxListBuilder();
|
||||
builder.Add(firstName);
|
||||
while (CurrentToken.Kind == TokenKind.Dot
|
||||
while (CurrentToken.Kind == TokenKind.DotToken
|
||||
&& !lastToken.TrailingTrivia.Any())
|
||||
{
|
||||
var dot = EatToken();
|
||||
builder.Add(dot);
|
||||
lastToken = EatToken(TokenKind.Identifier);
|
||||
lastToken = EatToken(TokenKind.IdentifierToken);
|
||||
builder.Add(lastToken);
|
||||
}
|
||||
|
||||
@ -503,14 +503,14 @@ namespace Parser.Internal
|
||||
private FunctionHandleSyntaxNode ParseFunctionHandle()
|
||||
{
|
||||
var atSign = EatToken();
|
||||
if (CurrentToken.Kind == TokenKind.Identifier)
|
||||
if (CurrentToken.Kind == TokenKind.IdentifierToken)
|
||||
{
|
||||
var compoundName = ParseCompoundName();
|
||||
return Factory.NamedFunctionHandleSyntax(
|
||||
atSign,
|
||||
compoundName);
|
||||
}
|
||||
else if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
else if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
var inputs = ParseFunctionInputDescription();
|
||||
var body = ParseExpression();
|
||||
@ -532,7 +532,7 @@ namespace Parser.Internal
|
||||
var operand = ParseSubExpression(options, newPrecedence);
|
||||
if (operand == null)
|
||||
{
|
||||
if (options.ParsingArrayElements && operation.Kind == TokenKind.Not)
|
||||
if (options.ParsingArrayElements && operation.Kind == TokenKind.TildeToken)
|
||||
{
|
||||
operand = Factory.EmptyExpressionSyntax();
|
||||
}
|
||||
@ -543,7 +543,7 @@ namespace Parser.Internal
|
||||
}
|
||||
lhs = Factory.UnaryPrefixOperationExpressionSyntax(operation, operand);
|
||||
}
|
||||
else if (CurrentToken.Kind == TokenKind.At)
|
||||
else if (CurrentToken.Kind == TokenKind.AtToken)
|
||||
{
|
||||
return ParseFunctionHandle();
|
||||
}
|
||||
@ -570,11 +570,11 @@ namespace Parser.Internal
|
||||
|
||||
EatToken();
|
||||
var rhs = ParseSubExpression(options, newPrecedence);
|
||||
if (rhs == null && token.Kind == TokenKind.Colon) // for parsing things like a{:}
|
||||
if (rhs == null && token.Kind == TokenKind.ColonToken) // for parsing things like a{:}
|
||||
{
|
||||
rhs = Factory.EmptyExpressionSyntax();
|
||||
}
|
||||
if (token.Kind == TokenKind.Assignment)
|
||||
if (token.Kind == TokenKind.EqualsToken)
|
||||
{
|
||||
lhs = Factory.AssignmentExpressionSyntax(lhs, token, rhs);
|
||||
}
|
||||
@ -595,7 +595,7 @@ namespace Parser.Internal
|
||||
private SyntaxList<SyntaxToken> ParseOptionalCommas()
|
||||
{
|
||||
var builder = new SyntaxListBuilder<SyntaxToken>();
|
||||
while (CurrentToken.Kind == TokenKind.Comma)
|
||||
while (CurrentToken.Kind == TokenKind.CommaToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
}
|
||||
@ -606,8 +606,8 @@ namespace Parser.Internal
|
||||
private SyntaxList<SyntaxToken> ParseOptionalSemicolonsOrCommas()
|
||||
{
|
||||
var builder = new SyntaxListBuilder<SyntaxToken>();
|
||||
while (CurrentToken.Kind == TokenKind.Comma
|
||||
|| CurrentToken.Kind == TokenKind.Semicolon)
|
||||
while (CurrentToken.Kind == TokenKind.CommaToken
|
||||
|| CurrentToken.Kind == TokenKind.SemicolonToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
}
|
||||
@ -763,7 +763,7 @@ namespace Parser.Internal
|
||||
|
||||
private AttributeAssignmentSyntaxNode ParseAttributeAssignment()
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.Assignment)
|
||||
if (CurrentToken.Kind == TokenKind.EqualsToken)
|
||||
{
|
||||
var assignmentSign = EatToken();
|
||||
var value = ParseExpression();
|
||||
@ -775,7 +775,7 @@ namespace Parser.Internal
|
||||
|
||||
private AttributeSyntaxNode ParseAttribute()
|
||||
{
|
||||
var name = Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier));
|
||||
var name = Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken));
|
||||
var assignment = ParseAttributeAssignment();
|
||||
return Factory.AttributeSyntax(name, assignment);
|
||||
}
|
||||
@ -785,11 +785,11 @@ namespace Parser.Internal
|
||||
var openingBracket = EatToken();
|
||||
var first = true;
|
||||
var builder = new SyntaxListBuilder();
|
||||
while (CurrentToken.Kind != TokenKind.ClosingBracket)
|
||||
while (CurrentToken.Kind != TokenKind.CloseParenthesisToken)
|
||||
{
|
||||
if (!first)
|
||||
{
|
||||
var comma = EatToken(TokenKind.Comma);
|
||||
var comma = EatToken(TokenKind.CommaToken);
|
||||
builder.Add(comma);
|
||||
}
|
||||
|
||||
@ -816,13 +816,13 @@ namespace Parser.Internal
|
||||
return ParseMethodDefinition();
|
||||
}
|
||||
|
||||
if (CurrentToken.Kind == TokenKind.OpeningSquareBracket
|
||||
|| CurrentToken.Kind == TokenKind.Identifier)
|
||||
if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken
|
||||
|| CurrentToken.Kind == TokenKind.IdentifierToken)
|
||||
{
|
||||
return ParseAbstractMethodDeclaration();
|
||||
}
|
||||
|
||||
if (CurrentToken.Kind == TokenKind.Semicolon)
|
||||
if (CurrentToken.Kind == TokenKind.SemicolonToken)
|
||||
{
|
||||
return Factory.EmptyStatementSyntax(EatToken());
|
||||
}
|
||||
@ -852,7 +852,7 @@ namespace Parser.Internal
|
||||
{
|
||||
var methodsKeyword = EatToken();
|
||||
AttributeListSyntaxNode attributes = null;
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
attributes = ParseAttributesList();
|
||||
}
|
||||
@ -870,7 +870,7 @@ namespace Parser.Internal
|
||||
|
||||
private GreenNode ParsePropertyDeclaration()
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.Comma)
|
||||
if (CurrentToken.Kind == TokenKind.CommaToken)
|
||||
{
|
||||
return EatToken();
|
||||
}
|
||||
@ -886,7 +886,7 @@ namespace Parser.Internal
|
||||
{
|
||||
var propertiesKeyword = EatToken();
|
||||
AttributeListSyntaxNode attributes = null;
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
attributes = ParseAttributesList();
|
||||
}
|
||||
@ -903,18 +903,18 @@ namespace Parser.Internal
|
||||
|
||||
private EnumerationItemValueSyntaxNode ParseEnumerationValue()
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
var builder = new SyntaxListBuilder();
|
||||
var openingBracket = EatToken(TokenKind.OpeningBracket);
|
||||
var openingBracket = EatToken(TokenKind.OpenParenthesisToken);
|
||||
var expression = ParseExpression() ?? Factory.EmptyExpressionSyntax();
|
||||
builder.Add(expression);
|
||||
while (CurrentToken.Kind == TokenKind.Comma)
|
||||
while (CurrentToken.Kind == TokenKind.CommaToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
builder.Add(ParseExpression());
|
||||
}
|
||||
var closingBracket = EatToken(TokenKind.ClosingBracket);
|
||||
var closingBracket = EatToken(TokenKind.CloseParenthesisToken);
|
||||
return Factory.EnumerationItemValueSyntax(openingBracket, builder.ToList(), closingBracket);
|
||||
}
|
||||
return null;
|
||||
@ -933,7 +933,7 @@ namespace Parser.Internal
|
||||
var enumerationKeyword = EatToken();
|
||||
var builder = new SyntaxListBuilder<EnumerationItemSyntaxNode>();
|
||||
AttributeListSyntaxNode attributes = null;
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
attributes = ParseAttributesList();
|
||||
}
|
||||
@ -955,7 +955,7 @@ namespace Parser.Internal
|
||||
{
|
||||
var eventsKeyword = EatToken();
|
||||
AttributeListSyntaxNode attributes = null;
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
attributes = ParseAttributesList();
|
||||
}
|
||||
@ -974,7 +974,7 @@ namespace Parser.Internal
|
||||
{
|
||||
var builder = new SyntaxListBuilder();
|
||||
builder.Add(ParseCompoundName());
|
||||
while (CurrentToken.Kind == TokenKind.BitwiseAnd)
|
||||
while (CurrentToken.Kind == TokenKind.AmpersandToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
builder.Add(ParseCompoundName());
|
||||
@ -994,13 +994,13 @@ namespace Parser.Internal
|
||||
{
|
||||
var classdefKeyword = EatToken();
|
||||
AttributeListSyntaxNode attributes = null;
|
||||
if (CurrentToken.Kind == TokenKind.OpeningBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenParenthesisToken)
|
||||
{
|
||||
attributes = ParseAttributesList();
|
||||
}
|
||||
var className = Factory.IdentifierNameSyntax(EatToken(TokenKind.Identifier));
|
||||
var className = Factory.IdentifierNameSyntax(EatToken(TokenKind.IdentifierToken));
|
||||
BaseClassListSyntaxNode baseClassList = null;
|
||||
if (CurrentToken.Kind == TokenKind.Less)
|
||||
if (CurrentToken.Kind == TokenKind.LessToken)
|
||||
{
|
||||
baseClassList = ParseBaseClassList();
|
||||
}
|
||||
@ -1046,7 +1046,7 @@ namespace Parser.Internal
|
||||
|
||||
private StatementSyntaxNode ParseStatement()
|
||||
{
|
||||
if (CurrentToken.Kind == TokenKind.Identifier)
|
||||
if (CurrentToken.Kind == TokenKind.IdentifierToken)
|
||||
{
|
||||
switch (CurrentToken.Text)
|
||||
{
|
||||
@ -1075,12 +1075,12 @@ namespace Parser.Internal
|
||||
}
|
||||
}
|
||||
|
||||
if (CurrentToken.Kind == TokenKind.OpeningSquareBracket)
|
||||
if (CurrentToken.Kind == TokenKind.OpenSquareBracketToken)
|
||||
{
|
||||
return ParseExpressionStatement();
|
||||
}
|
||||
|
||||
if (CurrentToken.Kind == TokenKind.Semicolon)
|
||||
if (CurrentToken.Kind == TokenKind.SemicolonToken)
|
||||
{
|
||||
return Factory.EmptyStatementSyntax(EatToken());
|
||||
}
|
||||
@ -1090,7 +1090,7 @@ namespace Parser.Internal
|
||||
private SyntaxList ParseStatementList()
|
||||
{
|
||||
var builder = new SyntaxListBuilder();
|
||||
while (CurrentToken.Kind != TokenKind.EndOfFile)
|
||||
while (CurrentToken.Kind != TokenKind.EndOfFileToken)
|
||||
{
|
||||
var node = ParseStatement();
|
||||
if (node == null)
|
||||
@ -1098,8 +1098,8 @@ namespace Parser.Internal
|
||||
break;
|
||||
}
|
||||
builder.Add(node);
|
||||
while (CurrentToken.Kind == TokenKind.Comma
|
||||
|| CurrentToken.Kind == TokenKind.Semicolon)
|
||||
while (CurrentToken.Kind == TokenKind.CommaToken
|
||||
|| CurrentToken.Kind == TokenKind.SemicolonToken)
|
||||
{
|
||||
builder.Add(EatToken());
|
||||
}
|
||||
|
||||
@ -38,41 +38,41 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case TokenKind.Assignment:
|
||||
case TokenKind.EqualsToken:
|
||||
return Precedence.Assignment;
|
||||
case TokenKind.LogicalOr:
|
||||
case TokenKind.PipePipeToken:
|
||||
return Precedence.LogicalOr;
|
||||
case TokenKind.LogicalAnd:
|
||||
case TokenKind.AmpersandAmpersandToken:
|
||||
return Precedence.LogicalAnd;
|
||||
case TokenKind.BitwiseOr:
|
||||
case TokenKind.PipeToken:
|
||||
return Precedence.BitwiseOr;
|
||||
case TokenKind.BitwiseAnd:
|
||||
case TokenKind.AmpersandToken:
|
||||
return Precedence.BitwiseAnd;
|
||||
case TokenKind.Less:
|
||||
case TokenKind.LessOrEqual:
|
||||
case TokenKind.Greater:
|
||||
case TokenKind.GreaterOrEqual:
|
||||
case TokenKind.Equality:
|
||||
case TokenKind.Inequality:
|
||||
case TokenKind.LessToken:
|
||||
case TokenKind.LessOrEqualsToken:
|
||||
case TokenKind.GreaterToken:
|
||||
case TokenKind.GreaterOrEqualsToken:
|
||||
case TokenKind.EqualsEqualsToken:
|
||||
case TokenKind.TildeEqualsToken:
|
||||
return Precedence.Relational;
|
||||
case TokenKind.Colon:
|
||||
case TokenKind.ColonToken:
|
||||
return Precedence.Colon;
|
||||
case TokenKind.Plus:
|
||||
case TokenKind.Minus:
|
||||
case TokenKind.PlusToken:
|
||||
case TokenKind.MinusToken:
|
||||
return Precedence.Additive;
|
||||
case TokenKind.Multiply:
|
||||
case TokenKind.DotMultiply:
|
||||
case TokenKind.Divide:
|
||||
case TokenKind.DotDivide:
|
||||
case TokenKind.Backslash:
|
||||
case TokenKind.DotBackslash:
|
||||
case TokenKind.StarToken:
|
||||
case TokenKind.DotStarToken:
|
||||
case TokenKind.SlashToken:
|
||||
case TokenKind.DotSlashToken:
|
||||
case TokenKind.BackslashToken:
|
||||
case TokenKind.DotBackslashToken:
|
||||
return Precedence.Multiplicative;
|
||||
case TokenKind.Not:
|
||||
case TokenKind.TildeToken:
|
||||
return Precedence.Unary;
|
||||
case TokenKind.Power:
|
||||
case TokenKind.DotPower:
|
||||
case TokenKind.Transpose:
|
||||
case TokenKind.DotTranspose:
|
||||
case TokenKind.CaretToken:
|
||||
case TokenKind.DotCaretToken:
|
||||
case TokenKind.ApostropheToken:
|
||||
case TokenKind.DotApostropheToken:
|
||||
return Precedence.Power;
|
||||
default:
|
||||
return Precedence.Expression;
|
||||
@ -113,9 +113,9 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (tokenKind)
|
||||
{
|
||||
case TokenKind.OpeningBrace:
|
||||
case TokenKind.OpeningBracket:
|
||||
case TokenKind.OpeningSquareBracket:
|
||||
case TokenKind.OpenBraceToken:
|
||||
case TokenKind.OpenParenthesisToken:
|
||||
case TokenKind.OpenSquareBracketToken:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
@ -126,9 +126,9 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (tokenKind)
|
||||
{
|
||||
case TokenKind.ClosingBrace:
|
||||
case TokenKind.ClosingBracket:
|
||||
case TokenKind.ClosingSquareBracket:
|
||||
case TokenKind.CloseBraceToken:
|
||||
case TokenKind.CloseParenthesisToken:
|
||||
case TokenKind.CloseSquareBracketToken:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
@ -144,12 +144,12 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (tokenKind)
|
||||
{
|
||||
case TokenKind.ClosingBrace:
|
||||
return TokenKind.OpeningBrace;
|
||||
case TokenKind.ClosingBracket:
|
||||
return TokenKind.OpeningBracket;
|
||||
case TokenKind.ClosingSquareBracket:
|
||||
return TokenKind.OpeningSquareBracket;
|
||||
case TokenKind.CloseBraceToken:
|
||||
return TokenKind.OpenBraceToken;
|
||||
case TokenKind.CloseParenthesisToken:
|
||||
return TokenKind.OpenParenthesisToken;
|
||||
case TokenKind.CloseSquareBracketToken:
|
||||
return TokenKind.OpenSquareBracketToken;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
@ -224,10 +224,10 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case TokenKind.Plus:
|
||||
case TokenKind.Minus:
|
||||
case TokenKind.Not:
|
||||
case TokenKind.QuestionMark:
|
||||
case TokenKind.PlusToken:
|
||||
case TokenKind.MinusToken:
|
||||
case TokenKind.TildeToken:
|
||||
case TokenKind.QuestionToken:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
@ -252,29 +252,29 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case TokenKind.Assignment:
|
||||
case TokenKind.LogicalOr:
|
||||
case TokenKind.LogicalAnd:
|
||||
case TokenKind.BitwiseOr:
|
||||
case TokenKind.BitwiseAnd:
|
||||
case TokenKind.Less:
|
||||
case TokenKind.LessOrEqual:
|
||||
case TokenKind.Greater:
|
||||
case TokenKind.GreaterOrEqual:
|
||||
case TokenKind.Equality:
|
||||
case TokenKind.Inequality:
|
||||
case TokenKind.Colon:
|
||||
case TokenKind.Plus:
|
||||
case TokenKind.Minus:
|
||||
case TokenKind.Multiply:
|
||||
case TokenKind.DotMultiply:
|
||||
case TokenKind.Divide:
|
||||
case TokenKind.DotDivide:
|
||||
case TokenKind.Backslash:
|
||||
case TokenKind.DotBackslash:
|
||||
case TokenKind.Not:
|
||||
case TokenKind.Power:
|
||||
case TokenKind.DotPower:
|
||||
case TokenKind.EqualsToken:
|
||||
case TokenKind.PipePipeToken:
|
||||
case TokenKind.AmpersandAmpersandToken:
|
||||
case TokenKind.PipeToken:
|
||||
case TokenKind.AmpersandToken:
|
||||
case TokenKind.LessToken:
|
||||
case TokenKind.LessOrEqualsToken:
|
||||
case TokenKind.GreaterToken:
|
||||
case TokenKind.GreaterOrEqualsToken:
|
||||
case TokenKind.EqualsEqualsToken:
|
||||
case TokenKind.TildeEqualsToken:
|
||||
case TokenKind.ColonToken:
|
||||
case TokenKind.PlusToken:
|
||||
case TokenKind.MinusToken:
|
||||
case TokenKind.StarToken:
|
||||
case TokenKind.DotStarToken:
|
||||
case TokenKind.SlashToken:
|
||||
case TokenKind.DotSlashToken:
|
||||
case TokenKind.BackslashToken:
|
||||
case TokenKind.DotBackslashToken:
|
||||
case TokenKind.TildeToken:
|
||||
case TokenKind.CaretToken:
|
||||
case TokenKind.DotCaretToken:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
@ -290,13 +290,13 @@ namespace Parser.Internal
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case TokenKind.Plus:
|
||||
case TokenKind.PlusToken:
|
||||
return TokenKind.UnaryPlus;
|
||||
case TokenKind.Minus:
|
||||
case TokenKind.MinusToken:
|
||||
return TokenKind.UnaryMinus;
|
||||
case TokenKind.Not:
|
||||
case TokenKind.TildeToken:
|
||||
return TokenKind.UnaryNot;
|
||||
case TokenKind.QuestionMark:
|
||||
case TokenKind.QuestionToken:
|
||||
return TokenKind.UnaryQuestionMark;
|
||||
default:
|
||||
throw new ArgumentException(nameof(kind));
|
||||
|
||||
@ -124,7 +124,7 @@ namespace Parser.Internal
|
||||
|
||||
public SyntaxIdentifier(
|
||||
string text
|
||||
) : base(TokenKind.Identifier)
|
||||
) : base(TokenKind.IdentifierToken)
|
||||
{
|
||||
_text = text;
|
||||
}
|
||||
|
||||
@ -42,7 +42,7 @@ namespace Parser.Internal
|
||||
IReadOnlyList<SyntaxTrivia> trailingTrivia)
|
||||
{
|
||||
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>(
|
||||
TokenKind.UnquotedStringLiteral,
|
||||
TokenKind.UnquotedStringLiteralToken,
|
||||
text,
|
||||
value,
|
||||
leadingTrivia,
|
||||
|
||||
@ -8,114 +8,115 @@
|
||||
None = 0,
|
||||
BadToken = 1,
|
||||
// The lexer puts a virtual "end of file" token at the end of the parsed file.
|
||||
EndOfFile = 2,
|
||||
EndOfFileToken = 2,
|
||||
// Identifier: could be a reserved word, a variable name, a class name, etc.
|
||||
Identifier = 3,
|
||||
IdentifierToken = 3,
|
||||
// Number literal: 123, 45.678, 2e-5, etc.
|
||||
NumberLiteral = 4,
|
||||
NumberLiteralToken = 4,
|
||||
// String literal: 'abc', '123', etc. The "usual" string literals are single-quoted and are just char arrays.
|
||||
StringLiteral = 5,
|
||||
StringLiteralToken = 5,
|
||||
// Double-quoted string literal: "abc", "123", etc. These are the "new" string literal that are more like strings
|
||||
// and less like char arrays (for example, char arrays could be columns instead of rows, or even multi-dimensional).
|
||||
DoubleQuotedStringLiteral = 6,
|
||||
DoubleQuotedStringLiteralToken = 6,
|
||||
// This is for supporting "command statements" like
|
||||
// > cd some/+folder/
|
||||
// In this example, "some/folder" should be treated as a string literal (for example, "+' there should be a part
|
||||
// of it, and not parsed as a binary operator).
|
||||
UnquotedStringLiteral = 7,
|
||||
UnquotedStringLiteralToken = 7,
|
||||
|
||||
// trivia
|
||||
|
||||
// Spaces, tabs, etc.
|
||||
Whitespace = 10,
|
||||
WhitespaceToken = 10,
|
||||
// New line characters.
|
||||
Newline = 11,
|
||||
NewlineToken = 11,
|
||||
// There are three types of comments:
|
||||
// * comments starting with % and lasting till the end of the line;
|
||||
// * comments starting with %{ and ending with %}, possibly spanning several lines;
|
||||
// * comments starting with ... and lasting till the end of the line.
|
||||
// At the moment, this token is used to accomodate all of them, so the other two
|
||||
// (MultilineComment = 13 and DotDotDot = 56 are not used).
|
||||
Comment = 12,
|
||||
CommentToken = 12,
|
||||
// Comments starting with %{ and ending with %}, possibly spanning several lines. Not used, see Comment = 12.
|
||||
MultilineComment = 13,
|
||||
MultilineCommentToken = 13,
|
||||
|
||||
// operators
|
||||
|
||||
// =
|
||||
Assignment = 20,
|
||||
EqualsToken = 20,
|
||||
// ==
|
||||
Equality = 21,
|
||||
EqualsEqualsToken = 21,
|
||||
// ~=
|
||||
Inequality = 22,
|
||||
TildeEqualsToken = 22,
|
||||
// &&
|
||||
LogicalAnd = 23,
|
||||
AmpersandAmpersandToken = 23,
|
||||
// ||
|
||||
LogicalOr = 24,
|
||||
PipePipeToken = 24,
|
||||
// &
|
||||
BitwiseAnd = 25,
|
||||
AmpersandToken = 25,
|
||||
// |
|
||||
BitwiseOr = 26,
|
||||
PipeToken = 26,
|
||||
// <
|
||||
Less = 27,
|
||||
LessToken = 27,
|
||||
// <=
|
||||
LessOrEqual = 28,
|
||||
LessOrEqualsToken = 28,
|
||||
// >
|
||||
Greater = 29,
|
||||
GreaterToken = 29,
|
||||
// >=
|
||||
GreaterOrEqual = 30,
|
||||
GreaterOrEqualsToken = 30,
|
||||
// ~
|
||||
Not = 31,
|
||||
TildeToken = 31,
|
||||
// +
|
||||
Plus = 32,
|
||||
PlusToken = 32,
|
||||
// -
|
||||
Minus = 33,
|
||||
MinusToken = 33,
|
||||
// *
|
||||
Multiply = 34,
|
||||
StarToken = 34,
|
||||
// /
|
||||
Divide = 35,
|
||||
SlashToken = 35,
|
||||
// ^
|
||||
Power = 36,
|
||||
CaretToken = 36,
|
||||
// \
|
||||
Backslash = 37,
|
||||
BackslashToken = 37,
|
||||
// ' (this is the same as starting string literal; we'll have some fun distinguishing those).
|
||||
Transpose = 38,
|
||||
ApostropheToken = 38,
|
||||
// .*
|
||||
DotMultiply = 39,
|
||||
DotStarToken = 39,
|
||||
// ./
|
||||
DotDivide = 40,
|
||||
DotSlashToken = 40,
|
||||
// .^
|
||||
DotPower = 41,
|
||||
DotCaretToken = 41,
|
||||
// .\
|
||||
DotBackslash = 42,
|
||||
DotBackslashToken = 42,
|
||||
// .'
|
||||
DotTranspose = 43,
|
||||
DotApostropheToken = 43,
|
||||
// @
|
||||
At = 44,
|
||||
AtToken = 44,
|
||||
// :
|
||||
Colon = 45,
|
||||
ColonToken = 45,
|
||||
// ?
|
||||
QuestionMark = 46,
|
||||
QuestionToken = 46,
|
||||
// ,
|
||||
Comma = 47,
|
||||
CommaToken = 47,
|
||||
// ;
|
||||
Semicolon = 48,
|
||||
SemicolonToken = 48,
|
||||
// {
|
||||
OpeningBrace = 49,
|
||||
OpenBraceToken = 49,
|
||||
// }
|
||||
ClosingBrace = 50,
|
||||
CloseBraceToken = 50,
|
||||
// [
|
||||
OpeningSquareBracket = 51,
|
||||
OpenSquareBracketToken = 51,
|
||||
// ]
|
||||
ClosingSquareBracket = 52,
|
||||
CloseSquareBracketToken = 52,
|
||||
// (
|
||||
OpeningBracket = 53,
|
||||
OpenParenthesisToken = 53,
|
||||
// )
|
||||
ClosingBracket = 54,
|
||||
CloseParenthesisToken = 54,
|
||||
// .
|
||||
Dot = 55,
|
||||
DotToken = 55,
|
||||
// Comments starting with ... and lasting till the end of the line. Not used, see Comment = 12.
|
||||
DotDotDot = 56,
|
||||
DotDotDotToken = 56,
|
||||
|
||||
// Unary tokens are not recognized during lexing; they are contextually recognized while parsing.
|
||||
UnaryPlus = 57,
|
||||
UnaryMinus = 58,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user