MParser/Parser/TokenKind.cs

namespace Parser
{
    // We use the same set of kinds for syntax tokens & syntax nodes.
    public enum TokenKind
    {
        // *****************
        // * SYNTAX TOKENS *
        // *****************

        None = 0,
        BadToken = 1,
        // The lexer puts a virtual "end of file" token at the end of the parsed file.
        EndOfFileToken = 2,
        // Identifier: could be a reserved word, a variable name, a class name, etc.
        IdentifierToken = 3,
        // Number literal: 123, 45.678, 2e-5, etc.
        NumberLiteralToken = 4,
        // String literal: 'abc', '123', etc. The "usual" string literals are single-quoted and are just char arrays.
        StringLiteralToken = 5,
        // Double-quoted string literal: "abc", "123", etc. These are the "new" string literal that are more like strings
        // and less like char arrays (for example, char arrays could be columns instead of rows, or even multi-dimensional).
        DoubleQuotedStringLiteralToken = 6,
        // This is for supporting "command statements" like
        // > cd some/+folder/
        // In this example, "some/folder" should be treated as a string literal (for example, "+' there should be a part
        // of it, and not parsed as a binary operator).
        UnquotedStringLiteralToken = 7,
        
        // trivia
        
        // Spaces, tabs, etc.
        WhitespaceToken = 10,
        // New line characters.
        NewlineToken = 11,
        // There are three types of comments:
        // * comments starting with % and lasting till the end of the line;
        // * comments starting with %{ and ending with %}, possibly spanning several lines;
        // * comments starting with ... and lasting till the end of the line.
        // At the moment, this token is used to accomodate all of them, so the other two
        // (MultilineComment = 13 and DotDotDot = 56 are not used).
        CommentToken = 12,
        // Comments starting with %{ and ending with %}, possibly spanning several lines. Not used, see Comment = 12.
        MultilineCommentToken = 13,
        
        // operators

        // =
        EqualsToken = 20,
        // ==
        EqualsEqualsToken = 21,
        // ~=
        TildeEqualsToken = 22,
        // &&
        AmpersandAmpersandToken = 23,
        // ||
        PipePipeToken = 24,
        // &
        AmpersandToken = 25,
        // |
        PipeToken = 26,
        // <
        LessToken = 27,
        // <=
        LessOrEqualsToken = 28,
        // >
        GreaterToken = 29,
        // >=
        GreaterOrEqualsToken = 30,
        // ~
        TildeToken = 31,
        // +
        PlusToken = 32,
        // -
        MinusToken = 33,
        // *
        StarToken = 34,
        // /
        SlashToken = 35,
        // ^
        CaretToken = 36,
        // \
        BackslashToken = 37,
        // ' (this is the same as starting string literal; we'll have some fun distinguishing those).
        ApostropheToken = 38,
        // .*
        DotStarToken = 39,
        // ./
        DotSlashToken = 40,
        // .^
        DotCaretToken = 41,
        // .\
        DotBackslashToken = 42,
        // .'
        DotApostropheToken = 43,
        // @
        AtToken = 44,
        // :
        ColonToken = 45,
        // ?
        QuestionToken = 46,
        // ,
        CommaToken = 47,
        // ;
        SemicolonToken = 48,
        // {
        OpenBraceToken = 49,
        // }
        CloseBraceToken = 50,
        // [
        OpenSquareBracketToken = 51,
        // ]
        CloseSquareBracketToken = 52,
        // (
        OpenParenthesisToken = 53,
        // )
        CloseParenthesisToken = 54,
        // .
        DotToken = 55,
        // Comments starting with ... and lasting till the end of the line.  Not used, see Comment = 12.
        DotDotDotToken = 56,

        // Unary tokens are not recognized during lexing; they are contextually recognized while parsing.
        UnaryPlus = 57,
        UnaryMinus = 58,
        UnaryNot = 59,
        UnaryQuestionMark = 60,

        // ****************
        // * SYNTAX NODES *
        // ****************

        // The whole file.
        File = 100,

        // a list of syntax nodes and/or tokens.
        List,

        // a semicolon that marks expression statements with discarded results.
        TrailingSemicolon,


        // STATEMENTS
        // The name ends with "Declaration" or "Statement".

        // result = abstractMethod(object)
        AbstractMethodDeclaration,

        // statement1
        // statement2;
        BlockStatement,

        // classdef MyClass < BaseClass, AnotherBaseClass
        //     properties
        //         y
        //     end
        //     methods
        //         <...>
        //     end
        // end
        ClassDeclaration,

        // function result = method(obj)
        //     <...>
        // end
        ConcreteMethodDeclaration,

        // 
        EmptyStatement,

        // a = 5;
        ExpressionStatement,

        // for a = 1:5
        //     process(a);
        // end
        ForStatement,

        // [output1, output2] = function(input1, input2)
        // <...>
        // end
        FunctionDeclaration,

        // if a < 5
        //     doSomething();
        // elseif a > 10
        //     doSomethingElse();
        // else
        //     GiveUp();
        // end
        IfStatement,

        // switch a
        //     case 1
        // <...>
        // end
        SwitchStatement,

        // try
        //     somethingWeird();
        // catch e
        //     dealWithIt(e);
        // end
        TryCatchStatement,

        // while a < 10
        //     doSomething();
        // end
        WhileStatement,


        // EXPRESSIONS
        // The name ends with "Expression".

        // [1, 2; 3 4]
        ArrayLiteralExpression,

        // a = 1:5
        AssignmentExpression,

        // +
        BinaryOperationExpression,

        // abc{2}
        CellArrayElementAccessExpression,

        // {1, 3, 'abc'}
        CellArrayLiteralExpression,

        // method@SuperClass(object)
        ClassInvokationExpression,

        // cd some/+folder/
        CommandExpression,

        // some.complex.name
        CompoundNameExpression,

        // "abc"
        DoubleQuotedStringLiteralExpression,

        // 
        EmptyExpression,

        // doSomething(5)
        FunctionCallExpression,

        // a
        IdentifierNameExpression,

        // struct.(field)
        IndirectMemberAccessExpression,

        // @(x) x + 1
        LambdaExpression,

        // object.member
        MemberAccessExpression,

        // @func
        NamedFunctionHandleExpression,

        // 123
        NumberLiteralExpression,

        // (1 + 2 * 3)
        ParenthesizedExpression,

        // 'abc'
        StringLiteralExpression,

        // -13
        UnaryPrefixOperationExpression,

        // [1 2 3]'
        UnaryPostfixOperationExpression,

        // abc
        UnquotedStringLiteralExpression,


        // PARTS OF STATEMENTS & EXPRESSIONS

        // (input1, input2)
        FunctionInputDescription,
        // [output1, output2] =
        FunctionOutputDescription,

        // case 1
        //     doSomething();
        SwitchCase,

        // elseif a > 10
        //     doSomethingElse();
        ElseifClause,
        // else
        //     GiveUp();
        ElseClause,
        // catch e
        //     dealWithIt(e);
        // end
        CatchClause,

        // = true
        AttributeAssignment,
        // Sealed = true
        Attribute,
        // (Sealed = true)
        AttributeList,

        // methods
        //     function result = method(obj)
        //     <...>
        //     end
        // end
        MethodsList,
        // properties
        //     x
        //     y
        // end
        PropertiesList,
        // < BaseClass, AnotherBaseClass
        BaseClassList,

        // (1)
        EnumerationItemValue,
        // One (1)
        EnumerationItem,
        // enumeration
        //     One (1)
        //     Two (2)
        // end
        EnumerationList,
        // events
        //     ToggleSomething
        // end
        EventsList,
        EndKeyword,
        Root
    }
}