mahalex/MParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge parser & lexer

Browse Source
This commit is contained in:
Alexander Luzgarev 2018-10-05 19:14:43 +02:00
parent 51a0cc863f
commit fb4cb901f2
71 changed files with 10265 additions and 3649 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ConsoleDemo/ConsoleDemo.csproj Normal file
View File

@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Parser\Parser.csproj" />
<ProjectReference Include="..\Semantics\Semantics.csproj" />
</ItemGroup>
</Project>

189
ConsoleDemo/DumbWalker.cs Normal file
View File

@ -0,0 +1,189 @@
using System;
using System.Linq;
using Parser;
using Semantics;
namespace ConsoleDemo
{
public class DumbWalker : SyntaxWalker
{
private bool _insideMethod;
private bool _insideFunction;
private VariableAssignments _variableAssignments;
private MethodAssignments _methodAssignments;
private Context _context;
public DumbWalker(Context context)
{
_context = context;
}
private void Assign(SyntaxNode lhs, SyntaxNode rhs)
{
switch (lhs.Kind)
{
case TokenKind.IdentifierName:
var name = ((IdentifierNameSyntaxNode) lhs).Name.Text;
Console.WriteLine($"Adding variable assignment for {name}");
_variableAssignments.Add(name, new Variable());
break;
default:
break;
}
}
public override void VisitAssignmentExpression(AssignmentExpressionSyntaxNode node)
{
if (_insideMethod || _insideFunction)
{
Console.Write($"Assignment: {node.Lhs} <- {node.Rhs}...");
if (IsDefined(node.Rhs))
{
Console.WriteLine("Ok.");
Assign(node.Lhs, node.Rhs);
}
else
{
Console.WriteLine("Right-hand side is not defined!");
}
}
}
private bool IsDefinedToken(SyntaxToken token)
{
switch (token.Kind)
{
case TokenKind.Comma:
return true;
default:
break;
}
return false;
}
private bool IsDefinedFunctionName(SyntaxNode node)
{
switch (node.Kind)
{
case TokenKind.IdentifierName:
var name = (IdentifierNameSyntaxNode) node;
if (_context.FindFunction(name.Name.Text))
{
return true;
}
if (_methodAssignments.Find(name.Text) != null)
{
return true;
}
break;
default:
break;
}
return false;
}
private bool IsDefined(SyntaxNode node)
{
Variable assignment;
switch (node.Kind)
{
case TokenKind.IdentifierName:
assignment = _variableAssignments.Find(node.Text);
if (assignment != null || node.Text == "end")
{
return true;
}
break;
case TokenKind.FunctionCall:
var functionCall = (FunctionCallExpressionSyntaxNode)node;
return
(IsDefined(functionCall.FunctionName) && IsDefined(functionCall.Nodes)) ||
(IsDefinedFunctionName(functionCall.FunctionName) && IsDefined(functionCall.Nodes));
case TokenKind.CellArrayElementAccess:
var cellArrayElementAccess = (CellArrayElementAccessExpressionSyntaxNode) node;
return IsDefined(cellArrayElementAccess.Expression) && IsDefined(cellArrayElementAccess.Nodes);
case TokenKind.List:
var list = (SyntaxNodeOrTokenList) node;
return list.All(x => x.IsNode ? IsDefined(x.AsNode()) : IsDefinedToken(x.AsToken()));
case TokenKind.NumberLiteralExpression:
return true;
case TokenKind.StringLiteralExpression:
return true;
case TokenKind.BinaryOperation:
var binaryOperation = (BinaryOperationExpressionSyntaxNode) node;
return IsDefined(binaryOperation.Lhs) && IsDefined(binaryOperation.Rhs);
case TokenKind.UnaryPrefixOperationExpression:
var unaryOperation = (UnaryPrefixOperationExpressionSyntaxNode) node;
return IsDefined(unaryOperation.Operand);
case TokenKind.ArrayLiteralExpression:
var arrayLiteral = (ArrayLiteralExpressionSyntaxNode) node;
return arrayLiteral.Nodes == null || IsDefined(arrayLiteral.Nodes);
default:
break;
}
return false;
}
public override void VisitFunctionDeclaration(FunctionDeclarationSyntaxNode node)
{
_insideFunction = true;
_variableAssignments = new VariableAssignments();
var parameterList = node.InputDescription.ParameterList;
foreach (var parameter in parameterList)
{
if (parameter.IsNode)
{
var parameterAsNode = parameter.AsNode();
Console.WriteLine($"Parameter node: {parameterAsNode}");
if (parameterAsNode.Kind == TokenKind.IdentifierName)
{
Console.WriteLine($"Adding variable assignment for {parameterAsNode.Text}");
_variableAssignments.Add(parameterAsNode.Text, new Variable());
}
else
{
Console.WriteLine($"Don't know how to add assignment for {parameterAsNode.Text}");
}
}
else
{
Console.WriteLine($"Parameter token: {parameter.AsToken()}");
}
}
base.VisitFunctionDeclaration(node);
_variableAssignments = null;
_insideFunction = false;
}
public override void VisitMethodDefinition(MethodDefinitionSyntaxNode node)
{
_insideMethod = true;
base.VisitMethodDefinition(node);
_insideMethod = false;
}
public override void VisitFile(FileSyntaxNode node)
{
_methodAssignments = new MethodAssignments();
foreach (var nodeOrToken in node.StatementList)
{
if (nodeOrToken.IsToken)
{
continue;
}
var n = nodeOrToken.AsNode();
if (n.Kind == TokenKind.FunctionDeclaration)
{
var functionDeclaration = (FunctionDeclarationSyntaxNode) n;
_methodAssignments.Add(functionDeclaration.Name.Text, new Variable());
}
}
base.VisitFile(node);
}
}
}

504
ConsoleDemo/PrettyPrinter.cs Normal file
View File

@ -0,0 +1,504 @@
using System;
using Parser;
using Parser.Internal;
namespace ProjectConsole
{
public class PrettyPrinter : SyntaxVisitor
{
public override void VisitFile(FileSyntaxNode node)
{
Visit(node.StatementList);
OutputKeyword(node.EndOfFile);
}
private void PrintToken(SyntaxToken token, ConsoleColor color, bool useBold = false)
{
if (token == default(SyntaxToken))
{
return;
}
Console.ForegroundColor = ConsoleColor.DarkGray;
foreach (var t in token.LeadingTrivia)
{
Console.Write(t);
}
Console.ForegroundColor = color;
if (useBold)
{
BoldOn();
}
Console.Write(token.Text);
if (useBold)
{
BoldOff();
}
Console.ForegroundColor = ConsoleColor.DarkGray;
foreach (var t in token.TrailingTrivia)
{
Console.Write(t);
}
}
private static void UnderlineOn()
{
Console.Write("\x1b[4m");
}
private static void UnderlineOff()
{
Console.Write("\x1b[0m");
}
private static void BoldOn()
{
Console.Write("\x1b[1m");
}
private static void BoldOff()
{
Console.Write("\x1b[0m");
}
private void OutputKeyword(SyntaxToken token)
{
PrintToken(token, ConsoleColor.Green, useBold: true);
}
private void OutputControlKeyword(SyntaxToken token)
{
PrintToken(token, ConsoleColor.Yellow, useBold: true);
}
private void OutputPunctuation(SyntaxToken token)
{
PrintToken(token, ConsoleColor.DarkBlue);
}
private void OutputOperator(SyntaxToken token)
{
PrintToken(token, ConsoleColor.Cyan);
}
private void OutputIdentifier(SyntaxToken token)
{
PrintToken(token, ConsoleColor.White, useBold: true);
}
private void OutputUnquotedStringLiteral(SyntaxToken token)
{
PrintToken(token, ConsoleColor.Blue);
}
private void OutputStringLiteral(SyntaxToken token)
{
PrintToken(token, ConsoleColor.Magenta);
}
private void OutputNumberLiteral(SyntaxToken token)
{
PrintToken(token, ConsoleColor.DarkGreen);
}
private void OutputBracket(SyntaxToken token)
{
PrintToken(token, ConsoleColor.DarkYellow);
}
public override void VisitBaseClassList(BaseClassListSyntaxNode node)
{
OutputPunctuation(node.LessSign);
Visit(node.BaseClasses);
}
public override void VisitClassDeclaration(ClassDeclarationSyntaxNode node)
{
OutputKeyword(node.ClassdefKeyword);
Visit(node.Attributes);
BoldOn();
Visit(node.ClassName);
BoldOff();
Visit(node.BaseClassList);
Visit(node.Nodes);
OutputKeyword(node.EndKeyword);
}
public override void DefaultVisit(SyntaxNode node)
{
Console.ForegroundColor = ConsoleColor.Gray;
foreach (var t in node.LeadingTrivia)
{
Console.Write(t);
}
Console.ForegroundColor = ConsoleColor.Gray;
Console.Write(node.Text);
Console.ForegroundColor = ConsoleColor.DarkGray;
foreach (var t in node.TrailingTrivia)
{
Console.Write(t);
}
}
public override void VisitList(SyntaxNodeOrTokenList list)
{
foreach (var nodeOrToken in list)
{
if (nodeOrToken.IsToken)
{
var token = nodeOrToken.AsToken();
if (token.Kind == TokenKind.Identifier)
{
OutputIdentifier(token);
}
else if (SyntaxFacts.IsBracket(token.Kind))
{
OutputBracket(token);
}
else
{
OutputPunctuation(token);
}
}
else
{
Visit(nodeOrToken.AsNode());
}
}
}
public override void VisitMethodsList(MethodsListSyntaxNode node)
{
OutputKeyword(node.MethodsKeyword);
Visit(node.Attributes);
Visit(node.Methods);
OutputKeyword(node.EndKeyword);
}
public override void VisitPropertiesList(PropertiesListSyntaxNode node)
{
OutputKeyword(node.PropertiesKeyword);
Visit(node.Attributes);
Visit(node.Properties);
OutputKeyword(node.EndKeyword);
}
public override void VisitMethodDefinition(MethodDefinitionSyntaxNode node)
{
OutputKeyword(node.FunctionKeyword);
Visit(node.OutputDescription);
BoldOn();
Visit(node.Name);
BoldOff();
Visit(node.InputDescription);
Visit(node.Commas);
Visit(node.Body);
OutputKeyword(node.EndKeyword);
}
public override void VisitFunctionDeclaration(FunctionDeclarationSyntaxNode node)
{
OutputKeyword(node.FunctionKeyword);
Visit(node.OutputDescription);
OutputIdentifier(node.Name);
Visit(node.InputDescription);
Visit(node.Commas);
Visit(node.Body);
OutputKeyword(node.EndKeyword);
}
public override void VisitIfStatement(IfStatementSyntaxNode node)
{
OutputControlKeyword(node.IfKeyword);
Visit(node.Condition);
Visit(node.OptionalCommas);
Visit(node.Body);
Visit(node.ElseifClauses);
Visit(node.ElseClause);
OutputControlKeyword(node.EndKeyword);
}
public override void VisitElseClause(ElseClause node)
{
OutputControlKeyword(node.ElseKeyword);
Visit(node.Body);
}
public override void VisitElseifClause(ElseifClause node)
{
OutputControlKeyword(node.ElseifKeyword);
Visit(node.Condition);
Visit(node.Body);
}
public override void VisitAbstractMethodDeclaration(AbstractMethodDeclarationSyntaxNode node)
{
Visit(node.OutputDescription);
BoldOn();
Visit(node.Name);
BoldOff();
Visit(node.InputDescription);
}
public override void VisitAssignmentExpression(AssignmentExpressionSyntaxNode node)
{
Visit(node.Lhs);
OutputOperator(node.AssignmentSign);
Visit(node.Rhs);
}
public override void VisitExpressionStatement(ExpressionStatementSyntaxNode node)
{
Visit(node.Expression);
}
public override void VisitArrayLiteralExpression(ArrayLiteralExpressionSyntaxNode node)
{
OutputBracket(node.OpeningSquareBracket);
Visit(node.Nodes);
OutputBracket(node.ClosingSquareBracket);
}
public override void VisitCellArrayLiteralExpression(CellArrayLiteralExpressionSyntaxNode node)
{
OutputBracket(node.OpeningBrace);
Visit(node.Nodes);
OutputBracket(node.ClosingBrace);
}
public override void VisitIdentifierName(IdentifierNameSyntaxNode node)
{
OutputIdentifier(node.Name);
}
public override void VisitForStatement(ForStatementSyntaxNode node)
{
OutputControlKeyword(node.ForKeyword);
Visit(node.Assignment);
Visit(node.OptionalCommas);
Visit(node.Body);
OutputControlKeyword(node.EndKeyword);
}
public override void VisitSwitchStatement(SwitchStatementSyntaxNode node)
{
OutputControlKeyword(node.SwitchKeyword);
Visit(node.SwitchExpression);
Visit(node.OptionalCommas);
Visit(node.Cases);
OutputControlKeyword(node.EndKeyword);
}
public override void VisitWhileStatement(WhileStatementSyntaxNode node)
{
OutputControlKeyword(node.WhileKeyword);
Visit(node.Condition);
Visit(node.OptionalCommas);
Visit(node.Body);
OutputControlKeyword(node.EndKeyword);
}
public override void VisitUnquotedStringLiteral(UnquotedStringLiteralSyntaxNode node)
{
OutputUnquotedStringLiteral(node.StringToken);
}
public override void VisitStringLiteral(StringLiteralSyntaxNode node)
{
OutputStringLiteral(node.StringToken);
}
public override void VisitBinaryOperationExpression(BinaryOperationExpressionSyntaxNode node)
{
Visit(node.Lhs);
OutputOperator(node.Operation);
Visit(node.Rhs);
}
public override void VisitFunctionCallExpression(FunctionCallExpressionSyntaxNode node)
{
Visit(node.FunctionName);
OutputBracket(node.OpeningBracket);
Visit(node.Nodes);
OutputBracket(node.ClosingBracket);
}
public override void VisitSwitchCase(SwitchCaseSyntaxNode node)
{
OutputControlKeyword(node.CaseKeyword);
Visit(node.CaseIdentifier);
Visit(node.OptionalCommas);
Visit(node.Body);
}
public override void VisitCatchClause(CatchClauseSyntaxNode node)
{
OutputControlKeyword(node.CatchKeyword);
Visit(node.CatchBody);
}
public override void VisitTryCatchStatement(TryCatchStatementSyntaxNode node)
{
OutputControlKeyword(node.TryKeyword);
Visit(node.TryBody);
Visit(node.CatchClause);
OutputControlKeyword(node.EndKeyword);
}
public override void VisitCommandExpression(CommandExpressionSyntaxNode node)
{
Visit(node.CommandName);
Visit(node.Arguments);
}
public override void VisitNumberLiteral(NumberLiteralSyntaxNode node)
{
OutputNumberLiteral(node.Number);
}
public override void VisitUnaryPrefixOperationExpression(UnaryPrefixOperationExpressionSyntaxNode node)
{
OutputOperator(node.Operation);
Visit(node.Operand);
}
public override void VisitUnaryPostixOperationExpression(UnaryPostixOperationExpressionSyntaxNode node)
{
Visit(node.Operand);
OutputOperator(node.Operation);
}
public override void VisitBaseClassInvokation(BaseClassInvokationSyntaxNode node)
{
Visit(node.MethodName);
OutputOperator(node.AtSign);
Visit(node.BaseClassNameAndArguments);
}
public override void VisitAttributeAssignment(AttributeAssignmentSyntaxNode node)
{
OutputOperator(node.AssignmentSign);
Visit(node.Value);
}
public override void VisitAttribute(AttributeSyntaxNode node)
{
Visit(node.Name);
Visit(node.Assignment);
}
public override void VisitAttributeList(AttributeListSyntaxNode node)
{
OutputBracket(node.OpeningBracket);
Visit(node.Nodes);
OutputBracket(node.ClosingBracket);
}
public override void VisitCellArrayElementAccessExpression(CellArrayElementAccessExpressionSyntaxNode node)
{
Visit(node.Expression);
OutputBracket(node.OpeningBrace);
Visit(node.Nodes);
OutputBracket(node.ClosingBrace);
}
public override void VisitCompoundName(CompoundNameSyntaxNode node)
{
Visit(node.Nodes);
}
public override void VisitDoubleQuotedStringLiteral(DoubleQuotedStringLiteralSyntaxNode node)
{
OutputStringLiteral(node.StringToken);
}
public override void VisitEmptyExpression(EmptyExpressionSyntaxNode node)
{
}
public override void VisitEmptyStatement(EmptyStatementSyntaxNode node)
{
OutputPunctuation(node.Semicolon);
}
public override void VisitEnumerationList(EnumerationListSyntaxNode node)
{
OutputKeyword(node.EnumerationKeyword);
Visit(node.Attributes);
Visit(node.Items);
OutputKeyword(node.EndKeyword);
}
public override void VisitEventsList(EventsListSyntaxNode node)
{
OutputKeyword(node.EventsKeyword);
Visit(node.Attributes);
Visit(node.Events);
OutputKeyword(node.EndKeyword);
}
public override void VisitEnumerationItemValue(EnumerationItemValueSyntaxNode node)
{
OutputPunctuation(node.OpeningBracket);
Visit(node.Values);
OutputPunctuation(node.ClosingBracket);
}
public override void VisitEnumerationItem(EnumerationItemSyntaxNode node)
{
Visit(node.Name);
Visit(node.Values);
Visit(node.Commas);
}
public override void VisitFunctionInputDescription(FunctionInputDescriptionSyntaxNode node)
{
OutputBracket(node.OpeningBracket);
Visit(node.ParameterList);
OutputBracket(node.ClosingBracket);
}
public override void VisitFunctionOutputDescription(FunctionOutputDescriptionSyntaxNode node)
{
Visit(node.OutputList);
OutputOperator(node.AssignmentSign);
}
public override void VisitIndirectMemberAccess(IndirectMemberAccessSyntaxNode node)
{
OutputBracket(node.OpeningBracket);
Visit(node.Expression);
OutputBracket(node.ClosingBracket);
}
public override void VisitLambda(LambdaSyntaxNode node)
{
OutputOperator(node.AtSign);
Visit(node.Input);
Visit(node.Body);
}
public override void VisitNamedFunctionHandle(NamedFunctionHandleSyntaxNode node)
{
OutputOperator(node.AtSign);
Visit(node.FunctionName);
}
public override void VisitMemberAccess(MemberAccessSyntaxNode node)
{
Visit(node.LeftOperand);
OutputOperator(node.Dot);
Visit(node.RightOperand);
}
public override void VisitParenthesizedExpression(ParenthesizedExpressionSyntaxNode node)
{
OutputBracket(node.OpeningBracket);
Visit(node.Expression);
OutputBracket(node.ClosingBracket);
}
}
}

162
ProjectConsole/Program.cs → ConsoleDemo/Program.cs
View File

@ -2,19 +2,19 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.Linq;
using Lexer;
using Parser; using Parser;
using ProjectConsole;
using Semantics;
namespace ProjectConsole namespace ConsoleDemo
{ {
class Program class Program
{ {
private static readonly string BaseDirectory; private static readonly string BaseDirectory;
private const string BaseDirectoryMacOs = @"/Applications/MATLAB_R2017b.app/toolbox/matlab/"; private const string BaseDirectoryMacOs = @"/Applications/MATLAB_R2017b.app/toolbox/matlab/";
private const string BaseDirectoryWindows = @"C:\Program Files\MATLAB\R2018a\toolbox\matlab\"; private const string BaseDirectoryWindows = @"D:\Program Files\MATLAB\R2018a\toolbox\matlab\";
private static HashSet<string> skipFiles = new HashSet<string> private static readonly HashSet<string> SkipFiles = new HashSet<string>
{ {
@"codetools\private\template.m", // this is a template, so it contains '$' characters. @"codetools\private\template.m", // this is a template, so it contains '$' characters.
@"plottools\+matlab\+graphics\+internal\+propertyinspector\+views\CategoricalHistogramPropertyView.m", // this one contains a 0xA0 character (probably it's 'non-breakable space' in Win-1252). @"plottools\+matlab\+graphics\+internal\+propertyinspector\+views\CategoricalHistogramPropertyView.m", // this one contains a 0xA0 character (probably it's 'non-breakable space' in Win-1252).
@ -26,32 +26,31 @@ namespace ProjectConsole
@"plottools/+matlab/+graphics/+internal/+propertyinspector/+views/PrimitiveHistogramPropertyView.m", // same @"plottools/+matlab/+graphics/+internal/+propertyinspector/+views/PrimitiveHistogramPropertyView.m", // same
}; };
static void ProcessFile(string fileName) private static MParser CreateParser(ITextWindow window)
{
return new MParser(window);
}
private static void ProcessFile(string fileName)
{ {
var text = File.ReadAllText(fileName); var text = File.ReadAllText(fileName);
Console.WriteLine($"Parsing {fileName}..."); //Console.Write($"Parsing {fileName}...");
var window = new TextWindowWithNull(text, fileName); var window = new TextWindowWithNull(text, fileName);
ILexer<Token> lexer = new MLexer(window, new PureTokenFactory(window)); var parser = CreateParser(window);
var tokens = lexer.ParseAll();
//AfterFunction(tokens);
//FirstToken(tokens);
var parser = new MParser(tokens);
var tree = parser.Parse(); var tree = parser.Parse();
var back = string.Join("", tokens.Select(token => token.FullText)); //Console.WriteLine("Done.");
if (text != back) var actual = tree.FullText;
if (actual != text)
{ {
throw new ApplicationException(); throw new ApplicationException();
} }
//var printer = new PrettyPrinter();
//printer.Visit(tree);
//Console.ReadKey();
} }
private static readonly int[] firstTokenCount;
private static readonly int[] afterFunctionCount;
static Program() static Program()
{ {
var maxKind = ((int[]) typeof(TokenKind).GetEnumValues()).Max();
firstTokenCount = new int[maxKind + 1];
afterFunctionCount = new int[maxKind + 1];
switch (Environment.OSVersion.Platform) switch (Environment.OSVersion.Platform)
{ {
case PlatformID.MacOSX: case PlatformID.MacOSX:
@ -64,57 +63,14 @@ namespace ProjectConsole
} }
} }
static void AfterFunction(List<Token> tokens) private static int ProcessDirectory(string directory)
{
for (var i = 0; i < tokens.Count; i++)
{
if (tokens[i].PureToken.Kind == TokenKind.Identifier &&
tokens[i].PureToken.LiteralText == "function")
{
var nextKind = tokens[i + 1].PureToken.Kind;
afterFunctionCount[(int) nextKind]++;
if (nextKind != TokenKind.Identifier && nextKind != TokenKind.OpeningSquareBracket)
{
Console.WriteLine("===EXAMPLE===");
Console.WriteLine($"{tokens[i]}{tokens[i+1]}");
}
}
}
}
static void FirstToken(List<Token> tokens)
{
var firstKind = tokens[0].PureToken.Kind;
firstTokenCount[(int) firstKind]++;
}
static void AfterFunctionFinish()
{
for (var i = 0; i < afterFunctionCount.Length; i++)
{
Console.WriteLine($"{(TokenKind)i}: {afterFunctionCount[i]}.");
}
}
static void FirstTokenFinish()
{
for (var i = 0; i < firstTokenCount.Length; i++)
{
if (firstTokenCount[i] != 0)
{
Console.WriteLine($"{(TokenKind) i}: {firstTokenCount[i]}.");
}
}
}
static int ProcessDirectory(string directory)
{ {
var counter = 0; var counter = 0;
var files = Directory.GetFiles(directory, "*.m"); var files = Directory.GetFiles(directory, "*.m");
foreach (var file in files) foreach (var file in files)
{ {
var relativePath = Path.GetRelativePath(BaseDirectory, file); var relativePath = Path.GetRelativePath(BaseDirectory, file);
if (skipFiles.Contains(relativePath)) if (SkipFiles.Contains(relativePath))
{ {
continue; continue;
} }
@ -131,7 +87,7 @@ namespace ProjectConsole
return counter; return counter;
} }
static void Main(string[] args) private static void ParserDemo()
{ {
Console.WriteLine("Hello World!"); Console.WriteLine("Hello World!");
var sw = new Stopwatch(); var sw = new Stopwatch();
@ -143,5 +99,79 @@ namespace ProjectConsole
//FirstTokenFinish(); //FirstTokenFinish();
Console.ReadKey(); Console.ReadKey();
} }
private static FileSyntaxNode GetTree(string fileName)
{
var text = File.ReadAllText(fileName);
var window = new TextWindowWithNull(text, fileName);
var parser = CreateParser(window);
var tree = parser.Parse();
return tree;
}
public static void SemanticsDemo()
{
var fileName = Path.Combine(
BaseDirectory,
"datatypes",
"@table",
"table.m");
var tree = GetTree(fileName);
var childNodesAndTokens = tree.GetChildNodesAndTokens();
var node = childNodesAndTokens[0].AsNode();
var classChildNodesAndTokens = node.GetChildNodesAndTokens();
var c = GetClass.FromTree(tree, fileName);
Console.WriteLine(c.Name);
foreach (var m in c.Methods)
{
Console.WriteLine($"* Method {m.Name}");
if (m.Description != "")
{
Console.WriteLine($"* Description: {m.Description}");
}
}
}
public static void ContextDemo()
{
var context = new Context();
context.ScanPath(BaseDirectory);
}
public static void DumbPrinterDemo()
{
var context = new Context();
context.ScanPath(BaseDirectory);
var fileName = Path.Combine(
BaseDirectory,
"specgraph",
"heatmap.m");
var tree = GetTree(fileName);
var printer = new DumbWalker(context);
printer.Visit(tree);
}
public static void UsageDemo()
{
var context = new Context();
context.ScanPath(BaseDirectory);
var fileName = Path.Combine(
BaseDirectory,
"specgraph",
"heatmap.m");
var tree = GetTree(fileName);
var printer = new UsageGathering(context);
printer.Visit(tree);
}
public static void Main(string[] args)
{
//ParserDemo();
//SemanticsDemo();
//ContextDemo();
//DumbPrinterDemo();
UsageDemo();
Console.ReadKey();
}
} }
} }

151
ConsoleDemo/UsageGathering.cs Normal file
View File

@ -0,0 +1,151 @@
using System;
using Parser;
using Semantics;
namespace ConsoleDemo
{
public class UsageGathering : SyntaxWalker
{
private MethodAssignments _methodAssignments;
private VariableAssignments _variableAssignments;
private bool _insideFunction;
private bool _insideMethod;
private Context _context;
public UsageGathering(Context context)
{
_context = context;
}
public override void VisitFunctionCallExpression(FunctionCallExpressionSyntaxNode node)
{
if (!(_insideFunction || _insideMethod))
{
return;
}
var name = node.FunctionName.Text;
if (_variableAssignments.Find(name) != null)
{
return;
}
Console.Write($"Function call: {name}...");
if (_context.FindFunction(name) || _methodAssignments.Find(name) != null)
{
Console.WriteLine("found.");
}
else
{
Console.WriteLine("NOT FOUND.");
}
base.VisitFunctionCallExpression(node);
}
private void Assign(SyntaxNode lhs, SyntaxNode rhs)
{
switch (lhs.Kind)
{
case TokenKind.IdentifierName:
var name = ((IdentifierNameSyntaxNode)lhs).Name.Text;
Console.WriteLine($"Adding variable assignment for {name}");
_variableAssignments.Add(name, new Variable());
break;
default:
break;
}
}
public override void VisitAssignmentExpression(AssignmentExpressionSyntaxNode node)
{
if (!(_insideFunction || _insideMethod))
{
return;
}
Assign(node.Lhs, node.Rhs);
base.VisitAssignmentExpression(node);
}
public override void VisitFile(FileSyntaxNode node)
{
_methodAssignments = new MethodAssignments();
foreach (var nodeOrToken in node.StatementList)
{
if (nodeOrToken.IsToken)
{
continue;
}
var n = nodeOrToken.AsNode();
if (n.Kind == TokenKind.FunctionDeclaration)
{
var functionDeclaration = (FunctionDeclarationSyntaxNode)n;
_methodAssignments.Add(functionDeclaration.Name.Text, new Variable());
}
}
base.VisitFile(node);
}
public override void VisitFunctionDeclaration(FunctionDeclarationSyntaxNode node)
{
_insideFunction = true;
_variableAssignments = new VariableAssignments();
var parameterList = node.InputDescription.ParameterList;
foreach (var parameter in parameterList)
{
if (parameter.IsNode)
{
var parameterAsNode = parameter.AsNode();
Console.WriteLine($"Parameter node: {parameterAsNode}");
if (parameterAsNode.Kind == TokenKind.IdentifierName)
{
Console.WriteLine($"Adding variable assignment for {parameterAsNode.Text}");
_variableAssignments.Add(parameterAsNode.Text, new Variable());
}
else
{
Console.WriteLine($"Don't know how to add assignment for {parameterAsNode.Text}");
}
}
else
{
Console.WriteLine($"Parameter token: {parameter.AsToken()}");
}
}
base.VisitFunctionDeclaration(node);
_variableAssignments = null;
_insideFunction = false;
}
public override void VisitMethodDefinition(MethodDefinitionSyntaxNode node)
{
_insideMethod = true;
_variableAssignments = new VariableAssignments();
var parameterList = node.InputDescription.ParameterList;
foreach (var parameter in parameterList)
{
if (parameter.IsNode)
{
var parameterAsNode = parameter.AsNode();
Console.WriteLine($"Parameter node: {parameterAsNode}");
if (parameterAsNode.Kind == TokenKind.IdentifierName)
{
Console.WriteLine($"Adding variable assignment for {parameterAsNode.Text}");
_variableAssignments.Add(parameterAsNode.Text, new Variable());
}
else
{
Console.WriteLine($"Don't know how to add assignment for {parameterAsNode.Text}");
}
}
else
{
Console.WriteLine($"Parameter token: {parameter.AsToken()}");
}
}
base.VisitMethodDefinition(node);
_variableAssignments = null;
_insideMethod = false;
}
}
}

14
Lexer.Tests/Lexer.Tests.csproj
View File

@ -1,14 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.7.0" />
<PackageReference Include="NUnit" Version="3.10.1" />
<PackageReference Include="NUnit3TestAdapter" Version="3.10.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Lexer\Lexer.csproj" />
</ItemGroup>
</Project>

276
Lexer.Tests/MLexerShould.cs
View File

@ -1,276 +0,0 @@
using System.Linq;
using NUnit.Framework;
namespace Lexer.Tests
{
public class MLexerShould
{
private static MLexer CreateLexer(string text)
{
var window = new TextWindowWithNull(text);
return new MLexer(window, new PureTokenFactory(window));
}
[Test]
public void ParseSequenceOfIdentifiers()
{
var sut = CreateLexer("undefined is not\n a function");
var tokens = sut.ParseAll();
Assert.AreEqual(6, tokens.Count);
CollectionAssert.AreEqual(
new[] {"undefined", "is", "not", "a", "function"},
tokens.Take(5).Select(token => token.PureToken.LiteralText));
CollectionAssert.AreEqual(
new[] { TokenKind.Identifier, TokenKind.UnquotedStringLiteral, TokenKind.UnquotedStringLiteral,
TokenKind.Identifier, TokenKind.UnquotedStringLiteral },
tokens.Take(5).Select(token => token.Kind));
}
[Test]
public void ParseIdentifierAndBrackets()
{
var sut = CreateLexer("undefined()");
var tokens = sut.ParseAll();
Assert.AreEqual(4, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.Identifier,
TokenKind.OpeningBracket,
TokenKind.ClosingBracket,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseTransposeSignAfterClosingSquareBracket()
{
var sut = CreateLexer("[undefined]'");
var tokens = sut.ParseAll();
Assert.AreEqual(5, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.OpeningSquareBracket,
TokenKind.Identifier,
TokenKind.ClosingSquareBracket,
TokenKind.Transpose,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseTransposeSignAfterClosingBrace()
{
var sut = CreateLexer("{undefined}'");
var tokens = sut.ParseAll();
Assert.AreEqual(5, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.OpeningBrace,
TokenKind.Identifier,
TokenKind.ClosingBrace,
TokenKind.Transpose,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseTransposeSignAfterClosingBracket()
{
var sut = CreateLexer("undefined()'");
var tokens = sut.ParseAll();
Assert.AreEqual(5, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.Identifier,
TokenKind.OpeningBracket,
TokenKind.ClosingBracket,
TokenKind.Transpose,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseTransposeSignAfterIdentifier()
{
var sut = CreateLexer("undefined'");
var tokens = sut.ParseAll();
Assert.AreEqual(3, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.Identifier,
TokenKind.Transpose,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseTransposeSignAfterDot()
{
var sut = CreateLexer("undefined.'");
var tokens = sut.ParseAll();
Assert.AreEqual(3, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.Identifier,
TokenKind.DotTranspose,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseDotPowerAfterNumber()
{
var sut = CreateLexer("26.^[1]");
var tokens = sut.ParseAll();
Assert.AreEqual(6, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.NumberLiteral,
TokenKind.DotPower,
TokenKind.OpeningSquareBracket,
TokenKind.NumberLiteral,
TokenKind.ClosingSquareBracket,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseDotInNumberBeforeSemicolon()
{
var sut = CreateLexer("42.;");
var tokens = sut.ParseAll();
Assert.AreEqual(3, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.NumberLiteral,
TokenKind.Semicolon,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseEAfterDotInANumber()
{
var sut = CreateLexer("42.e-5");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
CollectionAssert.AreEqual(
new[]
{
TokenKind.NumberLiteral,
TokenKind.EndOfFile
},
tokens.Select(token => token.PureToken.Kind));
}
[Test]
public void ParseEmptyLine()
{
var sut = CreateLexer("\n\nfunction shmunction\n\n\n");
var tokens = sut.ParseAll();
Assert.AreEqual(3, tokens.Count);
}
[Test]
public void ParseCommentsAfterDotDotDot()
{
var sut = CreateLexer("something ... #$@#%*^!@#\n");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
}
[TestCase("something ... #$@#%*^!@#\n")]
[TestCase("undefined is not a function")]
[TestCase("\n\nfunction shmunction\n\n\n")]
public void ReconstructTest(string s)
{
var sut = CreateLexer(s);
var tokens = sut.ParseAll();
var actual = string.Join("", tokens.Select(token => token.FullText));
Assert.AreEqual(s, actual);
}
[Test]
public void ParseStringLiteral()
{
var sut = CreateLexer("'just a string'");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(TokenKind.StringLiteral, tokens[0].Kind);
Assert.AreEqual("just a string", tokens[0].PureToken.Value);
}
[Test]
public void ParseStringLiteralWithEscapedQuotes()
{
var sut = CreateLexer("'just a ''string'''");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(TokenKind.StringLiteral, tokens[0].Kind);
Assert.AreEqual("just a 'string'", tokens[0].PureToken.Value);
}
[Test]
public void ParseDoubleQuotedStringLiteralWithEscapedQuotes()
{
var sut = CreateLexer("\"just a \"\"string\"\"\"");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(TokenKind.DoubleQuotedStringLiteral, tokens[0].Kind);
Assert.AreEqual("just a \"string\"", tokens[0].PureToken.Value);
}
[Test]
public void ParseNumberStartingWithDot()
{
var sut = CreateLexer(".42");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(TokenKind.NumberLiteral, tokens[0].Kind);
}
[TestCase("%{\nabc\n%}", true)]
[TestCase("%{ a\nabc\n%}", false)]
[TestCase("if %{\nabc\n%}", false)]
public void ParseMultilineComments(string text, bool isMultiline)
{
var sut = CreateLexer(text);
var tokens = sut.ParseAll();
if (isMultiline)
{
Assert.AreEqual(1, tokens.Count);
}
else
{
Assert.Less(1, tokens.Count);
}
}
[TestCase(".42i")]
[TestCase("42i")]
[TestCase("42e-1i")]
public void ParseComplexNumbers(string text)
{
var sut = CreateLexer(text);
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(TokenKind.NumberLiteral, tokens[0].Kind);
}
}
}

6
Lexer/IPosition.cs
View File

@ -1,6 +0,0 @@
namespace Lexer
{
public interface IPosition
{
}
}

5
Lexer/Lexer.csproj
View File

@ -1,5 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup>
</Project>

14
Lexer/PositionInsideFile.cs
View File

@ -1,14 +0,0 @@
namespace Lexer
{
public struct PositionInsideFile : IPosition
{
public string File { get; set; }
public int Line { get; set; }
public int Column { get; set; }
public override string ToString()
{
return $"line {Line}, column {Column}" + (File != null ? $" of {File}" : "");
}
}
}

20
Lexer/PureToken.cs
View File

@ -1,20 +0,0 @@
namespace Lexer
{
public struct PureToken
{
public TokenKind Kind { get; }
public string LiteralText { get; }
public object Value { get; }
public IPosition Position { get; }
public PureToken(TokenKind kind, string literalText, object value, IPosition position)
{
Kind = kind;
LiteralText = literalText;
Value = value;
Position = position;
}
public override string ToString() => LiteralText;
}
}

111
Lexer/PureTokenFactory.cs
View File

@ -1,111 +0,0 @@
namespace Lexer
{
public class PureTokenFactory
{
private ITextWindow Window { get; }
public PureTokenFactory(ITextWindow window)
{
Window = window;
}
private static readonly string[] PureTokenOfKind =
{
null, // None = 0,
null, // EndOfFile = 1,
null, // Identifier = 2,
null, // NumberLiteral = 3,
null, // StringLiteral = 4,
null, // DoubleQuotedStringLiteral = 5,
null, // UnquotedStringLiteral = 6
null, null, null, null, null, null, null, null, null, null, null, null, null,
"=", // Assignment = 20,
"==", // Equality = 21,
"~=", // Inequality = 22,
"&&", // LogicalAnd = 23,
"||", // LogicalOr = 24,
"&", // BitwiseAnd = 25,
"|", // BitwiseOr = 26,
"<", // Less = 27,
"<=", // LessOrEqual = 28,
">", // Greater = 29,
">=", // GreaterOrEqual = 30,
"~", // Not = 31,
"+", // Plus = 32,
"-", // Minus = 33,
"*", // Multiply = 34,
"/", // Divide = 35,
"^", // Power = 36,
"\\", // Backslash = 37,
"'", // Transpose = 38,
".*", // DotMultiply = 39,
"./", // DotDivide = 40,
".^", // DotPower = 41,
".\\", // DotBackslash = 42,
".'", // DotTranspose = 43,
"@", // At = 44,
":", // Colon = 45,
"?", // QuestionMark = 46,
",", // Comma = 47,
";", // Semicolon = 48,
"{", // OpeningBrace = 49,
"}", // ClosingBrace = 50,
"[", // OpeningSquareBracket = 51,
"]", // ClosingSquareBracket = 52,
"(", // OpeningBracket = 53,
")", // ClosingBracket = 54,
".", // Dot = 55,
"...", // DotDotDot = 56,
"+", // UnaryPlus = 57,
"-", // UnaryMinus = 58,
"~", // UnaryNot = 59,
"?", // UnaryQuestionMark = 60,
};
public PureToken CreatePunctuation(TokenKind kind)
{
return new PureToken(kind, PureTokenOfKind[(int)kind], null, Window.Position);
}
public PureToken CreateIdentifier(string s)
{
return new PureToken(TokenKind.Identifier, s, null, Window.Position);
}
public PureToken CreateNumberLiteral(string s)
{
return new PureToken(TokenKind.NumberLiteral, s, null, Window.Position); // TODO: actually parse number (here or in the lexer?)
}
private string EscapeStringLiteral(string s)
{
return s.Replace("'", "''");
}
public PureToken CreateStringLiteral(string s)
{
return new PureToken(TokenKind.StringLiteral, "'" + EscapeStringLiteral(s) + "'", s, Window.Position);
}
private string EscapeDoubleQuotedStringLiteral(string s)
{
return s.Replace("\"", "\"\"");
}
public PureToken CreateDoubleQuotedStringLiteral(string s)
{
return new PureToken(TokenKind.DoubleQuotedStringLiteral, "\"" + EscapeDoubleQuotedStringLiteral(s) + "\"", s, Window.Position);
}
public PureToken CreateUnquotedStringLiteral(string s)
{
return new PureToken(TokenKind.UnquotedStringLiteral, s, s, Window.Position);
}
public PureToken CreateEndOfFileToken()
{
return new PureToken(TokenKind.EndOfFile, "", null, Window.Position);
}
}
}

32
Lexer/Token.cs
View File

@ -1,32 +0,0 @@
using System.Collections.Generic;
using System.Linq;
namespace Lexer
{
public class Token
{
public List<Trivia> LeadingTrivia { get; }
public List<Trivia> TrailingTrivia { get; }
public PureToken PureToken { get; }
public string FullText { get; }
public TokenKind Kind => PureToken.Kind;
public Token(PureToken pureToken, List<Trivia> leadingTrivia, List<Trivia> trailingTrivia)
{
PureToken = pureToken;
LeadingTrivia = leadingTrivia;
TrailingTrivia = trailingTrivia;
FullText = BuildFullText();
}
private string BuildFullText()
{
var leading = LeadingTrivia.Select(t => t.LiteralText);
var token = PureToken.LiteralText;
var trailing = TrailingTrivia.Select(t => t.LiteralText);
return string.Join("", leading.Concat(new[] {token}).Concat(trailing));
}
public override string ToString() => FullText;
}
}

56
Lexer/TokenKind.cs
View File

@ -1,56 +0,0 @@
namespace Lexer
{
public enum TokenKind
{
None = 0,
EndOfFile = 1,
Identifier = 2,
NumberLiteral = 3,
StringLiteral = 4,
DoubleQuotedStringLiteral = 5,
UnquotedStringLiteral = 6,
Assignment = 20,
Equality = 21,
Inequality = 22,
LogicalAnd = 23,
LogicalOr = 24,
BitwiseAnd = 25,
BitwiseOr = 26,
Less = 27,
LessOrEqual = 28,
Greater = 29,
GreaterOrEqual = 30,
Not = 31,
Plus = 32,
Minus = 33,
Multiply = 34,
Divide = 35,
Power = 36,
Backslash = 37,
Transpose = 38,
DotMultiply = 39,
DotDivide = 40,
DotPower = 41,
DotBackslash = 42,
DotTranspose = 43,
At = 44,
Colon = 45,
QuestionMark = 46,
Comma = 47,
Semicolon = 48,
OpeningBrace = 49,
ClosingBrace = 50,
OpeningSquareBracket = 51,
ClosingSquareBracket = 52,
OpeningBracket = 53,
ClosingBracket = 54,
Dot = 55,
DotDotDot = 56,
// unary tokens are not recognized during lexing; they are contextually recognized while parsing.
UnaryPlus = 57,
UnaryMinus = 58,
UnaryNot = 59,
UnaryQuestionMark = 60,
}
}

14
Lexer/Trivia.cs
View File

@ -1,14 +0,0 @@
namespace Lexer
{
public class Trivia
{
public TriviaType Type { get; }
public string LiteralText { get; }
public Trivia(TriviaType type, string literalText)
{
Type = type;
LiteralText = literalText;
}
}
}

10
Lexer/TriviaType.cs
View File

@ -1,10 +0,0 @@
namespace Lexer
{
public enum TriviaType
{
Whitespace,
NewLine,
Comment,
MultiLineComment
}
}

486
Parser.Tests/MParserShould.cs
View File

@ -1,19 +1,13 @@
using System.Collections.Generic; using NUnit.Framework;
using System.Linq;
using Lexer;
using NUnit.Framework;
namespace Parser.Tests namespace Parser.Tests
{ {
public class MParserShould public class MParserShould
{ {
private static MParser CreateParser(string text) private static MParser GetSut(string text)
{ {
var window = new TextWindowWithNull(text); var window = new TextWindowWithNull(text);
var lexer = new MLexer(window, new PureTokenFactory(window)); var parser = new MParser(window);
var tokens = lexer.ParseAll();
var parser = new MParser(tokens);
return parser; return parser;
} }
@ -21,475 +15,11 @@ namespace Parser.Tests
public void ParseAssignmentExpression() public void ParseAssignmentExpression()
{ {
var text = "a = b"; var text = "a = b";
var sut = CreateParser(text); var sut = GetSut(text);
var actual = sut.ParseExpression(); var actual = sut.Parse();
Assert.IsInstanceOf<AssignmentExpressionNode>(actual); var assignment = actual.StatementList[0].AsNode();
Assert.AreEqual(text, actual.FullText); Assert.IsInstanceOf<ExpressionStatementSyntaxNode>(assignment);
} Assert.IsInstanceOf<AssignmentExpressionSyntaxNode>(((ExpressionStatementSyntaxNode)assignment).Expression);
[Test]
public void ParseSimpleStatement()
{
var text = "a = b";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<ExpressionStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseFunctionCallExpression()
{
var text = "func(a, 2, 'abc', d)";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<FunctionCallExpressionNode>(actual);
var f = actual as FunctionCallExpressionNode;
Assert.AreEqual(4, f?.Parameters.Parameters.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseArrayLiteralExpression()
{
var text = "[a, 2, 'text']";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var a = actual as ArrayLiteralExpressionNode;
Assert.AreEqual(3, a?.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseLeftAssociativeSamePrecedence()
{
var text = "2 + 3 + 4";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<BinaryOperationExpressionNode>(actual);
var e = (BinaryOperationExpressionNode)actual;
Assert.IsInstanceOf<BinaryOperationExpressionNode>(e.Lhs);
Assert.IsInstanceOf<NumberLiteralNode>(e.Rhs);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseLeftAssociativeRaisingPrecedence()
{
var text = "2 + 3 * 4";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<BinaryOperationExpressionNode>(actual);
var e = (BinaryOperationExpressionNode) actual;
Assert.AreEqual(TokenKind.Plus, e.Operation.Token.Kind);
Assert.IsInstanceOf<NumberLiteralNode>(e.Lhs);
Assert.IsInstanceOf<BinaryOperationExpressionNode>(e.Rhs);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseLeftAssociativeLoweringPrecedence()
{
var text = "2 * 3 + 4";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<BinaryOperationExpressionNode>(actual);
var e = (BinaryOperationExpressionNode) actual;
Assert.AreEqual(TokenKind.Plus, e.Operation.Token.Kind);
Assert.IsInstanceOf<BinaryOperationExpressionNode>(e.Lhs);
Assert.IsInstanceOf<NumberLiteralNode>(e.Rhs);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseUnaryOperators()
{
var text = "-42";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<UnaryPrefixOperationExpressionNode>(actual);
var e = (UnaryPrefixOperationExpressionNode) actual;
Assert.AreEqual(TokenKind.Minus, e.Operation.Token.Kind);
Assert.IsInstanceOf<NumberLiteralNode>(e.Operand);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseMemberAccess()
{
var text = "a.b.c";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<MemberAccessNode>(actual);
var m = (MemberAccessNode) actual;
Assert.IsInstanceOf<MemberAccessNode>(m.LeftOperand);
Assert.IsInstanceOf<IdentifierNameNode>(m.RightOperand);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseWhileStatement()
{
var text = "while a < b c = d end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<WhileStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseWhileStatementWithComma()
{
var text = "while a < b, c = d end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<WhileStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseIfStatement()
{
var text = "if 2 < 3 a = b end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<IfStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseIfElseStatement()
{
var text = "if 2 < 3 a = b else c = d end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<IfStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseParenthesizedExpression()
{
var text = "2 * (3 + 4)";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<BinaryOperationExpressionNode>(actual);
var e = (BinaryOperationExpressionNode) actual;
Assert.IsInstanceOf<NumberLiteralNode>(e.Lhs);
Assert.IsInstanceOf<ParenthesizedExpressionNode>(e.Rhs);
var p = (ParenthesizedExpressionNode) e.Rhs;
Assert.IsInstanceOf<BinaryOperationExpressionNode>(p.Expression);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseForStatement()
{
var text = "for i = 1:5 a = i end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<ForStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseEmptyArray()
{
var text = "[]";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var a = (ArrayLiteralExpressionNode) actual;
Assert.AreEqual(0, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseCellArrayLiteral()
{
var text = "{ 1 2, 3 }";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<CellArrayLiteralExpressionNode>(actual);
var a = (CellArrayLiteralExpressionNode) actual;
Assert.AreEqual(3, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseIndirectMemberAccess()
{
var text = "abc.(def)";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<MemberAccessNode>(actual);
var a = (MemberAccessNode) actual;
Assert.IsInstanceOf<IdentifierNameNode>(a.LeftOperand);
Assert.IsInstanceOf<IndirectMemberAccessNode>(a.RightOperand);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseMemberAccessAfterElementAccess()
{
var text = "a(1).b";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<MemberAccessNode>(actual);
var m = (MemberAccessNode) actual;
Assert.IsInstanceOf<FunctionCallExpressionNode>(m.LeftOperand);
Assert.IsInstanceOf<IdentifierNameNode>(m.RightOperand);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseFunctionDeclarationWithoutInputs()
{
var text = "function a = b a = 1";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<FunctionDeclarationNode>(actual);
var f = (FunctionDeclarationNode) actual;
Assert.AreEqual(f.InputDescription, null);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseCellArrayWithCellArrayLiteralInside()
{
var text = "{1 2 a {3}}";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<CellArrayLiteralExpressionNode>(actual);
var a = (CellArrayLiteralExpressionNode) actual;
Assert.AreEqual(4, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseCellArrayWithCellArrayAccessInside()
{
var text = "{1 2 a{3}}";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<CellArrayLiteralExpressionNode>(actual);
var a = (CellArrayLiteralExpressionNode) actual;
Assert.AreEqual(3, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseCellArrayWithElementInBracketsInside()
{
var text = "{1 2 a (3)}";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<CellArrayLiteralExpressionNode>(actual);
var a = (CellArrayLiteralExpressionNode) actual;
Assert.AreEqual(4, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseCellArrayWithFunctionCallInside()
{
var text = "{1 2 a(3)}";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<CellArrayLiteralExpressionNode>(actual);
var a = (CellArrayLiteralExpressionNode) actual;
Assert.AreEqual(3, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseArrayWithCellArrayLiteralInside()
{
var text = "[1 2 a {3}]";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var a = (ArrayLiteralExpressionNode) actual;
Assert.AreEqual(4, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseArrayWithCellArrayAccessInside()
{
var text = "[1 2 a{3}]";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var a = (ArrayLiteralExpressionNode) actual;
Assert.AreEqual(3, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseArrayWithElementInBracketsInside()
{
var text = "[1 2 a (3)]";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var a = (ArrayLiteralExpressionNode) actual;
Assert.AreEqual(4, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseArrayWithFunctionCallInside()
{
var text = "[1 2 a(3)]";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var a = (ArrayLiteralExpressionNode) actual;
Assert.AreEqual(3, a.Elements.Elements.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseFunctionHandle()
{
var text = "@sqrt";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<NamedFunctionHandleNode>(actual);
var f = (NamedFunctionHandleNode) actual;
Assert.AreEqual(1, f.FunctionName.Names.Count);
Assert.AreEqual("sqrt", f.FunctionName.Names[0].Token.PureToken.LiteralText);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseFunctionHandleWithCompoundName()
{
var text = "@a.b.c";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<NamedFunctionHandleNode>(actual);
var f = (NamedFunctionHandleNode) actual;
Assert.AreEqual(3, f.FunctionName.Names.Count);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseLambda()
{
var text = "@(x, y) x + y";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<LambdaNode>(actual);
var f = (LambdaNode) actual;
Assert.AreEqual(2, f.Input.Parameters.Parameters.Count);
Assert.IsInstanceOf<BinaryOperationExpressionNode>(f.Body);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseTildeAsResultReplacement()
{
var text = "[a, ~, b]";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<ArrayLiteralExpressionNode>(actual);
var f = (ArrayLiteralExpressionNode) actual;
Assert.AreEqual(3, f.Elements.Elements.Count);
Assert.IsInstanceOf<UnaryPrefixOperationExpressionNode>(f.Elements.Elements[1]);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseTildeAsFunctionInputReplacement()
{
var text = "function a(b, ~, c) end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<FunctionDeclarationNode>(actual);
var f = (FunctionDeclarationNode) actual;
Assert.AreEqual(3, f.InputDescription.Parameters.Parameters.Count);
CollectionAssert.AreEqual(new[] { "b", "~", "c" }, f.InputDescription.Parameters.Parameters.Select(p => (p as TokenNode).Token.PureToken.LiteralText));
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseDotTranspose()
{
var text = "a.'";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<UnaryPostfixOperationExpressionNode>(actual);
var e = (UnaryPostfixOperationExpressionNode) actual;
Assert.AreEqual(TokenKind.DotTranspose, e.Operation.Token.Kind);
Assert.AreEqual("a", (e.Operand as IdentifierNameNode)?.Token.PureToken.LiteralText);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseDoubleQuotedStringLiteral()
{
var text = "\"some string\"";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<DoubleQuotedStringLiteralNode>(actual);
var s = (DoubleQuotedStringLiteralNode) actual;
Assert.AreEqual("some string", s.Token.PureToken.Value);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseTryCatchStatement()
{
var text = "try a = b catch c = d end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<TryCatchStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseElseif()
{
var text = @"if a == 1
f()
elseif a == 2
g()
elseif a == 3
h()
end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<IfStatementNode>(actual);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseBaseClassInvokation()
{
var text = "a@b.c.d(e, f)";
var sut = CreateParser(text);
var actual = sut.ParseExpression();
Assert.IsInstanceOf<BaseClassInvokationNode>(actual);
var e = (BaseClassInvokationNode) actual;
Assert.AreEqual("a", e.MethodName.Token.PureToken.LiteralText);
Assert.IsInstanceOf<FunctionCallExpressionNode>(e.BaseClassNameAndArguments);
Assert.AreEqual(text, actual.FullText);
}
[Test]
public void ParseFunctionWithEmptyOutputsList()
{
var text = "function [] = a(b) end";
var sut = CreateParser(text);
var actual = sut.ParseStatement();
Assert.IsInstanceOf<FunctionDeclarationNode>(actual);
var f = (FunctionDeclarationNode) actual;
Assert.AreEqual(0, f.OutputDescription.Outputs.Count);
Assert.AreEqual(text, actual.FullText);
} }
} }
} }

7
Parser.Tests/Parser.Tests.csproj
View File

@ -1,6 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>netcoreapp2.0</TargetFramework> <TargetFramework>netcoreapp2.1</TargetFramework>
<IsPackable>false</IsPackable> <IsPackable>false</IsPackable>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
@ -9,9 +9,6 @@
<PackageReference Include="NUnit3TestAdapter" Version="3.10.0" /> <PackageReference Include="NUnit3TestAdapter" Version="3.10.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Parser\Parser.csproj"> <ProjectReference Include="..\Parser\Parser.csproj" />
<Project>{B20EDC10-E6E6-4430-8527-B95206DEF941}</Project>
<Name>Parser</Name>
</ProjectReference>
</ItemGroup> </ItemGroup>
</Project> </Project>

2
Lexer.Tests/TextWindowShould.cs → Parser.Tests/TextWindowShould.cs
View File

@ -1,6 +1,6 @@
using NUnit.Framework; using NUnit.Framework;
namespace Lexer.Tests namespace Parser.Tests
{ {
[TestFixture] [TestFixture]
public class TestWindowShould public class TestWindowShould

2
Lexer.Tests/TextWindowWithNullShould.cs → Parser.Tests/TextWindowWithNullShould.cs
View File

@ -1,6 +1,6 @@
using NUnit.Framework; using NUnit.Framework;
namespace Lexer.Tests namespace Parser.Tests
{ {
[TestFixture] [TestFixture]
public class TestWindowWithNullShould public class TestWindowWithNullShould

145
Parser/ChildNodesAndTokensList.cs Normal file
View File

@ -0,0 +1,145 @@
using System;
using System.Collections;
using System.Collections.Generic;
using Parser.Internal;
namespace Parser
{
public class ChildNodesAndTokensList : IReadOnlyList<SyntaxNodeOrToken>
{
private readonly SyntaxNode _node;
private readonly int _count;
internal ChildNodesAndTokensList(SyntaxNode node)
{
_node = node;
_count = CountChildNodes(node._green);
}
private int CountChildNodes(GreenNode green)
{
var counter = 0;
for (var i = 0; i < green.Slots; i++)
{
var child = green.GetSlot(i);
if (child == null)
{
continue;
}
if (child.IsList)
{
counter += child.Slots;
}
else
{
counter++;
}
}
return counter;
}
public IEnumerator<SyntaxNodeOrToken> GetEnumerator()
{
return new Enumerator(this);
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public int Count => _count;
public SyntaxNodeOrToken this[int index]
{
get
{
if (index < 0 || index >= Count)
{
throw new IndexOutOfRangeException();
}
return ThisInternal(index);
}
}
internal SyntaxNodeOrToken ThisInternal(int index)
{
var currentSlotIndex = 0;
GreenNode currentSlot = null;
while (true)
{
currentSlot = _node._green.GetSlot(currentSlotIndex);
if (currentSlot == null)
{
currentSlotIndex++;
continue;
}
var nodesInCurrentSlot = currentSlot.IsList ? currentSlot.Slots : 1;
if (index < nodesInCurrentSlot)
{
if (currentSlot.IsList)
{
var listSlot = _node.GetNode(currentSlotIndex);
var red = listSlot.GetNode(index);
if (red != null)
{
return red;
}
// this is a token
return new SyntaxToken(listSlot, listSlot._green.GetSlot(index));
}
else
{
var red = _node.GetNode(currentSlotIndex);
if (red != null)
{
return red;
}
// this is a token
return new SyntaxToken(_node, _node._green.GetSlot(currentSlotIndex));
}
}
index -= nodesInCurrentSlot;
currentSlotIndex++;
}
}
private struct Enumerator : IEnumerator<SyntaxNodeOrToken>
{
private int _index;
private readonly ChildNodesAndTokensList _list;
private readonly int _count;
internal Enumerator(ChildNodesAndTokensList list)
{
_index = -1;
_list = list;
_count = _list.Count;
}
public bool MoveNext()
{
_index++;
return _index < _count;
}
public void Reset()
{
throw new NotImplementedException();
}
public SyntaxNodeOrToken Current => _list[_index];
object IEnumerator.Current => Current;
public void Dispose()
{
}
}
}
}

4
Lexer/ILexer.cs → Parser/ILexer.cs
View File

@ -1,8 +1,8 @@
using System.Collections.Generic; using System.Collections.Generic;
namespace Lexer namespace Parser
{ {
public interface ILexer<T> where T : class public interface ILexer<T>
{ {
T NextToken(); T NextToken();
List<T> ParseAll(); List<T> ParseAll();

4
Lexer/ITextWindow.cs → Parser/ITextWindow.cs
View File

@ -1,4 +1,4 @@
namespace Lexer namespace Parser
{ {
public interface ITextWindow public interface ITextWindow
{ {
@ -10,6 +10,6 @@
char GetAndConsumeChar(); char GetAndConsumeChar();
string GetAndConsumeChars(int n); string GetAndConsumeChars(int n);
int CharactersLeft(); int CharactersLeft();
IPosition Position { get; } Position Position { get; }
} }
} }

217
Parser/Internal/GreenNode.cs Normal file
View File

@ -0,0 +1,217 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
namespace Parser.Internal
{
internal abstract class GreenNode
{
public TokenKind Kind { get; }
public int Slots { get; }
public abstract GreenNode GetSlot(int i);
public GreenNode(TokenKind kind)
{
Kind = kind;
}
public GreenNode(TokenKind kind, int slots)
{
Kind =kind;
Slots = slots;
}
internal abstract Parser.SyntaxNode CreateRed(Parser.SyntaxNode parent);
public virtual string Text
{
get
{
var sb = new StringBuilder();
using (var writer = new System.IO.StringWriter(sb, CultureInfo.InvariantCulture))
{
WriteTo(writer, leading: false, trailing: false);
}
return sb.ToString();
}
}
protected bool _isMissing = false;
internal bool IsMissing => _isMissing;
public override string ToString()
{
return base.ToString();
}
private void WriteTo(TextWriter writer, bool leading, bool trailing)
{
var stack = new Stack<(GreenNode node, bool leading, bool trailing)>();
stack.Push((this, leading, trailing));
WriteStackTo(writer, stack);
}
public virtual bool IsToken => false;
public virtual bool IsTrivia => false;
public virtual bool IsNode => true;
public virtual bool IsList => false;
public virtual void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
throw new NotImplementedException();
}
public virtual void WriteTriviaTo(TextWriter writer)
{
throw new NotImplementedException();
}
private int GetFirstNonNullChildIndex()
{
for (var i = 0; i < Slots; i++)
{
if (GetSlot(i) != null)
{
return i;
}
}
return Slots;
}
private int GetLastNonNullChildIndex()
{
for (var i = Slots - 1; i >= 0; i--)
{
if (GetSlot(i) != null)
{
return i;
}
}
return -1;
}
private GreenNode GetFirstTerminal()
{
var current = this;
while (true)
{
GreenNode next = null;
if (current.Slots == 0)
{
return current;
}
for (var i = 0; i < current.Slots; i++)
{
var child = current.GetSlot(i);
if (child == null)
{
continue;
}
next = child;
break;
}
if (next == null)
{
return null;
}
current = next;
}
}
private GreenNode GetLastTerminal()
{
var current = this;
while (true)
{
GreenNode next = null;
if (current.Slots == 0)
{
return current;
}
for (var i = current.Slots - 1; i >= 0; i--)
{
var child = current.GetSlot(i);
if (child == null)
{
continue;
}
next = child;
break;
}
if (next == null)
{
return null;
}
current = next;
}
}
public virtual IReadOnlyList<SyntaxTrivia> LeadingTrivia => GetFirstTerminal()?.LeadingTriviaCore ?? new List<SyntaxTrivia>();
public virtual IReadOnlyList<SyntaxTrivia> TrailingTrivia => GetLastTerminal()?.TrailingTriviaCore ?? new List<SyntaxTrivia>();
public abstract IReadOnlyList<SyntaxTrivia> LeadingTriviaCore { get; }
public abstract IReadOnlyList<SyntaxTrivia> TrailingTriviaCore { get; }
public virtual string FullText
{
get
{
var sb = new StringBuilder();
using (var writer = new System.IO.StringWriter(sb, CultureInfo.InvariantCulture))
{
WriteTo(writer, leading: true, trailing: true);
}
return sb.ToString();
}
}
private void WriteStackTo(TextWriter writer, Stack<(GreenNode node, bool leading, bool trailing)> stack)
{
while (stack.Count > 0)
{
var currentTriple = stack.Pop();
if (currentTriple.node.IsToken)
{
currentTriple.node.WriteTokenTo(writer, currentTriple.leading, currentTriple.trailing);
} else if (currentTriple.node.IsTrivia)
{
currentTriple.node.WriteTriviaTo(writer);
}
else
{
var firstIndex = currentTriple.node.GetFirstNonNullChildIndex();
var lastIndex = currentTriple.node.GetLastNonNullChildIndex();
for (var i = lastIndex; i >= firstIndex; i--)
{
var child = currentTriple.node.GetSlot(i);
if (child != null)
{
stack.Push((
child,
currentTriple.leading || i != firstIndex,
currentTriple.trailing || i != lastIndex));
}
}
}
}
}
}
}

570
Lexer/MLexer.cs → Parser/Internal/MLexerGreen.cs
View File

@ -1,46 +1,46 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
namespace Lexer namespace Parser.Internal
{ {
public class MLexer : ILexer<Token> internal class MLexerGreen : ILexer<(SyntaxToken, Position)>
{ {
private class TokenInfo
{
public TokenKind Kind { get; set; }
public string Text { get; set; }
public string StringValue { get; set; }
public double DoubleValue { get; set; }
}
private ITextWindow Window { get; } private ITextWindow Window { get; }
private Token LastToken { get; set; } private SyntaxToken LastToken { get; set; }
private int TokensSinceNewLine { get; set; } private int TokensSinceNewLine { get; set; }
private PureTokenFactory PureTokenFactory { get; }
private Stack<TokenKind> TokenStack { get; } private Stack<TokenKind> TokenStack { get; }
public MLexer(ITextWindow window, PureTokenFactory pureTokenFactory) public MLexerGreen(ITextWindow window)
{ {
Window = window; Window = window;
PureTokenFactory = pureTokenFactory;
TokenStack = new Stack<TokenKind>(); TokenStack = new Stack<TokenKind>();
} }
private static bool IsEolOrEof(char c) private SyntaxTrivia LexComment()
{
return c == '\n' || c == '\r' || c == '\0';
}
private Trivia LexComment()
{ {
if (TokensSinceNewLine == 0 && Window.PeekChar(1) == '{') if (TokensSinceNewLine == 0 && Window.PeekChar(1) == '{')
{ {
return LexMultilineComment(); return LexMultilineComment();
} }
var n = 1; var n = 1;
while (!IsEolOrEof(Window.PeekChar(n))) while (!SyntaxFacts.IsEolOrEof(Window.PeekChar(n)))
{ {
n++; n++;
} }
return new Trivia(TriviaType.Comment, Window.GetAndConsumeChars(n)); return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
} }
private Trivia LexMultilineComment() private SyntaxTrivia LexMultilineComment()
{ {
var n = 2; var n = 2;
var metPercentSign = false; var metPercentSign = false;
@ -50,7 +50,7 @@ namespace Lexer
var c = Window.PeekChar(n); var c = Window.PeekChar(n);
if (c == '\0') if (c == '\0')
{ {
throw new ParsingException($"Unexpected end of file while parsing multi-line comment."); throw new ParsingException("Unexpected end of file while parsing multi-line comment.");
} }
if (c == '\n') if (c == '\n')
@ -58,58 +58,62 @@ namespace Lexer
atFirstLine = false; atFirstLine = false;
} }
if (atFirstLine && !IsWhitespace(c)) // this is a one-line comment if (atFirstLine && !SyntaxFacts.IsWhitespace(c)) // this is a one-line comment
{ {
while (!IsEolOrEof(Window.PeekChar(n))) while (!SyntaxFacts.IsEolOrEof(Window.PeekChar(n)))
{ {
n++; n++;
} }
return new Trivia(TriviaType.Comment, Window.GetAndConsumeChars(n)); return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
} }
if (metPercentSign && c == '}') if (metPercentSign && c == '}')
{ {
return new Trivia(TriviaType.MultiLineComment, Window.GetAndConsumeChars(n+1)); return TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n+1));
} }
if (c == '%') metPercentSign = c == '%';
{
metPercentSign = true;
}
else
{
metPercentSign = false;
}
n++; n++;
} }
} }
private List<Trivia> LexCommentAfterDotDotDot() private List<SyntaxTrivia> LexCommentAfterDotDotDot()
{ {
var n = 0; var n = 0;
while (!IsEolOrEof(Window.PeekChar(n))) while (!SyntaxFacts.IsEolOrEof(Window.PeekChar(n)))
{ {
n++; n++;
} }
var comment = new Trivia(TriviaType.Comment, Window.GetAndConsumeChars(n)); var comment = TokenFactory.CreateTrivia(TokenKind.Comment, Window.GetAndConsumeChars(n));
var result = new List<Trivia> { comment }; var result = new List<SyntaxTrivia> { comment };
var character = Window.PeekChar(); var character = Window.PeekChar();
if (character == '\n' || character == '\r') if (character == '\n' || character == '\r')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
result.Add(new Trivia(TriviaType.Whitespace, character.ToString())); result.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, character.ToString()));
} }
return result; return result;
} }
private List<Trivia> LexTrivia(bool isTrailing) private List<SyntaxTrivia> LexTrivia(bool isTrailing)
{ {
var triviaList = new List<Trivia>(); var triviaList = new List<SyntaxTrivia>();
var whiteSpaceCache = new StringBuilder(); var whitespaceCache = new StringBuilder();
void FlushWhitespaceCache()
{
if (whitespaceCache.Length > 0)
{
triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Whitespace, whitespaceCache.ToString()));
}
whitespaceCache.Clear();
}
while (true) while (true)
{ {
var character = Window.PeekChar(); var character = Window.PeekChar();
@ -118,18 +122,13 @@ namespace Lexer
case ' ': case ' ':
case '\t': case '\t':
Window.ConsumeChar(); Window.ConsumeChar();
whiteSpaceCache.Append(character); whitespaceCache.Append(character);
break; break;
case '\r': case '\r':
case '\n': case '\n':
if (whiteSpaceCache.Length > 0) FlushWhitespaceCache();
{
triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
}
whiteSpaceCache.Clear();
Window.ConsumeChar(); Window.ConsumeChar();
triviaList.Add(new Trivia(TriviaType.NewLine, character.ToString())); triviaList.Add(TokenFactory.CreateTrivia(TokenKind.Newline, character.ToString()));
if (isTrailing) if (isTrailing)
{ {
return triviaList; return triviaList;
@ -137,39 +136,23 @@ namespace Lexer
break; break;
case '%': case '%':
if (whiteSpaceCache.Length > 0) FlushWhitespaceCache();
{
triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
}
whiteSpaceCache.Clear();
triviaList.Add(LexComment()); triviaList.Add(LexComment());
break; break;
case '.': case '.':
if (Window.PeekChar(1) == '.' && Window.PeekChar(2) == '.') if (Window.PeekChar(1) == '.' && Window.PeekChar(2) == '.')
{ {
if (whiteSpaceCache.Length > 0) FlushWhitespaceCache();
{
triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
}
whiteSpaceCache.Clear();
triviaList.AddRange(LexCommentAfterDotDotDot()); triviaList.AddRange(LexCommentAfterDotDotDot());
} }
else else
{ {
if (whiteSpaceCache.Length > 0) FlushWhitespaceCache();
{
triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
}
return triviaList; return triviaList;
} }
break; break;
default: default:
if (whiteSpaceCache.Length > 0) FlushWhitespaceCache();
{
triviaList.Add(new Trivia(TriviaType.Whitespace, whiteSpaceCache.ToString()));
}
return triviaList; return triviaList;
} }
} }
@ -180,7 +163,8 @@ namespace Lexer
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c == '_'); return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || (c == '_');
} }
private PureToken ContinueParsingIdentifier()
private bool ContinueLexingIdentifier(ref TokenInfo tokenInfo)
{ {
var n = 1; var n = 1;
while (IsLetterOrDigitOrUnderscore(Window.PeekChar(n))) while (IsLetterOrDigitOrUnderscore(Window.PeekChar(n)))
@ -189,7 +173,28 @@ namespace Lexer
} }
var identifier = Window.GetAndConsumeChars(n); var identifier = Window.GetAndConsumeChars(n);
return PureTokenFactory.CreateIdentifier(identifier); tokenInfo.Kind = TokenKind.Identifier;
tokenInfo.Text = identifier;
return true;
}
private bool ContinueParsingUnquotedStringLiteral(ref TokenInfo tokenInfo)
{
var n = 0;
while (true)
{
var c = Window.PeekChar(n);
if (c == ' ' || c == '\n' || c == '\0')
{
var literal = Window.GetAndConsumeChars(n);
tokenInfo.Kind = TokenKind.UnquotedStringLiteral;
tokenInfo.Text = literal;
tokenInfo.StringValue = literal;
return true;
}
n++;
}
} }
private enum NumberParsingState private enum NumberParsingState
@ -203,22 +208,7 @@ namespace Lexer
DigitsAfterE DigitsAfterE
} }
private static bool IsDigit(char c) private bool ContinueLexingNumber(ref TokenInfo tokenInfo)
{
return c >= '0' && c <= '9';
}
private static bool IsDigitOrDot(char c)
{
return c == '.' || (c >= '0' && c <= '9');
}
private static bool IsWhitespace(char c)
{
return c == ' ' || c == '\t' || c == '\n';
}
private PureToken? ContinueParsingNumber()
{ {
var state = NumberParsingState.Start; var state = NumberParsingState.Start;
var n = 0; var n = 0;
@ -231,7 +221,7 @@ namespace Lexer
switch (state) switch (state)
{ {
case NumberParsingState.Start: case NumberParsingState.Start:
if (IsDigitOrDot(c)) if (SyntaxFacts.IsDigitOrDot(c))
{ {
state = NumberParsingState.DigitsBeforeDot; state = NumberParsingState.DigitsBeforeDot;
} }
@ -241,7 +231,7 @@ namespace Lexer
} }
break; break;
case NumberParsingState.DigitsBeforeDot: case NumberParsingState.DigitsBeforeDot:
if (IsDigit(c)) if (SyntaxFacts.IsDigit(c))
{ {
} }
else if (c == '.') else if (c == '.')
@ -258,7 +248,7 @@ namespace Lexer
} }
break; break;
case NumberParsingState.AfterDot: case NumberParsingState.AfterDot:
if (IsDigit(c)) if (SyntaxFacts.IsDigit(c))
{ {
state = NumberParsingState.DigitsAfterDot; state = NumberParsingState.DigitsAfterDot;
} }
@ -266,7 +256,7 @@ namespace Lexer
{ {
state = NumberParsingState.AfterE; state = NumberParsingState.AfterE;
} }
else if (IsWhitespace(c) || c == ';' || c == ']' || c == ')' || c == '}') else if (SyntaxFacts.IsWhitespace(c) || c == ';' || c == ']' || c == ')' || c == '}')
{ {
success = true; success = true;
} }
@ -282,7 +272,7 @@ namespace Lexer
break; break;
case NumberParsingState.DigitsAfterDot: case NumberParsingState.DigitsAfterDot:
if (IsDigit(c)) if (SyntaxFacts.IsDigit(c))
{ {
} }
else if (c == 'e' || c == 'E') else if (c == 'e' || c == 'E')
@ -296,7 +286,7 @@ namespace Lexer
break; break;
case NumberParsingState.AfterE: case NumberParsingState.AfterE:
if (IsDigit(c)) if (SyntaxFacts.IsDigit(c))
{ {
state = NumberParsingState.DigitsAfterE; state = NumberParsingState.DigitsAfterE;
} }
@ -311,7 +301,7 @@ namespace Lexer
break; break;
case NumberParsingState.SignAfterE: case NumberParsingState.SignAfterE:
if (IsDigit(c)) if (SyntaxFacts.IsDigit(c))
{ {
state = NumberParsingState.DigitsAfterE; state = NumberParsingState.DigitsAfterE;
} }
@ -322,7 +312,7 @@ namespace Lexer
break; break;
case NumberParsingState.DigitsAfterE: case NumberParsingState.DigitsAfterE:
if (IsDigit(c)) if (SyntaxFacts.IsDigit(c))
{ {
} }
else else
@ -364,27 +354,36 @@ namespace Lexer
n++; n++;
} }
var s = Window.GetAndConsumeChars(n); var s = Window.GetAndConsumeChars(n);
return PureTokenFactory.CreateNumberLiteral(s);
tokenInfo.Kind = TokenKind.NumberLiteral;
tokenInfo.Text = s;
return true;
} }
return null; return false;
} }
private PureToken ContinueParsingStringLiteral() private bool ContinueLexingGeneralStringLiteral(ref TokenInfo tokenInfo, char quote)
{ {
Window.ConsumeChar(); Window.ConsumeChar();
var pieces = new List<string>(); var textBuilder = new StringBuilder();
textBuilder.Append(quote);
var valueBuilder = new StringBuilder();
var n = 0; var n = 0;
while (true) { while (true)
if (Window.PeekChar(n) == '\'') {
if (Window.PeekChar(n) == quote)
{ {
if (Window.PeekChar(n + 1) == '\'') if (Window.PeekChar(n + 1) == quote)
{ {
var piece = Window.GetAndConsumeChars(n); var piece = Window.GetAndConsumeChars(n);
pieces.Add(piece); textBuilder.Append(piece);
valueBuilder.Append(piece);
Window.ConsumeChar(); Window.ConsumeChar();
Window.ConsumeChar(); Window.ConsumeChar();
pieces.Add("'"); textBuilder.Append(quote);
textBuilder.Append(quote);
valueBuilder.Append(quote);
n = -1; n = -1;
} }
else else
@ -392,7 +391,7 @@ namespace Lexer
break; break;
} }
} }
if (IsEolOrEof(Window.PeekChar(n))) if (SyntaxFacts.IsEolOrEof(Window.PeekChar(n)))
{ {
throw new ParsingException("Unfinished string literal."); throw new ParsingException("Unfinished string literal.");
} }
@ -400,83 +399,37 @@ namespace Lexer
} }
var lastPiece = Window.GetAndConsumeChars(n); var lastPiece = Window.GetAndConsumeChars(n);
pieces.Add(lastPiece); textBuilder.Append(lastPiece);
var total = string.Join("", pieces); valueBuilder.Append(lastPiece);
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreateStringLiteral(total); textBuilder.Append(quote);
tokenInfo.Text = textBuilder.ToString();
tokenInfo.StringValue = valueBuilder.ToString();
return true;
} }
private PureToken ContinueParsingDoubleQuotedStringLiteral() private bool ContinueLexingStringLiteral(ref TokenInfo tokenInfo)
{ {
Window.ConsumeChar(); ContinueLexingGeneralStringLiteral(ref tokenInfo, '\'');
var n = 0; tokenInfo.Kind = TokenKind.StringLiteral;
var pieces = new List<string>(); return true;
while (true)
{
if (Window.PeekChar(n) == '"')
{
if (Window.PeekChar(n + 1) == '"')
{
var piece = Window.GetAndConsumeChars(n);
pieces.Add(piece);
Window.ConsumeChar();
Window.ConsumeChar();
pieces.Add("\"");
n = -1;
}
else
{
break;
}
}
if (IsEolOrEof(Window.PeekChar(n)))
{
throw new ParsingException("Unfinished double-quoted string literal.");
}
n++;
}
var lastPiece = Window.GetAndConsumeChars(n);
pieces.Add(lastPiece);
var total = string.Join("", pieces);
Window.ConsumeChar();
return PureTokenFactory.CreateDoubleQuotedStringLiteral(total);
} }
private PureToken ContinueParsingUnquotedStringLiteral() private bool ContinueLexingDoubleQuotedStringLiteral(ref TokenInfo tokenInfo)
{ {
var n = 0; ContinueLexingGeneralStringLiteral(ref tokenInfo, '"');
while (true) tokenInfo.Kind = TokenKind.StringLiteral;
{ return true;
var c = Window.PeekChar(n);
if (c == ' ' || c == '\n' || c == '\0')
{
var literal = Window.GetAndConsumeChars(n);
return PureTokenFactory.CreateUnquotedStringLiteral(literal);
}
n++;
}
} }
private static readonly HashSet<string> Keywords; private bool LexTokenWithoutTrivia(List<SyntaxTrivia> leadingTrivia, ref TokenInfo tokenInfo)
static MLexer()
{
Keywords = new HashSet<string>
{
"for", "if", "function", "while", "case", "try", "catch", "end",
"switch", "classdef", "elseif", "persistent", "else"
};
}
private PureToken LexTokenWithoutTrivia(List<Trivia> leadingTrivia)
{ {
var character = Window.PeekChar(); var character = Window.PeekChar();
if (character == '\0') if (character == '\0')
{ {
return PureTokenFactory.CreateEndOfFileToken(); tokenInfo.Kind = TokenKind.EndOfFile;
tokenInfo.Text = "";
return true;
} }
if (TokensSinceNewLine == 1 if (TokensSinceNewLine == 1
@ -485,16 +438,17 @@ namespace Lexer
&& LastToken.TrailingTrivia.Any() && LastToken.TrailingTrivia.Any()
&& character != '=' && character != '='
&& character != '(' && character != '('
&& !Keywords.Contains(LastToken.PureToken.LiteralText)) && !SyntaxFacts.Keywords.Contains(LastToken.Text))
{ {
return ContinueParsingUnquotedStringLiteral(); return ContinueParsingUnquotedStringLiteral(ref tokenInfo);
} }
if (LastToken?.Kind == TokenKind.UnquotedStringLiteral if (LastToken?.Kind == TokenKind.UnquotedStringLiteral
&& !TokenStack.Any() && !TokenStack.Any()
&& TokensSinceNewLine > 0) && TokensSinceNewLine > 0)
{ {
return ContinueParsingUnquotedStringLiteral(); return ContinueParsingUnquotedStringLiteral(ref tokenInfo);
} }
switch (character) switch (character)
{ {
case 'a': case 'a':
@ -549,7 +503,7 @@ namespace Lexer
case 'X': case 'X':
case 'Y': case 'Y':
case 'Z': case 'Z':
return ContinueParsingIdentifier(); return ContinueLexingIdentifier(ref tokenInfo);
case '0': case '0':
case '1': case '1':
case '2': case '2':
@ -560,31 +514,35 @@ namespace Lexer
case '7': case '7':
case '8': case '8':
case '9': case '9':
var possiblyNumberToken = ContinueParsingNumber(); var parsedNumber = ContinueLexingNumber(ref tokenInfo);
if (possiblyNumberToken == null) if (!parsedNumber)
{ {
throw new ParsingException($"Unexpected character \"{Window.PeekChar()}\" while parsing a number"); throw new ParsingException($"Unexpected character \"{Window.PeekChar()}\" while parsing a number");
} }
return true;
return (PureToken)possiblyNumberToken;
case '=': case '=':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Equality); tokenInfo.Kind = TokenKind.Equality;
}
else
{
tokenInfo.Kind = TokenKind.Assignment;
} }
return PureTokenFactory.CreatePunctuation(TokenKind.Assignment); return true;
case '.': case '.':
if (IsDigit(Window.PeekChar(1))) if (SyntaxFacts.IsDigit(Window.PeekChar(1)))
{ {
var possiblyNumberToken2 = ContinueParsingNumber(); var possiblyNumberToken2 = ContinueLexingNumber(ref tokenInfo);
if (possiblyNumberToken2 == null) if (!possiblyNumberToken2)
{ {
throw new ParsingException($"Unexpected character \"{Window.PeekChar()}\" while parsing a number"); throw new ParsingException($"Unexpected character \"{Window.PeekChar()}\" while parsing a number");
} }
return (PureToken)possiblyNumberToken2;
return true;
} }
Window.ConsumeChar(); Window.ConsumeChar();
var c = Window.PeekChar(); var c = Window.PeekChar();
@ -592,113 +550,164 @@ namespace Lexer
{ {
case '*': case '*':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.DotMultiply); tokenInfo.Kind = TokenKind.DotMultiply;
break;
case '/': case '/':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.DotDivide); tokenInfo.Kind = TokenKind.DotDivide;
break;
case '^': case '^':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.DotPower); tokenInfo.Kind = TokenKind.DotPower;
break;
case '\\': case '\\':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.DotBackslash); tokenInfo.Kind = TokenKind.DotBackslash;
break;
case '\'': case '\'':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.DotTranspose); tokenInfo.Kind = TokenKind.DotTranspose;
break;
default: default:
return PureTokenFactory.CreatePunctuation(TokenKind.Dot); tokenInfo.Kind = TokenKind.Dot;
break;
} }
return true;
case '(': case '(':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.OpeningBracket); tokenInfo.Kind = TokenKind.OpeningBracket;
return true;
case ')': case ')':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.ClosingBracket); tokenInfo.Kind = TokenKind.ClosingBracket;
return true;
case '[': case '[':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.OpeningSquareBracket); tokenInfo.Kind = TokenKind.OpeningSquareBracket;
return true;
case ']': case ']':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.ClosingSquareBracket); tokenInfo.Kind = TokenKind.ClosingSquareBracket;
return true;
case '{': case '{':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.OpeningBrace); tokenInfo.Kind = TokenKind.OpeningBrace;
return true;
case '}': case '}':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.ClosingBrace); tokenInfo.Kind = TokenKind.ClosingBrace;
return true;
case ',': case ',':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Comma); tokenInfo.Kind = TokenKind.Comma;
return true;
case ';': case ';':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Semicolon); tokenInfo.Kind = TokenKind.Semicolon;
return true;
case '&': case '&':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '&') if (Window.PeekChar() == '&')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.LogicalAnd); tokenInfo.Kind = TokenKind.LogicalAnd;
} }
return PureTokenFactory.CreatePunctuation(TokenKind.BitwiseAnd); else
{
tokenInfo.Kind = TokenKind.BitwiseAnd;
}
return true;
case '|': case '|':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '|') if (Window.PeekChar() == '|')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.LogicalOr); tokenInfo.Kind = TokenKind.LogicalOr;
} }
return PureTokenFactory.CreatePunctuation(TokenKind.BitwiseOr); else
{
tokenInfo.Kind = TokenKind.BitwiseOr;
}
return true;
case '<': case '<':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.LessOrEqual); tokenInfo.Kind = TokenKind.LessOrEqual;
} }
return PureTokenFactory.CreatePunctuation(TokenKind.Less); else
{
tokenInfo.Kind = TokenKind.Less;
}
return true;
case '>': case '>':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.GreaterOrEqual); tokenInfo.Kind = TokenKind.GreaterOrEqual;
} }
return PureTokenFactory.CreatePunctuation(TokenKind.Greater); else
{
tokenInfo.Kind = TokenKind.Greater;
}
return true;
case '~': case '~':
Window.ConsumeChar(); Window.ConsumeChar();
if (Window.PeekChar() == '=') if (Window.PeekChar() == '=')
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Inequality); tokenInfo.Kind = TokenKind.Inequality;
} }
return PureTokenFactory.CreatePunctuation(TokenKind.Not); else
{
tokenInfo.Kind = TokenKind.Not;
}
return true;
case '+': case '+':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Plus); tokenInfo.Kind = TokenKind.Plus;
return true;
case '-': case '-':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Minus); tokenInfo.Kind = TokenKind.Minus;
return true;
case '*': case '*':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Multiply); tokenInfo.Kind = TokenKind.Multiply;
return true;
case '/': case '/':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Divide); tokenInfo.Kind = TokenKind.Divide;
return true;
case '\\': case '\\':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Backslash); tokenInfo.Kind = TokenKind.Backslash;
return true;
case '^': case '^':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Power); tokenInfo.Kind = TokenKind.Power;
return true;
case '@': case '@':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.At); tokenInfo.Kind = TokenKind.At;
return true;
case ':': case ':':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Colon); tokenInfo.Kind = TokenKind.Colon;
return true;
case '?': case '?':
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.QuestionMark); tokenInfo.Kind = TokenKind.QuestionMark;
return true;
case '\'': case '\'':
if (LastToken != null && if (LastToken != null &&
(LastToken.Kind == TokenKind.ClosingBrace (LastToken.Kind == TokenKind.ClosingBrace
@ -709,14 +718,16 @@ namespace Lexer
if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0) if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0)
{ {
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.Transpose); tokenInfo.Kind = TokenKind.Transpose;
return true;
} }
} }
return ContinueParsingStringLiteral(); return ContinueLexingStringLiteral(ref tokenInfo);
case '"': case '"':
return ContinueParsingDoubleQuotedStringLiteral(); return ContinueLexingDoubleQuotedStringLiteral(ref tokenInfo);
case '\0': case '\0':
return PureTokenFactory.CreateEndOfFileToken(); tokenInfo.Kind = TokenKind.EndOfFile;
return true;
default: default:
throw new ParsingException( throw new ParsingException(
$"Unknown symbol \"{character}\" at {Window.Position}." $"Unknown symbol \"{character}\" at {Window.Position}."
@ -724,53 +735,14 @@ namespace Lexer
} }
} }
private bool IsOpeningToken(TokenKind tokenKind) public (SyntaxToken, Position) NextToken()
{
switch (tokenKind)
{
case TokenKind.OpeningBrace:
case TokenKind.OpeningBracket:
case TokenKind.OpeningSquareBracket:
return true;
default:
return false;
}
}
private bool IsClosingToken(TokenKind tokenKind)
{
switch (tokenKind)
{
case TokenKind.ClosingBrace:
case TokenKind.ClosingBracket:
case TokenKind.ClosingSquareBracket:
return true;
default:
return false;
}
}
private TokenKind? OpeningFromClosing(TokenKind tokenKind)
{
switch (tokenKind)
{
case TokenKind.ClosingBrace:
return TokenKind.OpeningBrace;
case TokenKind.ClosingBracket:
return TokenKind.OpeningBracket;
case TokenKind.ClosingSquareBracket:
return TokenKind.OpeningSquareBracket;
default:
return null;
}
}
public Token NextToken()
{ {
var leadingTrivia = LexTrivia(false); var leadingTrivia = LexTrivia(false);
var token = LexTokenWithoutTrivia(leadingTrivia); var position = Window.Position;
var tokenInfo = new TokenInfo();
LexTokenWithoutTrivia(leadingTrivia, ref tokenInfo);
var trailingTrivia = LexTrivia(true); var trailingTrivia = LexTrivia(true);
if (trailingTrivia.Where(t => t.Type == TriviaType.NewLine).Any()) if (trailingTrivia.Any(t => t.Kind == TokenKind.Newline))
{ {
TokensSinceNewLine = 0; TokensSinceNewLine = 0;
} }
@ -779,57 +751,103 @@ namespace Lexer
TokensSinceNewLine++; TokensSinceNewLine++;
} }
if (IsOpeningToken(token.Kind)) if (SyntaxFacts.IsOpeningToken(tokenInfo.Kind))
{ {
TokenStack.Push(token.Kind); TokenStack.Push(tokenInfo.Kind);
} }
if (IsClosingToken(token.Kind)) if (SyntaxFacts.IsClosingToken(tokenInfo.Kind))
{ {
if (TokenStack.TryPeek(out var t)) if (TokenStack.Count > 0)
{ {
if (t == OpeningFromClosing(token.Kind)) var t = TokenStack.Peek();
if (t == SyntaxFacts.OpeningFromClosing(tokenInfo.Kind))
{ {
TokenStack.Pop(); TokenStack.Pop();
} }
else else
{ {
throw new ParsingException($"Unmatched \"{token.LiteralText}\" at {token.Position}."); throw new ParsingException($"Unmatched \"{tokenInfo.Text}\" at {Window.Position}.");
} }
} }
else else
{ {
throw new ParsingException($"Unmatched \"{token.LiteralText}\" at {token.Position}."); throw new ParsingException($"Unmatched \"{tokenInfo.Text}\" at {Window.Position}.");
} }
} }
if (token.Kind == TokenKind.EndOfFile if (tokenInfo.Kind == TokenKind.EndOfFile
&& TokenStack.Any()) && TokenStack.Any())
{ {
throw new ParsingException($"Unmatched \"{TokenStack.Pop()}\" by the end of file."); throw new ParsingException($"Unmatched \"{TokenStack.Pop()}\" by the end of file.");
} }
var result = new Token(token, leadingTrivia, trailingTrivia); var result = Create(
tokenInfo,
leadingTrivia,
trailingTrivia);
LastToken = result; LastToken = result;
return result; return (result, position);
} }
public List<Token> ParseAll() private SyntaxToken Create(
TokenInfo tokenInfo,
List<SyntaxTrivia> leadingTrivia,
List<SyntaxTrivia> trailingTrivia)
{ {
var result = new List<Token>(); switch (tokenInfo.Kind)
{
case TokenKind.Identifier:
return TokenFactory.CreateIdentifier(
tokenInfo.Text,
leadingTrivia,
trailingTrivia);
case TokenKind.UnquotedStringLiteral:
return TokenFactory.CreateUnquotedStringLiteral(
tokenInfo.Text,
tokenInfo.StringValue,
leadingTrivia,
trailingTrivia);
case TokenKind.NumberLiteral:
return TokenFactory.CreateTokenWithValueAndTrivia<double>(
tokenInfo.Kind,
tokenInfo.Text,
tokenInfo.DoubleValue,
leadingTrivia,
trailingTrivia);
case TokenKind.StringLiteral:
return TokenFactory.CreateTokenWithValueAndTrivia<string>(
tokenInfo.Kind,
tokenInfo.Text,
tokenInfo.StringValue,
leadingTrivia,
trailingTrivia);
default:
return TokenFactory.CreateTokenWithTrivia(
tokenInfo.Kind,
leadingTrivia,
trailingTrivia);
}
}
public List<(SyntaxToken, Position)> ParseAll()
{
var result = new List<(SyntaxToken, Position)>();
while (true) while (true)
{ {
var token = NextToken(); var pair = NextToken();
var (token, _) = pair;
if (token == null) if (token == null)
{ {
throw new ParsingException($"Unexpected character: '{Window.PeekChar()}' at {Window.Position}."); throw new ParsingException($"Unexpected character: '{Window.PeekChar()}' at {Window.Position}.");
} }
result.Add(token); result.Add(pair);
if (token.PureToken.Kind == TokenKind.EndOfFile) if (token.Kind == TokenKind.EndOfFile)
{ {
return result; return result;
} }
} }
} }
} }
} }

1117
Parser/Internal/MParserGreen.cs Normal file
View File

File diff suppressed because it is too large Load Diff

555
Parser/Internal/SyntaxFactory.Generated.cs Normal file
View File

@ -0,0 +1,555 @@
namespace Parser.Internal
{
internal partial class SyntaxFactory
{
public FileSyntaxNode FileSyntax(
SyntaxList statementList,
SyntaxToken endOfFile)
{
return new FileSyntaxNode(
statementList,
endOfFile);
}
public FunctionDeclarationSyntaxNode FunctionDeclarationSyntax(
SyntaxToken functionKeyword,
FunctionOutputDescriptionSyntaxNode outputDescription,
SyntaxToken name,
FunctionInputDescriptionSyntaxNode inputDescription,
SyntaxList<SyntaxToken> commas,
SyntaxList body,
SyntaxToken endKeyword)
{
return new FunctionDeclarationSyntaxNode(
functionKeyword,
outputDescription,
name,
inputDescription,
commas,
body,
endKeyword);
}
public FunctionOutputDescriptionSyntaxNode FunctionOutputDescriptionSyntax(
SyntaxList outputList,
SyntaxToken assignmentSign)
{
return new FunctionOutputDescriptionSyntaxNode(
outputList,
assignmentSign);
}
public FunctionInputDescriptionSyntaxNode FunctionInputDescriptionSyntax(
SyntaxToken openingBracket,
SyntaxList parameterList,
SyntaxToken closingBracket)
{
return new FunctionInputDescriptionSyntaxNode(
openingBracket,
parameterList,
closingBracket);
}
public SwitchStatementSyntaxNode SwitchStatementSyntax(
SyntaxToken switchKeyword,
ExpressionSyntaxNode switchExpression,
SyntaxList<SyntaxToken> optionalCommas,
SyntaxList<SwitchCaseSyntaxNode> cases,
SyntaxToken endKeyword)
{
return new SwitchStatementSyntaxNode(
switchKeyword,
switchExpression,
optionalCommas,
cases,
endKeyword);
}
public SwitchCaseSyntaxNode SwitchCaseSyntax(
SyntaxToken caseKeyword,
ExpressionSyntaxNode caseIdentifier,
SyntaxList<SyntaxToken> optionalCommas,
SyntaxList body)
{
return new SwitchCaseSyntaxNode(
caseKeyword,
caseIdentifier,
optionalCommas,
body);
}
public WhileStatementSyntaxNode WhileStatementSyntax(
SyntaxToken whileKeyword,
ExpressionSyntaxNode condition,
SyntaxList<SyntaxToken> optionalCommas,
SyntaxList body,
SyntaxToken endKeyword)
{
return new WhileStatementSyntaxNode(
whileKeyword,
condition,
optionalCommas,
body,
endKeyword);
}
public ElseifClause ElseifClause(
SyntaxToken elseifKeyword,
ExpressionSyntaxNode condition,
SyntaxList<SyntaxToken> optionalCommas,
SyntaxList body)
{
return new ElseifClause(
elseifKeyword,
condition,
optionalCommas,
body);
}
public ElseClause ElseClause(
SyntaxToken elseKeyword,
SyntaxList body)
{
return new ElseClause(
elseKeyword,
body);
}
public IfStatementSyntaxNode IfStatementSyntax(
SyntaxToken ifKeyword,
ExpressionSyntaxNode condition,
SyntaxList<SyntaxToken> optionalCommas,
SyntaxList body,
SyntaxList<ElseifClause> elseifClauses,
ElseClause elseClause,
SyntaxToken endKeyword)
{
return new IfStatementSyntaxNode(
ifKeyword,
condition,
optionalCommas,
body,
elseifClauses,
elseClause,
endKeyword);
}
public ForStatementSyntaxNode ForStatementSyntax(
SyntaxToken forKeyword,
AssignmentExpressionSyntaxNode assignment,
SyntaxList<SyntaxToken> optionalCommas,
SyntaxList body,
SyntaxToken endKeyword)
{
return new ForStatementSyntaxNode(
forKeyword,
assignment,
optionalCommas,
body,
endKeyword);
}
public AssignmentExpressionSyntaxNode AssignmentExpressionSyntax(
ExpressionSyntaxNode lhs,
SyntaxToken assignmentSign,
ExpressionSyntaxNode rhs)
{
return new AssignmentExpressionSyntaxNode(
lhs,
assignmentSign,
rhs);
}
public CatchClauseSyntaxNode CatchClauseSyntax(
SyntaxToken catchKeyword,
SyntaxList catchBody)
{
return new CatchClauseSyntaxNode(
catchKeyword,
catchBody);
}
public TryCatchStatementSyntaxNode TryCatchStatementSyntax(
SyntaxToken tryKeyword,
SyntaxList tryBody,
CatchClauseSyntaxNode catchClause,
SyntaxToken endKeyword)
{
return new TryCatchStatementSyntaxNode(
tryKeyword,
tryBody,
catchClause,
endKeyword);
}
public ExpressionStatementSyntaxNode ExpressionStatementSyntax(
ExpressionSyntaxNode expression)
{
return new ExpressionStatementSyntaxNode(
expression);
}
public EmptyStatementSyntaxNode EmptyStatementSyntax(
SyntaxToken semicolon)
{
return new EmptyStatementSyntaxNode(
semicolon);
}
public EmptyExpressionSyntaxNode EmptyExpressionSyntax()
{
return new EmptyExpressionSyntaxNode();
}
public UnaryPrefixOperationExpressionSyntaxNode UnaryPrefixOperationExpressionSyntax(
SyntaxToken operation,
ExpressionSyntaxNode operand)
{
return new UnaryPrefixOperationExpressionSyntaxNode(
operation,
operand);
}
public CompoundNameSyntaxNode CompoundNameSyntax(
SyntaxList nodes)
{
return new CompoundNameSyntaxNode(
nodes);
}
public NamedFunctionHandleSyntaxNode NamedFunctionHandleSyntax(
SyntaxToken atSign,
CompoundNameSyntaxNode functionName)
{
return new NamedFunctionHandleSyntaxNode(
atSign,
functionName);
}
public LambdaSyntaxNode LambdaSyntax(
SyntaxToken atSign,
FunctionInputDescriptionSyntaxNode input,
ExpressionSyntaxNode body)
{
return new LambdaSyntaxNode(
atSign,
input,
body);
}
public BinaryOperationExpressionSyntaxNode BinaryOperationExpressionSyntax(
ExpressionSyntaxNode lhs,
SyntaxToken operation,
ExpressionSyntaxNode rhs)
{
return new BinaryOperationExpressionSyntaxNode(
lhs,
operation,
rhs);
}
public IdentifierNameSyntaxNode IdentifierNameSyntax(
SyntaxToken name)
{
return new IdentifierNameSyntaxNode(
name);
}
public NumberLiteralSyntaxNode NumberLiteralSyntax(
SyntaxToken number)
{
return new NumberLiteralSyntaxNode(
number);
}
public StringLiteralSyntaxNode StringLiteralSyntax(
SyntaxToken stringToken)
{
return new StringLiteralSyntaxNode(
stringToken);
}
public DoubleQuotedStringLiteralSyntaxNode DoubleQuotedStringLiteralSyntax(
SyntaxToken stringToken)
{
return new DoubleQuotedStringLiteralSyntaxNode(
stringToken);
}
public UnquotedStringLiteralSyntaxNode UnquotedStringLiteralSyntax(
SyntaxToken stringToken)
{
return new UnquotedStringLiteralSyntaxNode(
stringToken);
}
public ArrayLiteralExpressionSyntaxNode ArrayLiteralExpressionSyntax(
SyntaxToken openingSquareBracket,
SyntaxList nodes,
SyntaxToken closingSquareBracket)
{
return new ArrayLiteralExpressionSyntaxNode(
openingSquareBracket,
nodes,
closingSquareBracket);
}
public CellArrayLiteralExpressionSyntaxNode CellArrayLiteralExpressionSyntax(
SyntaxToken openingBrace,
SyntaxList nodes,
SyntaxToken closingBrace)
{
return new CellArrayLiteralExpressionSyntaxNode(
openingBrace,
nodes,
closingBrace);
}
public ParenthesizedExpressionSyntaxNode ParenthesizedExpressionSyntax(
SyntaxToken openingBracket,
ExpressionSyntaxNode expression,
SyntaxToken closingBracket)
{
return new ParenthesizedExpressionSyntaxNode(
openingBracket,
expression,
closingBracket);
}
public CellArrayElementAccessExpressionSyntaxNode CellArrayElementAccessExpressionSyntax(
ExpressionSyntaxNode expression,
SyntaxToken openingBrace,
SyntaxList nodes,
SyntaxToken closingBrace)
{
return new CellArrayElementAccessExpressionSyntaxNode(
expression,
openingBrace,
nodes,
closingBrace);
}
public FunctionCallExpressionSyntaxNode FunctionCallExpressionSyntax(
ExpressionSyntaxNode functionName,
SyntaxToken openingBracket,
SyntaxList nodes,
SyntaxToken closingBracket)
{
return new FunctionCallExpressionSyntaxNode(
functionName,
openingBracket,
nodes,
closingBracket);
}
public MemberAccessSyntaxNode MemberAccessSyntax(
SyntaxNode leftOperand,
SyntaxToken dot,
SyntaxNode rightOperand)
{
return new MemberAccessSyntaxNode(
leftOperand,
dot,
rightOperand);
}
public UnaryPostixOperationExpressionSyntaxNode UnaryPostixOperationExpressionSyntax(
ExpressionSyntaxNode operand,
SyntaxToken operation)
{
return new UnaryPostixOperationExpressionSyntaxNode(
operand,
operation);
}
public IndirectMemberAccessSyntaxNode IndirectMemberAccessSyntax(
SyntaxToken openingBracket,
ExpressionSyntaxNode expression,
SyntaxToken closingBracket)
{
return new IndirectMemberAccessSyntaxNode(
openingBracket,
expression,
closingBracket);
}
public CommandExpressionSyntaxNode CommandExpressionSyntax(
IdentifierNameSyntaxNode commandName,
SyntaxList<UnquotedStringLiteralSyntaxNode> arguments)
{
return new CommandExpressionSyntaxNode(
commandName,
arguments);
}
public BaseClassInvokationSyntaxNode BaseClassInvokationSyntax(
ExpressionSyntaxNode methodName,
SyntaxToken atSign,
ExpressionSyntaxNode baseClassNameAndArguments)
{
return new BaseClassInvokationSyntaxNode(
methodName,
atSign,
baseClassNameAndArguments);
}
public AttributeAssignmentSyntaxNode AttributeAssignmentSyntax(
SyntaxToken assignmentSign,
ExpressionSyntaxNode value)
{
return new AttributeAssignmentSyntaxNode(
assignmentSign,
value);
}
public AttributeSyntaxNode AttributeSyntax(
IdentifierNameSyntaxNode name,
AttributeAssignmentSyntaxNode assignment)
{
return new AttributeSyntaxNode(
name,
assignment);
}
public AttributeListSyntaxNode AttributeListSyntax(
SyntaxToken openingBracket,
SyntaxList nodes,
SyntaxToken closingBracket)
{
return new AttributeListSyntaxNode(
openingBracket,
nodes,
closingBracket);
}
public MethodDefinitionSyntaxNode MethodDefinitionSyntax(
SyntaxToken functionKeyword,
FunctionOutputDescriptionSyntaxNode outputDescription,
CompoundNameSyntaxNode name,
FunctionInputDescriptionSyntaxNode inputDescription,
SyntaxList<SyntaxToken> commas,
SyntaxList body,
SyntaxToken endKeyword)
{
return new MethodDefinitionSyntaxNode(
functionKeyword,
outputDescription,
name,
inputDescription,
commas,
body,
endKeyword);
}
public AbstractMethodDeclarationSyntaxNode AbstractMethodDeclarationSyntax(
FunctionOutputDescriptionSyntaxNode outputDescription,
CompoundNameSyntaxNode name,
FunctionInputDescriptionSyntaxNode inputDescription)
{
return new AbstractMethodDeclarationSyntaxNode(
outputDescription,
name,
inputDescription);
}
public MethodsListSyntaxNode MethodsListSyntax(
SyntaxToken methodsKeyword,
AttributeListSyntaxNode attributes,
SyntaxList methods,
SyntaxToken endKeyword)
{
return new MethodsListSyntaxNode(
methodsKeyword,
attributes,
methods,
endKeyword);
}
public PropertiesListSyntaxNode PropertiesListSyntax(
SyntaxToken propertiesKeyword,
AttributeListSyntaxNode attributes,
SyntaxList properties,
SyntaxToken endKeyword)
{
return new PropertiesListSyntaxNode(
propertiesKeyword,
attributes,
properties,
endKeyword);
}
public BaseClassListSyntaxNode BaseClassListSyntax(
SyntaxToken lessSign,
SyntaxList baseClasses)
{
return new BaseClassListSyntaxNode(
lessSign,
baseClasses);
}
public ClassDeclarationSyntaxNode ClassDeclarationSyntax(
SyntaxToken classdefKeyword,
AttributeListSyntaxNode attributes,
IdentifierNameSyntaxNode className,
BaseClassListSyntaxNode baseClassList,
SyntaxList nodes,
SyntaxToken endKeyword)
{
return new ClassDeclarationSyntaxNode(
classdefKeyword,
attributes,
className,
baseClassList,
nodes,
endKeyword);
}
public EnumerationItemValueSyntaxNode EnumerationItemValueSyntax(
SyntaxToken openingBracket,
SyntaxList values,
SyntaxToken closingBracket)
{
return new EnumerationItemValueSyntaxNode(
openingBracket,
values,
closingBracket);
}
public EnumerationItemSyntaxNode EnumerationItemSyntax(
IdentifierNameSyntaxNode name,
EnumerationItemValueSyntaxNode values,
SyntaxList<SyntaxToken> commas)
{
return new EnumerationItemSyntaxNode(
name,
values,
commas);
}
public EnumerationListSyntaxNode EnumerationListSyntax(
SyntaxToken enumerationKeyword,
AttributeListSyntaxNode attributes,
SyntaxList<EnumerationItemSyntaxNode> items,
SyntaxToken endKeyword)
{
return new EnumerationListSyntaxNode(
enumerationKeyword,
attributes,
items,
endKeyword);
}
public EventsListSyntaxNode EventsListSyntax(
SyntaxToken eventsKeyword,
AttributeListSyntaxNode attributes,
SyntaxList events,
SyntaxToken endKeyword)
{
return new EventsListSyntaxNode(
eventsKeyword,
attributes,
events,
endKeyword);
}
}
}

6
Parser/Internal/SyntaxFactory.cs Normal file
View File

@ -0,0 +1,6 @@
namespace Parser.Internal
{
internal partial class SyntaxFactory
{
}
}

276
Parser/Internal/SyntaxFacts.cs Normal file
View File

@ -0,0 +1,276 @@
using System;
using System.Collections.Generic;
namespace Parser.Internal
{
public static class SyntaxFacts
{
public static readonly HashSet<string> Keywords;
static SyntaxFacts()
{
Keywords = new HashSet<string>
{
"for", "if", "function", "while", "case", "try", "catch", "end",
"switch", "classdef", "elseif", "persistent", "else"
};
}
public enum Precedence
{
// see https://mathworks.com/help/matlab/matlab_prog/operator-precedence.html
Expression = 0,
Assignment,
LogicalOr,
LogicalAnd,
BitwiseOr,
BitwiseAnd,
Relational,
Colon,
Additive,
Multiplicative,
Unary,
WeirdPower,
Power
}
public static Precedence GetPrecedence(TokenKind kind)
{
switch (kind)
{
case TokenKind.Assignment:
return Precedence.Assignment;
case TokenKind.LogicalOr:
return Precedence.LogicalOr;
case TokenKind.LogicalAnd:
return Precedence.LogicalAnd;
case TokenKind.BitwiseOr:
return Precedence.BitwiseOr;
case TokenKind.BitwiseAnd:
return Precedence.BitwiseAnd;
case TokenKind.Less:
case TokenKind.LessOrEqual:
case TokenKind.Greater:
case TokenKind.GreaterOrEqual:
case TokenKind.Equality:
case TokenKind.Inequality:
return Precedence.Relational;
case TokenKind.Colon:
return Precedence.Colon;
case TokenKind.Plus:
case TokenKind.Minus:
return Precedence.Additive;
case TokenKind.Multiply:
case TokenKind.DotMultiply:
case TokenKind.Divide:
case TokenKind.DotDivide:
case TokenKind.Backslash:
case TokenKind.DotBackslash:
return Precedence.Multiplicative;
case TokenKind.Not:
return Precedence.Unary;
case TokenKind.Power:
case TokenKind.DotPower:
case TokenKind.Transpose:
case TokenKind.DotTranspose:
return Precedence.Power;
default:
return Precedence.Expression;
}
}
public static bool IsDigit(char c)
{
return c >= '0' && c <= '9';
}
public static bool IsDigitOrDot(char c)
{
return c == '.' || (c >= '0' && c <= '9');
}
public static bool IsEolOrEof(char c)
{
return c == '\n' || c == '\r' || c == '\0';
}
public static bool IsWhitespace(char c)
{
return c == ' ' || c == '\t' || c == '\n';
}
public static bool IsOpeningToken(TokenKind tokenKind)
{
switch (tokenKind)
{
case TokenKind.OpeningBrace:
case TokenKind.OpeningBracket:
case TokenKind.OpeningSquareBracket:
return true;
default:
return false;
}
}
public static bool IsClosingToken(TokenKind tokenKind)
{
switch (tokenKind)
{
case TokenKind.ClosingBrace:
case TokenKind.ClosingBracket:
case TokenKind.ClosingSquareBracket:
return true;
default:
return false;
}
}
public static bool IsBracket(TokenKind tokenKind)
{
return IsOpeningToken(tokenKind) || IsClosingToken(tokenKind);
}
public static TokenKind? OpeningFromClosing(TokenKind tokenKind)
{
switch (tokenKind)
{
case TokenKind.ClosingBrace:
return TokenKind.OpeningBrace;
case TokenKind.ClosingBracket:
return TokenKind.OpeningBracket;
case TokenKind.ClosingSquareBracket:
return TokenKind.OpeningSquareBracket;
default:
return null;
}
}
private static readonly string[] StringFromKind =
{
null, // None = 0,
"", // EndOfFile = 1,
null, // Identifier = 2,
null, // NumberLiteral = 3,
null, // StringLiteral = 4,
null, // DoubleQuotedStringLiteral = 5,
null, // UnquotedStringLiteral = 6
null, null, null, null, null, null, null, null, null, null, null, null, null,
"=", // Assignment = 20,
"==", // Equality = 21,
"~=", // Inequality = 22,
"&&", // LogicalAnd = 23,
"||", // LogicalOr = 24,
"&", // BitwiseAnd = 25,
"|", // BitwiseOr = 26,
"<", // Less = 27,
"<=", // LessOrEqual = 28,
">", // Greater = 29,
">=", // GreaterOrEqual = 30,
"~", // Not = 31,
"+", // Plus = 32,
"-", // Minus = 33,
"*", // Multiply = 34,
"/", // Divide = 35,
"^", // Power = 36,
"\\", // Backslash = 37,
"'", // Transpose = 38,
".*", // DotMultiply = 39,
"./", // DotDivide = 40,
".^", // DotPower = 41,
".\\", // DotBackslash = 42,
".'", // DotTranspose = 43,
"@", // At = 44,
":", // Colon = 45,
"?", // QuestionMark = 46,
",", // Comma = 47,
";", // Semicolon = 48,
"{", // OpeningBrace = 49,
"}", // ClosingBrace = 50,
"[", // OpeningSquareBracket = 51,
"]", // ClosingSquareBracket = 52,
"(", // OpeningBracket = 53,
")", // ClosingBracket = 54,
".", // Dot = 55,
"...", // DotDotDot = 56,
"+", // UnaryPlus = 57,
"-", // UnaryMinus = 58,
"~", // UnaryNot = 59,
"?", // UnaryQuestionMark = 60,
};
public static string GetText(TokenKind kind)
{
return StringFromKind[(int) kind];
}
public static bool IsUnaryOperator(TokenKind kind)
{
switch (kind)
{
case TokenKind.Plus:
case TokenKind.Minus:
case TokenKind.Not:
case TokenKind.QuestionMark:
return true;
default:
return false;
}
}
public static bool IsBinaryOperator(TokenKind kind)
{
switch (kind)
{
case TokenKind.Assignment:
case TokenKind.LogicalOr:
case TokenKind.LogicalAnd:
case TokenKind.BitwiseOr:
case TokenKind.BitwiseAnd:
case TokenKind.Less:
case TokenKind.LessOrEqual:
case TokenKind.Greater:
case TokenKind.GreaterOrEqual:
case TokenKind.Equality:
case TokenKind.Inequality:
case TokenKind.Colon:
case TokenKind.Plus:
case TokenKind.Minus:
case TokenKind.Multiply:
case TokenKind.DotMultiply:
case TokenKind.Divide:
case TokenKind.DotDivide:
case TokenKind.Backslash:
case TokenKind.DotBackslash:
case TokenKind.Not:
case TokenKind.Power:
case TokenKind.DotPower:
return true;
default:
return false;
}
}
public static bool IsLeftAssociative(TokenKind kind)
{
return true; // TODO: really?
}
public static TokenKind ConvertToUnaryTokenKind(TokenKind kind)
{
switch (kind)
{
case TokenKind.Plus:
return TokenKind.UnaryPlus;
case TokenKind.Minus:
return TokenKind.UnaryMinus;
case TokenKind.Not:
return TokenKind.UnaryNot;
case TokenKind.QuestionMark:
return TokenKind.UnaryQuestionMark;
default:
throw new ArgumentException(nameof(kind));
}
}
}
}

32
Parser/Internal/SyntaxList.cs Normal file
View File

@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
namespace Parser.Internal
{
internal class SyntaxList : SyntaxNode
{
private readonly GreenNode[] _elements;
protected SyntaxList(GreenNode[] elements) : base(TokenKind.List, elements.Length)
{
_elements = elements;
}
public override GreenNode GetSlot(int i)
{
return _elements[i];
}
public static SyntaxList List(GreenNode[] elements)
{
return new SyntaxList(elements);
}
public override bool IsList => true;
internal override Parser.SyntaxNode CreateRed(Parser.SyntaxNode parent)
{
return new Parser.SyntaxNodeOrTokenList(parent, this);
}
}
}

33
Parser/Internal/SyntaxListBuilder.cs Normal file
View File

@ -0,0 +1,33 @@
using System.Collections.Generic;
namespace Parser.Internal
{
internal class SyntaxListBuilder
{
private readonly List<GreenNode> _list;
public SyntaxListBuilder()
{
_list = new List<GreenNode>();
}
public void Add(GreenNode node)
{
_list.Add(node);
}
public void AddRange(SyntaxList list)
{
for (var i = 0; i < list.Slots; i++)
{
var element = list.GetSlot(i);
_list.Add(element);
}
}
public SyntaxList ToList()
{
return _list.Count == 0 ? null : SyntaxList.List(_list.ToArray());
}
}
}

25
Parser/Internal/SyntaxListBuilder`1.cs Normal file
View File

@ -0,0 +1,25 @@
using System.Collections.Generic;
namespace Parser.Internal
{
internal class SyntaxListBuilder<T> where T : GreenNode
{
private readonly List<T> _list;
public SyntaxListBuilder()
{
_list = new List<T>();
}
public void Add(T node)
{
_list.Add(node);
}
public SyntaxList<T> ToList()
{
return _list.Count == 0 ? null : SyntaxList<T>.List(_list.ToArray());
}
}
}

32
Parser/Internal/SyntaxList`1.cs Normal file
View File

@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
namespace Parser.Internal
{
internal class SyntaxList<T> : SyntaxNode where T : GreenNode
{
private readonly SyntaxList _list;
protected SyntaxList(T[] list) : base(TokenKind.List, list.Length)
{
_list = SyntaxList.List(list);
}
public override GreenNode GetSlot(int i)
{
return (T)_list.GetSlot(i);
}
public static SyntaxList<T> List(T[] elements)
{
return new SyntaxList<T>(elements);
}
public override bool IsList => true;
internal override Parser.SyntaxNode CreateRed(Parser.SyntaxNode parent)
{
return new Parser.SyntaxNodeOrTokenList(parent, this);
}
}
}

1652
Parser/Internal/SyntaxNode.Generated.cs Normal file
View File

File diff suppressed because it is too large Load Diff

82
Parser/Internal/SyntaxNode.cs Normal file
View File

@ -0,0 +1,82 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace Parser.Internal
{
internal abstract class SyntaxNode : GreenNode
{
protected SyntaxNode(TokenKind kind, int slots) : base(kind, slots)
{
}
public IEnumerable<SyntaxToken> DescendantTokens => CalculateChildTokens();
private IEnumerable<SyntaxToken> CalculateChildTokens()
{
for (var i = 0; i < Slots; i++)
{
var slot = GetSlot(i);
switch (slot)
{
case null:
continue;
case SyntaxToken token:
yield return token;
break;
case SyntaxNode node:
foreach (var t in node.DescendantTokens)
{
yield return t;
}
break;
}
}
}
protected virtual string CollectFullText()
{
var builder = new StringBuilder();
foreach (var token in DescendantTokens)
{
builder.Append(token.FullText);
}
return builder.ToString();
}
public override string FullText => CollectFullText();
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => throw new NotImplementedException();
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => throw new NotImplementedException();
}
internal abstract class StatementSyntaxNode : SyntaxNode
{
protected StatementSyntaxNode(TokenKind kind, int slots) : base(kind, slots + 1)
{
}
}
internal abstract class ExpressionSyntaxNode : SyntaxNode
{
protected ExpressionSyntaxNode(TokenKind kind, int slots) : base(kind, slots)
{
}
}
internal abstract class FunctionHandleSyntaxNode : ExpressionSyntaxNode
{
protected FunctionHandleSyntaxNode(TokenKind kind, int slots) : base(kind, slots)
{
}
}
internal abstract class MethodDeclarationSyntaxNode : StatementSyntaxNode
{
protected MethodDeclarationSyntaxNode(TokenKind kind, int slots) : base(kind, slots)
{
}
}
}

215
Parser/Internal/SyntaxToken.cs Normal file
View File

@ -0,0 +1,215 @@
using System;
using System.Collections.Generic;
using System.IO;
namespace Parser.Internal
{
internal abstract class SyntaxToken : GreenNode
{
internal class SyntaxTokenWithTrivia : SyntaxToken
{
private readonly string _text;
public SyntaxTokenWithTrivia(
TokenKind kind,
string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) : base(kind)
{
_text = text;
LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia;
}
public SyntaxTokenWithTrivia(
TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) : base(kind)
{
_text = base.Text;
LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia;
}
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
if (leading)
{
foreach (var trivia in LeadingTrivia)
{
writer.Write(trivia.Text);
}
}
base.WriteTokenTo(writer, leading, trailing);
if (trailing)
{
foreach (var trivia in TrailingTrivia)
{
writer.Write(trivia.Text);
}
}
}
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore { get; }
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore { get; }
}
internal class SyntaxTokenWithValue<T> : SyntaxToken
{
protected readonly string _text;
private readonly T _value;
public SyntaxTokenWithValue(
TokenKind kind,
string text,
T value) : base(kind)
{
_text = text;
_value = value;
}
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
writer.Write(_text);
}
public T Value => _value;
}
internal class SyntaxTokenWithValueAndTrivia<T> : SyntaxTokenWithValue<T>
{
public SyntaxTokenWithValueAndTrivia(
TokenKind kind,
string text,
T value,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) : base(kind, text, value)
{
LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia;
}
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore { get; }
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore { get; }
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
if (leading)
{
foreach (var trivia in LeadingTrivia)
{
writer.Write(trivia.Text);
}
}
base.WriteTokenTo(writer, leading, trailing);
if (trailing)
{
foreach (var trivia in TrailingTrivia)
{
writer.Write(trivia.Text);
}
}
}
}
internal class SyntaxIdentifier : SyntaxToken
{
private readonly string _text;
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
writer.Write(_text);
}
public SyntaxIdentifier(
string text
) : base(TokenKind.Identifier)
{
_text = text;
}
}
internal class SyntaxIdentifierWithTrivia : SyntaxIdentifier
{
private readonly IReadOnlyList<SyntaxTrivia> _leadingTrivia;
private readonly IReadOnlyList<SyntaxTrivia> _trailingTrivia;
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => _leadingTrivia;
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => _trailingTrivia;
public SyntaxIdentifierWithTrivia(
string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia
) : base(text)
{
_leadingTrivia = leadingTrivia;
_trailingTrivia = trailingTrivia;
}
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
if (leading)
{
foreach (var trivia in LeadingTrivia)
{
writer.Write(trivia.Text);
}
}
base.WriteTokenTo(writer, leading, trailing);
if (trailing)
{
foreach (var trivia in TrailingTrivia)
{
writer.Write(trivia.Text);
}
}
}
public override bool IsToken => true;
public override bool IsNode => false;
}
internal class MissingTokenWithTrivia : SyntaxTokenWithTrivia
{
public MissingTokenWithTrivia(
TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia
) : base(kind, leadingTrivia, trailingTrivia)
{
_isMissing = true;
}
public override string Text => "";
}
protected SyntaxToken(TokenKind kind) : base(kind, 0)
{
}
public virtual int Width => Text.Length;
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => new List<SyntaxTrivia>();
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => new List<SyntaxTrivia>();
public override GreenNode GetSlot(int i)
{
throw new System.InvalidOperationException();
}
internal override Parser.SyntaxNode CreateRed(Parser.SyntaxNode parent)
{
throw new InvalidOperationException();
}
public override bool IsToken => true;
public override bool IsNode => false;
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
writer.Write(SyntaxFacts.GetText(Kind));
}
}
}

41
Parser/Internal/SyntaxTrivia.cs Normal file
View File

@ -0,0 +1,41 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
namespace Parser.Internal
{
internal class SyntaxTrivia : GreenNode
{
private readonly string _text;
public SyntaxTrivia(TokenKind kind, string text) : base(kind, 0)
{
_text = text;
}
public override string Text => _text;
public int Width => _text.Length;
public override GreenNode GetSlot(int i)
{
throw new System.NotImplementedException();
}
internal override Parser.SyntaxNode CreateRed(Parser.SyntaxNode parent)
{
throw new InvalidOperationException();
}
public override bool IsTrivia => true;
public override bool IsNode => false;
public override void WriteTriviaTo(TextWriter writer)
{
writer.Write(_text);
}
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => new List<SyntaxTrivia>();
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => new List<SyntaxTrivia>();
}
}

60
Parser/Internal/TokenFactory.cs Normal file
View File

@ -0,0 +1,60 @@
using System.Collections.Generic;
namespace Parser.Internal
{
internal static class TokenFactory
{
public static SyntaxTrivia CreateTrivia(TokenKind kind, string text)
{
return new SyntaxTrivia(kind, text);
}
public static SyntaxToken CreateTokenWithTrivia(
TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
return new SyntaxToken.SyntaxTokenWithTrivia(kind, leadingTrivia, trailingTrivia);
}
public static SyntaxToken CreateIdentifier(
string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
return new SyntaxToken.SyntaxIdentifierWithTrivia(text, leadingTrivia, trailingTrivia);
}
public static SyntaxToken CreateTokenWithValueAndTrivia<T>(
TokenKind kind,
string text,
T value,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<T>(kind, text, value, leadingTrivia, trailingTrivia);
}
public static SyntaxToken CreateUnquotedStringLiteral(
string text,
string value,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>(
TokenKind.UnquotedStringLiteral,
text,
value,
leadingTrivia,
trailingTrivia);
}
public static SyntaxToken CreateMissing(
TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
return new SyntaxToken.MissingTokenWithTrivia(kind, leadingTrivia, trailingTrivia);
}
}
}

940
Parser/MParser.cs
View File

@ -1,939 +1,21 @@
using System; namespace Parser
using System.Collections.Generic;
using System.Linq;
using Lexer;
namespace Parser
{ {
public class MParser public class MParser
{ {
public enum Precedence private readonly ITextWindow _window;
public MParser(ITextWindow window)
{ {
// see https://mathworks.com/help/matlab/matlab_prog/operator-precedence.html _window = window;
Expression = 0,
Assignment,
LogicalOr,
LogicalAnd,
BitwiseOr,
BitwiseAnd,
Relational,
Colon,
Additive,
Multiplicative,
Unary,
WeirdPower,
Power
} }
private static Precedence GetPrecedence(TokenKind kind) public FileSyntaxNode Parse()
{ {
switch (kind) var lexer = new Internal.MLexerGreen(_window);
{ var tokens = lexer.ParseAll();
case TokenKind.Assignment: var parser = new Internal.MParserGreen(tokens, new Internal.SyntaxFactory());
return Precedence.Assignment; var green = parser.ParseFile();
case TokenKind.LogicalOr: return new FileSyntaxNode(null, green);
return Precedence.LogicalOr;
case TokenKind.LogicalAnd:
return Precedence.LogicalAnd;
case TokenKind.BitwiseOr:
return Precedence.BitwiseOr;
case TokenKind.BitwiseAnd:
return Precedence.BitwiseAnd;
case TokenKind.Less:
case TokenKind.LessOrEqual:
case TokenKind.Greater:
case TokenKind.GreaterOrEqual:
case TokenKind.Equality:
case TokenKind.Inequality:
return Precedence.Relational;
case TokenKind.Colon:
return Precedence.Colon;
case TokenKind.Plus:
case TokenKind.Minus:
return Precedence.Additive;
case TokenKind.Multiply:
case TokenKind.DotMultiply:
case TokenKind.Divide:
case TokenKind.DotDivide:
case TokenKind.Backslash:
case TokenKind.DotBackslash:
return Precedence.Multiplicative;
case TokenKind.Not:
return Precedence.Unary;
case TokenKind.Power:
case TokenKind.DotPower:
case TokenKind.Transpose:
case TokenKind.DotTranspose:
return Precedence.Power;
default:
return Precedence.Expression;
}
}
private List<Token> Tokens { get; }
private int _index;
private Token CurrentToken => Tokens[_index];
private Token PeekToken(int n) => Tokens[_index + n];
private SyntaxFactory Factory { get; }
public MParser(List<Token> tokens)
{
Tokens = tokens;
_index = 0;
Factory = new SyntaxFactory();
}
private Token EatToken()
{
var token = Tokens[_index];
//Console.WriteLine($"{token} at {token.PureToken.Position}");
_index++;
return token;
}
private Token EatToken(TokenKind kind)
{
var token = Tokens[_index];
//Console.WriteLine($"{token} at {token.PureToken.Position}");
if (token.Kind != kind)
{
throw new ParsingException($"Unexpected token \"{token.PureToken}\" instead of {kind} at {token.PureToken.Position}.");
}
_index++;
return token;
}
private Token EatIdentifier(string s)
{
var token = Tokens[_index];
//Console.WriteLine($"{token} at {token.PureToken.Position}");
if (token.PureToken.Kind != TokenKind.Identifier)
{
throw new ParsingException($"Unexpected token \"{token.PureToken}\" instead of identifier \"{s}\" at {token.PureToken.Position}.");
}
if (token.PureToken.LiteralText != s)
{
throw new ParsingException($"Unexpected identifier \"{token.PureToken.LiteralText}\" instead of \"{s}\" at {token.PureToken.Position}.");
}
_index++;
return token;
}
private void EatAll()
{
_index = Tokens.Count - 1;
}
private List<SyntaxNode> ParseFunctionOutputList()
{
var outputs = new List<Token>();
while (CurrentToken.Kind != TokenKind.ClosingSquareBracket)
{
if (outputs.Count > 0 && CurrentToken.Kind == TokenKind.Comma)
{
outputs.Add(EatToken());
}
outputs.Add(EatToken(TokenKind.Identifier));
}
return outputs.Select(token => new TokenNode(token) as SyntaxNode).ToList();
}
private FunctionOutputDescriptionNode ParseFunctionOutputDescription()
{
if (CurrentToken.Kind == TokenKind.Identifier)
{
if (PeekToken(1).Kind == TokenKind.Assignment)
{
var identifier = EatToken();
var assignmentSign = EatToken(TokenKind.Assignment);
return Factory.FunctionOutputDescription(
new List<SyntaxNode> { Factory.Token(identifier) },
Factory.Token(assignmentSign)
);
}
return null;
} else if (CurrentToken.Kind == TokenKind.OpeningSquareBracket)
{
var leftBracket = EatToken();
var outputs = ParseFunctionOutputList();
var rightBracket = EatToken(TokenKind.ClosingSquareBracket);
var nodes = new List<SyntaxNode> {Factory.Token(leftBracket)};
nodes.AddRange(outputs);
nodes.Add(Factory.Token(rightBracket));
var assignmentSign = EatToken(TokenKind.Assignment);
return Factory.FunctionOutputDescription(nodes, Factory.Token(assignmentSign));
}
throw new ParsingException($"Unexpected token {CurrentToken.PureToken} during parsing function output descritpion at {CurrentToken.PureToken.Position}.");
}
private ParameterListNode ParseParameterList()
{
var identifierTokens = new List<Token>();
while (CurrentToken.Kind != TokenKind.ClosingBracket)
{
if (identifierTokens.Count > 0)
{
identifierTokens.Add(EatToken(TokenKind.Comma));
}
if (CurrentToken.Kind == TokenKind.Not)
{
var notToken = EatToken();
identifierTokens.Add(notToken);
}
else
{
identifierTokens.Add(EatToken(TokenKind.Identifier));
}
}
return Factory.ParameterList(identifierTokens.Select(token => Factory.Token(token) as SyntaxNode).ToList());
}
private FunctionInputDescriptionNode ParseFunctionInputDescription()
{
if (CurrentToken.Kind == TokenKind.OpeningBracket)
{
var openingBracket = EatToken(TokenKind.OpeningBracket);
var parameterList = ParseParameterList();
var closingBracket = EatToken(TokenKind.ClosingBracket);
return Factory.FunctionInputDescription(
new TokenNode(openingBracket),
parameterList,
new TokenNode(closingBracket));
}
else
{
return null;
}
}
private TokenNode PossibleSemicolonOrComma()
{
if (CurrentToken.Kind == TokenKind.Semicolon
|| CurrentToken.Kind == TokenKind.Comma)
{
return Factory.Token(EatToken());
}
return null;
}
private FunctionDeclarationNode ParseFunctionDeclaration()
{
var functionKeyword = EatIdentifier("function");
var outputDescription = ParseFunctionOutputDescription();
var name = EatToken(TokenKind.Identifier);
var inputDescription = ParseFunctionInputDescription();
var body = ParseStatements();
TokenNode end = null;
if (CurrentToken.Kind == TokenKind.Identifier
&& CurrentToken.PureToken.LiteralText == "end")
{
end = Factory.Token(EatIdentifier("end"));
}
var semicolonOrComma = PossibleSemicolonOrComma();
return Factory.FunctionDeclaration(
Factory.Token(functionKeyword),
outputDescription,
Factory.Token(name),
inputDescription,
body,
end,
semicolonOrComma);
}
private StatementNode ParseClassDeclaration()
{
var node = new TokenNode(CurrentToken);
EatAll();
return null;
}
private FunctionCallParameterListNode ParseFunctionCallParameterList()
{
var nodes = new List<SyntaxNode>();
while (CurrentToken.Kind != TokenKind.ClosingBracket)
{
if (nodes.Count > 0)
{
nodes.Add(Factory.Token(EatToken(TokenKind.Comma)));
}
nodes.Add(ParseExpression());
}
return Factory.FunctionCallParameterList(nodes);
}
private ExpressionNode ParseMember()
{
if (CurrentToken.Kind == TokenKind.Identifier)
{
return Factory.IdentifierName(EatToken());
} else if (CurrentToken.Kind == TokenKind.OpeningBracket)
{
var openingBracket = EatToken();
var indirectMember = ParseExpression();
var closingBracket = EatToken(TokenKind.ClosingBracket);
return Factory.IndirectMemberAccess(
Factory.Token(openingBracket),
indirectMember,
Factory.Token(closingBracket));
}
throw new ParsingException($"Unexpected token {CurrentToken.PureToken} at {CurrentToken.PureToken.Position}.");
}
private ExpressionNode ParsePostfix(ParseOptions options, ExpressionNode expression)
{
while (true)
{
var token = CurrentToken;
switch(token.Kind) {
case TokenKind.OpeningBrace: // cell array element access
if (options.ParsingArrayElements && expression.TrailingTrivia.Any())
{
return expression;
}
var openingBrace = EatToken();
var indices = ParseArrayElementList(TokenKind.ClosingBrace);
var closingBrace = EatToken(TokenKind.ClosingBrace);
expression = Factory.CellArrayElementAccessExpression(
expression,
Factory.Token(openingBrace),
indices,
Factory.Token(closingBrace)
);
break;
case TokenKind.OpeningBracket: // function call
if (options.ParsingArrayElements && expression.TrailingTrivia.Any())
{
return expression;
}
var openingBracket = EatToken();
var parameters = ParseFunctionCallParameterList();
var closingBracket = EatToken(TokenKind.ClosingBracket);
expression = Factory.FunctionCallExpression(
expression,
Factory.Token(openingBracket),
parameters,
Factory.Token(closingBracket));
break;
case TokenKind.Dot: // member access
if (expression is IdentifierNameNode
|| expression is MemberAccessNode
|| expression is FunctionCallExpressionNode
|| expression is CellArrayElementAccessExpressionNode)
{
var dot = EatToken();
var member = ParseMember();
expression = Factory.MemberAccess(expression, Factory.Token(dot), member);
}
else
{
throw new ParsingException(
$"Unexpected token {token.PureToken} at {token.PureToken.Position}.");
}
break;
case TokenKind.Transpose:
case TokenKind.DotTranspose:
var operation = Factory.Token(EatToken());
expression = Factory.UnaryPostfixOperationExpression(expression, operation);
break;
case TokenKind.UnquotedStringLiteral:
if (expression is IdentifierNameNode idNameNode)
{
var arguments = new List<UnquotedStringLiteralNode>();
while (CurrentToken.Kind == TokenKind.UnquotedStringLiteral)
{
arguments.Add(Factory.UnquotedStringLiteral(EatToken()));
}
return Factory.CommandExpression(idNameNode, arguments);
}
else
{
throw new ParsingException($"Unexpected token \"{CurrentToken.PureToken.LiteralText}\" while parsing expression \"{expression.FullText}\" at {CurrentToken.PureToken.Position}.");
}
case TokenKind.At:
if (expression is IdentifierNameNode idNameNode2
&& !expression.TrailingTrivia.Any())
{
var atToken = Factory.Token(EatToken());
var baseClassNameWithArguments = ParseExpression();
return Factory.BaseClassInvokation(idNameNode2, atToken, baseClassNameWithArguments);
}
else
{
throw new ParsingException($"Unexpected token \"{CurrentToken.PureToken.LiteralText}\" at {CurrentToken.PureToken.Position}.");
}
default:
return expression;
}
}
}
private ArrayElementListNode ParseArrayElementList(TokenKind closing)
{
var nodes = new List<SyntaxNode> {};
while (CurrentToken.Kind != closing)
{
if (nodes.Count > 0)
{
if (CurrentToken.Kind == TokenKind.Comma
|| CurrentToken.Kind == TokenKind.Semicolon)
{
nodes.Add(Factory.Token(EatToken()));
}
}
var expression = ParseExpression(new ParseOptions {ParsingArrayElements = true});
if (expression != null)
{
nodes.Add(expression);
}
}
return Factory.ArrayElementList(nodes);
}
private ArrayLiteralExpressionNode ParseArrayLiteral()
{
var openingSquareBracket = EatToken(TokenKind.OpeningSquareBracket);
var elements = ParseArrayElementList(TokenKind.ClosingSquareBracket);
var closingSquareBracket = EatToken(TokenKind.ClosingSquareBracket);
return Factory.ArrayLiteralExpression(
Factory.Token(openingSquareBracket),
elements,
Factory.Token(closingSquareBracket));
}
private CellArrayLiteralExpressionNode ParseCellArrayLiteral()
{
var openingBrace = EatToken(TokenKind.OpeningBrace);
var elements = ParseArrayElementList(TokenKind.ClosingBrace);
var closingBrace = EatToken(TokenKind.ClosingBrace);
return Factory.CellArrayLiteralExpression(
Factory.Token(openingBrace),
elements,
Factory.Token(closingBrace));
}
private ParenthesizedExpressionNode ParseParenthesizedExpression()
{
var openParen = Factory.Token(EatToken(TokenKind.OpeningBracket));
var expression = ParseExpression();
var closeParen = Factory.Token(EatToken(TokenKind.ClosingBracket));
return Factory.ParenthesizedExpression(
openParen,
expression,
closeParen);
}
private ExpressionNode ParseTerm(ParseOptions options)
{
var token = CurrentToken;
ExpressionNode expression = null;
if (token.Kind == TokenKind.Identifier)
{
var term = EatToken();
expression = Factory.IdentifierName(term);
}
else if (token.Kind == TokenKind.NumberLiteral)
{
var number = EatToken();
expression = Factory.NumberLiteral(number);
}
else if (token.Kind == TokenKind.StringLiteral)
{
var str = EatToken();
expression = Factory.StringLiteral(str);
}
else if (token.Kind == TokenKind.DoubleQuotedStringLiteral)
{
var str = EatToken();
expression = Factory.DoubleQuotedStringLiteral(str);
}
else if (token.Kind == TokenKind.OpeningSquareBracket) // array literal expression
{
expression = ParseArrayLiteral();
}
else if (token.Kind == TokenKind.OpeningBrace) // cell array literal expression
{
expression = ParseCellArrayLiteral();
}
else if (token.Kind == TokenKind.Colon) // for parsing things like a{:}
{
expression = Factory.EmptyExpression();
}
else if (token.Kind == TokenKind.OpeningBracket)
{
expression = ParseParenthesizedExpression();
}
if (expression == null)
{
return null;
}
return ParsePostfix(options, expression);
}
internal struct ParseOptions
{
public bool ParsingArrayElements { get; set; }
public static ParseOptions Default = new ParseOptions { ParsingArrayElements = false };
}
public ExpressionNode ParseExpression()
{
return ParseExpression(ParseOptions.Default);
}
private ExpressionNode ParseExpression(ParseOptions options)
{
return ParseSubExpression(options, Precedence.Expression);
}
private bool IsUnaryOperator(TokenKind kind)
{
switch (kind)
{
case TokenKind.Plus:
case TokenKind.Minus:
case TokenKind.Not:
case TokenKind.QuestionMark:
return true;
default:
return false;
}
}
private bool IsBinaryOperator(TokenKind kind)
{
switch (kind)
{
case TokenKind.Assignment:
case TokenKind.LogicalOr:
case TokenKind.LogicalAnd:
case TokenKind.BitwiseOr:
case TokenKind.BitwiseAnd:
case TokenKind.Less:
case TokenKind.LessOrEqual:
case TokenKind.Greater:
case TokenKind.GreaterOrEqual:
case TokenKind.Equality:
case TokenKind.Inequality:
case TokenKind.Colon:
case TokenKind.Plus:
case TokenKind.Minus:
case TokenKind.Multiply:
case TokenKind.DotMultiply:
case TokenKind.Divide:
case TokenKind.DotDivide:
case TokenKind.Backslash:
case TokenKind.DotBackslash:
case TokenKind.Not:
case TokenKind.Power:
case TokenKind.DotPower:
return true;
default:
return false;
}
}
private bool IsLeftAssociative(TokenKind kind)
{
return true; // TODO: really?
}
private TokenKind ConvertToUnaryTokenKind(TokenKind kind)
{
switch (kind)
{
case TokenKind.Plus:
return TokenKind.UnaryPlus;
case TokenKind.Minus:
return TokenKind.UnaryMinus;
case TokenKind.Not:
return TokenKind.UnaryNot;
case TokenKind.QuestionMark:
return TokenKind.UnaryQuestionMark;
default:
throw new ArgumentException(nameof(kind));
}
}
private CompoundNameNode ParseCompoundName()
{
var lastToken = EatToken(TokenKind.Identifier);
var firstName = Factory.IdentifierName(lastToken);
var nodes = new List<SyntaxNode> {firstName};
while (CurrentToken.Kind == TokenKind.Dot
&& !lastToken.TrailingTrivia.Any())
{
var dot = Factory.Token(EatToken());
nodes.Add(dot);
lastToken = EatToken(TokenKind.Identifier);
nodes.Add(Factory.IdentifierName(lastToken));
}
return Factory.CompoundName(nodes);
}
private FunctionHandleNode ParseFunctionHandle()
{
var atSign = EatToken();
if (CurrentToken.Kind == TokenKind.Identifier)
{
var compoundName = ParseCompoundName();
return Factory.NamedFunctionHandle(
Factory.Token(atSign),
compoundName);
} else if (CurrentToken.Kind == TokenKind.OpeningBracket)
{
var inputs = ParseFunctionInputDescription();
var body = ParseExpression();
return Factory.Lambda(Factory.Token(atSign), inputs, body);
}
throw new ParsingException($"Unexpected token {CurrentToken.PureToken} while parsing function handle at {CurrentToken.PureToken.Position}.");
}
private ExpressionNode ParseSubExpression(ParseOptions options, Precedence precedence)
{
ExpressionNode lhs = null;
if (IsUnaryOperator(CurrentToken.Kind))
{
var operation = EatToken();
var unaryTokenKind = ConvertToUnaryTokenKind(operation.Kind);
var newPrecedence = GetPrecedence(unaryTokenKind);
var operand = ParseSubExpression(options, newPrecedence);
if (operand == null)
{
if (options.ParsingArrayElements && operation.Kind == TokenKind.Not)
{
operand = Factory.EmptyExpression();
}
else
{
throw new ParsingException($"Unexpected token {CurrentToken.Kind} at {operation.PureToken.Position}.");
}
}
lhs = Factory.UnaryPrefixOperationExpression(Factory.Token(operation), operand);
}
else if (CurrentToken.Kind == TokenKind.At)
{
return ParseFunctionHandle();
}
else
{
lhs = ParseTerm(options);
}
while (true)
{
var token = CurrentToken;
if (IsBinaryOperator(token.Kind))
{
var newPrecedence = GetPrecedence(token.Kind);
if (newPrecedence < precedence)
{
break;
}
if (newPrecedence == precedence && IsLeftAssociative(token.Kind))
{
break;
}
EatToken();
var rhs = ParseSubExpression(options, newPrecedence);
if (rhs == null && token.Kind == TokenKind.Colon) // for parsing things like a{:}
{
rhs = Factory.EmptyExpression();
}
if (token.Kind == TokenKind.Assignment)
{
lhs = Factory.AssignmentExpression(lhs, Factory.Token(token), rhs);
}
else
{
lhs = Factory.BinaryOperationExpression(lhs, Factory.Token(token), rhs);
}
}
else
{
break;
}
}
return lhs;
}
private List<TokenNode> ParseOptionalCommas()
{
var commas = new List<TokenNode>();
while (CurrentToken.Kind == TokenKind.Comma)
{
commas.Add(Factory.Token(EatToken()));
}
return commas;
}
private List<TokenNode> ParseOptionalSemicolonsOrCommas()
{
var commas = new List<TokenNode>();
while (CurrentToken.Kind == TokenKind.Comma
|| CurrentToken.Kind == TokenKind.Semicolon)
{
commas.Add(Factory.Token(EatToken()));
}
return commas;
}
private SwitchCaseNode ParseSwitchCase()
{
var caseKeyword = EatIdentifier("case");
var caseId = ParseExpression();
var commas = ParseOptionalCommas();
var statementList = ParseStatements();
return Factory.SwitchCase(Factory.Token(caseKeyword), caseId, statementList, commas);
}
private SwitchStatementNode ParseSwitchStatement()
{
var switchKeyword = EatIdentifier("switch");
var expression = ParseExpression();
var commas = ParseOptionalCommas();
var casesList = new List<SwitchCaseNode>();
while (CurrentToken.Kind == TokenKind.Identifier
&& CurrentToken.PureToken.LiteralText == "case")
{
casesList.Add(ParseSwitchCase());
}
var endKeyword = EatIdentifier("end");
return Factory.SwitchStatement(
Factory.Token(switchKeyword),
expression,
casesList,
Factory.Token(endKeyword),
commas);
}
public ExpressionStatementNode ParseExpressionStatement()
{
var statement = ParseExpression();
var possibleSemicolonOrComma = PossibleSemicolonOrComma();
return Factory.ExpressionStatement(statement, possibleSemicolonOrComma);
}
public WhileStatementNode ParseWhileStatement()
{
var whileKeyword = EatToken();
var condition = ParseExpression();
var commas = ParseOptionalCommas();
var body = ParseStatements();
var endKeyword = EatIdentifier("end");
var semicolonOrComma = PossibleSemicolonOrComma();
return Factory.WhileStatement(
Factory.Token(whileKeyword),
condition,
commas,
body,
Factory.Token(endKeyword),
semicolonOrComma);
}
public StatementNode ParseStatement()
{
var statement = ParseStatementCore();
return statement;
}
public IfStatementNode ParseIfStatement()
{
var ifKeyword = Factory.Token(EatToken());
var condition = ParseExpression();
var commas = ParseOptionalSemicolonsOrCommas();
var body = ParseStatements();
TokenNode elseKeyword = null;
StatementListNode elseBody = null;
if (CurrentToken.Kind == TokenKind.Identifier
&& CurrentToken.PureToken.LiteralText == "else")
{
elseKeyword = Factory.Token(EatToken());
elseBody = ParseStatements();
}
if (CurrentToken.Kind == TokenKind.Identifier
&& CurrentToken.PureToken.LiteralText == "elseif")
{
elseKeyword = null;
var ifStatement = ParseIfStatement();
elseBody = Factory.StatementList(new List<SyntaxNode> { ifStatement });
return Factory.IfStatement(
ifKeyword,
condition,
commas,
body,
null,
elseBody,
null,
null);
}
var endKeyword = Factory.Token(EatIdentifier("end"));
var possibleSemicolonOrComma = PossibleSemicolonOrComma();
return Factory.IfStatement(
ifKeyword,
condition,
commas,
body,
elseKeyword,
elseBody,
endKeyword,
possibleSemicolonOrComma);
}
public ForStatementNode ParseForStatement()
{
var forKeyword = Factory.Token(EatIdentifier("for"));
var expression = ParseExpression();
if (!(expression is AssignmentExpressionNode))
{
throw new ParsingException($"Unexpected expression \"{expression.FullText}\" while parsing FOR statement at {CurrentToken.PureToken.Position}.");
}
var forAssignment = (AssignmentExpressionNode) expression;
var commas = ParseOptionalSemicolonsOrCommas();
var body = ParseStatements();
var endKeyword = Factory.Token(EatIdentifier("end"));
return Factory.ForStatement(forKeyword, forAssignment, body, endKeyword, commas);
}
public TryCatchStatementNode ParseTryCatchStatement()
{
var tryKeyword = Factory.Token(EatIdentifier("try"));
var tryBody = ParseStatements();
if (CurrentToken.PureToken.LiteralText == "catch")
{
var catchKeyword = Factory.Token(EatIdentifier("catch"));
var catchBody = ParseStatements();
var endKeyword = Factory.Token(EatIdentifier("end"));
return Factory.TryCatchStatement(tryKeyword, tryBody, catchKeyword, catchBody, endKeyword);
}
else if (CurrentToken.PureToken.LiteralText == "end")
{
var endKeyword = Factory.Token(EatIdentifier("end"));
return Factory.TryCatchStatement(tryKeyword, tryBody, endKeyword);
}
else
{
throw new ParsingException($"Unexpected token {CurrentToken.PureToken} while parsing try/catch statement at {CurrentToken.PureToken.Position}.");
}
}
public StatementNode ParseStatementCore()
{
if (CurrentToken.Kind == TokenKind.Identifier)
{
if (CurrentToken.PureToken.LiteralText == "function")
{
return ParseFunctionDeclaration();
}
else if (CurrentToken.PureToken.LiteralText == "classdef")
{
return ParseClassDeclaration();
}
else if (CurrentToken.PureToken.LiteralText == "switch")
{
return ParseSwitchStatement();
}
else if (CurrentToken.PureToken.LiteralText == "while")
{
return ParseWhileStatement();
}
else if (CurrentToken.PureToken.LiteralText == "if")
{
return ParseIfStatement();
}
else if (CurrentToken.PureToken.LiteralText == "case")
{
return null;
}
else if (CurrentToken.PureToken.LiteralText == "else")
{
return null;
}
else if (CurrentToken.PureToken.LiteralText == "elseif")
{
return null;
}
else if (CurrentToken.PureToken.LiteralText == "end")
{
return null;
}
else if (CurrentToken.PureToken.LiteralText == "for")
{
return ParseForStatement();
}
else if (CurrentToken.PureToken.LiteralText == "try")
{
return ParseTryCatchStatement();
}
else if (CurrentToken.PureToken.LiteralText == "catch")
{
return null;
}
return ParseExpressionStatement();
}
if (CurrentToken.Kind == TokenKind.OpeningSquareBracket)
{
return ParseExpressionStatement();
}
if (CurrentToken.Kind == TokenKind.Semicolon)
{
return Factory.ExpressionStatement(Factory.EmptyExpression(), Factory.Token(EatToken()));
}
throw new ParsingException($"Unexpected token: \"{CurrentToken.PureToken}\" at {CurrentToken.PureToken.Position}");
}
private StatementListNode ParseStatements()
{
var statements = new List<SyntaxNode>();
while (CurrentToken.PureToken.Kind != TokenKind.EndOfFile)
{
var node = ParseStatement();
if (node == null)
{
break;
}
statements.Add(node);
}
return Factory.StatementList(statements);
}
private StatementListNode ParseFile()
{
return ParseStatements();
}
public StatementListNode Parse()
{
return ParseFile();
} }
} }
} }

4
Parser/Parser.csproj
View File

@ -1,8 +1,8 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>netcoreapp2.0</TargetFramework> <TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Lexer\Lexer.csproj" /> <PackageReference Include="System.Collections.Immutable" Version="1.5.0" />
</ItemGroup> </ItemGroup>
</Project> </Project>

2
Lexer/ParsingException.cs → Parser/ParsingException.cs
View File

@ -1,6 +1,6 @@
using System; using System;
namespace Lexer namespace Parser
{ {
public class ParsingException : Exception public class ParsingException : Exception
{ {

14
Parser/Position.cs Normal file
View File

@ -0,0 +1,14 @@
namespace Parser
{
public struct Position
{
public string FileName { get; set; }
public int Line { get; set; }
public int Column { get; set; }
public override string ToString()
{
return $"line {Line}, column {Column} of {FileName}.";
}
}
}

592
Parser/SyntaxFactory.cs
View File

@ -1,592 +0,0 @@
using System.Collections.Generic;
using System.Linq;
using Lexer;
namespace Parser
{
public class SyntaxFactory
{
private static List<SyntaxNode> RemoveNulls(List<SyntaxNode> children)
{
return children.Where(x => x != null).ToList();
}
public FunctionDeclarationNode FunctionDeclaration(
TokenNode functionKeyword,
FunctionOutputDescriptionNode outputDescription,
TokenNode name,
FunctionInputDescriptionNode inputDescription,
StatementListNode body,
TokenNode end,
TokenNode semicolonOrComma = null)
{
var children = new List<SyntaxNode>
{
functionKeyword,
outputDescription,
name,
inputDescription,
body,
end,
semicolonOrComma
};
return new FunctionDeclarationNode(
RemoveNulls(children),
functionKeyword,
outputDescription,
name,
inputDescription,
body,
end,
semicolonOrComma);
}
public FunctionOutputDescriptionNode FunctionOutputDescription(
List<SyntaxNode> nodes,
TokenNode equalitySign)
{
var children = new List<SyntaxNode>(nodes);
children.Add(equalitySign);
return new FunctionOutputDescriptionNode(
children,
children
.Where(node => node is TokenNode tokenNode && tokenNode.Token.Kind == TokenKind.Identifier)
.Select(node => (TokenNode)node)
.ToList(),
equalitySign
);
}
public ParameterListNode ParameterList(List<SyntaxNode> nodes)
{
return new ParameterListNode(
nodes,
nodes
.Where(
node => node is TokenNode tokenNode && tokenNode.Token.Kind != TokenKind.Comma
)
.ToList());
}
public StatementListNode StatementList(List<SyntaxNode> nodes)
{
return new StatementListNode(nodes);
}
public FunctionInputDescriptionNode FunctionInputDescription(
TokenNode openingBracket,
ParameterListNode parameterList,
TokenNode closingBracket)
{
var children = new List<SyntaxNode>
{
openingBracket,
parameterList,
closingBracket
};
return new FunctionInputDescriptionNode(children, openingBracket, parameterList, closingBracket);
}
public TokenNode Token(Token token)
{
return new TokenNode(token);
}
public SwitchStatementNode SwitchStatement(
TokenNode switchKeyword,
ExpressionNode switchExpression,
List<SwitchCaseNode> cases,
TokenNode endKeyword,
List<TokenNode> optionalCommasAfterExpression,
TokenNode semicolonOrComma = null)
{
var children = new List<SyntaxNode> { switchKeyword, switchExpression };
children.AddRange(optionalCommasAfterExpression);
children.AddRange(cases);
children.Add(endKeyword);
children.Add(semicolonOrComma);
return new SwitchStatementNode(
RemoveNulls(children),
switchKeyword,
switchExpression,
cases,
endKeyword,
semicolonOrComma,
optionalCommasAfterExpression);
}
public SwitchCaseNode SwitchCase(
TokenNode caseKeyword,
ExpressionNode caseIdentifier,
StatementListNode statementList,
List<TokenNode> optionalCommasAfterIdentifier)
{
var children = new List<SyntaxNode>
{
caseKeyword,
caseIdentifier
};
children.AddRange(optionalCommasAfterIdentifier);
children.Add(statementList);
return new SwitchCaseNode(
RemoveNulls(children),
caseKeyword,
caseIdentifier,
statementList,
optionalCommasAfterIdentifier);
}
public AssignmentExpressionNode AssignmentExpression(
ExpressionNode lhs,
TokenNode assignmentSign,
ExpressionNode rhs)
{
var children = new List<SyntaxNode>
{
lhs,
assignmentSign,
rhs
};
return new AssignmentExpressionNode(children, lhs, assignmentSign, rhs);
}
public UnaryPrefixOperationExpressionNode UnaryPrefixOperationExpression(
TokenNode operation,
ExpressionNode operand)
{
var children = new List<SyntaxNode>
{
operation,
operand
};
return new UnaryPrefixOperationExpressionNode(children, operation, operand);
}
public UnaryPostfixOperationExpressionNode UnaryPostfixOperationExpression(
ExpressionNode operand,
TokenNode operation)
{
var children = new List<SyntaxNode>
{
operand,
operation
};
return new UnaryPostfixOperationExpressionNode(children, operand, operation);
}
public BinaryOperationExpressionNode BinaryOperationExpression(
ExpressionNode lhs,
TokenNode operation,
ExpressionNode rhs)
{
var children = new List<SyntaxNode>
{
lhs,
operation,
rhs
};
return new BinaryOperationExpressionNode(children, lhs, operation, rhs);
}
public IdentifierNameNode IdentifierName(
Token identifier)
{
return new IdentifierNameNode(identifier);
}
public NumberLiteralNode NumberLiteral(
Token numberLiteral)
{
return new NumberLiteralNode(numberLiteral);
}
public StringLiteralNode StringLiteral(
Token stringLiteral)
{
return new StringLiteralNode(stringLiteral);
}
public DoubleQuotedStringLiteralNode DoubleQuotedStringLiteral(
Token stringLiteral)
{
return new DoubleQuotedStringLiteralNode(stringLiteral);
}
public UnquotedStringLiteralNode UnquotedStringLiteral(
Token stringLiteral)
{
return new UnquotedStringLiteralNode(stringLiteral);
}
public ExpressionStatementNode ExpressionStatement(ExpressionNode expression, TokenNode semicolonOrComma)
{
var children = new List<SyntaxNode> {expression, semicolonOrComma};
return new ExpressionStatementNode(
RemoveNulls(children),
expression,
semicolonOrComma);
}
public CellArrayElementAccessExpressionNode CellArrayElementAccessExpression(
ExpressionNode cellArray,
TokenNode openingBrace,
ArrayElementListNode indices,
TokenNode closingBrace)
{
var children = new List<SyntaxNode> {cellArray, openingBrace, indices, closingBrace};
return new CellArrayElementAccessExpressionNode(
children,
cellArray,
openingBrace,
indices,
closingBrace);
}
public FunctionCallExpressionNode FunctionCallExpression(
ExpressionNode functionName,
TokenNode openingBracket,
FunctionCallParameterListNode parameters,
TokenNode closingBracket)
{
var children = new List<SyntaxNode>
{
functionName,
openingBracket,
parameters,
closingBracket
};
return new FunctionCallExpressionNode(
children,
functionName,
openingBracket,
parameters,
closingBracket);
}
public FunctionCallParameterListNode FunctionCallParameterList(List<SyntaxNode> nodes)
{
return new FunctionCallParameterListNode(
nodes,
nodes
.OfType<ExpressionNode>()
.ToList());
}
public ArrayElementListNode ArrayElementList(List<SyntaxNode> nodes)
{
return new ArrayElementListNode(
nodes,
nodes
.OfType<ExpressionNode>()
.ToList());
}
public CompoundNameNode CompoundName(List<SyntaxNode> nodes)
{
return new CompoundNameNode(
nodes,
nodes
.OfType<IdentifierNameNode>()
.ToList());
}
public ArrayLiteralExpressionNode ArrayLiteralExpression(
TokenNode openingSquareBracket,
ArrayElementListNode elements,
TokenNode closingSquareBracket)
{
var children = new List<SyntaxNode>
{
openingSquareBracket,
elements,
closingSquareBracket
};
return new ArrayLiteralExpressionNode(
children,
openingSquareBracket,
elements,
closingSquareBracket);
}
public CellArrayLiteralExpressionNode CellArrayLiteralExpression(
TokenNode openingBrace,
ArrayElementListNode elements,
TokenNode closingBrace)
{
var children = new List<SyntaxNode>
{
openingBrace,
elements,
closingBrace
};
return new CellArrayLiteralExpressionNode(
children,
openingBrace,
elements,
closingBrace);
}
public EmptyExpressionNode EmptyExpression()
{
return new EmptyExpressionNode();
}
public MemberAccessNode MemberAccess(
SyntaxNode leftOperand,
TokenNode dot,
SyntaxNode rightOperand)
{
var children = new List<SyntaxNode>
{
leftOperand,
dot,
rightOperand
};
return new MemberAccessNode(
children,
leftOperand,
dot,
rightOperand);
}
public WhileStatementNode WhileStatement(
TokenNode whileKeyword,
ExpressionNode condition,
List<TokenNode> optionalCommasAfterCondition,
StatementListNode body,
TokenNode end,
TokenNode semicolonOrComma)
{
var children = new List<SyntaxNode>
{
whileKeyword,
condition,
};
children.AddRange(optionalCommasAfterCondition);
children.Add(body);
children.Add(end);
children.Add(semicolonOrComma);
return new WhileStatementNode(
RemoveNulls(children),
whileKeyword,
condition,
optionalCommasAfterCondition,
body,
end,
semicolonOrComma);
}
public StatementNode AppendSemicolonOrComma(StatementNode statement, TokenNode semicolonOrComma)
{
statement.SemicolonOrComma = semicolonOrComma;
statement.Children.Add(semicolonOrComma);
statement.Children[statement.Children.Count - 1].Parent = statement;
return statement;
}
public IfStatementNode IfStatement(
TokenNode ifKeyword,
ExpressionNode condition,
List<TokenNode> optionalCommasAfterCondition,
StatementListNode body,
TokenNode elseKeyword,
StatementListNode elseBody,
TokenNode endKeyword,
TokenNode possibleSemicolonOrComma)
{
var children = new List<SyntaxNode>
{
ifKeyword,
condition
};
children.AddRange(optionalCommasAfterCondition);
children.Add(body);
children.Add(elseKeyword);
children.Add(elseBody);
children.Add(endKeyword);
children.Add(possibleSemicolonOrComma);
return new IfStatementNode(
RemoveNulls(children),
ifKeyword,
condition,
optionalCommasAfterCondition,
body,
elseKeyword,
elseBody,
endKeyword,
possibleSemicolonOrComma);
}
public ParenthesizedExpressionNode ParenthesizedExpression(
TokenNode openParen,
ExpressionNode expression,
TokenNode closeParen)
{
var children = new List<SyntaxNode>
{
openParen,
expression,
closeParen
};
return new ParenthesizedExpressionNode(
children,
openParen,
expression,
closeParen);
}
public ForStatementNode ForStatement(
TokenNode forKeyword,
AssignmentExpressionNode forAssignment,
StatementListNode body,
TokenNode endKeyword,
List<TokenNode> optionalCommasAfterAssignment)
{
var children = new List<SyntaxNode>
{
forKeyword,
forAssignment,
};
children.AddRange(optionalCommasAfterAssignment);
children.Add(body);
children.Add(endKeyword);
return new ForStatementNode(
RemoveNulls(children),
forKeyword,
forAssignment,
body,
endKeyword,
optionalCommasAfterAssignment);
}
public IndirectMemberAccessNode IndirectMemberAccess(
TokenNode openingBracket,
ExpressionNode indirectMemberName,
TokenNode closingBracket)
{
var children = new List<SyntaxNode>
{
openingBracket,
indirectMemberName,
closingBracket
};
return new IndirectMemberAccessNode(
children,
openingBracket,
indirectMemberName,
closingBracket);
}
public NamedFunctionHandleNode NamedFunctionHandle(
TokenNode atSign,
CompoundNameNode functionName)
{
var children = new List<SyntaxNode>
{
atSign,
functionName
};
return new NamedFunctionHandleNode(
children,
atSign,
functionName);
}
public LambdaNode Lambda(
TokenNode atSign,
FunctionInputDescriptionNode input,
ExpressionNode body)
{
var children = new List<SyntaxNode>
{
atSign,
input,
body
};
return new LambdaNode(
children,
atSign,
input,
body);
}
public TryCatchStatementNode TryCatchStatement(
TokenNode tryKeyword,
StatementListNode tryBody,
TokenNode catchKeyword,
StatementListNode catchBody,
TokenNode endKeyword)
{
var children = new List<SyntaxNode>
{
tryKeyword,
tryBody,
catchKeyword,
catchBody,
endKeyword
};
return new TryCatchStatementNode(
children,
tryKeyword,
tryBody,
catchKeyword,
catchBody,
endKeyword);
}
public TryCatchStatementNode TryCatchStatement(
TokenNode tryKeyword,
StatementListNode tryBody,
TokenNode endKeyword)
{
var children = new List<SyntaxNode>
{
tryKeyword,
tryBody,
endKeyword
};
return new TryCatchStatementNode(
children,
tryKeyword,
tryBody,
null,
null,
endKeyword);
}
public CommandExpressionNode CommandExpression(
IdentifierNameNode identifierName,
List<UnquotedStringLiteralNode> arguments)
{
var children = new List<SyntaxNode>
{
identifierName
};
children.AddRange(arguments);
return new CommandExpressionNode(
children,
identifierName,
arguments);
}
public BaseClassInvokationNode BaseClassInvokation(
IdentifierNameNode methodName,
TokenNode atToken,
ExpressionNode baseClassNameAndArguments)
{
var children = new List<SyntaxNode>
{
methodName,
atToken,
baseClassNameAndArguments
};
return new BaseClassInvokationNode(
children,
methodName,
atToken,
baseClassNameAndArguments);
}
}
}

2631
Parser/SyntaxNode.Generated.cs Normal file
View File

File diff suppressed because it is too large Load Diff

770
Parser/SyntaxNode.cs
View File

@ -1,750 +1,100 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq; using System.Linq;
using Lexer;
namespace Parser namespace Parser
{ {
public class SyntaxNode public abstract class SyntaxNode
{ {
public SyntaxNode Parent { get; set; } private readonly SyntaxNode _parent;
public List<SyntaxNode> Children { get; } internal readonly Internal.GreenNode _green;
internal SyntaxNode(SyntaxNode parent, Internal.GreenNode green)
public virtual IEnumerable<Token> ChildTokens =>
Children.SelectMany(c => c.ChildTokens);
public SyntaxNode(List<SyntaxNode> children)
{ {
Children = children; _parent = parent;
if (children != null) _green = green;
{
foreach (var child in children)
{
child.Parent = this;
}
}
} }
public virtual string FullText => public TokenKind Kind => _green.Kind;
string.Join("", Children.Select(c => c.FullText));
public List<Trivia> TrailingTrivia public SyntaxNode Parent => _parent;
public ChildNodesAndTokensList GetChildNodesAndTokens()
{
return new ChildNodesAndTokensList(this);
}
internal abstract SyntaxNode GetNode(int index);
internal SyntaxNode GetRed(ref SyntaxNode field, int slot)
{
if (field == null)
{
var green = _green.GetSlot(slot);
if (green != null)
{
field = green.CreateRed(this);
}
}
return field;
}
public override string ToString()
{
return Text;
}
public virtual string Text => _green.Text;
public virtual string FullText => _green.FullText;
public virtual IReadOnlyList<SyntaxTrivia> LeadingTrivia
{ {
get get
{ {
if (ChildTokens.Any()) var p = Parent;
{ return _green.LeadingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
return ChildTokens.Last().TrailingTrivia;
}
else
{
return new List<Trivia>();
}
} }
} }
public virtual IReadOnlyList<SyntaxTrivia> TrailingTrivia
{
get
{
var p = Parent;
return _green.TrailingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
}
}
public abstract void Accept(SyntaxVisitor visitor);
} }
public class TokenNode : SyntaxNode public abstract class StatementSyntaxNode : SyntaxNode
{ {
public Token Token { get; } internal StatementSyntaxNode(SyntaxNode parent, Internal.GreenNode green) : base(parent, green)
public TokenNode(Token token)
: base(null)
{ {
Token = token;
}
public override string FullText => Token.FullText;
public override IEnumerable<Token> ChildTokens
{
get { yield return Token; }
} }
} }
public class OutputIdentifierNode : SyntaxNode public abstract class ExpressionSyntaxNode : SyntaxNode
{ {
public OutputIdentifierNode(List<SyntaxNode> children) : base(children) internal ExpressionSyntaxNode(SyntaxNode parent, Internal.GreenNode green) : base(parent, green)
{ {
} }
} }
public class FunctionOutputDescriptionNode : SyntaxNode public abstract class FunctionHandleSyntaxNode : ExpressionSyntaxNode
{ {
public List<TokenNode> Outputs { get; } internal FunctionHandleSyntaxNode(SyntaxNode parent, Internal.GreenNode green) : base(parent, green)
public TokenNode EqualitySign { get; }
public FunctionOutputDescriptionNode(
List<SyntaxNode> children,
List<TokenNode> outputs,
TokenNode equalitySign) : base(children)
{ {
Outputs = outputs;
EqualitySign = equalitySign;
} }
} }
public class FunctionInputDescriptionNode : SyntaxNode public abstract class MethodDeclarationSyntaxNode : StatementSyntaxNode
{ {
public TokenNode OpeningBracket { get; } internal MethodDeclarationSyntaxNode(SyntaxNode parent, Internal.GreenNode green) : base(parent, green)
public ParameterListNode Parameters { get; }
public TokenNode ClosingBracket { get; }
public FunctionInputDescriptionNode(
List<SyntaxNode> children,
TokenNode openingBracket,
ParameterListNode parameters,
TokenNode closingBracket) : base(children)
{
OpeningBracket = openingBracket;
Parameters = parameters;
ClosingBracket = closingBracket;
}
}
public class FunctionDeclarationNode : StatementNode
{
public TokenNode Token { get; }
public FunctionOutputDescriptionNode OutputDescription { get; }
public TokenNode Name { get; }
public FunctionInputDescriptionNode InputDescription { get; }
public StatementListNode Body { get; }
public TokenNode End { get; }
public FunctionDeclarationNode(
List<SyntaxNode> children,
TokenNode token,
FunctionOutputDescriptionNode outputDescription,
TokenNode name,
FunctionInputDescriptionNode inputDescription,
StatementListNode body,
TokenNode end,
TokenNode semicolonOrComma
) : base(children, semicolonOrComma)
{
Token = token;
OutputDescription = outputDescription;
Name = name;
InputDescription = inputDescription;
Body = body;
End = end;
}
}
public class StatementListNode : SyntaxNode
{
public List<SyntaxNode> Statements => Children;
public StatementListNode(List<SyntaxNode> children) : base(children)
{ {
} }
} }
public class ParameterListNode : SyntaxNode
{
public List<SyntaxNode> Parameters { get; }
public ParameterListNode(List<SyntaxNode> children, List<SyntaxNode> parameters) : base(children)
{
Parameters = parameters;
}
}
public class ExpressionNode : SyntaxNode
{
public ExpressionNode(List<SyntaxNode> children) : base(children)
{
}
}
public class AssignmentExpressionNode : ExpressionNode
{
public ExpressionNode Lhs { get; }
public TokenNode Assignment { get; }
public ExpressionNode Rhs { get; }
public AssignmentExpressionNode(
List<SyntaxNode> children,
ExpressionNode lhs,
TokenNode assignment,
ExpressionNode rhs) : base(children)
{
Lhs = lhs;
Assignment = assignment;
Rhs = rhs;
}
}
public class UnaryPrefixOperationExpressionNode : ExpressionNode
{
public TokenNode Operation { get; }
public ExpressionNode Operand { get; }
public UnaryPrefixOperationExpressionNode(
List<SyntaxNode> children,
TokenNode operation,
ExpressionNode operand) : base(children)
{
Operation = operation;
Operand = operand;
}
}
public class UnaryPostfixOperationExpressionNode : ExpressionNode
{
public ExpressionNode Operand { get; }
public TokenNode Operation { get; }
public UnaryPostfixOperationExpressionNode(
List<SyntaxNode> children,
ExpressionNode operand,
TokenNode operation) : base(children)
{
Operand = operand;
Operation = operation;
}
}
public class BinaryOperationExpressionNode : ExpressionNode
{
public ExpressionNode Lhs { get; }
public TokenNode Operation { get; }
public ExpressionNode Rhs { get; }
public BinaryOperationExpressionNode(
List<SyntaxNode> children,
ExpressionNode lhs,
TokenNode operation,
ExpressionNode rhs) : base(children)
{
Lhs = lhs;
Operation = operation;
Rhs = rhs;
}
}
public class SwitchStatementNode : StatementNode
{
public TokenNode SwitchKeyword { get; }
public ExpressionNode SwitchExpression { get; }
public List<TokenNode> OptionalCommasAfterExpression { get; }
public List<SwitchCaseNode> Cases { get; }
public TokenNode EndKeyword { get; }
public SwitchStatementNode(
List<SyntaxNode> children,
TokenNode switchKeyword,
ExpressionNode switchExpression,
List<SwitchCaseNode> cases,
TokenNode endKeyword,
TokenNode semicolonOrComma,
List<TokenNode> optionalCommasAfterExpression = null
) : base(children, semicolonOrComma)
{
SwitchKeyword = switchKeyword;
SwitchExpression = switchExpression;
OptionalCommasAfterExpression = optionalCommasAfterExpression;
Cases = cases;
EndKeyword = endKeyword;
}
}
public class SwitchCaseNode : SyntaxNode
{
public TokenNode CaseKeyword { get; }
public ExpressionNode CaseIdentifier { get; }
public List<TokenNode> OptionalCommasAfterIdentifier { get; }
public StatementListNode StatementList { get; }
public SwitchCaseNode(
List<SyntaxNode> children,
TokenNode caseKeyword,
ExpressionNode caseIdentifier,
StatementListNode statementList,
List<TokenNode> optionalCommasAfterIdentifier = null
) : base(children)
{
CaseKeyword = caseKeyword;
CaseIdentifier = caseIdentifier;
StatementList = statementList;
OptionalCommasAfterIdentifier = optionalCommasAfterIdentifier;
}
}
public class IdentifierNameNode : ExpressionNode
{
public Token Token { get; }
public IdentifierNameNode(Token token)
: base(null)
{
Token = token;
}
public override string FullText => Token.FullText;
public override IEnumerable<Token> ChildTokens
{
get { yield return Token; }
}
}
public class NumberLiteralNode : ExpressionNode
{
public Token Token { get; }
public NumberLiteralNode(Token token) : base(null)
{
Token = token;
}
public override string FullText => Token.FullText;
public override IEnumerable<Token> ChildTokens
{
get { yield return Token; }
}
}
public class StringLiteralNode : ExpressionNode
{
public Token Token { get; }
public StringLiteralNode(Token token) : base(null)
{
Token = token;
}
public override string FullText => Token.FullText;
public override IEnumerable<Token> ChildTokens
{
get { yield return Token; }
}
}
public class DoubleQuotedStringLiteralNode : ExpressionNode
{
public Token Token { get; }
public DoubleQuotedStringLiteralNode(Token token) : base(null)
{
Token = token;
}
public override string FullText => Token.FullText;
public override IEnumerable<Token> ChildTokens
{
get { yield return Token; }
}
}
public class UnquotedStringLiteralNode : ExpressionNode
{
public Token Token { get; }
public UnquotedStringLiteralNode(Token token) : base(null)
{
Token = token;
}
public override string FullText => Token.FullText;
public override IEnumerable<Token> ChildTokens
{
get { yield return Token; }
}
}
public class StatementNode : SyntaxNode
{
public TokenNode SemicolonOrComma { get; set; }
public StatementNode(List<SyntaxNode> children, TokenNode semicolonOrComma = null) : base(children)
{
SemicolonOrComma = semicolonOrComma;
}
}
public class ExpressionStatementNode : StatementNode
{
public ExpressionNode Expression { get; }
public ExpressionStatementNode(List<SyntaxNode> children, ExpressionNode expression, TokenNode semicolonOrComma)
: base(children, semicolonOrComma)
{
Expression = expression;
}
}
public class CellArrayElementAccessExpressionNode : ExpressionNode
{
public ExpressionNode CellArray { get; }
public TokenNode OpeningBrace { get; }
public ArrayElementListNode Indices { get; }
public TokenNode ClosingBrace { get; }
public CellArrayElementAccessExpressionNode(
List<SyntaxNode> children,
ExpressionNode cellArray,
TokenNode openingBrace,
ArrayElementListNode indices,
TokenNode closingBrace) : base(children)
{
CellArray = cellArray;
OpeningBrace = openingBrace;
Indices = indices;
ClosingBrace = closingBrace;
}
}
public class FunctionCallExpressionNode : ExpressionNode
{
public ExpressionNode FunctionName { get; }
public TokenNode OpeningBracket { get; }
public FunctionCallParameterListNode Parameters { get; }
public TokenNode ClosingBracket { get; }
public FunctionCallExpressionNode(
List<SyntaxNode> children,
ExpressionNode functionName,
TokenNode openingBracket,
FunctionCallParameterListNode parameters,
TokenNode closingBracket) : base(children)
{
FunctionName = functionName;
OpeningBracket = openingBracket;
Parameters = parameters;
ClosingBracket = closingBracket;
}
}
public class FunctionCallParameterListNode : SyntaxNode
{
public List<ExpressionNode> Parameters;
public FunctionCallParameterListNode(
List<SyntaxNode> children,
List<ExpressionNode> parameters) : base(children)
{
Parameters = parameters;
}
}
public class ArrayElementListNode : SyntaxNode
{
public List<ExpressionNode> Elements;
public ArrayElementListNode(
List<SyntaxNode> children,
List<ExpressionNode> elements) : base(children)
{
Elements = elements;
}
}
public class ArrayLiteralExpressionNode : ExpressionNode
{
public TokenNode OpeningSquareBracket { get; }
public ArrayElementListNode Elements { get; }
public TokenNode ClosingSquareBracket { get; }
public ArrayLiteralExpressionNode(
List<SyntaxNode> children,
TokenNode openingSquareBracket,
ArrayElementListNode elements,
TokenNode closingSquareBracket) : base(children)
{
OpeningSquareBracket = openingSquareBracket;
Elements = elements;
ClosingSquareBracket = closingSquareBracket;
}
}
public class CellArrayLiteralExpressionNode : ExpressionNode
{
public TokenNode OpeningBrace { get; }
public ArrayElementListNode Elements { get; }
public TokenNode ClosingBrace { get; }
public CellArrayLiteralExpressionNode(
List<SyntaxNode> children,
TokenNode openingBrace,
ArrayElementListNode elements,
TokenNode closingBrace) : base(children)
{
OpeningBrace = openingBrace;
Elements = elements;
ClosingBrace = closingBrace;
}
}
public class EmptyExpressionNode : ExpressionNode
{
public EmptyExpressionNode() : base(null)
{
}
public override IEnumerable<Token> ChildTokens
{
get { yield break; }
}
public override string FullText => "";
}
public class CompoundNameNode : ExpressionNode
{
public List<IdentifierNameNode> Names;
public CompoundNameNode(
List<SyntaxNode> children,
List<IdentifierNameNode> names
) : base(children)
{
Names = names;
}
}
public class MemberAccessNode : ExpressionNode
{
public SyntaxNode LeftOperand { get; }
public TokenNode Dot { get; }
public SyntaxNode RightOperand { get; }
public MemberAccessNode(
List<SyntaxNode> children,
SyntaxNode leftOperand,
TokenNode dot,
SyntaxNode rightOperand) : base(children)
{
LeftOperand = leftOperand;
Dot = dot;
RightOperand = rightOperand;
}
}
public class WhileStatementNode : StatementNode
{
public TokenNode WhileKeyword { get; }
public ExpressionNode Condition { get; }
public List<TokenNode> OptionalCommasAfterCondition { get; }
public StatementListNode Body { get; }
public TokenNode End { get; }
public WhileStatementNode(
List<SyntaxNode> children,
TokenNode whileKeyword,
ExpressionNode condition,
List<TokenNode> optionalCommasAfterCondition,
StatementListNode body,
TokenNode end,
TokenNode semicolonOrComma
) : base(children, semicolonOrComma)
{
WhileKeyword = whileKeyword;
Condition = condition;
OptionalCommasAfterCondition = optionalCommasAfterCondition;
Body = body;
End = end;
}
}
public class IfStatementNode : StatementNode
{
public TokenNode IfKeyword { get; }
public ExpressionNode Condition { get; }
public List<TokenNode> OptionalCommasAfterCondition { get; }
public StatementListNode Body { get; }
public TokenNode ElseKeyword { get; }
public StatementListNode ElseBody { get; }
public TokenNode EndKeyword { get; }
public IfStatementNode(
List<SyntaxNode> children,
TokenNode ifKeyword,
ExpressionNode condition,
List<TokenNode> optionalCommasAfterCondition,
StatementListNode body,
TokenNode elseKeyword,
StatementListNode elseBody,
TokenNode endKeyword,
TokenNode possibleSemicolonOrComma
) : base(children, possibleSemicolonOrComma)
{
IfKeyword = ifKeyword;
Condition = condition;
OptionalCommasAfterCondition = optionalCommasAfterCondition;
Body = body;
ElseKeyword = elseKeyword;
ElseBody = elseBody;
EndKeyword = endKeyword;
}
}
public class ParenthesizedExpressionNode : ExpressionNode
{
public TokenNode OpenParen { get; }
public ExpressionNode Expression { get; }
public TokenNode CloseParen { get; }
public ParenthesizedExpressionNode(
List<SyntaxNode> children,
TokenNode openParen,
ExpressionNode expression,
TokenNode closeParen) : base(children)
{
OpenParen = openParen;
Expression = expression;
CloseParen = closeParen;
}
}
public class ForStatementNode : StatementNode
{
public TokenNode ForKeyword { get; }
public AssignmentExpressionNode ForAssignment { get; }
public List<TokenNode> OptionalCommasAfterAssignment { get; }
public StatementListNode Body { get; }
public TokenNode EndKeyword { get; }
public ForStatementNode(
List<SyntaxNode> children,
TokenNode forKeyword,
AssignmentExpressionNode forAssignment,
StatementListNode body,
TokenNode endKeyword,
List<TokenNode> optionalCommasAfterAssignment
) : base(children)
{
ForKeyword = forKeyword;
ForAssignment = forAssignment;
Body = body;
EndKeyword = endKeyword;
OptionalCommasAfterAssignment = optionalCommasAfterAssignment;
}
}
public class IndirectMemberAccessNode : ExpressionNode
{
public TokenNode OpeningBracket { get; }
public ExpressionNode IndirectMemberName { get; }
public TokenNode ClosingBracket { get; }
public IndirectMemberAccessNode(
List<SyntaxNode> children,
TokenNode openingBracket,
ExpressionNode indirectMemberName,
TokenNode closingBracket) : base(children)
{
OpeningBracket = openingBracket;
IndirectMemberName = indirectMemberName;
ClosingBracket = closingBracket;
}
}
public abstract class FunctionHandleNode : ExpressionNode
{
protected FunctionHandleNode(
List<SyntaxNode> children) : base(children)
{
}
}
public class NamedFunctionHandleNode : FunctionHandleNode
{
public TokenNode AtSign { get; }
public CompoundNameNode FunctionName { get; }
public NamedFunctionHandleNode(
List<SyntaxNode> children,
TokenNode atSign,
CompoundNameNode functionName) : base(children)
{
AtSign = atSign;
FunctionName = functionName;
}
}
public class LambdaNode : FunctionHandleNode
{
public TokenNode AtSign { get; }
public FunctionInputDescriptionNode Input { get; }
public ExpressionNode Body { get; }
public LambdaNode(
List<SyntaxNode> children,
TokenNode atSign,
FunctionInputDescriptionNode input,
ExpressionNode body) : base(children)
{
AtSign = atSign;
Input = input;
Body = body;
}
}
public class TryCatchStatementNode : StatementNode
{
public TokenNode TryKeyword { get; }
public StatementListNode TryBody { get; }
public TokenNode CatchKeyword { get; }
public StatementListNode CatchBody { get; }
public TokenNode EndKeyword { get; }
public TryCatchStatementNode(
List<SyntaxNode> children,
TokenNode tryKeyword,
StatementListNode tryBody,
TokenNode catchKeyword,
StatementListNode catchBody,
TokenNode endKeyword
) : base(children)
{
TryKeyword = tryKeyword;
TryBody = tryBody;
CatchKeyword = catchKeyword;
CatchBody = catchBody;
EndKeyword = endKeyword;
}
}
public class CommandExpressionNode : ExpressionNode
{
public IdentifierNameNode CommandName { get; }
public List<UnquotedStringLiteralNode> Arguments { get; }
public CommandExpressionNode(
List<SyntaxNode> children,
IdentifierNameNode commandName,
List<UnquotedStringLiteralNode> arguments
) : base(children)
{
CommandName = commandName;
Arguments = arguments;
}
}
public class BaseClassInvokationNode : ExpressionNode
{
public IdentifierNameNode MethodName { get; }
public TokenNode AtToken { get; }
public ExpressionNode BaseClassNameAndArguments { get; }
public BaseClassInvokationNode(
List<SyntaxNode> children,
IdentifierNameNode methodName,
TokenNode atToken,
ExpressionNode baseClassNameAndArguments
) : base(children)
{
MethodName = methodName;
AtToken = atToken;
BaseClassNameAndArguments = baseClassNameAndArguments;
}
}
} }

55
Parser/SyntaxNodeOrToken.cs Normal file
View File

@ -0,0 +1,55 @@
namespace Parser
{
public struct SyntaxNodeOrToken
{
private readonly Internal.GreenNode _token;
private readonly SyntaxNode _nodeOrParent;
private readonly bool _isToken;
internal SyntaxNodeOrToken(SyntaxNode node)
{
_token = null;
_nodeOrParent = node;
_isToken = false;
}
internal SyntaxNodeOrToken(SyntaxNode parent, Internal.GreenNode token)
{
_token = token;
_nodeOrParent = parent;
_isToken = true;
}
public bool IsToken => _isToken;
public bool IsNode => !IsToken;
public SyntaxNode AsNode()
{
if (_isToken)
{
return default(SyntaxNode);
}
return _nodeOrParent;
}
public SyntaxToken AsToken()
{
if (!_isToken)
{
return default(SyntaxToken);
}
return new SyntaxToken(_nodeOrParent, _token);
}
public static implicit operator SyntaxNodeOrToken(SyntaxToken token)
{
return new SyntaxNodeOrToken(token.Parent, token.Token);
}
public static implicit operator SyntaxNodeOrToken(SyntaxNode node)
{
return new SyntaxNodeOrToken(node);
}
}
}

103
Parser/SyntaxNodeOrTokenList.cs Normal file
View File

@ -0,0 +1,103 @@
using System;
using System.Collections;
using System.Collections.Generic;
using Parser.Internal;
namespace Parser
{
public class SyntaxNodeOrTokenList : SyntaxNode, IReadOnlyCollection<SyntaxNodeOrToken>
{
internal SyntaxNodeOrTokenList(SyntaxNode parent, GreenNode green) : base(parent, green)
{
}
public SyntaxNodeOrToken this[int index]
{
get
{
if (_green != null && index < _green.Slots)
{
var green = _green.GetSlot(index);
if (green is Internal.SyntaxToken)
{
return new SyntaxToken(this, green);
}
else
{
return green.CreateRed(this);
}
}
throw new ArgumentOutOfRangeException();
}
}
internal class Enumerator : IEnumerator<SyntaxNodeOrToken>
{
private int _index;
private readonly SyntaxNodeOrTokenList _list;
internal Enumerator(SyntaxNodeOrTokenList list)
{
_index = -1;
_list = list;
}
public bool MoveNext()
{
var newIndex = _index + 1;
if (newIndex < _list.Count)
{
_index = newIndex;
return true;
}
return false;
}
public void Reset()
{
throw new NotSupportedException();
}
public SyntaxNodeOrToken Current => _list[_index];
object IEnumerator.Current => Current;
public void Dispose()
{
}
}
public IEnumerator<SyntaxNodeOrToken> GetEnumerator()
{
return new Enumerator(this);
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
public int Count => _green.Slots;
internal override SyntaxNode GetNode(int index)
{
if (index < _green.Slots)
{
var node = this[index];
if (node.IsNode)
{
return node.AsNode();
}
}
return null;
}
public override void Accept(SyntaxVisitor visitor)
{
visitor.VisitList(this);
}
}
}

80
Parser/SyntaxToken.cs Normal file
View File

@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
namespace Parser
{
public struct SyntaxToken : IEquatable<SyntaxToken>
{
private readonly SyntaxNode _parent;
private readonly Internal.GreenNode _token;
public TokenKind Kind => _token.Kind;
public override string ToString()
{
return _token.ToString();
}
internal SyntaxToken(SyntaxNode parent, Internal.GreenNode token)
{
_parent = parent;
_token = token ?? throw new ArgumentNullException(nameof(token));
}
public SyntaxNode Parent => _parent;
internal Internal.GreenNode Token => _token;
public bool Equals(SyntaxToken other)
{
return Equals(_parent, other._parent) && Equals(_token, other._token);
}
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
return obj is SyntaxToken token && Equals(token);
}
public override int GetHashCode()
{
unchecked
{
return ((_parent != null ? _parent.GetHashCode() : 0) * 397) ^ (_token != null ? _token.GetHashCode() : 0);
}
}
public static bool operator ==(SyntaxToken left, SyntaxToken right)
{
return left.Equals(right);
}
public static bool operator !=(SyntaxToken left, SyntaxToken right)
{
return !left.Equals(right);
}
public string Text => _token.Text;
public string FullText => _token.FullText;
public bool IsMissing => _token.IsMissing;
public IReadOnlyList<SyntaxTrivia> LeadingTrivia
{
get
{
var p = _parent;
return _token.LeadingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
}
}
public IReadOnlyList<SyntaxTrivia> TrailingTrivia
{
get
{
var p = _parent;
return _token.TrailingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
}
}
}
}

57
Parser/SyntaxTrivia.cs Normal file
View File

@ -0,0 +1,57 @@
using System;
namespace Parser
{
public struct SyntaxTrivia : IEquatable<SyntaxTrivia>
{
private readonly SyntaxNode _parent;
private readonly Internal.GreenNode _trivia;
internal SyntaxTrivia(SyntaxNode parent, Internal.GreenNode trivia)
{
_parent = parent;
_trivia = trivia;
}
public SyntaxNode Parent => _parent;
internal Internal.GreenNode Trivia => _trivia;
public bool Equals(SyntaxTrivia other)
{
return Equals(_parent, other._parent) && Equals(_trivia, other._trivia);
}
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
return obj is SyntaxTrivia trivia && Equals(trivia);
}
public override int GetHashCode()
{
unchecked
{
return ((_parent != null ? _parent.GetHashCode() : 0) * 397) ^ (_trivia != null ? _trivia.GetHashCode() : 0);
}
}
public static bool operator ==(SyntaxTrivia left, SyntaxTrivia right)
{
return left.Equals(right);
}
public static bool operator !=(SyntaxTrivia left, SyntaxTrivia right)
{
return !left.Equals(right);
}
public string Text => _trivia.Text;
public string FullText => _trivia.FullText;
public override string ToString()
{
return Text;
}
}
}

255
Parser/SyntaxVisitor.Generated.cs Normal file
View File

@ -0,0 +1,255 @@
namespace Parser
{
public partial class SyntaxVisitor
{
public virtual void VisitFile(FileSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitFunctionDeclaration(FunctionDeclarationSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitFunctionOutputDescription(FunctionOutputDescriptionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitFunctionInputDescription(FunctionInputDescriptionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitSwitchStatement(SwitchStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitSwitchCase(SwitchCaseSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitWhileStatement(WhileStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitElseifClause(ElseifClause node)
{
DefaultVisit(node);
}
public virtual void VisitElseClause(ElseClause node)
{
DefaultVisit(node);
}
public virtual void VisitIfStatement(IfStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitForStatement(ForStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitAssignmentExpression(AssignmentExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitCatchClause(CatchClauseSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitTryCatchStatement(TryCatchStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitExpressionStatement(ExpressionStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitEmptyStatement(EmptyStatementSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitEmptyExpression(EmptyExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitUnaryPrefixOperationExpression(UnaryPrefixOperationExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitCompoundName(CompoundNameSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitNamedFunctionHandle(NamedFunctionHandleSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitLambda(LambdaSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitBinaryOperationExpression(BinaryOperationExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitIdentifierName(IdentifierNameSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitNumberLiteral(NumberLiteralSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitStringLiteral(StringLiteralSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitDoubleQuotedStringLiteral(DoubleQuotedStringLiteralSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitUnquotedStringLiteral(UnquotedStringLiteralSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitArrayLiteralExpression(ArrayLiteralExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitCellArrayLiteralExpression(CellArrayLiteralExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitParenthesizedExpression(ParenthesizedExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitCellArrayElementAccessExpression(CellArrayElementAccessExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitFunctionCallExpression(FunctionCallExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitMemberAccess(MemberAccessSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitUnaryPostixOperationExpression(UnaryPostixOperationExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitIndirectMemberAccess(IndirectMemberAccessSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitCommandExpression(CommandExpressionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitBaseClassInvokation(BaseClassInvokationSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitAttributeAssignment(AttributeAssignmentSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitAttribute(AttributeSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitAttributeList(AttributeListSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitMethodDefinition(MethodDefinitionSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitAbstractMethodDeclaration(AbstractMethodDeclarationSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitMethodsList(MethodsListSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitPropertiesList(PropertiesListSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitBaseClassList(BaseClassListSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitClassDeclaration(ClassDeclarationSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitEnumerationItemValue(EnumerationItemValueSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitEnumerationItem(EnumerationItemSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitEnumerationList(EnumerationListSyntaxNode node)
{
DefaultVisit(node);
}
public virtual void VisitEventsList(EventsListSyntaxNode node)
{
DefaultVisit(node);
}
}
};

19
Parser/SyntaxVisitor.cs Normal file
View File

@ -0,0 +1,19 @@
namespace Parser
{
public abstract partial class SyntaxVisitor
{
public virtual void Visit(SyntaxNode node)
{
node?.Accept(this);
}
public virtual void DefaultVisit(SyntaxNode node)
{
}
public virtual void VisitList(SyntaxNodeOrTokenList list)
{
DefaultVisit(list);
}
}
}

24
Parser/SyntaxWalker.cs Normal file
View File

@ -0,0 +1,24 @@
namespace Parser
{
public abstract class SyntaxWalker : SyntaxVisitor
{
public override void DefaultVisit(SyntaxNode node)
{
foreach (var nodeOrToken in node.GetChildNodesAndTokens())
{
if (nodeOrToken.IsNode)
{
Visit(nodeOrToken.AsNode());
}
else
{
VisitToken(nodeOrToken.AsToken());
}
}
}
public virtual void VisitToken(SyntaxToken token)
{
}
}
}

10
Lexer/TextWindow.cs → Parser/TextWindow.cs
View File

@ -1,19 +1,19 @@
namespace Lexer namespace Parser
{ {
public class TextWindow : ITextWindow public class TextWindow : ITextWindow
{ {
protected readonly string Text; protected readonly string Text;
protected int Offset { get; set; } protected int Offset { get; set; }
private PositionInsideFile _position; private Position _position;
public IPosition Position => _position; public Position Position => _position;
public TextWindow(string text, string fileName = null) public TextWindow(string text, string fileName = null)
{ {
Text = text; Text = text;
Offset = 0; Offset = 0;
_position = new PositionInsideFile _position = new Position
{ {
File = fileName, FileName = fileName,
Line = 0, Line = 0,
Column = 0 Column = 0
}; };

2
Lexer/TextWindowWithNull.cs → Parser/TextWindowWithNull.cs
View File

@ -1,4 +1,4 @@
namespace Lexer namespace Parser
{ {
public class TextWindowWithNull : TextWindow public class TextWindowWithNull : TextWindow
{ {

115
Parser/TokenKind.cs Normal file
View File

@ -0,0 +1,115 @@
namespace Parser
{
public enum TokenKind
{
None = 0,
EndOfFile = 1,
Identifier = 2,
NumberLiteral = 3,
StringLiteral = 4,
DoubleQuotedStringLiteral = 5,
UnquotedStringLiteral = 6,
// trivia
Whitespace = 10,
Newline = 11,
Comment = 12,
MultilineComment = 13,
Assignment = 20,
Equality = 21,
Inequality = 22,
LogicalAnd = 23,
LogicalOr = 24,
BitwiseAnd = 25,
BitwiseOr = 26,
Less = 27,
LessOrEqual = 28,
Greater = 29,
GreaterOrEqual = 30,
Not = 31,
Plus = 32,
Minus = 33,
Multiply = 34,
Divide = 35,
Power = 36,
Backslash = 37,
Transpose = 38,
DotMultiply = 39,
DotDivide = 40,
DotPower = 41,
DotBackslash = 42,
DotTranspose = 43,
At = 44,
Colon = 45,
QuestionMark = 46,
Comma = 47,
Semicolon = 48,
OpeningBrace = 49,
ClosingBrace = 50,
OpeningSquareBracket = 51,
ClosingSquareBracket = 52,
OpeningBracket = 53,
ClosingBracket = 54,
Dot = 55,
DotDotDot = 56,
// unary tokens are not recognized during lexing; they are contextually recognized while parsing.
UnaryPlus = 57,
UnaryMinus = 58,
UnaryNot = 59,
UnaryQuestionMark = 60,
// syntax nodes
File = 100,
List,
FunctionDeclaration,
FunctionInputDescription,
FunctionOutputDescription,
SwitchStatement,
SwitchCase,
WhileStatement,
IfStatement,
ElseifClause,
ElseClause,
ForStatement,
AssignmentExpression,
CatchClause,
TryCatchStatement,
ExpressionStatement,
EmptyStatement,
EmptyExpression,
UnaryPrefixOperationExpression,
CompoundName,
NamedFunctionHandle,
Lambda,
BinaryOperation,
IdentifierName,
NumberLiteralExpression,
StringLiteralExpression,
DoubleQuotedStringLiteralExpression,
UnquotedStringLiteralExpression,
ArrayLiteralExpression,
CellArrayLiteralExpression,
ParenthesizedExpression,
CellArrayElementAccess,
FunctionCall,
MemberAccess,
UnaryPostfixOperationExpression,
IndirectMemberAccess,
Command,
ClassInvokation,
AttributeAssignment,
Attribute,
AttributeList,
MethodDefinition,
MethodsList,
PropertiesList,
BaseClassList,
ClassDeclaration,
EnumerationItemValue,
EnumerationItem,
EnumerationList,
AbstractMethodDeclaration,
EventsList,
}
}

32
Semantics/ClassContext.cs Normal file
View File

@ -0,0 +1,32 @@
using System.Collections.Generic;
namespace Semantics
{
internal class ClassContext
{
public string Path { get; }
public Dictionary<string, ClassContext> SubClasses { get; }
public Dictionary<string, FunctionContext> Methods { get; }
public Dictionary<string, FunctionContext> PrivateMethods { get; }
public ClassContext(
string path,
Dictionary<string, ClassContext> subClasses,
Dictionary<string, FunctionContext> methods,
Dictionary<string, FunctionContext> privateMethods)
{
Path = path;
SubClasses = subClasses;
Methods = methods;
PrivateMethods = privateMethods;
}
public ClassContext(string path)
{
Path = path;
SubClasses = new Dictionary<string, ClassContext>();
Methods = new Dictionary<string, FunctionContext>();
PrivateMethods = new Dictionary<string, FunctionContext>();
}
}
}

141
Semantics/Context.cs Normal file
View File

@ -0,0 +1,141 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Semantics
{
public class Context
{
internal PackageContext Root { get; }
public Context()
{
Root = new PackageContext(
null,
new Dictionary<string, PackageContext>(),
new Dictionary<string, ClassContext>(),
new Dictionary<string, FunctionContext>());
}
public bool FindFunction(string name)
{
return Root.Functions.ContainsKey(name);
}
private void ScanFunction(PackageContext context, string fileName)
{
var functionName = Path.GetFileNameWithoutExtension(fileName);
context.Functions[functionName] = new FunctionContext(fileName);
}
private void ScanMethod(ClassContext context, string fileName)
{
var methodName = Path.GetFileNameWithoutExtension(fileName);
context.Methods[methodName] = new FunctionContext(fileName);
}
private void ScanPrivateMethod(ClassContext context, string fileName)
{
var methodName = Path.GetFileNameWithoutExtension(fileName);
context.PrivateMethods[methodName] = new FunctionContext(fileName);
}
private string GetPackageNameFromFolder(string folderName)
{
return folderName.StartsWith('+') ? folderName.Substring(1, folderName.Length - 1) : null;
}
private string GetClassNameFromFolder(string folderName)
{
return folderName.StartsWith('@') ? folderName.Substring(1, folderName.Length - 1) : null;
}
private void ScanPrivateDirectory(ClassContext currentContext, string directory)
{
var files = Directory.GetFiles(directory, "*.m");
foreach (var fileName in files)
{
ScanPrivateMethod(currentContext, fileName);
}
var subDirectories = Directory.GetDirectories(directory);
foreach (var subDirectory in subDirectories)
{
Console.WriteLine($"A FOLDER INSIDE A PRIVATE SUBFOLDER WHAT TO DO? {subDirectory}");
}
}
private void ScanClassDirectory(ClassContext currentContext, string directory)
{
var files = Directory.GetFiles(directory, "*.m");
foreach (var fileName in files)
{
ScanMethod(currentContext, fileName);
}
var subDirectories = Directory.GetDirectories(directory);
foreach (var subDirectory in subDirectories)
{
var lastName = subDirectory.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Last();
if (lastName == "private")
{
ScanPrivateDirectory(currentContext, subDirectory);
continue;
}
var packageName = GetPackageNameFromFolder(lastName);
if (packageName != null)
{
Console.WriteLine($"A PACKAGE INSIDE A CLASS WHAT TO DO? {subDirectory}");
continue;
}
var className = GetClassNameFromFolder(lastName);
if (className != null)
{
currentContext.SubClasses[className] = new ClassContext(className);
ScanClassDirectory(currentContext.SubClasses[className], subDirectory);
continue;
}
ScanClassDirectory(currentContext, subDirectory); // Should this really work?
}
}
private void ScanDirectory(PackageContext currentContext, string directory)
{
var files = Directory.GetFiles(directory, "*.m");
foreach (var fileName in files)
{
ScanFunction(currentContext, fileName);
}
var subDirectories = Directory.GetDirectories(directory);
foreach (var subDirectory in subDirectories)
{
var lastName = subDirectory.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar).Last();
var packageName = GetPackageNameFromFolder(lastName);
if (packageName != null)
{
currentContext.SubPackages[packageName] = new PackageContext(packageName);
ScanDirectory(currentContext.SubPackages[packageName], subDirectory);
continue;
}
var className = GetClassNameFromFolder(lastName);
if (className != null)
{
currentContext.Classes[className] = new ClassContext(className);
ScanClassDirectory(currentContext.Classes[className], subDirectory);
continue;
}
ScanDirectory(currentContext, subDirectory);
}
}
public void ScanPath(string path)
{
ScanDirectory(Root, path);
}
}
}

12
Semantics/FunctionContext.cs Normal file
View File

@ -0,0 +1,12 @@
namespace Semantics
{
public class FunctionContext
{
public string FileName { get; }
public FunctionContext(string fileName)
{
FileName = fileName;
}
}
}

82
Semantics/GetClass.cs Normal file
View File

@ -0,0 +1,82 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Parser;
namespace Semantics
{
public class GetClass
{
private static MMethod MethodFromDefinition(MethodDefinitionSyntaxNode methodDefinition)
{
var name = methodDefinition.Name.Text;
var description = "";
description += string.Join("", methodDefinition.LeadingTrivia.Select(x => x.FullText));
if (methodDefinition.Body == null)
{
description += string.Join("", methodDefinition.EndKeyword.LeadingTrivia.Select(x => x.FullText));
}
else
{
description += string.Join("", methodDefinition.Body.LeadingTrivia.Select(x => x.FullText));
}
return new MMethod(name, description);
}
private static MMethod MethodFromDeclaration(AbstractMethodDeclarationSyntaxNode methodDeclaration)
{
var name = methodDeclaration.Name.Text;
var description = "";
description += string.Join("", methodDeclaration.LeadingTrivia.Select(x => x.FullText));
return new MMethod(name, description);
}
private static List<MMethod> MethodsFromList(MethodsListSyntaxNode methodsList)
{
var result = new List<MMethod>();
foreach (var method in methodsList.Methods)
{
if (method.IsToken)
{
continue;
}
if (method.AsNode() is MethodDefinitionSyntaxNode methodDefinition)
{
result.Add(MethodFromDefinition(methodDefinition));
}
if (method.AsNode() is AbstractMethodDeclarationSyntaxNode methodDeclaration)
{
result.Add(MethodFromDeclaration(methodDeclaration));
}
}
return result;
}
public static MClass FromTree(FileSyntaxNode tree, string fileName)
{
var classDeclaration = tree.StatementList[0].AsNode() as ClassDeclarationSyntaxNode;
if (classDeclaration == null)
{
return null;
}
var name = classDeclaration.ClassName.Text;
var methods = new List<MMethod>();
foreach (var s in classDeclaration.Nodes)
{
if (s.IsToken)
{
continue;
}
if (s.AsNode() is MethodsListSyntaxNode methodsList)
{
methods.AddRange(MethodsFromList(methodsList));
}
}
return new MClass(tree, name, fileName, methods);
}
}
}

21
Semantics/MClass.cs Normal file
View File

@ -0,0 +1,21 @@
using System.Collections.Generic;
using Parser;
namespace Semantics
{
public class MClass
{
public string Name { get; }
public string FileName { get; }
public FileSyntaxNode Tree { get; }
public List<MMethod> Methods { get; }
public MClass(FileSyntaxNode tree, string name, string fileName, List<MMethod> methods)
{
Tree = tree;
Name = name;
FileName = FileName;
Methods = methods;
}
}
}

15
Semantics/MMethod.cs Normal file
View File

@ -0,0 +1,15 @@
namespace Semantics
{
public class MMethod
{
public string Name { get; }
public string Description { get; }
public MClass Class { get; internal set; }
public MMethod(string name, string description)
{
Name = name;
Description = description;
}
}
}

30
Semantics/MethodAssignments.cs Normal file
View File

@ -0,0 +1,30 @@
using System.Collections.Generic;
namespace Semantics
{
public class MethodAssignments
{
public Context Context { get; }
private readonly Dictionary<string, Variable> _methods;
public MethodAssignments()
{
_methods = new Dictionary<string, Variable>();
}
public Variable Find(string name)
{
if (_methods.ContainsKey(name))
{
return _methods[name];
}
return null;
}
public void Add(string name, Variable variable)
{
_methods[name] = variable;
}
}
}

33
Semantics/PackageContext.cs Normal file
View File

@ -0,0 +1,33 @@
using System.Collections.Generic;
namespace Semantics
{
internal class PackageContext
{
public string Path { get; }
public Dictionary<string, PackageContext> SubPackages { get; }
public Dictionary<string, ClassContext> Classes { get; }
public Dictionary<string, FunctionContext> Functions { get; }
public PackageContext(
string path,
Dictionary<string, PackageContext> subPackages,
Dictionary<string, ClassContext> classes,
Dictionary<string, FunctionContext> functions)
{
Path = path;
SubPackages = subPackages;
Classes = classes;
Functions = functions;
}
public PackageContext(
string path)
{
Path = path;
SubPackages = new Dictionary<string, PackageContext>();
Classes = new Dictionary<string, ClassContext>();
Functions = new Dictionary<string, FunctionContext>();
}
}
}

3
ProjectConsole/ProjectConsole.csproj → Semantics/Semantics.csproj
View File

@ -1,7 +1,6 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <TargetFramework>netcoreapp2.1</TargetFramework>
<TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Parser\Parser.csproj" /> <ProjectReference Include="..\Parser\Parser.csproj" />

7
Semantics/Variable.cs Normal file
View File

@ -0,0 +1,7 @@
namespace Semantics
{
public class Variable
{
}
}

30
Semantics/VariableAssignments.cs Normal file
View File

@ -0,0 +1,30 @@
using System.Collections.Generic;
namespace Semantics
{
public class VariableAssignments
{
public Context Context { get; }
private readonly Dictionary<string, Variable> _variables;
public VariableAssignments()
{
_variables = new Dictionary<string, Variable>();
}
public Variable Find(string name)
{
if (_variables.ContainsKey(name))
{
return _variables[name];
}
return null;
}
public void Add(string name, Variable variable)
{
_variables[name] = variable;
}
}
}

26
Solution.sln
View File

@ -3,15 +3,15 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15 # Visual Studio 15
VisualStudioVersion = 15.0.27130.2026 VisualStudioVersion = 15.0.27130.2026
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProjectConsole", "ProjectConsole\ProjectConsole.csproj", "{5025FD8F-0F1A-43E5-A996-7753BC703D62}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConsoleDemo", "ConsoleDemo\ConsoleDemo.csproj", "{5025FD8F-0F1A-43E5-A996-7753BC703D62}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Parser", "Parser\Parser.csproj", "{B20EDC10-E6E6-4430-8527-B95206DEF941}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Parser", "Parser\Parser.csproj", "{1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Parser.Tests", "Parser.Tests\Parser.Tests.csproj", "{83008C72-2EFC-41EB-AC8D-023C6AE1709F}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Parser.Tests", "Parser.Tests\Parser.Tests.csproj", "{7BFEAD86-EAC3-43C8-9388-EBAB377938D4}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lexer", "Lexer\Lexer.csproj", "{1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SyntaxGenerator", "SyntaxGenerator\SyntaxGenerator.csproj", "{03487753-C1F5-4753-B576-593294ED86D1}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lexer.Tests", "Lexer.Tests\Lexer.Tests.csproj", "{7BFEAD86-EAC3-43C8-9388-EBAB377938D4}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semantics", "Semantics\Semantics.csproj", "{4595633B-7F9A-4771-B348-F12BB9DD7ABC}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -23,14 +23,6 @@ Global
{5025FD8F-0F1A-43E5-A996-7753BC703D62}.Debug|Any CPU.Build.0 = Debug|Any CPU {5025FD8F-0F1A-43E5-A996-7753BC703D62}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5025FD8F-0F1A-43E5-A996-7753BC703D62}.Release|Any CPU.ActiveCfg = Release|Any CPU {5025FD8F-0F1A-43E5-A996-7753BC703D62}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5025FD8F-0F1A-43E5-A996-7753BC703D62}.Release|Any CPU.Build.0 = Release|Any CPU {5025FD8F-0F1A-43E5-A996-7753BC703D62}.Release|Any CPU.Build.0 = Release|Any CPU
{B20EDC10-E6E6-4430-8527-B95206DEF941}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B20EDC10-E6E6-4430-8527-B95206DEF941}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B20EDC10-E6E6-4430-8527-B95206DEF941}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B20EDC10-E6E6-4430-8527-B95206DEF941}.Release|Any CPU.Build.0 = Release|Any CPU
{83008C72-2EFC-41EB-AC8D-023C6AE1709F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{83008C72-2EFC-41EB-AC8D-023C6AE1709F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{83008C72-2EFC-41EB-AC8D-023C6AE1709F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{83008C72-2EFC-41EB-AC8D-023C6AE1709F}.Release|Any CPU.Build.0 = Release|Any CPU
{1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}.Debug|Any CPU.Build.0 = Debug|Any CPU {1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B8E5BBC-E5CD-427B-A6C7-F30047AA4A39}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -39,6 +31,14 @@ Global
{7BFEAD86-EAC3-43C8-9388-EBAB377938D4}.Debug|Any CPU.Build.0 = Debug|Any CPU {7BFEAD86-EAC3-43C8-9388-EBAB377938D4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7BFEAD86-EAC3-43C8-9388-EBAB377938D4}.Release|Any CPU.ActiveCfg = Release|Any CPU {7BFEAD86-EAC3-43C8-9388-EBAB377938D4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7BFEAD86-EAC3-43C8-9388-EBAB377938D4}.Release|Any CPU.Build.0 = Release|Any CPU {7BFEAD86-EAC3-43C8-9388-EBAB377938D4}.Release|Any CPU.Build.0 = Release|Any CPU
{03487753-C1F5-4753-B576-593294ED86D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{03487753-C1F5-4753-B576-593294ED86D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{03487753-C1F5-4753-B576-593294ED86D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{03487753-C1F5-4753-B576-593294ED86D1}.Release|Any CPU.Build.0 = Release|Any CPU
{4595633B-7F9A-4771-B348-F12BB9DD7ABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4595633B-7F9A-4771-B348-F12BB9DD7ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4595633B-7F9A-4771-B348-F12BB9DD7ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4595633B-7F9A-4771-B348-F12BB9DD7ABC}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

360
SyntaxGenerator/GenerateSyntax.cs Normal file
View File

@ -0,0 +1,360 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml;
using System.Xml.Serialization;
namespace SyntaxGenerator
{
public class GenerateSyntax
{
private const string SyntaxTokenClassName = "SyntaxToken";
private const string InternalNamespace = "Parser.Internal";
private const string OuterNamespace = "Parser";
private static readonly List<(string visitorMethodName, string className)> Visitors = new List<(string, string)>();
private static string _outputPath;
static GenerateSyntax()
{
switch (Environment.OSVersion.Platform)
{
case PlatformID.MacOSX:
case PlatformID.Unix:
_outputPath = Path.Combine("..", "Parser");
break;
default:
_outputPath = Path.Combine("..", "..", "..", "..", "Parser");
break;
}
}
public static void TestOutput()
{
var field1 = new FieldDescription
{
FieldType = "SyntaxToken",
FieldName = "functionKeyword"
};
var field2 = new FieldDescription
{
FieldType = "FunctionOutputDescriptionSyntaxNode",
FieldName = "outputDescription"
};
var syntaxNode = new SyntaxNodeDescription
{
ClassName = "FunctionDeclarationSyntaxNode",
BaseClassName = "StatementSyntaxNode",
Fields = new[] {field1, field2}
};
var syntax = new SyntaxDescription
{
Nodes = new[] {syntaxNode, syntaxNode}
};
var serializer = new XmlSerializer(typeof(SyntaxDescription));
using (var writer = new StreamWriter("output.xml"))
{
serializer.Serialize(writer, syntax);
}
}
private static string GenerateInternalFieldDeclaration(FieldDescription field)
{
return $" internal readonly {field.FieldType} _{field.FieldName};\n";
}
private static string GeneratePrivateFieldDeclaration(FieldDescription field)
{
return $" private SyntaxNode _{field.FieldName};\n";
}
private static string GenerateFieldAssignmentInsideConstructor(FieldDescription field)
{
return $" _{field.FieldName} = {field.FieldName};\n";
}
private static string GenerateInternalConstructor(SyntaxNodeDescription node)
{
var arguments = string.Join(
",",
node.Fields.Select(field => $"\n {field.FieldType} {field.FieldName}"));
var header =
$" internal {node.ClassName}({arguments}) : base(TokenKind.{node.TokenKindName}, {node.Fields.Length})\n";
var assignments = string.Join(
"",
node.Fields.Select(GenerateFieldAssignmentInsideConstructor));
return header + " {\n" + assignments + " }\n";
}
private static string GenerateConstructor(SyntaxNodeDescription node)
{
var arguments = "SyntaxNode parent, Internal.GreenNode green";
var header =
$" internal {node.ClassName}({arguments}) : base(parent, green)\n";
return header + " {\n }\n";
}
private static string GenerateInternalGetSlot(SyntaxNodeDescription node)
{
var header = $" public override GreenNode GetSlot(int i)\n";
var cases = string.Join(
"",
node.Fields.Select((f, i) => $" case {i}: return _{f.FieldName};\n"));
var defaultCase = " default: return null;\n";
return header
+ " {\n switch (i)\n {\n"
+ cases
+ defaultCase
+ " }\n"
+ " }\n";
}
private static string GenerateGetSlot(SyntaxNodeDescription node, List<(FieldDescription field, int index)> pairs)
{
var header = $" internal override SyntaxNode GetNode(int i)\n";
var cases = string.Join(
"",
pairs.Select(pair => $" case {pair.index}: return GetRed(ref _{pair.field.FieldName}, {pair.index});\n"));
var defaultCase = " default: return null;\n";
return header
+ " {\n switch (i)\n {\n"
+ cases
+ defaultCase
+ " }\n"
+ " }\n";
}
private static string GenerateCreateRed(SyntaxNodeDescription node)
{
var header = $" internal override {OuterNamespace}.SyntaxNode CreateRed({OuterNamespace}.SyntaxNode parent)\n";
var text = $" return new {OuterNamespace}.{node.ClassName}(parent, this);\n";
return header + " {\n" + text + " }\n";
}
private static string GenerateInternalClass(SyntaxNodeDescription node)
{
var header = $" internal class {node.ClassName}";
if (node.BaseClassName != null)
{
header += $" : {node.BaseClassName}";
}
var fields = string.Join(
"",
node.Fields.Select(GenerateInternalFieldDeclaration));
var constructor = GenerateInternalConstructor(node);
var getSlot = GenerateInternalGetSlot(node);
var createRed = GenerateCreateRed(node);
return
header
+ "\n {\n"
+ fields + "\n"
+ constructor + "\n"
+ createRed + "\n"
+ getSlot + " }\n";
}
private static string Capitalize(string s)
{
return s[0].ToString().ToUpper() + s.Substring(1, s.Length - 1);
}
private static string GenerateTokenAccessor(SyntaxNodeDescription node, FieldDescription field)
{
var header = $" public SyntaxToken {Capitalize(field.FieldName)}\n";
var text =
$" get {{ return new SyntaxToken(this, (({InternalNamespace}.{node.ClassName})_green)._{field.FieldName}); }}";
return header + " {\n" + text + "\n }\n";
}
private static bool IsList(string type)
{
return type.StartsWith("SyntaxList");
}
private static string GenerateNodeAccessor(SyntaxNodeDescription node, FieldDescription field, int index)
{
var type = field.FieldType;
if (IsList(type))
{
type = "SyntaxNodeOrTokenList";
}
var header = $" public {type} {Capitalize(field.FieldName)}\n {{\n get\n {{\n";
var text =
$" var red = this.GetRed(ref this._{field.FieldName}, {index});\n"
+ $" if (red != null)\n"
+ $" return ({type})red;\n\n"
+ $" return default({type});\n";
return header + text + " }\n }\n";
}
private static string ConvertClassNameToVisitorName(string name)
{
if (name.EndsWith("SyntaxNode"))
{
name = name.Substring(0, name.Length - "SyntaxNode".Length);
}
return "Visit" + name;
}
private static string GenerateAccessMethod(SyntaxNodeDescription node)
{
var visitorName = ConvertClassNameToVisitorName(node.ClassName);
Visitors.Add((visitorName, node.ClassName));
var header = $" public override void Accept(SyntaxVisitor visitor)\n";
var body = $" visitor.{visitorName}(this);\n";
return header + " {\n" + body + " }\n";
}
private static string GenerateClass(SyntaxNodeDescription node)
{
var header = $" public class {node.ClassName}";
if (node.BaseClassName != null)
{
header += $" : {node.BaseClassName}";
}
var tokenSlots = node.Fields
.Select((f, i) => (field: f, index: i))
.Where(pair => pair.field.FieldType == SyntaxTokenClassName)
.ToList();
var nodeSlots = node.Fields
.Select((f, i) => (field: f, index: i))
.Where(pair => pair.field.FieldType != SyntaxTokenClassName)
.ToList();
var fields = string.Join(
"",
nodeSlots.Select(pair => GeneratePrivateFieldDeclaration(pair.field)));
var constructor = GenerateConstructor(node);
var tokenAccessors =
string.Join(
"\n",
tokenSlots.Select(pair => GenerateTokenAccessor(node, pair.field)));
var nodeAccessors =
string.Join(
"\n",
nodeSlots.Select(pair => GenerateNodeAccessor(node, pair.field, pair.index)));
var getSlot = GenerateGetSlot(node, nodeSlots);
var access = GenerateAccessMethod(node);
return
header
+ "\n {\n"
+ fields + "\n"
+ constructor + "\n"
+ tokenAccessors + "\n"
+ nodeAccessors + "\n"
+ getSlot + "\n"
+ access + "\n"
+ " }\n";
}
private static string GenerateInternalSyntaxNodeFile(SyntaxDescription syntax)
{
var header = $"namespace {InternalNamespace}\n";
var classes = string.Join(
"\n",
syntax.Nodes.Select(GenerateInternalClass)
);
return header + "{\n" + classes + "}\n";
}
private static string GenerateSyntaxNodeFile(SyntaxDescription syntax)
{
var header = $"namespace {OuterNamespace}\n";
var classes = string.Join(
"\n",
syntax.Nodes.Select(GenerateClass)
);
return header + "{\n" + classes + "}\n";
}
private static string FactoryMethodNameFromClassName(string className)
{
if (className.EndsWith("Node"))
{
return className.Substring(0, className.Length - 4);
}
else
{
return className;
}
}
private static string GenerateFactoryMethod(SyntaxNodeDescription node)
{
var methodName = FactoryMethodNameFromClassName(node.ClassName);
var header = $" public {node.ClassName} {methodName}";
var arguments = string.Join(
", ",
node.Fields.Select(field => $"\n {field.FieldType} {field.FieldName}"));
var constructorParameters = string.Join(
", ",
node.Fields.Select(field => $"\n {field.FieldName}"));
var returnStatement =
$" return new {node.ClassName}({constructorParameters});\n";
return header + "(" + arguments + ")\n {\n" + returnStatement + " }\n";
}
private static string GenerateSyntaxFactoryFile(SyntaxDescription syntax)
{
var header = $"namespace {InternalNamespace}\n{{\n internal partial class SyntaxFactory\n";
var methods = string.Join(
"\n",
syntax.Nodes.Select(GenerateFactoryMethod)
);
return header + " {\n" + methods + " }\n}";
}
private static string GenerateVisitor((string visitorMethodName, string className) info)
{
var header = $" public virtual void {info.visitorMethodName}({info.className} node)\n";
var body = $" DefaultVisit(node);\n";
return header + " {\n" + body + " }\n";
}
private static string GenerateSyntaxVisitorFile(SyntaxDescription syntax)
{
var header = $"namespace {OuterNamespace}\n{{\n public partial class SyntaxVisitor\n";
var methods = string.Join(
"\n",
Visitors.Select(GenerateVisitor));
return header + " {\n" + methods + " }\n};";
}
public static void Input()
{
var serializer = new XmlSerializer(typeof(SyntaxDescription));
using (var stream = new FileStream("input.xml", FileMode.Open))
{
var syntax = serializer.Deserialize(stream) as SyntaxDescription;
if (syntax == null)
{
Console.WriteLine("Couldn't deserialize syntax.");
return;
}
var internalSyntaxNodePath = Path.Combine(_outputPath, "Internal", "SyntaxNode.Generated.cs");
File.WriteAllText(internalSyntaxNodePath, GenerateInternalSyntaxNodeFile(syntax));
var internalSyntaxFactoryPath = Path.Combine(_outputPath, "Internal", "SyntaxFactory.Generated.cs");
File.WriteAllText(internalSyntaxFactoryPath, GenerateSyntaxFactoryFile(syntax));
var syntaxNodePath = Path.Combine(_outputPath, "SyntaxNode.Generated.cs");
File.WriteAllText(syntaxNodePath, GenerateSyntaxNodeFile(syntax));
var syntaxVisitorPath = Path.Combine(_outputPath, "SyntaxVisitor.Generated.cs");
File.WriteAllText(syntaxVisitorPath, GenerateSyntaxVisitorFile(syntax));
}
}
static void Main(string[] args)
{
Console.Write("Generating syntax...");
Input();
Console.WriteLine("Done.");
}
}
}

11
SyntaxGenerator/SyntaxGenerator.csproj Normal file
View File

@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<None Update="input.xml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

38
SyntaxGenerator/SyntaxNodeDescription.cs Normal file
View File

@ -0,0 +1,38 @@
using System.Xml.Serialization;
namespace SyntaxGenerator
{
[XmlRoot(ElementName = "Syntax")]
public class SyntaxDescription
{
[XmlElement(ElementName = "Class")]
public SyntaxNodeDescription[] Nodes { get; set; }
}
public class SyntaxNodeDescription
{
[XmlAttribute("Name")]
public string ClassName { get; set; }
[XmlAttribute("BaseClass")]
public string BaseClassName { get; set; }
[XmlAttribute("Kind")]
public string TokenKindName { get; set; }
[XmlElement(ElementName = "Field")]
public FieldDescription[] Fields
{
get => _fields;
set => _fields = value ?? new FieldDescription[0];
}
private FieldDescription[] _fields = new FieldDescription[0];
}
public class FieldDescription
{
[XmlAttribute("Type")]
public string FieldType { get; set; }
[XmlAttribute("Name")]
public string FieldName { get; set; }
}
}

253
SyntaxGenerator/input.xml Normal file
View File

@ -0,0 +1,253 @@
<?xml version="1.0" encoding="utf-8"?>
<Syntax xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<Class Name="FileSyntaxNode" BaseClass="SyntaxNode" Kind="File">
<Field Type="SyntaxList" Name="statementList" />
<Field Type="SyntaxToken" Name="endOfFile" />
</Class>
<Class Name="FunctionDeclarationSyntaxNode" BaseClass="StatementSyntaxNode" Kind="FunctionDeclaration">
<Field Type="SyntaxToken" Name="functionKeyword" />
<Field Type="FunctionOutputDescriptionSyntaxNode" Name="outputDescription" />
<Field Type="SyntaxToken" Name="name" />
<Field Type="FunctionInputDescriptionSyntaxNode" Name="inputDescription" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="commas" />
<Field Type="SyntaxList" Name="body" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="FunctionOutputDescriptionSyntaxNode" BaseClass="SyntaxNode" Kind="FunctionOutputDescription">
<Field Type="SyntaxList" Name="outputList" />
<Field Type="SyntaxToken" Name="assignmentSign" />
</Class>
<Class Name="FunctionInputDescriptionSyntaxNode" BaseClass="SyntaxNode" Kind="FunctionInputDescription">
<Field Type="SyntaxToken" Name="openingBracket" />
<Field Type="SyntaxList" Name="parameterList" />
<Field Type="SyntaxToken" Name="closingBracket" />
</Class>
<Class Name="SwitchStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="SwitchStatement">
<Field Type="SyntaxToken" Name="switchKeyword" />
<Field Type="ExpressionSyntaxNode" Name="switchExpression" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="optionalCommas" />
<Field Type="SyntaxList&lt;SwitchCaseSyntaxNode&gt;" Name="cases" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="SwitchCaseSyntaxNode" BaseClass="SyntaxNode" Kind="SwitchCase">
<Field Type="SyntaxToken" Name="caseKeyword" />
<Field Type="ExpressionSyntaxNode" Name="caseIdentifier" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="optionalCommas" />
<Field Type="SyntaxList" Name="body" />
</Class>
<Class Name="WhileStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="WhileStatement">
<Field Type="SyntaxToken" Name="whileKeyword" />
<Field Type="ExpressionSyntaxNode" Name="condition" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="optionalCommas" />
<Field Type="SyntaxList" Name="body" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="ElseifClause" BaseClass="SyntaxNode" Kind="ElseifClause">
<Field Type="SyntaxToken" Name="elseifKeyword" />
<Field Type="ExpressionSyntaxNode" Name="condition" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="optionalCommas" />
<Field Type="SyntaxList" Name="body" />
</Class>
<Class Name="ElseClause" BaseClass="SyntaxNode" Kind="ElseClause">
<Field Type="SyntaxToken" Name="elseKeyword" />
<Field Type="SyntaxList" Name="body" />
</Class>
<Class Name="IfStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="IfStatement">
<Field Type="SyntaxToken" Name="ifKeyword" />
<Field Type="ExpressionSyntaxNode" Name="condition" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="optionalCommas" />
<Field Type="SyntaxList" Name="body" />
<Field Type="SyntaxList&lt;ElseifClause&gt;" Name="elseifClauses" />
<Field Type="ElseClause" Name="elseClause" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="ForStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="ForStatement">
<Field Type="SyntaxToken" Name="forKeyword" />
<Field Type="AssignmentExpressionSyntaxNode" Name="assignment" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="optionalCommas" />
<Field Type="SyntaxList" Name="body" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="AssignmentExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="AssignmentExpression">
<Field Type="ExpressionSyntaxNode" Name="lhs" />
<Field Type="SyntaxToken" Name="assignmentSign" />
<Field Type="ExpressionSyntaxNode" Name="rhs" />
</Class>
<Class Name="CatchClauseSyntaxNode" BaseClass="SyntaxNode" Kind="CatchClause">
<Field Type="SyntaxToken" Name="catchKeyword" />
<Field Type="SyntaxList" Name="catchBody" />
</Class>
<Class Name="TryCatchStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="TryCatchStatement">
<Field Type="SyntaxToken" Name="tryKeyword" />
<Field Type="SyntaxList" Name="tryBody" />
<Field Type="CatchClauseSyntaxNode" Name="catchClause" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="ExpressionStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="ExpressionStatement">
<Field Type="ExpressionSyntaxNode" Name="expression" />
</Class>
<Class Name="EmptyStatementSyntaxNode" BaseClass="StatementSyntaxNode" Kind="EmptyStatement">
<Field Type="SyntaxToken" Name="semicolon" />
</Class>
<Class Name="EmptyExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="EmptyExpression">
</Class>
<Class Name="UnaryPrefixOperationExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="UnaryPrefixOperationExpression">
<Field Type="SyntaxToken" Name="operation" />
<Field Type="ExpressionSyntaxNode" Name="operand" />
</Class>
<Class Name="CompoundNameSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="CompoundName">
<Field Type="SyntaxList" Name="nodes" />
</Class>
<Class Name="NamedFunctionHandleSyntaxNode" BaseClass="FunctionHandleSyntaxNode" Kind="NamedFunctionHandle">
<Field Type="SyntaxToken" Name="atSign" />
<Field Type="CompoundNameSyntaxNode" Name="functionName" />
</Class>
<Class Name="LambdaSyntaxNode" BaseClass="FunctionHandleSyntaxNode" Kind="Lambda">
<Field Type="SyntaxToken" Name="atSign" />
<Field Type="FunctionInputDescriptionSyntaxNode" Name="input" />
<Field Type="ExpressionSyntaxNode" Name="body" />
</Class>
<Class Name="BinaryOperationExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="BinaryOperation">
<Field Type="ExpressionSyntaxNode" Name="lhs" />
<Field Type="SyntaxToken" Name="operation" />
<Field Type="ExpressionSyntaxNode" Name="rhs" />
</Class>
<Class Name="IdentifierNameSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="IdentifierName">
<Field Type="SyntaxToken" Name="name" />
</Class>
<Class Name="NumberLiteralSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="NumberLiteralExpression">
<Field Type="SyntaxToken" Name="number" />
</Class>
<Class Name="StringLiteralSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="StringLiteralExpression">
<Field Type="SyntaxToken" Name="stringToken" />
</Class>
<Class Name="DoubleQuotedStringLiteralSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="DoubleQuotedStringLiteralExpression">
<Field Type="SyntaxToken" Name="stringToken" />
</Class>
<Class Name="UnquotedStringLiteralSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="UnquotedStringLiteralExpression">
<Field Type="SyntaxToken" Name="stringToken" />
</Class>
<Class Name="ArrayLiteralExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="ArrayLiteralExpression">
<Field Type="SyntaxToken" Name="openingSquareBracket" />
<Field Type="SyntaxList" Name="nodes" />
<Field Type="SyntaxToken" Name="closingSquareBracket" />
</Class>
<Class Name="CellArrayLiteralExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="CellArrayLiteralExpression">
<Field Type="SyntaxToken" Name="openingBrace" />
<Field Type="SyntaxList" Name="nodes" />
<Field Type="SyntaxToken" Name="closingBrace" />
</Class>
<Class Name="ParenthesizedExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="ParenthesizedExpression">
<Field Type="SyntaxToken" Name="openingBracket" />
<Field Type="ExpressionSyntaxNode" Name="expression" />
<Field Type="SyntaxToken" Name="closingBracket" />
</Class>
<Class Name="CellArrayElementAccessExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="CellArrayElementAccess">
<Field Type="ExpressionSyntaxNode" Name="expression" />
<Field Type="SyntaxToken" Name="openingBrace" />
<Field Type="SyntaxList" Name="nodes" />
<Field Type="SyntaxToken" Name="closingBrace" />
</Class>
<Class Name="FunctionCallExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="FunctionCall">
<Field Type="ExpressionSyntaxNode" Name="functionName" />
<Field Type="SyntaxToken" Name="openingBracket" />
<Field Type="SyntaxList" Name="nodes" />
<Field Type="SyntaxToken" Name="closingBracket" />
</Class>
<Class Name="MemberAccessSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="MemberAccess">
<Field Type="SyntaxNode" Name="leftOperand" />
<Field Type="SyntaxToken" Name="dot" />
<Field Type="SyntaxNode" Name="rightOperand" />
</Class>
<Class Name="UnaryPostixOperationExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="UnaryPostfixOperationExpression">
<Field Type="ExpressionSyntaxNode" Name="operand" />
<Field Type="SyntaxToken" Name="operation" />
</Class>
<Class Name="IndirectMemberAccessSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="IndirectMemberAccess">
<Field Type="SyntaxToken" Name="openingBracket" />
<Field Type="ExpressionSyntaxNode" Name="expression" />
<Field Type="SyntaxToken" Name="closingBracket" />
</Class>
<Class Name="CommandExpressionSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="Command">
<Field Type="IdentifierNameSyntaxNode" Name="commandName" />
<Field Type="SyntaxList&lt;UnquotedStringLiteralSyntaxNode&gt;" Name="arguments" />
</Class>
<Class Name="BaseClassInvokationSyntaxNode" BaseClass="ExpressionSyntaxNode" Kind="ClassInvokation">
<Field Type="ExpressionSyntaxNode" Name="methodName" />
<Field Type="SyntaxToken" Name="atSign" />
<Field Type="ExpressionSyntaxNode" Name="baseClassNameAndArguments" />
</Class>
<Class Name="AttributeAssignmentSyntaxNode" BaseClass="SyntaxNode" Kind="AttributeAssignment">
<Field Type="SyntaxToken" Name="assignmentSign"/>
<Field Type="ExpressionSyntaxNode" Name="value" />
</Class>
<Class Name="AttributeSyntaxNode" BaseClass="SyntaxNode" Kind="Attribute">
<Field Type="IdentifierNameSyntaxNode" Name="name" />
<Field Type="AttributeAssignmentSyntaxNode" Name="assignment" />
</Class>
<Class Name="AttributeListSyntaxNode" BaseClass="SyntaxNode" Kind="AttributeList">
<Field Type="SyntaxToken" Name="openingBracket" />
<Field Type="SyntaxList" Name="nodes" />
<Field Type="SyntaxToken" Name="closingBracket" />
</Class>
<Class Name="MethodDefinitionSyntaxNode" BaseClass="MethodDeclarationSyntaxNode" Kind="MethodDefinition">
<Field Type="SyntaxToken" Name="functionKeyword" />
<Field Type="FunctionOutputDescriptionSyntaxNode" Name="outputDescription" />
<Field Type="CompoundNameSyntaxNode" Name="name" />
<Field Type="FunctionInputDescriptionSyntaxNode" Name="inputDescription" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="commas" />
<Field Type="SyntaxList" Name="body" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="AbstractMethodDeclarationSyntaxNode" BaseClass="MethodDeclarationSyntaxNode" Kind="AbstractMethodDeclaration">
<Field Type="FunctionOutputDescriptionSyntaxNode" Name="outputDescription" />
<Field Type="CompoundNameSyntaxNode" Name="name" />
<Field Type="FunctionInputDescriptionSyntaxNode" Name="inputDescription" />
</Class>
<Class Name="MethodsListSyntaxNode" BaseClass="SyntaxNode" Kind="MethodsList">
<Field Type="SyntaxToken" Name="methodsKeyword" />
<Field Type="AttributeListSyntaxNode" Name="attributes" />
<Field Type="SyntaxList" Name="methods" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="PropertiesListSyntaxNode" BaseClass="SyntaxNode" Kind="PropertiesList">
<Field Type="SyntaxToken" Name="propertiesKeyword" />
<Field Type="AttributeListSyntaxNode" Name="attributes" />
<Field Type="SyntaxList" Name="properties" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="BaseClassListSyntaxNode" BaseClass="SyntaxNode" Kind="BaseClassList">
<Field Type="SyntaxToken" Name="lessSign" />
<Field Type="SyntaxList" Name="baseClasses" />
</Class>
<Class Name="ClassDeclarationSyntaxNode" BaseClass="StatementSyntaxNode" Kind="ClassDeclaration">
<Field Type="SyntaxToken" Name="classdefKeyword" />
<Field Type="AttributeListSyntaxNode" Name="attributes" />
<Field Type="IdentifierNameSyntaxNode" Name="className" />
<Field Type="BaseClassListSyntaxNode" Name="baseClassList" />
<Field Type="SyntaxList" Name="nodes" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="EnumerationItemValueSyntaxNode" BaseClass="SyntaxNode" Kind="EnumerationItemValue">
<Field Type="SyntaxToken" Name="openingBracket" />
<Field Type="SyntaxList" Name="values" />
<Field Type="SyntaxToken" Name="closingBracket" />
</Class>
<Class Name="EnumerationItemSyntaxNode" BaseClass="SyntaxNode" Kind="EnumerationItem">
<Field Type="IdentifierNameSyntaxNode" Name="name" />
<Field Type="EnumerationItemValueSyntaxNode" Name="values" />
<Field Type="SyntaxList&lt;SyntaxToken&gt;" Name="commas" />
</Class>
<Class Name="EnumerationListSyntaxNode" BaseClass="SyntaxNode" Kind="EnumerationList">
<Field Type="SyntaxToken" Name="enumerationKeyword" />
<Field Type="AttributeListSyntaxNode" Name="attributes" />
<Field Type="SyntaxList&lt;EnumerationItemSyntaxNode&gt;" Name="items" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
<Class Name="EventsListSyntaxNode" BaseClass="SyntaxNode" Kind="EventsList">
<Field Type="SyntaxToken" Name="eventsKeyword" />
<Field Type="AttributeListSyntaxNode" Name="attributes" />
<Field Type="SyntaxList" Name="events" />
<Field Type="SyntaxToken" Name="endKeyword" />
</Class>
</Syntax>