mahalex/MParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement Span on tokens & nodes

Browse Source
This commit is contained in:
Alexander Luzgarev 2020-07-23 18:51:09 +02:00
parent 3584030768
commit e60cc1bb73
16 changed files with 307 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Parser.Tests/MParserShould.cs
View File

@ -129,5 +129,26 @@ namespace Parser.Tests
rhs.FullSpan.Start.Should().Be(16); rhs.FullSpan.Start.Should().Be(16);
rhs.FullSpan.End.Should().Be(17); rhs.FullSpan.End.Should().Be(17);
} }
[Fact]
public void ProvideSpan()
{
var text = "% Comment\n 2 + 3";
var sut = GetSut(text);
var actual = sut.Parse();
var statement = actual.Root.Body.Statements[0].AsNode() as ExpressionStatementSyntaxNode;
var expression = statement!.Expression as BinaryOperationExpressionSyntaxNode;
var lhs = expression!.Lhs;
var operation = expression.Operation;
var rhs = expression.Rhs;
expression.Span.Start.Should().Be(12);
expression.Span.End.Should().Be(17);
lhs.Span.Start.Should().Be(12);
lhs.Span.End.Should().Be(13);
operation.Span.Start.Should().Be(14);
operation.Span.End.Should().Be(15);
rhs.Span.Start.Should().Be(16);
rhs.Span.End.Should().Be(17);
}
} }
} }

8
Parser/Internal/GreenNode.cs
View File

@ -191,11 +191,11 @@ namespace Parser.Internal
} }
} }
public virtual IReadOnlyList<SyntaxTrivia> LeadingTrivia => GetFirstTerminal()?.LeadingTriviaCore ?? new List<SyntaxTrivia>(); public virtual GreenNode? LeadingTrivia => GetFirstTerminal()?.LeadingTriviaCore;
public virtual IReadOnlyList<SyntaxTrivia> TrailingTrivia => GetLastTerminal()?.TrailingTriviaCore ?? new List<SyntaxTrivia>(); public virtual GreenNode? TrailingTrivia => GetLastTerminal()?.TrailingTriviaCore;
public abstract IReadOnlyList<SyntaxTrivia> LeadingTriviaCore { get; } public virtual GreenNode? LeadingTriviaCore => null;
public abstract IReadOnlyList<SyntaxTrivia> TrailingTriviaCore { get; } public virtual GreenNode? TrailingTriviaCore => null;
public virtual string FullText public virtual string FullText
{ {

4
Parser/Internal/MLexerGreen.cs
View File

@ -493,7 +493,7 @@ namespace Parser.Internal
if (TokensSinceNewLine == 1 if (TokensSinceNewLine == 1
&& !TokenStack.Any() && !TokenStack.Any()
&& LastToken.Kind == TokenKind.IdentifierToken && LastToken.Kind == TokenKind.IdentifierToken
&& LastToken.TrailingTrivia.Any() && LastToken.TrailingTrivia is not null
&& character != '=' && character != '='
&& character != '(' && character != '('
&& !SyntaxFacts.Keywords.Contains(LastToken.Text)) && !SyntaxFacts.Keywords.Contains(LastToken.Text))
@ -773,7 +773,7 @@ namespace Parser.Internal
|| LastToken.Kind == TokenKind.CloseSquareBracketToken || LastToken.Kind == TokenKind.CloseSquareBracketToken
|| LastToken.Kind == TokenKind.IdentifierToken)) || LastToken.Kind == TokenKind.IdentifierToken))
{ {
if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0) if (LastToken.TrailingTrivia is null && leadingTrivia.Count == 0)
{ {
Window.ConsumeChar(); Window.ConsumeChar();
tokenInfo.Kind = TokenKind.ApostropheToken; tokenInfo.Kind = TokenKind.ApostropheToken;

25
Parser/Internal/MParserGreen.cs
View File

@ -348,7 +348,7 @@ namespace Parser.Internal
switch (token.Kind) switch (token.Kind)
{ {
case TokenKind.OpenBraceToken: // cell array element access case TokenKind.OpenBraceToken: // cell array element access
if (options.ParsingArrayElements && expression.TrailingTrivia.Any()) if (options.ParsingArrayElements && expression.TrailingTrivia is not null)
{ {
return expression; return expression;
} }
@ -363,7 +363,7 @@ namespace Parser.Internal
); );
break; break;
case TokenKind.OpenParenthesisToken: // function call case TokenKind.OpenParenthesisToken: // function call
if (options.ParsingArrayElements && expression.TrailingTrivia.Any()) if (options.ParsingArrayElements && expression.TrailingTrivia is not null)
{ {
return expression; return expression;
} }
@ -401,7 +401,7 @@ namespace Parser.Internal
case TokenKind.UnquotedStringLiteralToken: case TokenKind.UnquotedStringLiteralToken:
return ParseCommandExpression(expression); return ParseCommandExpression(expression);
case TokenKind.AtToken: case TokenKind.AtToken:
if (expression.TrailingTrivia.Any()) if (expression.TrailingTrivia is not null)
{ {
return expression; return expression;
} }
@ -435,7 +435,7 @@ namespace Parser.Internal
private ClassInvokationExpressionSyntaxNode ParseBaseClassInvokation(ExpressionSyntaxNode expression) private ClassInvokationExpressionSyntaxNode ParseBaseClassInvokation(ExpressionSyntaxNode expression)
{ {
if (expression is IdentifierNameExpressionSyntaxNode methodName if (expression is IdentifierNameExpressionSyntaxNode methodName
&& !expression.TrailingTrivia.Any()) && expression.TrailingTrivia is null)
{ {
var atToken = EatToken(); var atToken = EatToken();
var baseClassNameWithArguments = ParseExpression(); var baseClassNameWithArguments = ParseExpression();
@ -446,7 +446,7 @@ namespace Parser.Internal
return Factory.ClassInvokationExpressionSyntax(methodName, atToken, baseClassNameWithArguments); return Factory.ClassInvokationExpressionSyntax(methodName, atToken, baseClassNameWithArguments);
} }
if (expression is MemberAccessExpressionSyntaxNode memberAccess if (expression is MemberAccessExpressionSyntaxNode memberAccess
&& !expression.TrailingTrivia.Any()) && expression.TrailingTrivia is null)
{ {
var atToken = EatToken(); var atToken = EatToken();
var baseClassNameWithArguments = ParseExpression(); var baseClassNameWithArguments = ParseExpression();
@ -530,7 +530,7 @@ namespace Parser.Internal
var builder = new SyntaxListBuilder(); var builder = new SyntaxListBuilder();
builder.Add(firstName); builder.Add(firstName);
while (CurrentToken.Kind == TokenKind.DotToken while (CurrentToken.Kind == TokenKind.DotToken
&& !lastToken.TrailingTrivia.Any()) && lastToken.TrailingTrivia is null)
{ {
var dot = EatToken(); var dot = EatToken();
builder.Add(dot); builder.Add(dot);
@ -856,11 +856,18 @@ namespace Parser.Internal
return Factory.ExpressionStatementSyntax(expression, semicolon: null); return Factory.ExpressionStatementSyntax(expression, semicolon: null);
} }
private bool TriviaContainsNewLine(IReadOnlyList<SyntaxTrivia> trivia) private bool TriviaContainsNewLine(GreenNode? trivia)
{ {
foreach(var t in trivia) var triviaList = trivia as SyntaxList<SyntaxTrivia>;
if (triviaList is null)
{ {
if (t.Text.Contains('\n')) return false;
}
for (var i = 0; i < triviaList.Length; i++)
{
var text = triviaList[i].Text;
if (text.Contains('\n'))
{ {
return true; return true;
} }

2
Parser/Internal/SyntaxList.cs
View File

@ -58,5 +58,7 @@ namespace Parser.Internal
{ {
return new SyntaxList(_elements, diagnostics); return new SyntaxList(_elements, diagnostics);
} }
public int Length => _elements.Length;
} }
} }

12
Parser/Internal/SyntaxList`1.cs
View File

@ -1,4 +1,5 @@
using System.Linq; using System.Collections.Generic;
using System.Linq;
namespace Parser.Internal namespace Parser.Internal
{ {
@ -31,11 +32,18 @@ namespace Parser.Internal
return (T)_list.GetListSlot(i); return (T)_list.GetListSlot(i);
} }
public T this[int i] => (T)_list.GetListSlot(i);
public static SyntaxList<T> List(T[] elements) public static SyntaxList<T> List(T[] elements)
{ {
return new SyntaxList<T>(elements); return new SyntaxList<T>(elements);
} }
public static SyntaxList<T> List(IReadOnlyList<T> elements)
{
return new SyntaxList<T>(elements.ToArray());
}
public static SyntaxList<T> Empty => new SyntaxList<T>(new T[] { }); public static SyntaxList<T> Empty => new SyntaxList<T>(new T[] { });
public override bool IsList => true; public override bool IsList => true;
@ -49,5 +57,7 @@ namespace Parser.Internal
{ {
return new SyntaxList<T>(_list._elements.Select(x => (T)x).ToArray(), diagnostics); return new SyntaxList<T>(_list._elements.Select(x => (T)x).ToArray(), diagnostics);
} }
public int Length => _list.Length;
} }
} }

4
Parser/Internal/SyntaxNode.cs
View File

@ -53,8 +53,8 @@ namespace Parser.Internal
public override string FullText => CollectFullText(); public override string FullText => CollectFullText();
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => throw new NotImplementedException(); public override GreenNode? LeadingTriviaCore => throw new NotImplementedException();
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => throw new NotImplementedException(); public override GreenNode? TrailingTriviaCore => throw new NotImplementedException();
} }
internal abstract class StatementSyntaxNode : SyntaxNode internal abstract class StatementSyntaxNode : SyntaxNode

118
Parser/Internal/SyntaxToken.cs
View File

@ -1,7 +1,6 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
namespace Parser.Internal namespace Parser.Internal
{ {
@ -16,66 +15,66 @@ namespace Parser.Internal
public SyntaxTokenWithTrivia( public SyntaxTokenWithTrivia(
TokenKind kind, TokenKind kind,
string text, string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) : base(kind) GreenNode? trailingTrivia) : base(kind)
{ {
_text = text; _text = text;
LeadingTriviaCore = leadingTrivia; LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia; TrailingTriviaCore = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (_text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public SyntaxTokenWithTrivia( public SyntaxTokenWithTrivia(
TokenKind kind, TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) : base(kind) GreenNode? trailingTrivia) : base(kind)
{ {
_text = base.Text; _text = base.Text;
LeadingTriviaCore = leadingTrivia; LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia; TrailingTriviaCore = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (_text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (_text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public SyntaxTokenWithTrivia( public SyntaxTokenWithTrivia(
TokenKind kind, TokenKind kind,
string text, string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia, GreenNode? trailingTrivia,
TokenDiagnostic[] diagnostics) : base(kind, diagnostics) TokenDiagnostic[] diagnostics) : base(kind, diagnostics)
{ {
_text = text; _text = text;
LeadingTriviaCore = leadingTrivia; LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia; TrailingTriviaCore = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (_text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public SyntaxTokenWithTrivia( public SyntaxTokenWithTrivia(
TokenKind kind, TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia, GreenNode? trailingTrivia,
TokenDiagnostic[] diagnostics) : base(kind, diagnostics) TokenDiagnostic[] diagnostics) : base(kind, diagnostics)
{ {
_text = base.Text; _text = base.Text;
LeadingTriviaCore = leadingTrivia; LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia; TrailingTriviaCore = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (_text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (_text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing) public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{ {
if (leading) if (leading && LeadingTrivia is SyntaxList<SyntaxTrivia> leadingTrivia)
{ {
foreach (var trivia in LeadingTrivia) for (var i = 0; i < leadingTrivia.Length; i++)
{ {
writer.Write(trivia.Text); leadingTrivia[i].WriteTriviaTo(writer);
} }
} }
base.WriteTokenTo(writer, leading, trailing); base.WriteTokenTo(writer, leading, trailing);
if (trailing) if (trailing && TrailingTrivia is SyntaxList<SyntaxTrivia> trailingTrivia)
{ {
foreach (var trivia in TrailingTrivia) for (var i = 0; i < trailingTrivia.Length; i++)
{ {
writer.Write(trivia.Text); trailingTrivia[i].WriteTriviaTo(writer);
} }
} }
} }
@ -85,9 +84,9 @@ namespace Parser.Internal
return new SyntaxTokenWithTrivia(Kind, _text, LeadingTrivia, TrailingTrivia, diagnostics); return new SyntaxTokenWithTrivia(Kind, _text, LeadingTrivia, TrailingTrivia, diagnostics);
} }
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore { get; } public override GreenNode? LeadingTriviaCore { get; }
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore { get; } public override GreenNode? TrailingTriviaCore { get; }
} }
internal class SyntaxTokenWithValue<T> : SyntaxToken internal class SyntaxTokenWithValue<T> : SyntaxToken
@ -138,46 +137,47 @@ namespace Parser.Internal
TokenKind kind, TokenKind kind,
string text, string text,
T value, T value,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) : base(kind, text, value) GreenNode? trailingTrivia)
: base(kind, text, value)
{ {
LeadingTriviaCore = leadingTrivia; LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia; TrailingTriviaCore = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public SyntaxTokenWithValueAndTrivia( public SyntaxTokenWithValueAndTrivia(
TokenKind kind, TokenKind kind,
string text, string text,
T value, T value,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia, GreenNode? trailingTrivia,
TokenDiagnostic[] diagnostics) : base(kind, text, value, diagnostics) TokenDiagnostic[] diagnostics) : base(kind, text, value, diagnostics)
{ {
LeadingTriviaCore = leadingTrivia; LeadingTriviaCore = leadingTrivia;
TrailingTriviaCore = trailingTrivia; TrailingTriviaCore = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore { get; } public override GreenNode? LeadingTriviaCore { get; }
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore { get; } public override GreenNode? TrailingTriviaCore { get; }
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing) public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{ {
if (leading) if (leading && LeadingTrivia is SyntaxList<SyntaxTrivia> leadingTrivia)
{ {
foreach (var trivia in LeadingTrivia) for (var i = 0; i < leadingTrivia.Length; i++)
{ {
writer.Write(trivia.Text); leadingTrivia[i].WriteTriviaTo(writer);
} }
} }
base.WriteTokenTo(writer, leading, trailing); base.WriteTokenTo(writer, leading, trailing);
if (trailing) if (trailing && TrailingTrivia is SyntaxList<SyntaxTrivia> trailingTrivia)
{ {
foreach (var trivia in TrailingTrivia) for (var i = 0; i < trailingTrivia.Length; i++)
{ {
writer.Write(trivia.Text); trailingTrivia[i].WriteTriviaTo(writer);
} }
} }
} }
@ -220,49 +220,49 @@ namespace Parser.Internal
internal class SyntaxIdentifierWithTrivia : SyntaxIdentifier internal class SyntaxIdentifierWithTrivia : SyntaxIdentifier
{ {
private readonly IReadOnlyList<SyntaxTrivia> _leadingTrivia; private readonly GreenNode? _leadingTrivia;
private readonly IReadOnlyList<SyntaxTrivia> _trailingTrivia; private readonly GreenNode? _trailingTrivia;
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => _leadingTrivia; public override GreenNode? LeadingTriviaCore => _leadingTrivia;
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => _trailingTrivia; public override GreenNode? TrailingTriviaCore => _trailingTrivia;
public SyntaxIdentifierWithTrivia( public SyntaxIdentifierWithTrivia(
string text, string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia GreenNode? trailingTrivia)
) : base(text) : base(text)
{ {
_leadingTrivia = leadingTrivia; _leadingTrivia = leadingTrivia;
_trailingTrivia = trailingTrivia; _trailingTrivia = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public SyntaxIdentifierWithTrivia( public SyntaxIdentifierWithTrivia(
string text, string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia, GreenNode? trailingTrivia,
TokenDiagnostic[] diagnostics) : base(text, diagnostics) TokenDiagnostic[] diagnostics) : base(text, diagnostics)
{ {
_leadingTrivia = leadingTrivia; _leadingTrivia = leadingTrivia;
_trailingTrivia = trailingTrivia; _trailingTrivia = trailingTrivia;
_fullWidth = (leadingTrivia?.Sum(t => t.FullWidth) ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.Sum(t => t.FullWidth) ?? 0); _fullWidth = (leadingTrivia?.FullWidth ?? 0) + (text?.Length ?? 0) + (trailingTrivia?.FullWidth ?? 0);
} }
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing) public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{ {
if (leading) if (leading && LeadingTrivia is SyntaxList<SyntaxTrivia> leadingTrivia)
{ {
foreach (var trivia in LeadingTrivia) for (var i = 0; i < leadingTrivia.Length; i++)
{ {
writer.Write(trivia.Text); leadingTrivia[i].WriteTriviaTo(writer);
} }
} }
base.WriteTokenTo(writer, leading, trailing); base.WriteTokenTo(writer, leading, trailing);
if (trailing) if (trailing && TrailingTrivia is SyntaxList<SyntaxTrivia> trailingTrivia)
{ {
foreach (var trivia in TrailingTrivia) for (var i = 0; i < trailingTrivia.Length; i++)
{ {
writer.Write(trivia.Text); trailingTrivia[i].WriteTriviaTo(writer);
} }
} }
} }
@ -280,8 +280,8 @@ namespace Parser.Internal
{ {
public MissingTokenWithTrivia( public MissingTokenWithTrivia(
TokenKind kind, TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia GreenNode? trailingTrivia
) : base(kind, leadingTrivia, trailingTrivia) ) : base(kind, leadingTrivia, trailingTrivia)
{ {
_isMissing = true; _isMissing = true;
@ -289,8 +289,8 @@ namespace Parser.Internal
public MissingTokenWithTrivia( public MissingTokenWithTrivia(
TokenKind kind, TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia, GreenNode? leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia, GreenNode? trailingTrivia,
TokenDiagnostic[] diagnostics) : base(kind, leadingTrivia, trailingTrivia, diagnostics) TokenDiagnostic[] diagnostics) : base(kind, leadingTrivia, trailingTrivia, diagnostics)
{ {
_isMissing = true; _isMissing = true;
@ -312,7 +312,7 @@ namespace Parser.Internal
{ {
} }
internal static SyntaxToken NoneToken => new MissingTokenWithTrivia(TokenKind.None, s_EmptySyntaxTriviaList, s_EmptySyntaxTriviaList); internal static SyntaxToken NoneToken => new MissingTokenWithTrivia(TokenKind.None, null, null);
public virtual object? Value => null; public virtual object? Value => null;
@ -320,8 +320,8 @@ namespace Parser.Internal
public virtual int Width => Text.Length; public virtual int Width => Text.Length;
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => s_EmptySyntaxTriviaList; public override GreenNode? LeadingTriviaCore => null;
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => s_EmptySyntaxTriviaList; public override GreenNode? TrailingTriviaCore => null;
public override GreenNode? GetSlot(int i) public override GreenNode? GetSlot(int i)
{ {

9
Parser/Internal/SyntaxTrivia.cs
View File

@ -40,12 +40,17 @@ namespace Parser.Internal
writer.Write(_text); writer.Write(_text);
} }
public override void WriteTokenTo(TextWriter writer, bool leading, bool trailing)
{
writer.Write(_text);
}
public override GreenNode SetDiagnostics(TokenDiagnostic[] diagnostics) public override GreenNode SetDiagnostics(TokenDiagnostic[] diagnostics)
{ {
return new SyntaxTrivia(Kind, _text, diagnostics); return new SyntaxTrivia(Kind, _text, diagnostics);
} }
public override IReadOnlyList<SyntaxTrivia> LeadingTriviaCore => new List<SyntaxTrivia>(); public override GreenNode? LeadingTriviaCore => null;
public override IReadOnlyList<SyntaxTrivia> TrailingTriviaCore => new List<SyntaxTrivia>(); public override GreenNode? TrailingTriviaCore => null;
} }
} }

52
Parser/Internal/TokenFactory.cs
View File

@ -13,6 +13,16 @@ namespace Parser.Internal
TokenKind kind, TokenKind kind,
IReadOnlyList<SyntaxTrivia> leadingTrivia, IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
var leading = leadingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(leadingTrivia) : null;
var trailing = trailingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(trailingTrivia) : null;
return new SyntaxToken.SyntaxTokenWithTrivia(kind, leading, trailing);
}
public static SyntaxToken CreateTokenWithTrivia(
TokenKind kind,
GreenNode? leadingTrivia,
GreenNode? trailingTrivia)
{ {
return new SyntaxToken.SyntaxTokenWithTrivia(kind, leadingTrivia, trailingTrivia); return new SyntaxToken.SyntaxTokenWithTrivia(kind, leadingTrivia, trailingTrivia);
} }
@ -21,6 +31,16 @@ namespace Parser.Internal
string text, string text,
IReadOnlyList<SyntaxTrivia> leadingTrivia, IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
var leading = leadingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(leadingTrivia) : null;
var trailing = trailingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(trailingTrivia) : null;
return new SyntaxToken.SyntaxIdentifierWithTrivia(text, leading, trailing);
}
public static SyntaxToken CreateIdentifier(
string text,
GreenNode? leadingTrivia,
GreenNode? trailingTrivia)
{ {
return new SyntaxToken.SyntaxIdentifierWithTrivia(text, leadingTrivia, trailingTrivia); return new SyntaxToken.SyntaxIdentifierWithTrivia(text, leadingTrivia, trailingTrivia);
} }
@ -31,6 +51,18 @@ namespace Parser.Internal
T value, T value,
IReadOnlyList<SyntaxTrivia> leadingTrivia, IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
var leading = leadingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(leadingTrivia) : null;
var trailing = trailingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(trailingTrivia) : null;
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<T>(kind, text, value, leading, trailing);
}
public static SyntaxToken CreateTokenWithValueAndTrivia<T>(
TokenKind kind,
string text,
T value,
GreenNode? leadingTrivia,
GreenNode? trailingTrivia)
{ {
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<T>(kind, text, value, leadingTrivia, trailingTrivia); return new SyntaxToken.SyntaxTokenWithValueAndTrivia<T>(kind, text, value, leadingTrivia, trailingTrivia);
} }
@ -40,6 +72,22 @@ namespace Parser.Internal
string value, string value,
IReadOnlyList<SyntaxTrivia> leadingTrivia, IReadOnlyList<SyntaxTrivia> leadingTrivia,
IReadOnlyList<SyntaxTrivia> trailingTrivia) IReadOnlyList<SyntaxTrivia> trailingTrivia)
{
var leading = leadingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(leadingTrivia) : null;
var trailing = trailingTrivia.Count > 0 ? SyntaxList<SyntaxTrivia>.List(trailingTrivia) : null;
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>(
TokenKind.UnquotedStringLiteralToken,
text,
value,
leading,
trailing);
}
public static SyntaxToken CreateUnquotedStringLiteral(
string text,
string value,
GreenNode? leadingTrivia,
GreenNode? trailingTrivia)
{ {
return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>( return new SyntaxToken.SyntaxTokenWithValueAndTrivia<string>(
TokenKind.UnquotedStringLiteralToken, TokenKind.UnquotedStringLiteralToken,
@ -54,7 +102,9 @@ namespace Parser.Internal
IReadOnlyList<SyntaxTrivia>? leadingTrivia, IReadOnlyList<SyntaxTrivia>? leadingTrivia,
IReadOnlyList<SyntaxTrivia>? trailingTrivia) IReadOnlyList<SyntaxTrivia>? trailingTrivia)
{ {
return new SyntaxToken.MissingTokenWithTrivia(kind, leadingTrivia ?? SyntaxToken.s_EmptySyntaxTriviaList, trailingTrivia ?? SyntaxToken.s_EmptySyntaxTriviaList); var leading = (leadingTrivia is { } l && l.Count > 0) ? SyntaxList<SyntaxTrivia>.List(leadingTrivia) : null;
var trailing = (trailingTrivia is { } c && c.Count > 0) ? SyntaxList<SyntaxTrivia>.List(trailingTrivia) : null;
return new SyntaxToken.MissingTokenWithTrivia(kind, leading, trailing);
} }
} }
} }

27
Parser/SyntaxNavigator.cs Normal file
View File

@ -0,0 +1,27 @@
using System.Collections.Generic;
namespace Parser
{
public class SyntaxNavigator
{
public static SyntaxNavigator Singleton = new SyntaxNavigator();
public IEnumerable<SyntaxToken> EnumerateTokens(SyntaxNode node)
{
foreach (var child in node.GetChildNodesAndTokens())
{
if (child.IsNode)
{
foreach (var token in EnumerateTokens(child.AsNode()!))
{
yield return token;
}
}
if (child.IsToken)
{
yield return child.AsToken();
}
}
}
}
}

38
Parser/SyntaxNode.cs
View File

@ -1,6 +1,5 @@
using Parser.Internal; using Parser.Internal;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq; using System.Linq;
namespace Parser namespace Parser
@ -40,6 +39,15 @@ namespace Parser
public TextSpan FullSpan { get; } public TextSpan FullSpan { get; }
public TextSpan Span => CalculateSpan();
private TextSpan CalculateSpan()
{
var leadingTriviaWidth = LeadingTrivia?.Width ?? 0;
var trailingTriviaWidth = TrailingTrivia?.Width ?? 0;
return new TextSpan(Position + leadingTriviaWidth, _green.FullWidth - leadingTriviaWidth - trailingTriviaWidth);
}
public int FullWidth => _green.FullWidth; public int FullWidth => _green.FullWidth;
internal int GetChildPosition(int slot) internal int GetChildPosition(int slot)
@ -82,24 +90,32 @@ namespace Parser
public virtual string FullText => _green.FullText; public virtual string FullText => _green.FullText;
public virtual IReadOnlyList<SyntaxTrivia> LeadingTrivia public virtual SyntaxTriviaList? LeadingTrivia
{ {
get get
{ {
var p = Parent; return GetFirstToken()?.LeadingTrivia;
return _green.LeadingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
} }
} }
public virtual IReadOnlyList<SyntaxTrivia> TrailingTrivia public virtual SyntaxTriviaList? TrailingTrivia
{ {
get get
{ {
var p = Parent; return GetLastToken()?.TrailingTrivia;
return _green.TrailingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
} }
} }
public SyntaxToken? GetFirstToken()
{
return SyntaxNavigator.Singleton.EnumerateTokens(this).Select(t => (SyntaxToken?)t).FirstOrDefault();
}
public SyntaxToken? GetLastToken()
{
return SyntaxNavigator.Singleton.EnumerateTokens(this).Select(t => (SyntaxToken?)t).LastOrDefault();
}
public abstract void Accept(SyntaxVisitor visitor); public abstract void Accept(SyntaxVisitor visitor);
public SyntaxDiagnostic[] GetDiagnostics() public SyntaxDiagnostic[] GetDiagnostics()
@ -170,12 +186,16 @@ namespace Parser
internal override SyntaxNode? GetNode(int index) internal override SyntaxNode? GetNode(int index)
{ {
throw new System.NotImplementedException(); return index switch
{
0 => GetRed(ref _file!, 0),
_ => null,
};
} }
public override void Accept(SyntaxVisitor visitor) public override void Accept(SyntaxVisitor visitor)
{ {
throw new System.NotImplementedException(); visitor.VisitRoot(this);
} }

21
Parser/SyntaxToken.cs
View File

@ -33,6 +33,15 @@ namespace Parser
public TextSpan FullSpan { get; } public TextSpan FullSpan { get; }
public TextSpan Span => CalculateSpan();
public TextSpan CalculateSpan()
{
var leadingTriviaWidth = LeadingTrivia.Width;
var trailingTriviaWidth = TrailingTrivia.Width;
return new TextSpan(Position + leadingTriviaWidth, FullWidth - leadingTriviaWidth - trailingTriviaWidth);
}
public object? Value => _token.GetValue(); public object? Value => _token.GetValue();
public bool Equals(SyntaxToken other) public bool Equals(SyntaxToken other)
@ -69,21 +78,21 @@ namespace Parser
public int FullWidth => _token.FullWidth; public int FullWidth => _token.FullWidth;
public bool IsMissing => _token.IsMissing; public bool IsMissing => _token.IsMissing;
public IReadOnlyList<SyntaxTrivia> LeadingTrivia public SyntaxTriviaList LeadingTrivia
{ {
get get
{ {
var p = _parent; return new SyntaxTriviaList(this, Token.LeadingTriviaCore, this.Position);
return _token.LeadingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList();
} }
} }
public IReadOnlyList<SyntaxTrivia> TrailingTrivia public SyntaxTriviaList TrailingTrivia
{ {
get get
{ {
var p = _parent; var trailingGreen = Token.TrailingTriviaCore;
return _token.TrailingTrivia.Select(trivia => new SyntaxTrivia(p, trivia)).ToImmutableList(); var trailingTriviaWidth = trailingGreen?.FullWidth ?? 0;
return new SyntaxTriviaList(this, trailingGreen, this.Position + this.FullWidth - trailingTriviaWidth);
} }
} }
} }

52
Parser/SyntaxTriviaList.cs Normal file
View File

@ -0,0 +1,52 @@
using Parser.Internal;
using System;
using System.Collections;
using System.Collections.Generic;
namespace Parser
{
public struct SyntaxTriviaList : IReadOnlyList<SyntaxTrivia>
{
internal GreenNode? Node { get; }
internal SyntaxTriviaList(SyntaxToken token, GreenNode? node, int position)
{
Node = node;
Token = token;
Position = position;
}
public SyntaxToken Token { get; }
public int Position { get; }
public int Count => (Node as SyntaxList<Internal.SyntaxTrivia>)?.Length ?? 0;
public int Width => Node?.FullWidth ?? 0;
public SyntaxTrivia this[int index]
{
get
{
return Node switch
{
SyntaxList<Internal.SyntaxTrivia> triviaList => new SyntaxTrivia(Token.Parent, triviaList[index]),
_ => throw new IndexOutOfRangeException(),
};
}
}
public IEnumerator<SyntaxTrivia> GetEnumerator()
{
for (var i = 0; i < Count; i++)
{
yield return this[i];
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}

5
Parser/SyntaxVisitor.cs
View File

@ -15,5 +15,10 @@
{ {
DefaultVisit(list); DefaultVisit(list);
} }
public virtual void VisitRoot(RootSyntaxNode node)
{
DefaultVisit(node);
}
} }
} }

8
Semantics/GetClass.cs
View File

@ -11,14 +11,14 @@ namespace Semantics
{ {
var name = methodDefinition.Name.Text; var name = methodDefinition.Name.Text;
var description = ""; var description = "";
description += string.Join("", methodDefinition.LeadingTrivia.Select(x => x.FullText)); description += string.Join("", methodDefinition.LeadingTrivia?.Select(x => x.FullText));
if (methodDefinition.Body == null) if (methodDefinition.Body == null)
{ {
description += string.Join("", methodDefinition.EndKeyword.LeadingTrivia.Select(x => x.FullText)); description += string.Join("", methodDefinition.EndKeyword.LeadingTrivia?.Select(x => x.FullText));
} }
else else
{ {
description += string.Join("", methodDefinition.Body.LeadingTrivia.Select(x => x.FullText)); description += string.Join("", methodDefinition.Body.LeadingTrivia?.Select(x => x.FullText));
} }
return new MMethod(name, description); return new MMethod(name, description);
@ -28,7 +28,7 @@ namespace Semantics
{ {
var name = methodDeclaration.Name.Text; var name = methodDeclaration.Name.Text;
var description = ""; var description = "";
description += string.Join("", methodDeclaration.LeadingTrivia.Select(x => x.FullText)); description += string.Join("", methodDeclaration.LeadingTrivia?.Select(x => x.FullText));
return new MMethod(name, description); return new MMethod(name, description);
} }