mahalex/MParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correctly lex string literal as the first token

Browse Source
This commit is contained in:
Alexander Luzgarev 2018-03-31 22:18:04 +02:00
parent 00aae12361
commit 8acea8e90e
2 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Lexer.Tests/MLexerShould.cs
View File

@ -205,5 +205,15 @@ namespace Parser.Tests
var actual = string.Join("", tokens.Select(token => token.FullText)); var actual = string.Join("", tokens.Select(token => token.FullText));
Assert.AreEqual(s, actual); Assert.AreEqual(s, actual);
} }
[Test]
public void ParseStringLiteral()
{
var sut = CreateLexer("'just a string'");
var tokens = sut.ParseAll();
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(TokenKind.StringLiteral, tokens[0].Kind);
Assert.AreEqual("just a string", tokens[0].PureToken.Value);
}
} }
} }

9
Lexer/MLexer.cs
View File

@ -537,10 +537,11 @@ namespace Lexer
Window.ConsumeChar(); Window.ConsumeChar();
return PureTokenFactory.CreatePunctuation(TokenKind.QuestionMark); return PureTokenFactory.CreatePunctuation(TokenKind.QuestionMark);
case '\'': case '\'':
if (LastToken.PureToken.Kind == TokenKind.ClosingBrace if (LastToken != null &&
|| LastToken.PureToken.Kind == TokenKind.ClosingBracket (LastToken.Kind == TokenKind.ClosingBrace
|| LastToken.PureToken.Kind == TokenKind.ClosingSquareBracket || LastToken.Kind == TokenKind.ClosingBracket
|| LastToken.PureToken.Kind == TokenKind.Identifier) || LastToken.Kind == TokenKind.ClosingSquareBracket
|| LastToken.Kind == TokenKind.Identifier))
{ {
if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0) if (LastToken.TrailingTrivia.Count == 0 && leadingTrivia.Count == 0)
{ {