diff --git a/Parser.Tests/MLexerGreenTests.cs b/Parser.Tests/MLexerGreenTests.cs index 94d3d8b..36778c1 100644 --- a/Parser.Tests/MLexerGreenTests.cs +++ b/Parser.Tests/MLexerGreenTests.cs @@ -34,6 +34,33 @@ namespace Parser.Tests Assert.Equal(kind2, tokens[1].Kind); } + [Theory] + [MemberData(nameof(PairTokensWithSeparatorData))] + public void MLexerGreen_Parses_PairOfTokensWithSeparator(TokenKind kind1, string text1, string separatorText, TokenKind kind2, string text2) + { + var text = text1 + separatorText + text2; + var tokens = ParseText(text).ToArray(); + if (kind1 == TokenKind.Identifier && !ContainsNewLine(separatorText)) + { + Assert.Equal(TokenKind.Identifier, tokens[0].Kind); + foreach (var token in tokens.Skip(1)) + { + Assert.Equal(TokenKind.UnquotedStringLiteral, token.Kind); + } + } + else + { + Assert.Equal(2, tokens.Length); + Assert.Equal(kind1, tokens[0].Kind); + Assert.Equal(kind2, tokens[1].Kind); + } + } + + private static bool ContainsNewLine(string text) + { + return text.Contains('\r') || text.Contains('\n'); + } + public static IEnumerable SingleTokensData() { return GetTokens().Select(pair => new object[] { pair.kind, pair.text }); @@ -44,6 +71,11 @@ namespace Parser.Tests return GetPairsOfTokens().Select(data => new object[] {data.kind1, data.text1, data.kind2, data.text2}); } + public static IEnumerable PairTokensWithSeparatorData() + { + return GetPairsOfTokensWithSeparators().Select(data => new object[] {data.kind1, data.text1, data.separatorText, data.kind2, data.text2}); + } + public static IEnumerable<(TokenKind kind, string text)> GetTokens() { var fixedTokens = Enum.GetValues(typeof(TokenKind)) @@ -84,6 +116,23 @@ namespace Parser.Tests } } } + + public static IEnumerable<(TokenKind kind1, string text1, string separatorText, TokenKind kind2, string text2)> GetPairsOfTokensWithSeparators() + { + foreach (var token1 in GetTokens()) + { + foreach (var token2 in GetTokens()) + { + if (RequiresSeparator(token1.kind, token2.kind)) + { + foreach (var separatorText in GetSeparators()) + { + yield return (token1.kind, token1.text, separatorText, token2.kind, token2.text); + } + } + } + } + } private static bool RequiresSeparator(TokenKind kind1, TokenKind kind2) { @@ -213,5 +262,17 @@ namespace Parser.Tests } return false; } + + private static IEnumerable GetSeparators() + { + return new[] + { + " ", + " ", + "\r", + "\n", + "\r\n" + }; + } } }