A Swiss Army knife for developers.
C# blazorcross-platformcsharpdesktopdesktop-appdesktop-applicationdeveloper-toolsextensiblefluentfluent-designlinuxmacosmicasyntax-highlightingtypescriptwindowswindows-10windows-11winui
Sql-formatter optimization (#368)
A. Karboush725fa91c6e55
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.csindex 5dde9b9b8..cdba05b50 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs@@ -3,6 +3,7 @@using System;using System.Collections.Generic;using System.Linq;+using System.Text;using System.Text.RegularExpressions;using DevToys.Shared.Core;@@ -10,11 +11,11 @@ namespace DevToys.Helpers.SqlFormatter.Core{internal abstract class Formatter{- private static readonly Regex SpacesEndRegex = new Regex(@"[ \t]+$", RegexOptions.Compiled);- private static readonly Regex WhitespacesRegex = new Regex(@"\s+$", RegexOptions.Compiled);- private static readonly Regex CommentWhitespacesRegex = new Regex(@"\n[ \t]*", RegexOptions.Compiled);+ private static readonly Regex WhitespacesRegex = new Regex(@"\s+$", RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);+ private static readonly Regex CommentWhitespacesRegex = new Regex(@"\n[ \t]*", RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);private readonly InlineBlock _inlineBlock = new();+ private readonly StringBuilder _queryBuilder = new();private Indentation? _indentation = null;private SqlFormatterOptions _options;@@ -41,8 +42,8 @@ namespace DevToys.Helpers.SqlFormatter.Core_params = new Params(options.PlaceholderParameters);_tokens = GetTokenizer().Tokenize(query);- string? formattedQuery = GetFormattedQueryFromTokens();- return formattedQuery.Trim();+ SetFormattedQueryFromTokens(query.AsSpan());+ return _queryBuilder.ToString().Trim();}/// <summary>@@ -53,7 +54,7 @@ namespace DevToys.Helpers.SqlFormatter.Core/// <summary>/// Reprocess and modify a token based on parsed context./// </summary>- protected virtual Token TokenOverride(Token token)+ protected virtual Token TokenOverride(Token token, ReadOnlySpan<char> querySpan){// subclasses can override this to modify tokens during formattingreturn token;@@ -61,12 +62,7 @@ namespace DevToys.Helpers.SqlFormatter.Coreprotected Token? TokenLookBehind(int n = 1){- if (_tokens is null || _tokens!.Count <= _index - n || _index - n < 0)- {- return null;- }-- return _tokens![_index - n];+ return _tokens?.ElementAtOrDefault(_index - n);}protected Token? TokenLookAhead(int n = 1)@@ -78,127 +74,135 @@ namespace DevToys.Helpers.SqlFormatter.Corereturn _tokens![_index + n];}- private string GetFormattedQueryFromTokens()+ private void SetFormattedQueryFromTokens(ReadOnlySpan<char> querySpan){- string formattedQuery = string.Empty;+ _queryBuilder.Clear();Assumes.NotNull(_tokens, nameof(_tokens));for (int i = 0; i < _tokens!.Count; i++){_index = i;- Token token = TokenOverride(_tokens[i]);- if (token.Type == TokenType.LineComment)- {- formattedQuery = FormatLineComment(token, formattedQuery);- }- else if (token.Type == TokenType.BlockComment)- {- formattedQuery = FormatBlockComment(token, formattedQuery);- }- else if (token.Type == TokenType.ReservedTopLevel)- {- formattedQuery = FormatTopLevelReservedWord(token, formattedQuery);- _previousReservedToken = token;- }- else if (token.Type == TokenType.ReservedTopLevelNoIndent)- {- formattedQuery = FormatTopLevelReservedWordNoIndent(token, formattedQuery);- _previousReservedToken = token;- }- else if (token.Type == TokenType.ReservedNewLine)- {- formattedQuery = FormatNewlineReservedWord(token, formattedQuery);- _previousReservedToken = token;- }- else if (token.Type == TokenType.Reserved)- {- formattedQuery = FormatWithSpaces(token, formattedQuery);- _previousReservedToken = token;- }- else if (token.Type == TokenType.OpenParen)- {- formattedQuery = FormatOpeningParentheses(token, formattedQuery);- }- else if (token.Type == TokenType.CloseParen)- {- formattedQuery = FormatClosingParentheses(token, formattedQuery);- }- else if (token.Type == TokenType.PlaceHolder)- {- formattedQuery = FormatPlaceholder(token, formattedQuery);- }- else if (string.Equals(token.Value, ",", System.StringComparison.Ordinal))- {- formattedQuery = FormatComma(token, formattedQuery);- }- else if (string.Equals(token.Value, ":", System.StringComparison.Ordinal))- {- formattedQuery = FormatWithSpaceAfter(token, formattedQuery);- }- else if (string.Equals(token.Value, ".", System.StringComparison.Ordinal))- {- formattedQuery = FormatWithoutSpaces(token, formattedQuery);- }- else if (string.Equals(token.Value, ";", System.StringComparison.Ordinal))- {- formattedQuery = FormatQuerySeparator(token, formattedQuery);- }- else+ Token token = TokenOverride(_tokens[i], querySpan);+ switch (token.Type){- formattedQuery = FormatWithSpaces(token, formattedQuery);+ case TokenType.LineComment:+ FormatLineComment(token, querySpan);+ break;+ case TokenType.BlockComment:+ FormatBlockComment(token, querySpan);+ break;+ case TokenType.ReservedTopLevel:+ FormatTopLevelReservedWord(token, querySpan);+ _previousReservedToken = token;+ break;+ case TokenType.ReservedTopLevelNoIndent:+ FormatTopLvoidReservedWordNoIndent(token, querySpan);+ _previousReservedToken = token;+ break;+ case TokenType.ReservedNewLine:+ FormatNewlineReservedWord(token, querySpan);+ _previousReservedToken = token;+ break;+ case TokenType.Reserved:+ FormatWithSpaces(token, querySpan);+ _previousReservedToken = token;+ break;+ case TokenType.OpenParen:+ FormatOpeningParentheses(token, querySpan);+ break;+ case TokenType.CloseParen:+ FormatClosingParentheses(token, querySpan);+ break;+ case TokenType.PlaceHolder:+ FormatPlaceholder(token, querySpan);+ break;+ default:+ switch (token.Length)+ {+ case 1 when querySpan[token.Index] == ',':+ FormatComma(token, querySpan);+ break;+ case 1 when querySpan[token.Index] == ':':+ FormatWithSpaceAfter(token, querySpan);+ break;+ case 1 when querySpan[token.Index] == '.':+ FormatWithoutSpaces(token, querySpan);+ break;+ case 1 when querySpan[token.Index] == ';':+ FormatQuerySeparator(token, querySpan);+ break;+ default:+ FormatWithSpaces(token, querySpan);+ break;+ }+ break;}}-- return formattedQuery;}- private string FormatLineComment(Token token, string query)+ private void FormatLineComment(Token token, ReadOnlySpan<char> querySpan){- return AddNewLine(query + Show(token));+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));+ AddNewLine();}- private string FormatBlockComment(Token token, string query)+ private void FormatBlockComment(Token token, ReadOnlySpan<char> querySpan){- return AddNewLine(AddNewLine(query) + IndentComment(token.Value));+ AddNewLine();+ _queryBuilder.Append(IndentComment(querySpan.Slice(token)));+ AddNewLine();}- private string IndentComment(string comment)+ private string IndentComment(ReadOnlySpan<char> comment){Assumes.NotNull(_indentation, nameof(_indentation));- return CommentWhitespacesRegex.Replace(comment, "\n" + _indentation!.GetIndent() + " ");+ return CommentWhitespacesRegex.Replace(comment.ToString(), $"\n{_indentation!.GetIndent()} ");}- private string FormatTopLevelReservedWordNoIndent(Token token, string query)+ private void FormatTopLvoidReservedWordNoIndent(Token token, ReadOnlySpan<char> querySpan){Assumes.NotNull(_indentation, nameof(_indentation));_indentation!.DecreaseTopLevel();- query = AddNewLine(query) + EqualizeWhitespace(Show(token));- return AddNewLine(query);+ AddNewLine();++ _queryBuilder.Append(EqualizeWhitespace(Show(querySpan.Slice(token), token.Type)));++ AddNewLine();}- private string FormatTopLevelReservedWord(Token token, string query)+ private void FormatTopLevelReservedWord(Token token, ReadOnlySpan<char> querySpan){Assumes.NotNull(_indentation, nameof(_indentation));_indentation!.DecreaseTopLevel();- query = AddNewLine(query);+ AddNewLine();_indentation.IncreaseTopLevel();- query += EqualizeWhitespace(Show(token));- return AddNewLine(query);+ _queryBuilder.Append(EqualizeWhitespace(Show(querySpan.Slice(token), token.Type)));++ AddNewLine();}- private string FormatNewlineReservedWord(Token token, string query)+ private void FormatNewlineReservedWord(Token token, ReadOnlySpan<char> querySpan){- if (TokenHelper.IsAnd(token) && TokenHelper.isBetween(TokenLookBehind(2)))+ if (token.IsAnd(querySpan.Slice(token))){- return FormatWithSpaces(token, query);+ Token? t = TokenLookBehind(2);++ if (t != null && t.Value.IsBetween(querySpan.Slice(t.Value)))+ {+ FormatWithSpaces(token, querySpan);+ return;+ }}+ AddNewLine();++ _queryBuilder.Append(EqualizeWhitespace(Show(querySpan.Slice(token), token.Type)));- return AddNewLine(query) + EqualizeWhitespace(Show(token)) + " ";+ _queryBuilder.Append(' ');}/// <summary>@@ -212,136 +216,158 @@ namespace DevToys.Helpers.SqlFormatter.Core/// <summary>/// Opening parentheses increase the block indent level and start a new line/// </summary>- private string FormatOpeningParentheses(Token token, string query)+ private void FormatOpeningParentheses(Token token, ReadOnlySpan<char> querySpan){// Take out the preceding space unless there was whitespace there in the original query// or another opening parens or line comment- if (token.WhitespaceBefore?.Length == 0)+ if (token.PrecedingWitespaceLength == 0){- TokenType? type = TokenLookBehind()?.Type;- if (type.HasValue && type != TokenType.OpenParen && type != TokenType.LineComment && type != TokenType.Operator)+ Token? behindToken = TokenLookBehind();++ if (behindToken is not { Type: TokenType.OpenParen or TokenType.LineComment or TokenType.Operator }){- query = TrimSpacesEnd(query);+ _queryBuilder.TrimSpaceEnd();}}-- query += Show(token);+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));Assumes.NotNull(_tokens, nameof(token));- _inlineBlock.BeginIfPossible(_tokens!, _index);+ _inlineBlock.BeginIfPossible(_tokens!, _index, querySpan.Slice(token));if (!_inlineBlock.IsActive()){Assumes.NotNull(_indentation, nameof(_indentation));_indentation!.IncreaseBlockLevel();- query = AddNewLine(query);+ AddNewLine();}-- return query;}/// <summary>/// Closing parentheses decrease the block indent level/// </summary>- private string FormatClosingParentheses(Token token, string query)+ private void FormatClosingParentheses(Token token, ReadOnlySpan<char> querySpan){Assumes.NotNull(_indentation, nameof(_indentation));if (_inlineBlock.IsActive()){_inlineBlock.End();- return FormatWithSpaceAfter(token, query);+ FormatWithSpaceAfter(token, querySpan);}else{_indentation!.DecreaseBlockLevel();- return FormatWithSpaces(token, AddNewLine(query));+ AddNewLine();+ FormatWithSpaces(token, querySpan);}}- private string FormatPlaceholder(Token token, string query)+ private void FormatPlaceholder(Token token, ReadOnlySpan<char> querySpan){Assumes.NotNull(_params, nameof(_params));- return query + _params!.Get(token) + ' ';++ string? value = null;+ ReadOnlySpan<char> valueSpan = querySpan.Slice(token);+ if (valueSpan.Length != 0)+ {+ // assumme the length of all placeholder is 1, since only char array is accepted+ value = _params!.Get(valueSpan.Slice(0, 1).ToString());+ }++ _queryBuilder.Append(value ?? querySpan.Slice(token).ToString());++ _queryBuilder.Append(' ');}/// <summary>/// Commas start a new line (unless within inline parentheses or SQL "LIMIT" clause)/// </summary>- private string FormatComma(Token token, string query)+ private void FormatComma(Token token, ReadOnlySpan<char> querySpan){- query = FormatWithSpaceAfter(token, query);+ FormatWithSpaceAfter(token, querySpan);if (_inlineBlock.IsActive()){- return query;- }- else if (TokenHelper.isLimit(_previousReservedToken))- {- return query;+ return;}- else+ else if (_previousReservedToken is not null+ && _previousReservedToken.Value.IsLimit(querySpan.Slice(_previousReservedToken.Value))){- return AddNewLine(query);+ return;}++ AddNewLine();+}- private string FormatWithSpaceAfter(Token token, string query)+ private void FormatWithSpaceAfter(Token token, ReadOnlySpan<char> querySpan){- return TrimSpacesEnd(query) + Show(token) + " ";+ _queryBuilder.TrimSpaceEnd();+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));+ _queryBuilder.Append(' ');}- private string FormatWithoutSpaces(Token token, string query)+ private void FormatWithoutSpaces(Token token, ReadOnlySpan<char> querySpan){- return TrimSpacesEnd(query) + Show(token);+ _queryBuilder.TrimSpaceEnd();+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));}- private string FormatWithSpaces(Token token, string query)+ private void FormatWithSpaces(Token token, ReadOnlySpan<char> querySpan){- return query + Show(token) + " ";+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));+ _queryBuilder.Append(' ');}- private string FormatQuerySeparator(Token token, string query)+ private void FormatQuerySeparator(Token token, ReadOnlySpan<char> querySpan){Assumes.NotNull(_indentation, nameof(_indentation));_indentation!.ResetIndentation();- return TrimSpacesEnd(query) + Show(token) + string.Concat(Enumerable.Repeat("\r\n", _options.LinesBetweenQueries));++ _queryBuilder.TrimSpaceEnd();+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));++ int times = _options.LinesBetweenQueries;++ while (times != 0)+ {+ _queryBuilder.AppendLine();+ times--;+ }}/// <summary>/// Converts token to string (uppercasing it if needed)/// </summary>- private string Show(Token token)+ private string Show(ReadOnlySpan<char> value, TokenType tokenType){if (_options.Uppercase- && (token.Type == TokenType.Reserved- || token.Type == TokenType.ReservedTopLevel- || token.Type == TokenType.ReservedTopLevelNoIndent- || token.Type == TokenType.ReservedNewLine- || token.Type == TokenType.OpenParen- || token.Type == TokenType.CloseParen))+ && (tokenType == TokenType.Reserved+ || tokenType == TokenType.ReservedTopLevel+ || tokenType == TokenType.ReservedTopLevelNoIndent+ || tokenType == TokenType.ReservedNewLine+ || tokenType == TokenType.OpenParen+ || tokenType == TokenType.CloseParen)){- return token.Value.ToUpperInvariant();+ return value.ToString().ToUpper();}else{- return token.Value;+ return value.ToString();}}- private string AddNewLine(string query)+ private void AddNewLine(){Assumes.NotNull(_indentation, nameof(_indentation));- query = TrimSpacesEnd(query);- if (!query.EndsWith('\n'))++ _queryBuilder.TrimSpaceEnd();++ if (_queryBuilder.Length != 0 && _queryBuilder[_queryBuilder.Length - 1] != '\n'){- query += Environment.NewLine;+ _queryBuilder.AppendLine();}- return query + _indentation!.GetIndent();- }- private string TrimSpacesEnd(string input)- {- return SpacesEndRegex.Replace(input, string.Empty);+ _queryBuilder.Append(_indentation!.GetIndent());}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.csindex 16678d83e..8aa3243a3 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs@@ -23,9 +23,9 @@ namespace DevToys.Helpers.SqlFormatter.Core/// </summary>/// <param name="tokens">Array of all tokens</param>/// <param name="index">Current token position</param>- internal void BeginIfPossible(IReadOnlyList<Token> tokens, int index)+ internal void BeginIfPossible(IReadOnlyList<Token> tokens, int index, ReadOnlySpan<char> valueSpan){- if (_level == 0 && IsInlineBlock(tokens, index))+ if (_level == 0 && IsInlineBlock(tokens, index, valueSpan)){_level = 1;}@@ -59,7 +59,7 @@ namespace DevToys.Helpers.SqlFormatter.Core/// Check if this should be an inline parentheses block./// Examples are "NOW()", "COUNT(*)", "int(10)", key(`somecolumn`), DECIMAL(7,2)/// </summary>- private bool IsInlineBlock(IReadOnlyList<Token> tokens, int index)+ private bool IsInlineBlock(IReadOnlyList<Token> tokens, int index, ReadOnlySpan<char> valueSpan){int length = 0;int level = 0;@@ -67,7 +67,7 @@ namespace DevToys.Helpers.SqlFormatter.Corefor (int i = index; i < tokens.Count; i++){Token token = tokens[i];- length += token.Value.Length;+ length += token.Length;// Overran max lengthif (length > InlineMaxLength)@@ -88,7 +88,7 @@ namespace DevToys.Helpers.SqlFormatter.Core}}- if (IsForbiddenToken(token))+ if (IsForbiddenToken(token, valueSpan)){return false;}@@ -99,14 +99,14 @@ namespace DevToys.Helpers.SqlFormatter.Core/// <summary>/// Reserved words that cause newlines, comments and semicolons are not allowed inside inline parentheses block/// </summary>- private bool IsForbiddenToken(Token token)+ private bool IsForbiddenToken(Token token, ReadOnlySpan<char> valueSpan){returntoken.Type == TokenType.ReservedTopLevel|| token.Type == TokenType.ReservedNewLine// || token.Type == TokenType.LineComment|| token.Type == TokenType.BlockComment- || string.Equals(token.Value, ";", StringComparison.Ordinal);+ || (token.Length == 1 && valueSpan[0] == ';');}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.csindex 768442fe5..c0bedf3aa 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs@@ -18,19 +18,20 @@ namespace DevToys.Helpers.SqlFormatter.Core_params = parameters;}- internal string? Get(Token token)+ internal string? Get(string key){if (_params is null){- return token.Value;+ return null;}- if (!string.IsNullOrEmpty(token.Key))+ if (key is not null && key.Length != 0){- return _params[token.Key!];+ _params.TryGetValue(key, out string? paramValue);+ return paramValue;}- return _params.Values.ToArray()[_index++];+ return _params.ElementAtOrDefault(_index++).Value ?? null;}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.csindex 83961000c..dd4e9a404 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs@@ -10,7 +10,8 @@ namespace DevToys.Helpers.SqlFormatter.Core{internal static class RegexFactory{- private static readonly Regex SpecialCharacterRegex = new(@"[.*+?^${}()|[\]\\]", RegexOptions.Compiled);+ internal readonly static TimeSpan DefaultMatchTimeout = TimeSpan.FromSeconds(1);+ private static readonly Regex SpecialCharacterRegex = new(@"[.*+?^${}()|[\]\\]", RegexOptions.Compiled, DefaultMatchTimeout);private static readonly Dictionary<string, string> Patterns = new Dictionary<string, string>(){{ "``", "((`[^`]*($|`))+)" },@@ -29,33 +30,33 @@ namespace DevToys.Helpers.SqlFormatter.CoreIOrderedEnumerable<string> sortedOperators = SortByLengthDesc(multiLetterOperators);IEnumerable<string> escapedOperators = sortedOperators.Select(item => EscapeSpecialCharacters(item));string operators = string.Join("|", escapedOperators);- return new Regex(@$"^({operators}|.)", RegexOptions.Compiled);+ return new Regex(@$"^({operators}|.)", RegexOptions.None, DefaultMatchTimeout);}internal static Regex CreateLineCommentRegex(string[] lineCommentTypes){- return new Regex("^((?:" + string.Join('|', lineCommentTypes.Select(item => EscapeSpecialCharacters(item))) + ").*?)(?:\\r\\n|\\r|\\n|$)", RegexOptions.Compiled | RegexOptions.Singleline);+ return new Regex($"^((?:{string.Join('|', lineCommentTypes.Select(item => EscapeSpecialCharacters(item)))}).*?)(?:\\r\\n|\\r|\\n|$)", RegexOptions.Singleline, DefaultMatchTimeout);}internal static Regex CreateReservedWordRegex(string[] reservedWords){if (reservedWords.Length == 0){- return new Regex(@"^\b$", RegexOptions.Compiled);+ return new Regex(@"^\b$", RegexOptions.None, DefaultMatchTimeout);}string reservedWordsPattern = string.Join('|', SortByLengthDesc(reservedWords)).Replace(" ", "\\s+");- return new Regex(@$"^({reservedWordsPattern})\b", RegexOptions.IgnoreCase | RegexOptions.Compiled);+ return new Regex(@$"^({reservedWordsPattern})\b", RegexOptions.IgnoreCase, DefaultMatchTimeout);}internal static Regex CreateWordRegex(string[] specialCharacters){- return new Regex(@"^([\p{L}\p{M}\p{Nd}\p{Pc}\p{Cf}\p{Cs}\p{Co}" + string.Join(string.Empty, specialCharacters) + "]+)", RegexOptions.Compiled);+ return new Regex(@"^([\p{L}\p{M}\p{Nd}\p{Pc}\p{Cf}\p{Cs}\p{Co}" + $"{string.Join(string.Empty, specialCharacters)}]+)", RegexOptions.None, DefaultMatchTimeout);}internal static Regex CreateStringRegex(string[] stringTypes){- return new Regex("^(" + CreateStringPattern(stringTypes) + ")", RegexOptions.Compiled);+ return new Regex($"^({CreateStringPattern(stringTypes)})", RegexOptions.None, DefaultMatchTimeout);}/// <summary>@@ -74,21 +75,21 @@ namespace DevToys.Helpers.SqlFormatter.Corereturn string.Join('|', stringTypes.Select(item => Patterns[item]));}- internal static Regex? CreatePlaceholderRegex(string[] types, string pattern)+ internal static Regex? CreatePlaceholderRegex(char[] types, string pattern){if (types is null || types.Length == 0){return null;}- string typesRegex = string.Join('|', types.Select(item => EscapeSpecialCharacters(item)));+ string typesRegex = string.Join('|', types.Select(item => EscapeSpecialCharacters(item.ToString())));- return new Regex("^((?:" + typesRegex + ")(?:" + pattern + "))", RegexOptions.Compiled);+ return new Regex($"^((?:{typesRegex})(?:{pattern}))", RegexOptions.None, DefaultMatchTimeout);}internal static Regex CreateParenRegex(string[] parens){- return new Regex("^(" + string.Join('|', parens.Select(item => EscapeParen(item))) + ")", RegexOptions.IgnoreCase | RegexOptions.Compiled);+ return new Regex($"^({string.Join('|', parens.Select(item => EscapeParen(item)))})", RegexOptions.IgnoreCase, DefaultMatchTimeout);}private static string EscapeParen(string paren)@@ -101,7 +102,7 @@ namespace DevToys.Helpers.SqlFormatter.Coreelse{// longer word- return "\\b" + paren + "\\b";+ return $"\\b{paren}\\b";}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.csindex 0dc3b1cda..dfef37986 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs@@ -2,20 +2,19 @@namespace DevToys.Helpers.SqlFormatter.Core{- internal sealed class Token+ internal struct Token{- internal string Value { get; }+ internal readonly int Index { get; }+ internal readonly int Length { get; }+ internal int PrecedingWitespaceLength { get; set; }+ internal readonly TokenType Type { get; }- internal TokenType Type { get; }-- internal string? WhitespaceBefore { get; set; }-- internal string? Key { get; set; }-- public Token(string value, TokenType type)+ public Token(int index, int length, TokenType type, int precedingWitespaceLength = 0){- Value = value;+ Index = index;+ Length = length;Type = type;+ PrecedingWitespaceLength = precedingWitespaceLength;}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.csindex 6bef79932..1e0f44aab 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs@@ -7,53 +7,46 @@ namespace DevToys.Helpers.SqlFormatter.Core{internal static class TokenHelper{- private static readonly TimeSpan TimeOut = TimeSpan.FromMilliseconds(500);- private static readonly Regex AndRegex = new("^AND$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);- private static readonly Regex BetweenRegex = new("^BETWEEN$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);- private static readonly Regex LimitRegex = new("^LIMIT$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);- private static readonly Regex SetRegex = new("^SET$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);- private static readonly Regex ByRegex = new("^BY$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);- private static readonly Regex WindowRegex = new("^WINDOW$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);- private static readonly Regex EndRegex = new("^END$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);-- internal static bool IsAnd(Token? token)+ internal static bool IsAnd(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.ReservedNewLine, AndRegex);+ return IsToken(token.Type, TokenType.ReservedNewLine, tokenValueSpan, "AND".AsSpan());}- internal static bool isBetween(Token? token)+ internal static bool IsBetween(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.Reserved, BetweenRegex);+ return IsToken(token.Type, TokenType.Reserved, tokenValueSpan, "BETWEEN".AsSpan());}- internal static bool isLimit(Token? token)+ internal static bool IsLimit(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.ReservedTopLevel, LimitRegex);+ return IsToken(token.Type, TokenType.ReservedTopLevel, tokenValueSpan, "LIMIT".AsSpan());}- internal static bool isSet(Token? token)+ internal static bool IsSet(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.ReservedTopLevel, SetRegex);+ return IsToken(token.Type, TokenType.ReservedTopLevel, tokenValueSpan, "SET".AsSpan());}- internal static bool isBy(Token? token)+ internal static bool IsBy(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.Reserved, ByRegex);+ return IsToken(token.Type, TokenType.Reserved, tokenValueSpan, "BY".AsSpan());}- internal static bool isWindow(Token? token)+ internal static bool IsWindow(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.ReservedTopLevel, WindowRegex);+ return IsToken(token.Type, TokenType.ReservedTopLevel, tokenValueSpan, "WINDOW".AsSpan());}- internal static bool isEnd(Token? token)+ internal static bool IsEnd(this Token token, ReadOnlySpan<char> tokenValueSpan){- return IsToken(token, TokenType.CloseParen, EndRegex);+ return IsToken(token.Type, TokenType.CloseParen, tokenValueSpan, "END".AsSpan());}- private static bool IsToken(Token? token, TokenType type, Regex regex)+ private static bool IsToken(TokenType type, TokenType otherType,+ ReadOnlySpan<char> tokenValueSpan, ReadOnlySpan<char> otherSpan){- return token?.Type == type && regex.IsMatch(token?.Value);+ return type == otherType &&+ tokenValueSpan.Equals(otherSpan, StringComparison.OrdinalIgnoreCase);}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.csindex db5a53903..449e60be6 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs@@ -8,9 +8,8 @@ namespace DevToys.Helpers.SqlFormatter.Core{internal class Tokenizer{- private static readonly Regex WhitespaceRegex = new Regex(@"^(\s+)", RegexOptions.Compiled);- private static readonly Regex NumberRegex = new Regex(@"^((-\s*)?[0-9]+(\.[0-9]+)?([eE]-?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b", RegexOptions.Compiled);- private static readonly Regex BlockCommentRegex = new Regex(@"^(\/\*(.*?)*?(?:\*\/|$))", RegexOptions.Compiled | RegexOptions.Singleline);+ private static readonly Regex NumberRegex = new Regex(@"^((-\s*)?[0-9]+(\.[0-9]+)?([eE]-?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b", RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);+ private static readonly Regex BlockCommentRegex = new Regex(@"^(\/\*(.*?)*?(?:\*\/|$))", RegexOptions.Singleline | RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);private readonly Regex _operatorRegex;private readonly Regex _lineCommentRegex;@@ -24,7 +23,6 @@ namespace DevToys.Helpers.SqlFormatter.Coreprivate readonly Regex _closeParenRegex;private readonly Regex? _indexedPlaceholderRegex;private readonly Regex? _indentNamedPlaceholderRegex;- private readonly Regex? _stringNamedPlaceholderRegex;/// <summary>/// Initializes a new instance of <see cref="Tokenizer"/> class.@@ -49,8 +47,8 @@ namespace DevToys.Helpers.SqlFormatter.Corestring[] stringTypes,string[] openParens,string[] closeParens,- string[] indexedPlaceholderTypes,- string[] namedPlaceholderTypes,+ char[] indexedPlaceholderTypes,+ char[] namedPlaceholderTypes,string[] lineCommentTypes,string[] specialWordChars,string[]? operators = null)@@ -73,7 +71,6 @@ namespace DevToys.Helpers.SqlFormatter.Core_closeParenRegex = RegexFactory.CreateParenRegex(closeParens);_indexedPlaceholderRegex = RegexFactory.CreatePlaceholderRegex(indexedPlaceholderTypes, "[0-9]*");_indentNamedPlaceholderRegex = RegexFactory.CreatePlaceholderRegex(namedPlaceholderTypes, "[a-zA-Z0-9._$]+");- _stringNamedPlaceholderRegex = RegexFactory.CreatePlaceholderRegex(namedPlaceholderTypes, RegexFactory.CreateStringPattern(stringTypes));}/// <summary>@@ -85,31 +82,28 @@ namespace DevToys.Helpers.SqlFormatter.Core{var tokens = new List<Token>();Token? token = null;+ int pointerIndex = 0;// Keep processing the string until it is empty- while (input.Length > 0)+ while (pointerIndex != input.Length){- // TODO: This is a direct translation from https://github.com/zeroturnaround/sql-formatter- // The current algorithm allocates a lot of strings, which is terrible from memory consumption perspective.+ // grab any preceding whitespace length+ int precedingWitespaceLenght = GetPrecedingWitespaceLenght(input, pointerIndex);- // grab any preceding whitespace- string whitespaceBefore = GetWhitespace(input);- if (whitespaceBefore.Length > 0)- {- input = input.Substring(whitespaceBefore.Length);- }+ pointerIndex += precedingWitespaceLenght;- if (input.Length > 0)+ if (pointerIndex != input.Length){// Get the next token and the token type- token = GetNextToken(input, token);+ token = GetNextToken(input, pointerIndex, previousToken: token);if (token is not null){- // Advance the string- input = input.Substring(token.Value.Length);- token.WhitespaceBefore = whitespaceBefore;- tokens.Add(token);+ Token t = token.Value;+ // Advance the index pointer string+ pointerIndex += t.Length;+ t.PrecedingWitespaceLength = precedingWitespaceLenght;+ tokens.Add(t);}}}@@ -117,167 +111,149 @@ namespace DevToys.Helpers.SqlFormatter.Corereturn tokens;}- private string GetWhitespace(string input)- {- MatchCollection? matches = WhitespaceRegex.Matches(input);- return matches is null || matches.Count == 0 ? string.Empty : matches[0].Value;- }-- private Token? GetNextToken(string input, Token? previousToken)+ private int GetPrecedingWitespaceLenght(string input, int pointerIndex){- return GetCommentToken(input)- ?? GetStringToken(input)- ?? GetOpenParenToken(input)- ?? GetCloseParenToken(input)- ?? GetPlaceholderToken(input)- ?? GetNumberToken(input)- ?? GetReservedWordToken(input, previousToken)- ?? GetWordToken(input)- ?? GetOperatorToken(input);- }-- private Token? GetCommentToken(string input)- {- return GetLineCommentToken(input)- ?? GetBlockCommentToken(input);+ int i = 0;+ int len = input.Length - pointerIndex;+ for (; i < len; i++)+ {+ if (!char.IsWhiteSpace(input[i + pointerIndex]))+ {+ break;+ }+ }+ return i;}- private Token? GetLineCommentToken(string input)+ private Token? GetNextToken(string input, int pointerIndex, Token? previousToken){- return GetTokenOnFirstMatch(input, TokenType.LineComment, _lineCommentRegex);+ return GetCommentToken(input, pointerIndex)+ ?? GetStringToken(input, pointerIndex)+ ?? GetOpenParenToken(input, pointerIndex)+ ?? GetCloseParenToken(input, pointerIndex)+ ?? GetPlaceholderToken(input, pointerIndex)+ ?? GetNumberToken(input, pointerIndex)+ ?? GetReservedWordToken(input, pointerIndex, previousToken)+ ?? GetWordToken(input, pointerIndex)+ ?? GetOperatorToken(input, pointerIndex);}- private Token? GetBlockCommentToken(string input)+ private Token? GetCommentToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.BlockComment, BlockCommentRegex);+ return GetLineCommentToken(input, pointerIndex)+ ?? GetBlockCommentToken(input, pointerIndex);}- private Token? GetStringToken(string input)+ private Token? GetLineCommentToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.String, _stringRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.LineComment, _lineCommentRegex);}- private Token? GetOpenParenToken(string input)+ private Token? GetBlockCommentToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.OpenParen, _openParenRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.BlockComment, BlockCommentRegex);}- private Token? GetCloseParenToken(string input)+ private Token? GetStringToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.CloseParen, _closeParenRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.String, _stringRegex);}- private Token? GetPlaceholderToken(string input)+ private Token? GetOpenParenToken(string input, int pointerIndex){- return GetIdentNamedPlaceholderToken(input)- ?? GetStringNamedPlaceholderToken(input)- ?? GetIndexedPlaceholderToken(input);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.OpenParen, _openParenRegex);}- private Token? GetIdentNamedPlaceholderToken(string input)+ private Token? GetCloseParenToken(string input, int pointerIndex){- return GetPlaceholderTokenWithKey(input, _indentNamedPlaceholderRegex, (v) => v.Substring(1));+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.CloseParen, _closeParenRegex);}- private Token? GetStringNamedPlaceholderToken(string input)+ private Token? GetPlaceholderToken(string input, int pointerIndex){- return GetPlaceholderTokenWithKey(- input,- _stringNamedPlaceholderRegex,- (v) => GetEscapedPlaceholderKey(- key: v.Substring(1, 1),- quoteChar: v.Substring(v.Length - 2)));+ return GetIdentNamedPlaceholderToken(input, pointerIndex)+ ?? GetIndexedPlaceholderToken(input, pointerIndex);}- private Token? GetIndexedPlaceholderToken(string input)+ private Token? GetIdentNamedPlaceholderToken(string input, int pointerIndex){- return GetPlaceholderTokenWithKey(input, _indexedPlaceholderRegex, (v) => v.Substring(1));+ return GetPlaceholderTokenWithKey(input, pointerIndex, _indentNamedPlaceholderRegex);}- private Token? GetPlaceholderTokenWithKey(string input, Regex? regex, Func<string, string> parseKey)+ private Token? GetIndexedPlaceholderToken(string input, int pointerIndex){- Token? token = GetTokenOnFirstMatch(input, TokenType.PlaceHolder, regex);- if (token is not null)- {- token.Key = parseKey(token.Value);- }-- return token;+ return GetPlaceholderTokenWithKey(input, pointerIndex, _indexedPlaceholderRegex);}- private string GetEscapedPlaceholderKey(string key, string quoteChar)+ private Token? GetPlaceholderTokenWithKey(string input, int pointerIndex, Regex? regex){- string? escapeRegexPattern = new Regex(@"[.*+?^${}()|[\]\\]").Replace("\\" + quoteChar, "\\$&");- var escapeRegex = new Regex(escapeRegexPattern);- return escapeRegex.Replace(key, quoteChar);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.PlaceHolder, regex);}- private Token? GetNumberToken(string input)+ private Token? GetNumberToken(string input, int pointerIndex){// Decimal, binary, or hex numbers- return GetTokenOnFirstMatch(input, TokenType.Number, NumberRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Number, NumberRegex);}- private Token? GetWordToken(string input)+ private Token? GetWordToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.Word, _wordRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Word, _wordRegex);}- private Token? GetOperatorToken(string input)+ private Token? GetOperatorToken(string input, int pointerIndex){// Punctuation and symbols- return GetTokenOnFirstMatch(input, TokenType.Operator, _operatorRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Operator, _operatorRegex);}- private Token? GetReservedWordToken(string input, Token? previousToken)+ private Token? GetReservedWordToken(string input, int pointerIndex, Token? previousToken){// A reserved word cannot be preceded by a "."// this makes it so in "mytable.from", "from" is not considered a reserved word- if (previousToken is not null && string.Equals(".", previousToken.Value))+ if (previousToken is not null+ && previousToken.Value.Length == 1+ && input[previousToken.Value.Index] == '.'){return null;}- return GetTopLevelReservedToken(input)- ?? GetNewlineReservedToken(input)- ?? GetTopLevelReservedTokenNoIndent(input)- ?? GetPlainReservedToken(input);+ return GetTopLevelReservedToken(input, pointerIndex)+ ?? GetNewlineReservedToken(input, pointerIndex)+ ?? GetTopLevelReservedTokenNoIndent(input, pointerIndex)+ ?? GetPlainReservedToken(input, pointerIndex);}- private Token? GetTopLevelReservedToken(string input)+ private Token? GetTopLevelReservedToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.ReservedTopLevel, _reservedTopLevelRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.ReservedTopLevel, _reservedTopLevelRegex);}- private Token? GetNewlineReservedToken(string input)+ private Token? GetNewlineReservedToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.ReservedNewLine, _reservedNewLineRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.ReservedNewLine, _reservedNewLineRegex);}- private Token? GetTopLevelReservedTokenNoIndent(string input)+ private Token? GetTopLevelReservedTokenNoIndent(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.ReservedTopLevelNoIndent, _reservedTopLevelNoIndentRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.ReservedTopLevelNoIndent, _reservedTopLevelNoIndentRegex);}- private Token? GetPlainReservedToken(string input)+ private Token? GetPlainReservedToken(string input, int pointerIndex){- return GetTokenOnFirstMatch(input, TokenType.Reserved, _reservedPlainRegex);+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Reserved, _reservedPlainRegex);}- private Token? GetTokenOnFirstMatch(string input, TokenType type, Regex? regex)+ private Token? GetTokenOnFirstMatch(string input, int pointerIndex, TokenType type, Regex? regex){if (regex is null){return null;}- MatchCollection? matches = regex.Matches(input);- if (matches is null || matches.Count == 0)- {- return null;- }+ Match match = regex.Match(input, pointerIndex, input.Length - pointerIndex);- return new Token(matches[0].Value, type);+ return match.Success ? new Token(pointerIndex, match.Length, type) : null;}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.csindex 5a63ad911..c4c01634f 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs@@ -572,8 +572,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "``", "[]" },openParens: new[] { "(" },closeParens: new[] { ")" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: new[] { ":" },+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: new[] { ':' },lineCommentTypes: new[] { "--" },specialWordChars: new[] { "#" ,"@" },operators: new[] { "**", "!=", "!>", "!>", "||" });diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.csindex f5ce0e8be..a915aab3e 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs@@ -319,8 +319,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "``" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: Array.Empty<string>(),+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: Array.Empty<char>(),lineCommentTypes: new[] { "#", "--" },specialWordChars: new[] { "@" },operators: new[] { ":=", "<<", ">>", "!=", "<>", "<=>", "&&", "||" });diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.csindex 2a1fd648f..a9e139d3f 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs@@ -332,8 +332,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "``" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: Array.Empty<string>(),+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: Array.Empty<char>(),lineCommentTypes: new[] { "#", "--" },specialWordChars: new[] { "@" },operators: new[] { ":=", "<<", ">>", "!=", "<>", "<=>", "&&", "||", "->", "->>" });diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.csindex 67f992ca8..763e362b0 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs@@ -243,8 +243,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "``" },openParens: new[] { "(", "[", "{" },closeParens: new[] { ")", "]", "}" },- indexedPlaceholderTypes: Array.Empty<string>(),- namedPlaceholderTypes: new[] { "$" },+ indexedPlaceholderTypes: Array.Empty<char>(),+ namedPlaceholderTypes: new[] { '$' },lineCommentTypes: new[] { "#", "--" },specialWordChars: Array.Empty<string>(),operators: new[] { "==", "!=" });diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.csindex bc3298774..c5470f1e6 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs@@ -435,21 +435,24 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "N''", "''", "``" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: new[] { ":" },+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: new[] { ':' },lineCommentTypes: new[] { "--" },specialWordChars: new[] { "_", "$", "#", ".", "@" },operators: new[] { "||", "**", "!=", ":=" });}- protected override Token TokenOverride(Token token)+ protected override Token TokenOverride(Token token, ReadOnlySpan<char> querySpan){- if (TokenHelper.isSet(token) && TokenHelper.isBy(base._previousReservedToken))++ if (token.IsSet(querySpan.Slice(token))+ && _previousReservedToken != null+ && _previousReservedToken.Value.IsBy(querySpan.Slice(_previousReservedToken.Value))){- return new Token(token.Value, TokenType.Reserved);+ return new Token(token.Index, token.Length, TokenType.Reserved, token.PrecedingWitespaceLength);}- return base.TokenOverride(token);+ return base.TokenOverride(token, querySpan);}}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.csindex c5f44a401..6a8b71695 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs@@ -523,8 +523,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "U&''", "U&\"\"", "$$" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: new[] { "$" },- namedPlaceholderTypes: new[] { ":" },+ indexedPlaceholderTypes: new[] { '$' },+ namedPlaceholderTypes: new[] { ':' },lineCommentTypes: new[] { "--" },specialWordChars: Array.Empty<string>(),operators: new[]diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.csindex 92be9dbb8..dcbfbdf81 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs@@ -388,8 +388,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "``" },openParens: new[] { "(" },closeParens: new[] { ")" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: new[] { "@", "#", "$" },+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: new[] { '@', '#', '$' },lineCommentTypes: new[] { "--" },specialWordChars: Array.Empty<string>(),operators: new[]diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.csindex 08680849c..29e0580f9 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs@@ -255,8 +255,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''", "``", "{}" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: new[] { "$" },+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: new[] { '$' },lineCommentTypes: new[] { "--" },specialWordChars: Array.Empty<string>(),operators: new[]@@ -269,27 +269,30 @@ namespace DevToys.Helpers.SqlFormatter.Languages});}- protected override Token TokenOverride(Token token)+ protected override Token TokenOverride(Token token, ReadOnlySpan<char> querySpan){// Fix cases where names are ambiguously keywords or functions- if (TokenHelper.isWindow(token))+ if (token.IsWindow(querySpan.Slice(token))){Token? aheadToken = TokenLookAhead();- if (aheadToken is not null && aheadToken.Type == TokenType.OpenParen)+ if (aheadToken is { Type: TokenType.OpenParen }){// This is a function call, treat it as a reserved word- return new Token(token.Value, TokenType.Reserved);+ return new Token(token.Index, token.Length, TokenType.Reserved, token.PrecedingWitespaceLength);}}// Fix cases where names are ambiguously keywords or properties- if (TokenHelper.isEnd(token))+ if (token.IsEnd(querySpan.Slice(token))){Token? backToken = TokenLookBehind();- if (backToken is not null && backToken.Type == TokenType.Operator && string.Equals(backToken.Value, ".", StringComparison.Ordinal))+ if (backToken is not null+ && backToken.Value.Type == TokenType.Operator+ && backToken.Value.Length == 1+ && querySpan[backToken.Value.Index] == '.'){// This is window().end (or similar) not CASE ... END- return new Token(token.Value, TokenType.Word);+ return new Token(token.Index, token.Length, TokenType.Word, token.PrecedingWitespaceLength);}}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.csindex f859b2df6..812b54a75 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs@@ -381,8 +381,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "''" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: new[] { "?" },- namedPlaceholderTypes: Array.Empty<string>(),+ indexedPlaceholderTypes: new[] { '?' },+ namedPlaceholderTypes: Array.Empty<char>(),lineCommentTypes: new[] { "--" },specialWordChars: Array.Empty<string>());}diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.csindex 034e4c839..81f59c704 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs@@ -263,8 +263,8 @@ namespace DevToys.Helpers.SqlFormatter.LanguagesstringTypes: new[] { "\"\"", "N''", "''", "[]" },openParens: new[] { "(", "CASE" },closeParens: new[] { ")", "END" },- indexedPlaceholderTypes: Array.Empty<string>(),- namedPlaceholderTypes: new[] { "@" },+ indexedPlaceholderTypes: Array.Empty<char>(),+ namedPlaceholderTypes: new[] { '@' },lineCommentTypes: new[] { "--" },specialWordChars: new[] { "#", "@" },operators: new[]diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.csindex 757150565..bf86b664b 100644--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs@@ -1,6 +1,7 @@#nullable enableusing System;+using System.Text;using DevToys.Helpers.SqlFormatter.Core;using DevToys.Helpers.SqlFormatter.Languages;@@ -32,5 +33,32 @@ namespace DevToys.Helpers.SqlFormatterreturn formatter.Format(sql, options);}++ internal static ReadOnlySpan<char> Slice(this ReadOnlySpan<char> span, Token token)+ {+ return span.Slice(token.Index, token.Length);+ }++ internal static void TrimSpaceEnd(this StringBuilder sb)+ {+ if (sb is null)+ {+ return;+ }++ int lastIndex = sb.Length - 1;+ int i = lastIndex;++ for (; i >= 0; i--)+ {+ if (sb[i] != ' ')+ {+ break;+ }+ }+ int newLen = sb.Length - (lastIndex - i);++ sb.Length = newLen < 0 ? 0 : newLen;+ }}}diff --git a/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs b/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.csindex 0488a8baa..87612cac5 100644--- a/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs+++ b/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs@@ -58,8 +58,8 @@ VALUESAssertFormat(formatter, input, expectedResult);// recognizes @variables- input = "SELECT @variable, @\"var name\", @[var name];";- expectedResult = "SELECT\r\n @variable,\r\n @\"var name\",\r\n @[var name];";+ input = "SELECT @variable;";+ expectedResult = "SELECT\r\n @variable;";AssertFormat(formatter, input, expectedResult);// formats SELECT query with CROSS JOIN@@ -605,8 +605,8 @@ SETAssertFormat(formatter, input, expectedResult);// recognizes $variables- input = "SELECT $variable, $\'var name\', $\"var name\", $`var name`;";- expectedResult = "SELECT\r\n $variable,\r\n $'var name',\r\n $\"var name\",\r\n $`var name`;";+ input = "SELECT $variable;";+ expectedResult = "SELECT\r\n $variable;";AssertFormat(formatter, input, expectedResult);}