Skip to content

A Swiss Army knife for developers.

Sql-formatter optimization (#368)

A. Karboush725fa91c6e55
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs
index 5dde9b9b8..cdba05b50 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Formatter.cs
@@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
+using System.Text;
using System.Text.RegularExpressions;
using DevToys.Shared.Core;
@@ -10,11 +11,11 @@ namespace DevToys.Helpers.SqlFormatter.Core
{
internal abstract class Formatter
{
- private static readonly Regex SpacesEndRegex = new Regex(@"[ \t]+$", RegexOptions.Compiled);
- private static readonly Regex WhitespacesRegex = new Regex(@"\s+$", RegexOptions.Compiled);
- private static readonly Regex CommentWhitespacesRegex = new Regex(@"\n[ \t]*", RegexOptions.Compiled);
+ private static readonly Regex WhitespacesRegex = new Regex(@"\s+$", RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);
+ private static readonly Regex CommentWhitespacesRegex = new Regex(@"\n[ \t]*", RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);
private readonly InlineBlock _inlineBlock = new();
+ private readonly StringBuilder _queryBuilder = new();
private Indentation? _indentation = null;
private SqlFormatterOptions _options;
@@ -41,8 +42,8 @@ namespace DevToys.Helpers.SqlFormatter.Core
_params = new Params(options.PlaceholderParameters);
_tokens = GetTokenizer().Tokenize(query);
- string? formattedQuery = GetFormattedQueryFromTokens();
- return formattedQuery.Trim();
+ SetFormattedQueryFromTokens(query.AsSpan());
+ return _queryBuilder.ToString().Trim();
}
/// <summary>
@@ -53,7 +54,7 @@ namespace DevToys.Helpers.SqlFormatter.Core
/// <summary>
/// Reprocess and modify a token based on parsed context.
/// </summary>
- protected virtual Token TokenOverride(Token token)
+ protected virtual Token TokenOverride(Token token, ReadOnlySpan<char> querySpan)
{
// subclasses can override this to modify tokens during formatting
return token;
@@ -61,12 +62,7 @@ namespace DevToys.Helpers.SqlFormatter.Core
protected Token? TokenLookBehind(int n = 1)
{
- if (_tokens is null || _tokens!.Count <= _index - n || _index - n < 0)
- {
- return null;
- }
-
- return _tokens![_index - n];
+ return _tokens?.ElementAtOrDefault(_index - n);
}
protected Token? TokenLookAhead(int n = 1)
@@ -78,127 +74,135 @@ namespace DevToys.Helpers.SqlFormatter.Core
return _tokens![_index + n];
}
- private string GetFormattedQueryFromTokens()
+ private void SetFormattedQueryFromTokens(ReadOnlySpan<char> querySpan)
{
- string formattedQuery = string.Empty;
+ _queryBuilder.Clear();
Assumes.NotNull(_tokens, nameof(_tokens));
for (int i = 0; i < _tokens!.Count; i++)
{
_index = i;
- Token token = TokenOverride(_tokens[i]);
- if (token.Type == TokenType.LineComment)
- {
- formattedQuery = FormatLineComment(token, formattedQuery);
- }
- else if (token.Type == TokenType.BlockComment)
- {
- formattedQuery = FormatBlockComment(token, formattedQuery);
- }
- else if (token.Type == TokenType.ReservedTopLevel)
- {
- formattedQuery = FormatTopLevelReservedWord(token, formattedQuery);
- _previousReservedToken = token;
- }
- else if (token.Type == TokenType.ReservedTopLevelNoIndent)
- {
- formattedQuery = FormatTopLevelReservedWordNoIndent(token, formattedQuery);
- _previousReservedToken = token;
- }
- else if (token.Type == TokenType.ReservedNewLine)
- {
- formattedQuery = FormatNewlineReservedWord(token, formattedQuery);
- _previousReservedToken = token;
- }
- else if (token.Type == TokenType.Reserved)
- {
- formattedQuery = FormatWithSpaces(token, formattedQuery);
- _previousReservedToken = token;
- }
- else if (token.Type == TokenType.OpenParen)
- {
- formattedQuery = FormatOpeningParentheses(token, formattedQuery);
- }
- else if (token.Type == TokenType.CloseParen)
- {
- formattedQuery = FormatClosingParentheses(token, formattedQuery);
- }
- else if (token.Type == TokenType.PlaceHolder)
- {
- formattedQuery = FormatPlaceholder(token, formattedQuery);
- }
- else if (string.Equals(token.Value, ",", System.StringComparison.Ordinal))
- {
- formattedQuery = FormatComma(token, formattedQuery);
- }
- else if (string.Equals(token.Value, ":", System.StringComparison.Ordinal))
- {
- formattedQuery = FormatWithSpaceAfter(token, formattedQuery);
- }
- else if (string.Equals(token.Value, ".", System.StringComparison.Ordinal))
- {
- formattedQuery = FormatWithoutSpaces(token, formattedQuery);
- }
- else if (string.Equals(token.Value, ";", System.StringComparison.Ordinal))
- {
- formattedQuery = FormatQuerySeparator(token, formattedQuery);
- }
- else
+ Token token = TokenOverride(_tokens[i], querySpan);
+ switch (token.Type)
{
- formattedQuery = FormatWithSpaces(token, formattedQuery);
+ case TokenType.LineComment:
+ FormatLineComment(token, querySpan);
+ break;
+ case TokenType.BlockComment:
+ FormatBlockComment(token, querySpan);
+ break;
+ case TokenType.ReservedTopLevel:
+ FormatTopLevelReservedWord(token, querySpan);
+ _previousReservedToken = token;
+ break;
+ case TokenType.ReservedTopLevelNoIndent:
+ FormatTopLvoidReservedWordNoIndent(token, querySpan);
+ _previousReservedToken = token;
+ break;
+ case TokenType.ReservedNewLine:
+ FormatNewlineReservedWord(token, querySpan);
+ _previousReservedToken = token;
+ break;
+ case TokenType.Reserved:
+ FormatWithSpaces(token, querySpan);
+ _previousReservedToken = token;
+ break;
+ case TokenType.OpenParen:
+ FormatOpeningParentheses(token, querySpan);
+ break;
+ case TokenType.CloseParen:
+ FormatClosingParentheses(token, querySpan);
+ break;
+ case TokenType.PlaceHolder:
+ FormatPlaceholder(token, querySpan);
+ break;
+ default:
+ switch (token.Length)
+ {
+ case 1 when querySpan[token.Index] == ',':
+ FormatComma(token, querySpan);
+ break;
+ case 1 when querySpan[token.Index] == ':':
+ FormatWithSpaceAfter(token, querySpan);
+ break;
+ case 1 when querySpan[token.Index] == '.':
+ FormatWithoutSpaces(token, querySpan);
+ break;
+ case 1 when querySpan[token.Index] == ';':
+ FormatQuerySeparator(token, querySpan);
+ break;
+ default:
+ FormatWithSpaces(token, querySpan);
+ break;
+ }
+ break;
}
}
-
- return formattedQuery;
}
- private string FormatLineComment(Token token, string query)
+ private void FormatLineComment(Token token, ReadOnlySpan<char> querySpan)
{
- return AddNewLine(query + Show(token));
+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));
+ AddNewLine();
}
- private string FormatBlockComment(Token token, string query)
+ private void FormatBlockComment(Token token, ReadOnlySpan<char> querySpan)
{
- return AddNewLine(AddNewLine(query) + IndentComment(token.Value));
+ AddNewLine();
+ _queryBuilder.Append(IndentComment(querySpan.Slice(token)));
+ AddNewLine();
}
- private string IndentComment(string comment)
+ private string IndentComment(ReadOnlySpan<char> comment)
{
Assumes.NotNull(_indentation, nameof(_indentation));
- return CommentWhitespacesRegex.Replace(comment, "\n" + _indentation!.GetIndent() + " ");
+ return CommentWhitespacesRegex.Replace(comment.ToString(), $"\n{_indentation!.GetIndent()} ");
}
- private string FormatTopLevelReservedWordNoIndent(Token token, string query)
+ private void FormatTopLvoidReservedWordNoIndent(Token token, ReadOnlySpan<char> querySpan)
{
Assumes.NotNull(_indentation, nameof(_indentation));
_indentation!.DecreaseTopLevel();
- query = AddNewLine(query) + EqualizeWhitespace(Show(token));
- return AddNewLine(query);
+ AddNewLine();
+
+ _queryBuilder.Append(EqualizeWhitespace(Show(querySpan.Slice(token), token.Type)));
+
+ AddNewLine();
}
- private string FormatTopLevelReservedWord(Token token, string query)
+ private void FormatTopLevelReservedWord(Token token, ReadOnlySpan<char> querySpan)
{
Assumes.NotNull(_indentation, nameof(_indentation));
_indentation!.DecreaseTopLevel();
- query = AddNewLine(query);
+ AddNewLine();
_indentation.IncreaseTopLevel();
- query += EqualizeWhitespace(Show(token));
- return AddNewLine(query);
+ _queryBuilder.Append(EqualizeWhitespace(Show(querySpan.Slice(token), token.Type)));
+
+ AddNewLine();
}
- private string FormatNewlineReservedWord(Token token, string query)
+ private void FormatNewlineReservedWord(Token token, ReadOnlySpan<char> querySpan)
{
- if (TokenHelper.IsAnd(token) && TokenHelper.isBetween(TokenLookBehind(2)))
+ if (token.IsAnd(querySpan.Slice(token)))
{
- return FormatWithSpaces(token, query);
+ Token? t = TokenLookBehind(2);
+
+ if (t != null && t.Value.IsBetween(querySpan.Slice(t.Value)))
+ {
+ FormatWithSpaces(token, querySpan);
+ return;
+ }
}
+ AddNewLine();
+
+ _queryBuilder.Append(EqualizeWhitespace(Show(querySpan.Slice(token), token.Type)));
- return AddNewLine(query) + EqualizeWhitespace(Show(token)) + " ";
+ _queryBuilder.Append(' ');
}
/// <summary>
@@ -212,136 +216,158 @@ namespace DevToys.Helpers.SqlFormatter.Core
/// <summary>
/// Opening parentheses increase the block indent level and start a new line
/// </summary>
- private string FormatOpeningParentheses(Token token, string query)
+ private void FormatOpeningParentheses(Token token, ReadOnlySpan<char> querySpan)
{
// Take out the preceding space unless there was whitespace there in the original query
// or another opening parens or line comment
- if (token.WhitespaceBefore?.Length == 0)
+ if (token.PrecedingWitespaceLength == 0)
{
- TokenType? type = TokenLookBehind()?.Type;
- if (type.HasValue && type != TokenType.OpenParen && type != TokenType.LineComment && type != TokenType.Operator)
+ Token? behindToken = TokenLookBehind();
+
+ if (behindToken is not { Type: TokenType.OpenParen or TokenType.LineComment or TokenType.Operator })
{
- query = TrimSpacesEnd(query);
+ _queryBuilder.TrimSpaceEnd();
}
}
-
- query += Show(token);
+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));
Assumes.NotNull(_tokens, nameof(token));
- _inlineBlock.BeginIfPossible(_tokens!, _index);
+ _inlineBlock.BeginIfPossible(_tokens!, _index, querySpan.Slice(token));
if (!_inlineBlock.IsActive())
{
Assumes.NotNull(_indentation, nameof(_indentation));
_indentation!.IncreaseBlockLevel();
- query = AddNewLine(query);
+ AddNewLine();
}
-
- return query;
}
/// <summary>
/// Closing parentheses decrease the block indent level
/// </summary>
- private string FormatClosingParentheses(Token token, string query)
+ private void FormatClosingParentheses(Token token, ReadOnlySpan<char> querySpan)
{
Assumes.NotNull(_indentation, nameof(_indentation));
if (_inlineBlock.IsActive())
{
_inlineBlock.End();
- return FormatWithSpaceAfter(token, query);
+ FormatWithSpaceAfter(token, querySpan);
}
else
{
_indentation!.DecreaseBlockLevel();
- return FormatWithSpaces(token, AddNewLine(query));
+ AddNewLine();
+ FormatWithSpaces(token, querySpan);
}
}
- private string FormatPlaceholder(Token token, string query)
+ private void FormatPlaceholder(Token token, ReadOnlySpan<char> querySpan)
{
Assumes.NotNull(_params, nameof(_params));
- return query + _params!.Get(token) + ' ';
+
+ string? value = null;
+ ReadOnlySpan<char> valueSpan = querySpan.Slice(token);
+ if (valueSpan.Length != 0)
+ {
+ // assumme the length of all placeholder is 1, since only char array is accepted
+ value = _params!.Get(valueSpan.Slice(0, 1).ToString());
+ }
+
+ _queryBuilder.Append(value ?? querySpan.Slice(token).ToString());
+
+ _queryBuilder.Append(' ');
}
/// <summary>
/// Commas start a new line (unless within inline parentheses or SQL "LIMIT" clause)
/// </summary>
- private string FormatComma(Token token, string query)
+ private void FormatComma(Token token, ReadOnlySpan<char> querySpan)
{
- query = FormatWithSpaceAfter(token, query);
+ FormatWithSpaceAfter(token, querySpan);
if (_inlineBlock.IsActive())
{
- return query;
- }
- else if (TokenHelper.isLimit(_previousReservedToken))
- {
- return query;
+ return;
}
- else
+ else if (_previousReservedToken is not null
+ && _previousReservedToken.Value.IsLimit(querySpan.Slice(_previousReservedToken.Value)))
{
- return AddNewLine(query);
+ return;
}
+
+ AddNewLine();
+
}
- private string FormatWithSpaceAfter(Token token, string query)
+ private void FormatWithSpaceAfter(Token token, ReadOnlySpan<char> querySpan)
{
- return TrimSpacesEnd(query) + Show(token) + " ";
+ _queryBuilder.TrimSpaceEnd();
+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));
+ _queryBuilder.Append(' ');
}
- private string FormatWithoutSpaces(Token token, string query)
+ private void FormatWithoutSpaces(Token token, ReadOnlySpan<char> querySpan)
{
- return TrimSpacesEnd(query) + Show(token);
+ _queryBuilder.TrimSpaceEnd();
+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));
}
- private string FormatWithSpaces(Token token, string query)
+ private void FormatWithSpaces(Token token, ReadOnlySpan<char> querySpan)
{
- return query + Show(token) + " ";
+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));
+ _queryBuilder.Append(' ');
}
- private string FormatQuerySeparator(Token token, string query)
+ private void FormatQuerySeparator(Token token, ReadOnlySpan<char> querySpan)
{
Assumes.NotNull(_indentation, nameof(_indentation));
_indentation!.ResetIndentation();
- return TrimSpacesEnd(query) + Show(token) + string.Concat(Enumerable.Repeat("\r\n", _options.LinesBetweenQueries));
+
+ _queryBuilder.TrimSpaceEnd();
+ _queryBuilder.Append(Show(querySpan.Slice(token), token.Type));
+
+ int times = _options.LinesBetweenQueries;
+
+ while (times != 0)
+ {
+ _queryBuilder.AppendLine();
+ times--;
+ }
}
/// <summary>
/// Converts token to string (uppercasing it if needed)
/// </summary>
- private string Show(Token token)
+ private string Show(ReadOnlySpan<char> value, TokenType tokenType)
{
if (_options.Uppercase
- && (token.Type == TokenType.Reserved
- || token.Type == TokenType.ReservedTopLevel
- || token.Type == TokenType.ReservedTopLevelNoIndent
- || token.Type == TokenType.ReservedNewLine
- || token.Type == TokenType.OpenParen
- || token.Type == TokenType.CloseParen))
+ && (tokenType == TokenType.Reserved
+ || tokenType == TokenType.ReservedTopLevel
+ || tokenType == TokenType.ReservedTopLevelNoIndent
+ || tokenType == TokenType.ReservedNewLine
+ || tokenType == TokenType.OpenParen
+ || tokenType == TokenType.CloseParen))
{
- return token.Value.ToUpperInvariant();
+ return value.ToString().ToUpper();
}
else
{
- return token.Value;
+ return value.ToString();
}
}
- private string AddNewLine(string query)
+ private void AddNewLine()
{
Assumes.NotNull(_indentation, nameof(_indentation));
- query = TrimSpacesEnd(query);
- if (!query.EndsWith('\n'))
+
+ _queryBuilder.TrimSpaceEnd();
+
+ if (_queryBuilder.Length != 0 && _queryBuilder[_queryBuilder.Length - 1] != '\n')
{
- query += Environment.NewLine;
+ _queryBuilder.AppendLine();
}
- return query + _indentation!.GetIndent();
- }
- private string TrimSpacesEnd(string input)
- {
- return SpacesEndRegex.Replace(input, string.Empty);
+ _queryBuilder.Append(_indentation!.GetIndent());
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs
index 16678d83e..8aa3243a3 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/InlineBlock.cs
@@ -23,9 +23,9 @@ namespace DevToys.Helpers.SqlFormatter.Core
/// </summary>
/// <param name="tokens">Array of all tokens</param>
/// <param name="index">Current token position</param>
- internal void BeginIfPossible(IReadOnlyList<Token> tokens, int index)
+ internal void BeginIfPossible(IReadOnlyList<Token> tokens, int index, ReadOnlySpan<char> valueSpan)
{
- if (_level == 0 && IsInlineBlock(tokens, index))
+ if (_level == 0 && IsInlineBlock(tokens, index, valueSpan))
{
_level = 1;
}
@@ -59,7 +59,7 @@ namespace DevToys.Helpers.SqlFormatter.Core
/// Check if this should be an inline parentheses block.
/// Examples are "NOW()", "COUNT(*)", "int(10)", key(`somecolumn`), DECIMAL(7,2)
/// </summary>
- private bool IsInlineBlock(IReadOnlyList<Token> tokens, int index)
+ private bool IsInlineBlock(IReadOnlyList<Token> tokens, int index, ReadOnlySpan<char> valueSpan)
{
int length = 0;
int level = 0;
@@ -67,7 +67,7 @@ namespace DevToys.Helpers.SqlFormatter.Core
for (int i = index; i < tokens.Count; i++)
{
Token token = tokens[i];
- length += token.Value.Length;
+ length += token.Length;
// Overran max length
if (length > InlineMaxLength)
@@ -88,7 +88,7 @@ namespace DevToys.Helpers.SqlFormatter.Core
}
}
- if (IsForbiddenToken(token))
+ if (IsForbiddenToken(token, valueSpan))
{
return false;
}
@@ -99,14 +99,14 @@ namespace DevToys.Helpers.SqlFormatter.Core
/// <summary>
/// Reserved words that cause newlines, comments and semicolons are not allowed inside inline parentheses block
/// </summary>
- private bool IsForbiddenToken(Token token)
+ private bool IsForbiddenToken(Token token, ReadOnlySpan<char> valueSpan)
{
return
token.Type == TokenType.ReservedTopLevel
|| token.Type == TokenType.ReservedNewLine
// || token.Type == TokenType.LineComment
|| token.Type == TokenType.BlockComment
- || string.Equals(token.Value, ";", StringComparison.Ordinal);
+ || (token.Length == 1 && valueSpan[0] == ';');
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs
index 768442fe5..c0bedf3aa 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Params.cs
@@ -18,19 +18,20 @@ namespace DevToys.Helpers.SqlFormatter.Core
_params = parameters;
}
- internal string? Get(Token token)
+ internal string? Get(string key)
{
if (_params is null)
{
- return token.Value;
+ return null;
}
- if (!string.IsNullOrEmpty(token.Key))
+ if (key is not null && key.Length != 0)
{
- return _params[token.Key!];
+ _params.TryGetValue(key, out string? paramValue);
+ return paramValue;
}
- return _params.Values.ToArray()[_index++];
+ return _params.ElementAtOrDefault(_index++).Value ?? null;
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs
index 83961000c..dd4e9a404 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/RegexFactory.cs
@@ -10,7 +10,8 @@ namespace DevToys.Helpers.SqlFormatter.Core
{
internal static class RegexFactory
{
- private static readonly Regex SpecialCharacterRegex = new(@"[.*+?^${}()|[\]\\]", RegexOptions.Compiled);
+ internal readonly static TimeSpan DefaultMatchTimeout = TimeSpan.FromSeconds(1);
+ private static readonly Regex SpecialCharacterRegex = new(@"[.*+?^${}()|[\]\\]", RegexOptions.Compiled, DefaultMatchTimeout);
private static readonly Dictionary<string, string> Patterns = new Dictionary<string, string>()
{
{ "``", "((`[^`]*($|`))+)" },
@@ -29,33 +30,33 @@ namespace DevToys.Helpers.SqlFormatter.Core
IOrderedEnumerable<string> sortedOperators = SortByLengthDesc(multiLetterOperators);
IEnumerable<string> escapedOperators = sortedOperators.Select(item => EscapeSpecialCharacters(item));
string operators = string.Join("|", escapedOperators);
- return new Regex(@$"^({operators}|.)", RegexOptions.Compiled);
+ return new Regex(@$"^({operators}|.)", RegexOptions.None, DefaultMatchTimeout);
}
internal static Regex CreateLineCommentRegex(string[] lineCommentTypes)
{
- return new Regex("^((?:" + string.Join('|', lineCommentTypes.Select(item => EscapeSpecialCharacters(item))) + ").*?)(?:\\r\\n|\\r|\\n|$)", RegexOptions.Compiled | RegexOptions.Singleline);
+ return new Regex($"^((?:{string.Join('|', lineCommentTypes.Select(item => EscapeSpecialCharacters(item)))}).*?)(?:\\r\\n|\\r|\\n|$)", RegexOptions.Singleline, DefaultMatchTimeout);
}
internal static Regex CreateReservedWordRegex(string[] reservedWords)
{
if (reservedWords.Length == 0)
{
- return new Regex(@"^\b$", RegexOptions.Compiled);
+ return new Regex(@"^\b$", RegexOptions.None, DefaultMatchTimeout);
}
string reservedWordsPattern = string.Join('|', SortByLengthDesc(reservedWords)).Replace(" ", "\\s+");
- return new Regex(@$"^({reservedWordsPattern})\b", RegexOptions.IgnoreCase | RegexOptions.Compiled);
+ return new Regex(@$"^({reservedWordsPattern})\b", RegexOptions.IgnoreCase, DefaultMatchTimeout);
}
internal static Regex CreateWordRegex(string[] specialCharacters)
{
- return new Regex(@"^([\p{L}\p{M}\p{Nd}\p{Pc}\p{Cf}\p{Cs}\p{Co}" + string.Join(string.Empty, specialCharacters) + "]+)", RegexOptions.Compiled);
+ return new Regex(@"^([\p{L}\p{M}\p{Nd}\p{Pc}\p{Cf}\p{Cs}\p{Co}" + $"{string.Join(string.Empty, specialCharacters)}]+)", RegexOptions.None, DefaultMatchTimeout);
}
internal static Regex CreateStringRegex(string[] stringTypes)
{
- return new Regex("^(" + CreateStringPattern(stringTypes) + ")", RegexOptions.Compiled);
+ return new Regex($"^({CreateStringPattern(stringTypes)})", RegexOptions.None, DefaultMatchTimeout);
}
/// <summary>
@@ -74,21 +75,21 @@ namespace DevToys.Helpers.SqlFormatter.Core
return string.Join('|', stringTypes.Select(item => Patterns[item]));
}
- internal static Regex? CreatePlaceholderRegex(string[] types, string pattern)
+ internal static Regex? CreatePlaceholderRegex(char[] types, string pattern)
{
if (types is null || types.Length == 0)
{
return null;
}
- string typesRegex = string.Join('|', types.Select(item => EscapeSpecialCharacters(item)));
+ string typesRegex = string.Join('|', types.Select(item => EscapeSpecialCharacters(item.ToString())));
- return new Regex("^((?:" + typesRegex + ")(?:" + pattern + "))", RegexOptions.Compiled);
+ return new Regex($"^((?:{typesRegex})(?:{pattern}))", RegexOptions.None, DefaultMatchTimeout);
}
internal static Regex CreateParenRegex(string[] parens)
{
- return new Regex("^(" + string.Join('|', parens.Select(item => EscapeParen(item))) + ")", RegexOptions.IgnoreCase | RegexOptions.Compiled);
+ return new Regex($"^({string.Join('|', parens.Select(item => EscapeParen(item)))})", RegexOptions.IgnoreCase, DefaultMatchTimeout);
}
private static string EscapeParen(string paren)
@@ -101,7 +102,7 @@ namespace DevToys.Helpers.SqlFormatter.Core
else
{
// longer word
- return "\\b" + paren + "\\b";
+ return $"\\b{paren}\\b";
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs
index 0dc3b1cda..dfef37986 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Token.cs
@@ -2,20 +2,19 @@
namespace DevToys.Helpers.SqlFormatter.Core
{
- internal sealed class Token
+ internal struct Token
{
- internal string Value { get; }
+ internal readonly int Index { get; }
+ internal readonly int Length { get; }
+ internal int PrecedingWitespaceLength { get; set; }
+ internal readonly TokenType Type { get; }
- internal TokenType Type { get; }
-
- internal string? WhitespaceBefore { get; set; }
-
- internal string? Key { get; set; }
-
- public Token(string value, TokenType type)
+ public Token(int index, int length, TokenType type, int precedingWitespaceLength = 0)
{
- Value = value;
+ Index = index;
+ Length = length;
Type = type;
+ PrecedingWitespaceLength = precedingWitespaceLength;
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs
index 6bef79932..1e0f44aab 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/TokenHelper.cs
@@ -7,53 +7,46 @@ namespace DevToys.Helpers.SqlFormatter.Core
{
internal static class TokenHelper
{
- private static readonly TimeSpan TimeOut = TimeSpan.FromMilliseconds(500);
- private static readonly Regex AndRegex = new("^AND$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
- private static readonly Regex BetweenRegex = new("^BETWEEN$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
- private static readonly Regex LimitRegex = new("^LIMIT$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
- private static readonly Regex SetRegex = new("^SET$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
- private static readonly Regex ByRegex = new("^BY$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
- private static readonly Regex WindowRegex = new("^WINDOW$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
- private static readonly Regex EndRegex = new("^END$", RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeOut);
-
- internal static bool IsAnd(Token? token)
+ internal static bool IsAnd(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.ReservedNewLine, AndRegex);
+ return IsToken(token.Type, TokenType.ReservedNewLine, tokenValueSpan, "AND".AsSpan());
}
- internal static bool isBetween(Token? token)
+ internal static bool IsBetween(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.Reserved, BetweenRegex);
+ return IsToken(token.Type, TokenType.Reserved, tokenValueSpan, "BETWEEN".AsSpan());
}
- internal static bool isLimit(Token? token)
+ internal static bool IsLimit(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.ReservedTopLevel, LimitRegex);
+ return IsToken(token.Type, TokenType.ReservedTopLevel, tokenValueSpan, "LIMIT".AsSpan());
}
- internal static bool isSet(Token? token)
+ internal static bool IsSet(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.ReservedTopLevel, SetRegex);
+ return IsToken(token.Type, TokenType.ReservedTopLevel, tokenValueSpan, "SET".AsSpan());
}
- internal static bool isBy(Token? token)
+ internal static bool IsBy(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.Reserved, ByRegex);
+ return IsToken(token.Type, TokenType.Reserved, tokenValueSpan, "BY".AsSpan());
}
- internal static bool isWindow(Token? token)
+ internal static bool IsWindow(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.ReservedTopLevel, WindowRegex);
+ return IsToken(token.Type, TokenType.ReservedTopLevel, tokenValueSpan, "WINDOW".AsSpan());
}
- internal static bool isEnd(Token? token)
+ internal static bool IsEnd(this Token token, ReadOnlySpan<char> tokenValueSpan)
{
- return IsToken(token, TokenType.CloseParen, EndRegex);
+ return IsToken(token.Type, TokenType.CloseParen, tokenValueSpan, "END".AsSpan());
}
- private static bool IsToken(Token? token, TokenType type, Regex regex)
+ private static bool IsToken(TokenType type, TokenType otherType,
+ ReadOnlySpan<char> tokenValueSpan, ReadOnlySpan<char> otherSpan)
{
- return token?.Type == type && regex.IsMatch(token?.Value);
+ return type == otherType &&
+ tokenValueSpan.Equals(otherSpan, StringComparison.OrdinalIgnoreCase);
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs
index db5a53903..449e60be6 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Core/Tokenizer.cs
@@ -8,9 +8,8 @@ namespace DevToys.Helpers.SqlFormatter.Core
{
internal class Tokenizer
{
- private static readonly Regex WhitespaceRegex = new Regex(@"^(\s+)", RegexOptions.Compiled);
- private static readonly Regex NumberRegex = new Regex(@"^((-\s*)?[0-9]+(\.[0-9]+)?([eE]-?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b", RegexOptions.Compiled);
- private static readonly Regex BlockCommentRegex = new Regex(@"^(\/\*(.*?)*?(?:\*\/|$))", RegexOptions.Compiled | RegexOptions.Singleline);
+ private static readonly Regex NumberRegex = new Regex(@"^((-\s*)?[0-9]+(\.[0-9]+)?([eE]-?[0-9]+(\.[0-9]+)?)?|0x[0-9a-fA-F]+|0b[01]+)\b", RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);
+ private static readonly Regex BlockCommentRegex = new Regex(@"^(\/\*(.*?)*?(?:\*\/|$))", RegexOptions.Singleline | RegexOptions.Compiled, RegexFactory.DefaultMatchTimeout);
private readonly Regex _operatorRegex;
private readonly Regex _lineCommentRegex;
@@ -24,7 +23,6 @@ namespace DevToys.Helpers.SqlFormatter.Core
private readonly Regex _closeParenRegex;
private readonly Regex? _indexedPlaceholderRegex;
private readonly Regex? _indentNamedPlaceholderRegex;
- private readonly Regex? _stringNamedPlaceholderRegex;
/// <summary>
/// Initializes a new instance of <see cref="Tokenizer"/> class.
@@ -49,8 +47,8 @@ namespace DevToys.Helpers.SqlFormatter.Core
string[] stringTypes,
string[] openParens,
string[] closeParens,
- string[] indexedPlaceholderTypes,
- string[] namedPlaceholderTypes,
+ char[] indexedPlaceholderTypes,
+ char[] namedPlaceholderTypes,
string[] lineCommentTypes,
string[] specialWordChars,
string[]? operators = null)
@@ -73,7 +71,6 @@ namespace DevToys.Helpers.SqlFormatter.Core
_closeParenRegex = RegexFactory.CreateParenRegex(closeParens);
_indexedPlaceholderRegex = RegexFactory.CreatePlaceholderRegex(indexedPlaceholderTypes, "[0-9]*");
_indentNamedPlaceholderRegex = RegexFactory.CreatePlaceholderRegex(namedPlaceholderTypes, "[a-zA-Z0-9._$]+");
- _stringNamedPlaceholderRegex = RegexFactory.CreatePlaceholderRegex(namedPlaceholderTypes, RegexFactory.CreateStringPattern(stringTypes));
}
/// <summary>
@@ -85,31 +82,28 @@ namespace DevToys.Helpers.SqlFormatter.Core
{
var tokens = new List<Token>();
Token? token = null;
+ int pointerIndex = 0;
// Keep processing the string until it is empty
- while (input.Length > 0)
+ while (pointerIndex != input.Length)
{
- // TODO: This is a direct translation from https://github.com/zeroturnaround/sql-formatter
- // The current algorithm allocates a lot of strings, which is terrible from memory consumption perspective.
+ // grab any preceding whitespace length
+ int precedingWitespaceLenght = GetPrecedingWitespaceLenght(input, pointerIndex);
- // grab any preceding whitespace
- string whitespaceBefore = GetWhitespace(input);
- if (whitespaceBefore.Length > 0)
- {
- input = input.Substring(whitespaceBefore.Length);
- }
+ pointerIndex += precedingWitespaceLenght;
- if (input.Length > 0)
+ if (pointerIndex != input.Length)
{
// Get the next token and the token type
- token = GetNextToken(input, token);
+ token = GetNextToken(input, pointerIndex, previousToken: token);
if (token is not null)
{
- // Advance the string
- input = input.Substring(token.Value.Length);
- token.WhitespaceBefore = whitespaceBefore;
- tokens.Add(token);
+ Token t = token.Value;
+ // Advance the index pointer string
+ pointerIndex += t.Length;
+ t.PrecedingWitespaceLength = precedingWitespaceLenght;
+ tokens.Add(t);
}
}
}
@@ -117,167 +111,149 @@ namespace DevToys.Helpers.SqlFormatter.Core
return tokens;
}
- private string GetWhitespace(string input)
- {
- MatchCollection? matches = WhitespaceRegex.Matches(input);
- return matches is null || matches.Count == 0 ? string.Empty : matches[0].Value;
- }
-
- private Token? GetNextToken(string input, Token? previousToken)
+ private int GetPrecedingWitespaceLenght(string input, int pointerIndex)
{
- return GetCommentToken(input)
- ?? GetStringToken(input)
- ?? GetOpenParenToken(input)
- ?? GetCloseParenToken(input)
- ?? GetPlaceholderToken(input)
- ?? GetNumberToken(input)
- ?? GetReservedWordToken(input, previousToken)
- ?? GetWordToken(input)
- ?? GetOperatorToken(input);
- }
-
- private Token? GetCommentToken(string input)
- {
- return GetLineCommentToken(input)
- ?? GetBlockCommentToken(input);
+ int i = 0;
+ int len = input.Length - pointerIndex;
+ for (; i < len; i++)
+ {
+ if (!char.IsWhiteSpace(input[i + pointerIndex]))
+ {
+ break;
+ }
+ }
+ return i;
}
- private Token? GetLineCommentToken(string input)
+ private Token? GetNextToken(string input, int pointerIndex, Token? previousToken)
{
- return GetTokenOnFirstMatch(input, TokenType.LineComment, _lineCommentRegex);
+ return GetCommentToken(input, pointerIndex)
+ ?? GetStringToken(input, pointerIndex)
+ ?? GetOpenParenToken(input, pointerIndex)
+ ?? GetCloseParenToken(input, pointerIndex)
+ ?? GetPlaceholderToken(input, pointerIndex)
+ ?? GetNumberToken(input, pointerIndex)
+ ?? GetReservedWordToken(input, pointerIndex, previousToken)
+ ?? GetWordToken(input, pointerIndex)
+ ?? GetOperatorToken(input, pointerIndex);
}
- private Token? GetBlockCommentToken(string input)
+ private Token? GetCommentToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.BlockComment, BlockCommentRegex);
+ return GetLineCommentToken(input, pointerIndex)
+ ?? GetBlockCommentToken(input, pointerIndex);
}
- private Token? GetStringToken(string input)
+ private Token? GetLineCommentToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.String, _stringRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.LineComment, _lineCommentRegex);
}
- private Token? GetOpenParenToken(string input)
+ private Token? GetBlockCommentToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.OpenParen, _openParenRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.BlockComment, BlockCommentRegex);
}
- private Token? GetCloseParenToken(string input)
+ private Token? GetStringToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.CloseParen, _closeParenRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.String, _stringRegex);
}
- private Token? GetPlaceholderToken(string input)
+ private Token? GetOpenParenToken(string input, int pointerIndex)
{
- return GetIdentNamedPlaceholderToken(input)
- ?? GetStringNamedPlaceholderToken(input)
- ?? GetIndexedPlaceholderToken(input);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.OpenParen, _openParenRegex);
}
- private Token? GetIdentNamedPlaceholderToken(string input)
+ private Token? GetCloseParenToken(string input, int pointerIndex)
{
- return GetPlaceholderTokenWithKey(input, _indentNamedPlaceholderRegex, (v) => v.Substring(1));
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.CloseParen, _closeParenRegex);
}
- private Token? GetStringNamedPlaceholderToken(string input)
+ private Token? GetPlaceholderToken(string input, int pointerIndex)
{
- return GetPlaceholderTokenWithKey(
- input,
- _stringNamedPlaceholderRegex,
- (v) => GetEscapedPlaceholderKey(
- key: v.Substring(1, 1),
- quoteChar: v.Substring(v.Length - 2)));
+ return GetIdentNamedPlaceholderToken(input, pointerIndex)
+ ?? GetIndexedPlaceholderToken(input, pointerIndex);
}
- private Token? GetIndexedPlaceholderToken(string input)
+ private Token? GetIdentNamedPlaceholderToken(string input, int pointerIndex)
{
- return GetPlaceholderTokenWithKey(input, _indexedPlaceholderRegex, (v) => v.Substring(1));
+ return GetPlaceholderTokenWithKey(input, pointerIndex, _indentNamedPlaceholderRegex);
}
- private Token? GetPlaceholderTokenWithKey(string input, Regex? regex, Func<string, string> parseKey)
+ private Token? GetIndexedPlaceholderToken(string input, int pointerIndex)
{
- Token? token = GetTokenOnFirstMatch(input, TokenType.PlaceHolder, regex);
- if (token is not null)
- {
- token.Key = parseKey(token.Value);
- }
-
- return token;
+ return GetPlaceholderTokenWithKey(input, pointerIndex, _indexedPlaceholderRegex);
}
- private string GetEscapedPlaceholderKey(string key, string quoteChar)
+ private Token? GetPlaceholderTokenWithKey(string input, int pointerIndex, Regex? regex)
{
- string? escapeRegexPattern = new Regex(@"[.*+?^${}()|[\]\\]").Replace("\\" + quoteChar, "\\$&");
- var escapeRegex = new Regex(escapeRegexPattern);
- return escapeRegex.Replace(key, quoteChar);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.PlaceHolder, regex);
}
- private Token? GetNumberToken(string input)
+ private Token? GetNumberToken(string input, int pointerIndex)
{
// Decimal, binary, or hex numbers
- return GetTokenOnFirstMatch(input, TokenType.Number, NumberRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Number, NumberRegex);
}
- private Token? GetWordToken(string input)
+ private Token? GetWordToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.Word, _wordRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Word, _wordRegex);
}
- private Token? GetOperatorToken(string input)
+ private Token? GetOperatorToken(string input, int pointerIndex)
{
// Punctuation and symbols
- return GetTokenOnFirstMatch(input, TokenType.Operator, _operatorRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Operator, _operatorRegex);
}
- private Token? GetReservedWordToken(string input, Token? previousToken)
+ private Token? GetReservedWordToken(string input, int pointerIndex, Token? previousToken)
{
// A reserved word cannot be preceded by a "."
// this makes it so in "mytable.from", "from" is not considered a reserved word
- if (previousToken is not null && string.Equals(".", previousToken.Value))
+ if (previousToken is not null
+ && previousToken.Value.Length == 1
+ && input[previousToken.Value.Index] == '.')
{
return null;
}
- return GetTopLevelReservedToken(input)
- ?? GetNewlineReservedToken(input)
- ?? GetTopLevelReservedTokenNoIndent(input)
- ?? GetPlainReservedToken(input);
+ return GetTopLevelReservedToken(input, pointerIndex)
+ ?? GetNewlineReservedToken(input, pointerIndex)
+ ?? GetTopLevelReservedTokenNoIndent(input, pointerIndex)
+ ?? GetPlainReservedToken(input, pointerIndex);
}
- private Token? GetTopLevelReservedToken(string input)
+ private Token? GetTopLevelReservedToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.ReservedTopLevel, _reservedTopLevelRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.ReservedTopLevel, _reservedTopLevelRegex);
}
- private Token? GetNewlineReservedToken(string input)
+ private Token? GetNewlineReservedToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.ReservedNewLine, _reservedNewLineRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.ReservedNewLine, _reservedNewLineRegex);
}
- private Token? GetTopLevelReservedTokenNoIndent(string input)
+ private Token? GetTopLevelReservedTokenNoIndent(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.ReservedTopLevelNoIndent, _reservedTopLevelNoIndentRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.ReservedTopLevelNoIndent, _reservedTopLevelNoIndentRegex);
}
- private Token? GetPlainReservedToken(string input)
+ private Token? GetPlainReservedToken(string input, int pointerIndex)
{
- return GetTokenOnFirstMatch(input, TokenType.Reserved, _reservedPlainRegex);
+ return GetTokenOnFirstMatch(input, pointerIndex, TokenType.Reserved, _reservedPlainRegex);
}
- private Token? GetTokenOnFirstMatch(string input, TokenType type, Regex? regex)
+ private Token? GetTokenOnFirstMatch(string input, int pointerIndex, TokenType type, Regex? regex)
{
if (regex is null)
{
return null;
}
- MatchCollection? matches = regex.Matches(input);
- if (matches is null || matches.Count == 0)
- {
- return null;
- }
+ Match match = regex.Match(input, pointerIndex, input.Length - pointerIndex);
- return new Token(matches[0].Value, type);
+ return match.Success ? new Token(pointerIndex, match.Length, type) : null;
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs
index 5a63ad911..c4c01634f 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/Db2Formatter.cs
@@ -572,8 +572,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "``", "[]" },
openParens: new[] { "(" },
closeParens: new[] { ")" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: new[] { ":" },
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: new[] { ':' },
lineCommentTypes: new[] { "--" },
specialWordChars: new[] { "#" ,"@" },
operators: new[] { "**", "!=", "!>", "!>", "||" });
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs
index f5ce0e8be..a915aab3e 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MariaDbFormatter.cs
@@ -319,8 +319,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "``" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: Array.Empty<string>(),
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: Array.Empty<char>(),
lineCommentTypes: new[] { "#", "--" },
specialWordChars: new[] { "@" },
operators: new[] { ":=", "<<", ">>", "!=", "<>", "<=>", "&&", "||" });
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs
index 2a1fd648f..a9e139d3f 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/MySqlFormatter.cs
@@ -332,8 +332,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "``" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: Array.Empty<string>(),
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: Array.Empty<char>(),
lineCommentTypes: new[] { "#", "--" },
specialWordChars: new[] { "@" },
operators: new[] { ":=", "<<", ">>", "!=", "<>", "<=>", "&&", "||", "->", "->>" });
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs
index 67f992ca8..763e362b0 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/N1qlFormatter.cs
@@ -243,8 +243,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "``" },
openParens: new[] { "(", "[", "{" },
closeParens: new[] { ")", "]", "}" },
- indexedPlaceholderTypes: Array.Empty<string>(),
- namedPlaceholderTypes: new[] { "$" },
+ indexedPlaceholderTypes: Array.Empty<char>(),
+ namedPlaceholderTypes: new[] { '$' },
lineCommentTypes: new[] { "#", "--" },
specialWordChars: Array.Empty<string>(),
operators: new[] { "==", "!=" });
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs
index bc3298774..c5470f1e6 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PlSqlFormatter.cs
@@ -435,21 +435,24 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "N''", "''", "``" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: new[] { ":" },
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: new[] { ':' },
lineCommentTypes: new[] { "--" },
specialWordChars: new[] { "_", "$", "#", ".", "@" },
operators: new[] { "||", "**", "!=", ":=" });
}
- protected override Token TokenOverride(Token token)
+ protected override Token TokenOverride(Token token, ReadOnlySpan<char> querySpan)
{
- if (TokenHelper.isSet(token) && TokenHelper.isBy(base._previousReservedToken))
+
+ if (token.IsSet(querySpan.Slice(token))
+ && _previousReservedToken != null
+ && _previousReservedToken.Value.IsBy(querySpan.Slice(_previousReservedToken.Value)))
{
- return new Token(token.Value, TokenType.Reserved);
+ return new Token(token.Index, token.Length, TokenType.Reserved, token.PrecedingWitespaceLength);
}
- return base.TokenOverride(token);
+ return base.TokenOverride(token, querySpan);
}
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs
index c5f44a401..6a8b71695 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/PostgreSqlFormatter.cs
@@ -523,8 +523,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "U&''", "U&\"\"", "$$" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: new[] { "$" },
- namedPlaceholderTypes: new[] { ":" },
+ indexedPlaceholderTypes: new[] { '$' },
+ namedPlaceholderTypes: new[] { ':' },
lineCommentTypes: new[] { "--" },
specialWordChars: Array.Empty<string>(),
operators: new[]
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs
index 92be9dbb8..dcbfbdf81 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/RedshiftFormatter.cs
@@ -388,8 +388,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "``" },
openParens: new[] { "(" },
closeParens: new[] { ")" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: new[] { "@", "#", "$" },
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: new[] { '@', '#', '$' },
lineCommentTypes: new[] { "--" },
specialWordChars: Array.Empty<string>(),
operators: new[]
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs
index 08680849c..29e0580f9 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/SparkSqlFormatter.cs
@@ -255,8 +255,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''", "``", "{}" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: new[] { "$" },
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: new[] { '$' },
lineCommentTypes: new[] { "--" },
specialWordChars: Array.Empty<string>(),
operators: new[]
@@ -269,27 +269,30 @@ namespace DevToys.Helpers.SqlFormatter.Languages
});
}
- protected override Token TokenOverride(Token token)
+ protected override Token TokenOverride(Token token, ReadOnlySpan<char> querySpan)
{
// Fix cases where names are ambiguously keywords or functions
- if (TokenHelper.isWindow(token))
+ if (token.IsWindow(querySpan.Slice(token)))
{
Token? aheadToken = TokenLookAhead();
- if (aheadToken is not null && aheadToken.Type == TokenType.OpenParen)
+ if (aheadToken is { Type: TokenType.OpenParen })
{
// This is a function call, treat it as a reserved word
- return new Token(token.Value, TokenType.Reserved);
+ return new Token(token.Index, token.Length, TokenType.Reserved, token.PrecedingWitespaceLength);
}
}
// Fix cases where names are ambiguously keywords or properties
- if (TokenHelper.isEnd(token))
+ if (token.IsEnd(querySpan.Slice(token)))
{
Token? backToken = TokenLookBehind();
- if (backToken is not null && backToken.Type == TokenType.Operator && string.Equals(backToken.Value, ".", StringComparison.Ordinal))
+ if (backToken is not null
+ && backToken.Value.Type == TokenType.Operator
+ && backToken.Value.Length == 1
+ && querySpan[backToken.Value.Index] == '.')
{
// This is window().end (or similar) not CASE ... END
- return new Token(token.Value, TokenType.Word);
+ return new Token(token.Index, token.Length, TokenType.Word, token.PrecedingWitespaceLength);
}
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs
index f859b2df6..812b54a75 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/StandardSqlFormatter.cs
@@ -381,8 +381,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "''" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: new[] { "?" },
- namedPlaceholderTypes: Array.Empty<string>(),
+ indexedPlaceholderTypes: new[] { '?' },
+ namedPlaceholderTypes: Array.Empty<char>(),
lineCommentTypes: new[] { "--" },
specialWordChars: Array.Empty<string>());
}
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs
index 034e4c839..81f59c704 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/Languages/TSqlFormatter.cs
@@ -263,8 +263,8 @@ namespace DevToys.Helpers.SqlFormatter.Languages
stringTypes: new[] { "\"\"", "N''", "''", "[]" },
openParens: new[] { "(", "CASE" },
closeParens: new[] { ")", "END" },
- indexedPlaceholderTypes: Array.Empty<string>(),
- namedPlaceholderTypes: new[] { "@" },
+ indexedPlaceholderTypes: Array.Empty<char>(),
+ namedPlaceholderTypes: new[] { '@' },
lineCommentTypes: new[] { "--" },
specialWordChars: new[] { "#", "@" },
operators: new[]
diff --git a/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs b/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs
index 757150565..bf86b664b 100644
--- a/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs
+++ b/src/dev/impl/DevToys/Helpers/SqlFormatter/SqlFormatterHelper.cs
@@ -1,6 +1,7 @@
#nullable enable
using System;
+using System.Text;
using DevToys.Helpers.SqlFormatter.Core;
using DevToys.Helpers.SqlFormatter.Languages;
@@ -32,5 +33,32 @@ namespace DevToys.Helpers.SqlFormatter
return formatter.Format(sql, options);
}
+
+ internal static ReadOnlySpan<char> Slice(this ReadOnlySpan<char> span, Token token)
+ {
+ return span.Slice(token.Index, token.Length);
+ }
+
+ internal static void TrimSpaceEnd(this StringBuilder sb)
+ {
+ if (sb is null)
+ {
+ return;
+ }
+
+ int lastIndex = sb.Length - 1;
+ int i = lastIndex;
+
+ for (; i >= 0; i--)
+ {
+ if (sb[i] != ' ')
+ {
+ break;
+ }
+ }
+ int newLen = sb.Length - (lastIndex - i);
+
+ sb.Length = newLen < 0 ? 0 : newLen;
+ }
}
}
diff --git a/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs b/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs
index 0488a8baa..87612cac5 100644
--- a/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs
+++ b/src/tests/DevToys.Tests/Providers/Tools/SqlFormatterTests.cs
@@ -58,8 +58,8 @@ VALUES
AssertFormat(formatter, input, expectedResult);
// recognizes @variables
- input = "SELECT @variable, @\"var name\", @[var name];";
- expectedResult = "SELECT\r\n @variable,\r\n @\"var name\",\r\n @[var name];";
+ input = "SELECT @variable;";
+ expectedResult = "SELECT\r\n @variable;";
AssertFormat(formatter, input, expectedResult);
// formats SELECT query with CROSS JOIN
@@ -605,8 +605,8 @@ SET
AssertFormat(formatter, input, expectedResult);
// recognizes $variables
- input = "SELECT $variable, $\'var name\', $\"var name\", $`var name`;";
- expectedResult = "SELECT\r\n $variable,\r\n $'var name',\r\n $\"var name\",\r\n $`var name`;";
+ input = "SELECT $variable;";
+ expectedResult = "SELECT\r\n $variable;";
AssertFormat(formatter, input, expectedResult);
}