The minimalistic document archiving platform.
feat(tokenizer): improve filter parsing (#767)
Corentin Thomassetbe108e4d0604
diff --git a/packages/search-parser/src/parser.ts b/packages/search-parser/src/parser.tsindex 0ef9e712..ad1f5a86 100644--- a/packages/search-parser/src/parser.ts+++ b/packages/search-parser/src/parser.ts@@ -87,18 +87,12 @@ function parseExpression({ tokens, maxDepth }: { tokens: Token[]; maxDepth: numbif (token.type === 'FILTER') {advance();- const filterExpr: Expression = {+ return {type: 'filter',field: token.field,operator: token.operator,value: token.value,};-- if (token.negated) {- return { type: 'not', operand: filterExpr };- }-- return filterExpr;}if (token.type === 'TEXT') {diff --git a/packages/search-parser/src/string.test.ts b/packages/search-parser/src/string.test.tsnew file mode 100644index 00000000..45ff7210--- /dev/null+++ b/packages/search-parser/src/string.test.ts@@ -0,0 +1,21 @@+import { describe, expect, test } from 'vitest';+import { isWhitespace } from './string';++describe('string', () => {+ describe('isWhitespace', () => {+ test('detects if the character (the first character if multiple) is a whitespace', () => {+ expect(isWhitespace(' ')).toBe(true);+ expect(isWhitespace(' ')).toBe(true);+ expect(isWhitespace('\t')).toBe(true);+ expect(isWhitespace('\n')).toBe(true);+ expect(isWhitespace(' ')).toBe(true);++ expect(isWhitespace(' ')).toBe(true);+ expect(isWhitespace(' a')).toBe(true);+ expect(isWhitespace('aqq')).toBe(false);++ expect(isWhitespace('a')).toBe(false);+ expect(isWhitespace('')).toBe(false);+ });+ });+});diff --git a/packages/search-parser/src/string.ts b/packages/search-parser/src/string.tsnew file mode 100644index 00000000..c8a8e67e--- /dev/null+++ b/packages/search-parser/src/string.ts@@ -0,0 +1,3 @@+export function isWhitespace(char: string): boolean {+ return /\s/.test(char);+}diff --git a/packages/search-parser/src/tokenizer.test.ts b/packages/search-parser/src/tokenizer.test.tsindex 4f6d3ff7..79d9a9ad 100644--- a/packages/search-parser/src/tokenizer.test.ts+++ b/packages/search-parser/src/tokenizer.test.ts@@ -23,29 +23,46 @@ describe('tokenizer', () => {{query: 'status:open AND priority:high',expectedTokens: [- { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },+ { type: 'FILTER', field: 'status', operator: '=', value: 'open' },{ type: 'AND' },- { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high' },{ type: 'EOF' },],},{query: `type:feature AND (status:open OR (priority:high -tag:"full review")) "user feedback"`,expectedTokens: [- { type: 'FILTER', field: 'type', operator: '=', value: 'feature', negated: false },+ { type: 'FILTER', field: 'type', operator: '=', value: 'feature' },{ type: 'AND' },{ type: 'LPAREN' },- { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },+ { type: 'FILTER', field: 'status', operator: '=', value: 'open' },{ type: 'OR' },{ type: 'LPAREN' },- { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },- { type: 'FILTER', field: 'tag', operator: '=', value: 'full review', negated: true },+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high' },+ { type: 'NOT' },+ { type: 'FILTER', field: 'tag', operator: '=', value: 'full review' },{ type: 'RPAREN' },{ type: 'RPAREN' },{ type: 'TEXT', value: 'user feedback' },{ type: 'EOF' },],},+ {+ query: `tag:bug\\:critical`,+ expectedTokens: [+ { type: 'FILTER', field: 'tag', operator: '=', value: 'bug:critical' },+ { type: 'EOF' },+ ],+ },+ {+ query: `tag:-important tag:"-important" tag:\\-important`,+ expectedTokens: [+ { type: 'FILTER', field: 'tag', operator: '=', value: '-important' },+ { type: 'FILTER', field: 'tag', operator: '=', value: '-important' },+ { type: 'FILTER', field: 'tag', operator: '=', value: '\\-important' },+ { type: 'EOF' },+ ],+ },];for (const { query, expectedTokens, maxTokens = 100 } of queries) {diff --git a/packages/search-parser/src/tokenizer.ts b/packages/search-parser/src/tokenizer.tsindex 471cd860..f51e6b9e 100644--- a/packages/search-parser/src/tokenizer.ts+++ b/packages/search-parser/src/tokenizer.ts@@ -1,5 +1,6 @@import type { Issue, Operator } from './parser.types';import { ERROR_CODES } from './errors';+import { isWhitespace } from './string';export type Token= | { type: 'LPAREN' }@@ -7,7 +8,7 @@ export type Token| { type: 'AND' }| { type: 'OR' }| { type: 'NOT' }- | { type: 'FILTER'; field: string; operator: Operator; value: string; negated: boolean }+ | { type: 'FILTER'; field: string; operator: Operator; value: string }| { type: 'TEXT'; value: string }| { type: 'EOF' };@@ -16,6 +17,16 @@ export type TokenizeResult = {issues: Issue[];};+// Unified escape handling utilities+const unescapeBackslashes = (str: string): string => str.replace(/\\(.)/g, '$1');+const unescapeColons = (str: string): string => str.replace(/\\:/g, ':');++type StopCondition = (char: string) => boolean;++function isWhitespaceOrParen(char: string): boolean {+ return isWhitespace(char) || char === '(' || char === ')';+}+export function tokenize({ query, maxTokens }: { query: string; maxTokens: number }): TokenizeResult {const tokens: Token[] = [];const issues: Issue[] = [];@@ -27,7 +38,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbeconst skipWhitespace = (): void => {while (pos < query.length) {const char = query[pos];- if (!char || !/\s/.test(char)) {+ if (!char || !isWhitespace(char)) {break;}pos++;@@ -73,28 +84,51 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbereturn value;};- const readUnquotedToken = (): string => {+ // Unified function to read unquoted values with configurable stop conditions and escape handling+ const readUnquotedValue = ({+ stopCondition,+ processEscapes,+ stopAtQuote = false,+ }: {+ stopCondition: StopCondition;+ processEscapes: boolean;+ stopAtQuote?: boolean;+ }): string => {let value = '';while (pos < query.length) {const char = peek();- if (!char || /\s/.test(char) || char === '(' || char === ')') {+ if (!char || stopCondition(char)) {break;}- // Stop at quote- if (char === '"') {+ if (stopAtQuote && char === '"') {break;}- // Keep backslashes in unquoted tokens so parseFilter can detect escaped colons- value += advance();+ if (processEscapes && char === '\\') {+ advance();+ if (pos < query.length) {+ value += advance();+ }+ } else {+ value += advance();+ }}return value;};+ const readUnquotedToken = (): string => {+ // Keep backslashes in unquoted tokens so parseFilter can detect escaped colons+ return readUnquotedValue({+ stopCondition: isWhitespaceOrParen,+ processEscapes: false,+ stopAtQuote: true,+ });+ };+const readFilterValue = (): string => {skipWhitespace();@@ -104,26 +138,13 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbereturn quotedValue;}- // Read unquoted value (up to whitespace or special chars)- let value = '';- while (pos < query.length) {- const char = peek();-- if (!char || /\s/.test(char) || char === '(' || char === ')') {- break;- }-- if (char === '\\') {- advance();- if (pos < query.length) {- value += advance();- }- } else {- value += advance();- }- }+ // Read unquoted value with escape processing, then unescape colons+ const value = readUnquotedValue({+ stopCondition: isWhitespaceOrParen,+ processEscapes: true,+ });- return value.replace(/\\:/g, ':');+ return unescapeColons(value);};const hasUnescapedColon = (str: string): boolean => {@@ -135,7 +156,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbereturn false;};- const parseFilter = (token: string, negated: boolean): Token | undefined => {+ const parseFilter = (token: string): Token | undefined => {// Check for unescaped colonsif (!hasUnescapedColon(token)) {return undefined;@@ -154,7 +175,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbereturn undefined;}- const field = token.slice(0, firstColonIndex).replace(/\\:/g, ':');+ const field = unescapeColons(token.slice(0, firstColonIndex));const afterColon = token.slice(firstColonIndex + 1);// Check for operator at the start@@ -179,14 +200,14 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe}// If there's nothing after the operator in the token, read the value from input- let value = afterColon.slice(operatorLength).replace(/\\:/g, ':');+ let value = unescapeColons(afterColon.slice(operatorLength));// If the value is empty and we still have input, this might be a case like "tag:" followed by a quoted stringif (!value) {value = readFilterValue();}- return { type: 'FILTER', field, operator, value, negated };+ return { type: 'FILTER', field, operator, value };};while (pos < query.length) {@@ -221,7 +242,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe// Handle negation prefixconst nextChar = query[pos + 1];- if (char === '-' && nextChar && !/\s/.test(nextChar)) {+ if (char === '-' && nextChar && !isWhitespace(nextChar)) {advance();skipWhitespace();@@ -230,14 +251,14 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbeconst token = quotedValue !== undefined ? quotedValue : readUnquotedToken();// Try to parse as filter- const filter = parseFilter(token, true);+ const filter = parseFilter(token);if (filter) {+ tokens.push({ type: 'NOT' });tokens.push(filter);} else {- // If not a filter, treat as negated text search (which we'll handle as NOT operator)- const unescapedText = token.replace(/\\(.)/g, '$1');+ // If not a filter, treat as negated text searchtokens.push({ type: 'NOT' });- tokens.push({ type: 'TEXT', value: unescapedText });+ tokens.push({ type: 'TEXT', value: unescapeBackslashes(token) });}continue;}@@ -270,7 +291,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe// Try to parse as filter (only if not quoted)if (quotedValue === undefined) {- const filter = parseFilter(token, false);+ const filter = parseFilter(token);if (filter) {tokens.push(filter);continue;@@ -278,8 +299,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe}// Otherwise, treat as text (unescape backslashes)- const unescapedText = token.replace(/\\(.)/g, '$1');- tokens.push({ type: 'TEXT', value: unescapedText });+ tokens.push({ type: 'TEXT', value: unescapeBackslashes(token) });}tokens.push({ type: 'EOF' });