The minimalistic document archiving platform.
fix(tokenizer): properly handle quoted negated filter (#766)
Corentin Thomasset2d8ea0fad759
diff --git a/packages/search-parser/src/tokenizer.test.ts b/packages/search-parser/src/tokenizer.test.tsnew file mode 100644index 00000000..4f6d3ff7--- /dev/null+++ b/packages/search-parser/src/tokenizer.test.ts@@ -0,0 +1,59 @@+import { describe, expect, test } from 'vitest';+import { tokenize } from './tokenizer';++describe('tokenizer', () => {+ describe('tokenize', () => {+ describe('tokenizes queries', () => {+ const queries: { query: string; expectedTokens: unknown[]; maxTokens?: number }[] = [+ {+ query: 'foobar',+ expectedTokens: [+ { type: 'TEXT', value: 'foobar' },+ { type: 'EOF' },+ ],+ },+ {+ query: '-foobar',+ expectedTokens: [+ { type: 'NOT' },+ { type: 'TEXT', value: 'foobar' },+ { type: 'EOF' },+ ],+ },+ {+ query: 'status:open AND priority:high',+ expectedTokens: [+ { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },+ { type: 'AND' },+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },+ { type: 'EOF' },+ ],+ },+ {+ query: `type:feature AND (status:open OR (priority:high -tag:"full review")) "user feedback"`,+ expectedTokens: [+ { type: 'FILTER', field: 'type', operator: '=', value: 'feature', negated: false },+ { type: 'AND' },+ { type: 'LPAREN' },+ { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },+ { type: 'OR' },+ { type: 'LPAREN' },+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },+ { type: 'FILTER', field: 'tag', operator: '=', value: 'full review', negated: true },+ { type: 'RPAREN' },+ { type: 'RPAREN' },+ { type: 'TEXT', value: 'user feedback' },+ { type: 'EOF' },+ ],+ },+ ];++ for (const { query, expectedTokens, maxTokens = 100 } of queries) {+ test(`tokenizes "${query}"`, () => {+ const result = tokenize({ query, maxTokens });+ expect(result.tokens).to.eql(expectedTokens);+ });+ }+ });+ });+});diff --git a/packages/search-parser/src/tokenizer.ts b/packages/search-parser/src/tokenizer.tsindex 9904b0e6..471cd860 100644--- a/packages/search-parser/src/tokenizer.ts+++ b/packages/search-parser/src/tokenizer.ts@@ -73,7 +73,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbereturn value;};- const readUnquotedToken = (stopAtQuote = false): string => {+ const readUnquotedToken = (): string => {let value = '';while (pos < query.length) {@@ -83,8 +83,8 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbebreak;}- // Stop at quote if requested (for filter field names before quoted values)- if (stopAtQuote && char === '"') {+ // Stop at quote+ if (char === '"') {break;}@@ -244,7 +244,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe// Read quoted or unquoted tokenconst quotedValue = readQuotedString();- const token = quotedValue !== undefined ? quotedValue : readUnquotedToken(true);+ const token = quotedValue !== undefined ? quotedValue : readUnquotedToken();if (!token) {advance(); // Skip invalid character