Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

fix(tokenizer): properly handle quoted negated filter (#766)

Corentin Thomasset2d8ea0fad759
diff --git a/packages/search-parser/src/tokenizer.test.ts b/packages/search-parser/src/tokenizer.test.ts
new file mode 100644
index 00000000..4f6d3ff7
--- /dev/null
+++ b/packages/search-parser/src/tokenizer.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, test } from 'vitest';
+import { tokenize } from './tokenizer';
+
+describe('tokenizer', () => {
+ describe('tokenize', () => {
+ describe('tokenizes queries', () => {
+ const queries: { query: string; expectedTokens: unknown[]; maxTokens?: number }[] = [
+ {
+ query: 'foobar',
+ expectedTokens: [
+ { type: 'TEXT', value: 'foobar' },
+ { type: 'EOF' },
+ ],
+ },
+ {
+ query: '-foobar',
+ expectedTokens: [
+ { type: 'NOT' },
+ { type: 'TEXT', value: 'foobar' },
+ { type: 'EOF' },
+ ],
+ },
+ {
+ query: 'status:open AND priority:high',
+ expectedTokens: [
+ { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },
+ { type: 'AND' },
+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },
+ { type: 'EOF' },
+ ],
+ },
+ {
+ query: `type:feature AND (status:open OR (priority:high -tag:"full review")) "user feedback"`,
+ expectedTokens: [
+ { type: 'FILTER', field: 'type', operator: '=', value: 'feature', negated: false },
+ { type: 'AND' },
+ { type: 'LPAREN' },
+ { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },
+ { type: 'OR' },
+ { type: 'LPAREN' },
+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },
+ { type: 'FILTER', field: 'tag', operator: '=', value: 'full review', negated: true },
+ { type: 'RPAREN' },
+ { type: 'RPAREN' },
+ { type: 'TEXT', value: 'user feedback' },
+ { type: 'EOF' },
+ ],
+ },
+ ];
+
+ for (const { query, expectedTokens, maxTokens = 100 } of queries) {
+ test(`tokenizes "${query}"`, () => {
+ const result = tokenize({ query, maxTokens });
+ expect(result.tokens).to.eql(expectedTokens);
+ });
+ }
+ });
+ });
+});
diff --git a/packages/search-parser/src/tokenizer.ts b/packages/search-parser/src/tokenizer.ts
index 9904b0e6..471cd860 100644
--- a/packages/search-parser/src/tokenizer.ts
+++ b/packages/search-parser/src/tokenizer.ts
@@ -73,7 +73,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
return value;
};
- const readUnquotedToken = (stopAtQuote = false): string => {
+ const readUnquotedToken = (): string => {
let value = '';
while (pos < query.length) {
@@ -83,8 +83,8 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
break;
}
- // Stop at quote if requested (for filter field names before quoted values)
- if (stopAtQuote && char === '"') {
+ // Stop at quote
+ if (char === '"') {
break;
}
@@ -244,7 +244,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
// Read quoted or unquoted token
const quotedValue = readQuotedString();
- const token = quotedValue !== undefined ? quotedValue : readUnquotedToken(true);
+ const token = quotedValue !== undefined ? quotedValue : readUnquotedToken();
if (!token) {
advance(); // Skip invalid character