Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

feat(tokenizer): improve filter parsing (#767)

Corentin Thomassetbe108e4d0604
diff --git a/packages/search-parser/src/parser.ts b/packages/search-parser/src/parser.ts
index 0ef9e712..ad1f5a86 100644
--- a/packages/search-parser/src/parser.ts
+++ b/packages/search-parser/src/parser.ts
@@ -87,18 +87,12 @@ function parseExpression({ tokens, maxDepth }: { tokens: Token[]; maxDepth: numb
if (token.type === 'FILTER') {
advance();
- const filterExpr: Expression = {
+ return {
type: 'filter',
field: token.field,
operator: token.operator,
value: token.value,
};
-
- if (token.negated) {
- return { type: 'not', operand: filterExpr };
- }
-
- return filterExpr;
}
if (token.type === 'TEXT') {
diff --git a/packages/search-parser/src/string.test.ts b/packages/search-parser/src/string.test.ts
new file mode 100644
index 00000000..45ff7210
--- /dev/null
+++ b/packages/search-parser/src/string.test.ts
@@ -0,0 +1,21 @@
+import { describe, expect, test } from 'vitest';
+import { isWhitespace } from './string';
+
+describe('string', () => {
+ describe('isWhitespace', () => {
+ test('detects if the character (the first character if multiple) is a whitespace', () => {
+ expect(isWhitespace(' ')).toBe(true);
+ expect(isWhitespace(' ')).toBe(true);
+ expect(isWhitespace('\t')).toBe(true);
+ expect(isWhitespace('\n')).toBe(true);
+ expect(isWhitespace(' ')).toBe(true);
+
+ expect(isWhitespace(' ')).toBe(true);
+ expect(isWhitespace(' a')).toBe(true);
+ expect(isWhitespace('aqq')).toBe(false);
+
+ expect(isWhitespace('a')).toBe(false);
+ expect(isWhitespace('')).toBe(false);
+ });
+ });
+});
diff --git a/packages/search-parser/src/string.ts b/packages/search-parser/src/string.ts
new file mode 100644
index 00000000..c8a8e67e
--- /dev/null
+++ b/packages/search-parser/src/string.ts
@@ -0,0 +1,3 @@
+export function isWhitespace(char: string): boolean {
+ return /\s/.test(char);
+}
diff --git a/packages/search-parser/src/tokenizer.test.ts b/packages/search-parser/src/tokenizer.test.ts
index 4f6d3ff7..79d9a9ad 100644
--- a/packages/search-parser/src/tokenizer.test.ts
+++ b/packages/search-parser/src/tokenizer.test.ts
@@ -23,29 +23,46 @@ describe('tokenizer', () => {
{
query: 'status:open AND priority:high',
expectedTokens: [
- { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },
+ { type: 'FILTER', field: 'status', operator: '=', value: 'open' },
{ type: 'AND' },
- { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },
+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high' },
{ type: 'EOF' },
],
},
{
query: `type:feature AND (status:open OR (priority:high -tag:"full review")) "user feedback"`,
expectedTokens: [
- { type: 'FILTER', field: 'type', operator: '=', value: 'feature', negated: false },
+ { type: 'FILTER', field: 'type', operator: '=', value: 'feature' },
{ type: 'AND' },
{ type: 'LPAREN' },
- { type: 'FILTER', field: 'status', operator: '=', value: 'open', negated: false },
+ { type: 'FILTER', field: 'status', operator: '=', value: 'open' },
{ type: 'OR' },
{ type: 'LPAREN' },
- { type: 'FILTER', field: 'priority', operator: '=', value: 'high', negated: false },
- { type: 'FILTER', field: 'tag', operator: '=', value: 'full review', negated: true },
+ { type: 'FILTER', field: 'priority', operator: '=', value: 'high' },
+ { type: 'NOT' },
+ { type: 'FILTER', field: 'tag', operator: '=', value: 'full review' },
{ type: 'RPAREN' },
{ type: 'RPAREN' },
{ type: 'TEXT', value: 'user feedback' },
{ type: 'EOF' },
],
},
+ {
+ query: `tag:bug\\:critical`,
+ expectedTokens: [
+ { type: 'FILTER', field: 'tag', operator: '=', value: 'bug:critical' },
+ { type: 'EOF' },
+ ],
+ },
+ {
+ query: `tag:-important tag:"-important" tag:\\-important`,
+ expectedTokens: [
+ { type: 'FILTER', field: 'tag', operator: '=', value: '-important' },
+ { type: 'FILTER', field: 'tag', operator: '=', value: '-important' },
+ { type: 'FILTER', field: 'tag', operator: '=', value: '\\-important' },
+ { type: 'EOF' },
+ ],
+ },
];
for (const { query, expectedTokens, maxTokens = 100 } of queries) {
diff --git a/packages/search-parser/src/tokenizer.ts b/packages/search-parser/src/tokenizer.ts
index 471cd860..f51e6b9e 100644
--- a/packages/search-parser/src/tokenizer.ts
+++ b/packages/search-parser/src/tokenizer.ts
@@ -1,5 +1,6 @@
import type { Issue, Operator } from './parser.types';
import { ERROR_CODES } from './errors';
+import { isWhitespace } from './string';
export type Token
= | { type: 'LPAREN' }
@@ -7,7 +8,7 @@ export type Token
| { type: 'AND' }
| { type: 'OR' }
| { type: 'NOT' }
- | { type: 'FILTER'; field: string; operator: Operator; value: string; negated: boolean }
+ | { type: 'FILTER'; field: string; operator: Operator; value: string }
| { type: 'TEXT'; value: string }
| { type: 'EOF' };
@@ -16,6 +17,16 @@ export type TokenizeResult = {
issues: Issue[];
};
+// Unified escape handling utilities
+const unescapeBackslashes = (str: string): string => str.replace(/\\(.)/g, '$1');
+const unescapeColons = (str: string): string => str.replace(/\\:/g, ':');
+
+type StopCondition = (char: string) => boolean;
+
+function isWhitespaceOrParen(char: string): boolean {
+ return isWhitespace(char) || char === '(' || char === ')';
+}
+
export function tokenize({ query, maxTokens }: { query: string; maxTokens: number }): TokenizeResult {
const tokens: Token[] = [];
const issues: Issue[] = [];
@@ -27,7 +38,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
const skipWhitespace = (): void => {
while (pos < query.length) {
const char = query[pos];
- if (!char || !/\s/.test(char)) {
+ if (!char || !isWhitespace(char)) {
break;
}
pos++;
@@ -73,28 +84,51 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
return value;
};
- const readUnquotedToken = (): string => {
+ // Unified function to read unquoted values with configurable stop conditions and escape handling
+ const readUnquotedValue = ({
+ stopCondition,
+ processEscapes,
+ stopAtQuote = false,
+ }: {
+ stopCondition: StopCondition;
+ processEscapes: boolean;
+ stopAtQuote?: boolean;
+ }): string => {
let value = '';
while (pos < query.length) {
const char = peek();
- if (!char || /\s/.test(char) || char === '(' || char === ')') {
+ if (!char || stopCondition(char)) {
break;
}
- // Stop at quote
- if (char === '"') {
+ if (stopAtQuote && char === '"') {
break;
}
- // Keep backslashes in unquoted tokens so parseFilter can detect escaped colons
- value += advance();
+ if (processEscapes && char === '\\') {
+ advance();
+ if (pos < query.length) {
+ value += advance();
+ }
+ } else {
+ value += advance();
+ }
}
return value;
};
+ const readUnquotedToken = (): string => {
+ // Keep backslashes in unquoted tokens so parseFilter can detect escaped colons
+ return readUnquotedValue({
+ stopCondition: isWhitespaceOrParen,
+ processEscapes: false,
+ stopAtQuote: true,
+ });
+ };
+
const readFilterValue = (): string => {
skipWhitespace();
@@ -104,26 +138,13 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
return quotedValue;
}
- // Read unquoted value (up to whitespace or special chars)
- let value = '';
- while (pos < query.length) {
- const char = peek();
-
- if (!char || /\s/.test(char) || char === '(' || char === ')') {
- break;
- }
-
- if (char === '\\') {
- advance();
- if (pos < query.length) {
- value += advance();
- }
- } else {
- value += advance();
- }
- }
+ // Read unquoted value with escape processing, then unescape colons
+ const value = readUnquotedValue({
+ stopCondition: isWhitespaceOrParen,
+ processEscapes: true,
+ });
- return value.replace(/\\:/g, ':');
+ return unescapeColons(value);
};
const hasUnescapedColon = (str: string): boolean => {
@@ -135,7 +156,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
return false;
};
- const parseFilter = (token: string, negated: boolean): Token | undefined => {
+ const parseFilter = (token: string): Token | undefined => {
// Check for unescaped colons
if (!hasUnescapedColon(token)) {
return undefined;
@@ -154,7 +175,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
return undefined;
}
- const field = token.slice(0, firstColonIndex).replace(/\\:/g, ':');
+ const field = unescapeColons(token.slice(0, firstColonIndex));
const afterColon = token.slice(firstColonIndex + 1);
// Check for operator at the start
@@ -179,14 +200,14 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
}
// If there's nothing after the operator in the token, read the value from input
- let value = afterColon.slice(operatorLength).replace(/\\:/g, ':');
+ let value = unescapeColons(afterColon.slice(operatorLength));
// If the value is empty and we still have input, this might be a case like "tag:" followed by a quoted string
if (!value) {
value = readFilterValue();
}
- return { type: 'FILTER', field, operator, value, negated };
+ return { type: 'FILTER', field, operator, value };
};
while (pos < query.length) {
@@ -221,7 +242,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
// Handle negation prefix
const nextChar = query[pos + 1];
- if (char === '-' && nextChar && !/\s/.test(nextChar)) {
+ if (char === '-' && nextChar && !isWhitespace(nextChar)) {
advance();
skipWhitespace();
@@ -230,14 +251,14 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
const token = quotedValue !== undefined ? quotedValue : readUnquotedToken();
// Try to parse as filter
- const filter = parseFilter(token, true);
+ const filter = parseFilter(token);
if (filter) {
+ tokens.push({ type: 'NOT' });
tokens.push(filter);
} else {
- // If not a filter, treat as negated text search (which we'll handle as NOT operator)
- const unescapedText = token.replace(/\\(.)/g, '$1');
+ // If not a filter, treat as negated text search
tokens.push({ type: 'NOT' });
- tokens.push({ type: 'TEXT', value: unescapedText });
+ tokens.push({ type: 'TEXT', value: unescapeBackslashes(token) });
}
continue;
}
@@ -270,7 +291,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
// Try to parse as filter (only if not quoted)
if (quotedValue === undefined) {
- const filter = parseFilter(token, false);
+ const filter = parseFilter(token);
if (filter) {
tokens.push(filter);
continue;
@@ -278,8 +299,7 @@ export function tokenize({ query, maxTokens }: { query: string; maxTokens: numbe
}
// Otherwise, treat as text (unescape backslashes)
- const unescapedText = token.replace(/\\(.)/g, '$1');
- tokens.push({ type: 'TEXT', value: unescapedText });
+ tokens.push({ type: 'TEXT', value: unescapeBackslashes(token) });
}
tokens.push({ type: 'EOF' });