Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

refactor(search-parser): treat text as first-class expressions in AST (#744)

Corentin Thomasset38bfa010f870
diff --git a/.gitignore b/.gitignore
index f25b4c9e..88735271 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,4 +44,5 @@ ingestion
*.traineddata
.eslintcache
-.claude
\ No newline at end of file
+.claude
+.opencode
\ No newline at end of file
diff --git a/packages/search-parser/src/index.ts b/packages/search-parser/src/index.ts
index 680e0b56..25406bfa 100644
--- a/packages/search-parser/src/index.ts
+++ b/packages/search-parser/src/index.ts
@@ -15,4 +15,5 @@ export type {
Operator,
OrExpression,
ParsedQuery,
+ TextExpression,
} from './parser.types';
diff --git a/packages/search-parser/src/parser.test.ts b/packages/search-parser/src/parser.test.ts
index f6c182d0..887083b6 100644
--- a/packages/search-parser/src/parser.test.ts
+++ b/packages/search-parser/src/parser.test.ts
@@ -6,39 +6,40 @@ describe('parseSearchQuery', () => {
test('returns empty expression and no search terms for empty query', () => {
expect(parseSearchQuery({ query: '' })).toEqual({
expression: { type: 'and', operands: [] },
- search: undefined,
issues: [],
});
});
test('extracts full-text search terms when no filters present', () => {
expect(parseSearchQuery({ query: 'my invoice' })).toEqual({
- expression: { type: 'and', operands: [] },
- search: 'my invoice',
+ expression: {
+ type: 'and',
+ operands: [
+ { type: 'text', value: 'my' },
+ { type: 'text', value: 'invoice' },
+ ],
+ },
issues: [],
});
});
test('handles quoted search terms', () => {
expect(parseSearchQuery({ query: '"my special invoice"' })).toEqual({
- expression: { type: 'and', operands: [] },
- search: 'my special invoice',
+ expression: { type: 'text', value: 'my special invoice' },
issues: [],
});
});
test('handles escaped quotes in search terms', () => {
expect(parseSearchQuery({ query: '"my \\"special\\" invoice"' })).toEqual({
- expression: { type: 'and', operands: [] },
- search: 'my "special" invoice',
+ expression: { type: 'text', value: 'my "special" invoice' },
issues: [],
});
});
test('handles escaped colons to prevent filter parsing', () => {
expect(parseSearchQuery({ query: 'tag\\:invoice' })).toEqual({
- expression: { type: 'and', operands: [] },
- search: 'tag:invoice',
+ expression: { type: 'text', value: 'tag:invoice' },
issues: [],
});
});
@@ -53,7 +54,6 @@ describe('parseSearchQuery', () => {
operator: '=',
value: 'invoice',
},
- search: undefined,
issues: [],
});
});
@@ -66,7 +66,6 @@ describe('parseSearchQuery', () => {
operator: '=',
value: 'invoice',
},
- search: undefined,
issues: [],
});
});
@@ -79,7 +78,6 @@ describe('parseSearchQuery', () => {
operator: '=',
value: 'my invoices',
},
- search: undefined,
issues: [],
});
});
@@ -92,7 +90,6 @@ describe('parseSearchQuery', () => {
operator: '=',
value: 'my "special" invoices',
},
- search: undefined,
issues: [],
});
});
@@ -105,7 +102,6 @@ describe('parseSearchQuery', () => {
operator: '=',
value: 'my::special::tag',
},
- search: undefined,
issues: [],
});
});
@@ -118,7 +114,6 @@ describe('parseSearchQuery', () => {
operator: '=',
value: 'value',
},
- search: undefined,
issues: [],
});
});
@@ -131,7 +126,6 @@ describe('parseSearchQuery', () => {
operator: '>',
value: '2024-01-01',
},
- search: undefined,
issues: [],
});
@@ -142,7 +136,6 @@ describe('parseSearchQuery', () => {
operator: '<',
value: '2024-01-01',
},
- search: undefined,
issues: [],
});
@@ -153,7 +146,6 @@ describe('parseSearchQuery', () => {
operator: '>=',
value: '2024-01-01',
},
- search: undefined,
issues: [],
});
@@ -164,7 +156,6 @@ describe('parseSearchQuery', () => {
operator: '<=',
value: '2024-01-01',
},
- search: undefined,
issues: [],
});
});
@@ -180,7 +171,6 @@ describe('parseSearchQuery', () => {
value: 'personal',
},
},
- search: undefined,
issues: [],
});
});
@@ -190,12 +180,13 @@ describe('parseSearchQuery', () => {
test('combines filter with search terms', () => {
expect(parseSearchQuery({ query: 'tag:invoice my invoice' })).toEqual({
expression: {
- type: 'filter',
- field: 'tag',
- operator: '=',
- value: 'invoice',
+ type: 'and',
+ operands: [
+ { type: 'filter', field: 'tag', operator: '=', value: 'invoice' },
+ { type: 'text', value: 'my' },
+ { type: 'text', value: 'invoice' },
+ ],
},
- search: 'my invoice',
issues: [],
});
});
@@ -203,12 +194,13 @@ describe('parseSearchQuery', () => {
test('extracts search terms before and after filters', () => {
expect(parseSearchQuery({ query: 'foo tag:invoice bar' })).toEqual({
expression: {
- type: 'filter',
- field: 'tag',
- operator: '=',
- value: 'invoice',
+ type: 'and',
+ operands: [
+ { type: 'text', value: 'foo' },
+ { type: 'filter', field: 'tag', operator: '=', value: 'invoice' },
+ { type: 'text', value: 'bar' },
+ ],
},
- search: 'foo bar',
issues: [],
});
});
@@ -224,7 +216,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'receipt' },
],
},
- search: undefined,
issues: [],
});
});
@@ -238,7 +229,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'receipt' },
],
},
- search: undefined,
issues: [],
});
});
@@ -253,7 +243,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'status', operator: '=', value: 'active' },
],
},
- search: undefined,
issues: [],
});
});
@@ -269,7 +258,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'receipt' },
],
},
- search: undefined,
issues: [],
});
});
@@ -284,7 +272,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'bill' },
],
},
- search: undefined,
issues: [],
});
});
@@ -302,7 +289,6 @@ describe('parseSearchQuery', () => {
value: 'personal',
},
},
- search: undefined,
issues: [],
});
});
@@ -319,7 +305,6 @@ describe('parseSearchQuery', () => {
],
},
},
- search: undefined,
issues: [],
});
});
@@ -335,7 +320,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'receipt' },
],
},
- search: undefined,
issues: [],
});
});
@@ -355,7 +339,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'createdAt', operator: '>', value: '2024-01-01' },
],
},
- search: undefined,
issues: [],
});
});
@@ -375,7 +358,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'status', operator: '=', value: 'active' },
],
},
- search: undefined,
issues: [],
});
});
@@ -397,7 +379,6 @@ describe('parseSearchQuery', () => {
},
],
},
- search: undefined,
issues: [],
});
});
@@ -414,7 +395,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'receipt' },
],
},
- search: undefined,
issues: [],
});
});
@@ -434,7 +414,6 @@ describe('parseSearchQuery', () => {
{ type: 'filter', field: 'tag', operator: '=', value: 'company' },
],
},
- search: undefined,
issues: [],
});
});
@@ -443,7 +422,10 @@ describe('parseSearchQuery', () => {
describe('when handling malformed queries', () => {
test('handles unclosed quoted string gracefully', () => {
const result = parseSearchQuery({ query: '"unclosed string' });
- expect(result.search).toBe('unclosed string');
+ expect(result.expression).toEqual({
+ type: 'text',
+ value: 'unclosed string',
+ });
expect(result.issues).toContainEqual({
code: 'unclosed-quoted-string',
message: 'Unclosed quoted string',
@@ -535,9 +517,10 @@ describe('parseSearchQuery', () => {
type: 'not',
operand: { type: 'filter', field: 'tag', operator: '=', value: 'personal' },
},
+ { type: 'text', value: 'my' },
+ { type: 'text', value: 'document' },
],
},
- search: 'my document',
issues: [],
});
});
@@ -561,7 +544,134 @@ describe('parseSearchQuery', () => {
},
],
},
- search: undefined,
+ issues: [],
+ });
+ });
+ });
+
+ describe('when parsing text with boolean operators', () => {
+ test('parses text OR text as OR expression', () => {
+ expect(parseSearchQuery({ query: 'foo OR bar' })).toEqual({
+ expression: {
+ type: 'or',
+ operands: [
+ { type: 'text', value: 'foo' },
+ { type: 'text', value: 'bar' },
+ ],
+ },
+ issues: [],
+ });
+ });
+
+ test('parses text with implicit AND correctly', () => {
+ expect(parseSearchQuery({ query: 'foo bar baz' })).toEqual({
+ expression: {
+ type: 'and',
+ operands: [
+ { type: 'text', value: 'foo' },
+ { type: 'text', value: 'bar' },
+ { type: 'text', value: 'baz' },
+ ],
+ },
+ issues: [],
+ });
+ });
+
+ test('negated text creates NOT expression', () => {
+ expect(parseSearchQuery({ query: '-foo' })).toEqual({
+ expression: {
+ type: 'not',
+ operand: { type: 'text', value: 'foo' },
+ },
+ issues: [],
+ });
+ });
+
+ test('handles complex text with operators', () => {
+ expect(
+ parseSearchQuery({
+ query: '(tag:invoice foo) OR (tag:receipt bar)',
+ }),
+ ).toEqual({
+ expression: {
+ type: 'or',
+ operands: [
+ {
+ type: 'and',
+ operands: [
+ { type: 'filter', field: 'tag', operator: '=', value: 'invoice' },
+ { type: 'text', value: 'foo' },
+ ],
+ },
+ {
+ type: 'and',
+ operands: [
+ { type: 'filter', field: 'tag', operator: '=', value: 'receipt' },
+ { type: 'text', value: 'bar' },
+ ],
+ },
+ ],
+ },
+ issues: [],
+ });
+ });
+
+ test('parses multiple ORs with text and filters', () => {
+ expect(parseSearchQuery({ query: 'foo OR bar OR baz' })).toEqual({
+ expression: {
+ type: 'or',
+ operands: [
+ { type: 'text', value: 'foo' },
+ { type: 'text', value: 'bar' },
+ { type: 'text', value: 'baz' },
+ ],
+ },
+ issues: [],
+ });
+ });
+
+ test('parses quoted text with spaces', () => {
+ expect(parseSearchQuery({ query: '"hello world"' })).toEqual({
+ expression: { type: 'text', value: 'hello world' },
+ issues: [],
+ });
+ });
+
+ test('parses negated quoted text', () => {
+ expect(parseSearchQuery({ query: '-"hello world"' })).toEqual({
+ expression: {
+ type: 'not',
+ operand: { type: 'text', value: 'hello world' },
+ },
+ issues: [],
+ });
+ });
+
+ test('parses text mixed with explicit NOT operator', () => {
+ expect(parseSearchQuery({ query: 'NOT foo' })).toEqual({
+ expression: {
+ type: 'not',
+ operand: { type: 'text', value: 'foo' },
+ },
+ issues: [],
+ });
+ });
+
+ test('respects AND precedence over OR in text expressions', () => {
+ expect(parseSearchQuery({ query: 'foo OR bar AND baz' })).toEqual({
+ expression: {
+ type: 'or',
+ operands: [
+ { type: 'text', value: 'foo' },
+ {
+ type: 'and',
+ operands: [
+ { type: 'text', value: 'bar' },
+ { type: 'text', value: 'baz' },
+ ],
+ },
+ ],
+ },
issues: [],
});
});
diff --git a/packages/search-parser/src/parser.ts b/packages/search-parser/src/parser.ts
index d3979062..24ccb1a1 100644
--- a/packages/search-parser/src/parser.ts
+++ b/packages/search-parser/src/parser.ts
@@ -16,17 +16,15 @@ export function parseSearchQuery(
): ParsedQuery {
const { tokens, issues: tokenizerIssues } = tokenize({ query, maxTokens });
- const { expression, issues: parserIssues, search } = parseExpression({ tokens, maxDepth });
+ const { expression, issues: parserIssues } = parseExpression({ tokens, maxDepth });
return {
expression,
- search,
issues: [...tokenizerIssues, ...parserIssues],
};
}
function parseExpression({ tokens, maxDepth }: { tokens: Token[]; maxDepth: number }): ParsedQuery {
- const searchTerms: string[] = [];
const parserIssues: Issue[] = [];
let currentTokenIndex = 0;
@@ -91,8 +89,10 @@ function parseExpression({ tokens, maxDepth }: { tokens: Token[]; maxDepth: numb
if (token.type === 'TEXT') {
advance();
- searchTerms.push(token.value);
- return undefined; // Text doesn't produce an expression
+ return {
+ type: 'text',
+ value: token.value,
+ };
}
return undefined;
@@ -198,11 +198,9 @@ function parseExpression({ tokens, maxDepth }: { tokens: Token[]; maxDepth: numb
// Build final result
const finalExpression: Expression = expression ?? { type: 'and', operands: [] };
- const search = searchTerms.length > 0 ? searchTerms.join(' ') : undefined;
return {
expression: finalExpression,
- search,
issues: parserIssues,
};
}
diff --git a/packages/search-parser/src/parser.types.ts b/packages/search-parser/src/parser.types.ts
index caba3469..92466813 100644
--- a/packages/search-parser/src/parser.types.ts
+++ b/packages/search-parser/src/parser.types.ts
@@ -5,7 +5,6 @@ export type Issue = {
export type ParsedQuery = {
expression: Expression;
- search: string | undefined;
issues: Issue[];
};
@@ -13,7 +12,8 @@ export type Expression
= | AndExpression
| OrExpression
| NotExpression
- | FilterExpression;
+ | FilterExpression
+ | TextExpression;
export type AndExpression = {
type: 'and';
@@ -38,3 +38,8 @@ export type FilterExpression = {
operator: Operator;
value: string;
};
+
+export type TextExpression = {
+ type: 'text';
+ value: string;
+};