Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: fix invalid search queries & support short terms in boolean queries (#10517)

Abdullah Atta1dcb77f3b95a
diff --git a/packages/core/__tests__/lookup.test.js b/packages/core/__tests__/lookup.test.js
index 1ba083674..e5b8867c5 100644
--- a/packages/core/__tests__/lookup.test.js
+++ b/packages/core/__tests__/lookup.test.js
@@ -46,6 +46,48 @@ test("search notes", () =>
expect(await db.lookup.notes("hb kb ch").ids()).toHaveLength(1);
}));
+test("search notes with unbalanced quotes", () =>
+ noteTest({
+ content: { type: "tiptap", data: `<p>he said "hello" to the world</p>` }
+ }).then(async ({ db }) => {
+ expect(await db.lookup.notes('"hello').ids()).toHaveLength(1);
+ expect(await db.lookup.notes('hello"').ids()).toHaveLength(1);
+ expect(await db.lookup.notes('said "hello').ids()).toHaveLength(1);
+ }));
+
+test("search notes with short terms in boolean queries", () =>
+ databaseTest().then(async (db) => {
+ const notes = {
+ world: await db.notes.add({
+ title: "first",
+ content: { type: "tiptap", data: "<p>hello world</p>" }
+ }),
+ onyx: await db.notes.add({
+ title: "second",
+ content: { type: "tiptap", data: "<p>the onyx stone</p>" }
+ }),
+ helloOnyx: await db.notes.add({
+ title: "third",
+ content: { type: "tiptap", data: "<p>hello onyx</p>" }
+ }),
+ title: await db.notes.add({
+ title: "yx in title",
+ content: { type: "tiptap", data: "<p>nothing here</p>" }
+ })
+ };
+ const search = async (query) => (await db.lookup.notes(query).ids()).sort();
+ const ids = (...keys) => keys.map((key) => notes[key]).sort();
+
+ expect(await search("yx")).toEqual(ids("onyx", "helloOnyx", "title"));
+ expect(await search("yx OR world")).toEqual(
+ ids("world", "onyx", "helloOnyx", "title")
+ );
+ expect(await search("yx hello")).toEqual(ids("helloOnyx"));
+ expect(await search("hello NOT yx")).toEqual(ids("world"));
+ expect(await search("yx NOT stone")).toEqual(ids("helloOnyx", "title"));
+ expect(await search("hello yx OR stone")).toEqual(ids("onyx", "helloOnyx"));
+ }));
+
test("search notes (remove diacritics)", () =>
noteTest({
content: {
@@ -207,6 +249,49 @@ describe("notesWithHighlighting", () => {
expect(item.content[0][0].prefix).toBe("<img src=x onerror=alert(1)> ");
}));
+ test("search notes with unbalanced quotes should highlight the match", () =>
+ noteTest({
+ title: "quotes",
+ content: {
+ type: "tiptap",
+ data: `<p>he said "hello" to the world</p>`
+ }
+ }).then(async ({ db }) => {
+ for (const query of ['"hello', 'hello"']) {
+ const filtered = await db.lookup.notesWithHighlighting(
+ query,
+ db.notes.all
+ );
+ const item = await filtered.item(0);
+ expect(item.item).toBeDefined();
+ expect(item.item.content.length).toBeGreaterThan(0);
+ }
+ }));
+
+ test("search notes with short terms in boolean queries", () =>
+ databaseTest().then(async (db) => {
+ await db.notes.add({
+ title: "first",
+ content: { type: "tiptap", data: "<p>hello world</p>" }
+ });
+ await db.notes.add({
+ title: "second",
+ content: { type: "tiptap", data: "<p>the onyx stone</p>" }
+ });
+
+ const filtered = await db.lookup.notesWithHighlighting(
+ "yx OR world",
+ db.notes.all
+ );
+ expect(filtered.length).toBe(2);
+ const matches = [];
+ for (let i = 0; i < filtered.length; i++) {
+ const { item } = await filtered.item(i);
+ matches.push(item.content[0][0].match);
+ }
+ expect(matches.sort()).toEqual(["world", "yx"]);
+ }));
+
test("search should be diacritic agnostic", () =>
databaseTest().then(async (db) => {
await db.notes.add({ title: "outdoor café" });
diff --git a/packages/core/__tests__/search-fuzz.test.ts b/packages/core/__tests__/search-fuzz.test.ts
new file mode 100644
index 000000000..434019f25
--- /dev/null
+++ b/packages/core/__tests__/search-fuzz.test.ts
@@ -0,0 +1,102 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { databaseTest } from "./utils/index.js";
+import { logger } from "../src/logger.js";
+import { test, expect, vi } from "vitest";
+
+const SEED = 0x6e6e;
+const ITERATIONS = 1000;
+const VOCABULARY = [
+ // terms
+ "ab",
+ "x",
+ "hello",
+ "world",
+ "foo*",
+ "钢",
+ "é",
+ "a.b",
+ "x:y",
+ "^ab",
+ "{x}",
+ "-x",
+ "*",
+ "(",
+ ")",
+ "“smart”",
+ "’",
+ "–",
+ "NEAR",
+ "NEAR(",
+ "hello\tworld",
+ "a\nb",
+ // operators
+ "AND",
+ "OR",
+ "NOT",
+ // quotes
+ '"',
+ '""',
+ 'a"b',
+ '"foo',
+ 'foo"',
+ '"x y"',
+ // fields
+ "title:",
+ "content:"
+];
+
+// mulberry32
+function random(seed: number) {
+ return () => {
+ seed = (seed + 0x6d2b79f5) | 0;
+ let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+}
+
+test("search should not fail for any query", () =>
+ databaseTest().then(async (db) => {
+ await db.notes.add({
+ title: "hello world",
+ content: { type: "tiptap", data: "<p>hello world ab x 钢</p>" }
+ });
+ const errors: string[] = [];
+ const spy = vi
+ .spyOn(logger, "error")
+ .mockImplementation((_e, _message, context) => {
+ errors.push((context as { query: string }).query);
+ });
+
+ const next = random(SEED);
+ for (let i = 0; i < ITERATIONS; ++i) {
+ const length = 1 + Math.floor(next() * 5);
+ const query = Array.from(
+ { length },
+ () => VOCABULARY[Math.floor(next() * VOCABULARY.length)]
+ ).join(" ");
+ await db.lookup.notes(query).ids();
+ await db.lookup.notesWithHighlighting(query, db.notes.all);
+ }
+
+ spy.mockRestore();
+ expect(errors).toEqual([]);
+ }));
diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.ts
index c502d46ac..5f6981be0 100644
--- a/packages/core/src/api/lookup.ts
+++ b/packages/core/src/api/lookup.ts
@@ -33,16 +33,24 @@ import {
import { DatabaseSchema, RawDatabaseSchema } from "../database/index.js";
import {
AnyColumnWithTable,
+ Expression,
ExpressionBuilder,
Kysely,
SelectQueryBuilder,
+ SqlBool,
sql
} from "@streetwriters/kysely";
import { FilteredSelector } from "../database/sql-collection.js";
import { VirtualizedGrouping } from "../utils/virtualized-grouping.js";
import { logger } from "../logger.js";
import { rebuildSearchIndex } from "../database/fts.js";
-import { QueryTokens, transformQuery } from "../utils/query-transformer.js";
+import {
+ QueryExpression,
+ QueryTokens,
+ hasShortTerms,
+ serializeExpression,
+ transformQuery
+} from "../utils/query-transformer.js";
import { getSortSelectors, groupArray } from "../utils/grouping.js";
import { fuzzy } from "../utils/fuzzy.js";
import { extractMatchingBlocks } from "../utils/html-parser.js";
@@ -62,6 +70,12 @@ type FuzzySearchField<T> = {
ignore?: boolean;
};
+type SearchMatch = {
+ id: string;
+ rank: number;
+ type: "content" | "title";
+};
+
const MATCH_TAG_NAME = "nn-search-result";
const MATCH_TAG_REGEX = new RegExp(
`<${MATCH_TAG_NAME}\\s+id="(.+?)">(.*?)<\\/${MATCH_TAG_NAME}>`,
@@ -78,7 +92,7 @@ export default class Lookup {
const ftsResults: string[] =
(await this.ftsQueryBuilder(
- { content: content?.query, title: title?.query },
+ { content: content?.expression, title: title?.expression },
excludedIds,
notes
)
@@ -92,18 +106,7 @@ export default class Lookup {
})
.then((r) => r.map((r) => r.id))) || [];
- const regexMatches = await this.regexQueryBuilder(
- {
- content: filterSmallTokens(content?.tokens),
- title: filterSmallTokens(title?.tokens)
- },
- (!!content || !!title) && ftsResults.length > 0 ? ftsResults : notes
- )
- ?.select("results.id")
- .execute();
- if (!regexMatches) return ftsResults;
-
- return regexMatches.map((r) => r.id);
+ return ftsResults;
}, notes || this.db.notes.all);
}
@@ -221,7 +224,7 @@ export default class Lookup {
console.time("gather matches");
const ftsResults =
(await this.ftsQueryBuilder(
- { content: content?.query, title: title?.query },
+ { content: content?.expression, title: title?.expression },
excludedIds,
notes
)
@@ -231,18 +234,6 @@ export default class Lookup {
logger.error(e, `Error while searching`, { query });
return [];
})) || [];
-
- const ftsIds = ftsResults.map((r) => r.id);
- const regexMatches =
- (await this.regexQueryBuilder(
- {
- content: filterSmallTokens(content?.tokens),
- title: filterSmallTokens(title?.tokens)
- },
- (!!content || !!title) && ftsIds.length > 0 ? ftsIds : notes
- )
- ?.select(["results.id", "results.type", sql<number>`1`.as("rank")])
- .execute()) || [];
console.timeEnd("gather matches");
console.time("sorting matches");
@@ -251,25 +242,24 @@ export default class Lookup {
values: { id: string; types: string[]; rank: number }[];
};
let matches: Matches = { ids: [], values: [] };
- for (const array of [ftsResults, regexMatches])
- for (const { id, type, rank } of array) {
- const index = matches.ids.indexOf(id);
- const match =
- index === -1
- ? {
- id,
- types: [],
- rank: 0
- }
- : matches.values[index];
- match.types.push(type);
- match.rank += rank || 0;
+ for (const { id, type, rank } of ftsResults) {
+ const index = matches.ids.indexOf(id);
+ const match =
+ index === -1
+ ? {
+ id,
+ types: [],
+ rank: 0
+ }
+ : matches.values[index];
+ match.types.push(type);
+ match.rank += rank || 0;
- if (index === -1) {
- matches.ids.push(id);
- matches.values.push(match);
- }
+ if (index === -1) {
+ matches.ids.push(id);
+ matches.values.push(match);
}
+ }
if (!sortOptions || sortOptions.sortBy === "relevance") {
matches.values.sort(
@@ -432,8 +422,8 @@ export default class Lookup {
private ftsQueryBuilder(
queries: {
- title?: string;
- content?: string;
+ title?: QueryExpression;
+ content?: QueryExpression;
},
excludedIds: string[] = [],
filter?: FilteredSelector<Note>
@@ -445,8 +435,43 @@ export default class Lookup {
function buildTitleQuery(
eb:
| Kysely<RawDatabaseSchema>
- | ExpressionBuilder<RawDatabaseSchema, never>
+ | ExpressionBuilder<RawDatabaseSchema, never>,
+ expression: QueryExpression
) {
+ if (hasShortTerms(expression))
+ return eb
+ .selectFrom("notes")
+ .$if(!!filter, (eb) =>
+ eb.where("id", "in", filter!.filter.select("id"))
+ )
+ .$if(excludedIds.length > 0, (eb) =>
+ eb.where("id", "not in", excludedIds)
+ )
+ .where("title", "is not", null)
+ .where("deleted", "is not", true)
+ .where((eb) =>
+ compileExpression(
+ eb,
+ expression,
+ (query) =>
+ eb(
+ "notes.id",
+ "in",
+ eb
+ .selectFrom("notes_fts")
+ .where("title", "match", query)
+ .select("id")
+ ),
+ (regex) => eb("notes.title", "regexp", sql<string>`${regex}`)
+ )
+ )
+ .select([
+ "id",
+ sql<number>`0`.as("rank"),
+ sql<string>`'title'`.as("type")
+ ])
+ .$castTo<SearchMatch>();
+
return eb
.selectFrom("notes_fts")
.$if(!!filter, (eb) =>
@@ -455,16 +480,54 @@ export default class Lookup {
.$if(excludedIds.length > 0, (eb) =>
eb.where("id", "not in", excludedIds)
)
- .where("title", "match", queries.title)
+ .where("title", "match", serializeExpression(expression))
.where("rank", "=", sql<number>`'bm25(1.0, 10.0)'`)
- .select(["id", "rank", sql<string>`'title'`.as("type")]);
+ .select(["id", "rank", sql<string>`'title'`.as("type")])
+ .$castTo<SearchMatch>();
}
function buildContentQuery(
eb:
| Kysely<RawDatabaseSchema>
- | ExpressionBuilder<RawDatabaseSchema, never>
+ | ExpressionBuilder<RawDatabaseSchema, never>,
+ expression: QueryExpression
) {
+ if (hasShortTerms(expression))
+ return eb
+ .selectFrom("content")
+ .$if(!!filter, (eb) =>
+ eb.where("noteId", "in", filter!.filter.select("id"))
+ )
+ .$if(excludedIds.length > 0, (eb) =>
+ eb.where("noteId", "not in", excludedIds)
+ )
+ .where("noteId", "is not", null)
+ .where("data", "is not", null)
+ .where("deleted", "is not", true)
+ .where("locked", "is not", true)
+ .where((eb) =>
+ compileExpression(
+ eb,
+ expression,
+ (query) =>
+ eb(
+ "content.noteId",
+ "in",
+ eb
+ .selectFrom("content_fts")
+ .where("data", "match", query)
+ .select("noteId")
+ ),
+ (regex) => eb("content.data", "regexp", sql<string>`${regex}`)
+ )
+ )
+ .select([
+ "noteId as id",
+ sql<number>`0`.as("rank"),
+ sql<string>`'content'`.as("type")
+ ])
+ .$castTo<SearchMatch>();
+
return eb
.selectFrom("content_fts")
.$if(!!filter, (eb) =>
@@ -473,100 +536,26 @@ export default class Lookup {
.$if(excludedIds.length > 0, (eb) =>
eb.where("noteId", "not in", excludedIds)
)
- .where("data", "match", queries.content)
+ .where("data", "match", serializeExpression(expression))
.where("rank", "=", sql<number>`'bm25(1.0, 1.0, 10.0)'`)
.select(["noteId as id", "rank", sql<string>`'content'`.as("type")])
- .$castTo<{
- id: string;
- rank: number;
- type: "content" | "title";
- }>();
+ .$castTo<SearchMatch>();
}
if (queries.content && queries.title)
return db.selectFrom((eb) =>
- buildTitleQuery(eb)
- .unionAll((eb) => buildContentQuery(eb))
- .as("results")
- );
- else if (queries.content)
- return db.selectFrom((eb) => buildContentQuery(eb).as("results"));
- else if (queries.title)
- return db.selectFrom((eb) => buildTitleQuery(eb).as("results"));
- }
-
- private regexQueryBuilder(
- queries: {
- title?: string[];
- content?: string[];
- },
- ids?: string[] | FilteredSelector<Note>
- ) {
- if (!queries.content?.length && !queries.title?.length) return;
-
- const buildRegex = (queries: string[]) =>
- queries
- .filter((q) => q && q.length > 0)
- .map((q) => q.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
- .join("|");
-
- function buildTitleQuery(
- eb: Kysely<DatabaseSchema> | ExpressionBuilder<DatabaseSchema, never>,
- queries: string[]
- ) {
- const regex = buildRegex(queries);
- return eb
- .selectFrom("notes")
- .$if(!!ids, (eb) =>
- eb.where(
- "id",
- "in",
- Array.isArray(ids) ? ids! : ids!.filter.select("id")
- )
- )
- .where("title", "regexp", sql<string>`${regex}`)
- .select(["id", sql<string>`'title'`.as("type")]);
- }
-
- function buildContentQuery(
- eb: Kysely<DatabaseSchema> | ExpressionBuilder<DatabaseSchema, never>,
- queries: string[]
- ) {
- const regex = buildRegex(queries);
- return eb
- .selectFrom("content")
- .where("content.locked", "!=", true)
- .$if(!!ids, (eb) =>
- eb.where(
- "noteId",
- "in",
- Array.isArray(ids) ? ids! : ids!.filter.select("id")
- )
- )
- .where("data", "regexp", sql<string>`${regex}`)
- .select(["noteId as id", sql<string>`'content'`.as("type")])
- .$castTo<{
- id: string;
- type: "content" | "title";
- }>();
- }
-
- if (queries.content && queries.title)
- return this.db.sql().selectFrom((eb) =>
buildTitleQuery(eb, queries.title!)
.unionAll((eb) => buildContentQuery(eb, queries.content!))
.as("results")
);
else if (queries.content)
- return this.db
- .sql()
- .selectFrom((eb) =>
- buildContentQuery(eb, queries.content!).as("results")
- );
+ return db.selectFrom((eb) =>
+ buildContentQuery(eb, queries.content!).as("results")
+ );
else if (queries.title)
- return this.db
- .sql()
- .selectFrom((eb) => buildTitleQuery(eb, queries.title!).as("results"));
+ return db.selectFrom((eb) =>
+ buildTitleQuery(eb, queries.title!).as("results")
+ );
}
notebooks(query: string) {
@@ -1155,11 +1144,11 @@ function getMatchScore(
return score;
}
-function filterSmallTokens(tokens: QueryTokens | undefined) {
- if (!tokens) return;
- return [...tokens.andTokens, ...tokens.orTokens].filter(
- (token) => token.length < 3
- );
+// tokens are escaped as FTS5 strings (e.g. `"hello` becomes `"""hello"`)
+function unquoteToken(token: string) {
+ if (token.length < 2 || !token.startsWith('"') || !token.endsWith('"'))
+ return token;
+ return token.slice(1, -1).replace(/""/g, '"');
}
function transformTokens(tokens: QueryTokens | undefined) {
@@ -1172,13 +1161,13 @@ function transformTokens(tokens: QueryTokens | undefined) {
};
const andTokens = tokens.andTokens.map((t) =>
- removeDiacritics(t.replace(/"(.+)"/g, "$1").toLowerCase())
+ removeDiacritics(unquoteToken(t).toLowerCase())
);
const orTokens = tokens.orTokens.map((t) =>
- removeDiacritics(t.replace(/"(.+)"/g, "$1").toLowerCase())
+ removeDiacritics(unquoteToken(t).toLowerCase())
);
const notTokens = tokens.notTokens.map((t) =>
- removeDiacritics(t.replace(/"(.+)"/g, "$1").toLowerCase())
+ removeDiacritics(unquoteToken(t).toLowerCase())
);
return {
andTokens,
@@ -1188,6 +1177,33 @@ function transformTokens(tokens: QueryTokens | undefined) {
};
}
+/**
+ * Compiles a query expression containing short terms into an SQL
+ * expression. Short terms are matched as substrings using regex
+ * while everything else is matched using FTS.
+ */
+function compileExpression<DB, TB extends keyof DB>(
+ eb: ExpressionBuilder<DB, TB>,
+ expression: QueryExpression,
+ match: (query: string) => Expression<SqlBool>,
+ regex: (pattern: string) => Expression<SqlBool>
+): Expression<SqlBool> {
+ if (!hasShortTerms(expression)) return match(serializeExpression(expression));
+ if (expression.type === "term")
+ return regex(expression.value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
+
+ const left = compileExpression(eb, expression.left, match, regex);
+ const right = compileExpression(eb, expression.right, match, regex);
+ switch (expression.type) {
+ case "AND":
+ return eb.and([left, right]);
+ case "OR":
+ return eb.or([left, right]);
+ case "NOT":
+ return eb.and([left, eb.not(right)]);
+ }
+}
+
function createSearchResultTag(content: string, id: string) {
return `<${MATCH_TAG_NAME} id="${id}">${content}</${MATCH_TAG_NAME}>`;
}
diff --git a/packages/core/src/utils/__tests__/query-transformer.test.ts b/packages/core/src/utils/__tests__/query-transformer.test.ts
index 5e76fbe6d..de16b9b71 100644
--- a/packages/core/src/utils/__tests__/query-transformer.test.ts
+++ b/packages/core/src/utils/__tests__/query-transformer.test.ts
@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import { expect, test } from "vitest";
-import { transformQuery } from "../query-transformer.js";
+import { serializeExpression, transformQuery } from "../query-transformer.js";
const TRANSFORM_QUERY_TESTS: [string, string | undefined][] = [
["hello world", `hello AND world`],
@@ -69,7 +69,17 @@ const TRANSFORM_QUERY_TESTS: [string, string | undefined][] = [
[">he", `">he"`],
["something<hello", `"something<hello"`],
["<", `"<"`],
- [">", `">"`]
+ [">", `">"`],
+ ["ab OR hello", `ab OR hello`],
+ ["hello ab OR world", `hello AND ab OR world`],
+ ["hello NOT ab", `hello NOT ab`],
+ ['a"b hello', `"a""b hello"`],
+ ['"foo', `"""foo"`],
+ ['foo"', `"foo"""`],
+ ['"" OR hello', `hello`],
+ ['hello OR ""', `hello`],
+ ['"" NOT hello', ``],
+ ['hello NOT ""', `hello`]
];
for (const [input, expectedOutput] of TRANSFORM_QUERY_TESTS) {
@@ -111,3 +121,33 @@ test("should separate repeated quoted tag filters", () => {
"three four"
]);
});
+
+test("should parse query with FTS5 operator precedence", () => {
+ const term = (value: string) => ({ type: "term", value });
+ expect(
+ transformQuery("hello OR ab world NOT foo").content?.expression
+ ).toEqual({
+ type: "OR",
+ left: term("hello"),
+ right: {
+ type: "AND",
+ left: term("ab"),
+ right: { type: "NOT", left: term("world"), right: term("foo") }
+ }
+ });
+});
+
+test("should add parentheses when serializing nested expressions", () => {
+ const term = (value: string) => ({ type: "term" as const, value });
+ expect(
+ serializeExpression({
+ type: "AND",
+ left: { type: "OR", left: term("foo"), right: term("bar") },
+ right: {
+ type: "NOT",
+ left: term("baz"),
+ right: { type: "OR", left: term("qux"), right: term("quux") }
+ }
+ })
+ ).toBe("(foo OR bar) AND baz NOT (qux OR quux)");
+});
diff --git a/packages/core/src/utils/query-transformer.ts b/packages/core/src/utils/query-transformer.ts
index 489a00844..16d2130b2 100644
--- a/packages/core/src/utils/query-transformer.ts
+++ b/packages/core/src/utils/query-transformer.ts
@@ -143,7 +143,7 @@ function escapeSQLString(str: string): string {
str.startsWith("%") ||
str.endsWith("%");
if (hasInvalidSymbol || isWildcard) {
- return `"${str}"`;
+ return `"${str.replace(/"/g, '""')}"`;
}
// if (isWildcard) {
@@ -304,6 +304,107 @@ function generateSQL(ast: QueryNode): string {
.join(" ");
}
+export type QueryTerm = { type: "term"; value: string };
+export type QueryExpression =
+ | QueryTerm
+ | {
+ type: OperatorNode["type"];
+ left: QueryExpression;
+ right: QueryExpression;
+ };
+
+// same as FTS5: NOT binds tighter than AND which binds tighter than OR
+const PRECEDENCE: Record<OperatorNode["type"], number> = {
+ OR: 1,
+ AND: 2,
+ NOT: 3
+};
+
+/**
+ * Terms shorter than 3 characters cannot be matched as substrings
+ * by the trigram tokenizer.
+ */
+function isShortTerm(term: QueryTerm) {
+ return term.value.length < 3;
+}
+
+export function hasShortTerms(expression: QueryExpression): boolean {
+ if (expression.type === "term") return isShortTerm(expression);
+ return hasShortTerms(expression.left) || hasShortTerms(expression.right);
+}
+
+export function serializeExpression(expression: QueryExpression): string {
+ if (expression.type === "term") return expression.value;
+
+ const precedence = PRECEDENCE[expression.type];
+ const serialize = (child: QueryExpression, needsParens: boolean) =>
+ needsParens
+ ? `(${serializeExpression(child)})`
+ : serializeExpression(child);
+ const left = serialize(
+ expression.left,
+ expression.left.type !== "term" &&
+ PRECEDENCE[expression.left.type] < precedence
+ );
+ // operators are left associative so the right operand
+ // needs parentheses even if it has the same precedence
+ const right = serialize(
+ expression.right,
+ expression.right.type !== "term" &&
+ PRECEDENCE[expression.right.type] <= precedence
+ );
+ return `${left} ${expression.type} ${right}`;
+}
+
+function buildExpression(ast: QueryNode): QueryExpression | undefined {
+ // words in a phrase are ANDed together
+ const items: (QueryTerm | undefined | OperatorNode["type"])[] = [];
+ for (const child of ast.children) {
+ if (child.type === "phrase") {
+ child.value.forEach((value, index) => {
+ if (index > 0) items.push("AND");
+ // an empty phrase ("") cannot match anything so we drop it
+ items.push(value === '""' ? undefined : { type: "term", value });
+ });
+ } else if (
+ child.type === "AND" ||
+ child.type === "OR" ||
+ child.type === "NOT"
+ ) {
+ items.push(child.type);
+ }
+ }
+ if (items.length === 0) return;
+
+ // items alternate between terms & operators (guaranteed by
+ // transformQueryNode) so we can use simple precedence climbing.
+ let index = 0;
+ const parse = (minPrecedence: number): QueryExpression | undefined => {
+ let left: QueryExpression | undefined = items[index++] as
+ | QueryTerm
+ | undefined;
+ while (index < items.length) {
+ const operator = items[index] as OperatorNode["type"];
+ const precedence = PRECEDENCE[operator];
+ if (precedence < minPrecedence) break;
+ index++;
+ left = combine(operator, left, parse(precedence + 1));
+ }
+ return left;
+ };
+ return parse(1);
+}
+
+function combine(
+ operator: OperatorNode["type"],
+ left: QueryExpression | undefined,
+ right: QueryExpression | undefined
+): QueryExpression | undefined {
+ if (!left) return operator === "NOT" ? undefined : right;
+ if (!right) return left;
+ return { type: operator, left, right };
+}
+
// Main transformer: returns (QueryNode | FieldPhraseNode)[]
export function transformQuery(query: string): {
[K in keyof typeof SUPPORTED_FIELDS]?: ReturnType<
@@ -344,9 +445,12 @@ export function transformQuery(query: string): {
}
function serializeQuery(node: QueryNode | FieldPhraseNode) {
+ const ast = node.type === "query" ? node : node.value;
+ const expression = buildExpression(ast);
return {
- query: generateSQL(node.type === "query" ? node : node.value),
- tokens: tokenizeAst(node.type === "query" ? node : node.value)
+ query: expression ? serializeExpression(expression) : "",
+ expression,
+ tokens: tokenizeAst(ast)
};
}