Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: search query should be AND by default instead of OR

Abdullah Atta7e88ec88ee76
diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.ts
index f7aea041d..6fad0408e 100644
--- a/packages/core/src/api/lookup.ts
+++ b/packages/core/src/api/lookup.ts
@@ -26,6 +26,7 @@ import { FilteredSelector } from "../database/sql-collection";
import { VirtualizedGrouping } from "../utils/virtualized-grouping";
import { logger } from "../logger";
import { rebuildSearchIndex } from "../database/fts";
+import { transformQuery } from "../utils/query-transformer";
type SearchResults<T> = {
sorted: (limit?: number) => Promise<VirtualizedGrouping<T>>;
@@ -86,6 +87,7 @@ export default class Lookup {
return results.map((r) => r.id);
}
+ query = transformQuery(query);
const results = await db
.selectFrom((eb) =>
eb
diff --git a/packages/core/src/utils/__tests__/query-transformer.test.ts b/packages/core/src/utils/__tests__/query-transformer.test.ts
new file mode 100644
index 000000000..c9b54a58b
--- /dev/null
+++ b/packages/core/src/utils/__tests__/query-transformer.test.ts
@@ -0,0 +1,72 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { expect, test } from "vitest";
+import { transformQuery } from "../query-transformer";
+
+const TRANSFORM_QUERY_TESTS = [
+ ["hello world", `"hello world"`],
+ ["hello world OR bar", `"hello world" OR "bar"`],
+ ["hello world OR bar NOT baz", `"hello world" OR "bar" NOT "baz"`],
+ ["hello world OR NOT AND", `"hello world"`],
+ ["hello world OR NOT AND something", `"hello world" AND "something"`],
+ ["hello world -foo", `"hello world -foo"`],
+ ["hello world phrase-with-dash", `"hello world phrase-with-dash"`],
+ ["hello world phrase-with-dash*", '"hello world phrase-with-dash*"'],
+ ["example + foo + bar", `"example + foo + bar"`],
+ ["example OR foo NOT bar", `"example" OR "foo" NOT "bar"`],
+ [
+ 'example "quoted phrase" "another quoted phrase"',
+ `"example ""quoted phrase"" ""another quoted phrase"""`
+ ],
+ ['"phrase-with-dash*"', `""phrase-with-dash*""`],
+ ['-foo + bar OR "quoted-phrase"', '"-foo + bar" OR ""quoted-phrase""'],
+ [
+ 'phrase-with-dash* + "quoted-phrase"',
+ '"phrase-with-dash* + ""quoted-phrase"""'
+ ],
+ [
+ 'example -foo + bar + "quoted-dash-phrase*" OR "another-quoted-phrase"',
+ '"example -foo + bar + ""quoted-dash-phrase*""" OR ""another-quoted-phrase""'
+ ],
+ ["", ""],
+ ["foo", `"foo"`],
+ ['"quoted"', '""quoted""'],
+ ["-foo -bar", `"-foo -bar"`],
+ ["foo + + bar", `"foo + + bar"`],
+ ["foo + OR", `"foo +"`],
+ ['"special -phrase*"', '""special -phrase*""'],
+ ["foo* + bar*", `"foo* + bar*"`],
+ ["(foo + bar) -baz", `"(foo + bar) -baz"`],
+ ['"phrase with "quotes""', '""phrase with ""quotes""""'],
+ ['foo + "bar -baz" OR "qux*"', '"foo + ""bar -baz""" OR ""qux*""'],
+ ["foo + bar + ", `"foo + bar +"`],
+ ["+foo bar", `"+foo bar"`],
+ ["foo*bar*", `"foo*bar*"`],
+ ['"escaped "quotes""', '""escaped ""quotes""""'],
+ ["-hello-world", `"-hello-world"`],
+ ["-hello-world*", '"-hello-world*"'],
+ ["*helo*", `"*helo*"`]
+];
+
+for (const [input, expectedOutput] of TRANSFORM_QUERY_TESTS) {
+ test(`should transform "${input}" into a valid SQL query`, () => {
+ expect(transformQuery(input)).toBe(expectedOutput);
+ });
+}
diff --git a/packages/core/src/utils/query-transformer.ts b/packages/core/src/utils/query-transformer.ts
new file mode 100644
index 000000000..422533a17
--- /dev/null
+++ b/packages/core/src/utils/query-transformer.ts
@@ -0,0 +1,162 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+type ASTNode = QueryNode | PhraseNode | OperatorNode;
+
+type QueryNode = {
+ type: "query";
+ children: ASTNode[];
+};
+
+type PhraseNode = {
+ type: "phrase";
+ value: string[];
+};
+
+type OperatorNode = {
+ type: "AND" | "OR" | "NOT";
+};
+
+function escapeSQLString(str: string): string {
+ if (str.startsWith('"') && str.endsWith('"')) {
+ const innerStr = str.slice(1, -1).replace(/"/g, '""');
+ return `"${innerStr}"`;
+ }
+
+ // const isSubstitution = str.startsWith("-");
+ // if (isSubstitution) {
+ // return str.replace(/^-(.+)/g, (_, rest) => {
+ // return `-${escapeSQLString(rest)}`;
+ // });
+ // }
+
+ // const isWildcard = str.endsWith("*");
+ // if (isWildcard) {
+ // return str.replace(/(.+?)(\*?$)/gm, (_, text, end) => {
+ // return `${escapeSQLString(text)}${end}`;
+ // });
+ // }
+
+ // if (str.includes("-")) {
+ // return `"${str.replace(/"/g, '""')}"`;
+ // }
+
+ return str.replace(/"/g, '""');
+}
+
+function tokenize(query: string): string[] {
+ const tokens: string[] = [];
+ let buffer = "";
+ let isQuoted = false;
+
+ for (let i = 0; i < query.length; ++i) {
+ const char = query[i];
+ if (char === '"') {
+ isQuoted = !isQuoted;
+ }
+ if (char === " " && !isQuoted) {
+ if (buffer.length > 0) {
+ tokens.push(buffer);
+ buffer = "";
+ }
+ } else {
+ buffer += char;
+ }
+ }
+ if (buffer.length > 0) tokens.push(buffer);
+
+ return tokens;
+}
+
+function parseTokens(tokens: string[]): QueryNode {
+ const ast: QueryNode = { type: "query", children: [] };
+ let currentPhrase: string[] = [];
+
+ for (const token of tokens) {
+ if (token === "AND" || token === "OR" || token === "NOT") {
+ if (currentPhrase.length > 0) {
+ ast.children.push({ type: "phrase", value: currentPhrase });
+ currentPhrase = [];
+ }
+ ast.children.push({ type: token });
+ } else {
+ currentPhrase.push(token);
+ }
+ }
+
+ if (currentPhrase.length > 0) {
+ ast.children.push({ type: "phrase", value: currentPhrase });
+ }
+
+ return ast;
+}
+
+function transformAST(ast: QueryNode): QueryNode {
+ const transformedAST: QueryNode = { ...ast, children: [] };
+ let lastWasPhrase = false;
+
+ for (let i = 0; i < ast.children.length; i++) {
+ const child = ast.children[i];
+
+ if (child.type === "phrase") {
+ if (lastWasPhrase) {
+ transformedAST.children.push({ type: "AND" });
+ }
+ // const transformedPhrase = child.value;//.map(escapeSQLString);
+ transformedAST.children.push({
+ type: "phrase",
+ value: child.value
+ });
+ lastWasPhrase = true;
+ } else if (
+ child.type === "AND" ||
+ child.type === "OR" ||
+ child.type === "NOT"
+ ) {
+ if (
+ lastWasPhrase &&
+ i + 1 < ast.children.length &&
+ ast.children[i + 1].type === "phrase"
+ ) {
+ transformedAST.children.push(child);
+ lastWasPhrase = false;
+ }
+ }
+ }
+
+ return transformedAST;
+}
+
+function generateSQL(ast: QueryNode): string {
+ return ast.children
+ .map((child) => {
+ if (child.type === "phrase") {
+ return `"${escapeSQLString(child.value.join(" "))}"`;
+ }
+ if (child.type === "AND" || child.type === "OR" || child.type === "NOT") {
+ return child.type;
+ }
+ return "";
+ })
+ .join(" ");
+}
+
+export function transformQuery(query: string): string {
+ return generateSQL(transformAST(parseTokens(tokenize(query))));
+}