A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: filter out stop words in search query
Abdullah Atta4f1bb994f237
diff --git a/packages/core/src/utils/query-transformer.ts b/packages/core/src/utils/query-transformer.tsindex 81fb4c34c..b8843fdcc 100644--- a/packages/core/src/utils/query-transformer.ts+++ b/packages/core/src/utils/query-transformer.ts@@ -17,6 +17,8 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/+import { EN_STOPWORDS } from "./stop-words";+type ASTNode = QueryNode | PhraseNode | OperatorNode;type QueryNode = {@@ -161,7 +163,9 @@ function generateSQL(ast: QueryNode): string {}export function transformQuery(query: string) {- const tokens = tokenize(query);+ const tokens = tokenize(query).filter(+ (token) => !EN_STOPWORDS.includes(token)+ );const largeTokens = tokens.filter((token) => token.length >= 3 || token === "OR");diff --git a/packages/core/src/utils/stop-words.ts b/packages/core/src/utils/stop-words.tsnew file mode 100644index 000000000..feff544a7--- /dev/null+++ b/packages/core/src/utils/stop-words.ts@@ -0,0 +1,321 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/+export const EN_STOPWORDS = [+ "a",+ "about",+ "above",+ "across",+ "actually",+ "after",+ "again",+ "against",+ "all",+ "almost",+ "alone",+ "along",+ "already",+ "also",+ "although",+ "always",+ "among",+ "an",+ "and",+ "another",+ "any",+ "anybody",+ "anyone",+ "anything",+ "anywhere",+ "are",+ "around",+ "as",+ "ask",+ "asked",+ "asking",+ "asks",+ "at",+ "away",+ "b",+ "be",+ "became",+ "because",+ "become",+ "becomes",+ "becoming",+ "been",+ "before",+ "behind",+ "being",+ "best",+ "better",+ "between",+ "both",+ "but",+ "by",+ "c",+ "came",+ "can",+ "certain",+ "certainly",+ "clearly",+ "come",+ "consider",+ "considering",+ "could",+ "d",+ "did",+ "different",+ "do",+ "does",+ "doing",+ "done",+ "down",+ "downwards",+ "during",+ "e",+ "each",+ "eg",+ "eight",+ "either",+ "enough",+ "even",+ "ever",+ "every",+ "everybody",+ "everyone",+ "everything",+ "everywhere",+ "ex",+ "exactly",+ "example",+ "except",+ "f",+ "fact",+ "facts",+ "far",+ "few",+ "first",+ "five",+ "for",+ "four",+ "from",+ "further",+ "g",+ "get",+ "gets",+ "given",+ "gives",+ "go",+ "going",+ "got",+ "h",+ "had",+ "has",+ "have",+ "having",+ "he",+ "her",+ "here",+ "herself",+ "him",+ "himself",+ "his",+ "how",+ "however",+ "i",+ "ie",+ "if",+ "in",+ "into",+ "is",+ "it",+ "its",+ "itself",+ "j",+ "just",+ "k",+ "keep",+ "keeps",+ "knew",+ "know",+ "known",+ "knows",+ "l",+ "last",+ "later",+ "least",+ "less",+ "let",+ "like",+ "likely",+ "m",+ "many",+ "may",+ "me",+ "might",+ "more",+ "most",+ "mostly",+ "much",+ "must",+ "my",+ "myself",+ "n",+ "necessary",+ "need",+ "needs",+ "never",+ "new",+ "next",+ "nine",+ "no",+ "nobody",+ "non",+ "not",+ "nothing",+ "now",+ "nowhere",+ "o",+ "of",+ "off",+ "often",+ "old",+ "on",+ "once",+ "one",+ "only",+ "or",+ "other",+ "others",+ "our",+ "out",+ "over",+ "p",+ "per",+ "perhaps",+ "please",+ "possible",+ "put",+ "q",+ "quite",+ "r",+ "rather",+ "really",+ "right",+ "s",+ "said",+ "same",+ "saw",+ "say",+ "says",+ "second",+ "see",+ "seem",+ "seemed",+ "seems",+ "seven",+ "several",+ "shall",+ "she",+ "should",+ "since",+ "six",+ "so",+ "some",+ "somebody",+ "someone",+ "something",+ "somewhere",+ "still",+ "such",+ "sure",+ "t",+ "take",+ "taken",+ "ten",+ "than",+ "that",+ "the",+ "their",+ "them",+ "then",+ "there",+ "therefore",+ "therein",+ "thereupon",+ "these",+ "they",+ "think",+ "third",+ "this",+ "those",+ "though",+ "three",+ "through",+ "thus",+ "to",+ "together",+ "too",+ "took",+ "toward",+ "two",+ "u",+ "under",+ "until",+ "up",+ "upon",+ "us",+ "use",+ "used",+ "uses",+ "v",+ "very",+ "w",+ "want",+ "wanted",+ "wants",+ "was",+ "way",+ "we",+ "well",+ "went",+ "were",+ "what",+ "when",+ "where",+ "whether",+ "which",+ "while",+ "who",+ "whole",+ "whose",+ "why",+ "will",+ "with",+ "within",+ "without",+ "would",+ "x",+ "y",+ "yet",+ "you",+ "your",+ "yours",+ "z"+];