Skip to content

Open-Source API Development Ecosystem • https://hoppscotch.io • Offline, On-Prem & Cloud • Web, Desktop & CLI • Open-Source Alternative to Postman, Insomnia

refactor: use trigram search index instead of full text search (#3900)

Andrew Bastin0e96665254ae
diff --git a/packages/hoppscotch-backend/prisma/migrations/20240226053141_full_text_search_additions/migration.sql b/packages/hoppscotch-backend/prisma/migrations/20240226053141_full_text_search_additions/migration.sql
index 8d5ee2fc2..25ca672f0 100644
--- a/packages/hoppscotch-backend/prisma/migrations/20240226053141_full_text_search_additions/migration.sql
+++ b/packages/hoppscotch-backend/prisma/migrations/20240226053141_full_text_search_additions/migration.sql
@@ -1,17 +1,22 @@
--- AlterTable
-ALTER TABLE
+-- This is a custom migration file which is not generated by Prisma.
+-- The aim of this migration is to add text search indices to the TeamCollection and TeamRequest tables.
+
+-- Create Extension
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
+-- Create GIN Trigram Index for Team Collection title
+CREATE INDEX
+ "TeamCollection_title_trgm_idx"
+ON
"TeamCollection"
-ADD
- titleSearch tsvector GENERATED ALWAYS AS (to_tsvector('english', title)) STORED;
+USING
+ GIN (title gin_trgm_ops);
--- AlterTable
-ALTER TABLE
+-- Create GIN Trigram Index for Team Collection title
+CREATE INDEX
+ "TeamRequest_title_trgm_idx"
+ON
"TeamRequest"
-ADD
- titleSearch tsvector GENERATED ALWAYS AS (to_tsvector('english', title)) STORED;
-
--- CreateIndex
-CREATE INDEX "TeamCollection_textSearch_idx" ON "TeamCollection" USING GIN (titleSearch);
+USING
+ GIN (title gin_trgm_ops);
--- CreateIndex
-CREATE INDEX "TeamRequest_textSearch_idx" ON "TeamRequest" USING GIN (titleSearch);
diff --git a/packages/hoppscotch-backend/src/team-collection/team-collection.service.ts b/packages/hoppscotch-backend/src/team-collection/team-collection.service.ts
index 6def6a2fe..76dd45411 100644
--- a/packages/hoppscotch-backend/src/team-collection/team-collection.service.ts
+++ b/packages/hoppscotch-backend/src/team-collection/team-collection.service.ts
@@ -20,7 +20,7 @@ import {
TEAM_COLL_PARENT_TREE_GEN_FAILED,
} from '../errors';
import { PubSubService } from '../pubsub/pubsub.service';
-import { isValidLength } from 'src/utils';
+import { escapeSqlLikeString, isValidLength } from 'src/utils';
import * as E from 'fp-ts/Either';
import * as O from 'fp-ts/Option';
import { Prisma, TeamCollection as DBTeamCollection } from '@prisma/client';
@@ -1125,7 +1125,7 @@ export class TeamCollectionService {
id: searchResults[i].id,
path: !fetchedParentTree
? []
- : ([fetchedParentTree.right] as CollectionSearchNode[]),
+ : (fetchedParentTree.right as CollectionSearchNode[]),
});
}
@@ -1148,14 +1148,20 @@ export class TeamCollectionService {
skip: number,
) {
const query = Prisma.sql`
- select id,title,'collection' AS type
- from "TeamCollection"
- where "TeamCollection"."teamID"=${teamID}
- and titlesearch @@ to_tsquery(${searchQuery})
- order by ts_rank(titlesearch,to_tsquery(${searchQuery}))
- limit ${take}
+ SELECT
+ id,title,'collection' AS type
+ FROM
+ "TeamCollection"
+ WHERE
+ "TeamCollection"."teamID"=${teamID}
+ AND
+ title ILIKE ${`%${escapeSqlLikeString(searchQuery)}%`}
+ ORDER BY
+ similarity(title, ${searchQuery})
+ LIMIT ${take}
OFFSET ${skip === 0 ? 0 : (skip - 1) * take};
`;
+
try {
const res = await this.prisma.$queryRaw<SearchQueryReturnType[]>(query);
return E.right(res);
@@ -1180,12 +1186,17 @@ export class TeamCollectionService {
skip: number,
) {
const query = Prisma.sql`
- select id,title,request->>'method' as method,'request' AS type
- from "TeamRequest"
- where "TeamRequest"."teamID"=${teamID}
- and titlesearch @@ to_tsquery(${searchQuery})
- order by ts_rank(titlesearch,to_tsquery(${searchQuery}))
- limit ${take}
+ SELECT
+ id,title,request->>'method' as method,'request' AS type
+ FROM
+ "TeamRequest"
+ WHERE
+ "TeamRequest"."teamID"=${teamID}
+ AND
+ title ILIKE ${`%${escapeSqlLikeString(searchQuery)}%`}
+ ORDER BY
+ similarity(title, ${searchQuery})
+ LIMIT ${take}
OFFSET ${skip === 0 ? 0 : (skip - 1) * take};
`;
@@ -1250,45 +1261,53 @@ export class TeamCollectionService {
* @returns The parent tree of the parent collections
*/
private generateParentTree(parentCollections: ParentTreeQueryReturnType[]) {
- function findChildren(id) {
+ function findChildren(id: string): CollectionSearchNode[] {
const collection = parentCollections.filter((item) => item.id === id)[0];
if (collection.parentID == null) {
- return {
- id: collection.id,
- title: collection.title,
- type: 'collection',
- path: [],
- };
+ return <CollectionSearchNode[]>[
+ {
+ id: collection.id,
+ title: collection.title,
+ type: 'collection' as const,
+ path: [],
+ },
+ ];
}
- const res = {
- id: collection.id,
- title: collection.title,
- type: 'collection',
- path: findChildren(collection.parentID),
- };
+ const res = <CollectionSearchNode[]>[
+ {
+ id: collection.id,
+ title: collection.title,
+ type: 'collection' as const,
+ path: findChildren(collection.parentID),
+ },
+ ];
return res;
}
if (parentCollections.length > 0) {
if (parentCollections[0].parentID == null) {
- return {
+ return <CollectionSearchNode[]>[
+ {
+ id: parentCollections[0].id,
+ title: parentCollections[0].title,
+ type: 'collection',
+ path: [],
+ },
+ ];
+ }
+
+ return <CollectionSearchNode[]>[
+ {
id: parentCollections[0].id,
title: parentCollections[0].title,
type: 'collection',
- path: [],
- };
- }
-
- return {
- id: parentCollections[0].id,
- title: parentCollections[0].title,
- type: 'collection',
- path: findChildren(parentCollections[0].parentID),
- };
+ path: findChildren(parentCollections[0].parentID),
+ },
+ ];
}
- return null;
+ return <CollectionSearchNode[]>[];
}
/**
diff --git a/packages/hoppscotch-backend/src/utils.ts b/packages/hoppscotch-backend/src/utils.ts
index ea809a4ed..34095eb63 100644
--- a/packages/hoppscotch-backend/src/utils.ts
+++ b/packages/hoppscotch-backend/src/utils.ts
@@ -250,3 +250,40 @@ export function checkEnvironmentAuthProvider(
}
}
}
+
+/**
+ * Adds escape backslashes to the input so that it can be used inside
+ * SQL LIKE/ILIKE queries. Inspired by PHP's `mysql_real_escape_string`
+ * function.
+ *
+ * Eg. "100%" -> "100\\%"
+ *
+ * Source: https://stackoverflow.com/a/32648526
+ */
+export function escapeSqlLikeString(str: string) {
+ if (typeof str != 'string')
+ return str;
+
+ return str.replace(/[\0\x08\x09\x1a\n\r"'\\\%]/g, function (char) {
+ switch (char) {
+ case "\0":
+ return "\\0";
+ case "\x08":
+ return "\\b";
+ case "\x09":
+ return "\\t";
+ case "\x1a":
+ return "\\z";
+ case "\n":
+ return "\\n";
+ case "\r":
+ return "\\r";
+ case "\"":
+ case "'":
+ case "\\":
+ case "%":
+ return "\\"+char; // prepends a backslash to backslash, percent,
+ // and double/single quotes
+ }
+ });
+}