Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

feat(search): index document and organization ids in fts search index (#733)

Corentin Thomasset848671b98237
diff --git a/.changeset/eager-grapes-raise.md b/.changeset/eager-grapes-raise.md
new file mode 100644
index 00000000..ef03cc17
--- /dev/null
+++ b/.changeset/eager-grapes-raise.md
@@ -0,0 +1,5 @@
+---
+"@papra/docker": patch
+---
+
+Improved search speed by using document and organization ids in index
diff --git a/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts b/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts
index 8f51e127..f75ce62e 100644
--- a/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts
+++ b/apps/papra-server/src/migrations/list/0002-documents-fts.migration.ts
@@ -6,6 +6,7 @@ export const documentsFtsMigration = {
up: async ({ db }) => {
await db.batch([
+ db.run(sql`DROP TABLE IF EXISTS documents_fts`),
db.run(sql`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(id UNINDEXED, name, original_name, content, prefix='2 3 4')`),
db.run(sql`INSERT INTO documents_fts(id, name, original_name, content) SELECT id, name, original_name, content FROM documents`),
db.run(sql`
diff --git a/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.test.ts b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.test.ts
new file mode 100644
index 00000000..2386a280
--- /dev/null
+++ b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.test.ts
@@ -0,0 +1,39 @@
+import { sql } from 'drizzle-orm';
+import { describe, expect, test } from 'vitest';
+import { setupDatabase } from '../../modules/app/database/database';
+import { initialSchemaSetupMigration } from './0001-initial-schema-setup.migration';
+import { documentsFtsMigration } from './0002-documents-fts.migration';
+import { dropFts5TriggersMigration } from './0013-drop-fts-5-triggers.migration';
+import { indexDocumentsFtsIdsMigration } from './0015-index-documents-fts-ids.migration';
+
+describe('0015-index-documents-fts-ids migration', () => {
+ describe('index-documents-fts-ids', () => {
+ test('the recreation of the FTS table with document_id and organization_id included as searchable columns still permits full text search', async () => {
+ const { db } = setupDatabase({ url: ':memory:' });
+ // Recreate a documents table and populate it with sample data
+ await initialSchemaSetupMigration.up({ db });
+
+ await db.batch([
+ db.run(sql`INSERT INTO organizations(id, name, created_at, updated_at) VALUES ('org_1', 'Test Organization', 0, 0)`),
+ db.run(sql`INSERT INTO documents(id, organization_id, original_name, name, mime_type, original_storage_key, original_sha256_hash, content, created_at, updated_at)
+ VALUES
+ ('doc1', 'org_1', 'Test Document', 'Test Document', 'text/plain', 'key1', 'hash1', 'This is a sample document content about testing.',0,0),
+ ('doc2', 'org_1', 'Another Document', 'Another Document', 'text/plain', 'key2', 'hash2', 'This document discusses database migrations.',0,0)
+ `),
+ ]);
+
+ await documentsFtsMigration.up({ db });
+ await dropFts5TriggersMigration.up({ db });
+
+ const { rows: searchBeforeMigration } = await db.run(sql`SELECT id FROM documents_fts WHERE documents_fts = 'Test*'`);
+
+ expect(searchBeforeMigration).to.eql([{ id: 'doc1' }]);
+
+ await indexDocumentsFtsIdsMigration.up({ db });
+
+ const { rows: searchAfterMigration } = await db.run(sql`SELECT document_id FROM documents_fts WHERE documents_fts = 'Test*'`);
+
+ expect(searchAfterMigration).to.eql([{ document_id: 'doc1' }]);
+ });
+ });
+});
diff --git a/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.ts b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.ts
new file mode 100644
index 00000000..840bc9a2
--- /dev/null
+++ b/apps/papra-server/src/migrations/list/0015-index-documents-fts-ids.migration.ts
@@ -0,0 +1,38 @@
+import type { Migration } from '../migrations.types';
+import { sql } from 'drizzle-orm';
+
+export const indexDocumentsFtsIdsMigration = {
+ name: 'index-documents-fts-ids',
+
+ up: async ({ db }) => {
+ await db.batch([
+ // Drop old FTS table
+ db.run(sql`DROP TABLE IF EXISTS documents_fts`),
+
+ // Recreate FTS table with document_id and organization_id as indexed for efficient joins and removed original_name
+ db.run(sql`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+ document_id,
+ organization_id,
+ name,
+ content,
+ prefix='2 3 4'
+ )`),
+
+ // Populate the new FTS table
+ db.run(sql`INSERT INTO documents_fts(document_id, organization_id, name, content)
+ SELECT id, organization_id, name, content FROM documents`),
+ ]);
+ },
+
+ down: async ({ db }) => {
+ await db.batch([
+ db.run(sql`DROP TABLE IF EXISTS documents_fts`),
+
+ // Recreate old FTS table with original schema
+ db.run(sql`CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(id UNINDEXED, name, original_name, content, prefix='2 3 4')`),
+
+ // Populate with old structure
+ db.run(sql`INSERT INTO documents_fts(id, name, original_name, content) SELECT id, name, original_name, content FROM documents`),
+ ]);
+ },
+} satisfies Migration;
diff --git a/apps/papra-server/src/migrations/migrations.registry.test.ts b/apps/papra-server/src/migrations/migrations.registry.test.ts
index 40b0b741..b320aa48 100644
--- a/apps/papra-server/src/migrations/migrations.registry.test.ts
+++ b/apps/papra-server/src/migrations/migrations.registry.test.ts
@@ -100,7 +100,7 @@ describe('migrations registry', () => {
CREATE TABLE "auth_verifications" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "identifier" text NOT NULL, "value" text NOT NULL, "expires_at" integer NOT NULL );
CREATE TABLE "document_activity_log" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "document_id" text NOT NULL, "event" text NOT NULL, "event_data" text, "user_id" text, "tag_id" text, FOREIGN KEY ("document_id") REFERENCES "documents"("id") ON UPDATE cascade ON DELETE cascade, FOREIGN KEY ("user_id") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null, FOREIGN KEY ("tag_id") REFERENCES "tags"("id") ON UPDATE cascade ON DELETE set null );
CREATE TABLE "documents" ( "id" text PRIMARY KEY NOT NULL, "created_at" integer NOT NULL, "updated_at" integer NOT NULL, "is_deleted" integer DEFAULT false NOT NULL, "deleted_at" integer, "organization_id" text NOT NULL, "created_by" text, "deleted_by" text, "original_name" text NOT NULL, "original_size" integer DEFAULT 0 NOT NULL, "original_storage_key" text NOT NULL, "original_sha256_hash" text NOT NULL, "name" text NOT NULL, "mime_type" text NOT NULL, "content" text DEFAULT '' NOT NULL, file_encryption_key_wrapped TEXT, file_encryption_kek_version TEXT, file_encryption_algorithm TEXT, FOREIGN KEY ("organization_id") REFERENCES "organizations"("id") ON UPDATE cascade ON DELETE cascade, FOREIGN KEY ("created_by") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null, FOREIGN KEY ("deleted_by") REFERENCES "users"("id") ON UPDATE cascade ON DELETE set null );
- CREATE VIRTUAL TABLE documents_fts USING fts5(id UNINDEXED, name, original_name, content, prefix='2 3 4');
+ CREATE VIRTUAL TABLE documents_fts USING fts5( document_id, organization_id, name, content, prefix='2 3 4' );
CREATE TABLE 'documents_fts_config'(k PRIMARY KEY, v) WITHOUT ROWID;
CREATE TABLE 'documents_fts_content'(id INTEGER PRIMARY KEY, c0, c1, c2, c3);
CREATE TABLE 'documents_fts_data'(id INTEGER PRIMARY KEY, block BLOB);
diff --git a/apps/papra-server/src/migrations/migrations.registry.ts b/apps/papra-server/src/migrations/migrations.registry.ts
index 4d670ce5..342d9c31 100644
--- a/apps/papra-server/src/migrations/migrations.registry.ts
+++ b/apps/papra-server/src/migrations/migrations.registry.ts
@@ -14,6 +14,7 @@ import { softDeleteOrganizationsMigration } from './list/0011-soft-delete-organi
import { taggingRuleConditionMatchModeMigration } from './list/0012-tagging-rule-condition-match-mode.migration';
import { dropFts5TriggersMigration } from './list/0013-drop-fts-5-triggers.migration';
import { twoFactorAuthenticationMigration } from './list/0014-two-factor-authentication.migration';
+import { indexDocumentsFtsIdsMigration } from './list/0015-index-documents-fts-ids.migration';
export const migrations: Migration[] = [
initialSchemaSetupMigration,
@@ -30,4 +31,5 @@ export const migrations: Migration[] = [
taggingRuleConditionMatchModeMigration,
dropFts5TriggersMigration,
twoFactorAuthenticationMigration,
+ indexDocumentsFtsIdsMigration,
];
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts
index e4a47812..5cda28fc 100644
--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts
+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.repository.ts
@@ -27,10 +27,10 @@ async function searchOrganizationDocuments({ organizationId, searchQuery, pageIn
.from(documentsTable)
.innerJoin(
documentsFtsTable,
- eq(documentsFtsTable.id, documentsTable.id),
+ eq(documentsFtsTable.documentId, documentsTable.id),
)
.where(and(
- eq(documentsTable.organizationId, organizationId),
+ eq(documentsFtsTable.organizationId, organizationId),
eq(documentsTable.isDeleted, false),
eq(documentsFtsTable, formattedSearchQuery), // Match and eq works the same for FTS5 virtual tables
))
@@ -45,9 +45,9 @@ async function indexDocument({ document, db }: { document: DocumentSearchableDat
await db
.insert(documentsFtsTable)
.values({
- id: document.id,
+ documentId: document.id,
+ organizationId: document.organizationId,
name: document.name,
- originalName: document.originalName,
content: document.content,
});
}
@@ -66,11 +66,11 @@ async function updateDocument({ documentId, document, db }: { documentId: string
await db
.update(documentsFtsTable)
.set(dataToUpdate)
- .where(eq(documentsFtsTable.id, documentId));
+ .where(eq(documentsFtsTable.documentId, documentId));
}
async function deleteDocument({ documentId, db }: { documentId: string; db: Database }) {
await db
.delete(documentsFtsTable)
- .where(eq(documentsFtsTable.id, documentId));
+ .where(eq(documentsFtsTable.documentId, documentId));
}
diff --git a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts
index c12dcf0f..b560ae26 100644
--- a/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts
+++ b/apps/papra-server/src/modules/documents/document-search/database-fts5/database-fts5.tables.ts
@@ -1,8 +1,8 @@
import { sqliteTable, text } from 'drizzle-orm/sqlite-core';
export const documentsFtsTable = sqliteTable('documents_fts', {
- id: text('id').notNull(),
+ documentId: text('document_id').notNull(),
+ organizationId: text('organization_id').notNull(),
name: text('name').notNull(),
- originalName: text('original_name').notNull(),
content: text('content').notNull(),
});