Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

feat(api): rebuild every search index in one job (#3616)

Hampusd20b31fa095d
diff --git a/fluxer_admin/openapi-admin.json b/fluxer_admin/openapi-admin.json
index 567115215..ad7ba5d1d 100644
--- a/fluxer_admin/openapi-admin.json
+++ b/fluxer_admin/openapi-admin.json
@@ -7173,7 +7173,7 @@
"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}
}
},
- "description": "Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and favorite_memes requires user_id. Requires GUILD_LOOKUP permission.",
+ "description": "Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and all rebuilds every index, one at a time. Queueing all while one is running returns the running refresh ID. Requires GUILD_LOOKUP permission.",
"security": [{"adminApiKey": []}],
"parameters": [
{
@@ -7189,9 +7189,9 @@
"audit_logs",
"channel_messages",
"guild_members",
- "favorite_memes",
"discovery",
- "threads"
+ "threads",
+ "all"
],
"x-enumDescriptions": [
"Guild search index",
@@ -7200,9 +7200,9 @@
"Audit log search index",
"Channel message search index",
"Guild member search index",
- "Favourite meme search index",
"Discovery guild search index",
- "Thread search index"
+ "Thread search index",
+ "Every search index, with messages in every channel and DM"
],
"enum": [
"guilds",
@@ -7211,9 +7211,9 @@
"audit_logs",
"channel_messages",
"guild_members",
- "favorite_memes",
"discovery",
- "threads"
+ "threads",
+ "all"
],
"type": "string"
},
@@ -14107,10 +14107,6 @@
"guild_id": {
"description": "Specific guild ID to reindex",
"allOf": [{"$ref": "#/components/schemas/SnowflakeType"}]
- },
- "user_id": {
- "description": "Specific user ID to reindex",
- "allOf": [{"$ref": "#/components/schemas/SnowflakeType"}]
}
}
},
@@ -14143,6 +14139,10 @@
"type": "string"
},
"index_type": {"type": "string", "description": "Type of index being refreshed"},
+ "phase": {
+ "description": "The index an `all` rebuild is working on, or queued before it starts",
+ "type": "string"
+ },
"total": {"description": "Total number of items to index", "type": "number"},
"indexed": {"description": "Number of items indexed so far", "type": "number"},
"started_at": {"description": "ISO 8601 timestamp when the job started", "type": "string"},
diff --git a/fluxer_admin/src/api/search.rs b/fluxer_admin/src/api/search.rs
index 2ac4cba92..de3efdf33 100644
--- a/fluxer_admin/src/api/search.rs
+++ b/fluxer_admin/src/api/search.rs
@@ -18,7 +18,6 @@ impl AdminApiClient {
.map_err(|e| ApiError::Parse(e.to_string()))?;
let body = generated_types::RefreshSearchIndexRequest {
guild_id: guild_id.map(snowflake),
- user_id: None,
};
let response = self
.generated()
@@ -53,6 +52,7 @@ fn index_refresh_status(
generated_types::IndexRefreshStatusResponse::Variant1 {
status,
index_type,
+ phase,
total,
indexed,
started_at,
@@ -62,6 +62,7 @@ fn index_refresh_status(
} => Ok(IndexRefreshStatusResponse::Progress {
status: status.to_string(),
index_type: Some(index_type),
+ phase,
total: total
.map(|value| float_to_u64(value, "total"))
.transpose()?,
diff --git a/fluxer_admin/src/api/types/search.rs b/fluxer_admin/src/api/types/search.rs
index 9e0f75ec6..ea612bbbd 100644
--- a/fluxer_admin/src/api/types/search.rs
+++ b/fluxer_admin/src/api/types/search.rs
@@ -17,6 +17,7 @@ pub enum IndexRefreshStatusResponse {
Progress {
status: String,
index_type: Option<String>,
+ phase: Option<String>,
total: Option<u64>,
indexed: Option<u64>,
started_at: Option<String>,
diff --git a/fluxer_admin/src/routes/system.rs b/fluxer_admin/src/routes/system.rs
index e68c9d647..1d302bb1d 100644
--- a/fluxer_admin/src/routes/system.rs
+++ b/fluxer_admin/src/routes/system.rs
@@ -157,8 +157,10 @@ async fn search_index_page(
let job_id_str = query.job_id.unwrap_or_default();
let mut status: Option<String> = None;
+ let mut phase: Option<String> = None;
let mut total: Option<u64> = None;
let mut indexed: Option<u64> = None;
+ let mut error: Option<String> = None;
if !job_id_str.is_empty() {
let client =
@@ -170,13 +172,17 @@ async fn search_index_page(
}
crate::api::types::IndexRefreshStatusResponse::Progress {
status: s,
+ phase: p,
total: t,
indexed: i,
+ error: e,
..
} => {
status = Some(s);
+ phase = p;
total = t;
indexed = i;
+ error = e;
}
},
Err(error) => {
@@ -189,11 +195,12 @@ async fn search_index_page(
Some(templates::pages::search_index::RefreshStatus {
job_id: &job_id_str,
status: status.as_deref(),
+ phase: phase.as_deref(),
total,
indexed,
started_at: None,
completed_at: None,
- error: None,
+ error: error.as_deref(),
})
} else {
None
diff --git a/fluxer_admin/src/templates/pages/search_index.rs b/fluxer_admin/src/templates/pages/search_index.rs
index c90341192..200188ad8 100644
--- a/fluxer_admin/src/templates/pages/search_index.rs
+++ b/fluxer_admin/src/templates/pages/search_index.rs
@@ -6,7 +6,9 @@ use crate::{
middleware::auth::AuthContext,
templates::{
components::{
- auto_refresh::auto_refresh, form::csrf_input, page_container::page_header,
+ auto_refresh::auto_refresh,
+ form::{csrf_input, submit_button},
+ page_container::page_header,
section_card::section_card_simple,
},
layout::admin_layout,
@@ -43,6 +45,25 @@ fn disabled_reindex_button(title: &str) -> Markup {
}
}
+fn reindex_everything(base: &str, csrf_token: &str) -> Markup {
+ section_card_simple(
+ "Reindex everything",
+ html! {
+ p class="text-sm text-neutral-500 mb-3" {
+ "Rebuilds every index on this page, the members and threads of every guild, \
+ and the messages in every channel and DM, one at a time. Search shows \
+ partial results until it finishes, which can take hours on a large instance."
+ }
+ form method="post" action={(base) "/search-index?action=reindex"} class="w-fit"
+ onsubmit="return confirm('Rebuild every search index? Search shows partial results until it finishes.')" {
+ (csrf_input(csrf_token))
+ input type="hidden" name="index_type" value="all";
+ (submit_button("Reindex everything"))
+ }
+ },
+ )
+}
+
fn reindex_controls(base: &str, csrf_token: &str) -> Markup {
section_card_simple(
"Global Search Indexes",
@@ -76,6 +97,21 @@ fn reindex_controls(base: &str, csrf_token: &str) -> Markup {
)
}
+fn phase_label(phase: &str) -> Option<&'static str> {
+ match phase {
+ "queued" => Some("Waiting for a worker to start"),
+ "users" => Some("Rebuilding users"),
+ "guilds" => Some("Rebuilding guilds"),
+ "discovery" => Some("Rebuilding the Explore index"),
+ "reports" => Some("Rebuilding reports"),
+ "audit_logs" => Some("Rebuilding audit logs"),
+ "guild_members" => Some("Rebuilding guild members, counted in guilds"),
+ "threads" => Some("Rebuilding threads, counted in guilds"),
+ "messages" => Some("Rebuilding messages, counted in channels"),
+ _ => None,
+ }
+}
+
fn format_status_label(status: &str) -> &str {
match status {
"in_progress" => "In progress",
@@ -89,6 +125,7 @@ fn format_status_label(status: &str) -> &str {
pub struct RefreshStatus<'a> {
pub job_id: &'a str,
pub status: Option<&'a str>,
+ pub phase: Option<&'a str>,
pub total: Option<u64>,
pub indexed: Option<u64>,
pub started_at: Option<&'a str>,
@@ -121,6 +158,9 @@ fn status_section(base: &str, rs: &RefreshStatus) -> Markup {
}
@if is_in_progress {
+ @if let Some(label) = rs.phase.and_then(phase_label) {
+ p class="text-sm text-neutral-700" { (label) }
+ }
@if let (Some(idx), Some(tot)) = (rs.indexed, rs.total) {
@let pct = (u128::from(idx) * 100).checked_div(u128::from(tot)).unwrap_or(0);
div class="space-y-2" {
@@ -182,6 +222,7 @@ pub fn search_index_page(
let content = html! {
(page_header("Search Index Management", None))
div class="space-y-6" {
+ (reindex_everything(base, csrf_token))
(reindex_controls(base, csrf_token))
@if let Some(rs) = refresh {
(status_section(base, rs))
diff --git a/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts b/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts
index a12ae8157..3e062ee17 100644
--- a/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts
+++ b/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts
@@ -28,6 +28,13 @@ export type WorkerTaskHandler<Payload = Record<string, unknown>> = (
helpers: WorkerTaskHelpers,
) => Promise<WorkerTaskResult> | Promise<void>;
+export class JobFailedError extends Error {
+ constructor(message: string) {
+ super(message);
+ this.name = 'JobFailedError';
+ }
+}
+
export class JobCancelledError extends Error {
constructor(message = 'Job cancelled by admin') {
super(message);
diff --git a/fluxer_api/src/api/admin/controllers/SearchAdminController.ts b/fluxer_api/src/api/admin/controllers/SearchAdminController.ts
index 7266c6509..59dbd56e6 100644
--- a/fluxer_api/src/api/admin/controllers/SearchAdminController.ts
+++ b/fluxer_api/src/api/admin/controllers/SearchAdminController.ts
@@ -26,7 +26,7 @@ export function SearchAdminController(app: HonoApp) {
operationId: 'create_admin_search_index_refresh',
summary: 'Refresh a search index',
description:
- 'Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and favorite_memes requires user_id. Requires GUILD_LOOKUP permission.',
+ 'Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and all rebuilds every index, one at a time. Queueing all while one is running returns the running refresh ID. Requires GUILD_LOOKUP permission.',
responseSchema: RefreshSearchIndexResponse,
statusCode: 200,
security: 'adminApiKey',
@@ -37,10 +37,10 @@ export function SearchAdminController(app: HonoApp) {
const adminUserId = ctx.get('adminUserId');
const auditLogReason = ctx.get('auditLogReason');
const {index_name} = ctx.req.valid('param');
- const {guild_id, user_id} = ctx.req.valid('json');
+ const {guild_id} = ctx.req.valid('json');
return ctx.json(
await adminService.searchService.refreshSearchIndex(
- {index_type: index_name, guild_id, user_id},
+ {index_type: index_name, guild_id},
adminUserId,
auditLogReason,
),
diff --git a/fluxer_api/src/api/admin/services/AdminSearchService.ts b/fluxer_api/src/api/admin/services/AdminSearchService.ts
index 1a0afac3d..385430b37 100644
--- a/fluxer_api/src/api/admin/services/AdminSearchService.ts
+++ b/fluxer_api/src/api/admin/services/AdminSearchService.ts
@@ -8,7 +8,13 @@ import {createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes';
import {isSyntheticUserId} from '@app/api/constants/Core';
import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';
import {Logger} from '@app/api/Logger';
-import {getGuildSearchService, getUserSearchService} from '@app/api/SearchFactory';
+import {getGuildSearchService, getMessageSearchService, getUserSearchService} from '@app/api/SearchFactory';
+import {
+ claimSearchReindexAll,
+ finishSearchReindexAll,
+ INDEX_REFRESH_STATUS_TTL_SECONDS,
+ indexRefreshStatusKey,
+} from '@app/api/search/SearchReindexAllRun';
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
import type {UserSearchFilters} from '@fluxer/schema/src/contracts/search/SearchDocumentTypes';
@@ -21,15 +27,14 @@ interface RefreshSearchIndexJobPayload extends WorkerJobPayload {
| 'reports'
| 'audit_logs'
| 'channel_messages'
- | 'favorite_memes'
| 'guild_members'
| 'discovery'
- | 'threads';
+ | 'threads'
+ | 'all';
admin_user_id: string;
audit_log_reason: string | null;
job_id: string;
guild_id?: string;
- user_id?: string;
}
interface AdminSearchServiceDeps {
@@ -171,17 +176,16 @@ export class AdminSearchService {
| 'audit_logs'
| 'channel_messages'
| 'guild_members'
- | 'favorite_memes'
| 'discovery'
- | 'threads';
+ | 'threads'
+ | 'all';
guild_id?: bigint;
- user_id?: bigint;
},
adminUserId: UserID,
auditLogReason: string | null,
) {
const {auditService} = this.deps;
- const {worker: workerService, snowflake: snowflakeService} = this.deps.apiContext.services;
+ const {snowflake: snowflakeService} = this.deps.apiContext.services;
const jobId = (await snowflakeService.generate()).toString();
const payload: RefreshSearchIndexJobPayload = {
index_type: data.index_type,
@@ -207,28 +211,23 @@ export class AdminSearchService {
}
payload.guild_id = data.guild_id.toString();
}
- if (data.index_type === 'favorite_memes') {
- if (!data.user_id) {
- throw InputValidationError.create('user_id', 'user_id is required for the favorite_memes index type');
- }
- payload.user_id = data.user_id.toString();
+ let queuedJobId = jobId;
+ if (data.index_type === 'all') {
+ queuedJobId = await this.queueReindexAll(payload);
+ } else {
+ await this.queueRefresh(payload);
}
- await workerService.addJob('refreshSearchIndex', payload, {
- jobKey: `refreshSearchIndex_${data.index_type}_${jobId}`,
- maxAttempts: 1,
- requireLedger: true,
- });
- Logger.debug({index_type: data.index_type, job_id: jobId}, 'Queued search index refresh job');
+ Logger.debug({index_type: data.index_type, job_id: queuedJobId}, 'Queued search index refresh job');
const metadata = new Map([
['index_type', data.index_type],
- ['job_id', jobId],
+ ['job_id', queuedJobId],
]);
+ if (queuedJobId !== jobId) {
+ metadata.set('already_running', 'true');
+ }
if (data.guild_id) {
metadata.set('guild_id', data.guild_id.toString());
}
- if (data.user_id) {
- metadata.set('user_id', data.user_id.toString());
- }
await auditService.createAuditLog({
adminUserId,
targetType: 'search_index',
@@ -239,23 +238,56 @@ export class AdminSearchService {
});
return {
success: true,
- job_id: jobId,
+ job_id: queuedJobId,
};
}
+ private async queueRefresh(payload: RefreshSearchIndexJobPayload): Promise<void> {
+ await this.deps.apiContext.services.worker.addJob('refreshSearchIndex', payload, {
+ jobKey: `refreshSearchIndex_${payload.index_type}_${payload.job_id}`,
+ maxAttempts: 1,
+ requireLedger: true,
+ });
+ }
+
+ private async queueReindexAll(payload: RefreshSearchIndexJobPayload): Promise<string> {
+ if (!getMessageSearchService()) {
+ throw new FeatureTemporarilyDisabledError();
+ }
+ const {kv} = this.deps.apiContext.services;
+ const running = await claimSearchReindexAll(kv, payload.job_id);
+ if (running) {
+ return running;
+ }
+ const queued = {status: 'in_progress', index_type: 'all', phase: 'queued', started_at: new Date().toISOString()};
+ try {
+ await kv.set(
+ indexRefreshStatusKey(payload.job_id),
+ JSON.stringify(queued),
+ 'EX',
+ INDEX_REFRESH_STATUS_TTL_SECONDS,
+ );
+ await this.queueRefresh(payload);
+ } catch (error) {
+ await finishSearchReindexAll(kv, payload.job_id);
+ throw error;
+ }
+ return payload.job_id;
+ }
+
async getIndexRefreshStatus(jobId: string) {
const {cache: cacheService} = this.deps.apiContext.services;
- const statusKey = `index_refresh_status:${jobId}`;
const status = await cacheService.get<{
status: 'in_progress' | 'completed' | 'failed';
index_type: string;
+ phase?: string;
total?: number;
indexed?: number;
started_at?: string;
completed_at?: string;
failed_at?: string;
error?: string;
- }>(statusKey);
+ }>(indexRefreshStatusKey(jobId));
if (!status) {
return {
status: 'not_found' as const,
diff --git a/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts b/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts
index 2316703d9..9a878151f 100644
--- a/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts
+++ b/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts
@@ -7,14 +7,14 @@ export const SearchAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [
{
method: 'POST',
route: '/admin/search/indexes/:index_name/refreshes',
- async prepare({admin}) {
+ async prepare() {
return {
- request: {path: '/admin/search/indexes/favorite_memes/refreshes', body: {user_id: admin.userId}},
+ request: {path: '/admin/search/indexes/users/refreshes', body: {}},
expected: {
action: 'queue_refresh_index',
targetType: 'search_index',
targetId: '0',
- metadata: {index_type: 'favorite_memes', job_id: expect.any(String), user_id: admin.userId},
+ metadata: {index_type: 'users', job_id: expect.any(String)},
},
};
},
diff --git a/fluxer_api/src/api/search/SearchReindexAllRun.ts b/fluxer_api/src/api/search/SearchReindexAllRun.ts
new file mode 100644
index 000000000..67796bfe2
--- /dev/null
+++ b/fluxer_api/src/api/search/SearchReindexAllRun.ts
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
+import {seconds} from 'itty-time';
+
+const RUN_KEY = 'index_refresh:all';
+const RUN_TTL_SECONDS = seconds('1 hour');
+const CLAIM_ATTEMPTS = 3;
+
+export const INDEX_REFRESH_STATUS_TTL_SECONDS = seconds('1 hour');
+
+export function indexRefreshStatusKey(jobId: string): string {
+ return `index_refresh_status:${jobId}`;
+}
+
+export async function claimSearchReindexAll(kvClient: IKVProvider, jobId: string): Promise<string | null> {
+ for (let attempt = 0; attempt < CLAIM_ATTEMPTS; attempt++) {
+ if (await kvClient.acquireLock(RUN_KEY, jobId, RUN_TTL_SECONDS)) return null;
+ const running = await kvClient.get(RUN_KEY);
+ if (running) return running;
+ }
+ throw new Error('Could not claim the search reindex run');
+}
+
+export async function holdSearchReindexAll(kvClient: IKVProvider, jobId: string): Promise<boolean> {
+ return (
+ (await kvClient.extendLock(RUN_KEY, jobId, RUN_TTL_SECONDS)) ||
+ (await kvClient.acquireLock(RUN_KEY, jobId, RUN_TTL_SECONDS))
+ );
+}
+
+export async function finishSearchReindexAll(kvClient: IKVProvider, jobId: string): Promise<void> {
+ await kvClient.releaseLock(RUN_KEY, jobId);
+}
diff --git a/fluxer_api/src/api/search/tests/SearchReindexAll.test.ts b/fluxer_api/src/api/search/tests/SearchReindexAll.test.ts
new file mode 100644
index 000000000..9fd622aa6
--- /dev/null
+++ b/fluxer_api/src/api/search/tests/SearchReindexAll.test.ts
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {createAdminWithAcls} from '@app/api/admin/tests/AdminTestUtils';
+import {createTestAccount, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {resetApiServicesForTesting} from '@app/api/CreateApiContext';
+import {createGuild} from '@app/api/guild/tests/GuildTestUtils';
+import {markChannelAsIndexed, sendMessage} from '@app/api/message/tests/MessageTestUtils';
+import {getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';
+import {getMessageSearchService} from '@app/api/SearchFactory';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {NoopWorkerService} from '@app/api/test/NoopWorkerService';
+import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService';
+import {createBuilder} from '@app/api/test/TestRequestBuilder';
+import refreshSearchIndex from '@app/api/worker/tasks/RefreshSearchIndex';
+import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';
+import {initializeWorkerDependencies} from '@app/api/worker/WorkerDependencies';
+import {AdminACLs} from '@fluxer/constants/src/AdminACLs';
+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it} from 'vitest';
+
+interface RefreshResponse {
+ job_id: string;
+}
+
+interface StatusResponse {
+ status: string;
+ index_type?: string;
+}
+
+describe('rebuilding every search index', () => {
+ let harness: ApiTestHarness;
+ let worker: SyncTaskWorkerService;
+ let admin: TestAccount;
+
+ beforeAll(async () => {
+ harness = await createApiTestHarness({search: 'enabled'});
+ });
+
+ beforeEach(async () => {
+ await harness.reset();
+ setWorkerDependencies(await initializeWorkerDependencies(getSnowflakeService()));
+ worker = new SyncTaskWorkerService({refreshSearchIndex}, {deferred: true});
+ setInjectedWorkerService(worker);
+ resetApiServicesForTesting();
+ admin = await createAdminWithAcls(harness, [AdminACLs.GUILD_LOOKUP]);
+ });
+
+ afterEach(() => {
+ setInjectedWorkerService(new NoopWorkerService());
+ resetApiServicesForTesting();
+ });
+
+ afterAll(async () => {
+ clearWorkerDependencies();
+ await harness.shutdown();
+ });
+
+ function queueAll(): Promise<RefreshResponse> {
+ return createBuilder<RefreshResponse>(harness, admin.token)
+ .post('/admin/search/indexes/all/refreshes')
+ .body({})
+ .expect(200)
+ .execute();
+ }
+
+ it('queues one rebuild at a time and brings back wiped messages', async () => {
+ const account = await createTestAccount(harness);
+ const guild = await createGuild(harness, account.token, 'Reindex Guild');
+ const channelId = guild.system_channel_id!;
+ const content = `reindex-everything-${Date.now()}`;
+ const message = await sendMessage(harness, account.token, channelId, content);
+ await markChannelAsIndexed(harness, channelId);
+ const searchService = getMessageSearchService()!;
+ await searchService.deleteAllDocuments();
+ const search = () => searchService.searchMessages(content, {channelIds: [channelId]}, {hitsPerPage: 25, page: 1});
+ expect((await search()).hits).toEqual([]);
+
+ const first = await queueAll();
+ const second = await queueAll();
+ expect(second.job_id).toBe(first.job_id);
+ expect(worker.pending.filter((job) => job.taskType === 'refreshSearchIndex')).toHaveLength(1);
+
+ await worker.drain({maxAttempts: 1});
+ const status = await createBuilder<StatusResponse>(harness, admin.token)
+ .get(`/admin/search/index-refreshes/${first.job_id}`)
+ .expect(200)
+ .execute();
+ expect(status).toMatchObject({status: 'completed', index_type: 'all'});
+ expect((await search()).hits.map((hit) => hit.id)).toEqual([message.id]);
+
+ const third = await queueAll();
+ expect(third.job_id).not.toBe(first.job_id);
+ });
+});
diff --git a/fluxer_api/src/api/worker/WorkerRunner.ts b/fluxer_api/src/api/worker/WorkerRunner.ts
index 28c39aed1..bca199bd9 100644
--- a/fluxer_api/src/api/worker/WorkerRunner.ts
+++ b/fluxer_api/src/api/worker/WorkerRunner.ts
@@ -16,7 +16,7 @@ import {
} from '@app/api/worker/WorkerHeartbeat';
import type {ConsumerMessages, FetchOptions, JsMsg} from '@nats-io/jetstream';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
-import {JobCancelledError, type WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
+import {JobCancelledError, JobFailedError, type WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
const MAX_DLQ_PUBLISH_ATTEMPTS = 3;
const DLQ_RETRY_DELAY_MS = 250;
@@ -470,7 +470,7 @@ export class WorkerRunner {
return false;
}
const deliveryCount = msg.info.deliveryCount;
- const isLastDelivery = deliveryCount >= this.maxDeliver;
+ const isLastDelivery = deliveryCount >= this.maxDeliver || error instanceof JobFailedError;
const errorMessage = error instanceof Error ? error.message : String(error);
if (archiveTask || isLastDelivery) {
Logger.error(
diff --git a/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts b/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts
index 16232fb05..886f6796e 100644
--- a/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts
+++ b/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts
@@ -1,10 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-import type {MessageID, UserID} from '@app/api/BrandedTypes';
+import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes';
import {createChannelID} from '@app/api/BrandedTypes';
import {isMessageSearchIndexable} from '@app/api/channel/services/message/MessageSearchService';
import {Logger} from '@app/api/Logger';
import {getMessageSearchService} from '@app/api/SearchFactory';
+import type {IMessageSearchService} from '@app/api/search/IMessageSearchService';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';
import {seconds} from 'itty-time';
@@ -17,6 +18,50 @@ const PayloadSchema = z.object({
});
const BULK_BATCH_SIZE = 5000;
const COMPLETION_KEY_TTL_SECONDS = seconds('1 day');
+
+export async function indexChannelMessageHistory(
+ searchService: IMessageSearchService,
+ channelId: ChannelID,
+): Promise<number> {
+ const {channelRepository, userRepository} = getWorkerDependencies();
+ const authorBotCache = new Map<UserID, boolean>();
+ let cursor: MessageID | undefined;
+ let totalIndexed = 0;
+ while (true) {
+ const messages = await channelRepository.messages.listMessages(channelId, cursor, BULK_BATCH_SIZE);
+ if (messages.length === 0) {
+ break;
+ }
+ const newAuthorIds = [
+ ...new Set(messages.map((m) => m.authorId).filter((id): id is UserID => id !== null && !authorBotCache.has(id))),
+ ];
+ if (newAuthorIds.length > 0) {
+ const users = await userRepository.listUsers(newAuthorIds);
+ for (const user of users) {
+ authorBotCache.set(user.id, user.isBot);
+ }
+ }
+ const indexable = messages.filter(isMessageSearchIndexable);
+ await searchService.bulkIndexMessages(indexable, authorBotCache);
+ totalIndexed += messages.length;
+ cursor = messages[messages.length - 1]!.id;
+ Logger.debug(
+ {channelId: channelId.toString(), indexed: totalIndexed, batch: messages.length},
+ 'Bulk indexed message batch',
+ );
+ if (messages.length < BULK_BATCH_SIZE) {
+ break;
+ }
+ }
+ const channel = await channelRepository.channelData.findUnique(channelId);
+ if (channel?.isThread()) {
+ await channelRepository.channelData.patchIndexedAt(channelId, new Date());
+ } else if (channel) {
+ await channelRepository.channelData.upsert({...channel.toRow(), indexed_at: new Date()});
+ }
+ return totalIndexed;
+}
+
const indexChannelMessages: WorkerTaskHandler = async (payload) => {
const validated = PayloadSchema.parse(payload);
const searchService = getMessageSearchService();
@@ -24,39 +69,9 @@ const indexChannelMessages: WorkerTaskHandler = async (payload) => {
throw new Error('Search service is not available in this worker process');
}
const channelId = createChannelID(BigInt(validated.channelId));
- const {channelRepository, userRepository, kvClient} = getWorkerDependencies();
- const authorBotCache = new Map<UserID, boolean>();
- let cursor: MessageID | undefined;
- let totalIndexed = 0;
+ const {kvClient} = getWorkerDependencies();
try {
- while (true) {
- const messages = await channelRepository.messages.listMessages(channelId, cursor, BULK_BATCH_SIZE);
- if (messages.length === 0) {
- break;
- }
- const newAuthorIds = [
- ...new Set(
- messages.map((m) => m.authorId).filter((id): id is UserID => id !== null && !authorBotCache.has(id)),
- ),
- ];
- if (newAuthorIds.length > 0) {
- const users = await userRepository.listUsers(newAuthorIds);
- for (const user of users) {
- authorBotCache.set(user.id, user.isBot);
- }
- }
- const indexable = messages.filter(isMessageSearchIndexable);
- await searchService.bulkIndexMessages(indexable, authorBotCache);
- totalIndexed += messages.length;
- cursor = messages[messages.length - 1]!.id;
- Logger.debug(
- {channelId: channelId.toString(), indexed: totalIndexed, batch: messages.length},
- 'Bulk indexed message batch',
- );
- if (messages.length < BULK_BATCH_SIZE) {
- break;
- }
- }
+ const totalIndexed = await indexChannelMessageHistory(searchService, channelId);
if (validated.completionKey && validated.channelCount) {
await kvClient.sadd(validated.completionKey, validated.channelId);
await kvClient.expire(validated.completionKey, COMPLETION_KEY_TTL_SECONDS);
@@ -71,12 +86,6 @@ const indexChannelMessages: WorkerTaskHandler = async (payload) => {
Logger.info({completionKey: validated.completionKey}, 'All channels indexed');
}
}
- const channel = await channelRepository.channelData.findUnique(channelId);
- if (channel?.isThread()) {
- await channelRepository.channelData.patchIndexedAt(channelId, new Date());
- } else if (channel) {
- await channelRepository.channelData.upsert({...channel.toRow(), indexed_at: new Date()});
- }
Logger.info({channelId: channelId.toString(), totalIndexed}, 'Bulk channel indexing complete');
} catch (error) {
Logger.error({error, channelId: channelId.toString()}, 'Failed to bulk index channel messages');
diff --git a/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts b/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts
index b1a03bdd1..8a348fb46 100644
--- a/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts
+++ b/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-import type {GuildID, ReportID, UserID} from '@app/api/BrandedTypes';
+import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository';
+import type {ChannelID, GuildID, ReportID, UserID} from '@app/api/BrandedTypes';
import {createGuildID} from '@app/api/BrandedTypes';
import {GuildDiscoveryRepository} from '@app/api/guild/repositories/GuildDiscoveryRepository';
import {Logger} from '@app/api/Logger';
@@ -17,13 +18,25 @@ import {
import type {IGuildMemberSearchService} from '@app/api/search/IGuildMemberSearchService';
import type {IMessageSearchService} from '@app/api/search/IMessageSearchService';
import {deleteChannelMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCleanup';
+import {
+ finishSearchReindexAll,
+ holdSearchReindexAll,
+ INDEX_REFRESH_STATUS_TTL_SECONDS,
+ indexRefreshStatusKey,
+} from '@app/api/search/SearchReindexAllRun';
import {backfillThreadSearch} from '@app/api/search/thread/ThreadSearchService';
+import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';
+import {indexChannelMessageHistory} from '@app/api/worker/tasks/IndexChannelMessages';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
-import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';
-import {seconds} from 'itty-time';
+import {
+ JobCancelledError,
+ JobFailedError,
+ type WorkerTaskHandler,
+ type WorkerTaskHelpers,
+} from '@pkgs/worker/src/contracts/WorkerTask';
import {z} from 'zod';
const INDEX_TYPES = [
@@ -35,6 +48,7 @@ const INDEX_TYPES = [
'guild_members',
'discovery',
'threads',
+ 'all',
] as const;
type IndexType = (typeof INDEX_TYPES)[number];
@@ -45,7 +59,6 @@ const PayloadSchema = z
job_id: z.string(),
admin_user_id: z.string(),
guild_id: z.string().optional(),
- user_id: z.string().optional(),
})
.refine(
(data) => {
@@ -64,7 +77,8 @@ const PayloadSchema = z
type RefreshSearchIndexPayload = z.infer<typeof PayloadSchema>;
const BATCH_SIZE = 1000;
-const PROGRESS_TTL = seconds('1 hour');
+const CHANNEL_PAGE_SIZE = 500;
+const CHANNEL_CONCURRENCY = 2;
function requireSearchService<T>(service: T | null): T {
if (!service) {
@@ -74,32 +88,19 @@ function requireSearchService<T>(service: T | null): T {
}
async function setProgress(kvClient: IKVProvider, progressKey: string, data: Record<string, unknown>): Promise<void> {
- await kvClient.set(progressKey, JSON.stringify(data), 'EX', PROGRESS_TTL);
+ await kvClient.set(progressKey, JSON.stringify(data), 'EX', INDEX_REFRESH_STATUS_TTL_SECONDS);
}
-async function reportInProgress(
- kvClient: IKVProvider,
- progressKey: string,
- indexType: IndexType,
- indexed: number,
-): Promise<void> {
- await setProgress(kvClient, progressKey, {
- status: 'in_progress',
- index_type: indexType,
- total: indexed,
- indexed,
- started_at: new Date().toISOString(),
- });
-}
+type ReportProgress = (indexed: number, total?: number, phase?: IndexType | 'messages') => Promise<void>;
+
+const ignoreProgress: ReportProgress = async () => {};
interface PaginateAndIndexOptions<TCursor, TItem> {
fetchPage: (cursor: TCursor | undefined) => Promise<Array<TItem>>;
indexBatch: (items: Array<TItem>) => Promise<number>;
getCursor: (item: TItem) => TCursor;
label: string;
- kvClient: IKVProvider;
- progressKey: string;
- indexType: IndexType;
+ report: ReportProgress;
}
async function paginateAndIndex<TCursor, TItem>(options: PaginateAndIndexOptions<TCursor, TItem>): Promise<number> {
@@ -113,7 +114,7 @@ async function paginateAndIndex<TCursor, TItem>(options: PaginateAndIndexOptions
indexedCount += batchCount;
cursor = options.getCursor(items[items.length - 1]!);
if (batchCount > 0) {
- await reportInProgress(options.kvClient, options.progressKey, options.indexType, indexedCount);
+ await options.report(indexedCount);
Logger.debug({count: batchCount, total: indexedCount}, `Indexed ${options.label} batch`);
}
}
@@ -126,11 +127,10 @@ async function paginateAndIndex<TCursor, TItem>(options: PaginateAndIndexOptions
type IndexHandler = (
payload: RefreshSearchIndexPayload,
helpers: WorkerTaskHelpers,
- kvClient: IKVProvider,
- progressKey: string,
+ report: ReportProgress,
) => Promise<number>;
-const refreshGuilds: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {
+const refreshGuilds: IndexHandler = async (_payload, _helpers, report) => {
const {guildRepository} = getWorkerDependencies();
const searchService = requireSearchService(getGuildSearchService());
await searchService.deleteAllDocuments();
@@ -142,12 +142,10 @@ const refreshGuilds: IndexHandler = async (_payload, _helpers, kvClient, progres
},
getCursor: (guild) => guild.id,
label: 'guild',
- kvClient,
- progressKey,
- indexType: 'guilds',
+ report,
});
};
-const refreshUsers: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {
+const refreshUsers: IndexHandler = async (_payload, _helpers, report) => {
const {userRepository} = getWorkerDependencies();
const searchService = requireSearchService(getUserSearchService());
await searchService.deleteAllDocuments();
@@ -160,13 +158,13 @@ const refreshUsers: IndexHandler = async (_payload, _helpers, kvClient, progress
if (users.length === 0) continue;
await searchService.indexUsers(users);
indexedCount += users.length;
- await reportInProgress(kvClient, progressKey, 'users', indexedCount);
+ await report(indexedCount);
Logger.debug({count: users.length, total: indexedCount}, 'Indexed user batch');
} while (pageState);
Logger.debug({count: indexedCount}, 'Refreshed user search index');
return indexedCount;
};
-const refreshReports: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {
+const refreshReports: IndexHandler = async (_payload, _helpers, report) => {
const {reportRepository} = getWorkerDependencies();
const searchService = requireSearchService(getReportSearchService());
await searchService.deleteAllDocuments();
@@ -178,12 +176,10 @@ const refreshReports: IndexHandler = async (_payload, _helpers, kvClient, progre
},
getCursor: (report) => report.reportId,
label: 'report',
- kvClient,
- progressKey,
- indexType: 'reports',
+ report,
});
};
-const refreshAuditLogs: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {
+const refreshAuditLogs: IndexHandler = async (_payload, _helpers, report) => {
const {adminRepository} = getWorkerDependencies();
const searchService = requireSearchService(getAuditLogSearchService());
await searchService.deleteAllDocuments();
@@ -195,13 +191,11 @@ const refreshAuditLogs: IndexHandler = async (_payload, _helpers, kvClient, prog
},
getCursor: (log) => log.logId,
label: 'audit log',
- kvClient,
- progressKey,
- indexType: 'audit_logs',
+ report,
});
};
-const refreshChannelMessages: IndexHandler = async (payload, helpers, kvClient, _progressKey) => {
- const {channelRepository} = getWorkerDependencies();
+const refreshChannelMessages: IndexHandler = async (payload, helpers) => {
+ const {channelRepository, kvClient} = getWorkerDependencies();
const guildId = createGuildID(BigInt(payload.guild_id!));
const searchService = requireSearchService<IMessageSearchService>(getMessageSearchService());
const guildChannels = await channelRepository.channelData.listGuildChannels(guildId, 'complete');
@@ -240,7 +234,7 @@ const refreshChannelMessages: IndexHandler = async (payload, helpers, kvClient,
);
return indexedChannels.length;
};
-const refreshGuildMembers: IndexHandler = async (payload, _helpers, kvClient, progressKey) => {
+const refreshGuildMembers: IndexHandler = async (payload, _helpers, report) => {
const {guildRepository, userRepository} = getWorkerDependencies();
const guildId = createGuildID(BigInt(payload.guild_id!));
const searchService = requireSearchService<IGuildMemberSearchService>(getGuildMemberSearchService());
@@ -262,9 +256,7 @@ const refreshGuildMembers: IndexHandler = async (payload, _helpers, kvClient, pr
},
getCursor: (member) => member.userId,
label: 'guild member',
- kvClient,
- progressKey,
- indexType: 'guild_members',
+ report,
});
const guild = await guildRepository.findUnique(guildId);
if (guild) {
@@ -273,7 +265,7 @@ const refreshGuildMembers: IndexHandler = async (payload, _helpers, kvClient, pr
return indexedCount;
};
const DISCOVERY_BATCH_SIZE = 200;
-const refreshDiscovery: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {
+const refreshDiscovery: IndexHandler = async (_payload, _helpers, report) => {
const {guildRepository} = getWorkerDependencies();
const searchService = requireSearchService(getGuildSearchService());
const discoveryRepository = new GuildDiscoveryRepository();
@@ -308,13 +300,7 @@ const refreshDiscovery: IndexHandler = async (_payload, _helpers, kvClient, prog
}
await Promise.all(updates);
synced += updates.length;
- await setProgress(kvClient, progressKey, {
- status: 'in_progress',
- index_type: 'discovery',
- total: guildIds.length,
- indexed: synced,
- started_at: new Date().toISOString(),
- });
+ await report(synced, guildIds.length);
}
return synced;
};
@@ -325,6 +311,126 @@ const refreshThreads: IndexHandler = async (payload) => {
await channelRepository.threads.clearGuildSearchBackfilled(guildId);
return backfillThreadSearch(channelRepository, guildId);
};
+async function forEachGuild(visit: (guildId: GuildID) => Promise<void>): Promise<void> {
+ const {guildRepository} = getWorkerDependencies();
+ let cursor: GuildID | undefined;
+ while (true) {
+ const guilds = await guildRepository.listAllGuildsPaginated(BATCH_SIZE, cursor);
+ for (const guild of guilds) {
+ await visit(guild.id);
+ }
+ if (guilds.length < BATCH_SIZE) return;
+ cursor = guilds[guilds.length - 1]!.id;
+ }
+}
+
+async function forEachChannelWithMessages(visit: (channelIds: Array<ChannelID>) => Promise<void>): Promise<void> {
+ const repository = new AttachmentDecayRepository();
+ let pageState: string | null = null;
+ let previous: ChannelID | null = null;
+ do {
+ const page = await repository.scanChannelsWithMessagesPage(CHANNEL_PAGE_SIZE, pageState);
+ pageState = page.pageState;
+ const channelIds = page.channelIds.filter((channelId) => channelId !== previous);
+ previous = page.channelIds.at(-1) ?? previous;
+ if (channelIds.length > 0) {
+ await visit(channelIds);
+ }
+ } while (pageState);
+}
+
+async function throwIfCancelled(helpers: WorkerTaskHelpers): Promise<void> {
+ if (await helpers.shouldCancel()) {
+ throw new JobCancelledError();
+ }
+}
+
+async function refreshEveryGuild(
+ handler: IndexHandler,
+ payload: RefreshSearchIndexPayload,
+ helpers: WorkerTaskHelpers,
+ guildCount: number,
+ report: ReportProgress,
+): Promise<number> {
+ let visited = 0;
+ let indexed = 0;
+ await report(0, guildCount);
+ await forEachGuild(async (guildId) => {
+ indexed += await handler({...payload, guild_id: guildId.toString()}, helpers, ignoreProgress);
+ visited++;
+ await report(visited, Math.max(guildCount, visited));
+ await throwIfCancelled(helpers);
+ });
+ return indexed;
+}
+
+async function refreshEveryChannel(
+ searchService: IMessageSearchService,
+ helpers: WorkerTaskHelpers,
+ report: ReportProgress,
+): Promise<number> {
+ await searchService.deleteAllDocuments();
+ let channelCount = 0;
+ await forEachChannelWithMessages(async (channelIds) => {
+ channelCount += channelIds.length;
+ });
+ let visited = 0;
+ let indexed = 0;
+ await report(0, channelCount);
+ await forEachChannelWithMessages(async (channelIds) => {
+ const counts = await mapWithConcurrency(channelIds, CHANNEL_CONCURRENCY, async (channelId) => {
+ const count = await indexChannelMessageHistory(searchService, channelId);
+ await searchService.refreshIndex().catch((error) => {
+ Logger.warn({error, channelId: channelId.toString()}, 'Search refresh after channel reindex failed');
+ });
+ return count;
+ });
+ indexed += counts.reduce((sum, count) => sum + count, 0);
+ visited += channelIds.length;
+ await report(visited, Math.max(channelCount, visited));
+ await throwIfCancelled(helpers);
+ });
+ return indexed;
+}
+
+const refreshAll: IndexHandler = async (payload, helpers, report) => {
+ let indexed = 0;
+ let guildCount = 0;
+ const globalHandlers = [
+ ['users', refreshUsers],
+ ['guilds', refreshGuilds],
+ ['discovery', refreshDiscovery],
+ ['reports', refreshReports],
+ ['audit_logs', refreshAuditLogs],
+ ] as const;
+ for (const [indexType, handler] of globalHandlers) {
+ const count = await handler(payload, helpers, (done, total) => report(done, total, indexType));
+ if (indexType === 'guilds') {
+ guildCount = count;
+ }
+ indexed += count;
+ await throwIfCancelled(helpers);
+ }
+ const memberSearch = getGuildMemberSearchService();
+ if (memberSearch) {
+ await memberSearch.deleteAllDocuments();
+ indexed += await refreshEveryGuild(refreshGuildMembers, payload, helpers, guildCount, (done, total) =>
+ report(done, total, 'guild_members'),
+ );
+ }
+ const threadSearch = getThreadSearchService();
+ if (threadSearch) {
+ await threadSearch.deleteAllDocuments();
+ indexed += await refreshEveryGuild(refreshThreads, payload, helpers, guildCount, (done, total) =>
+ report(done, total, 'threads'),
+ );
+ }
+ const messageSearch = getMessageSearchService();
+ if (messageSearch) {
+ indexed += await refreshEveryChannel(messageSearch, helpers, (done, total) => report(done, total, 'messages'));
+ }
+ return indexed;
+};
const INDEX_HANDLERS: Record<IndexType, IndexHandler> = {
guilds: refreshGuilds,
users: refreshUsers,
@@ -334,32 +440,69 @@ const INDEX_HANDLERS: Record<IndexType, IndexHandler> = {
guild_members: refreshGuildMembers,
discovery: refreshDiscovery,
threads: refreshThreads,
+ all: refreshAll,
};
-const refreshSearchIndex: WorkerTaskHandler = async (payload, helpers) => {
- const validated = PayloadSchema.parse(payload);
- helpers.logger.debug({payload: validated}, 'Processing refreshSearchIndex task');
- const {kvClient} = getWorkerDependencies();
- const progressKey = `index_refresh_status:${validated.job_id}`;
- await reportInProgress(kvClient, progressKey, validated.index_type, 0);
+async function runRefresh(
+ payload: RefreshSearchIndexPayload,
+ helpers: WorkerTaskHelpers,
+ kvClient: IKVProvider,
+): Promise<number> {
+ const progressKey = indexRefreshStatusKey(payload.job_id);
+ const indexType = payload.index_type;
+ const report: ReportProgress = async (indexed, total = indexed, phase) => {
+ if (indexType === 'all') {
+ await holdSearchReindexAll(kvClient, payload.job_id);
+ }
+ await setProgress(kvClient, progressKey, {
+ status: 'in_progress',
+ index_type: indexType,
+ ...(phase ? {phase} : {}),
+ total,
+ indexed,
+ started_at: new Date().toISOString(),
+ });
+ };
+ await report(0);
try {
- const handler = INDEX_HANDLERS[validated.index_type];
- const indexedCount = await handler(validated, helpers, kvClient, progressKey);
+ const indexedCount = await INDEX_HANDLERS[indexType](payload, helpers, report);
await setProgress(kvClient, progressKey, {
status: 'completed',
- index_type: validated.index_type,
+ index_type: indexType,
total: indexedCount,
indexed: indexedCount,
completed_at: new Date().toISOString(),
});
+ return indexedCount;
} catch (error) {
- Logger.error({error, payload: validated}, 'Failed to refresh search index');
+ Logger.error({error, payload}, 'Failed to refresh search index');
+ const message = error instanceof Error ? error.message : 'Unknown error';
await setProgress(kvClient, progressKey, {
status: 'failed',
- index_type: validated.index_type,
- error: error instanceof Error ? error.message : 'Unknown error',
+ index_type: indexType,
+ error: message,
failed_at: new Date().toISOString(),
});
- throw error;
+ if (error instanceof JobCancelledError) {
+ throw error;
+ }
+ throw new JobFailedError(message);
+ }
+}
+const refreshSearchIndex: WorkerTaskHandler = async (payload, helpers) => {
+ const validated = PayloadSchema.parse(payload);
+ helpers.logger.debug({payload: validated}, 'Processing refreshSearchIndex task');
+ const {kvClient} = getWorkerDependencies();
+ if (validated.index_type !== 'all') {
+ return {indexed: await runRefresh(validated, helpers, kvClient)};
+ }
+ if (!(await holdSearchReindexAll(kvClient, validated.job_id))) {
+ helpers.logger.info({jobId: validated.job_id}, 'Another search reindex is running, stopping this one');
+ return {superseded: true};
+ }
+ try {
+ return {indexed: await runRefresh(validated, helpers, kvClient)};
+ } finally {
+ await finishSearchReindexAll(kvClient, validated.job_id);
}
};
diff --git a/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx b/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx
index 03ed45c02..2dcac160f 100644
--- a/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx
+++ b/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx
@@ -21,14 +21,14 @@ A rebuild clears the index first, and search over that index returns partial res
| discovery | Instance | Discovery details of approved listings in the `guilds` index |
| channel_messages<sup>1</sup> | Guild | Messages in every channel and thread of one guild |
| guild_members | Guild | Members of one guild |
-| favorite_memes<sup>2</sup> | User | Favorited [memes](/http-api/memes/) of one user |
-| threads<sup>3</sup> | Guild | Threads used by [Search threads](/http-api/threads/#search-threads) |
+| threads<sup>2</sup> | Guild | Threads used by [Search threads](/http-api/threads/#search-threads) |
+| all<sup>3</sup> | Instance | Every index above, for every guild, and messages in every channel, thread and DM |
<sup>1</sup> Progress shows `completed` once every channel is queued for indexing, and `total` and `indexed` count channels. Messages keep indexing after that.
-<sup>2</sup> The rebuild is queued but always fails, and its progress reads as `not_found`.
+<sup>2</sup> [Search threads](/http-api/threads/#search-threads) returns 202 for the guild until the rebuild completes.
-<sup>3</sup> [Search threads](/http-api/threads/#search-threads) returns 202 for the guild until the rebuild completes.
+<sup>3</sup> Rebuilds one index at a time and clears each one first, including `threads`. Only one `all` rebuild runs at a time.
## Search index refresh object
@@ -64,8 +64,9 @@ An unknown or expired `job_id` returns the not found structure, and every other
| --- | --- | --- |
| status | string | One of `in_progress`, `completed`, or `failed` |
| index_type | string | The [search index name](#search-index-names) being rebuilt |
-| total? | number | Equal to `indexed` until `status` is `completed`, except for `discovery`, where it is the number of approved listings |
-| indexed? | number | The number of documents written so far |
+| phase? | string | For `all`, the index it is on, `messages`, or `queued` before it starts |
+| total? | number | Equal to `indexed` until `status` is `completed`, except for `discovery`, where it is the number of approved listings, and in the `all` phases that count guilds or channels |
+| indexed? | number | The number of documents written so far, or of guilds or channels done in the `guild_members`, `threads` and `messages` phases of `all` |
| started_at? | ISO8601 timestamp | The time the rebuild last reported progress |
| completed_at? | ISO8601 timestamp | The time the rebuild finished, when `status` is `completed` |
| failed_at? | ISO8601 timestamp | The time the rebuild failed, when `status` is `failed` |
@@ -75,7 +76,7 @@ An unknown or expired `job_id` returns the not found structure, and every other
<RouteHeader method="POST" path="/v1/admin/search/indexes/{index_name}/refreshes" auditReason />
-Queues a rebuild of the index and returns its `job_id`. You need the `guild:lookup` [ACL](/admin-api/#acl-evaluation). A failed rebuild is not retried, so queue a new one.
+Queues a rebuild of the index and returns its `job_id`. You need the `guild:lookup` [ACL](/admin-api/#acl-evaluation). A failed rebuild is not retried, so queue a new one. While an `all` rebuild runs, queueing `all` again returns its `job_id`.
### Path parameters
@@ -88,14 +89,14 @@ Queues a rebuild of the index and returns its `job_id`. You need the `guild:look
| Field | Type | Description |
| --- | --- | --- |
| guild_id? | snowflake | The ID of the guild to rebuild, needed by `channel_messages`, `guild_members` and `threads` |
-| user_id? | snowflake | The ID of the user to rebuild, needed by `favorite_memes` |
### Response
| Status | Body | Condition |
| --- | --- | --- |
-| 200 | [search index refresh](#search-index-refresh-object) object | The rebuild was queued |
-| 400 | [error response](/admin-api/#error-response) | `INVALID_FORM_BODY` when the index needs `guild_id` or `user_id` and it is missing |
+| 200 | [search index refresh](#search-index-refresh-object) object | The rebuild was queued or is already running |
+| 400 | [error response](/admin-api/#error-response) | `INVALID_FORM_BODY` when the index needs `guild_id` and it is missing |
+| 403 | [error response](/admin-api/#error-response) | `FEATURE_TEMPORARILY_DISABLED` when `index_name` is `all` on an instance with no search backend |
Fluxer records an Admin audit log entry with the action `queue_refresh_index` and the target type `search_index`.
diff --git a/packages/schema/src/domains/admin/AdminSchemas.ts b/packages/schema/src/domains/admin/AdminSchemas.ts
index 8e40f77db..5af146210 100644
--- a/packages/schema/src/domains/admin/AdminSchemas.ts
+++ b/packages/schema/src/domains/admin/AdminSchemas.ts
@@ -158,9 +158,9 @@ const SearchIndexTypeEnum = createNamedStringLiteralUnion(
['audit_logs', 'audit_logs', 'Audit log search index'],
['channel_messages', 'channel_messages', 'Channel message search index'],
['guild_members', 'guild_members', 'Guild member search index'],
- ['favorite_memes', 'favorite_memes', 'Favourite meme search index'],
['discovery', 'discovery', 'Discovery guild search index'],
['threads', 'threads', 'Thread search index'],
+ ['all', 'all', 'Every search index, with messages in every channel and DM'],
],
'Type of search index to refresh',
);
@@ -308,7 +308,6 @@ export type SearchIndexNameParam = z.infer<typeof SearchIndexNameParam>;
export const RefreshSearchIndexRequest = z.object({
guild_id: SnowflakeType.optional().describe('Specific guild ID to reindex'),
- user_id: SnowflakeType.optional().describe('Specific user ID to reindex'),
});
export type RefreshSearchIndexRequest = z.infer<typeof RefreshSearchIndexRequest>;
@@ -1257,6 +1256,7 @@ export const IndexRefreshStatusResponse = z.union([
z.object({
status: IndexRefreshStatusEnum,
index_type: z.string().describe('Type of index being refreshed'),
+ phase: z.string().optional().describe('The index an `all` rebuild is working on, or queued before it starts'),
total: z.number().optional().describe('Total number of items to index'),
indexed: z.number().optional().describe('Number of items indexed so far'),
started_at: z.string().optional().describe('ISO 8601 timestamp when the job started'),