A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(api): rebuild every search index in one job (#3616)
Hampusd20b31fa095d
diff --git a/fluxer_admin/openapi-admin.json b/fluxer_admin/openapi-admin.jsonindex 567115215..ad7ba5d1d 100644--- a/fluxer_admin/openapi-admin.json+++ b/fluxer_admin/openapi-admin.json@@ -7173,7 +7173,7 @@"content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}}},- "description": "Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and favorite_memes requires user_id. Requires GUILD_LOOKUP permission.",+ "description": "Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and all rebuilds every index, one at a time. Queueing all while one is running returns the running refresh ID. Requires GUILD_LOOKUP permission.","security": [{"adminApiKey": []}],"parameters": [{@@ -7189,9 +7189,9 @@"audit_logs","channel_messages","guild_members",- "favorite_memes","discovery",- "threads"+ "threads",+ "all"],"x-enumDescriptions": ["Guild search index",@@ -7200,9 +7200,9 @@"Audit log search index","Channel message search index","Guild member search index",- "Favourite meme search index","Discovery guild search index",- "Thread search index"+ "Thread search index",+ "Every search index, with messages in every channel and DM"],"enum": ["guilds",@@ -7211,9 +7211,9 @@"audit_logs","channel_messages","guild_members",- "favorite_memes","discovery",- "threads"+ "threads",+ "all"],"type": "string"},@@ -14107,10 +14107,6 @@"guild_id": {"description": "Specific guild ID to reindex","allOf": [{"$ref": "#/components/schemas/SnowflakeType"}]- },- "user_id": {- "description": "Specific user ID to reindex",- "allOf": [{"$ref": "#/components/schemas/SnowflakeType"}]}}},@@ -14143,6 +14139,10 @@"type": "string"},"index_type": {"type": "string", "description": "Type of index being refreshed"},+ "phase": {+ "description": "The index an `all` rebuild is working on, or queued before it starts",+ "type": "string"+ },"total": {"description": "Total number of items to index", "type": "number"},"indexed": {"description": "Number of items indexed so far", "type": "number"},"started_at": {"description": "ISO 8601 timestamp when the job started", "type": "string"},diff --git a/fluxer_admin/src/api/search.rs b/fluxer_admin/src/api/search.rsindex 2ac4cba92..de3efdf33 100644--- a/fluxer_admin/src/api/search.rs+++ b/fluxer_admin/src/api/search.rs@@ -18,7 +18,6 @@ impl AdminApiClient {.map_err(|e| ApiError::Parse(e.to_string()))?;let body = generated_types::RefreshSearchIndexRequest {guild_id: guild_id.map(snowflake),- user_id: None,};let response = self.generated()@@ -53,6 +52,7 @@ fn index_refresh_status(generated_types::IndexRefreshStatusResponse::Variant1 {status,index_type,+ phase,total,indexed,started_at,@@ -62,6 +62,7 @@ fn index_refresh_status(} => Ok(IndexRefreshStatusResponse::Progress {status: status.to_string(),index_type: Some(index_type),+ phase,total: total.map(|value| float_to_u64(value, "total")).transpose()?,diff --git a/fluxer_admin/src/api/types/search.rs b/fluxer_admin/src/api/types/search.rsindex 9e0f75ec6..ea612bbbd 100644--- a/fluxer_admin/src/api/types/search.rs+++ b/fluxer_admin/src/api/types/search.rs@@ -17,6 +17,7 @@ pub enum IndexRefreshStatusResponse {Progress {status: String,index_type: Option<String>,+ phase: Option<String>,total: Option<u64>,indexed: Option<u64>,started_at: Option<String>,diff --git a/fluxer_admin/src/routes/system.rs b/fluxer_admin/src/routes/system.rsindex e68c9d647..1d302bb1d 100644--- a/fluxer_admin/src/routes/system.rs+++ b/fluxer_admin/src/routes/system.rs@@ -157,8 +157,10 @@ async fn search_index_page(let job_id_str = query.job_id.unwrap_or_default();let mut status: Option<String> = None;+ let mut phase: Option<String> = None;let mut total: Option<u64> = None;let mut indexed: Option<u64> = None;+ let mut error: Option<String> = None;if !job_id_str.is_empty() {let client =@@ -170,13 +172,17 @@ async fn search_index_page(}crate::api::types::IndexRefreshStatusResponse::Progress {status: s,+ phase: p,total: t,indexed: i,+ error: e,..} => {status = Some(s);+ phase = p;total = t;indexed = i;+ error = e;}},Err(error) => {@@ -189,11 +195,12 @@ async fn search_index_page(Some(templates::pages::search_index::RefreshStatus {job_id: &job_id_str,status: status.as_deref(),+ phase: phase.as_deref(),total,indexed,started_at: None,completed_at: None,- error: None,+ error: error.as_deref(),})} else {Nonediff --git a/fluxer_admin/src/templates/pages/search_index.rs b/fluxer_admin/src/templates/pages/search_index.rsindex c90341192..200188ad8 100644--- a/fluxer_admin/src/templates/pages/search_index.rs+++ b/fluxer_admin/src/templates/pages/search_index.rs@@ -6,7 +6,9 @@ use crate::{middleware::auth::AuthContext,templates::{components::{- auto_refresh::auto_refresh, form::csrf_input, page_container::page_header,+ auto_refresh::auto_refresh,+ form::{csrf_input, submit_button},+ page_container::page_header,section_card::section_card_simple,},layout::admin_layout,@@ -43,6 +45,25 @@ fn disabled_reindex_button(title: &str) -> Markup {}}+fn reindex_everything(base: &str, csrf_token: &str) -> Markup {+ section_card_simple(+ "Reindex everything",+ html! {+ p class="text-sm text-neutral-500 mb-3" {+ "Rebuilds every index on this page, the members and threads of every guild, \+ and the messages in every channel and DM, one at a time. Search shows \+ partial results until it finishes, which can take hours on a large instance."+ }+ form method="post" action={(base) "/search-index?action=reindex"} class="w-fit"+ onsubmit="return confirm('Rebuild every search index? Search shows partial results until it finishes.')" {+ (csrf_input(csrf_token))+ input type="hidden" name="index_type" value="all";+ (submit_button("Reindex everything"))+ }+ },+ )+}+fn reindex_controls(base: &str, csrf_token: &str) -> Markup {section_card_simple("Global Search Indexes",@@ -76,6 +97,21 @@ fn reindex_controls(base: &str, csrf_token: &str) -> Markup {)}+fn phase_label(phase: &str) -> Option<&'static str> {+ match phase {+ "queued" => Some("Waiting for a worker to start"),+ "users" => Some("Rebuilding users"),+ "guilds" => Some("Rebuilding guilds"),+ "discovery" => Some("Rebuilding the Explore index"),+ "reports" => Some("Rebuilding reports"),+ "audit_logs" => Some("Rebuilding audit logs"),+ "guild_members" => Some("Rebuilding guild members, counted in guilds"),+ "threads" => Some("Rebuilding threads, counted in guilds"),+ "messages" => Some("Rebuilding messages, counted in channels"),+ _ => None,+ }+}+fn format_status_label(status: &str) -> &str {match status {"in_progress" => "In progress",@@ -89,6 +125,7 @@ fn format_status_label(status: &str) -> &str {pub struct RefreshStatus<'a> {pub job_id: &'a str,pub status: Option<&'a str>,+ pub phase: Option<&'a str>,pub total: Option<u64>,pub indexed: Option<u64>,pub started_at: Option<&'a str>,@@ -121,6 +158,9 @@ fn status_section(base: &str, rs: &RefreshStatus) -> Markup {}@if is_in_progress {+ @if let Some(label) = rs.phase.and_then(phase_label) {+ p class="text-sm text-neutral-700" { (label) }+ }@if let (Some(idx), Some(tot)) = (rs.indexed, rs.total) {@let pct = (u128::from(idx) * 100).checked_div(u128::from(tot)).unwrap_or(0);div class="space-y-2" {@@ -182,6 +222,7 @@ pub fn search_index_page(let content = html! {(page_header("Search Index Management", None))div class="space-y-6" {+ (reindex_everything(base, csrf_token))(reindex_controls(base, csrf_token))@if let Some(rs) = refresh {(status_section(base, rs))diff --git a/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts b/fluxer_api/pkgs/worker/src/contracts/WorkerTask.tsindex a12ae8157..3e062ee17 100644--- a/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts+++ b/fluxer_api/pkgs/worker/src/contracts/WorkerTask.ts@@ -28,6 +28,13 @@ export type WorkerTaskHandler<Payload = Record<string, unknown>> = (helpers: WorkerTaskHelpers,) => Promise<WorkerTaskResult> | Promise<void>;+export class JobFailedError extends Error {+ constructor(message: string) {+ super(message);+ this.name = 'JobFailedError';+ }+}+export class JobCancelledError extends Error {constructor(message = 'Job cancelled by admin') {super(message);diff --git a/fluxer_api/src/api/admin/controllers/SearchAdminController.ts b/fluxer_api/src/api/admin/controllers/SearchAdminController.tsindex 7266c6509..59dbd56e6 100644--- a/fluxer_api/src/api/admin/controllers/SearchAdminController.ts+++ b/fluxer_api/src/api/admin/controllers/SearchAdminController.ts@@ -26,7 +26,7 @@ export function SearchAdminController(app: HonoApp) {operationId: 'create_admin_search_index_refresh',summary: 'Refresh a search index',description:- 'Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and favorite_memes requires user_id. Requires GUILD_LOOKUP permission.',+ 'Trigger a full or partial rebuild of the named search index. Creates a background job and returns its refresh ID for status tracking. The channel_messages, guild_members and threads indexes are rebuilt one guild at a time and require guild_id, and all rebuilds every index, one at a time. Queueing all while one is running returns the running refresh ID. Requires GUILD_LOOKUP permission.',responseSchema: RefreshSearchIndexResponse,statusCode: 200,security: 'adminApiKey',@@ -37,10 +37,10 @@ export function SearchAdminController(app: HonoApp) {const adminUserId = ctx.get('adminUserId');const auditLogReason = ctx.get('auditLogReason');const {index_name} = ctx.req.valid('param');- const {guild_id, user_id} = ctx.req.valid('json');+ const {guild_id} = ctx.req.valid('json');return ctx.json(await adminService.searchService.refreshSearchIndex(- {index_type: index_name, guild_id, user_id},+ {index_type: index_name, guild_id},adminUserId,auditLogReason,),diff --git a/fluxer_api/src/api/admin/services/AdminSearchService.ts b/fluxer_api/src/api/admin/services/AdminSearchService.tsindex 1a0afac3d..385430b37 100644--- a/fluxer_api/src/api/admin/services/AdminSearchService.ts+++ b/fluxer_api/src/api/admin/services/AdminSearchService.ts@@ -8,7 +8,13 @@ import {createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes';import {isSyntheticUserId} from '@app/api/constants/Core';import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';import {Logger} from '@app/api/Logger';-import {getGuildSearchService, getUserSearchService} from '@app/api/SearchFactory';+import {getGuildSearchService, getMessageSearchService, getUserSearchService} from '@app/api/SearchFactory';+import {+ claimSearchReindexAll,+ finishSearchReindexAll,+ INDEX_REFRESH_STATUS_TTL_SECONDS,+ indexRefreshStatusKey,+} from '@app/api/search/SearchReindexAllRun';import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';import type {UserSearchFilters} from '@fluxer/schema/src/contracts/search/SearchDocumentTypes';@@ -21,15 +27,14 @@ interface RefreshSearchIndexJobPayload extends WorkerJobPayload {| 'reports'| 'audit_logs'| 'channel_messages'- | 'favorite_memes'| 'guild_members'| 'discovery'- | 'threads';+ | 'threads'+ | 'all';admin_user_id: string;audit_log_reason: string | null;job_id: string;guild_id?: string;- user_id?: string;}interface AdminSearchServiceDeps {@@ -171,17 +176,16 @@ export class AdminSearchService {| 'audit_logs'| 'channel_messages'| 'guild_members'- | 'favorite_memes'| 'discovery'- | 'threads';+ | 'threads'+ | 'all';guild_id?: bigint;- user_id?: bigint;},adminUserId: UserID,auditLogReason: string | null,) {const {auditService} = this.deps;- const {worker: workerService, snowflake: snowflakeService} = this.deps.apiContext.services;+ const {snowflake: snowflakeService} = this.deps.apiContext.services;const jobId = (await snowflakeService.generate()).toString();const payload: RefreshSearchIndexJobPayload = {index_type: data.index_type,@@ -207,28 +211,23 @@ export class AdminSearchService {}payload.guild_id = data.guild_id.toString();}- if (data.index_type === 'favorite_memes') {- if (!data.user_id) {- throw InputValidationError.create('user_id', 'user_id is required for the favorite_memes index type');- }- payload.user_id = data.user_id.toString();+ let queuedJobId = jobId;+ if (data.index_type === 'all') {+ queuedJobId = await this.queueReindexAll(payload);+ } else {+ await this.queueRefresh(payload);}- await workerService.addJob('refreshSearchIndex', payload, {- jobKey: `refreshSearchIndex_${data.index_type}_${jobId}`,- maxAttempts: 1,- requireLedger: true,- });- Logger.debug({index_type: data.index_type, job_id: jobId}, 'Queued search index refresh job');+ Logger.debug({index_type: data.index_type, job_id: queuedJobId}, 'Queued search index refresh job');const metadata = new Map([['index_type', data.index_type],- ['job_id', jobId],+ ['job_id', queuedJobId],]);+ if (queuedJobId !== jobId) {+ metadata.set('already_running', 'true');+ }if (data.guild_id) {metadata.set('guild_id', data.guild_id.toString());}- if (data.user_id) {- metadata.set('user_id', data.user_id.toString());- }await auditService.createAuditLog({adminUserId,targetType: 'search_index',@@ -239,23 +238,56 @@ export class AdminSearchService {});return {success: true,- job_id: jobId,+ job_id: queuedJobId,};}+ private async queueRefresh(payload: RefreshSearchIndexJobPayload): Promise<void> {+ await this.deps.apiContext.services.worker.addJob('refreshSearchIndex', payload, {+ jobKey: `refreshSearchIndex_${payload.index_type}_${payload.job_id}`,+ maxAttempts: 1,+ requireLedger: true,+ });+ }++ private async queueReindexAll(payload: RefreshSearchIndexJobPayload): Promise<string> {+ if (!getMessageSearchService()) {+ throw new FeatureTemporarilyDisabledError();+ }+ const {kv} = this.deps.apiContext.services;+ const running = await claimSearchReindexAll(kv, payload.job_id);+ if (running) {+ return running;+ }+ const queued = {status: 'in_progress', index_type: 'all', phase: 'queued', started_at: new Date().toISOString()};+ try {+ await kv.set(+ indexRefreshStatusKey(payload.job_id),+ JSON.stringify(queued),+ 'EX',+ INDEX_REFRESH_STATUS_TTL_SECONDS,+ );+ await this.queueRefresh(payload);+ } catch (error) {+ await finishSearchReindexAll(kv, payload.job_id);+ throw error;+ }+ return payload.job_id;+ }+async getIndexRefreshStatus(jobId: string) {const {cache: cacheService} = this.deps.apiContext.services;- const statusKey = `index_refresh_status:${jobId}`;const status = await cacheService.get<{status: 'in_progress' | 'completed' | 'failed';index_type: string;+ phase?: string;total?: number;indexed?: number;started_at?: string;completed_at?: string;failed_at?: string;error?: string;- }>(statusKey);+ }>(indexRefreshStatusKey(jobId));if (!status) {return {status: 'not_found' as const,diff --git a/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts b/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.tsindex 2316703d9..9a878151f 100644--- a/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts+++ b/fluxer_api/src/api/admin/tests/audit_coverage/SearchAdminAuditCases.ts@@ -7,14 +7,14 @@ export const SearchAdminAuditCases: ReadonlyArray<AdminAuditCoverageCase> = [{method: 'POST',route: '/admin/search/indexes/:index_name/refreshes',- async prepare({admin}) {+ async prepare() {return {- request: {path: '/admin/search/indexes/favorite_memes/refreshes', body: {user_id: admin.userId}},+ request: {path: '/admin/search/indexes/users/refreshes', body: {}},expected: {action: 'queue_refresh_index',targetType: 'search_index',targetId: '0',- metadata: {index_type: 'favorite_memes', job_id: expect.any(String), user_id: admin.userId},+ metadata: {index_type: 'users', job_id: expect.any(String)},},};},diff --git a/fluxer_api/src/api/search/SearchReindexAllRun.ts b/fluxer_api/src/api/search/SearchReindexAllRun.tsnew file mode 100644index 000000000..67796bfe2--- /dev/null+++ b/fluxer_api/src/api/search/SearchReindexAllRun.ts@@ -0,0 +1,34 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';+import {seconds} from 'itty-time';++const RUN_KEY = 'index_refresh:all';+const RUN_TTL_SECONDS = seconds('1 hour');+const CLAIM_ATTEMPTS = 3;++export const INDEX_REFRESH_STATUS_TTL_SECONDS = seconds('1 hour');++export function indexRefreshStatusKey(jobId: string): string {+ return `index_refresh_status:${jobId}`;+}++export async function claimSearchReindexAll(kvClient: IKVProvider, jobId: string): Promise<string | null> {+ for (let attempt = 0; attempt < CLAIM_ATTEMPTS; attempt++) {+ if (await kvClient.acquireLock(RUN_KEY, jobId, RUN_TTL_SECONDS)) return null;+ const running = await kvClient.get(RUN_KEY);+ if (running) return running;+ }+ throw new Error('Could not claim the search reindex run');+}++export async function holdSearchReindexAll(kvClient: IKVProvider, jobId: string): Promise<boolean> {+ return (+ (await kvClient.extendLock(RUN_KEY, jobId, RUN_TTL_SECONDS)) ||+ (await kvClient.acquireLock(RUN_KEY, jobId, RUN_TTL_SECONDS))+ );+}++export async function finishSearchReindexAll(kvClient: IKVProvider, jobId: string): Promise<void> {+ await kvClient.releaseLock(RUN_KEY, jobId);+}diff --git a/fluxer_api/src/api/search/tests/SearchReindexAll.test.ts b/fluxer_api/src/api/search/tests/SearchReindexAll.test.tsnew file mode 100644index 000000000..9fd622aa6--- /dev/null+++ b/fluxer_api/src/api/search/tests/SearchReindexAll.test.ts@@ -0,0 +1,93 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {createAdminWithAcls} from '@app/api/admin/tests/AdminTestUtils';+import {createTestAccount, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';+import {resetApiServicesForTesting} from '@app/api/CreateApiContext';+import {createGuild} from '@app/api/guild/tests/GuildTestUtils';+import {markChannelAsIndexed, sendMessage} from '@app/api/message/tests/MessageTestUtils';+import {getSnowflakeService, setInjectedWorkerService} from '@app/api/middleware/ServiceRegistry';+import {getMessageSearchService} from '@app/api/SearchFactory';+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';+import {NoopWorkerService} from '@app/api/test/NoopWorkerService';+import {SyncTaskWorkerService} from '@app/api/test/SyncTaskWorkerService';+import {createBuilder} from '@app/api/test/TestRequestBuilder';+import refreshSearchIndex from '@app/api/worker/tasks/RefreshSearchIndex';+import {clearWorkerDependencies, setWorkerDependencies} from '@app/api/worker/WorkerContext';+import {initializeWorkerDependencies} from '@app/api/worker/WorkerDependencies';+import {AdminACLs} from '@fluxer/constants/src/AdminACLs';+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it} from 'vitest';++interface RefreshResponse {+ job_id: string;+}++interface StatusResponse {+ status: string;+ index_type?: string;+}++describe('rebuilding every search index', () => {+ let harness: ApiTestHarness;+ let worker: SyncTaskWorkerService;+ let admin: TestAccount;++ beforeAll(async () => {+ harness = await createApiTestHarness({search: 'enabled'});+ });++ beforeEach(async () => {+ await harness.reset();+ setWorkerDependencies(await initializeWorkerDependencies(getSnowflakeService()));+ worker = new SyncTaskWorkerService({refreshSearchIndex}, {deferred: true});+ setInjectedWorkerService(worker);+ resetApiServicesForTesting();+ admin = await createAdminWithAcls(harness, [AdminACLs.GUILD_LOOKUP]);+ });++ afterEach(() => {+ setInjectedWorkerService(new NoopWorkerService());+ resetApiServicesForTesting();+ });++ afterAll(async () => {+ clearWorkerDependencies();+ await harness.shutdown();+ });++ function queueAll(): Promise<RefreshResponse> {+ return createBuilder<RefreshResponse>(harness, admin.token)+ .post('/admin/search/indexes/all/refreshes')+ .body({})+ .expect(200)+ .execute();+ }++ it('queues one rebuild at a time and brings back wiped messages', async () => {+ const account = await createTestAccount(harness);+ const guild = await createGuild(harness, account.token, 'Reindex Guild');+ const channelId = guild.system_channel_id!;+ const content = `reindex-everything-${Date.now()}`;+ const message = await sendMessage(harness, account.token, channelId, content);+ await markChannelAsIndexed(harness, channelId);+ const searchService = getMessageSearchService()!;+ await searchService.deleteAllDocuments();+ const search = () => searchService.searchMessages(content, {channelIds: [channelId]}, {hitsPerPage: 25, page: 1});+ expect((await search()).hits).toEqual([]);++ const first = await queueAll();+ const second = await queueAll();+ expect(second.job_id).toBe(first.job_id);+ expect(worker.pending.filter((job) => job.taskType === 'refreshSearchIndex')).toHaveLength(1);++ await worker.drain({maxAttempts: 1});+ const status = await createBuilder<StatusResponse>(harness, admin.token)+ .get(`/admin/search/index-refreshes/${first.job_id}`)+ .expect(200)+ .execute();+ expect(status).toMatchObject({status: 'completed', index_type: 'all'});+ expect((await search()).hits.map((hit) => hit.id)).toEqual([message.id]);++ const third = await queueAll();+ expect(third.job_id).not.toBe(first.job_id);+ });+});diff --git a/fluxer_api/src/api/worker/WorkerRunner.ts b/fluxer_api/src/api/worker/WorkerRunner.tsindex 28c39aed1..bca199bd9 100644--- a/fluxer_api/src/api/worker/WorkerRunner.ts+++ b/fluxer_api/src/api/worker/WorkerRunner.ts@@ -16,7 +16,7 @@ import {} from '@app/api/worker/WorkerHeartbeat';import type {ConsumerMessages, FetchOptions, JsMsg} from '@nats-io/jetstream';import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';-import {JobCancelledError, type WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';+import {JobCancelledError, JobFailedError, type WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';const MAX_DLQ_PUBLISH_ATTEMPTS = 3;const DLQ_RETRY_DELAY_MS = 250;@@ -470,7 +470,7 @@ export class WorkerRunner {return false;}const deliveryCount = msg.info.deliveryCount;- const isLastDelivery = deliveryCount >= this.maxDeliver;+ const isLastDelivery = deliveryCount >= this.maxDeliver || error instanceof JobFailedError;const errorMessage = error instanceof Error ? error.message : String(error);if (archiveTask || isLastDelivery) {Logger.error(diff --git a/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts b/fluxer_api/src/api/worker/tasks/IndexChannelMessages.tsindex 16232fb05..886f6796e 100644--- a/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts+++ b/fluxer_api/src/api/worker/tasks/IndexChannelMessages.ts@@ -1,10 +1,11 @@// SPDX-License-Identifier: AGPL-3.0-or-later-import type {MessageID, UserID} from '@app/api/BrandedTypes';+import type {ChannelID, MessageID, UserID} from '@app/api/BrandedTypes';import {createChannelID} from '@app/api/BrandedTypes';import {isMessageSearchIndexable} from '@app/api/channel/services/message/MessageSearchService';import {Logger} from '@app/api/Logger';import {getMessageSearchService} from '@app/api/SearchFactory';+import type {IMessageSearchService} from '@app/api/search/IMessageSearchService';import {getWorkerDependencies} from '@app/api/worker/WorkerContext';import type {WorkerTaskHandler} from '@pkgs/worker/src/contracts/WorkerTask';import {seconds} from 'itty-time';@@ -17,6 +18,50 @@ const PayloadSchema = z.object({});const BULK_BATCH_SIZE = 5000;const COMPLETION_KEY_TTL_SECONDS = seconds('1 day');++export async function indexChannelMessageHistory(+ searchService: IMessageSearchService,+ channelId: ChannelID,+): Promise<number> {+ const {channelRepository, userRepository} = getWorkerDependencies();+ const authorBotCache = new Map<UserID, boolean>();+ let cursor: MessageID | undefined;+ let totalIndexed = 0;+ while (true) {+ const messages = await channelRepository.messages.listMessages(channelId, cursor, BULK_BATCH_SIZE);+ if (messages.length === 0) {+ break;+ }+ const newAuthorIds = [+ ...new Set(messages.map((m) => m.authorId).filter((id): id is UserID => id !== null && !authorBotCache.has(id))),+ ];+ if (newAuthorIds.length > 0) {+ const users = await userRepository.listUsers(newAuthorIds);+ for (const user of users) {+ authorBotCache.set(user.id, user.isBot);+ }+ }+ const indexable = messages.filter(isMessageSearchIndexable);+ await searchService.bulkIndexMessages(indexable, authorBotCache);+ totalIndexed += messages.length;+ cursor = messages[messages.length - 1]!.id;+ Logger.debug(+ {channelId: channelId.toString(), indexed: totalIndexed, batch: messages.length},+ 'Bulk indexed message batch',+ );+ if (messages.length < BULK_BATCH_SIZE) {+ break;+ }+ }+ const channel = await channelRepository.channelData.findUnique(channelId);+ if (channel?.isThread()) {+ await channelRepository.channelData.patchIndexedAt(channelId, new Date());+ } else if (channel) {+ await channelRepository.channelData.upsert({...channel.toRow(), indexed_at: new Date()});+ }+ return totalIndexed;+}+const indexChannelMessages: WorkerTaskHandler = async (payload) => {const validated = PayloadSchema.parse(payload);const searchService = getMessageSearchService();@@ -24,39 +69,9 @@ const indexChannelMessages: WorkerTaskHandler = async (payload) => {throw new Error('Search service is not available in this worker process');}const channelId = createChannelID(BigInt(validated.channelId));- const {channelRepository, userRepository, kvClient} = getWorkerDependencies();- const authorBotCache = new Map<UserID, boolean>();- let cursor: MessageID | undefined;- let totalIndexed = 0;+ const {kvClient} = getWorkerDependencies();try {- while (true) {- const messages = await channelRepository.messages.listMessages(channelId, cursor, BULK_BATCH_SIZE);- if (messages.length === 0) {- break;- }- const newAuthorIds = [- ...new Set(- messages.map((m) => m.authorId).filter((id): id is UserID => id !== null && !authorBotCache.has(id)),- ),- ];- if (newAuthorIds.length > 0) {- const users = await userRepository.listUsers(newAuthorIds);- for (const user of users) {- authorBotCache.set(user.id, user.isBot);- }- }- const indexable = messages.filter(isMessageSearchIndexable);- await searchService.bulkIndexMessages(indexable, authorBotCache);- totalIndexed += messages.length;- cursor = messages[messages.length - 1]!.id;- Logger.debug(- {channelId: channelId.toString(), indexed: totalIndexed, batch: messages.length},- 'Bulk indexed message batch',- );- if (messages.length < BULK_BATCH_SIZE) {- break;- }- }+ const totalIndexed = await indexChannelMessageHistory(searchService, channelId);if (validated.completionKey && validated.channelCount) {await kvClient.sadd(validated.completionKey, validated.channelId);await kvClient.expire(validated.completionKey, COMPLETION_KEY_TTL_SECONDS);@@ -71,12 +86,6 @@ const indexChannelMessages: WorkerTaskHandler = async (payload) => {Logger.info({completionKey: validated.completionKey}, 'All channels indexed');}}- const channel = await channelRepository.channelData.findUnique(channelId);- if (channel?.isThread()) {- await channelRepository.channelData.patchIndexedAt(channelId, new Date());- } else if (channel) {- await channelRepository.channelData.upsert({...channel.toRow(), indexed_at: new Date()});- }Logger.info({channelId: channelId.toString(), totalIndexed}, 'Bulk channel indexing complete');} catch (error) {Logger.error({error, channelId: channelId.toString()}, 'Failed to bulk index channel messages');diff --git a/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts b/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.tsindex b1a03bdd1..8a348fb46 100644--- a/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts+++ b/fluxer_api/src/api/worker/tasks/RefreshSearchIndex.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-later-import type {GuildID, ReportID, UserID} from '@app/api/BrandedTypes';+import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository';+import type {ChannelID, GuildID, ReportID, UserID} from '@app/api/BrandedTypes';import {createGuildID} from '@app/api/BrandedTypes';import {GuildDiscoveryRepository} from '@app/api/guild/repositories/GuildDiscoveryRepository';import {Logger} from '@app/api/Logger';@@ -17,13 +18,25 @@ import {import type {IGuildMemberSearchService} from '@app/api/search/IGuildMemberSearchService';import type {IMessageSearchService} from '@app/api/search/IMessageSearchService';import {deleteChannelMessageSearchDocuments} from '@app/api/search/MessageSearchIndexCleanup';+import {+ finishSearchReindexAll,+ holdSearchReindexAll,+ INDEX_REFRESH_STATUS_TTL_SECONDS,+ indexRefreshStatusKey,+} from '@app/api/search/SearchReindexAllRun';import {backfillThreadSearch} from '@app/api/search/thread/ThreadSearchService';+import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';+import {indexChannelMessageHistory} from '@app/api/worker/tasks/IndexChannelMessages';import {getWorkerDependencies} from '@app/api/worker/WorkerContext';import {DiscoveryApplicationStatus} from '@fluxer/constants/src/DiscoveryConstants';import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';-import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';-import {seconds} from 'itty-time';+import {+ JobCancelledError,+ JobFailedError,+ type WorkerTaskHandler,+ type WorkerTaskHelpers,+} from '@pkgs/worker/src/contracts/WorkerTask';import {z} from 'zod';const INDEX_TYPES = [@@ -35,6 +48,7 @@ const INDEX_TYPES = ['guild_members','discovery','threads',+ 'all',] as const;type IndexType = (typeof INDEX_TYPES)[number];@@ -45,7 +59,6 @@ const PayloadSchema = zjob_id: z.string(),admin_user_id: z.string(),guild_id: z.string().optional(),- user_id: z.string().optional(),}).refine((data) => {@@ -64,7 +77,8 @@ const PayloadSchema = ztype RefreshSearchIndexPayload = z.infer<typeof PayloadSchema>;const BATCH_SIZE = 1000;-const PROGRESS_TTL = seconds('1 hour');+const CHANNEL_PAGE_SIZE = 500;+const CHANNEL_CONCURRENCY = 2;function requireSearchService<T>(service: T | null): T {if (!service) {@@ -74,32 +88,19 @@ function requireSearchService<T>(service: T | null): T {}async function setProgress(kvClient: IKVProvider, progressKey: string, data: Record<string, unknown>): Promise<void> {- await kvClient.set(progressKey, JSON.stringify(data), 'EX', PROGRESS_TTL);+ await kvClient.set(progressKey, JSON.stringify(data), 'EX', INDEX_REFRESH_STATUS_TTL_SECONDS);}-async function reportInProgress(- kvClient: IKVProvider,- progressKey: string,- indexType: IndexType,- indexed: number,-): Promise<void> {- await setProgress(kvClient, progressKey, {- status: 'in_progress',- index_type: indexType,- total: indexed,- indexed,- started_at: new Date().toISOString(),- });-}+type ReportProgress = (indexed: number, total?: number, phase?: IndexType | 'messages') => Promise<void>;++const ignoreProgress: ReportProgress = async () => {};interface PaginateAndIndexOptions<TCursor, TItem> {fetchPage: (cursor: TCursor | undefined) => Promise<Array<TItem>>;indexBatch: (items: Array<TItem>) => Promise<number>;getCursor: (item: TItem) => TCursor;label: string;- kvClient: IKVProvider;- progressKey: string;- indexType: IndexType;+ report: ReportProgress;}async function paginateAndIndex<TCursor, TItem>(options: PaginateAndIndexOptions<TCursor, TItem>): Promise<number> {@@ -113,7 +114,7 @@ async function paginateAndIndex<TCursor, TItem>(options: PaginateAndIndexOptionsindexedCount += batchCount;cursor = options.getCursor(items[items.length - 1]!);if (batchCount > 0) {- await reportInProgress(options.kvClient, options.progressKey, options.indexType, indexedCount);+ await options.report(indexedCount);Logger.debug({count: batchCount, total: indexedCount}, `Indexed ${options.label} batch`);}}@@ -126,11 +127,10 @@ async function paginateAndIndex<TCursor, TItem>(options: PaginateAndIndexOptionstype IndexHandler = (payload: RefreshSearchIndexPayload,helpers: WorkerTaskHelpers,- kvClient: IKVProvider,- progressKey: string,+ report: ReportProgress,) => Promise<number>;-const refreshGuilds: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {+const refreshGuilds: IndexHandler = async (_payload, _helpers, report) => {const {guildRepository} = getWorkerDependencies();const searchService = requireSearchService(getGuildSearchService());await searchService.deleteAllDocuments();@@ -142,12 +142,10 @@ const refreshGuilds: IndexHandler = async (_payload, _helpers, kvClient, progres},getCursor: (guild) => guild.id,label: 'guild',- kvClient,- progressKey,- indexType: 'guilds',+ report,});};-const refreshUsers: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {+const refreshUsers: IndexHandler = async (_payload, _helpers, report) => {const {userRepository} = getWorkerDependencies();const searchService = requireSearchService(getUserSearchService());await searchService.deleteAllDocuments();@@ -160,13 +158,13 @@ const refreshUsers: IndexHandler = async (_payload, _helpers, kvClient, progressif (users.length === 0) continue;await searchService.indexUsers(users);indexedCount += users.length;- await reportInProgress(kvClient, progressKey, 'users', indexedCount);+ await report(indexedCount);Logger.debug({count: users.length, total: indexedCount}, 'Indexed user batch');} while (pageState);Logger.debug({count: indexedCount}, 'Refreshed user search index');return indexedCount;};-const refreshReports: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {+const refreshReports: IndexHandler = async (_payload, _helpers, report) => {const {reportRepository} = getWorkerDependencies();const searchService = requireSearchService(getReportSearchService());await searchService.deleteAllDocuments();@@ -178,12 +176,10 @@ const refreshReports: IndexHandler = async (_payload, _helpers, kvClient, progre},getCursor: (report) => report.reportId,label: 'report',- kvClient,- progressKey,- indexType: 'reports',+ report,});};-const refreshAuditLogs: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {+const refreshAuditLogs: IndexHandler = async (_payload, _helpers, report) => {const {adminRepository} = getWorkerDependencies();const searchService = requireSearchService(getAuditLogSearchService());await searchService.deleteAllDocuments();@@ -195,13 +191,11 @@ const refreshAuditLogs: IndexHandler = async (_payload, _helpers, kvClient, prog},getCursor: (log) => log.logId,label: 'audit log',- kvClient,- progressKey,- indexType: 'audit_logs',+ report,});};-const refreshChannelMessages: IndexHandler = async (payload, helpers, kvClient, _progressKey) => {- const {channelRepository} = getWorkerDependencies();+const refreshChannelMessages: IndexHandler = async (payload, helpers) => {+ const {channelRepository, kvClient} = getWorkerDependencies();const guildId = createGuildID(BigInt(payload.guild_id!));const searchService = requireSearchService<IMessageSearchService>(getMessageSearchService());const guildChannels = await channelRepository.channelData.listGuildChannels(guildId, 'complete');@@ -240,7 +234,7 @@ const refreshChannelMessages: IndexHandler = async (payload, helpers, kvClient,);return indexedChannels.length;};-const refreshGuildMembers: IndexHandler = async (payload, _helpers, kvClient, progressKey) => {+const refreshGuildMembers: IndexHandler = async (payload, _helpers, report) => {const {guildRepository, userRepository} = getWorkerDependencies();const guildId = createGuildID(BigInt(payload.guild_id!));const searchService = requireSearchService<IGuildMemberSearchService>(getGuildMemberSearchService());@@ -262,9 +256,7 @@ const refreshGuildMembers: IndexHandler = async (payload, _helpers, kvClient, pr},getCursor: (member) => member.userId,label: 'guild member',- kvClient,- progressKey,- indexType: 'guild_members',+ report,});const guild = await guildRepository.findUnique(guildId);if (guild) {@@ -273,7 +265,7 @@ const refreshGuildMembers: IndexHandler = async (payload, _helpers, kvClient, prreturn indexedCount;};const DISCOVERY_BATCH_SIZE = 200;-const refreshDiscovery: IndexHandler = async (_payload, _helpers, kvClient, progressKey) => {+const refreshDiscovery: IndexHandler = async (_payload, _helpers, report) => {const {guildRepository} = getWorkerDependencies();const searchService = requireSearchService(getGuildSearchService());const discoveryRepository = new GuildDiscoveryRepository();@@ -308,13 +300,7 @@ const refreshDiscovery: IndexHandler = async (_payload, _helpers, kvClient, prog}await Promise.all(updates);synced += updates.length;- await setProgress(kvClient, progressKey, {- status: 'in_progress',- index_type: 'discovery',- total: guildIds.length,- indexed: synced,- started_at: new Date().toISOString(),- });+ await report(synced, guildIds.length);}return synced;};@@ -325,6 +311,126 @@ const refreshThreads: IndexHandler = async (payload) => {await channelRepository.threads.clearGuildSearchBackfilled(guildId);return backfillThreadSearch(channelRepository, guildId);};+async function forEachGuild(visit: (guildId: GuildID) => Promise<void>): Promise<void> {+ const {guildRepository} = getWorkerDependencies();+ let cursor: GuildID | undefined;+ while (true) {+ const guilds = await guildRepository.listAllGuildsPaginated(BATCH_SIZE, cursor);+ for (const guild of guilds) {+ await visit(guild.id);+ }+ if (guilds.length < BATCH_SIZE) return;+ cursor = guilds[guilds.length - 1]!.id;+ }+}++async function forEachChannelWithMessages(visit: (channelIds: Array<ChannelID>) => Promise<void>): Promise<void> {+ const repository = new AttachmentDecayRepository();+ let pageState: string | null = null;+ let previous: ChannelID | null = null;+ do {+ const page = await repository.scanChannelsWithMessagesPage(CHANNEL_PAGE_SIZE, pageState);+ pageState = page.pageState;+ const channelIds = page.channelIds.filter((channelId) => channelId !== previous);+ previous = page.channelIds.at(-1) ?? previous;+ if (channelIds.length > 0) {+ await visit(channelIds);+ }+ } while (pageState);+}++async function throwIfCancelled(helpers: WorkerTaskHelpers): Promise<void> {+ if (await helpers.shouldCancel()) {+ throw new JobCancelledError();+ }+}++async function refreshEveryGuild(+ handler: IndexHandler,+ payload: RefreshSearchIndexPayload,+ helpers: WorkerTaskHelpers,+ guildCount: number,+ report: ReportProgress,+): Promise<number> {+ let visited = 0;+ let indexed = 0;+ await report(0, guildCount);+ await forEachGuild(async (guildId) => {+ indexed += await handler({...payload, guild_id: guildId.toString()}, helpers, ignoreProgress);+ visited++;+ await report(visited, Math.max(guildCount, visited));+ await throwIfCancelled(helpers);+ });+ return indexed;+}++async function refreshEveryChannel(+ searchService: IMessageSearchService,+ helpers: WorkerTaskHelpers,+ report: ReportProgress,+): Promise<number> {+ await searchService.deleteAllDocuments();+ let channelCount = 0;+ await forEachChannelWithMessages(async (channelIds) => {+ channelCount += channelIds.length;+ });+ let visited = 0;+ let indexed = 0;+ await report(0, channelCount);+ await forEachChannelWithMessages(async (channelIds) => {+ const counts = await mapWithConcurrency(channelIds, CHANNEL_CONCURRENCY, async (channelId) => {+ const count = await indexChannelMessageHistory(searchService, channelId);+ await searchService.refreshIndex().catch((error) => {+ Logger.warn({error, channelId: channelId.toString()}, 'Search refresh after channel reindex failed');+ });+ return count;+ });+ indexed += counts.reduce((sum, count) => sum + count, 0);+ visited += channelIds.length;+ await report(visited, Math.max(channelCount, visited));+ await throwIfCancelled(helpers);+ });+ return indexed;+}++const refreshAll: IndexHandler = async (payload, helpers, report) => {+ let indexed = 0;+ let guildCount = 0;+ const globalHandlers = [+ ['users', refreshUsers],+ ['guilds', refreshGuilds],+ ['discovery', refreshDiscovery],+ ['reports', refreshReports],+ ['audit_logs', refreshAuditLogs],+ ] as const;+ for (const [indexType, handler] of globalHandlers) {+ const count = await handler(payload, helpers, (done, total) => report(done, total, indexType));+ if (indexType === 'guilds') {+ guildCount = count;+ }+ indexed += count;+ await throwIfCancelled(helpers);+ }+ const memberSearch = getGuildMemberSearchService();+ if (memberSearch) {+ await memberSearch.deleteAllDocuments();+ indexed += await refreshEveryGuild(refreshGuildMembers, payload, helpers, guildCount, (done, total) =>+ report(done, total, 'guild_members'),+ );+ }+ const threadSearch = getThreadSearchService();+ if (threadSearch) {+ await threadSearch.deleteAllDocuments();+ indexed += await refreshEveryGuild(refreshThreads, payload, helpers, guildCount, (done, total) =>+ report(done, total, 'threads'),+ );+ }+ const messageSearch = getMessageSearchService();+ if (messageSearch) {+ indexed += await refreshEveryChannel(messageSearch, helpers, (done, total) => report(done, total, 'messages'));+ }+ return indexed;+};const INDEX_HANDLERS: Record<IndexType, IndexHandler> = {guilds: refreshGuilds,users: refreshUsers,@@ -334,32 +440,69 @@ const INDEX_HANDLERS: Record<IndexType, IndexHandler> = {guild_members: refreshGuildMembers,discovery: refreshDiscovery,threads: refreshThreads,+ all: refreshAll,};-const refreshSearchIndex: WorkerTaskHandler = async (payload, helpers) => {- const validated = PayloadSchema.parse(payload);- helpers.logger.debug({payload: validated}, 'Processing refreshSearchIndex task');- const {kvClient} = getWorkerDependencies();- const progressKey = `index_refresh_status:${validated.job_id}`;- await reportInProgress(kvClient, progressKey, validated.index_type, 0);+async function runRefresh(+ payload: RefreshSearchIndexPayload,+ helpers: WorkerTaskHelpers,+ kvClient: IKVProvider,+): Promise<number> {+ const progressKey = indexRefreshStatusKey(payload.job_id);+ const indexType = payload.index_type;+ const report: ReportProgress = async (indexed, total = indexed, phase) => {+ if (indexType === 'all') {+ await holdSearchReindexAll(kvClient, payload.job_id);+ }+ await setProgress(kvClient, progressKey, {+ status: 'in_progress',+ index_type: indexType,+ ...(phase ? {phase} : {}),+ total,+ indexed,+ started_at: new Date().toISOString(),+ });+ };+ await report(0);try {- const handler = INDEX_HANDLERS[validated.index_type];- const indexedCount = await handler(validated, helpers, kvClient, progressKey);+ const indexedCount = await INDEX_HANDLERS[indexType](payload, helpers, report);await setProgress(kvClient, progressKey, {status: 'completed',- index_type: validated.index_type,+ index_type: indexType,total: indexedCount,indexed: indexedCount,completed_at: new Date().toISOString(),});+ return indexedCount;} catch (error) {- Logger.error({error, payload: validated}, 'Failed to refresh search index');+ Logger.error({error, payload}, 'Failed to refresh search index');+ const message = error instanceof Error ? error.message : 'Unknown error';await setProgress(kvClient, progressKey, {status: 'failed',- index_type: validated.index_type,- error: error instanceof Error ? error.message : 'Unknown error',+ index_type: indexType,+ error: message,failed_at: new Date().toISOString(),});- throw error;+ if (error instanceof JobCancelledError) {+ throw error;+ }+ throw new JobFailedError(message);+ }+}+const refreshSearchIndex: WorkerTaskHandler = async (payload, helpers) => {+ const validated = PayloadSchema.parse(payload);+ helpers.logger.debug({payload: validated}, 'Processing refreshSearchIndex task');+ const {kvClient} = getWorkerDependencies();+ if (validated.index_type !== 'all') {+ return {indexed: await runRefresh(validated, helpers, kvClient)};+ }+ if (!(await holdSearchReindexAll(kvClient, validated.job_id))) {+ helpers.logger.info({jobId: validated.job_id}, 'Another search reindex is running, stopping this one');+ return {superseded: true};+ }+ try {+ return {indexed: await runRefresh(validated, helpers, kvClient)};+ } finally {+ await finishSearchReindexAll(kvClient, validated.job_id);}};diff --git a/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx b/fluxer_docs/src/content/docs/admin-api/search-indexes.mdxindex 03ed45c02..2dcac160f 100644--- a/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx+++ b/fluxer_docs/src/content/docs/admin-api/search-indexes.mdx@@ -21,14 +21,14 @@ A rebuild clears the index first, and search over that index returns partial res| discovery | Instance | Discovery details of approved listings in the `guilds` index || channel_messages<sup>1</sup> | Guild | Messages in every channel and thread of one guild || guild_members | Guild | Members of one guild |-| favorite_memes<sup>2</sup> | User | Favorited [memes](/http-api/memes/) of one user |-| threads<sup>3</sup> | Guild | Threads used by [Search threads](/http-api/threads/#search-threads) |+| threads<sup>2</sup> | Guild | Threads used by [Search threads](/http-api/threads/#search-threads) |+| all<sup>3</sup> | Instance | Every index above, for every guild, and messages in every channel, thread and DM |<sup>1</sup> Progress shows `completed` once every channel is queued for indexing, and `total` and `indexed` count channels. Messages keep indexing after that.-<sup>2</sup> The rebuild is queued but always fails, and its progress reads as `not_found`.+<sup>2</sup> [Search threads](/http-api/threads/#search-threads) returns 202 for the guild until the rebuild completes.-<sup>3</sup> [Search threads](/http-api/threads/#search-threads) returns 202 for the guild until the rebuild completes.+<sup>3</sup> Rebuilds one index at a time and clears each one first, including `threads`. Only one `all` rebuild runs at a time.## Search index refresh object@@ -64,8 +64,9 @@ An unknown or expired `job_id` returns the not found structure, and every other| --- | --- | --- || status | string | One of `in_progress`, `completed`, or `failed` || index_type | string | The [search index name](#search-index-names) being rebuilt |-| total? | number | Equal to `indexed` until `status` is `completed`, except for `discovery`, where it is the number of approved listings |-| indexed? | number | The number of documents written so far |+| phase? | string | For `all`, the index it is on, `messages`, or `queued` before it starts |+| total? | number | Equal to `indexed` until `status` is `completed`, except for `discovery`, where it is the number of approved listings, and in the `all` phases that count guilds or channels |+| indexed? | number | The number of documents written so far, or of guilds or channels done in the `guild_members`, `threads` and `messages` phases of `all` || started_at? | ISO8601 timestamp | The time the rebuild last reported progress || completed_at? | ISO8601 timestamp | The time the rebuild finished, when `status` is `completed` || failed_at? | ISO8601 timestamp | The time the rebuild failed, when `status` is `failed` |@@ -75,7 +76,7 @@ An unknown or expired `job_id` returns the not found structure, and every other<RouteHeader method="POST" path="/v1/admin/search/indexes/{index_name}/refreshes" auditReason />-Queues a rebuild of the index and returns its `job_id`. You need the `guild:lookup` [ACL](/admin-api/#acl-evaluation). A failed rebuild is not retried, so queue a new one.+Queues a rebuild of the index and returns its `job_id`. You need the `guild:lookup` [ACL](/admin-api/#acl-evaluation). A failed rebuild is not retried, so queue a new one. While an `all` rebuild runs, queueing `all` again returns its `job_id`.### Path parameters@@ -88,14 +89,14 @@ Queues a rebuild of the index and returns its `job_id`. You need the `guild:look| Field | Type | Description || --- | --- | --- || guild_id? | snowflake | The ID of the guild to rebuild, needed by `channel_messages`, `guild_members` and `threads` |-| user_id? | snowflake | The ID of the user to rebuild, needed by `favorite_memes` |### Response| Status | Body | Condition || --- | --- | --- |-| 200 | [search index refresh](#search-index-refresh-object) object | The rebuild was queued |-| 400 | [error response](/admin-api/#error-response) | `INVALID_FORM_BODY` when the index needs `guild_id` or `user_id` and it is missing |+| 200 | [search index refresh](#search-index-refresh-object) object | The rebuild was queued or is already running |+| 400 | [error response](/admin-api/#error-response) | `INVALID_FORM_BODY` when the index needs `guild_id` and it is missing |+| 403 | [error response](/admin-api/#error-response) | `FEATURE_TEMPORARILY_DISABLED` when `index_name` is `all` on an instance with no search backend |Fluxer records an Admin audit log entry with the action `queue_refresh_index` and the target type `search_index`.diff --git a/packages/schema/src/domains/admin/AdminSchemas.ts b/packages/schema/src/domains/admin/AdminSchemas.tsindex 8e40f77db..5af146210 100644--- a/packages/schema/src/domains/admin/AdminSchemas.ts+++ b/packages/schema/src/domains/admin/AdminSchemas.ts@@ -158,9 +158,9 @@ const SearchIndexTypeEnum = createNamedStringLiteralUnion(['audit_logs', 'audit_logs', 'Audit log search index'],['channel_messages', 'channel_messages', 'Channel message search index'],['guild_members', 'guild_members', 'Guild member search index'],- ['favorite_memes', 'favorite_memes', 'Favourite meme search index'],['discovery', 'discovery', 'Discovery guild search index'],['threads', 'threads', 'Thread search index'],+ ['all', 'all', 'Every search index, with messages in every channel and DM'],],'Type of search index to refresh',);@@ -308,7 +308,6 @@ export type SearchIndexNameParam = z.infer<typeof SearchIndexNameParam>;export const RefreshSearchIndexRequest = z.object({guild_id: SnowflakeType.optional().describe('Specific guild ID to reindex'),- user_id: SnowflakeType.optional().describe('Specific user ID to reindex'),});export type RefreshSearchIndexRequest = z.infer<typeof RefreshSearchIndexRequest>;@@ -1257,6 +1256,7 @@ export const IndexRefreshStatusResponse = z.union([z.object({status: IndexRefreshStatusEnum,index_type: z.string().describe('Type of index being refreshed'),+ phase: z.string().optional().describe('The index an `all` rebuild is working on, or queued before it starts'),total: z.number().optional().describe('Total number of items to index'),indexed: z.number().optional().describe('Number of items indexed so far'),started_at: z.string().optional().describe('ISO 8601 timestamp when the job started'),