Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat: make it possible to trigger admin jobs for a certain duration

Mohamed Bassem0a1821ba85b5
diff --git a/apps/cli/src/commands/admin.ts b/apps/cli/src/commands/admin.ts
index 9c3d7c2d..f63ed1a4 100644
--- a/apps/cli/src/commands/admin.ts
+++ b/apps/cli/src/commands/admin.ts
@@ -5,7 +5,7 @@ import {
printStatusMessage,
} from "@/lib/output";
import { getAPIClient } from "@/lib/trpc";
-import { Command } from "@commander-js/extra-typings";
+import { Command, InvalidArgumentError } from "@commander-js/extra-typings";
import { getBorderCharacters, table } from "table";
export const adminCmd = new Command()
@@ -19,6 +19,43 @@ function toHumanReadableSize(size: number): string {
return (size / Math.pow(1024, i)).toFixed(2) + " " + sizes[i];
}
+function parseDuration(value: string): number {
+ const match = /^(\d+(?:\.\d+)?)(s|m|h|d|w)?$/.exec(
+ value.trim().toLowerCase(),
+ );
+ if (!match) {
+ throw new InvalidArgumentError(
+ "duration must be a positive number of seconds or use s, m, h, d, or w (for example, 1h)",
+ );
+ }
+
+ const unitSeconds = {
+ s: 1,
+ m: 60,
+ h: 60 * 60,
+ d: 24 * 60 * 60,
+ w: 7 * 24 * 60 * 60,
+ } as const;
+ const duration =
+ Number(match[1]) *
+ (match[2] ? unitSeconds[match[2] as keyof typeof unitSeconds] : 1);
+ if (!Number.isSafeInteger(duration) || duration <= 0) {
+ throw new InvalidArgumentError(
+ "duration must resolve to a positive whole number of seconds",
+ );
+ }
+ return duration;
+}
+
+const modifiedWithinOptionDescription =
+ "only process bookmarks modified within this window (for example, 30m, 1h, or 2d)";
+
+function modifiedWithinStatusSuffix(modifiedWithinSeconds?: number) {
+ return modifiedWithinSeconds === undefined
+ ? ""
+ : ` modified within the last ${modifiedWithinSeconds} seconds`;
+}
+
// --- Users subcommand ---
const usersCmd = new Command()
@@ -404,6 +441,11 @@ jobsCmd
"filter by crawl status (success, failure, pending, all)",
)
.option("--run-inference", "also re-run inference after crawling", false)
+ .option(
+ "--modified-within <duration>",
+ modifiedWithinOptionDescription,
+ parseDuration,
+ )
.action(async (opts) => {
const api = getAPIClient();
const status = opts.status as "success" | "failure" | "pending" | "all";
@@ -411,10 +453,11 @@ jobsCmd
await api.admin.recrawlLinks.mutate({
crawlStatus: status,
runInference: opts.runInference,
+ modifiedWithinSeconds: opts.modifiedWithin,
});
printStatusMessage(
true,
- `Recrawl queued for all links with crawl status: ${status}`,
+ `Recrawl queued for links with crawl status ${status}${modifiedWithinStatusSuffix(opts.modifiedWithin)}`,
);
} catch (error) {
printErrorMessageWithReason(
@@ -427,11 +470,21 @@ jobsCmd
jobsCmd
.command("reindex-all")
.description("reindex all bookmarks for search")
- .action(async () => {
+ .option(
+ "--modified-within <duration>",
+ modifiedWithinOptionDescription,
+ parseDuration,
+ )
+ .action(async (opts) => {
const api = getAPIClient();
try {
- await api.admin.reindexAllBookmarks.mutate();
- printStatusMessage(true, "Reindex queued for all bookmarks");
+ await api.admin.reindexAllBookmarks.mutate({
+ modifiedWithinSeconds: opts.modifiedWithin,
+ });
+ printStatusMessage(
+ true,
+ `Reindex queued for bookmarks${modifiedWithinStatusSuffix(opts.modifiedWithin)}`,
+ );
} catch (error) {
printErrorMessageWithReason(
"Failed to queue mass reindex",
@@ -447,6 +500,11 @@ jobsCmd
"--status <status>",
"filter by tagging status (success, failure, pending, all)",
)
+ .option(
+ "--modified-within <duration>",
+ modifiedWithinOptionDescription,
+ parseDuration,
+ )
.action(async (opts) => {
const api = getAPIClient();
const status = opts.status as "success" | "failure" | "pending" | "all";
@@ -454,10 +512,11 @@ jobsCmd
await api.admin.reRunInferenceOnAllBookmarks.mutate({
type: "tag",
status,
+ modifiedWithinSeconds: opts.modifiedWithin,
});
printStatusMessage(
true,
- `Retag queued for all bookmarks with tagging status: ${status}`,
+ `Retag queued for bookmarks with tagging status ${status}${modifiedWithinStatusSuffix(opts.modifiedWithin)}`,
);
} catch (error) {
printErrorMessageWithReason(
@@ -474,6 +533,11 @@ jobsCmd
"--status <status>",
"filter by summarization status (success, failure, pending, all)",
)
+ .option(
+ "--modified-within <duration>",
+ modifiedWithinOptionDescription,
+ parseDuration,
+ )
.action(async (opts) => {
const api = getAPIClient();
const status = opts.status as "success" | "failure" | "pending" | "all";
@@ -481,10 +545,11 @@ jobsCmd
await api.admin.reRunInferenceOnAllBookmarks.mutate({
type: "summarize",
status,
+ modifiedWithinSeconds: opts.modifiedWithin,
});
printStatusMessage(
true,
- `Resummarize queued for all bookmarks with summarization status: ${status}`,
+ `Resummarize queued for bookmarks with summarization status ${status}${modifiedWithinStatusSuffix(opts.modifiedWithin)}`,
);
} catch (error) {
printErrorMessageWithReason(
@@ -501,14 +566,22 @@ jobsCmd
"--status <status>",
"filter by embedding status (failure, pending, all)",
)
+ .option(
+ "--modified-within <duration>",
+ modifiedWithinOptionDescription,
+ parseDuration,
+ )
.action(async (opts) => {
const api = getAPIClient();
const status = opts.status as "failure" | "pending" | "all";
try {
- await api.admin.regenerateAllBookmarkEmbeddings.mutate({ status });
+ await api.admin.regenerateAllBookmarkEmbeddings.mutate({
+ status,
+ modifiedWithinSeconds: opts.modifiedWithin,
+ });
printStatusMessage(
true,
- `Embedding regeneration queued for all bookmarks with embedding status: ${status}`,
+ `Embedding regeneration queued for bookmarks with embedding status ${status}${modifiedWithinStatusSuffix(opts.modifiedWithin)}`,
);
} catch (error) {
printErrorMessageWithReason(
@@ -521,11 +594,21 @@ jobsCmd
jobsCmd
.command("reprocess-assets")
.description("reprocess all asset bookmarks in fix mode")
- .action(async () => {
+ .option(
+ "--modified-within <duration>",
+ modifiedWithinOptionDescription,
+ parseDuration,
+ )
+ .action(async (opts) => {
const api = getAPIClient();
try {
- await api.admin.reprocessAssetsFixMode.mutate();
- printStatusMessage(true, "Asset reprocessing queued for all assets");
+ await api.admin.reprocessAssetsFixMode.mutate({
+ modifiedWithinSeconds: opts.modifiedWithin,
+ });
+ printStatusMessage(
+ true,
+ `Asset reprocessing queued for assets${modifiedWithinStatusSuffix(opts.modifiedWithin)}`,
+ );
} catch (error) {
printErrorMessageWithReason(
"Failed to queue asset reprocessing",
diff --git a/packages/api/routes/admin.ts b/packages/api/routes/admin.ts
index 523c0e5b..662f9908 100644
--- a/packages/api/routes/admin.ts
+++ b/packages/api/routes/admin.ts
@@ -2,7 +2,10 @@ import { zValidator } from "@hono/zod-validator";
import { Hono } from "hono";
import { z } from "zod";
-import { updateUserSchema } from "@karakeep/shared/types/admin";
+import {
+ updateUserSchema,
+ zAdminJobModifiedWithinSecondsSchema,
+} from "@karakeep/shared/types/admin";
import { adminAuthMiddleware } from "../middlewares/auth";
@@ -32,6 +35,7 @@ const app = new Hono()
.enum(["success", "failure", "pending", "all"])
.default("all"),
runInference: z.boolean().default(false),
+ modifiedWithinSeconds: zAdminJobModifiedWithinSecondsSchema.optional(),
}),
),
async (c) => {
@@ -42,10 +46,20 @@ const app = new Hono()
)
// POST /admin/jobs/trigger/reindex
- .post("/jobs/trigger/reindex", async (c) => {
- await c.var.api.admin.reindexAllBookmarks();
- return c.json({ success: true }, 200);
- })
+ .post(
+ "/jobs/trigger/reindex",
+ zValidator(
+ "json",
+ z.object({
+ modifiedWithinSeconds: zAdminJobModifiedWithinSecondsSchema.optional(),
+ }),
+ ),
+ async (c) => {
+ const body = c.req.valid("json");
+ await c.var.api.admin.reindexAllBookmarks(body);
+ return c.json({ success: true }, 200);
+ },
+ )
// POST /admin/jobs/trigger/inference
.post(
@@ -55,6 +69,7 @@ const app = new Hono()
z.object({
type: z.enum(["tag", "summarize"]),
status: z.enum(["success", "failure", "pending", "all"]).default("all"),
+ modifiedWithinSeconds: zAdminJobModifiedWithinSecondsSchema.optional(),
}),
),
async (c) => {
diff --git a/packages/open-api/karakeep-openapi-spec.json b/packages/open-api/karakeep-openapi-spec.json
index 1a118f4d..acfc9b4b 100644
--- a/packages/open-api/karakeep-openapi-spec.json
+++ b/packages/open-api/karakeep-openapi-spec.json
@@ -4384,7 +4384,7 @@
"/admin/jobs/trigger/recrawl": {
"post": {
"operationId": "adminTriggerRecrawl",
- "description": "Trigger a recrawl of link bookmarks. You can filter by crawl status to target specific bookmarks (e.g., only failed ones). Optionally run AI inference after crawling. Requires admin role.",
+ "description": "Trigger a recrawl of link bookmarks. You can filter by crawl status to target specific bookmarks (e.g., only failed ones) and by how recently they were modified. Optionally run AI inference after crawling. Requires admin role.",
"summary": "Trigger recrawl of links (admin)",
"tags": [
"Admin"
@@ -4416,11 +4416,18 @@
"type": "boolean",
"default": false,
"description": "Whether to run AI inference after crawling."
+ },
+ "modifiedWithinSeconds": {
+ "type": "integer",
+ "minimum": 0,
+ "exclusiveMinimum": true,
+ "description": "Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks."
}
},
"example": {
"crawlStatus": "failure",
- "runInference": false
+ "runInference": false,
+ "modifiedWithinSeconds": 3600
}
}
}
@@ -4483,7 +4490,7 @@
"/admin/jobs/trigger/reindex": {
"post": {
"operationId": "adminTriggerReindex",
- "description": "Trigger a reindex of all bookmarks in the search engine. This clears the existing index and re-queues all bookmarks for indexing. Requires admin role.",
+ "description": "Trigger a reindex of bookmarks in the search engine. Without modifiedWithinSeconds, this clears the existing index and re-queues all bookmarks. When set, only bookmarks modified within that many seconds are re-queued and the existing index is preserved. Requires admin role.",
"summary": "Trigger reindex of all bookmarks (admin)",
"tags": [
"Admin"
@@ -4493,6 +4500,27 @@
"bearerAuth": []
}
],
+ "requestBody": {
+ "description": "Optional time window for the reindex job.",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "properties": {
+ "modifiedWithinSeconds": {
+ "type": "integer",
+ "minimum": 0,
+ "exclusiveMinimum": true,
+ "description": "Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks."
+ }
+ },
+ "example": {
+ "modifiedWithinSeconds": 3600
+ }
+ }
+ }
+ }
+ },
"responses": {
"200": {
"description": "Reindex jobs triggered successfully.",
@@ -4540,7 +4568,7 @@
"/admin/jobs/trigger/inference": {
"post": {
"operationId": "adminTriggerInference",
- "description": "Trigger AI inference (tagging or summarization) on bookmarks. You can filter by status to target specific bookmarks (e.g., only failed ones). Requires admin role.",
+ "description": "Trigger AI inference (tagging or summarization) on bookmarks. You can filter by status and by how recently bookmarks were modified. Requires admin role.",
"summary": "Trigger AI inference on bookmarks (admin)",
"tags": [
"Admin"
@@ -4576,6 +4604,12 @@
],
"default": "all",
"description": "Filter bookmarks by their inference status. Use 'failure' to retry only failed ones."
+ },
+ "modifiedWithinSeconds": {
+ "type": "integer",
+ "minimum": 0,
+ "exclusiveMinimum": true,
+ "description": "Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks."
}
},
"required": [
@@ -4583,7 +4617,8 @@
],
"example": {
"type": "tag",
- "status": "failure"
+ "status": "failure",
+ "modifiedWithinSeconds": 3600
}
}
}
diff --git a/packages/open-api/lib/admin.ts b/packages/open-api/lib/admin.ts
index 354543a7..2497b4b2 100644
--- a/packages/open-api/lib/admin.ts
+++ b/packages/open-api/lib/admin.ts
@@ -1,7 +1,10 @@
import { OpenAPIRegistry } from "@asteasolutions/zod-to-openapi";
import * as z from "zod";
-import { updateUserSchema } from "@karakeep/shared/types/admin";
+import {
+ updateUserSchema,
+ zAdminJobModifiedWithinSecondsSchema,
+} from "@karakeep/shared/types/admin";
import { BearerAuth } from "./common";
import { ErrorSchema, UnauthorizedResponse } from "./errors";
@@ -98,7 +101,8 @@ registry.registerPath({
path: "/admin/jobs/trigger/recrawl",
description:
"Trigger a recrawl of link bookmarks. You can filter by crawl status to target specific bookmarks " +
- "(e.g., only failed ones). Optionally run AI inference after crawling. Requires admin role.",
+ "(e.g., only failed ones) and by how recently they were modified. Optionally run AI inference " +
+ "after crawling. Requires admin role.",
summary: "Trigger recrawl of links (admin)",
tags: ["Admin"],
security: [{ [BearerAuth.name]: [] }],
@@ -119,11 +123,14 @@ registry.registerPath({
.boolean()
.default(false)
.describe("Whether to run AI inference after crawling."),
+ modifiedWithinSeconds:
+ zAdminJobModifiedWithinSecondsSchema.optional(),
})
.openapi({
example: {
crawlStatus: "failure",
runInference: false,
+ modifiedWithinSeconds: 3600,
},
}),
},
@@ -157,11 +164,31 @@ registry.registerPath({
method: "post",
path: "/admin/jobs/trigger/reindex",
description:
- "Trigger a reindex of all bookmarks in the search engine. This clears the existing index and " +
- "re-queues all bookmarks for indexing. Requires admin role.",
+ "Trigger a reindex of bookmarks in the search engine. Without modifiedWithinSeconds, this " +
+ "clears the existing index and re-queues all bookmarks. When set, only bookmarks modified " +
+ "within that many seconds are re-queued and the existing index is preserved. Requires admin role.",
summary: "Trigger reindex of all bookmarks (admin)",
tags: ["Admin"],
security: [{ [BearerAuth.name]: [] }],
+ request: {
+ body: {
+ description: "Optional time window for the reindex job.",
+ content: {
+ "application/json": {
+ schema: z
+ .object({
+ modifiedWithinSeconds:
+ zAdminJobModifiedWithinSecondsSchema.optional(),
+ })
+ .openapi({
+ example: {
+ modifiedWithinSeconds: 3600,
+ },
+ }),
+ },
+ },
+ },
+ },
responses: {
200: {
description: "Reindex jobs triggered successfully.",
@@ -182,7 +209,7 @@ registry.registerPath({
path: "/admin/jobs/trigger/inference",
description:
"Trigger AI inference (tagging or summarization) on bookmarks. You can filter by status " +
- "to target specific bookmarks (e.g., only failed ones). Requires admin role.",
+ "and by how recently bookmarks were modified. Requires admin role.",
summary: "Trigger AI inference on bookmarks (admin)",
tags: ["Admin"],
security: [{ [BearerAuth.name]: [] }],
@@ -205,11 +232,14 @@ registry.registerPath({
.describe(
"Filter bookmarks by their inference status. Use 'failure' to retry only failed ones.",
),
+ modifiedWithinSeconds:
+ zAdminJobModifiedWithinSecondsSchema.optional(),
})
.openapi({
example: {
type: "tag",
status: "failure",
+ modifiedWithinSeconds: 3600,
},
}),
},
diff --git a/packages/sdk/src/karakeep-api.d.ts b/packages/sdk/src/karakeep-api.d.ts
index 89564cf2..fb00e672 100644
--- a/packages/sdk/src/karakeep-api.d.ts
+++ b/packages/sdk/src/karakeep-api.d.ts
@@ -595,7 +595,7 @@ export interface paths {
put?: never;
/**
* Trigger recrawl of links (admin)
- * @description Trigger a recrawl of link bookmarks. You can filter by crawl status to target specific bookmarks (e.g., only failed ones). Optionally run AI inference after crawling. Requires admin role.
+ * @description Trigger a recrawl of link bookmarks. You can filter by crawl status to target specific bookmarks (e.g., only failed ones) and by how recently they were modified. Optionally run AI inference after crawling. Requires admin role.
*/
post: operations["adminTriggerRecrawl"];
delete?: never;
@@ -615,7 +615,7 @@ export interface paths {
put?: never;
/**
* Trigger reindex of all bookmarks (admin)
- * @description Trigger a reindex of all bookmarks in the search engine. This clears the existing index and re-queues all bookmarks for indexing. Requires admin role.
+ * @description Trigger a reindex of bookmarks in the search engine. Without modifiedWithinSeconds, this clears the existing index and re-queues all bookmarks. When set, only bookmarks modified within that many seconds are re-queued and the existing index is preserved. Requires admin role.
*/
post: operations["adminTriggerReindex"];
delete?: never;
@@ -635,7 +635,7 @@ export interface paths {
put?: never;
/**
* Trigger AI inference on bookmarks (admin)
- * @description Trigger AI inference (tagging or summarization) on bookmarks. You can filter by status to target specific bookmarks (e.g., only failed ones). Requires admin role.
+ * @description Trigger AI inference (tagging or summarization) on bookmarks. You can filter by status and by how recently bookmarks were modified. Requires admin role.
*/
post: operations["adminTriggerInference"];
delete?: never;
@@ -3140,6 +3140,8 @@ export interface operations {
* @default false
*/
runInference?: boolean;
+ /** @description Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks. */
+ modifiedWithinSeconds?: number;
};
};
};
@@ -3192,7 +3194,15 @@ export interface operations {
path?: never;
cookie?: never;
};
- requestBody?: never;
+ /** @description Optional time window for the reindex job. */
+ requestBody?: {
+ content: {
+ "application/json": {
+ /** @description Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks. */
+ modifiedWithinSeconds?: number;
+ };
+ };
+ };
responses: {
/** @description Reindex jobs triggered successfully. */
200: {
@@ -3248,6 +3258,8 @@ export interface operations {
* @enum {string}
*/
status?: "success" | "failure" | "pending" | "all";
+ /** @description Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks. */
+ modifiedWithinSeconds?: number;
};
};
};
diff --git a/packages/shared/types/admin.ts b/packages/shared/types/admin.ts
index d7f938b9..726e27b2 100644
--- a/packages/shared/types/admin.ts
+++ b/packages/shared/types/admin.ts
@@ -18,6 +18,14 @@ export const updateUserSchema = z.object({
browserCrawlingEnabled: z.boolean().nullable().optional(),
});
+export const zAdminJobModifiedWithinSecondsSchema = z
+ .number()
+ .int()
+ .positive()
+ .describe(
+ "Only process bookmarks modified within this many seconds. Omit to process all matching bookmarks.",
+ );
+
export const resetPasswordSchema = z
.object({
userId: z.string(),
diff --git a/packages/trpc/routers/admin.test.ts b/packages/trpc/routers/admin.test.ts
index b923f608..ee397305 100644
--- a/packages/trpc/routers/admin.test.ts
+++ b/packages/trpc/routers/admin.test.ts
@@ -1,7 +1,21 @@
import { eq } from "drizzle-orm";
-import { assert, beforeEach, describe, expect, test } from "vitest";
+import {
+ afterEach,
+ assert,
+ beforeEach,
+ describe,
+ expect,
+ test,
+ vi,
+} from "vitest";
-import { bookmarkLinks, users } from "@karakeep/db/schema";
+import {
+ bookmarkAssets,
+ bookmarkLinks,
+ bookmarks,
+ users,
+} from "@karakeep/db/schema";
+import { QueuePriority } from "@karakeep/shared-server";
import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
import type { CustomTestContext } from "../testUtils";
@@ -9,14 +23,356 @@ import {
buildTestContext,
getApiCaller,
getApiKeyCallerForPlainKey,
+ getTestQueueMocks,
} from "../testUtils";
+const testQueueMocks = getTestQueueMocks();
+
+const adminJobMocks = vi.hoisted(() => ({
+ searchClearIndex: vi.fn(async () => undefined),
+ vectorClearIndex: vi.fn(async () => undefined),
+ getSearchClient: vi.fn(),
+ getVectorStoreClient: vi.fn(),
+}));
+
+vi.mock("@karakeep/shared/search", () => ({
+ getSearchClient: adminJobMocks.getSearchClient,
+}));
+
+vi.mock("@karakeep/shared/vectorStore", () => ({
+ getVectorStoreClient: adminJobMocks.getVectorStoreClient,
+}));
+
beforeEach<CustomTestContext>(async (context) => {
+ vi.clearAllMocks();
const testContext = await buildTestContext(true);
Object.assign(context, testContext);
});
describe("Admin Routes", () => {
+ describe("bulk bookmark jobs", () => {
+ const now = new Date("2026-07-26T12:00:00.000Z");
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(now);
+ adminJobMocks.getSearchClient.mockResolvedValue({
+ clearIndex: adminJobMocks.searchClearIndex,
+ });
+ adminJobMocks.getVectorStoreClient.mockResolvedValue({
+ clearIndex: adminJobMocks.vectorClearIndex,
+ });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ async function getAdminApi(db: CustomTestContext["db"]) {
+ const [adminUser] = await db
+ .insert(users)
+ .values({
+ name: "Admin User",
+ email: "[email protected]",
+ role: "admin",
+ })
+ .returning();
+ return getApiCaller(db, adminUser.id, adminUser.email, "admin").admin;
+ }
+
+ async function setModifiedAt(
+ db: CustomTestContext["db"],
+ bookmarkId: string,
+ modifiedAt: Date,
+ ) {
+ await db
+ .update(bookmarks)
+ .set({ modifiedAt })
+ .where(eq(bookmarks.id, bookmarkId));
+ }
+
+ test<CustomTestContext>("combines modifiedWithinSeconds with the inference status filter", async ({
+ apiCallers,
+ db,
+ }) => {
+ const adminApi = await getAdminApi(db);
+ const [oldPending, recentPending, recentSuccess] = await Promise.all([
+ apiCallers[0].bookmarks.createBookmark({
+ text: "old pending",
+ type: BookmarkTypes.TEXT,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ text: "recent pending",
+ type: BookmarkTypes.TEXT,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ text: "recent success",
+ type: BookmarkTypes.TEXT,
+ }),
+ ]);
+ await Promise.all([
+ setModifiedAt(
+ db,
+ oldPending.id,
+ new Date(now.getTime() - 2 * 60 * 60 * 1000),
+ ),
+ setModifiedAt(
+ db,
+ recentPending.id,
+ new Date(now.getTime() - 30 * 60 * 1000),
+ ),
+ db
+ .update(bookmarks)
+ .set({
+ modifiedAt: new Date(now.getTime() - 30 * 60 * 1000),
+ taggingStatus: "success",
+ })
+ .where(eq(bookmarks.id, recentSuccess.id)),
+ ]);
+ testQueueMocks.openAIEnqueue.mockClear();
+
+ await adminApi.reRunInferenceOnAllBookmarks({
+ type: "tag",
+ status: "pending",
+ modifiedWithinSeconds: 60 * 60,
+ });
+
+ expect(testQueueMocks.openAIEnqueue).toHaveBeenCalledTimes(1);
+ expect(testQueueMocks.openAIEnqueue).toHaveBeenCalledWith(
+ { bookmarkId: recentPending.id, type: "tag" },
+ { priority: QueuePriority.Low },
+ );
+ });
+
+ test<CustomTestContext>("filters recrawls using the bookmark modified date", async ({
+ apiCallers,
+ db,
+ }) => {
+ const adminApi = await getAdminApi(db);
+ const [oldLink, recentLink] = await Promise.all([
+ apiCallers[0].bookmarks.createBookmark({
+ url: "https://old.example.com",
+ type: BookmarkTypes.LINK,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ url: "https://recent.example.com",
+ type: BookmarkTypes.LINK,
+ }),
+ ]);
+ await Promise.all([
+ setModifiedAt(
+ db,
+ oldLink.id,
+ new Date(now.getTime() - 2 * 60 * 60 * 1000),
+ ),
+ setModifiedAt(
+ db,
+ recentLink.id,
+ new Date(now.getTime() - 30 * 60 * 1000),
+ ),
+ ]);
+ testQueueMocks.lowPriorityCrawlerEnqueue.mockClear();
+
+ await adminApi.recrawlLinks({
+ crawlStatus: "all",
+ runInference: false,
+ modifiedWithinSeconds: 60 * 60,
+ });
+
+ expect(testQueueMocks.lowPriorityCrawlerEnqueue).toHaveBeenCalledTimes(1);
+ expect(testQueueMocks.lowPriorityCrawlerEnqueue).toHaveBeenCalledWith(
+ {
+ bookmarkId: recentLink.id,
+ runInference: false,
+ },
+ expect.objectContaining({
+ priority: QueuePriority.Low,
+ }),
+ );
+ });
+
+ test<CustomTestContext>("preserves the search index for a time-limited reindex", async ({
+ apiCallers,
+ db,
+ }) => {
+ const adminApi = await getAdminApi(db);
+ const [oldBookmark, recentBookmark] = await Promise.all([
+ apiCallers[0].bookmarks.createBookmark({
+ text: "old",
+ type: BookmarkTypes.TEXT,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ text: "recent",
+ type: BookmarkTypes.TEXT,
+ }),
+ ]);
+ await Promise.all([
+ setModifiedAt(
+ db,
+ oldBookmark.id,
+ new Date(now.getTime() - 2 * 60 * 60 * 1000),
+ ),
+ setModifiedAt(
+ db,
+ recentBookmark.id,
+ new Date(now.getTime() - 30 * 60 * 1000),
+ ),
+ ]);
+ testQueueMocks.triggerSearchReindex.mockClear();
+
+ await adminApi.reindexAllBookmarks({
+ modifiedWithinSeconds: 60 * 60,
+ });
+
+ expect(adminJobMocks.getSearchClient).not.toHaveBeenCalled();
+ expect(testQueueMocks.triggerSearchReindex).toHaveBeenCalledTimes(1);
+ expect(testQueueMocks.triggerSearchReindex).toHaveBeenCalledWith(
+ recentBookmark.id,
+ { priority: QueuePriority.Low },
+ );
+ });
+
+ test<CustomTestContext>("preserves the vector index for time-limited embedding regeneration", async ({
+ apiCallers,
+ db,
+ }) => {
+ const adminApi = await getAdminApi(db);
+ const [oldBookmark, recentBookmark] = await Promise.all([
+ apiCallers[0].bookmarks.createBookmark({
+ text: "old",
+ type: BookmarkTypes.TEXT,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ text: "recent",
+ type: BookmarkTypes.TEXT,
+ }),
+ ]);
+ await Promise.all([
+ setModifiedAt(
+ db,
+ oldBookmark.id,
+ new Date(now.getTime() - 2 * 60 * 60 * 1000),
+ ),
+ setModifiedAt(
+ db,
+ recentBookmark.id,
+ new Date(now.getTime() - 30 * 60 * 1000),
+ ),
+ ]);
+ testQueueMocks.embeddingsEnqueue.mockClear();
+
+ await adminApi.regenerateAllBookmarkEmbeddings({
+ status: "all",
+ modifiedWithinSeconds: 60 * 60,
+ });
+
+ expect(adminJobMocks.getVectorStoreClient).not.toHaveBeenCalled();
+ expect(testQueueMocks.embeddingsEnqueue).toHaveBeenCalledTimes(1);
+ expect(testQueueMocks.embeddingsEnqueue).toHaveBeenCalledWith(
+ {
+ bookmarkId: recentBookmark.id,
+ type: "embed",
+ force: true,
+ runTaggingOnComplete: false,
+ },
+ {
+ priority: QueuePriority.Low,
+ groupId: "admin",
+ },
+ );
+ });
+
+ test<CustomTestContext>("filters asset reprocessing using the bookmark modified date", async ({
+ apiCallers,
+ db,
+ }) => {
+ const adminApi = await getAdminApi(db);
+ const [oldAsset, recentAsset] = await Promise.all([
+ apiCallers[0].bookmarks.createBookmark({
+ text: "old asset",
+ type: BookmarkTypes.TEXT,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ text: "recent asset",
+ type: BookmarkTypes.TEXT,
+ }),
+ ]);
+ await db.insert(bookmarkAssets).values([
+ {
+ id: oldAsset.id,
+ assetId: "old-asset-id",
+ assetType: "image",
+ },
+ {
+ id: recentAsset.id,
+ assetId: "recent-asset-id",
+ assetType: "image",
+ },
+ ]);
+ await Promise.all([
+ setModifiedAt(
+ db,
+ oldAsset.id,
+ new Date(now.getTime() - 2 * 60 * 60 * 1000),
+ ),
+ setModifiedAt(
+ db,
+ recentAsset.id,
+ new Date(now.getTime() - 30 * 60 * 1000),
+ ),
+ ]);
+ testQueueMocks.assetPreprocessingEnqueue.mockClear();
+
+ await adminApi.reprocessAssetsFixMode({
+ modifiedWithinSeconds: 60 * 60,
+ });
+
+ expect(testQueueMocks.assetPreprocessingEnqueue).toHaveBeenCalledTimes(1);
+ expect(testQueueMocks.assetPreprocessingEnqueue).toHaveBeenCalledWith(
+ {
+ bookmarkId: recentAsset.id,
+ fixMode: true,
+ },
+ { priority: QueuePriority.Low },
+ );
+ });
+
+ test<CustomTestContext>("keeps the existing unbounded reindex behavior", async ({
+ apiCallers,
+ db,
+ }) => {
+ const adminApi = await getAdminApi(db);
+ const [firstBookmark, secondBookmark] = await Promise.all([
+ apiCallers[0].bookmarks.createBookmark({
+ text: "first",
+ type: BookmarkTypes.TEXT,
+ }),
+ apiCallers[0].bookmarks.createBookmark({
+ text: "second",
+ type: BookmarkTypes.TEXT,
+ }),
+ ]);
+ testQueueMocks.triggerSearchReindex.mockClear();
+ testQueueMocks.embeddingsEnqueue.mockClear();
+
+ await adminApi.reindexAllBookmarks();
+ await adminApi.regenerateAllBookmarkEmbeddings({ status: "all" });
+
+ expect(adminJobMocks.searchClearIndex).toHaveBeenCalledTimes(1);
+ expect(adminJobMocks.vectorClearIndex).toHaveBeenCalledTimes(1);
+ expect(
+ testQueueMocks.triggerSearchReindex.mock.calls
+ .map(([bookmarkId]) => bookmarkId)
+ .sort(),
+ ).toEqual([firstBookmark.id, secondBookmark.id].sort());
+ expect(
+ testQueueMocks.embeddingsEnqueue.mock.calls
+ .map(([payload]) => payload.bookmarkId)
+ .sort(),
+ ).toEqual([firstBookmark.id, secondBookmark.id].sort());
+ });
+ });
+
test<CustomTestContext>("admin API key uses granular admin scopes", async ({
apiCallers,
db,
diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts
index 76518181..a47f7226 100644
--- a/packages/trpc/routers/admin.ts
+++ b/packages/trpc/routers/admin.ts
@@ -1,10 +1,11 @@
import * as dns from "dns";
import { TRPCError } from "@trpc/server";
-import { and, asc, count, eq, gt, inArray, or, sum } from "drizzle-orm";
+import { and, asc, count, eq, gt, gte, inArray, or, sum } from "drizzle-orm";
import { z } from "zod";
import {
assets,
+ bookmarkAssets,
bookmarkLinks,
bookmarks,
subscriptions,
@@ -34,6 +35,7 @@ import {
resetPasswordSchema,
updateUserSchema,
zAdminCreateUserSchema,
+ zAdminJobModifiedWithinSecondsSchema,
} from "@karakeep/shared/types/admin";
import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
import { setUrlHostnameFromResolvedAddress } from "@karakeep/shared/utils/url";
@@ -50,6 +52,15 @@ const adminJobsProcedure = createAdminScopedProcedure("jobs");
const adminSystemProcedure = createAdminScopedProcedure("system");
const adminUsersProcedure = createAdminScopedProcedure("users");
+function modifiedWithin(modifiedWithinSeconds?: number) {
+ return modifiedWithinSeconds === undefined
+ ? undefined
+ : gte(
+ bookmarks.modifiedAt,
+ new Date(Date.now() - modifiedWithinSeconds * 1000),
+ );
+}
+
export const adminAppRouter = router({
stats: adminSystemProcedure
.output(
@@ -256,17 +267,22 @@ export const adminAppRouter = router({
z.object({
crawlStatus: z.enum(["success", "failure", "pending", "all"]),
runInference: z.boolean(),
+ modifiedWithinSeconds: zAdminJobModifiedWithinSecondsSchema.optional(),
}),
)
.mutation(async ({ ctx, input }) => {
- const bookmarkIds = await ctx.db.query.bookmarkLinks.findMany({
- columns: {
- id: true,
- },
- ...(input.crawlStatus === "all"
- ? {}
- : { where: eq(bookmarkLinks.crawlStatus, input.crawlStatus) }),
- });
+ const bookmarkIds = await ctx.db
+ .select({ id: bookmarkLinks.id })
+ .from(bookmarkLinks)
+ .innerJoin(bookmarks, eq(bookmarkLinks.id, bookmarks.id))
+ .where(
+ and(
+ input.crawlStatus === "all"
+ ? undefined
+ : eq(bookmarkLinks.crawlStatus, input.crawlStatus),
+ modifiedWithin(input.modifiedWithinSeconds),
+ ),
+ );
await Promise.all(
bookmarkIds.map((b) => {
@@ -281,36 +297,49 @@ export const adminAppRouter = router({
}),
);
}),
- reindexAllBookmarks: adminBookmarksProcedure.mutation(async ({ ctx }) => {
- const searchIdx = await getSearchClient();
- await searchIdx?.clearIndex();
- const bookmarkIds = await ctx.db.query.bookmarks.findMany({
- columns: {
- id: true,
- },
- });
-
- await Promise.all(
- bookmarkIds.map((b) =>
- triggerSearchReindex(b.id, {
- priority: QueuePriority.Low,
- }),
- ),
- );
- }),
+ reindexAllBookmarks: adminBookmarksProcedure
+ .input(
+ z
+ .object({
+ modifiedWithinSeconds:
+ zAdminJobModifiedWithinSecondsSchema.optional(),
+ })
+ .optional(),
+ )
+ .mutation(async ({ ctx, input }) => {
+ if (input?.modifiedWithinSeconds === undefined) {
+ const searchIdx = await getSearchClient();
+ await searchIdx?.clearIndex();
+ }
+ const bookmarkIds = await ctx.db
+ .select({ id: bookmarks.id })
+ .from(bookmarks)
+ .where(modifiedWithin(input?.modifiedWithinSeconds));
+
+ await Promise.all(
+ bookmarkIds.map((b) =>
+ triggerSearchReindex(b.id, {
+ priority: QueuePriority.Low,
+ }),
+ ),
+ );
+ }),
regenerateAllBookmarkEmbeddings: adminBookmarksProcedure
.input(
z
.object({
status: z.enum(["failure", "pending", "all"]).default("all"),
+ modifiedWithinSeconds:
+ zAdminJobModifiedWithinSecondsSchema.optional(),
})
.optional(),
)
.mutation(async ({ ctx, input }) => {
const status = input?.status ?? "all";
+ const modifiedAtFilter = modifiedWithin(input?.modifiedWithinSeconds);
- const vectorStore = await getVectorStoreClient();
- if (status === "all") {
+ if (status === "all" && input?.modifiedWithinSeconds === undefined) {
+ const vectorStore = await getVectorStoreClient();
await vectorStore?.clearIndex();
}
@@ -329,6 +358,7 @@ export const adminAppRouter = router({
status === "all"
? undefined
: eq(bookmarks.embeddingStatus, status),
+ modifiedAtFilter,
cursor ? gt(bookmarks.id, cursor) : undefined,
),
)
@@ -369,50 +399,58 @@ export const adminAppRouter = router({
}
}
}),
- reprocessAssetsFixMode: adminBookmarksProcedure.mutation(async ({ ctx }) => {
- const bookmarkIds = await ctx.db.query.bookmarkAssets.findMany({
- columns: {
- id: true,
- },
- });
-
- await Promise.all(
- bookmarkIds.map((b) =>
- AssetPreprocessingQueue.enqueue(
- {
- bookmarkId: b.id,
- fixMode: true,
- },
- {
- priority: QueuePriority.Low,
- },
+ reprocessAssetsFixMode: adminBookmarksProcedure
+ .input(
+ z
+ .object({
+ modifiedWithinSeconds:
+ zAdminJobModifiedWithinSecondsSchema.optional(),
+ })
+ .optional(),
+ )
+ .mutation(async ({ ctx, input }) => {
+ const bookmarkIds = await ctx.db
+ .select({ id: bookmarkAssets.id })
+ .from(bookmarkAssets)
+ .innerJoin(bookmarks, eq(bookmarkAssets.id, bookmarks.id))
+ .where(modifiedWithin(input?.modifiedWithinSeconds));
+
+ await Promise.all(
+ bookmarkIds.map((b) =>
+ AssetPreprocessingQueue.enqueue(
+ {
+ bookmarkId: b.id,
+ fixMode: true,
+ },
+ {
+ priority: QueuePriority.Low,
+ },
+ ),
),
- ),
- );
- }),
+ );
+ }),
reRunInferenceOnAllBookmarks: adminBookmarksProcedure
.input(
z.object({
type: z.enum(["tag", "summarize"]),
status: z.enum(["success", "failure", "pending", "all"]),
+ modifiedWithinSeconds: zAdminJobModifiedWithinSecondsSchema.optional(),
}),
)
.mutation(async ({ input, ctx }) => {
- const bookmarkIds = await ctx.db.query.bookmarks.findMany({
- columns: {
- id: true,
- },
- ...{
- tag:
- input.status === "all"
- ? {}
- : { where: eq(bookmarks.taggingStatus, input.status) },
- summarize:
- input.status === "all"
- ? {}
- : { where: eq(bookmarks.summarizationStatus, input.status) },
- }[input.type],
- });
+ const statusFilter =
+ input.status === "all"
+ ? undefined
+ : eq(
+ input.type === "tag"
+ ? bookmarks.taggingStatus
+ : bookmarks.summarizationStatus,
+ input.status,
+ );
+ const bookmarkIds = await ctx.db
+ .select({ id: bookmarks.id })
+ .from(bookmarks)
+ .where(and(statusFilter, modifiedWithin(input.modifiedWithinSeconds)));
await Promise.all(
bookmarkIds.map((b) =>
diff --git a/packages/trpc/testUtils.ts b/packages/trpc/testUtils.ts
index be5b9bcb..3a038c71 100644
--- a/packages/trpc/testUtils.ts
+++ b/packages/trpc/testUtils.ts
@@ -7,6 +7,21 @@ import type { Context } from "./index";
import { createCallerFactory } from "./index";
import { appRouter } from "./routers/_app";
+const testQueueMocks = vi.hoisted(() => ({
+ assetPreprocessingEnqueue: vi.fn(),
+ embeddingsEnqueue: vi.fn(),
+ linkCrawlerEnqueue: vi.fn(),
+ lowPriorityCrawlerEnqueue: vi.fn(),
+ openAIEnqueue: vi.fn(),
+ ruleEngineEnqueue: vi.fn(),
+ searchIndexingEnqueue: vi.fn(),
+ triggerSearchReindex: vi.fn(),
+}));
+
+export function getTestQueueMocks() {
+ return testQueueMocks;
+}
+
export function getTestDB() {
return getInMemoryDB(true);
}
@@ -105,22 +120,28 @@ export function defaultBeforeEach(seedDB = true) {
(await original()) as typeof import("@karakeep/shared-server");
return {
...mod,
+ AssetPreprocessingQueue: {
+ enqueue: testQueueMocks.assetPreprocessingEnqueue,
+ },
LinkCrawlerQueue: {
- enqueue: vi.fn(),
+ enqueue: testQueueMocks.linkCrawlerEnqueue,
+ },
+ LowPriorityCrawlerQueue: {
+ enqueue: testQueueMocks.lowPriorityCrawlerEnqueue,
},
OpenAIQueue: {
- enqueue: vi.fn(),
+ enqueue: testQueueMocks.openAIEnqueue,
},
EmbeddingsQueue: {
- enqueue: vi.fn(),
+ enqueue: testQueueMocks.embeddingsEnqueue,
},
SearchIndexingQueue: {
- enqueue: vi.fn(),
+ enqueue: testQueueMocks.searchIndexingEnqueue,
},
RuleEngineQueue: {
- enqueue: vi.fn(),
+ enqueue: testQueueMocks.ruleEngineEnqueue,
},
- triggerSearchReindex: vi.fn(),
+ triggerSearchReindex: testQueueMocks.triggerSearchReindex,
};
});
Object.assign(context, await buildTestContext(seedDB));