Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

chore: add tests for the inference worker

Mohamed Bassem364cb8a4b03e
diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.yml
index 74c3df2a..ea47c3f9 100644
--- a/packages/e2e_tests/docker-compose.yml
+++ b/packages/e2e_tests/docker-compose.yml
@@ -16,6 +16,9 @@ services:
BROWSER_WEB_URL: http://chrome:9222
CRAWLER_NUM_WORKERS: 6
CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx
+ OPENAI_API_KEY: aimock-test-key
+ OPENAI_BASE_URL: http://aimock:4010/v1
+ INFERENCE_ENABLE_AUTO_SUMMARIZATION: "true"
meilisearch:
image: getmeili/meilisearch:v1.41.0
restart: unless-stopped
@@ -49,6 +52,28 @@ services:
command: server /data --console-address ":9001"
volumes:
- minio_data:/data
+ aimock:
+ image: ghcr.io/copilotkit/aimock:latest
+ restart: unless-stopped
+ ports:
+ - "${AIMOCK_PORT:-4010}:4010"
+ volumes:
+ - ./setup/aimock:/fixtures:ro
+ command:
+ - --fixtures
+ - /fixtures
+ - --host
+ - 0.0.0.0
+ healthcheck:
+ test:
+ - CMD
+ - wget
+ - --spider
+ - -q
+ - http://127.0.0.1:4010/health
+ interval: 5s
+ timeout: 3s
+ retries: 20
volumes:
minio_data:
diff --git a/packages/e2e_tests/setup/aimock/inference.json b/packages/e2e_tests/setup/aimock/inference.json
new file mode 100644
index 00000000..20651396
--- /dev/null
+++ b/packages/e2e_tests/setup/aimock/inference.json
@@ -0,0 +1,23 @@
+{
+ "fixtures": [
+ {
+ "match": {
+ "model": "gpt-4.1-mini",
+ "responseFormat": "json_schema",
+ "userMessage": "automatic tagging for a read-it-later/bookmarking app"
+ },
+ "response": {
+ "content": "{\"tags\":[\"ai-generated\",\"karakeep\",\"worker-test\"]}"
+ }
+ },
+ {
+ "match": {
+ "model": "gpt-4.1-mini",
+ "userMessage": "Summarize the following content responding ONLY with the summary."
+ },
+ "response": {
+ "content": "This page contains a short Hello World test document used to verify Karakeep's inference worker end-to-end."
+ }
+ }
+ ]
+}
diff --git a/packages/e2e_tests/setup/startContainers.ts b/packages/e2e_tests/setup/startContainers.ts
index 4d2a89cf..7a8b2b40 100644
--- a/packages/e2e_tests/setup/startContainers.ts
+++ b/packages/e2e_tests/setup/startContainers.ts
@@ -29,31 +29,54 @@ async function waitForHealthy(port: number, timeout = 60000): Promise<void> {
);
}
+async function waitForAimockHealthy(
+ port: number,
+ timeout = 60000,
+): Promise<void> {
+ return waitUntil(
+ async () => {
+ const response = await fetch(`http://localhost:${port}/health`);
+ return response.status === 200;
+ },
+ "Aimock is healthy",
+ timeout,
+ );
+}
+
export default async function ({ provide }: GlobalSetupContext) {
const __dirname = path.dirname(fileURLToPath(import.meta.url));
- const port = await getRandomPort();
+ const karakeepPort = await getRandomPort();
+ const aimockPort = await getRandomPort();
const buildArg = process.env.E2E_TEST_NO_BUILD ? "" : "--build";
- console.log(`Starting docker compose on port ${port}...`);
+ console.log(
+ `Starting docker compose on ports karakeep=${karakeepPort} aimock=${aimockPort}...`,
+ );
execSync(`docker compose up ${buildArg} -d`, {
cwd: __dirname,
stdio: "inherit",
env: {
...process.env,
- KARAKEEP_PORT: port.toString(),
+ KARAKEEP_PORT: karakeepPort.toString(),
+ AIMOCK_PORT: aimockPort.toString(),
},
});
- console.log("Waiting for service to become healthy...");
- await waitForHealthy(port);
+ console.log("Waiting for services to become healthy...");
+ await Promise.all([
+ waitForHealthy(karakeepPort),
+ waitForAimockHealthy(aimockPort),
+ ]);
// Wait 5 seconds for the worker to start
await new Promise((resolve) => setTimeout(resolve, 5000));
- provide("karakeepPort", port);
+ provide("karakeepPort", karakeepPort);
+ provide("aimockPort", aimockPort);
- process.env.KARAKEEP_PORT = port.toString();
+ process.env.KARAKEEP_PORT = karakeepPort.toString();
+ process.env.AIMOCK_PORT = aimockPort.toString();
return async () => {
console.log("Capturing docker logs...");
@@ -61,7 +84,14 @@ export default async function ({ provide }: GlobalSetupContext) {
const logsDir = path.join(__dirname, "docker-logs");
execSync(`mkdir -p "${logsDir}"`, { cwd: __dirname });
- const services = ["web", "meilisearch", "chrome", "nginx", "minio"];
+ const services = [
+ "web",
+ "meilisearch",
+ "chrome",
+ "nginx",
+ "minio",
+ "aimock",
+ ];
for (const service of services) {
try {
execSync(
@@ -91,5 +121,6 @@ export default async function ({ provide }: GlobalSetupContext) {
declare module "vitest" {
export interface ProvidedContext {
karakeepPort: number;
+ aimockPort: number;
}
}
diff --git a/packages/e2e_tests/tests/workers/inference.test.ts b/packages/e2e_tests/tests/workers/inference.test.ts
new file mode 100644
index 00000000..527fb38d
--- /dev/null
+++ b/packages/e2e_tests/tests/workers/inference.test.ts
@@ -0,0 +1,133 @@
+import { assert, beforeEach, describe, expect, inject, it } from "vitest";
+
+import { createKarakeepClient } from "@karakeep/sdk";
+
+import { createTestUser } from "../../utils/api";
+import { waitUntil } from "../../utils/general";
+import { getTrpcClient } from "../../utils/trpc";
+
+describe("Inference Worker Tests", () => {
+ const port = inject("karakeepPort");
+
+ if (!port) {
+ throw new Error("Missing required environment variables");
+ }
+
+ let client: ReturnType<typeof createKarakeepClient>;
+ let trpc: ReturnType<typeof getTrpcClient>;
+ let apiKey: string;
+
+ beforeEach(async () => {
+ apiKey = await createTestUser();
+ client = createKarakeepClient({
+ baseUrl: `http://localhost:${port}/api/v1/`,
+ headers: {
+ "Content-Type": "application/json",
+ authorization: `Bearer ${apiKey}`,
+ },
+ });
+ trpc = getTrpcClient(apiKey);
+ });
+
+ it("auto-tags text bookmarks", async () => {
+ await trpc.users.updateSettings.mutate({
+ autoTaggingEnabled: true,
+ });
+
+ const { data: createdBookmark, error } = await client.POST("/bookmarks", {
+ body: {
+ type: "text",
+ title: "Inference text bookmark",
+ text: "Karakeep should tag this bookmark through the inference worker.",
+ },
+ });
+
+ if (error) {
+ throw error;
+ }
+ assert(createdBookmark);
+
+ await waitUntil(async () => {
+ const { data: bookmark } = await client.GET("/bookmarks/{bookmarkId}", {
+ params: {
+ path: {
+ bookmarkId: createdBookmark.id,
+ },
+ },
+ });
+
+ return bookmark?.taggingStatus === "success";
+ }, "Text bookmark tagging completes");
+
+ const { data: bookmark } = await client.GET("/bookmarks/{bookmarkId}", {
+ params: {
+ path: {
+ bookmarkId: createdBookmark.id,
+ },
+ },
+ });
+
+ assert(bookmark);
+ expect(bookmark.taggingStatus).toBe("success");
+ expect(bookmark.tags.map((tag) => tag.name)).toEqual(
+ expect.arrayContaining(["ai-generated", "karakeep", "worker-test"]),
+ );
+ }, 120000);
+
+ it("auto-tags and summarizes crawled link bookmarks", async () => {
+ await trpc.users.updateSettings.mutate({
+ autoTaggingEnabled: true,
+ autoSummarizationEnabled: true,
+ });
+
+ const { data: createdBookmark, error } = await client.POST("/bookmarks", {
+ body: {
+ type: "link",
+ title: "Inference link bookmark",
+ url: "http://nginx:80/hello.html",
+ },
+ });
+
+ if (error) {
+ throw error;
+ }
+ assert(createdBookmark);
+
+ await waitUntil(
+ async () => {
+ const { data: bookmark } = await client.GET("/bookmarks/{bookmarkId}", {
+ params: {
+ path: {
+ bookmarkId: createdBookmark.id,
+ },
+ },
+ });
+
+ return (
+ bookmark?.taggingStatus === "success" &&
+ bookmark?.summarizationStatus === "success"
+ );
+ },
+ "Link bookmark inference completes",
+ 120000,
+ );
+
+ const { data: bookmark } = await client.GET("/bookmarks/{bookmarkId}", {
+ params: {
+ path: {
+ bookmarkId: createdBookmark.id,
+ },
+ },
+ });
+
+ assert(bookmark);
+ expect(bookmark.taggingStatus).toBe("success");
+ expect(bookmark.summarizationStatus).toBe("success");
+ expect(bookmark.summary).toBe(
+ "This page contains a short Hello World test document used to verify Karakeep's inference worker end-to-end.",
+ );
+ expect(bookmark.tags.map((tag) => tag.name)).toEqual(
+ expect.arrayContaining(["ai-generated", "karakeep", "worker-test"]),
+ );
+ }, 120000);
+});
diff --git a/packages/e2e_tests/utils/api.ts b/packages/e2e_tests/utils/api.ts
index 9f9052fc..45aea93c 100644
--- a/packages/e2e_tests/utils/api.ts
+++ b/packages/e2e_tests/utils/api.ts
@@ -53,5 +53,11 @@ export async function createTestUser() {
keyName: "test-key",
});
+ const authedTrpc = getTrpcClient(key);
+ await authedTrpc.users.updateSettings.mutate({
+ autoTaggingEnabled: false,
+ autoSummarizationEnabled: false,
+ });
+
return key;
}