Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

chore: Add seed snapshot generator (#2829)

Mohamed Bassem53da5e2bbe94
diff --git a/.dockerignore b/.dockerignore
index 748c949b..96954fb7 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -19,6 +19,7 @@ apps/landing
apps/browser-extension
packages/e2e_tests
packages/benchmarks
+tools/seed-snapshot
# Aider
.aider*
diff --git a/package.json b/package.json
index b0a5b7dc..c355d5c4 100644
--- a/package.json
+++ b/package.json
@@ -11,6 +11,8 @@
"db:generate": "pnpm --filter @karakeep/db run generate",
"db:migrate": "pnpm --filter @karakeep/db run migrate",
"db:studio": "pnpm --filter @karakeep/db studio",
+ "seed:apply": "pnpm --filter @karakeep/seed-snapshot run apply",
+ "seed:snapshot": "pnpm --filter @karakeep/seed-snapshot run snapshot",
"workers": "pnpm --filter @karakeep/workers run start",
"web": "pnpm --filter @karakeep/web run dev",
"web:portless": "pnpm --filter @karakeep/web run dev:portless",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 71c9a6f8..2e3d52c6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1618,6 +1618,31 @@ importers:
specifier: ^5.9
version: 5.9.3
+ tools/seed-snapshot:
+ dependencies:
+ '@karakeep/shared':
+ specifier: workspace:^0.1.0
+ version: link:../../packages/shared
+ '@karakeep/trpc':
+ specifier: workspace:^0.1.0
+ version: link:../../packages/trpc
+ '@trpc/client':
+ specifier: ^11.9.0
+ superjson:
+ specifier: ^2.2.1
+ version: 2.2.1
+ devDependencies:
+ '@karakeep/tsconfig':
+ specifier: workspace:^0.1.0
+ version: link:../../tooling/typescript
+ oxlint:
+ specifier: ^1.60.0
+ version: 1.60.0
+ tsx:
+ specifier: ^4.8.1
+ version: 4.21.0
+
packages:
'@0no-co/[email protected]':
diff --git a/snapshots/seed-data-2026-05-20-163735.json b/snapshots/seed-data-2026-05-20-163735.json
new file mode 100644
index 00000000..cff6efe7
--- /dev/null
+++ b/snapshots/seed-data-2026-05-20-163735.json
@@ -0,0 +1,256 @@
+{
+ "createdAt": "2026-05-20T15:37:35.334Z",
+ "sourceCommit": "021cb12579bdf4cc5e2b50f09170c23f99ed6eab",
+ "archive": "seed-data-2026-05-20-163735.tar.gz",
+ "users": [
+ {
+ "email": "[email protected]",
+ "password": "test1234",
+ "seeded": true,
+ "apiKeyName": "[email protected]",
+ "counts": {
+ "bookmarks": 20,
+ "tags": 3,
+ "manualLists": 2,
+ "smartLists": 2
+ },
+ "bookmarks": [
+ {
+ "id": "km2jba5wyxzsa07ea1rvlkw6",
+ "url": "https://www.nasa.gov/",
+ "title": "Seed snapshot bookmark 1",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "kxriupr76n9swxb1c5ypzk7x",
+ "url": "https://www.nationalgeographic.com/",
+ "title": "Seed snapshot bookmark 2",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "smw5grpmamknt52h2egre7w9",
+ "url": "https://www.theverge.com/",
+ "title": "Seed snapshot bookmark 3",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "t47py0t3mi1abngrzbh0ve67",
+ "url": "https://www.wired.com/",
+ "title": "Seed snapshot bookmark 4",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "exe0kj1nr1yheba71hmhnpr7",
+ "url": "https://www.apple.com/ipad-pro/",
+ "title": "Seed snapshot bookmark 5",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "yx72o5hos53bmbjhoxcv9z2g",
+ "url": "https://www.figma.com/",
+ "title": "Seed snapshot bookmark 6",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "xkoqt13nxufsk5b1y60fxo5k",
+ "url": "https://linear.app/",
+ "title": "Seed snapshot bookmark 7",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "thv2c1k4z4reuu7yc6482i8t",
+ "url": "https://www.notion.com/",
+ "title": "Seed snapshot bookmark 8",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "psg7m6mhwm1xfnpyqskb6jy5",
+ "url": "https://stripe.com/",
+ "title": "Seed snapshot bookmark 9",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "gmab7mbludxywqxx8hz9qxt6",
+ "url": "https://www.shopify.com/",
+ "title": "Seed snapshot bookmark 10",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "e2ej14zclzd0gw91u17fwux9",
+ "url": "https://developer.mozilla.org/en-US/docs/Web",
+ "title": "Seed snapshot bookmark 11",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "j0b03m8ks0cmgo8lkokku3mn",
+ "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript",
+ "title": "Seed snapshot bookmark 12",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "aflpy1knahedicy7hihiwz0k",
+ "url": "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",
+ "title": "Seed snapshot bookmark 13",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "lc8sdxnnbh6lxrp79ou39fnb",
+ "url": "https://developer.mozilla.org/en-US/docs/Web/CSS",
+ "title": "Seed snapshot bookmark 14",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "sb4rtshzmsa3sasllcn54gnz",
+ "url": "https://web.dev/learn/html",
+ "title": "Seed snapshot bookmark 15",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "pk4h0d754vjs6dm3508c5q78",
+ "url": "https://web.dev/learn/css",
+ "title": "Seed snapshot bookmark 16",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "jqdfmk4yb9ekec4kkg8a69cg",
+ "url": "https://web.dev/learn/javascript",
+ "title": "Seed snapshot bookmark 17",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "zcbcwbus69sc701khrtmyccz",
+ "url": "https://www.raspberrypi.com/products/raspberry-pi-5/",
+ "title": "Seed snapshot bookmark 18",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "uhvuf05w0ldy0k3bqbcslofu",
+ "url": "https://www.postgresql.org/docs/current/index.html",
+ "title": "Seed snapshot bookmark 19",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "q5mf9byhs1nekffbs25ty8rs",
+ "url": "https://docs.docker.com/get-started/",
+ "title": "Seed snapshot bookmark 20",
+ "crawlStatus": "success"
+ }
+ ],
+ "tags": [
+ "research",
+ "docs",
+ "product"
+ ],
+ "lists": [
+ {
+ "name": "Reading Queue",
+ "type": "manual",
+ "parentName": null,
+ "query": null
+ },
+ {
+ "name": "Reference",
+ "type": "manual",
+ "parentName": "Reading Queue",
+ "query": null
+ },
+ {
+ "name": "Research Smart",
+ "type": "smart",
+ "parentName": null,
+ "query": "tag:research"
+ },
+ {
+ "name": "Favorites Smart",
+ "type": "smart",
+ "parentName": "Reading Queue",
+ "query": "is:fav"
+ }
+ ]
+ },
+ {
+ "email": "[email protected]",
+ "password": "test1234",
+ "seeded": true,
+ "apiKeyName": "[email protected]",
+ "counts": {
+ "bookmarks": 4,
+ "tags": 3,
+ "manualLists": 2,
+ "smartLists": 2
+ },
+ "bookmarks": [
+ {
+ "id": "c9ybiun3c0lwfqvbzuumasvq",
+ "url": "https://github.com/features/actions",
+ "title": "Seed snapshot bookmark 1",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "z09y6r8bmn0c9j0z687esgus",
+ "url": "https://www.cloudflare.com/",
+ "title": "Seed snapshot bookmark 2",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "k9bkj37wdby9yxn726fsbc4f",
+ "url": "https://web.dev/",
+ "title": "Seed snapshot bookmark 3",
+ "crawlStatus": "success"
+ },
+ {
+ "id": "tvsyx6oswqu8loayf4gc7i7y",
+ "url": "https://www.raspberrypi.com/products/raspberry-pi-5/",
+ "title": "Seed snapshot bookmark 4",
+ "crawlStatus": "success"
+ }
+ ],
+ "tags": [
+ "engineering",
+ "design",
+ "ops"
+ ],
+ "lists": [
+ {
+ "name": "Workbench",
+ "type": "manual",
+ "parentName": null,
+ "query": null
+ },
+ {
+ "name": "Deep Dives",
+ "type": "manual",
+ "parentName": "Workbench",
+ "query": null
+ },
+ {
+ "name": "Engineering Smart",
+ "type": "smart",
+ "parentName": null,
+ "query": "tag:engineering"
+ },
+ {
+ "name": "Archived Smart",
+ "type": "smart",
+ "parentName": "Workbench",
+ "query": "is:archived"
+ }
+ ]
+ },
+ {
+ "email": "[email protected]",
+ "password": "test1234",
+ "seeded": false,
+ "apiKeyName": "[email protected]",
+ "counts": {
+ "bookmarks": 0,
+ "tags": 0,
+ "manualLists": 0,
+ "smartLists": 0
+ },
+ "bookmarks": [],
+ "tags": [],
+ "lists": []
+ }
+ ]
+}
diff --git a/snapshots/seed-data-2026-05-20-163735.tar.gz b/snapshots/seed-data-2026-05-20-163735.tar.gz
new file mode 100644
index 00000000..5c42603f
Binary files /dev/null and b/snapshots/seed-data-2026-05-20-163735.tar.gz differ
diff --git a/tools/seed-snapshot/.gitignore b/tools/seed-snapshot/.gitignore
new file mode 100644
index 00000000..5f65e0ba
--- /dev/null
+++ b/tools/seed-snapshot/.gitignore
@@ -0,0 +1,2 @@
+.tmp/
+docker-logs/
diff --git a/tools/seed-snapshot/docker-compose.yml b/tools/seed-snapshot/docker-compose.yml
new file mode 100644
index 00000000..98697924
--- /dev/null
+++ b/tools/seed-snapshot/docker-compose.yml
@@ -0,0 +1,35 @@
+services:
+ web:
+ build:
+ dockerfile: docker/Dockerfile
+ context: ../../
+ target: aio
+ restart: unless-stopped
+ ports:
+ - "${KARAKEEP_PORT:-3000}:3000"
+ environment:
+ DATA_DIR: /data
+ NEXTAUTH_SECRET: seed-snapshot-secret
+ NEXTAUTH_URL: http://localhost:${KARAKEEP_PORT:-3000}
+ MEILI_MASTER_KEY: dummy
+ MEILI_ADDR: http://meilisearch:7700
+ BROWSER_WEB_URL: http://chrome:9222
+ CRAWLER_NUM_WORKERS: 4
+ volumes:
+ - "${SEED_SNAPSHOT_DATA_DIR:?SEED_SNAPSHOT_DATA_DIR is required}:/data"
+ meilisearch:
+ image: getmeili/meilisearch:v1.41.0
+ restart: unless-stopped
+ environment:
+ MEILI_NO_ANALYTICS: "true"
+ MEILI_MASTER_KEY: dummy
+ chrome:
+ image: gcr.io/zenika-hub/alpine-chrome:124
+ restart: unless-stopped
+ command:
+ - --no-sandbox
+ - --disable-gpu
+ - --disable-dev-shm-usage
+ - --remote-debugging-address=0.0.0.0
+ - --remote-debugging-port=9222
+ - --hide-scrollbars
diff --git a/tools/seed-snapshot/package.json b/tools/seed-snapshot/package.json
new file mode 100644
index 00000000..b80f31e2
--- /dev/null
+++ b/tools/seed-snapshot/package.json
@@ -0,0 +1,27 @@
+{
+ "$schema": "https://json.schemastore.org/package.json",
+ "name": "@karakeep/seed-snapshot",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "apply": "tsx src/apply.ts",
+ "snapshot": "tsx src/index.ts",
+ "lint": "oxlint --ignore-path=../../.gitignore .",
+ "lint:fix": "oxlint --ignore-path=../../.gitignore . --fix",
+ "format": "oxfmt --ignore-path=../../.gitignore --check .",
+ "format:fix": "oxfmt --ignore-path=../../.gitignore .",
+ "typecheck": "tsc --noEmit"
+ },
+ "dependencies": {
+ "@karakeep/shared": "workspace:^0.1.0",
+ "@karakeep/trpc": "workspace:^0.1.0",
+ "@trpc/client": "^11.9.0",
+ "superjson": "^2.2.1"
+ },
+ "devDependencies": {
+ "@karakeep/tsconfig": "workspace:^0.1.0",
+ "oxlint": "^1.60.0",
+ "tsx": "^4.8.1"
+ }
+}
diff --git a/tools/seed-snapshot/src/apply.ts b/tools/seed-snapshot/src/apply.ts
new file mode 100644
index 00000000..f68e7f58
--- /dev/null
+++ b/tools/seed-snapshot/src/apply.ts
@@ -0,0 +1,90 @@
+import { execFileSync } from "child_process";
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const SNAPSHOT_PATTERN = /^seed-data-\d{4}-\d{2}-\d{2}-\d{6}\.tar\.gz$/;
+
+function logStep(title: string): void {
+ console.log(`\n== ${title}`);
+}
+
+function logInfo(message: string): void {
+ console.log(` -- ${message}`);
+}
+
+function logSuccess(message: string): void {
+ console.log(` OK ${message}`);
+}
+
+function hasForceFlag(): boolean {
+ return process.argv.slice(2).includes("--force");
+}
+
+async function pathExists(targetPath: string): Promise<boolean> {
+ try {
+ await fs.stat(targetPath);
+ return true;
+ } catch (error) {
+ if (
+ error &&
+ typeof error === "object" &&
+ "code" in error &&
+ error.code === "ENOENT"
+ ) {
+ return false;
+ }
+ throw error;
+ }
+}
+
+async function findLatestSnapshot(snapshotsDir: string): Promise<string> {
+ const entries = await fs.readdir(snapshotsDir);
+ const snapshots = entries
+ .filter((entry) => SNAPSHOT_PATTERN.test(entry))
+ .sort();
+
+ const latest = snapshots.at(-1);
+ if (!latest) {
+ throw new Error(`No seed snapshots found in ${snapshotsDir}`);
+ }
+
+ return path.join(snapshotsDir, latest);
+}
+
+async function main(): Promise<void> {
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ const repoRoot = path.join(__dirname, "..", "..", "..");
+ const snapshotsDir = path.join(repoRoot, "snapshots");
+ const dataDir = path.join(repoRoot, "data");
+ const force = hasForceFlag();
+
+ logStep("Finding latest seed snapshot");
+ const snapshotPath = await findLatestSnapshot(snapshotsDir);
+ logInfo(`Using ${path.relative(repoRoot, snapshotPath)}`);
+
+ if (await pathExists(dataDir)) {
+ if (!force) {
+ throw new Error(
+ `Refusing to overwrite existing data dir at ${dataDir}. Re-run with --force to replace it.`,
+ );
+ }
+
+ logStep("Removing existing data dir");
+ await fs.rm(dataDir, { force: true, recursive: true });
+ }
+
+ logStep("Applying seed snapshot");
+ await fs.mkdir(dataDir, { recursive: true });
+ execFileSync("tar", ["-xzf", snapshotPath, "-C", dataDir], {
+ cwd: repoRoot,
+ stdio: "inherit",
+ });
+
+ logSuccess(`Applied snapshot to ${dataDir}`);
+}
+
+main().catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+});
diff --git a/tools/seed-snapshot/src/index.ts b/tools/seed-snapshot/src/index.ts
new file mode 100644
index 00000000..f031a8d5
--- /dev/null
+++ b/tools/seed-snapshot/src/index.ts
@@ -0,0 +1,623 @@
+import { execFileSync, execSync } from "child_process";
+import fs from "fs/promises";
+import net from "net";
+import path from "path";
+import { fileURLToPath } from "url";
+
+import { createTRPCClient, httpBatchLink } from "@trpc/client";
+import superjson from "superjson";
+
+import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+import type {
+ ZBookmark,
+ ZBookmarkedLink,
+} from "@karakeep/shared/types/bookmarks";
+import type { ZBookmarkList } from "@karakeep/shared/types/lists";
+import type { ZTagBasic } from "@karakeep/shared/types/tags";
+import type { AppRouter } from "@karakeep/trpc/routers/_app";
+
+type TrpcClient = ReturnType<typeof getTrpcClient>;
+type CrawlStatus = NonNullable<ZBookmarkedLink["crawlStatus"]>;
+
+const PASSWORD = "test1234";
+const USER_DATASETS = {
+ test1: {
+ tagNames: ["research", "docs", "product"],
+ urls: [
+ "https://www.nasa.gov/",
+ "https://www.nationalgeographic.com/",
+ "https://www.theverge.com/",
+ "https://www.wired.com/",
+ "https://www.apple.com/ipad-pro/",
+ "https://www.figma.com/",
+ "https://linear.app/",
+ "https://www.notion.com/",
+ "https://stripe.com/",
+ "https://www.shopify.com/",
+ "https://developer.mozilla.org/en-US/docs/Web",
+ "https://developer.mozilla.org/en-US/docs/Web/JavaScript",
+ "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API",
+ "https://developer.mozilla.org/en-US/docs/Web/CSS",
+ "https://web.dev/learn/html",
+ "https://web.dev/learn/css",
+ "https://web.dev/learn/javascript",
+ "https://www.raspberrypi.com/products/raspberry-pi-5/",
+ "https://www.postgresql.org/docs/current/index.html",
+ "https://docs.docker.com/get-started/",
+ ],
+ lists: {
+ root: "Reading Queue",
+ child: "Reference",
+ smartTag: "Research Smart",
+ smartFav: "Favorites Smart",
+ },
+ },
+ test2: {
+ tagNames: ["engineering", "design", "ops"],
+ urls: [
+ "https://github.com/features/actions",
+ "https://www.cloudflare.com/",
+ "https://web.dev/",
+ "https://www.raspberrypi.com/products/raspberry-pi-5/",
+ ],
+ lists: {
+ root: "Workbench",
+ child: "Deep Dives",
+ smartTag: "Engineering Smart",
+ smartFav: "Archived Smart",
+ },
+ },
+} as const;
+
+type UserDataset = (typeof USER_DATASETS)[keyof typeof USER_DATASETS];
+
+const USERS = [
+ {
+ email: "[email protected]",
+ name: "Seed Test 1",
+ datasetKey: "test1",
+ },
+ {
+ email: "[email protected]",
+ name: "Seed Test 2",
+ datasetKey: "test2",
+ },
+ { email: "[email protected]", name: "Seed Test 3", datasetKey: null },
+] as const;
+
+interface SeededBookmark {
+ id: string;
+ url: string;
+ title: string;
+ crawlStatus: CrawlStatus | null;
+}
+
+interface SeededUserManifest {
+ email: string;
+ password: string;
+ seeded: boolean;
+ apiKeyName: string;
+ counts: {
+ bookmarks: number;
+ tags: number;
+ manualLists: number;
+ smartLists: number;
+ };
+ bookmarks: SeededBookmark[];
+ tags: string[];
+ lists: Array<{
+ name: string;
+ type: "manual" | "smart";
+ parentName: string | null;
+ query: string | null;
+ }>;
+}
+
+interface SnapshotManifest {
+ createdAt: string;
+ sourceCommit: string | null;
+ archive: string;
+ users: SeededUserManifest[];
+}
+
+function logStep(title: string): void {
+ console.log(`\n== ${title}`);
+}
+
+function logInfo(message: string): void {
+ console.log(` -- ${message}`);
+}
+
+function logSuccess(message: string): void {
+ console.log(` OK ${message}`);
+}
+
+function logWarn(message: string): void {
+ console.log(` !! ${message}`);
+}
+
+function sleep(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitUntil(
+ fn: () => Promise<boolean>,
+ description: string,
+ timeoutMs = 60000,
+ intervalMs = 1000,
+): Promise<void> {
+ const start = Date.now();
+ while (Date.now() - start < timeoutMs) {
+ try {
+ if (await fn()) return;
+ } catch {
+ // Retry until timeout. Services may still be warming up.
+ }
+ await sleep(intervalMs);
+ }
+ throw new Error(`${description} timed out after ${timeoutMs}ms`);
+}
+
+async function getRandomPort(): Promise<number> {
+ const server = net.createServer();
+ return new Promise<number>((resolve, reject) => {
+ server.unref();
+ server.on("error", reject);
+ server.listen(0, () => {
+ const address = server.address() as net.AddressInfo;
+ server.close(() => resolve(address.port));
+ });
+ });
+}
+
+function getTrpcClient(
+ apiKey?: string,
+): ReturnType<typeof createTRPCClient<AppRouter>> {
+ const port = process.env.KARAKEEP_PORT;
+ if (!port) {
+ throw new Error("KARAKEEP_PORT is not set. Did you start the containers?");
+ }
+
+ return createTRPCClient<AppRouter>({
+ links: [
+ httpBatchLink({
+ transformer: superjson,
+ url: `http://localhost:${port}/api/trpc`,
+ headers() {
+ return {
+ authorization: apiKey ? `Bearer ${apiKey}` : undefined,
+ };
+ },
+ }),
+ ],
+ });
+}
+
+async function waitForHealthy(port: number): Promise<void> {
+ await waitUntil(
+ async () => {
+ const res = await fetch(`http://localhost:${port}/api/health`);
+ return res.status === 200;
+ },
+ "Karakeep stack to become healthy",
+ Number(process.env.SEED_SNAPSHOT_HEALTH_TIMEOUT_MS ?? 120_000),
+ 1_000,
+ );
+}
+
+async function captureDockerLogs(
+ composeDir: string,
+ composeEnv: NodeJS.ProcessEnv,
+): Promise<void> {
+ const logsDir = path.join(composeDir, "docker-logs");
+ await fs.mkdir(logsDir, { recursive: true });
+
+ for (const service of ["web", "meilisearch", "chrome"]) {
+ try {
+ execSync(
+ `/bin/sh -c 'docker compose logs ${service} > "${logsDir}/${service}.log" 2>&1'`,
+ {
+ cwd: composeDir,
+ env: composeEnv,
+ stdio: "ignore",
+ },
+ );
+ logInfo(`Captured logs for ${service}`);
+ } catch (error) {
+ logWarn(`Failed to capture logs for ${service}: ${error}`);
+ }
+ }
+}
+
+async function startContainers(composeDir: string, dataDir: string) {
+ const port = await getRandomPort();
+ const skipBuild =
+ process.env.SEED_SNAPSHOT_NO_BUILD === "1" ||
+ process.env.SEED_SNAPSHOT_SKIP_BUILD === "1";
+ const buildArg = skipBuild ? "" : "--build";
+ const composeEnv = {
+ ...process.env,
+ KARAKEEP_PORT: String(port),
+ SEED_SNAPSHOT_DATA_DIR: dataDir,
+ };
+
+ logStep(`Starting docker compose on port ${port}`);
+ execSync(`docker compose up ${buildArg} -d`, {
+ cwd: composeDir,
+ stdio: "inherit",
+ env: composeEnv,
+ });
+
+ process.env.KARAKEEP_PORT = String(port);
+
+ try {
+ logInfo("Waiting for Karakeep to report healthy...");
+ await waitForHealthy(port);
+ await sleep(5_000);
+ logSuccess("Containers are ready");
+ } catch (error) {
+ logWarn("Karakeep did not become healthy; stopping docker compose");
+ await captureDockerLogs(composeDir, composeEnv);
+ execSync("docker compose down", {
+ cwd: composeDir,
+ env: composeEnv,
+ stdio: "inherit",
+ });
+ throw error;
+ }
+
+ let stopped = false;
+ return {
+ port,
+ async stop(): Promise<void> {
+ if (stopped) return;
+ stopped = true;
+ logStep("Collecting docker logs");
+ await captureDockerLogs(composeDir, composeEnv);
+ logStep("Stopping docker compose");
+ execSync("docker compose down", {
+ cwd: composeDir,
+ env: composeEnv,
+ stdio: "inherit",
+ });
+ },
+ };
+}
+
+async function createUser(
+ authlessClient: TrpcClient,
+ email: string,
+ name: string,
+): Promise<TrpcClient> {
+ logInfo(`Creating ${email}`);
+ await authlessClient.users.create.mutate({
+ name,
+ email,
+ password: PASSWORD,
+ confirmPassword: PASSWORD,
+ });
+
+ const keyName = `seed-snapshot-${email}`;
+ const { key } = await authlessClient.apiKeys.exchange.mutate({
+ email,
+ password: PASSWORD,
+ keyName,
+ });
+
+ return getTrpcClient(key);
+}
+
+async function seedTags(
+ client: TrpcClient,
+ tagNames: readonly string[],
+): Promise<ZTagBasic[]> {
+ const tags: ZTagBasic[] = [];
+ for (const name of tagNames) {
+ tags.push(await client.tags.create.mutate({ name }));
+ }
+ return tags;
+}
+
+async function createLists(
+ client: TrpcClient,
+ dataset: UserDataset,
+): Promise<{
+ manualLists: ZBookmarkList[];
+ smartLists: ZBookmarkList[];
+}> {
+ const root = await client.lists.create.mutate({
+ name: dataset.lists.root,
+ description: "Seed snapshot root manual list",
+ icon: "book",
+ type: "manual",
+ });
+ const child = await client.lists.create.mutate({
+ name: dataset.lists.child,
+ description: "Seed snapshot nested manual list",
+ icon: "folder",
+ type: "manual",
+ parentId: root.id,
+ });
+ const smartTag = await client.lists.create.mutate({
+ name: dataset.lists.smartTag,
+ description: "Seed snapshot smart list by tag",
+ icon: "tag",
+ type: "smart",
+ query: `tag:${dataset.tagNames[0]}`,
+ });
+ const smartFav = await client.lists.create.mutate({
+ name: dataset.lists.smartFav,
+ description: "Seed snapshot smart list by state",
+ icon: "star",
+ type: "smart",
+ query: dataset.lists.smartFav.includes("Archived")
+ ? "is:archived"
+ : "is:fav",
+ parentId: root.id,
+ });
+
+ return {
+ manualLists: [root, child],
+ smartLists: [smartTag, smartFav],
+ };
+}
+
+async function createBookmarks(
+ client: TrpcClient,
+ dataset: UserDataset,
+ tags: ZTagBasic[],
+ lists: ZBookmarkList[],
+): Promise<SeededBookmark[]> {
+ const bookmarks: SeededBookmark[] = [];
+ for (const [index, url] of dataset.urls.entries()) {
+ const bookmark = await client.bookmarks.createBookmark.mutate({
+ type: BookmarkTypes.LINK,
+ url,
+ title: `Seed snapshot bookmark ${index + 1}`,
+ source: "api",
+ summary: `Seeded bookmark for ${url}`,
+ favourited: index % 2 === 0,
+ archived: index === dataset.urls.length - 1,
+ createdAt: new Date(Date.UTC(2025, 0, index + 1, 12, 0, 0)),
+ });
+
+ const attachedTags = [
+ tags[index % tags.length],
+ tags[(index + 1) % tags.length],
+ ];
+ await client.bookmarks.updateTags.mutate({
+ bookmarkId: bookmark.id,
+ attach: attachedTags.map((tag) => ({
+ tagId: tag.id,
+ tagName: tag.name,
+ })),
+ detach: [],
+ });
+
+ await client.lists.addToList.mutate({
+ listId: lists[index % lists.length].id,
+ bookmarkId: bookmark.id,
+ });
+
+ bookmarks.push({
+ id: bookmark.id,
+ url,
+ title: bookmark.title ?? `Seed snapshot bookmark ${index + 1}`,
+ crawlStatus: null,
+ });
+ }
+ return bookmarks;
+}
+
+function getTerminalCrawlStatus(bookmark: ZBookmark): CrawlStatus | null {
+ if (bookmark.content.type !== BookmarkTypes.LINK) {
+ return null;
+ }
+ const status = bookmark.content.crawlStatus ?? null;
+ return status === "success" || status === "failure" ? status : null;
+}
+
+async function waitForCrawls(
+ client: TrpcClient,
+ bookmarks: SeededBookmark[],
+): Promise<void> {
+ const pending = new Set(bookmarks.map((bookmark) => bookmark.id));
+ await waitUntil(
+ async () => {
+ for (const bookmark of bookmarks) {
+ if (!pending.has(bookmark.id)) continue;
+
+ const latest = await client.bookmarks.getBookmark.query({
+ bookmarkId: bookmark.id,
+ });
+ const terminalStatus = getTerminalCrawlStatus(latest);
+ if (terminalStatus) {
+ bookmark.crawlStatus = terminalStatus;
+ pending.delete(bookmark.id);
+ logInfo(`${bookmark.url} crawled with status ${terminalStatus}`);
+ }
+ }
+ return pending.size === 0;
+ },
+ "Seed bookmark crawls to reach terminal status",
+ Number(process.env.SEED_SNAPSHOT_CRAWL_TIMEOUT_MS ?? 180_000),
+ 3_000,
+ );
+}
+
+async function seedUser(
+ client: TrpcClient,
+ user: (typeof USERS)[number],
+ dataset: UserDataset,
+): Promise<SeededUserManifest> {
+ logStep(`Seeding ${user.email}`);
+ const tags = await seedTags(client, dataset.tagNames);
+ const { manualLists, smartLists } = await createLists(client, dataset);
+ const bookmarks = await createBookmarks(client, dataset, tags, manualLists);
+ await waitForCrawls(client, bookmarks);
+
+ return {
+ email: user.email,
+ password: PASSWORD,
+ seeded: true,
+ apiKeyName: `seed-snapshot-${user.email}`,
+ counts: {
+ bookmarks: bookmarks.length,
+ tags: tags.length,
+ manualLists: manualLists.length,
+ smartLists: smartLists.length,
+ },
+ bookmarks,
+ tags: tags.map((tag) => tag.name),
+ lists: [...manualLists, ...smartLists].map((list) => ({
+ name: list.name,
+ type: list.type,
+ parentName:
+ list.parentId === manualLists[0]?.id ? manualLists[0].name : null,
+ query: list.query ?? null,
+ })),
+ };
+}
+
+async function createUsersAndSeedData(): Promise<SeededUserManifest[]> {
+ const authlessClient = getTrpcClient();
+ const manifests: SeededUserManifest[] = [];
+
+ logStep("Creating seed users");
+ for (const user of USERS) {
+ const client = await createUser(authlessClient, user.email, user.name);
+ if (user.datasetKey) {
+ manifests.push(
+ await seedUser(client, user, USER_DATASETS[user.datasetKey]),
+ );
+ } else {
+ manifests.push({
+ email: user.email,
+ password: PASSWORD,
+ seeded: false,
+ apiKeyName: `seed-snapshot-${user.email}`,
+ counts: {
+ bookmarks: 0,
+ tags: 0,
+ manualLists: 0,
+ smartLists: 0,
+ },
+ bookmarks: [],
+ tags: [],
+ lists: [],
+ });
+ logSuccess(`${user.email} left empty`);
+ }
+ }
+
+ return manifests;
+}
+
+function formatSnapshotTimestamp(date: Date): string {
+ const pad = (num: number) => String(num).padStart(2, "0");
+ return [
+ date.getUTCFullYear(),
+ "-",
+ pad(date.getUTCMonth() + 1),
+ "-",
+ pad(date.getUTCDate()),
+ "-",
+ pad(date.getUTCHours()),
+ pad(date.getUTCMinutes()),
+ pad(date.getUTCSeconds()),
+ ].join("");
+}
+
+function getSourceCommit(repoRoot: string): string | null {
+ try {
+ return execSync("git rev-parse HEAD", {
+ cwd: repoRoot,
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "ignore"],
+ }).trim();
+ } catch {
+ return null;
+ }
+}
+
+async function writeSnapshot(params: {
+ dataDir: string;
+ repoRoot: string;
+ users: SeededUserManifest[];
+}): Promise<void> {
+ const snapshotsDir = path.join(params.repoRoot, "snapshots");
+ await fs.mkdir(snapshotsDir, { recursive: true });
+
+ const createdAt = new Date();
+ const timestamp = formatSnapshotTimestamp(createdAt);
+ const archiveName = `seed-data-${timestamp}.tar.gz`;
+ const manifestName = `seed-data-${timestamp}.json`;
+ const archivePath = path.join(snapshotsDir, archiveName);
+ const manifestPath = path.join(snapshotsDir, manifestName);
+
+ logStep("Archiving data directory");
+ execFileSync(
+ "tar",
+ ["--exclude", "._*", "-czf", archivePath, "-C", params.dataDir, "."],
+ {
+ cwd: params.repoRoot,
+ env: {
+ ...process.env,
+ COPYFILE_DISABLE: "1",
+ },
+ stdio: "inherit",
+ },
+ );
+
+ const manifest: SnapshotManifest = {
+ createdAt: createdAt.toISOString(),
+ sourceCommit: getSourceCommit(params.repoRoot),
+ archive: archiveName,
+ users: params.users,
+ };
+ await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+
+ logSuccess(`Wrote ${archivePath}`);
+ logSuccess(`Wrote ${manifestPath}`);
+}
+
+async function main(): Promise<void> {
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ const packageRoot = path.join(__dirname, "..");
+ const repoRoot = path.join(packageRoot, "..", "..");
+ const composeDir = packageRoot;
+ const dataDir = path.join(packageRoot, ".tmp", "data");
+
+ await fs.rm(dataDir, { force: true, recursive: true });
+ await fs.mkdir(dataDir, { recursive: true });
+ await fs.chmod(dataDir, 0o777);
+
+ const running = await startContainers(composeDir, dataDir);
+
+ const stopContainers = async () => {
+ await running.stop();
+ };
+
+ const handleSignal = async (signal: NodeJS.Signals) => {
+ logWarn(`Received ${signal}, shutting down...`);
+ await stopContainers();
+ process.exit(1);
+ };
+
+ process.on("SIGINT", handleSignal);
+ process.on("SIGTERM", handleSignal);
+
+ try {
+ const users = await createUsersAndSeedData();
+ await stopContainers();
+ await writeSnapshot({ dataDir, repoRoot, users });
+ logSuccess("Seed snapshot complete");
+ } catch (error) {
+ logWarn("Seed snapshot failed");
+ console.error(error);
+ await stopContainers();
+ process.exitCode = 1;
+ }
+}
+
+main();
diff --git a/tools/seed-snapshot/tsconfig.json b/tools/seed-snapshot/tsconfig.json
new file mode 100644
index 00000000..ae9547cb
--- /dev/null
+++ b/tools/seed-snapshot/tsconfig.json
@@ -0,0 +1,9 @@
+{
+ "$schema": "https://json.schemastore.org/tsconfig",
+ "extends": "@karakeep/tsconfig/node.json",
+ "include": ["src/**/*.ts"],
+ "exclude": ["node_modules"],
+ "compilerOptions": {
+ "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+ }
+}