Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

refactor: move assetdb into a plugin

Mohamed Bassemc9ab52c43e4f
diff --git a/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts b/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts
index eed746c9..efbd645f 100644
--- a/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts
+++ b/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts
@@ -4,13 +4,13 @@ import type { ZAdminMaintenanceMigrateLargeLinkHtmlTask } from "@karakeep/shared
import type { DequeuedJob } from "@karakeep/shared/queueing";
import { db } from "@karakeep/db";
import { AssetTypes, bookmarkLinks, bookmarks } from "@karakeep/db/schema";
-import { QuotaService } from "@karakeep/shared-server";
import {
ASSET_TYPES,
deleteAsset,
newAssetId,
+ QuotaService,
saveAsset,
-} from "@karakeep/shared/assetdb";
+} from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { tryCatch } from "@karakeep/shared/tryCatch";
diff --git a/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts b/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts
index 792f9417..c6d9829f 100644
--- a/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts
+++ b/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts
@@ -3,11 +3,12 @@ import { eq } from "drizzle-orm";
import { db } from "@karakeep/db";
import { assets } from "@karakeep/db/schema";
import {
+ deleteAsset,
+ getAllAssets,
ZAdminMaintenanceTidyAssetsTask,
ZTidyAssetsRequest,
zTidyAssetsRequestSchema,
} from "@karakeep/shared-server";
-import { deleteAsset, getAllAssets } from "@karakeep/shared/assetdb";
import logger from "@karakeep/shared/logger";
import { DequeuedJob } from "@karakeep/shared/queueing";
diff --git a/apps/workers/workers/assetPreprocessingWorker.ts b/apps/workers/workers/assetPreprocessingWorker.ts
index 9e291653..83ae403b 100644
--- a/apps/workers/workers/assetPreprocessingWorker.ts
+++ b/apps/workers/workers/assetPreprocessingWorker.ts
@@ -18,12 +18,14 @@ import {
addLogFields,
AssetPreprocessingQueue,
EmbeddingsQueue,
+ newAssetId,
OpenAIQueue,
QuotaService,
+ readAsset,
+ saveAsset,
StorageQuotaError,
triggerSearchReindex,
} from "@karakeep/shared-server";
-import { newAssetId, readAsset, saveAsset } from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import { InferenceClientFactory } from "@karakeep/shared/inference";
import logger from "@karakeep/shared/logger";
diff --git a/apps/workers/workers/backupWorker.ts b/apps/workers/workers/backupWorker.ts
index a4358c84..e9ee5778 100644
--- a/apps/workers/workers/backupWorker.ts
+++ b/apps/workers/workers/backupWorker.ts
@@ -22,8 +22,8 @@ import {
addLogFields,
BackupQueue,
QuotaService,
+ saveAssetFromFile,
} from "@karakeep/shared-server";
-import { saveAssetFromFile } from "@karakeep/shared/assetdb";
import {
toExportFormat,
toExportListFormat,
diff --git a/apps/workers/workers/crawler/assetStorage.ts b/apps/workers/workers/crawler/assetStorage.ts
index 2067fae9..a52e3aab 100644
--- a/apps/workers/workers/crawler/assetStorage.ts
+++ b/apps/workers/workers/crawler/assetStorage.ts
@@ -15,15 +15,17 @@ import { fetchWithProxy, getBookmarkDomain } from "network";
import type { RunProxyConfig } from "network";
import { db } from "@karakeep/db";
-import { getTracer, QuotaService, withSpan } from "@karakeep/shared-server";
import {
ASSET_TYPES,
getAssetSize,
+ getTracer,
IMAGE_ASSET_TYPES,
newAssetId,
+ QuotaService,
saveAsset,
saveAssetFromFile,
-} from "@karakeep/shared/assetdb";
+ withSpan,
+} from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { tryCatch } from "@karakeep/shared/tryCatch";
diff --git a/apps/workers/workers/crawler/crawlAndParse.ts b/apps/workers/workers/crawler/crawlAndParse.ts
index b239acd9..a157182f 100644
--- a/apps/workers/workers/crawler/crawlAndParse.ts
+++ b/apps/workers/workers/crawler/crawlAndParse.ts
@@ -21,16 +21,14 @@ import {
} from "@karakeep/db/schema";
import {
addLogFields,
+ ASSET_TYPES,
AssetPreprocessingQueue,
getTracer,
+ readAsset,
setSpanAttributes,
+ silentDeleteAsset,
withSpan,
} from "@karakeep/shared-server";
-import {
- ASSET_TYPES,
- readAsset,
- silentDeleteAsset,
-} from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
diff --git a/apps/workers/workers/crawler/probe.ts b/apps/workers/workers/crawler/probe.ts
index 8d43468d..aa114f53 100644
--- a/apps/workers/workers/crawler/probe.ts
+++ b/apps/workers/workers/crawler/probe.ts
@@ -11,11 +11,11 @@ import { db } from "@karakeep/db";
import { bookmarkLinks } from "@karakeep/db/schema";
import {
addLogFields,
+ ASSET_TYPES,
getTracer,
setSpanAttributes,
withSpan,
} from "@karakeep/shared-server";
-import { ASSET_TYPES } from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { tryCatch } from "@karakeep/shared/tryCatch";
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 5a3792a5..b19c13b4 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -14,19 +14,17 @@ import { db } from "@karakeep/db";
import { bookmarkLinks, bookmarks } from "@karakeep/db/schema";
import {
addLogFields,
+ ASSET_TYPES,
EmbeddingsQueue,
getTracer,
+ IMAGE_ASSET_TYPES,
OpenAIQueue,
+ SUPPORTED_UPLOAD_ASSET_TYPES,
triggerSearchReindex,
VideoWorkerQueue,
withSpan,
zCrawlLinkRequestSchema,
} from "@karakeep/shared-server";
-import {
- ASSET_TYPES,
- IMAGE_ASSET_TYPES,
- SUPPORTED_UPLOAD_ASSET_TYPES,
-} from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import {
diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.ts
index 051e7de8..7b049b68 100644
--- a/apps/workers/workers/inference/tagging.ts
+++ b/apps/workers/workers/inference/tagging.ts
@@ -20,10 +20,11 @@ import {
} from "@karakeep/db/schema";
import {
addLogFields,
+ ASSET_TYPES,
+ readAsset,
setSpanAttributes,
triggerSearchReindex,
} from "@karakeep/shared-server";
-import { ASSET_TYPES, readAsset } from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { buildImagePrompt } from "@karakeep/shared/prompts";
diff --git a/apps/workers/workers/videoWorker.ts b/apps/workers/workers/videoWorker.ts
index d2ed04cc..32723b88 100644
--- a/apps/workers/workers/videoWorker.ts
+++ b/apps/workers/workers/videoWorker.ts
@@ -14,18 +14,16 @@ import { db } from "@karakeep/db";
import { AssetTypes } from "@karakeep/db/schema";
import {
addLogFields,
+ ASSET_TYPES,
+ newAssetId,
QuotaService,
+ saveAssetFromFile,
+ silentDeleteAsset,
StorageQuotaError,
VideoWorkerQueue,
ZVideoRequest,
zvideoRequestSchema,
} from "@karakeep/shared-server";
-import {
- ASSET_TYPES,
- newAssetId,
- saveAssetFromFile,
- silentDeleteAsset,
-} from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";
diff --git a/docs/docs/08-development/02-directories.md b/docs/docs/08-development/02-directories.md
index 712e86a1..9ae5eb9d 100644
--- a/docs/docs/08-development/02-directories.md
+++ b/docs/docs/08-development/02-directories.md
@@ -2,21 +2,23 @@
## Apps
-| Directory | Description |
-| ------------------------ | ------------------------------------------------------ |
-| `apps/web` | The main web app |
-| `apps/workers` | The background workers logic |
-| `apps/mobile` | The react native based mobile app |
-| `apps/browser-extension` | The browser extension |
+| Directory | Description |
+| ------------------------ | -------------------------------------------------------- |
+| `apps/web` | The main web app |
+| `apps/workers` | The background workers logic |
+| `apps/mobile` | The react native based mobile app |
+| `apps/browser-extension` | The browser extension |
| `apps/landing` | The landing page of [karakeep.app](https://karakeep.app) |
## Shared Packages
-| Directory | Description |
-| ----------------- | ---------------------------------------------------------------------------- |
-| `packages/db` | The database schema and migrations |
-| `packages/trpc` | Where most of the business logic lies built as TRPC routes |
-| `packages/shared` | Some shared code between the different apps (e.g. loggers, configs, assetdb) |
+| Directory | Description |
+| ------------------------ | -------------------------------------------------------------------------- |
+| `packages/db` | The database schema and migrations |
+| `packages/trpc` | Where most of the business logic lies built as TRPC routes |
+| `packages/shared` | Shared code between the different apps, such as loggers and configs |
+| `packages/shared-server` | Shared server-only services, such as queues and asset storage |
+| `packages/plugins` | Implementations of pluggable services, including filesystem and S3 storage |
## Toolings
diff --git a/packages/api/utils/assets.ts b/packages/api/utils/assets.ts
index eb81bc2f..844ff2fe 100644
--- a/packages/api/utils/assets.ts
+++ b/packages/api/utils/assets.ts
@@ -5,7 +5,7 @@ import {
createAssetReadStream,
getAssetSize,
readAssetMetadata,
-} from "@karakeep/shared/assetdb";
+} from "@karakeep/shared-server";
import { toWebReadableStream } from "./upload";
diff --git a/packages/api/utils/upload.ts b/packages/api/utils/upload.ts
index a7ebe92c..6f139927 100644
--- a/packages/api/utils/upload.ts
+++ b/packages/api/utils/upload.ts
@@ -6,12 +6,13 @@ import { pipeline } from "stream/promises";
import { fileTypeFromBlob, supportedMimeTypes } from "file-type";
import { assets, AssetTypes } from "@karakeep/db/schema";
-import { QuotaService, StorageQuotaError } from "@karakeep/shared-server";
import {
newAssetId,
+ QuotaService,
saveAssetFromFile,
+ StorageQuotaError,
SUPPORTED_UPLOAD_ASSET_TYPES,
-} from "@karakeep/shared/assetdb";
+} from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
import { AuthedContext } from "@karakeep/trpc";
diff --git a/packages/e2e_tests/package.json b/packages/e2e_tests/package.json
index c0a577b2..d58de306 100644
--- a/packages/e2e_tests/package.json
+++ b/packages/e2e_tests/package.json
@@ -16,6 +16,7 @@
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1014.0",
+ "@karakeep/plugins": "workspace:^0.1.0",
"@karakeep/sdk": "workspace:*",
"@karakeep/shared": "workspace:^0.1.0",
"@karakeep/trpc": "workspace:^0.1.0",
diff --git a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts b/packages/e2e_tests/tests/assetdb/assetdb-utils.ts
index a8e29ab4..74f16a87 100644
--- a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts
+++ b/packages/e2e_tests/tests/assetdb/assetdb-utils.ts
@@ -9,13 +9,10 @@ import {
S3Client,
} from "@aws-sdk/client-s3";
-import {
- ASSET_TYPES,
- AssetMetadata,
- AssetStore,
- LocalFileSystemAssetStore,
- S3AssetStore,
-} from "@karakeep/shared/assetdb";
+import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";
+import { ASSET_TYPES } from "@karakeep/shared/assetdb";
+import { LocalFileSystemAssetStore } from "@karakeep/plugins/assetstore-filesystem/implementation";
+import { S3AssetStore } from "@karakeep/plugins/assetstore-s3/implementation";
export interface TestAssetData {
userId: string;
diff --git a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts b/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts
index 66d227d5..34729af0 100644
--- a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts
+++ b/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts
@@ -1,7 +1,8 @@
import * as fs from "fs";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { ASSET_TYPES, AssetStore } from "@karakeep/shared/assetdb";
+import type { AssetStore } from "@karakeep/shared/assetdb";
+import { ASSET_TYPES } from "@karakeep/shared/assetdb";
import {
assertAssetExists,
diff --git a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts
index 36ff837f..2421926a 100644
--- a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts
+++ b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts
@@ -2,7 +2,7 @@ import * as fs from "fs";
import * as path from "path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { LocalFileSystemAssetStore } from "@karakeep/shared/assetdb";
+import { LocalFileSystemAssetStore } from "@karakeep/plugins/assetstore-filesystem/implementation";
import {
assertAssetNotExists,
diff --git a/packages/e2e_tests/tests/assetdb/s3-store.test.ts b/packages/e2e_tests/tests/assetdb/s3-store.test.ts
index c573750e..32998db8 100644
--- a/packages/e2e_tests/tests/assetdb/s3-store.test.ts
+++ b/packages/e2e_tests/tests/assetdb/s3-store.test.ts
@@ -1,7 +1,7 @@
import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
-import { S3AssetStore } from "@karakeep/shared/assetdb";
+import { S3AssetStore } from "@karakeep/plugins/assetstore-s3/implementation";
import {
assertAssetExists,
diff --git a/packages/plugins/assetstore-filesystem/index.ts b/packages/plugins/assetstore-filesystem/index.ts
new file mode 100644
index 00000000..c09f7af3
--- /dev/null
+++ b/packages/plugins/assetstore-filesystem/index.ts
@@ -0,0 +1,24 @@
+import type { AssetStore } from "@karakeep/shared/assetdb";
+import serverConfig from "@karakeep/shared/config";
+import { PluginManager, PluginType } from "@karakeep/shared/plugins";
+import type { PluginProvider } from "@karakeep/shared/plugins";
+
+export class FileSystemAssetStoreProvider implements PluginProvider<AssetStore> {
+ private client: AssetStore | null = null;
+
+ async getClient(): Promise<AssetStore> {
+ if (!this.client) {
+ const { LocalFileSystemAssetStore } = await import("./src");
+ if (!this.client) {
+ this.client = new LocalFileSystemAssetStore(serverConfig.assetsDir);
+ }
+ }
+ return this.client;
+ }
+}
+
+PluginManager.register({
+ type: PluginType.AssetStore,
+ name: "Filesystem",
+ provider: new FileSystemAssetStoreProvider(),
+});
diff --git a/packages/plugins/assetstore-filesystem/src/index.ts b/packages/plugins/assetstore-filesystem/src/index.ts
new file mode 100644
index 00000000..a07fc52e
--- /dev/null
+++ b/packages/plugins/assetstore-filesystem/src/index.ts
@@ -0,0 +1,206 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { Glob } from "glob";
+
+import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";
+import {
+ SUPPORTED_ASSET_TYPES,
+ zAssetMetadataSchema,
+} from "@karakeep/shared/assetdb";
+
+export class LocalFileSystemAssetStore implements AssetStore {
+ constructor(private readonly rootPath: string) {}
+
+ private getAssetDir(userId: string, assetId: string) {
+ return path.join(this.rootPath, userId, assetId);
+ }
+
+ private async isPathExists(filePath: string) {
+ return fs.promises
+ .access(filePath)
+ .then(() => true)
+ .catch(() => false);
+ }
+
+ async saveAsset({
+ userId,
+ assetId,
+ asset,
+ metadata,
+ }: {
+ userId: string;
+ assetId: string;
+ asset: Buffer;
+ metadata: AssetMetadata;
+ }) {
+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
+ throw new Error("Unsupported asset type");
+ }
+ const assetDir = this.getAssetDir(userId, assetId);
+ await fs.promises.mkdir(assetDir, { recursive: true });
+
+ await Promise.all([
+ fs.promises.writeFile(
+ path.join(assetDir, "asset.bin"),
+ Uint8Array.from(asset),
+ ),
+ fs.promises.writeFile(
+ path.join(assetDir, "metadata.json"),
+ JSON.stringify(metadata),
+ ),
+ ]);
+ }
+
+ async saveAssetFromFile({
+ userId,
+ assetId,
+ assetPath,
+ metadata,
+ }: {
+ userId: string;
+ assetId: string;
+ assetPath: string;
+ metadata: AssetMetadata;
+ }) {
+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
+ throw new Error("Unsupported asset type");
+ }
+ const assetDir = this.getAssetDir(userId, assetId);
+ await fs.promises.mkdir(assetDir, { recursive: true });
+
+ await Promise.all([
+ fs.promises.copyFile(assetPath, path.join(assetDir, "asset.bin")),
+ fs.promises.writeFile(
+ path.join(assetDir, "metadata.json"),
+ JSON.stringify(metadata),
+ ),
+ ]);
+ await fs.promises.rm(assetPath);
+ }
+
+ async readAsset({
+ userId,
+ assetId,
+ start,
+ end,
+ }: {
+ userId: string;
+ assetId: string;
+ start?: number;
+ end?: number;
+ }) {
+ const assetDir = this.getAssetDir(userId, assetId);
+
+ const readAssetFile =
+ start !== undefined || end !== undefined
+ ? (async () => {
+ const fd = await fs.promises.open(
+ path.join(assetDir, "asset.bin"),
+ "r",
+ );
+ try {
+ const stat = await fd.stat();
+ const offset = start ?? 0;
+ const effectiveEnd = Math.min(
+ end !== undefined ? end + 1 : stat.size,
+ stat.size,
+ );
+ const length = effectiveEnd - offset;
+ const buffer = Buffer.alloc(length);
+ const { bytesRead } = await fd.read(buffer, 0, length, offset);
+ return bytesRead < length
+ ? buffer.subarray(0, bytesRead)
+ : buffer;
+ } finally {
+ await fd.close();
+ }
+ })()
+ : fs.promises.readFile(path.join(assetDir, "asset.bin"));
+
+ const [asset, metadataStr] = await Promise.all([
+ readAssetFile,
+ fs.promises.readFile(path.join(assetDir, "metadata.json"), {
+ encoding: "utf8",
+ }),
+ ]);
+
+ const metadata = zAssetMetadataSchema.parse(JSON.parse(metadataStr));
+ return { asset, metadata };
+ }
+
+ async createAssetReadStream({
+ userId,
+ assetId,
+ start,
+ end,
+ }: {
+ userId: string;
+ assetId: string;
+ start?: number;
+ end?: number;
+ }) {
+ const assetDir = this.getAssetDir(userId, assetId);
+ const assetPath = path.join(assetDir, "asset.bin");
+ if (!(await this.isPathExists(assetPath))) {
+ throw new Error(`Asset ${assetId} not found`);
+ }
+
+ return fs.createReadStream(assetPath, { start, end });
+ }
+
+ async readAssetMetadata({
+ userId,
+ assetId,
+ }: {
+ userId: string;
+ assetId: string;
+ }) {
+ const assetDir = this.getAssetDir(userId, assetId);
+
+ const metadataStr = await fs.promises.readFile(
+ path.join(assetDir, "metadata.json"),
+ { encoding: "utf8" },
+ );
+
+ return zAssetMetadataSchema.parse(JSON.parse(metadataStr));
+ }
+
+ async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {
+ const assetDir = this.getAssetDir(userId, assetId);
+ const stat = await fs.promises.stat(path.join(assetDir, "asset.bin"));
+ return stat.size;
+ }
+
+ async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {
+ const assetDir = this.getAssetDir(userId, assetId);
+ if (!(await this.isPathExists(assetDir))) {
+ return;
+ }
+ await fs.promises.rm(assetDir, { recursive: true });
+ }
+
+ async deleteUserAssets({ userId }: { userId: string }) {
+ const userDir = path.join(this.rootPath, userId);
+ if (!(await this.isPathExists(userDir))) {
+ return;
+ }
+ await fs.promises.rm(userDir, { recursive: true });
+ }
+
+ async *getAllAssets() {
+ const assets = new Glob("/**/**/asset.bin", {
+ maxDepth: 3,
+ root: this.rootPath,
+ cwd: this.rootPath,
+ absolute: false,
+ });
+ for await (const file of assets) {
+ const [userId, assetId] = file.split("/").slice(0, 2);
+ const [size, metadata] = await Promise.all([
+ this.getAssetSize({ userId, assetId }),
+ this.readAssetMetadata({ userId, assetId }),
+ ]);
+ yield { userId, assetId, ...metadata, size };
+ }
+ }
+}
diff --git a/packages/plugins/assetstore-filesystem/src/tests/provider.test.ts b/packages/plugins/assetstore-filesystem/src/tests/provider.test.ts
new file mode 100644
index 00000000..c8c4caa6
--- /dev/null
+++ b/packages/plugins/assetstore-filesystem/src/tests/provider.test.ts
@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vitest";
+
+import { FileSystemAssetStoreProvider } from "../../index";
+
+describe("FileSystemAssetStoreProvider", () => {
+ it("returns one client across concurrent calls", async () => {
+ const provider = new FileSystemAssetStoreProvider();
+
+ const [first, second] = await Promise.all([
+ provider.getClient(),
+ provider.getClient(),
+ ]);
+
+ expect(first).toBe(second);
+ });
+});
diff --git a/packages/plugins/assetstore-s3/index.ts b/packages/plugins/assetstore-s3/index.ts
new file mode 100644
index 00000000..87972071
--- /dev/null
+++ b/packages/plugins/assetstore-s3/index.ts
@@ -0,0 +1,68 @@
+import type { AssetStore } from "@karakeep/shared/assetdb";
+import serverConfig from "@karakeep/shared/config";
+import { PluginManager, PluginType } from "@karakeep/shared/plugins";
+import type { PluginProvider } from "@karakeep/shared/plugins";
+
+interface S3AssetStoreOptions {
+ region?: string;
+ endpoint: string;
+ forcePathStyle: boolean;
+ bucket: string;
+ accessKeyId: string;
+ secretAccessKey: string;
+}
+
+export class S3AssetStoreProvider implements PluginProvider<AssetStore> {
+ private client: AssetStore | null = null;
+
+ constructor(private readonly options: S3AssetStoreOptions) {}
+
+ async getClient(): Promise<AssetStore> {
+ if (!this.client) {
+ const [{ S3Client }, { S3AssetStore }] = await Promise.all([
+ import("@aws-sdk/client-s3"),
+ import("./src"),
+ ]);
+ if (!this.client) {
+ const s3Client = new S3Client({
+ region: this.options.region,
+ endpoint: this.options.endpoint,
+ forcePathStyle: this.options.forcePathStyle,
+ credentials: {
+ accessKeyId: this.options.accessKeyId,
+ secretAccessKey: this.options.secretAccessKey,
+ },
+ });
+ this.client = new S3AssetStore(s3Client, this.options.bucket);
+ }
+ }
+ return this.client;
+ }
+}
+
+const config = serverConfig.assetStore;
+if (config.type === "s3") {
+ if (!config.s3.bucket) {
+ throw new Error(
+ "ASSET_STORE_S3_BUCKET is required when using S3 asset store",
+ );
+ }
+ if (!config.s3.accessKeyId || !config.s3.secretAccessKey) {
+ throw new Error(
+ "ASSET_STORE_S3_ACCESS_KEY_ID and ASSET_STORE_S3_SECRET_ACCESS_KEY are required when using S3 asset store",
+ );
+ }
+
+ PluginManager.register({
+ type: PluginType.AssetStore,
+ name: "S3",
+ provider: new S3AssetStoreProvider({
+ region: config.s3.region,
+ endpoint: config.s3.endpoint!,
+ forcePathStyle: config.s3.forcePathStyle,
+ bucket: config.s3.bucket,
+ accessKeyId: config.s3.accessKeyId,
+ secretAccessKey: config.s3.secretAccessKey,
+ }),
+ });
+}
diff --git a/packages/plugins/assetstore-s3/src/index.ts b/packages/plugins/assetstore-s3/src/index.ts
new file mode 100644
index 00000000..92dcbc22
--- /dev/null
+++ b/packages/plugins/assetstore-s3/src/index.ts
@@ -0,0 +1,284 @@
+import * as fs from "node:fs";
+import { Readable } from "node:stream";
+import {
+ DeleteObjectCommand,
+ DeleteObjectsCommand,
+ GetObjectCommand,
+ HeadObjectCommand,
+ ListObjectsV2Command,
+ PutObjectCommand,
+ S3Client,
+} from "@aws-sdk/client-s3";
+
+import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";
+import { SUPPORTED_ASSET_TYPES } from "@karakeep/shared/assetdb";
+import logger from "@karakeep/shared/logger";
+
+export class S3AssetStore implements AssetStore {
+ constructor(
+ private readonly s3Client: S3Client,
+ private readonly bucketName: string,
+ ) {}
+
+ private getAssetKey(userId: string, assetId: string) {
+ return `${userId}/${assetId}`;
+ }
+
+ private metadataToS3Metadata(
+ metadata: AssetMetadata,
+ ): Record<string, string> {
+ return {
+ ...(metadata.fileName
+ ? { "x-amz-meta-file-name": metadata.fileName }
+ : {}),
+ "x-amz-meta-content-type": metadata.contentType,
+ };
+ }
+
+ private s3MetadataToMetadata(
+ s3Metadata: Record<string, string> | undefined,
+ ): AssetMetadata {
+ if (!s3Metadata) {
+ throw new Error("No metadata found in S3 object");
+ }
+
+ return {
+ contentType: s3Metadata["x-amz-meta-content-type"] || "",
+ fileName: s3Metadata["x-amz-meta-file-name"] ?? null,
+ };
+ }
+
+ async saveAsset({
+ userId,
+ assetId,
+ asset,
+ metadata,
+ }: {
+ userId: string;
+ assetId: string;
+ asset: Buffer;
+ metadata: AssetMetadata;
+ }) {
+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
+ throw new Error("Unsupported asset type");
+ }
+
+ await this.s3Client.send(
+ new PutObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ Body: asset,
+ ContentType: metadata.contentType,
+ Metadata: this.metadataToS3Metadata(metadata),
+ }),
+ );
+ }
+
+ async saveAssetFromFile({
+ userId,
+ assetId,
+ assetPath,
+ metadata,
+ }: {
+ userId: string;
+ assetId: string;
+ assetPath: string;
+ metadata: AssetMetadata;
+ }) {
+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
+ throw new Error("Unsupported asset type");
+ }
+
+ const asset = fs.createReadStream(assetPath);
+ await this.s3Client.send(
+ new PutObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ Body: asset,
+ ContentType: metadata.contentType,
+ Metadata: this.metadataToS3Metadata(metadata),
+ }),
+ );
+ await fs.promises.rm(assetPath);
+ }
+
+ async readAsset({
+ userId,
+ assetId,
+ start,
+ end,
+ }: {
+ userId: string;
+ assetId: string;
+ start?: number;
+ end?: number;
+ }) {
+ const range =
+ start !== undefined || end !== undefined
+ ? `bytes=${start ?? 0}-${end ?? ""}`
+ : undefined;
+
+ const response = await this.s3Client.send(
+ new GetObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ Range: range,
+ }),
+ );
+
+ if (!response.Body) {
+ throw new Error("Asset not found");
+ }
+
+ const assetBuffer = await this.streamToBuffer(response.Body as Readable);
+ const metadata = this.s3MetadataToMetadata(response.Metadata);
+
+ return { asset: assetBuffer, metadata };
+ }
+
+ async createAssetReadStream({
+ userId,
+ assetId,
+ start,
+ end,
+ }: {
+ userId: string;
+ assetId: string;
+ start?: number;
+ end?: number;
+ }) {
+ const range =
+ start !== undefined || end !== undefined
+ ? `bytes=${start ?? 0}-${end ?? ""}`
+ : undefined;
+
+ const response = await this.s3Client.send(
+ new GetObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ Range: range,
+ }),
+ );
+ if (!response.Body) {
+ throw new Error("Asset not found");
+ }
+ return response.Body as NodeJS.ReadableStream;
+ }
+
+ async readAssetMetadata({
+ userId,
+ assetId,
+ }: {
+ userId: string;
+ assetId: string;
+ }) {
+ const response = await this.s3Client.send(
+ new HeadObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ }),
+ );
+
+ return this.s3MetadataToMetadata(response.Metadata);
+ }
+
+ async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {
+ const response = await this.s3Client.send(
+ new HeadObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ }),
+ );
+
+ return response.ContentLength || 0;
+ }
+
+ async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {
+ await this.s3Client.send(
+ new DeleteObjectCommand({
+ Bucket: this.bucketName,
+ Key: this.getAssetKey(userId, assetId),
+ }),
+ );
+ }
+
+ async deleteUserAssets({ userId }: { userId: string }) {
+ let continuationToken: string | undefined;
+
+ do {
+ const listResponse = await this.s3Client.send(
+ new ListObjectsV2Command({
+ Bucket: this.bucketName,
+ Prefix: `${userId}/`,
+ ContinuationToken: continuationToken,
+ }),
+ );
+
+ if (listResponse.Contents && listResponse.Contents.length > 0) {
+ await this.s3Client.send(
+ new DeleteObjectsCommand({
+ Bucket: this.bucketName,
+ Delete: {
+ Objects: listResponse.Contents.map((object) => ({
+ Key: object.Key!,
+ })),
+ },
+ }),
+ );
+ }
+
+ continuationToken = listResponse.NextContinuationToken;
+ } while (continuationToken);
+ }
+
+ async *getAllAssets() {
+ let continuationToken: string | undefined;
+
+ do {
+ const listResponse = await this.s3Client.send(
+ new ListObjectsV2Command({
+ Bucket: this.bucketName,
+ ContinuationToken: continuationToken,
+ }),
+ );
+
+ if (listResponse.Contents) {
+ for (const object of listResponse.Contents) {
+ if (!object.Key) continue;
+
+ const pathParts = object.Key.split("/");
+ if (pathParts.length === 2) {
+ const userId = pathParts[0];
+ const assetId = pathParts[1];
+
+ try {
+ const headResponse = await this.s3Client.send(
+ new HeadObjectCommand({
+ Bucket: this.bucketName,
+ Key: object.Key,
+ }),
+ );
+
+ const metadata = this.s3MetadataToMetadata(headResponse.Metadata);
+ const size = headResponse.ContentLength || 0;
+
+ yield { userId, assetId, ...metadata, size };
+ } catch (error) {
+ logger.warn(`Failed to read asset ${userId}/${assetId}:`, error);
+ }
+ }
+ }
+ }
+
+ continuationToken = listResponse.NextContinuationToken;
+ } while (continuationToken);
+ }
+
+ private async streamToBuffer(stream: Readable): Promise<Buffer> {
+ const chunks: Buffer[] = [];
+ for await (const chunk of stream) {
+ chunks.push(chunk);
+ }
+ return Buffer.concat(chunks);
+ }
+}
diff --git a/packages/plugins/assetstore-s3/src/tests/provider.test.ts b/packages/plugins/assetstore-s3/src/tests/provider.test.ts
new file mode 100644
index 00000000..66489d7f
--- /dev/null
+++ b/packages/plugins/assetstore-s3/src/tests/provider.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+
+import { S3AssetStoreProvider } from "../../index";
+
+describe("S3AssetStoreProvider", () => {
+ it("returns one client across concurrent calls", async () => {
+ const provider = new S3AssetStoreProvider({
+ region: "us-east-1",
+ endpoint: "http://localhost:9000",
+ forcePathStyle: true,
+ bucket: "test",
+ accessKeyId: "test",
+ secretAccessKey: "test",
+ });
+
+ const [first, second] = await Promise.all([
+ provider.getClient(),
+ provider.getClient(),
+ ]);
+
+ expect(first).toBe(second);
+ });
+});
diff --git a/packages/plugins/package.json b/packages/plugins/package.json
index 69321979..c39f864e 100644
--- a/packages/plugins/package.json
+++ b/packages/plugins/package.json
@@ -5,6 +5,10 @@
"private": true,
"type": "module",
"exports": {
+ "./assetstore-filesystem": "./assetstore-filesystem/index.ts",
+ "./assetstore-filesystem/implementation": "./assetstore-filesystem/src/index.ts",
+ "./assetstore-s3": "./assetstore-s3/index.ts",
+ "./assetstore-s3/implementation": "./assetstore-s3/src/index.ts",
"./queue-liteque": "./queue-liteque/index.ts",
"./queue-restate": "./queue-restate/index.ts",
"./ratelimit-memory": "./ratelimit-memory/index.ts",
@@ -21,10 +25,12 @@
"test": "vitest"
},
"dependencies": {
+ "@aws-sdk/client-s3": "^3.1014.0",
"@karakeep/shared": "workspace:*",
"@restatedev/restate-sdk": "^1.16.2",
"@restatedev/restate-sdk-clients": "^1.16.2",
"async-mutex": "^0.4.1",
+ "glob": "^11.0.0",
"liteque": "^0.9.0",
"meilisearch": "^0.58.0",
"redis": "^5.11.0"
diff --git a/packages/shared-server/src/assetdb.ts b/packages/shared-server/src/assetdb.ts
new file mode 100644
index 00000000..1e33266a
--- /dev/null
+++ b/packages/shared-server/src/assetdb.ts
@@ -0,0 +1,160 @@
+import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";
+import { PluginManager, PluginType } from "@karakeep/shared/plugins";
+import type { QuotaApproved } from "@karakeep/shared/storageQuota";
+
+import { loadAllPlugins } from "./plugins";
+
+export {
+ ASSET_TYPES,
+ IMAGE_ASSET_TYPES,
+ SUPPORTED_ASSET_TYPES,
+ SUPPORTED_BOOKMARK_ASSET_TYPES,
+ SUPPORTED_UPLOAD_ASSET_TYPES,
+ VIDEO_ASSET_TYPES,
+ zAssetMetadataSchema,
+} from "@karakeep/shared/assetdb";
+export type {
+ AssetInfo,
+ AssetMetadata,
+ AssetStore,
+} from "@karakeep/shared/assetdb";
+
+async function getAssetStore(): Promise<AssetStore> {
+ await loadAllPlugins();
+ const store = await PluginManager.getClient(PluginType.AssetStore);
+ if (!store) {
+ throw new Error("No asset store plugin is registered");
+ }
+ return store;
+}
+
+export function newAssetId() {
+ return crypto.randomUUID();
+}
+
+export async function saveAsset({
+ userId,
+ assetId,
+ asset,
+ metadata,
+ quotaApproved,
+}: {
+ userId: string;
+ assetId: string;
+ asset: Buffer;
+ metadata: AssetMetadata;
+ quotaApproved: QuotaApproved;
+}) {
+ if (quotaApproved.userId !== userId) {
+ throw new Error("Quota approval is for a different user");
+ }
+ if (quotaApproved.approvedSize < asset.byteLength) {
+ throw new Error("Asset size exceeds approved quota");
+ }
+
+ const store = await getAssetStore();
+ return store.saveAsset({ userId, assetId, asset, metadata });
+}
+
+export async function saveAssetFromFile({
+ userId,
+ assetId,
+ assetPath,
+ metadata,
+ quotaApproved,
+}: {
+ userId: string;
+ assetId: string;
+ assetPath: string;
+ metadata: AssetMetadata;
+ quotaApproved: QuotaApproved;
+}) {
+ if (quotaApproved.userId !== userId) {
+ throw new Error("Quota approval is for a different user");
+ }
+
+ const store = await getAssetStore();
+ return store.saveAssetFromFile({ userId, assetId, assetPath, metadata });
+}
+
+export async function readAsset({
+ userId,
+ assetId,
+ start,
+ end,
+}: {
+ userId: string;
+ assetId: string;
+ start?: number;
+ end?: number;
+}) {
+ const store = await getAssetStore();
+ return store.readAsset({ userId, assetId, start, end });
+}
+
+export async function createAssetReadStream({
+ userId,
+ assetId,
+ start,
+ end,
+}: {
+ userId: string;
+ assetId: string;
+ start?: number;
+ end?: number;
+}) {
+ const store = await getAssetStore();
+ return store.createAssetReadStream({ userId, assetId, start, end });
+}
+
+export async function readAssetMetadata({
+ userId,
+ assetId,
+}: {
+ userId: string;
+ assetId: string;
+}) {
+ const store = await getAssetStore();
+ return store.readAssetMetadata({ userId, assetId });
+}
+
+export async function getAssetSize({
+ userId,
+ assetId,
+}: {
+ userId: string;
+ assetId: string;
+}) {
+ const store = await getAssetStore();
+ return store.getAssetSize({ userId, assetId });
+}
+
+export async function silentDeleteAsset(
+ userId: string,
+ assetId: string | undefined,
+) {
+ if (assetId) {
+ await deleteAsset({ userId, assetId }).catch(() => ({}));
+ }
+}
+
+export async function deleteAsset({
+ userId,
+ assetId,
+}: {
+ userId: string;
+ assetId: string;
+}) {
+ const store = await getAssetStore();
+ return store.deleteAsset({ userId, assetId });
+}
+
+export async function deleteUserAssets({ userId }: { userId: string }) {
+ const store = await getAssetStore();
+ return store.deleteUserAssets({ userId });
+}
+
+export async function* getAllAssets() {
+ const store = await getAssetStore();
+ yield* store.getAllAssets();
+}
diff --git a/packages/shared-server/src/index.ts b/packages/shared-server/src/index.ts
index 580d45ed..d5c9ab17 100644
--- a/packages/shared-server/src/index.ts
+++ b/packages/shared-server/src/index.ts
@@ -1,4 +1,5 @@
export { loadAllPlugins } from "./plugins";
+export * from "./assetdb";
export { QuotaService, StorageQuotaError } from "./services/quotaService";
export * from "./queues";
export * from "./eventLogger";
diff --git a/packages/shared-server/src/plugins.ts b/packages/shared-server/src/plugins.ts
index dc9fa547..057ad16c 100644
--- a/packages/shared-server/src/plugins.ts
+++ b/packages/shared-server/src/plugins.ts
@@ -23,6 +23,9 @@ export async function loadAllPlugins() {
}
pluginLoaderState.loading = (async () => {
// Load plugins here. Order of plugin loading matter.
+ // Asset store providers (order matters - last one wins)
+ await import("@karakeep/plugins/assetstore-filesystem");
+ await import("@karakeep/plugins/assetstore-s3");
// Queue provider(s)
await import("@karakeep/plugins/queue-liteque");
await import("@karakeep/plugins/queue-restate");
diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts
index 8f98d0fd..d1b0f8a6 100644
--- a/packages/shared/assetdb.ts
+++ b/packages/shared/assetdb.ts
@@ -1,25 +1,5 @@
-import * as fs from "fs";
-import * as path from "path";
-import { Readable } from "stream";
-import {
- _Object,
- DeleteObjectCommand,
- DeleteObjectsCommand,
- GetObjectCommand,
- HeadObjectCommand,
- ListObjectsV2Command,
- PutObjectCommand,
- S3Client,
-} from "@aws-sdk/client-s3";
-import { Glob } from "glob";
import { z } from "zod";
-import serverConfig from "./config";
-import logger from "./logger";
-import { QuotaApproved } from "./storageQuota";
-
-const ROOT_PATH = serverConfig.assetsDir;
-
export const enum ASSET_TYPES {
IMAGE_GIF = "image/gif",
IMAGE_JPEG = "image/jpeg",
@@ -126,694 +106,3 @@ export interface AssetStore {
getAllAssets(): AsyncGenerator<AssetInfo>;
}
-
-export function newAssetId() {
- return crypto.randomUUID();
-}
-
-class LocalFileSystemAssetStore implements AssetStore {
- private rootPath: string;
-
- constructor(rootPath: string) {
- this.rootPath = rootPath;
- }
-
- private getAssetDir(userId: string, assetId: string) {
- return path.join(this.rootPath, userId, assetId);
- }
-
- private async isPathExists(filePath: string) {
- return fs.promises
- .access(filePath)
- .then(() => true)
- .catch(() => false);
- }
-
- async saveAsset({
- userId,
- assetId,
- asset,
- metadata,
- }: {
- userId: string;
- assetId: string;
- asset: Buffer;
- metadata: AssetMetadata;
- }) {
- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
- throw new Error("Unsupported asset type");
- }
- const assetDir = this.getAssetDir(userId, assetId);
- await fs.promises.mkdir(assetDir, { recursive: true });
-
- await Promise.all([
- fs.promises.writeFile(
- path.join(assetDir, "asset.bin"),
- Uint8Array.from(asset),
- ),
- fs.promises.writeFile(
- path.join(assetDir, "metadata.json"),
- JSON.stringify(metadata),
- ),
- ]);
- }
-
- async saveAssetFromFile({
- userId,
- assetId,
- assetPath,
- metadata,
- }: {
- userId: string;
- assetId: string;
- assetPath: string;
- metadata: AssetMetadata;
- }) {
- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
- throw new Error("Unsupported asset type");
- }
- const assetDir = this.getAssetDir(userId, assetId);
- await fs.promises.mkdir(assetDir, { recursive: true });
-
- await Promise.all([
- fs.promises.copyFile(assetPath, path.join(assetDir, "asset.bin")),
- fs.promises.writeFile(
- path.join(assetDir, "metadata.json"),
- JSON.stringify(metadata),
- ),
- ]);
- await fs.promises.rm(assetPath);
- }
-
- async readAsset({
- userId,
- assetId,
- start,
- end,
- }: {
- userId: string;
- assetId: string;
- start?: number;
- end?: number;
- }) {
- const assetDir = this.getAssetDir(userId, assetId);
-
- const readAssetFile =
- start !== undefined || end !== undefined
- ? (async () => {
- const fd = await fs.promises.open(
- path.join(assetDir, "asset.bin"),
- "r",
- );
- try {
- const stat = await fd.stat();
- const offset = start ?? 0;
- const effectiveEnd = Math.min(
- end !== undefined ? end + 1 : stat.size,
- stat.size,
- );
- const length = effectiveEnd - offset;
- const buffer = Buffer.alloc(length);
- const { bytesRead } = await fd.read(buffer, 0, length, offset);
- return bytesRead < length
- ? buffer.subarray(0, bytesRead)
- : buffer;
- } finally {
- await fd.close();
- }
- })()
- : fs.promises.readFile(path.join(assetDir, "asset.bin"));
-
- const [asset, metadataStr] = await Promise.all([
- readAssetFile,
- fs.promises.readFile(path.join(assetDir, "metadata.json"), {
- encoding: "utf8",
- }),
- ]);
-
- const metadata = zAssetMetadataSchema.parse(JSON.parse(metadataStr));
- return { asset, metadata };
- }
-
- async createAssetReadStream({
- userId,
- assetId,
- start,
- end,
- }: {
- userId: string;
- assetId: string;
- start?: number;
- end?: number;
- }) {
- const assetDir = this.getAssetDir(userId, assetId);
- const assetPath = path.join(assetDir, "asset.bin");
- if (!(await this.isPathExists(assetPath))) {
- throw new Error(`Asset ${assetId} not found`);
- }
-
- return fs.createReadStream(path.join(assetDir, "asset.bin"), {
- start,
- end,
- });
- }
-
- async readAssetMetadata({
- userId,
- assetId,
- }: {
- userId: string;
- assetId: string;
- }) {
- const assetDir = this.getAssetDir(userId, assetId);
-
- const metadataStr = await fs.promises.readFile(
- path.join(assetDir, "metadata.json"),
- {
- encoding: "utf8",
- },
- );
-
- return zAssetMetadataSchema.parse(JSON.parse(metadataStr));
- }
-
- async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {
- const assetDir = this.getAssetDir(userId, assetId);
- const stat = await fs.promises.stat(path.join(assetDir, "asset.bin"));
- return stat.size;
- }
-
- async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {
- const assetDir = this.getAssetDir(userId, assetId);
- if (!(await this.isPathExists(assetDir))) {
- return;
- }
- await fs.promises.rm(assetDir, { recursive: true });
- }
-
- async deleteUserAssets({ userId }: { userId: string }) {
- const userDir = path.join(this.rootPath, userId);
- const dirExists = await this.isPathExists(userDir);
- if (!dirExists) {
- return;
- }
- await fs.promises.rm(userDir, { recursive: true });
- }
-
- async *getAllAssets() {
- const g = new Glob(`/**/**/asset.bin`, {
- maxDepth: 3,
- root: this.rootPath,
- cwd: this.rootPath,
- absolute: false,
- });
- for await (const file of g) {
- const [userId, assetId] = file.split("/").slice(0, 2);
- const [size, metadata] = await Promise.all([
- this.getAssetSize({ userId, assetId }),
- this.readAssetMetadata({ userId, assetId }),
- ]);
- yield {
- userId,
- assetId,
- ...metadata,
- size,
- };
- }
- }
-}
-
-class S3AssetStore implements AssetStore {
- private s3Client: S3Client;
- private bucketName: string;
-
- constructor(s3Client: S3Client, bucketName: string) {
- this.s3Client = s3Client;
- this.bucketName = bucketName;
- }
-
- private getAssetKey(userId: string, assetId: string) {
- return `${userId}/${assetId}`;
- }
-
- private metadataToS3Metadata(
- metadata: AssetMetadata,
- ): Record<string, string> {
- return {
- ...(metadata.fileName
- ? { "x-amz-meta-file-name": metadata.fileName }
- : {}),
- "x-amz-meta-content-type": metadata.contentType,
- };
- }
-
- private s3MetadataToMetadata(
- s3Metadata: Record<string, string> | undefined,
- ): AssetMetadata {
- if (!s3Metadata) {
- throw new Error("No metadata found in S3 object");
- }
-
- return {
- contentType: s3Metadata["x-amz-meta-content-type"] || "",
- fileName: s3Metadata["x-amz-meta-file-name"] ?? null,
- };
- }
-
- async saveAsset({
- userId,
- assetId,
- asset,
- metadata,
- }: {
- userId: string;
- assetId: string;
- asset: Buffer;
- metadata: AssetMetadata;
- }) {
- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
- throw new Error("Unsupported asset type");
- }
-
- await this.s3Client.send(
- new PutObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- Body: asset,
- ContentType: metadata.contentType,
- Metadata: this.metadataToS3Metadata(metadata),
- }),
- );
- }
-
- async saveAssetFromFile({
- userId,
- assetId,
- assetPath,
- metadata,
- }: {
- userId: string;
- assetId: string;
- assetPath: string;
- metadata: AssetMetadata;
- }) {
- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
- throw new Error("Unsupported asset type");
- }
-
- const asset = fs.createReadStream(assetPath);
- await this.s3Client.send(
- new PutObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- Body: asset,
- ContentType: metadata.contentType,
- Metadata: this.metadataToS3Metadata(metadata),
- }),
- );
- await fs.promises.rm(assetPath);
- }
-
- async readAsset({
- userId,
- assetId,
- start,
- end,
- }: {
- userId: string;
- assetId: string;
- start?: number;
- end?: number;
- }) {
- const range =
- start !== undefined || end !== undefined
- ? `bytes=${start ?? 0}-${end ?? ""}`
- : undefined;
-
- const response = await this.s3Client.send(
- new GetObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- Range: range,
- }),
- );
-
- if (!response.Body) {
- throw new Error("Asset not found");
- }
-
- const assetBuffer = await this.streamToBuffer(response.Body as Readable);
- const metadata = this.s3MetadataToMetadata(response.Metadata);
-
- return { asset: assetBuffer, metadata };
- }
-
- async createAssetReadStream({
- userId,
- assetId,
- start,
- end,
- }: {
- userId: string;
- assetId: string;
- start?: number;
- end?: number;
- }) {
- const range =
- start !== undefined || end !== undefined
- ? `bytes=${start ?? 0}-${end ?? ""}`
- : undefined;
-
- const command = new GetObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- Range: range,
- });
-
- const response = await this.s3Client.send(command);
- if (!response.Body) {
- throw new Error("Asset not found");
- }
- return response.Body as NodeJS.ReadableStream;
- }
-
- async readAssetMetadata({
- userId,
- assetId,
- }: {
- userId: string;
- assetId: string;
- }) {
- const response = await this.s3Client.send(
- new HeadObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- }),
- );
-
- return this.s3MetadataToMetadata(response.Metadata);
- }
-
- async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {
- const response = await this.s3Client.send(
- new HeadObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- }),
- );
-
- return response.ContentLength || 0;
- }
-
- async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {
- await this.s3Client.send(
- new DeleteObjectCommand({
- Bucket: this.bucketName,
- Key: this.getAssetKey(userId, assetId),
- }),
- );
- }
-
- async deleteUserAssets({ userId }: { userId: string }) {
- let continuationToken: string | undefined;
-
- do {
- const listResponse = await this.s3Client.send(
- new ListObjectsV2Command({
- Bucket: this.bucketName,
- Prefix: `${userId}/`,
- ContinuationToken: continuationToken,
- }),
- );
-
- if (listResponse.Contents && listResponse.Contents.length > 0) {
- await this.s3Client.send(
- new DeleteObjectsCommand({
- Bucket: this.bucketName,
- Delete: {
- Objects: listResponse.Contents.map((obj) => ({
- Key: obj.Key!,
- })),
- },
- }),
- );
- }
-
- continuationToken = listResponse.NextContinuationToken;
- } while (continuationToken);
- }
-
- async *getAllAssets() {
- let continuationToken: string | undefined;
-
- do {
- const listResponse = await this.s3Client.send(
- new ListObjectsV2Command({
- Bucket: this.bucketName,
- ContinuationToken: continuationToken,
- }),
- );
-
- if (listResponse.Contents) {
- for (const obj of listResponse.Contents) {
- if (!obj.Key) continue;
-
- const pathParts = obj.Key.split("/");
- if (pathParts.length === 2) {
- const userId = pathParts[0];
- const assetId = pathParts[1];
-
- try {
- const headResponse = await this.s3Client.send(
- new HeadObjectCommand({
- Bucket: this.bucketName,
- Key: obj.Key,
- }),
- );
-
- const metadata = this.s3MetadataToMetadata(headResponse.Metadata);
- const size = headResponse.ContentLength || 0;
-
- yield {
- userId,
- assetId,
- ...metadata,
- size,
- };
- } catch (error) {
- logger.warn(`Failed to read asset ${userId}/${assetId}:`, error);
- }
- }
- }
- }
-
- continuationToken = listResponse.NextContinuationToken;
- } while (continuationToken);
- }
-
- private async streamToBuffer(stream: Readable): Promise<Buffer> {
- const chunks: Buffer[] = [];
- for await (const chunk of stream) {
- chunks.push(chunk);
- }
- return Buffer.concat(chunks);
- }
-}
-
-function createDefaultAssetStore(): AssetStore {
- const config = serverConfig.assetStore;
-
- if (config.type === "s3") {
- if (!config.s3.bucket) {
- throw new Error(
- "ASSET_STORE_S3_BUCKET is required when using S3 asset store",
- );
- }
- if (!config.s3.accessKeyId || !config.s3.secretAccessKey) {
- throw new Error(
- "ASSET_STORE_S3_ACCESS_KEY_ID and ASSET_STORE_S3_SECRET_ACCESS_KEY are required when using S3 asset store",
- );
- }
-
- const s3Client = new S3Client({
- region: config.s3.region,
- endpoint: config.s3.endpoint,
- forcePathStyle: config.s3.forcePathStyle,
- credentials: {
- accessKeyId: config.s3.accessKeyId,
- secretAccessKey: config.s3.secretAccessKey,
- },
- });
-
- return new S3AssetStore(s3Client, config.s3.bucket);
- }
-
- return new LocalFileSystemAssetStore(ROOT_PATH);
-}
-
-const defaultAssetStore = createDefaultAssetStore();
-
-export { LocalFileSystemAssetStore, S3AssetStore };
-
-/**
- * Example usage of S3AssetStore:
- *
- * import { S3Client } from "@aws-sdk/client-s3";
- * import { S3AssetStore } from "@karakeep/shared/assetdb";
- *
- * const s3Client = new S3Client({
- * region: "us-east-1",
- * credentials: {
- * accessKeyId: "your-access-key",
- * secretAccessKey: "your-secret-key"
- * }
- * });
- *
- * const s3AssetStore = new S3AssetStore(s3Client, "your-bucket-name");
- *
- * // Use s3AssetStore instead of the default file system store
- * await s3AssetStore.saveAsset({
- * userId: "user123",
- * assetId: "asset456",
- * asset: buffer,
- * metadata: { contentType: "image/jpeg", fileName: "photo.jpg" }
- * });
- */
-
-export async function saveAsset({
- userId,
- assetId,
- asset,
- metadata,
- quotaApproved,
-}: {
- userId: string;
- assetId: string;
- asset: Buffer;
- metadata: z.infer<typeof zAssetMetadataSchema>;
- quotaApproved: QuotaApproved;
-}) {
- // Verify the quota approval is for the correct user and size
- if (quotaApproved.userId !== userId) {
- throw new Error("Quota approval is for a different user");
- }
- if (quotaApproved.approvedSize < asset.byteLength) {
- throw new Error("Asset size exceeds approved quota");
- }
-
- return defaultAssetStore.saveAsset({ userId, assetId, asset, metadata });
-}
-
-export async function saveAssetFromFile({
- userId,
- assetId,
- assetPath,
- metadata,
- quotaApproved,
-}: {
- userId: string;
- assetId: string;
- assetPath: string;
- metadata: z.infer<typeof zAssetMetadataSchema>;
- quotaApproved: QuotaApproved;
-}) {
- // Verify the quota approval is for the correct user
- if (quotaApproved.userId !== userId) {
- throw new Error("Quota approval is for a different user");
- }
-
- // For file-based saves, we'll verify the file size matches the approved size
- // when the underlying store implementation reads the file
-
- return defaultAssetStore.saveAssetFromFile({
- userId,
- assetId,
- assetPath,
- metadata,
- });
-}
-
-export async function readAsset({
- userId,
- assetId,
- start,
- end,
-}: {
- userId: string;
- assetId: string;
- start?: number;
- end?: number;
-}) {
- return defaultAssetStore.readAsset({ userId, assetId, start, end });
-}
-
-export async function createAssetReadStream({
- userId,
- assetId,
- start,
- end,
-}: {
- userId: string;
- assetId: string;
- start?: number;
- end?: number;
-}) {
- return defaultAssetStore.createAssetReadStream({
- userId,
- assetId,
- start,
- end,
- });
-}
-
-export async function readAssetMetadata({
- userId,
- assetId,
-}: {
- userId: string;
- assetId: string;
-}) {
- return defaultAssetStore.readAssetMetadata({ userId, assetId });
-}
-
-export async function getAssetSize({
- userId,
- assetId,
-}: {
- userId: string;
- assetId: string;
-}) {
- return defaultAssetStore.getAssetSize({ userId, assetId });
-}
-
-/**
- * Deletes the passed in asset if it exists and ignores any errors
- * @param userId the id of the user the asset belongs to
- * @param assetId the id of the asset to delete
- */
-export async function silentDeleteAsset(
- userId: string,
- assetId: string | undefined,
-) {
- if (assetId) {
- await deleteAsset({ userId, assetId }).catch(() => ({}));
- }
-}
-
-export async function deleteAsset({
- userId,
- assetId,
-}: {
- userId: string;
- assetId: string;
-}) {
- return defaultAssetStore.deleteAsset({ userId, assetId });
-}
-
-export async function deleteUserAssets({ userId }: { userId: string }) {
- return defaultAssetStore.deleteUserAssets({ userId });
-}
-
-export async function* getAllAssets() {
- yield* defaultAssetStore.getAllAssets();
-}
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 9be91c3a..d9eceb33 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -5,9 +5,7 @@
"private": true,
"type": "module",
"dependencies": {
- "@aws-sdk/client-s3": "^3.1014.0",
"cheerio": "^1.2.0",
- "glob": "^11.0.0",
"html-to-text": "^9.0.5",
"js-tiktoken": "^1.0.20",
"nodemailer": "^7.0.13",
diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.ts
index 974b9215..efe8853f 100644
--- a/packages/shared/plugins.ts
+++ b/packages/shared/plugins.ts
@@ -1,5 +1,6 @@
// Implementation inspired from Outline
+import type { AssetStore } from "./assetdb";
import type { QueueClient } from "./queueing";
import type { RateLimitClient } from "./ratelimiting";
import type { VectorStoreClient } from "./vectorStore";
@@ -7,6 +8,7 @@ import logger from "./logger";
import { SearchIndexClient } from "./search";
export enum PluginType {
+ AssetStore = "assetstore",
Search = "search",
Queue = "queue",
RateLimit = "ratelimit",
@@ -14,6 +16,7 @@ export enum PluginType {
}
interface PluginTypeMap {
+ [PluginType.AssetStore]: AssetStore;
[PluginType.Search]: SearchIndexClient;
[PluginType.Queue]: QueueClient;
[PluginType.RateLimit]: RateLimitClient;
@@ -37,6 +40,7 @@ const pluginProvidersKey = "__karakeep_plugins_providers__";
function createProviderMap(): ProviderMap {
return {
+ [PluginType.AssetStore]: [],
[PluginType.Search]: [],
[PluginType.Queue]: [],
[PluginType.RateLimit]: [],
diff --git a/packages/trpc/models/assets.ts b/packages/trpc/models/assets.ts
index 44366a4f..8fb4842b 100644
--- a/packages/trpc/models/assets.ts
+++ b/packages/trpc/models/assets.ts
@@ -3,7 +3,7 @@ import { and, desc, eq, sql } from "drizzle-orm";
import { z } from "zod";
import { assets } from "@karakeep/db/schema";
-import { deleteAsset } from "@karakeep/shared/assetdb";
+import { deleteAsset } from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
import { createSignedToken } from "@karakeep/shared/signedTokens";
import { zAssetSignedTokenSchema } from "@karakeep/shared/types/assets";
diff --git a/packages/trpc/models/backups.ts b/packages/trpc/models/backups.ts
index 7f0d0598..82f7b91f 100644
--- a/packages/trpc/models/backups.ts
+++ b/packages/trpc/models/backups.ts
@@ -3,8 +3,7 @@ import { and, desc, eq, lt } from "drizzle-orm";
import { z } from "zod";
import { assets, backupsTable } from "@karakeep/db/schema";
-import { BackupQueue } from "@karakeep/shared-server";
-import { deleteAsset } from "@karakeep/shared/assetdb";
+import { BackupQueue, deleteAsset } from "@karakeep/shared-server";
import { zBackupSchema } from "@karakeep/shared/types/backups";
import { AuthedContext } from "..";
diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.ts
index dbf1ca63..0fe06904 100644
--- a/packages/trpc/models/bookmarks.ts
+++ b/packages/trpc/models/bookmarks.ts
@@ -31,10 +31,14 @@ import {
rssFeedImportsTable,
tagsOnBookmarks,
} from "@karakeep/db/schema";
-import { EmbeddingsQueue, SearchIndexingQueue } from "@karakeep/shared-server";
+import {
+ deleteAsset,
+ EmbeddingsQueue,
+ readAsset,
+ SearchIndexingQueue,
+} from "@karakeep/shared-server";
import { WebhooksService } from "./webhooks.service";
-import { deleteAsset, readAsset } from "@karakeep/shared/assetdb";
import { getAlignedExpiry } from "@karakeep/shared/signedTokens";
import {
BookmarkTypes,
diff --git a/packages/trpc/models/users.ts b/packages/trpc/models/users.ts
index 7cf23c97..a0d52010 100644
--- a/packages/trpc/models/users.ts
+++ b/packages/trpc/models/users.ts
@@ -19,7 +19,7 @@ import {
users,
verificationTokens,
} from "@karakeep/db/schema";
-import { deleteAsset, deleteUserAssets } from "@karakeep/shared/assetdb";
+import { deleteAsset, deleteUserAssets } from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
import {
zResetPasswordSchema,
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 20f3e3c7..7269560b 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -26,9 +26,9 @@ import {
OpenAIQueue,
QueuePriority,
QuotaService,
+ SUPPORTED_BOOKMARK_ASSET_TYPES,
triggerSearchReindex,
} from "@karakeep/shared-server";
-import { SUPPORTED_BOOKMARK_ASSET_TYPES } from "@karakeep/shared/assetdb";
import serverConfig from "@karakeep/shared/config";
import {
EmbeddingClientFactory,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 2c166e30..df2cbb98 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1277,6 +1277,9 @@ importers:
'@aws-sdk/client-s3':
specifier: ^3.1014.0
version: 3.1014.0
+ '@karakeep/plugins':
+ specifier: workspace:^0.1.0
+ version: link:../plugins
'@karakeep/sdk':
specifier: workspace:*
version: link:../sdk
@@ -1333,6 +1336,9 @@ importers:
packages/plugins:
dependencies:
+ '@aws-sdk/client-s3':
+ specifier: ^3.1014.0
+ version: 3.1014.0
'@karakeep/shared':
specifier: workspace:*
version: link:../shared
@@ -1345,6 +1351,9 @@ importers:
async-mutex:
specifier: ^0.4.1
version: 0.4.1
+ glob:
+ specifier: ^11.0.0
+ version: 11.0.2
liteque:
specifier: ^0.9.0
@@ -1392,15 +1401,9 @@ importers:
packages/shared:
dependencies:
- '@aws-sdk/client-s3':
- specifier: ^3.1014.0
- version: 3.1014.0
cheerio:
specifier: ^1.2.0
version: 1.2.0
- glob:
- specifier: ^11.0.0
- version: 11.0.2
html-to-text:
specifier: ^9.0.5
version: 9.0.5