A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
refactor: move assetdb into a plugin
Mohamed Bassemc9ab52c43e4f
diff --git a/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts b/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.tsindex eed746c9..efbd645f 100644--- a/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts+++ b/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts@@ -4,13 +4,13 @@ import type { ZAdminMaintenanceMigrateLargeLinkHtmlTask } from "@karakeep/sharedimport type { DequeuedJob } from "@karakeep/shared/queueing";import { db } from "@karakeep/db";import { AssetTypes, bookmarkLinks, bookmarks } from "@karakeep/db/schema";-import { QuotaService } from "@karakeep/shared-server";import {ASSET_TYPES,deleteAsset,newAssetId,+ QuotaService,saveAsset,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { tryCatch } from "@karakeep/shared/tryCatch";diff --git a/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts b/apps/workers/workers/adminMaintenance/tasks/tidyAssets.tsindex 792f9417..c6d9829f 100644--- a/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts+++ b/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts@@ -3,11 +3,12 @@ import { eq } from "drizzle-orm";import { db } from "@karakeep/db";import { assets } from "@karakeep/db/schema";import {+ deleteAsset,+ getAllAssets,ZAdminMaintenanceTidyAssetsTask,ZTidyAssetsRequest,zTidyAssetsRequestSchema,} from "@karakeep/shared-server";-import { deleteAsset, getAllAssets } from "@karakeep/shared/assetdb";import logger from "@karakeep/shared/logger";import { DequeuedJob } from "@karakeep/shared/queueing";diff --git a/apps/workers/workers/assetPreprocessingWorker.ts b/apps/workers/workers/assetPreprocessingWorker.tsindex 9e291653..83ae403b 100644--- a/apps/workers/workers/assetPreprocessingWorker.ts+++ b/apps/workers/workers/assetPreprocessingWorker.ts@@ -18,12 +18,14 @@ import {addLogFields,AssetPreprocessingQueue,EmbeddingsQueue,+ newAssetId,OpenAIQueue,QuotaService,+ readAsset,+ saveAsset,StorageQuotaError,triggerSearchReindex,} from "@karakeep/shared-server";-import { newAssetId, readAsset, saveAsset } from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import { InferenceClientFactory } from "@karakeep/shared/inference";import logger from "@karakeep/shared/logger";diff --git a/apps/workers/workers/backupWorker.ts b/apps/workers/workers/backupWorker.tsindex a4358c84..e9ee5778 100644--- a/apps/workers/workers/backupWorker.ts+++ b/apps/workers/workers/backupWorker.ts@@ -22,8 +22,8 @@ import {addLogFields,BackupQueue,QuotaService,+ saveAssetFromFile,} from "@karakeep/shared-server";-import { saveAssetFromFile } from "@karakeep/shared/assetdb";import {toExportFormat,toExportListFormat,diff --git a/apps/workers/workers/crawler/assetStorage.ts b/apps/workers/workers/crawler/assetStorage.tsindex 2067fae9..a52e3aab 100644--- a/apps/workers/workers/crawler/assetStorage.ts+++ b/apps/workers/workers/crawler/assetStorage.ts@@ -15,15 +15,17 @@ import { fetchWithProxy, getBookmarkDomain } from "network";import type { RunProxyConfig } from "network";import { db } from "@karakeep/db";-import { getTracer, QuotaService, withSpan } from "@karakeep/shared-server";import {ASSET_TYPES,getAssetSize,+ getTracer,IMAGE_ASSET_TYPES,newAssetId,+ QuotaService,saveAsset,saveAssetFromFile,-} from "@karakeep/shared/assetdb";+ withSpan,+} from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { tryCatch } from "@karakeep/shared/tryCatch";diff --git a/apps/workers/workers/crawler/crawlAndParse.ts b/apps/workers/workers/crawler/crawlAndParse.tsindex b239acd9..a157182f 100644--- a/apps/workers/workers/crawler/crawlAndParse.ts+++ b/apps/workers/workers/crawler/crawlAndParse.ts@@ -21,16 +21,14 @@ import {} from "@karakeep/db/schema";import {addLogFields,+ ASSET_TYPES,AssetPreprocessingQueue,getTracer,+ readAsset,setSpanAttributes,+ silentDeleteAsset,withSpan,} from "@karakeep/shared-server";-import {- ASSET_TYPES,- readAsset,- silentDeleteAsset,-} from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";diff --git a/apps/workers/workers/crawler/probe.ts b/apps/workers/workers/crawler/probe.tsindex 8d43468d..aa114f53 100644--- a/apps/workers/workers/crawler/probe.ts+++ b/apps/workers/workers/crawler/probe.ts@@ -11,11 +11,11 @@ import { db } from "@karakeep/db";import { bookmarkLinks } from "@karakeep/db/schema";import {addLogFields,+ ASSET_TYPES,getTracer,setSpanAttributes,withSpan,} from "@karakeep/shared-server";-import { ASSET_TYPES } from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { tryCatch } from "@karakeep/shared/tryCatch";diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 5a3792a5..b19c13b4 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -14,19 +14,17 @@ import { db } from "@karakeep/db";import { bookmarkLinks, bookmarks } from "@karakeep/db/schema";import {addLogFields,+ ASSET_TYPES,EmbeddingsQueue,getTracer,+ IMAGE_ASSET_TYPES,OpenAIQueue,+ SUPPORTED_UPLOAD_ASSET_TYPES,triggerSearchReindex,VideoWorkerQueue,withSpan,zCrawlLinkRequestSchema,} from "@karakeep/shared-server";-import {- ASSET_TYPES,- IMAGE_ASSET_TYPES,- SUPPORTED_UPLOAD_ASSET_TYPES,-} from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import {diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.tsindex 051e7de8..7b049b68 100644--- a/apps/workers/workers/inference/tagging.ts+++ b/apps/workers/workers/inference/tagging.ts@@ -20,10 +20,11 @@ import {} from "@karakeep/db/schema";import {addLogFields,+ ASSET_TYPES,+ readAsset,setSpanAttributes,triggerSearchReindex,} from "@karakeep/shared-server";-import { ASSET_TYPES, readAsset } from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { buildImagePrompt } from "@karakeep/shared/prompts";diff --git a/apps/workers/workers/videoWorker.ts b/apps/workers/workers/videoWorker.tsindex d2ed04cc..32723b88 100644--- a/apps/workers/workers/videoWorker.ts+++ b/apps/workers/workers/videoWorker.ts@@ -14,18 +14,16 @@ import { db } from "@karakeep/db";import { AssetTypes } from "@karakeep/db/schema";import {addLogFields,+ ASSET_TYPES,+ newAssetId,QuotaService,+ saveAssetFromFile,+ silentDeleteAsset,StorageQuotaError,VideoWorkerQueue,ZVideoRequest,zvideoRequestSchema,} from "@karakeep/shared-server";-import {- ASSET_TYPES,- newAssetId,- saveAssetFromFile,- silentDeleteAsset,-} from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";diff --git a/docs/docs/08-development/02-directories.md b/docs/docs/08-development/02-directories.mdindex 712e86a1..9ae5eb9d 100644--- a/docs/docs/08-development/02-directories.md+++ b/docs/docs/08-development/02-directories.md@@ -2,21 +2,23 @@## Apps-| Directory | Description |-| ------------------------ | ------------------------------------------------------ |-| `apps/web` | The main web app |-| `apps/workers` | The background workers logic |-| `apps/mobile` | The react native based mobile app |-| `apps/browser-extension` | The browser extension |+| Directory | Description |+| ------------------------ | -------------------------------------------------------- |+| `apps/web` | The main web app |+| `apps/workers` | The background workers logic |+| `apps/mobile` | The react native based mobile app |+| `apps/browser-extension` | The browser extension || `apps/landing` | The landing page of [karakeep.app](https://karakeep.app) |## Shared Packages-| Directory | Description |-| ----------------- | ---------------------------------------------------------------------------- |-| `packages/db` | The database schema and migrations |-| `packages/trpc` | Where most of the business logic lies built as TRPC routes |-| `packages/shared` | Some shared code between the different apps (e.g. loggers, configs, assetdb) |+| Directory | Description |+| ------------------------ | -------------------------------------------------------------------------- |+| `packages/db` | The database schema and migrations |+| `packages/trpc` | Where most of the business logic lies built as TRPC routes |+| `packages/shared` | Shared code between the different apps, such as loggers and configs |+| `packages/shared-server` | Shared server-only services, such as queues and asset storage |+| `packages/plugins` | Implementations of pluggable services, including filesystem and S3 storage |## Toolingsdiff --git a/packages/api/utils/assets.ts b/packages/api/utils/assets.tsindex eb81bc2f..844ff2fe 100644--- a/packages/api/utils/assets.ts+++ b/packages/api/utils/assets.ts@@ -5,7 +5,7 @@ import {createAssetReadStream,getAssetSize,readAssetMetadata,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server";import { toWebReadableStream } from "./upload";diff --git a/packages/api/utils/upload.ts b/packages/api/utils/upload.tsindex a7ebe92c..6f139927 100644--- a/packages/api/utils/upload.ts+++ b/packages/api/utils/upload.ts@@ -6,12 +6,13 @@ import { pipeline } from "stream/promises";import { fileTypeFromBlob, supportedMimeTypes } from "file-type";import { assets, AssetTypes } from "@karakeep/db/schema";-import { QuotaService, StorageQuotaError } from "@karakeep/shared-server";import {newAssetId,+ QuotaService,saveAssetFromFile,+ StorageQuotaError,SUPPORTED_UPLOAD_ASSET_TYPES,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import { AuthedContext } from "@karakeep/trpc";diff --git a/packages/e2e_tests/package.json b/packages/e2e_tests/package.jsonindex c0a577b2..d58de306 100644--- a/packages/e2e_tests/package.json+++ b/packages/e2e_tests/package.json@@ -16,6 +16,7 @@},"dependencies": {"@aws-sdk/client-s3": "^3.1014.0",+ "@karakeep/plugins": "workspace:^0.1.0","@karakeep/sdk": "workspace:*","@karakeep/shared": "workspace:^0.1.0","@karakeep/trpc": "workspace:^0.1.0",diff --git a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts b/packages/e2e_tests/tests/assetdb/assetdb-utils.tsindex a8e29ab4..74f16a87 100644--- a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts+++ b/packages/e2e_tests/tests/assetdb/assetdb-utils.ts@@ -9,13 +9,10 @@ import {S3Client,} from "@aws-sdk/client-s3";-import {- ASSET_TYPES,- AssetMetadata,- AssetStore,- LocalFileSystemAssetStore,- S3AssetStore,-} from "@karakeep/shared/assetdb";+import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";+import { ASSET_TYPES } from "@karakeep/shared/assetdb";+import { LocalFileSystemAssetStore } from "@karakeep/plugins/assetstore-filesystem/implementation";+import { S3AssetStore } from "@karakeep/plugins/assetstore-s3/implementation";export interface TestAssetData {userId: string;diff --git a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts b/packages/e2e_tests/tests/assetdb/interface-compliance.test.tsindex 66d227d5..34729af0 100644--- a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts+++ b/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts@@ -1,7 +1,8 @@import * as fs from "fs";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { ASSET_TYPES, AssetStore } from "@karakeep/shared/assetdb";+import type { AssetStore } from "@karakeep/shared/assetdb";+import { ASSET_TYPES } from "@karakeep/shared/assetdb";import {assertAssetExists,diff --git a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.tsindex 36ff837f..2421926a 100644--- a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts+++ b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts@@ -2,7 +2,7 @@ import * as fs from "fs";import * as path from "path";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { LocalFileSystemAssetStore } from "@karakeep/shared/assetdb";+import { LocalFileSystemAssetStore } from "@karakeep/plugins/assetstore-filesystem/implementation";import {assertAssetNotExists,diff --git a/packages/e2e_tests/tests/assetdb/s3-store.test.ts b/packages/e2e_tests/tests/assetdb/s3-store.test.tsindex c573750e..32998db8 100644--- a/packages/e2e_tests/tests/assetdb/s3-store.test.ts+++ b/packages/e2e_tests/tests/assetdb/s3-store.test.ts@@ -1,7 +1,7 @@import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { S3AssetStore } from "@karakeep/shared/assetdb";+import { S3AssetStore } from "@karakeep/plugins/assetstore-s3/implementation";import {assertAssetExists,diff --git a/packages/plugins/assetstore-filesystem/index.ts b/packages/plugins/assetstore-filesystem/index.tsnew file mode 100644index 00000000..c09f7af3--- /dev/null+++ b/packages/plugins/assetstore-filesystem/index.ts@@ -0,0 +1,24 @@+import type { AssetStore } from "@karakeep/shared/assetdb";+import serverConfig from "@karakeep/shared/config";+import { PluginManager, PluginType } from "@karakeep/shared/plugins";+import type { PluginProvider } from "@karakeep/shared/plugins";++export class FileSystemAssetStoreProvider implements PluginProvider<AssetStore> {+ private client: AssetStore | null = null;++ async getClient(): Promise<AssetStore> {+ if (!this.client) {+ const { LocalFileSystemAssetStore } = await import("./src");+ if (!this.client) {+ this.client = new LocalFileSystemAssetStore(serverConfig.assetsDir);+ }+ }+ return this.client;+ }+}++PluginManager.register({+ type: PluginType.AssetStore,+ name: "Filesystem",+ provider: new FileSystemAssetStoreProvider(),+});diff --git a/packages/plugins/assetstore-filesystem/src/index.ts b/packages/plugins/assetstore-filesystem/src/index.tsnew file mode 100644index 00000000..a07fc52e--- /dev/null+++ b/packages/plugins/assetstore-filesystem/src/index.ts@@ -0,0 +1,206 @@+import * as fs from "node:fs";+import * as path from "node:path";+import { Glob } from "glob";++import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";+import {+ SUPPORTED_ASSET_TYPES,+ zAssetMetadataSchema,+} from "@karakeep/shared/assetdb";++export class LocalFileSystemAssetStore implements AssetStore {+ constructor(private readonly rootPath: string) {}++ private getAssetDir(userId: string, assetId: string) {+ return path.join(this.rootPath, userId, assetId);+ }++ private async isPathExists(filePath: string) {+ return fs.promises+ .access(filePath)+ .then(() => true)+ .catch(() => false);+ }++ async saveAsset({+ userId,+ assetId,+ asset,+ metadata,+ }: {+ userId: string;+ assetId: string;+ asset: Buffer;+ metadata: AssetMetadata;+ }) {+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {+ throw new Error("Unsupported asset type");+ }+ const assetDir = this.getAssetDir(userId, assetId);+ await fs.promises.mkdir(assetDir, { recursive: true });++ await Promise.all([+ fs.promises.writeFile(+ path.join(assetDir, "asset.bin"),+ Uint8Array.from(asset),+ ),+ fs.promises.writeFile(+ path.join(assetDir, "metadata.json"),+ JSON.stringify(metadata),+ ),+ ]);+ }++ async saveAssetFromFile({+ userId,+ assetId,+ assetPath,+ metadata,+ }: {+ userId: string;+ assetId: string;+ assetPath: string;+ metadata: AssetMetadata;+ }) {+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {+ throw new Error("Unsupported asset type");+ }+ const assetDir = this.getAssetDir(userId, assetId);+ await fs.promises.mkdir(assetDir, { recursive: true });++ await Promise.all([+ fs.promises.copyFile(assetPath, path.join(assetDir, "asset.bin")),+ fs.promises.writeFile(+ path.join(assetDir, "metadata.json"),+ JSON.stringify(metadata),+ ),+ ]);+ await fs.promises.rm(assetPath);+ }++ async readAsset({+ userId,+ assetId,+ start,+ end,+ }: {+ userId: string;+ assetId: string;+ start?: number;+ end?: number;+ }) {+ const assetDir = this.getAssetDir(userId, assetId);++ const readAssetFile =+ start !== undefined || end !== undefined+ ? (async () => {+ const fd = await fs.promises.open(+ path.join(assetDir, "asset.bin"),+ "r",+ );+ try {+ const stat = await fd.stat();+ const offset = start ?? 0;+ const effectiveEnd = Math.min(+ end !== undefined ? end + 1 : stat.size,+ stat.size,+ );+ const length = effectiveEnd - offset;+ const buffer = Buffer.alloc(length);+ const { bytesRead } = await fd.read(buffer, 0, length, offset);+ return bytesRead < length+ ? buffer.subarray(0, bytesRead)+ : buffer;+ } finally {+ await fd.close();+ }+ })()+ : fs.promises.readFile(path.join(assetDir, "asset.bin"));++ const [asset, metadataStr] = await Promise.all([+ readAssetFile,+ fs.promises.readFile(path.join(assetDir, "metadata.json"), {+ encoding: "utf8",+ }),+ ]);++ const metadata = zAssetMetadataSchema.parse(JSON.parse(metadataStr));+ return { asset, metadata };+ }++ async createAssetReadStream({+ userId,+ assetId,+ start,+ end,+ }: {+ userId: string;+ assetId: string;+ start?: number;+ end?: number;+ }) {+ const assetDir = this.getAssetDir(userId, assetId);+ const assetPath = path.join(assetDir, "asset.bin");+ if (!(await this.isPathExists(assetPath))) {+ throw new Error(`Asset ${assetId} not found`);+ }++ return fs.createReadStream(assetPath, { start, end });+ }++ async readAssetMetadata({+ userId,+ assetId,+ }: {+ userId: string;+ assetId: string;+ }) {+ const assetDir = this.getAssetDir(userId, assetId);++ const metadataStr = await fs.promises.readFile(+ path.join(assetDir, "metadata.json"),+ { encoding: "utf8" },+ );++ return zAssetMetadataSchema.parse(JSON.parse(metadataStr));+ }++ async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {+ const assetDir = this.getAssetDir(userId, assetId);+ const stat = await fs.promises.stat(path.join(assetDir, "asset.bin"));+ return stat.size;+ }++ async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {+ const assetDir = this.getAssetDir(userId, assetId);+ if (!(await this.isPathExists(assetDir))) {+ return;+ }+ await fs.promises.rm(assetDir, { recursive: true });+ }++ async deleteUserAssets({ userId }: { userId: string }) {+ const userDir = path.join(this.rootPath, userId);+ if (!(await this.isPathExists(userDir))) {+ return;+ }+ await fs.promises.rm(userDir, { recursive: true });+ }++ async *getAllAssets() {+ const assets = new Glob("/**/**/asset.bin", {+ maxDepth: 3,+ root: this.rootPath,+ cwd: this.rootPath,+ absolute: false,+ });+ for await (const file of assets) {+ const [userId, assetId] = file.split("/").slice(0, 2);+ const [size, metadata] = await Promise.all([+ this.getAssetSize({ userId, assetId }),+ this.readAssetMetadata({ userId, assetId }),+ ]);+ yield { userId, assetId, ...metadata, size };+ }+ }+}diff --git a/packages/plugins/assetstore-filesystem/src/tests/provider.test.ts b/packages/plugins/assetstore-filesystem/src/tests/provider.test.tsnew file mode 100644index 00000000..c8c4caa6--- /dev/null+++ b/packages/plugins/assetstore-filesystem/src/tests/provider.test.ts@@ -0,0 +1,16 @@+import { describe, expect, it } from "vitest";++import { FileSystemAssetStoreProvider } from "../../index";++describe("FileSystemAssetStoreProvider", () => {+ it("returns one client across concurrent calls", async () => {+ const provider = new FileSystemAssetStoreProvider();++ const [first, second] = await Promise.all([+ provider.getClient(),+ provider.getClient(),+ ]);++ expect(first).toBe(second);+ });+});diff --git a/packages/plugins/assetstore-s3/index.ts b/packages/plugins/assetstore-s3/index.tsnew file mode 100644index 00000000..87972071--- /dev/null+++ b/packages/plugins/assetstore-s3/index.ts@@ -0,0 +1,68 @@+import type { AssetStore } from "@karakeep/shared/assetdb";+import serverConfig from "@karakeep/shared/config";+import { PluginManager, PluginType } from "@karakeep/shared/plugins";+import type { PluginProvider } from "@karakeep/shared/plugins";++interface S3AssetStoreOptions {+ region?: string;+ endpoint: string;+ forcePathStyle: boolean;+ bucket: string;+ accessKeyId: string;+ secretAccessKey: string;+}++export class S3AssetStoreProvider implements PluginProvider<AssetStore> {+ private client: AssetStore | null = null;++ constructor(private readonly options: S3AssetStoreOptions) {}++ async getClient(): Promise<AssetStore> {+ if (!this.client) {+ const [{ S3Client }, { S3AssetStore }] = await Promise.all([+ import("@aws-sdk/client-s3"),+ import("./src"),+ ]);+ if (!this.client) {+ const s3Client = new S3Client({+ region: this.options.region,+ endpoint: this.options.endpoint,+ forcePathStyle: this.options.forcePathStyle,+ credentials: {+ accessKeyId: this.options.accessKeyId,+ secretAccessKey: this.options.secretAccessKey,+ },+ });+ this.client = new S3AssetStore(s3Client, this.options.bucket);+ }+ }+ return this.client;+ }+}++const config = serverConfig.assetStore;+if (config.type === "s3") {+ if (!config.s3.bucket) {+ throw new Error(+ "ASSET_STORE_S3_BUCKET is required when using S3 asset store",+ );+ }+ if (!config.s3.accessKeyId || !config.s3.secretAccessKey) {+ throw new Error(+ "ASSET_STORE_S3_ACCESS_KEY_ID and ASSET_STORE_S3_SECRET_ACCESS_KEY are required when using S3 asset store",+ );+ }++ PluginManager.register({+ type: PluginType.AssetStore,+ name: "S3",+ provider: new S3AssetStoreProvider({+ region: config.s3.region,+ endpoint: config.s3.endpoint!,+ forcePathStyle: config.s3.forcePathStyle,+ bucket: config.s3.bucket,+ accessKeyId: config.s3.accessKeyId,+ secretAccessKey: config.s3.secretAccessKey,+ }),+ });+}diff --git a/packages/plugins/assetstore-s3/src/index.ts b/packages/plugins/assetstore-s3/src/index.tsnew file mode 100644index 00000000..92dcbc22--- /dev/null+++ b/packages/plugins/assetstore-s3/src/index.ts@@ -0,0 +1,284 @@+import * as fs from "node:fs";+import { Readable } from "node:stream";+import {+ DeleteObjectCommand,+ DeleteObjectsCommand,+ GetObjectCommand,+ HeadObjectCommand,+ ListObjectsV2Command,+ PutObjectCommand,+ S3Client,+} from "@aws-sdk/client-s3";++import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";+import { SUPPORTED_ASSET_TYPES } from "@karakeep/shared/assetdb";+import logger from "@karakeep/shared/logger";++export class S3AssetStore implements AssetStore {+ constructor(+ private readonly s3Client: S3Client,+ private readonly bucketName: string,+ ) {}++ private getAssetKey(userId: string, assetId: string) {+ return `${userId}/${assetId}`;+ }++ private metadataToS3Metadata(+ metadata: AssetMetadata,+ ): Record<string, string> {+ return {+ ...(metadata.fileName+ ? { "x-amz-meta-file-name": metadata.fileName }+ : {}),+ "x-amz-meta-content-type": metadata.contentType,+ };+ }++ private s3MetadataToMetadata(+ s3Metadata: Record<string, string> | undefined,+ ): AssetMetadata {+ if (!s3Metadata) {+ throw new Error("No metadata found in S3 object");+ }++ return {+ contentType: s3Metadata["x-amz-meta-content-type"] || "",+ fileName: s3Metadata["x-amz-meta-file-name"] ?? null,+ };+ }++ async saveAsset({+ userId,+ assetId,+ asset,+ metadata,+ }: {+ userId: string;+ assetId: string;+ asset: Buffer;+ metadata: AssetMetadata;+ }) {+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {+ throw new Error("Unsupported asset type");+ }++ await this.s3Client.send(+ new PutObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ Body: asset,+ ContentType: metadata.contentType,+ Metadata: this.metadataToS3Metadata(metadata),+ }),+ );+ }++ async saveAssetFromFile({+ userId,+ assetId,+ assetPath,+ metadata,+ }: {+ userId: string;+ assetId: string;+ assetPath: string;+ metadata: AssetMetadata;+ }) {+ if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {+ throw new Error("Unsupported asset type");+ }++ const asset = fs.createReadStream(assetPath);+ await this.s3Client.send(+ new PutObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ Body: asset,+ ContentType: metadata.contentType,+ Metadata: this.metadataToS3Metadata(metadata),+ }),+ );+ await fs.promises.rm(assetPath);+ }++ async readAsset({+ userId,+ assetId,+ start,+ end,+ }: {+ userId: string;+ assetId: string;+ start?: number;+ end?: number;+ }) {+ const range =+ start !== undefined || end !== undefined+ ? `bytes=${start ?? 0}-${end ?? ""}`+ : undefined;++ const response = await this.s3Client.send(+ new GetObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ Range: range,+ }),+ );++ if (!response.Body) {+ throw new Error("Asset not found");+ }++ const assetBuffer = await this.streamToBuffer(response.Body as Readable);+ const metadata = this.s3MetadataToMetadata(response.Metadata);++ return { asset: assetBuffer, metadata };+ }++ async createAssetReadStream({+ userId,+ assetId,+ start,+ end,+ }: {+ userId: string;+ assetId: string;+ start?: number;+ end?: number;+ }) {+ const range =+ start !== undefined || end !== undefined+ ? `bytes=${start ?? 0}-${end ?? ""}`+ : undefined;++ const response = await this.s3Client.send(+ new GetObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ Range: range,+ }),+ );+ if (!response.Body) {+ throw new Error("Asset not found");+ }+ return response.Body as NodeJS.ReadableStream;+ }++ async readAssetMetadata({+ userId,+ assetId,+ }: {+ userId: string;+ assetId: string;+ }) {+ const response = await this.s3Client.send(+ new HeadObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ }),+ );++ return this.s3MetadataToMetadata(response.Metadata);+ }++ async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {+ const response = await this.s3Client.send(+ new HeadObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ }),+ );++ return response.ContentLength || 0;+ }++ async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {+ await this.s3Client.send(+ new DeleteObjectCommand({+ Bucket: this.bucketName,+ Key: this.getAssetKey(userId, assetId),+ }),+ );+ }++ async deleteUserAssets({ userId }: { userId: string }) {+ let continuationToken: string | undefined;++ do {+ const listResponse = await this.s3Client.send(+ new ListObjectsV2Command({+ Bucket: this.bucketName,+ Prefix: `${userId}/`,+ ContinuationToken: continuationToken,+ }),+ );++ if (listResponse.Contents && listResponse.Contents.length > 0) {+ await this.s3Client.send(+ new DeleteObjectsCommand({+ Bucket: this.bucketName,+ Delete: {+ Objects: listResponse.Contents.map((object) => ({+ Key: object.Key!,+ })),+ },+ }),+ );+ }++ continuationToken = listResponse.NextContinuationToken;+ } while (continuationToken);+ }++ async *getAllAssets() {+ let continuationToken: string | undefined;++ do {+ const listResponse = await this.s3Client.send(+ new ListObjectsV2Command({+ Bucket: this.bucketName,+ ContinuationToken: continuationToken,+ }),+ );++ if (listResponse.Contents) {+ for (const object of listResponse.Contents) {+ if (!object.Key) continue;++ const pathParts = object.Key.split("/");+ if (pathParts.length === 2) {+ const userId = pathParts[0];+ const assetId = pathParts[1];++ try {+ const headResponse = await this.s3Client.send(+ new HeadObjectCommand({+ Bucket: this.bucketName,+ Key: object.Key,+ }),+ );++ const metadata = this.s3MetadataToMetadata(headResponse.Metadata);+ const size = headResponse.ContentLength || 0;++ yield { userId, assetId, ...metadata, size };+ } catch (error) {+ logger.warn(`Failed to read asset ${userId}/${assetId}:`, error);+ }+ }+ }+ }++ continuationToken = listResponse.NextContinuationToken;+ } while (continuationToken);+ }++ private async streamToBuffer(stream: Readable): Promise<Buffer> {+ const chunks: Buffer[] = [];+ for await (const chunk of stream) {+ chunks.push(chunk);+ }+ return Buffer.concat(chunks);+ }+}diff --git a/packages/plugins/assetstore-s3/src/tests/provider.test.ts b/packages/plugins/assetstore-s3/src/tests/provider.test.tsnew file mode 100644index 00000000..66489d7f--- /dev/null+++ b/packages/plugins/assetstore-s3/src/tests/provider.test.ts@@ -0,0 +1,23 @@+import { describe, expect, it } from "vitest";++import { S3AssetStoreProvider } from "../../index";++describe("S3AssetStoreProvider", () => {+ it("returns one client across concurrent calls", async () => {+ const provider = new S3AssetStoreProvider({+ region: "us-east-1",+ endpoint: "http://localhost:9000",+ forcePathStyle: true,+ bucket: "test",+ accessKeyId: "test",+ secretAccessKey: "test",+ });++ const [first, second] = await Promise.all([+ provider.getClient(),+ provider.getClient(),+ ]);++ expect(first).toBe(second);+ });+});diff --git a/packages/plugins/package.json b/packages/plugins/package.jsonindex 69321979..c39f864e 100644--- a/packages/plugins/package.json+++ b/packages/plugins/package.json@@ -5,6 +5,10 @@"private": true,"type": "module","exports": {+ "./assetstore-filesystem": "./assetstore-filesystem/index.ts",+ "./assetstore-filesystem/implementation": "./assetstore-filesystem/src/index.ts",+ "./assetstore-s3": "./assetstore-s3/index.ts",+ "./assetstore-s3/implementation": "./assetstore-s3/src/index.ts","./queue-liteque": "./queue-liteque/index.ts","./queue-restate": "./queue-restate/index.ts","./ratelimit-memory": "./ratelimit-memory/index.ts",@@ -21,10 +25,12 @@"test": "vitest"},"dependencies": {+ "@aws-sdk/client-s3": "^3.1014.0","@karakeep/shared": "workspace:*","@restatedev/restate-sdk": "^1.16.2","@restatedev/restate-sdk-clients": "^1.16.2","async-mutex": "^0.4.1",+ "glob": "^11.0.0","liteque": "^0.9.0","meilisearch": "^0.58.0","redis": "^5.11.0"diff --git a/packages/shared-server/src/assetdb.ts b/packages/shared-server/src/assetdb.tsnew file mode 100644index 00000000..1e33266a--- /dev/null+++ b/packages/shared-server/src/assetdb.ts@@ -0,0 +1,160 @@+import type { AssetMetadata, AssetStore } from "@karakeep/shared/assetdb";+import { PluginManager, PluginType } from "@karakeep/shared/plugins";+import type { QuotaApproved } from "@karakeep/shared/storageQuota";++import { loadAllPlugins } from "./plugins";++export {+ ASSET_TYPES,+ IMAGE_ASSET_TYPES,+ SUPPORTED_ASSET_TYPES,+ SUPPORTED_BOOKMARK_ASSET_TYPES,+ SUPPORTED_UPLOAD_ASSET_TYPES,+ VIDEO_ASSET_TYPES,+ zAssetMetadataSchema,+} from "@karakeep/shared/assetdb";+export type {+ AssetInfo,+ AssetMetadata,+ AssetStore,+} from "@karakeep/shared/assetdb";++async function getAssetStore(): Promise<AssetStore> {+ await loadAllPlugins();+ const store = await PluginManager.getClient(PluginType.AssetStore);+ if (!store) {+ throw new Error("No asset store plugin is registered");+ }+ return store;+}++export function newAssetId() {+ return crypto.randomUUID();+}++export async function saveAsset({+ userId,+ assetId,+ asset,+ metadata,+ quotaApproved,+}: {+ userId: string;+ assetId: string;+ asset: Buffer;+ metadata: AssetMetadata;+ quotaApproved: QuotaApproved;+}) {+ if (quotaApproved.userId !== userId) {+ throw new Error("Quota approval is for a different user");+ }+ if (quotaApproved.approvedSize < asset.byteLength) {+ throw new Error("Asset size exceeds approved quota");+ }++ const store = await getAssetStore();+ return store.saveAsset({ userId, assetId, asset, metadata });+}++export async function saveAssetFromFile({+ userId,+ assetId,+ assetPath,+ metadata,+ quotaApproved,+}: {+ userId: string;+ assetId: string;+ assetPath: string;+ metadata: AssetMetadata;+ quotaApproved: QuotaApproved;+}) {+ if (quotaApproved.userId !== userId) {+ throw new Error("Quota approval is for a different user");+ }++ const store = await getAssetStore();+ return store.saveAssetFromFile({ userId, assetId, assetPath, metadata });+}++export async function readAsset({+ userId,+ assetId,+ start,+ end,+}: {+ userId: string;+ assetId: string;+ start?: number;+ end?: number;+}) {+ const store = await getAssetStore();+ return store.readAsset({ userId, assetId, start, end });+}++export async function createAssetReadStream({+ userId,+ assetId,+ start,+ end,+}: {+ userId: string;+ assetId: string;+ start?: number;+ end?: number;+}) {+ const store = await getAssetStore();+ return store.createAssetReadStream({ userId, assetId, start, end });+}++export async function readAssetMetadata({+ userId,+ assetId,+}: {+ userId: string;+ assetId: string;+}) {+ const store = await getAssetStore();+ return store.readAssetMetadata({ userId, assetId });+}++export async function getAssetSize({+ userId,+ assetId,+}: {+ userId: string;+ assetId: string;+}) {+ const store = await getAssetStore();+ return store.getAssetSize({ userId, assetId });+}++export async function silentDeleteAsset(+ userId: string,+ assetId: string | undefined,+) {+ if (assetId) {+ await deleteAsset({ userId, assetId }).catch(() => ({}));+ }+}++export async function deleteAsset({+ userId,+ assetId,+}: {+ userId: string;+ assetId: string;+}) {+ const store = await getAssetStore();+ return store.deleteAsset({ userId, assetId });+}++export async function deleteUserAssets({ userId }: { userId: string }) {+ const store = await getAssetStore();+ return store.deleteUserAssets({ userId });+}++export async function* getAllAssets() {+ const store = await getAssetStore();+ yield* store.getAllAssets();+}diff --git a/packages/shared-server/src/index.ts b/packages/shared-server/src/index.tsindex 580d45ed..d5c9ab17 100644--- a/packages/shared-server/src/index.ts+++ b/packages/shared-server/src/index.ts@@ -1,4 +1,5 @@export { loadAllPlugins } from "./plugins";+export * from "./assetdb";export { QuotaService, StorageQuotaError } from "./services/quotaService";export * from "./queues";export * from "./eventLogger";diff --git a/packages/shared-server/src/plugins.ts b/packages/shared-server/src/plugins.tsindex dc9fa547..057ad16c 100644--- a/packages/shared-server/src/plugins.ts+++ b/packages/shared-server/src/plugins.ts@@ -23,6 +23,9 @@ export async function loadAllPlugins() {}pluginLoaderState.loading = (async () => {// Load plugins here. Order of plugin loading matter.+ // Asset store providers (order matters - last one wins)+ await import("@karakeep/plugins/assetstore-filesystem");+ await import("@karakeep/plugins/assetstore-s3");// Queue provider(s)await import("@karakeep/plugins/queue-liteque");await import("@karakeep/plugins/queue-restate");diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.tsindex 8f98d0fd..d1b0f8a6 100644--- a/packages/shared/assetdb.ts+++ b/packages/shared/assetdb.ts@@ -1,25 +1,5 @@-import * as fs from "fs";-import * as path from "path";-import { Readable } from "stream";-import {- _Object,- DeleteObjectCommand,- DeleteObjectsCommand,- GetObjectCommand,- HeadObjectCommand,- ListObjectsV2Command,- PutObjectCommand,- S3Client,-} from "@aws-sdk/client-s3";-import { Glob } from "glob";import { z } from "zod";-import serverConfig from "./config";-import logger from "./logger";-import { QuotaApproved } from "./storageQuota";--const ROOT_PATH = serverConfig.assetsDir;-export const enum ASSET_TYPES {IMAGE_GIF = "image/gif",IMAGE_JPEG = "image/jpeg",@@ -126,694 +106,3 @@ export interface AssetStore {getAllAssets(): AsyncGenerator<AssetInfo>;}--export function newAssetId() {- return crypto.randomUUID();-}--class LocalFileSystemAssetStore implements AssetStore {- private rootPath: string;-- constructor(rootPath: string) {- this.rootPath = rootPath;- }-- private getAssetDir(userId: string, assetId: string) {- return path.join(this.rootPath, userId, assetId);- }-- private async isPathExists(filePath: string) {- return fs.promises- .access(filePath)- .then(() => true)- .catch(() => false);- }-- async saveAsset({- userId,- assetId,- asset,- metadata,- }: {- userId: string;- assetId: string;- asset: Buffer;- metadata: AssetMetadata;- }) {- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {- throw new Error("Unsupported asset type");- }- const assetDir = this.getAssetDir(userId, assetId);- await fs.promises.mkdir(assetDir, { recursive: true });-- await Promise.all([- fs.promises.writeFile(- path.join(assetDir, "asset.bin"),- Uint8Array.from(asset),- ),- fs.promises.writeFile(- path.join(assetDir, "metadata.json"),- JSON.stringify(metadata),- ),- ]);- }-- async saveAssetFromFile({- userId,- assetId,- assetPath,- metadata,- }: {- userId: string;- assetId: string;- assetPath: string;- metadata: AssetMetadata;- }) {- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {- throw new Error("Unsupported asset type");- }- const assetDir = this.getAssetDir(userId, assetId);- await fs.promises.mkdir(assetDir, { recursive: true });-- await Promise.all([- fs.promises.copyFile(assetPath, path.join(assetDir, "asset.bin")),- fs.promises.writeFile(- path.join(assetDir, "metadata.json"),- JSON.stringify(metadata),- ),- ]);- await fs.promises.rm(assetPath);- }-- async readAsset({- userId,- assetId,- start,- end,- }: {- userId: string;- assetId: string;- start?: number;- end?: number;- }) {- const assetDir = this.getAssetDir(userId, assetId);-- const readAssetFile =- start !== undefined || end !== undefined- ? (async () => {- const fd = await fs.promises.open(- path.join(assetDir, "asset.bin"),- "r",- );- try {- const stat = await fd.stat();- const offset = start ?? 0;- const effectiveEnd = Math.min(- end !== undefined ? end + 1 : stat.size,- stat.size,- );- const length = effectiveEnd - offset;- const buffer = Buffer.alloc(length);- const { bytesRead } = await fd.read(buffer, 0, length, offset);- return bytesRead < length- ? buffer.subarray(0, bytesRead)- : buffer;- } finally {- await fd.close();- }- })()- : fs.promises.readFile(path.join(assetDir, "asset.bin"));-- const [asset, metadataStr] = await Promise.all([- readAssetFile,- fs.promises.readFile(path.join(assetDir, "metadata.json"), {- encoding: "utf8",- }),- ]);-- const metadata = zAssetMetadataSchema.parse(JSON.parse(metadataStr));- return { asset, metadata };- }-- async createAssetReadStream({- userId,- assetId,- start,- end,- }: {- userId: string;- assetId: string;- start?: number;- end?: number;- }) {- const assetDir = this.getAssetDir(userId, assetId);- const assetPath = path.join(assetDir, "asset.bin");- if (!(await this.isPathExists(assetPath))) {- throw new Error(`Asset ${assetId} not found`);- }-- return fs.createReadStream(path.join(assetDir, "asset.bin"), {- start,- end,- });- }-- async readAssetMetadata({- userId,- assetId,- }: {- userId: string;- assetId: string;- }) {- const assetDir = this.getAssetDir(userId, assetId);-- const metadataStr = await fs.promises.readFile(- path.join(assetDir, "metadata.json"),- {- encoding: "utf8",- },- );-- return zAssetMetadataSchema.parse(JSON.parse(metadataStr));- }-- async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {- const assetDir = this.getAssetDir(userId, assetId);- const stat = await fs.promises.stat(path.join(assetDir, "asset.bin"));- return stat.size;- }-- async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {- const assetDir = this.getAssetDir(userId, assetId);- if (!(await this.isPathExists(assetDir))) {- return;- }- await fs.promises.rm(assetDir, { recursive: true });- }-- async deleteUserAssets({ userId }: { userId: string }) {- const userDir = path.join(this.rootPath, userId);- const dirExists = await this.isPathExists(userDir);- if (!dirExists) {- return;- }- await fs.promises.rm(userDir, { recursive: true });- }-- async *getAllAssets() {- const g = new Glob(`/**/**/asset.bin`, {- maxDepth: 3,- root: this.rootPath,- cwd: this.rootPath,- absolute: false,- });- for await (const file of g) {- const [userId, assetId] = file.split("/").slice(0, 2);- const [size, metadata] = await Promise.all([- this.getAssetSize({ userId, assetId }),- this.readAssetMetadata({ userId, assetId }),- ]);- yield {- userId,- assetId,- ...metadata,- size,- };- }- }-}--class S3AssetStore implements AssetStore {- private s3Client: S3Client;- private bucketName: string;-- constructor(s3Client: S3Client, bucketName: string) {- this.s3Client = s3Client;- this.bucketName = bucketName;- }-- private getAssetKey(userId: string, assetId: string) {- return `${userId}/${assetId}`;- }-- private metadataToS3Metadata(- metadata: AssetMetadata,- ): Record<string, string> {- return {- ...(metadata.fileName- ? { "x-amz-meta-file-name": metadata.fileName }- : {}),- "x-amz-meta-content-type": metadata.contentType,- };- }-- private s3MetadataToMetadata(- s3Metadata: Record<string, string> | undefined,- ): AssetMetadata {- if (!s3Metadata) {- throw new Error("No metadata found in S3 object");- }-- return {- contentType: s3Metadata["x-amz-meta-content-type"] || "",- fileName: s3Metadata["x-amz-meta-file-name"] ?? null,- };- }-- async saveAsset({- userId,- assetId,- asset,- metadata,- }: {- userId: string;- assetId: string;- asset: Buffer;- metadata: AssetMetadata;- }) {- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {- throw new Error("Unsupported asset type");- }-- await this.s3Client.send(- new PutObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- Body: asset,- ContentType: metadata.contentType,- Metadata: this.metadataToS3Metadata(metadata),- }),- );- }-- async saveAssetFromFile({- userId,- assetId,- assetPath,- metadata,- }: {- userId: string;- assetId: string;- assetPath: string;- metadata: AssetMetadata;- }) {- if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {- throw new Error("Unsupported asset type");- }-- const asset = fs.createReadStream(assetPath);- await this.s3Client.send(- new PutObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- Body: asset,- ContentType: metadata.contentType,- Metadata: this.metadataToS3Metadata(metadata),- }),- );- await fs.promises.rm(assetPath);- }-- async readAsset({- userId,- assetId,- start,- end,- }: {- userId: string;- assetId: string;- start?: number;- end?: number;- }) {- const range =- start !== undefined || end !== undefined- ? `bytes=${start ?? 0}-${end ?? ""}`- : undefined;-- const response = await this.s3Client.send(- new GetObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- Range: range,- }),- );-- if (!response.Body) {- throw new Error("Asset not found");- }-- const assetBuffer = await this.streamToBuffer(response.Body as Readable);- const metadata = this.s3MetadataToMetadata(response.Metadata);-- return { asset: assetBuffer, metadata };- }-- async createAssetReadStream({- userId,- assetId,- start,- end,- }: {- userId: string;- assetId: string;- start?: number;- end?: number;- }) {- const range =- start !== undefined || end !== undefined- ? `bytes=${start ?? 0}-${end ?? ""}`- : undefined;-- const command = new GetObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- Range: range,- });-- const response = await this.s3Client.send(command);- if (!response.Body) {- throw new Error("Asset not found");- }- return response.Body as NodeJS.ReadableStream;- }-- async readAssetMetadata({- userId,- assetId,- }: {- userId: string;- assetId: string;- }) {- const response = await this.s3Client.send(- new HeadObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- }),- );-- return this.s3MetadataToMetadata(response.Metadata);- }-- async getAssetSize({ userId, assetId }: { userId: string; assetId: string }) {- const response = await this.s3Client.send(- new HeadObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- }),- );-- return response.ContentLength || 0;- }-- async deleteAsset({ userId, assetId }: { userId: string; assetId: string }) {- await this.s3Client.send(- new DeleteObjectCommand({- Bucket: this.bucketName,- Key: this.getAssetKey(userId, assetId),- }),- );- }-- async deleteUserAssets({ userId }: { userId: string }) {- let continuationToken: string | undefined;-- do {- const listResponse = await this.s3Client.send(- new ListObjectsV2Command({- Bucket: this.bucketName,- Prefix: `${userId}/`,- ContinuationToken: continuationToken,- }),- );-- if (listResponse.Contents && listResponse.Contents.length > 0) {- await this.s3Client.send(- new DeleteObjectsCommand({- Bucket: this.bucketName,- Delete: {- Objects: listResponse.Contents.map((obj) => ({- Key: obj.Key!,- })),- },- }),- );- }-- continuationToken = listResponse.NextContinuationToken;- } while (continuationToken);- }-- async *getAllAssets() {- let continuationToken: string | undefined;-- do {- const listResponse = await this.s3Client.send(- new ListObjectsV2Command({- Bucket: this.bucketName,- ContinuationToken: continuationToken,- }),- );-- if (listResponse.Contents) {- for (const obj of listResponse.Contents) {- if (!obj.Key) continue;-- const pathParts = obj.Key.split("/");- if (pathParts.length === 2) {- const userId = pathParts[0];- const assetId = pathParts[1];-- try {- const headResponse = await this.s3Client.send(- new HeadObjectCommand({- Bucket: this.bucketName,- Key: obj.Key,- }),- );-- const metadata = this.s3MetadataToMetadata(headResponse.Metadata);- const size = headResponse.ContentLength || 0;-- yield {- userId,- assetId,- ...metadata,- size,- };- } catch (error) {- logger.warn(`Failed to read asset ${userId}/${assetId}:`, error);- }- }- }- }-- continuationToken = listResponse.NextContinuationToken;- } while (continuationToken);- }-- private async streamToBuffer(stream: Readable): Promise<Buffer> {- const chunks: Buffer[] = [];- for await (const chunk of stream) {- chunks.push(chunk);- }- return Buffer.concat(chunks);- }-}--function createDefaultAssetStore(): AssetStore {- const config = serverConfig.assetStore;-- if (config.type === "s3") {- if (!config.s3.bucket) {- throw new Error(- "ASSET_STORE_S3_BUCKET is required when using S3 asset store",- );- }- if (!config.s3.accessKeyId || !config.s3.secretAccessKey) {- throw new Error(- "ASSET_STORE_S3_ACCESS_KEY_ID and ASSET_STORE_S3_SECRET_ACCESS_KEY are required when using S3 asset store",- );- }-- const s3Client = new S3Client({- region: config.s3.region,- endpoint: config.s3.endpoint,- forcePathStyle: config.s3.forcePathStyle,- credentials: {- accessKeyId: config.s3.accessKeyId,- secretAccessKey: config.s3.secretAccessKey,- },- });-- return new S3AssetStore(s3Client, config.s3.bucket);- }-- return new LocalFileSystemAssetStore(ROOT_PATH);-}--const defaultAssetStore = createDefaultAssetStore();--export { LocalFileSystemAssetStore, S3AssetStore };--/**- * Example usage of S3AssetStore:- *- * import { S3Client } from "@aws-sdk/client-s3";- * import { S3AssetStore } from "@karakeep/shared/assetdb";- *- * const s3Client = new S3Client({- * region: "us-east-1",- * credentials: {- * accessKeyId: "your-access-key",- * secretAccessKey: "your-secret-key"- * }- * });- *- * const s3AssetStore = new S3AssetStore(s3Client, "your-bucket-name");- *- * // Use s3AssetStore instead of the default file system store- * await s3AssetStore.saveAsset({- * userId: "user123",- * assetId: "asset456",- * asset: buffer,- * metadata: { contentType: "image/jpeg", fileName: "photo.jpg" }- * });- */--export async function saveAsset({- userId,- assetId,- asset,- metadata,- quotaApproved,-}: {- userId: string;- assetId: string;- asset: Buffer;- metadata: z.infer<typeof zAssetMetadataSchema>;- quotaApproved: QuotaApproved;-}) {- // Verify the quota approval is for the correct user and size- if (quotaApproved.userId !== userId) {- throw new Error("Quota approval is for a different user");- }- if (quotaApproved.approvedSize < asset.byteLength) {- throw new Error("Asset size exceeds approved quota");- }-- return defaultAssetStore.saveAsset({ userId, assetId, asset, metadata });-}--export async function saveAssetFromFile({- userId,- assetId,- assetPath,- metadata,- quotaApproved,-}: {- userId: string;- assetId: string;- assetPath: string;- metadata: z.infer<typeof zAssetMetadataSchema>;- quotaApproved: QuotaApproved;-}) {- // Verify the quota approval is for the correct user- if (quotaApproved.userId !== userId) {- throw new Error("Quota approval is for a different user");- }-- // For file-based saves, we'll verify the file size matches the approved size- // when the underlying store implementation reads the file-- return defaultAssetStore.saveAssetFromFile({- userId,- assetId,- assetPath,- metadata,- });-}--export async function readAsset({- userId,- assetId,- start,- end,-}: {- userId: string;- assetId: string;- start?: number;- end?: number;-}) {- return defaultAssetStore.readAsset({ userId, assetId, start, end });-}--export async function createAssetReadStream({- userId,- assetId,- start,- end,-}: {- userId: string;- assetId: string;- start?: number;- end?: number;-}) {- return defaultAssetStore.createAssetReadStream({- userId,- assetId,- start,- end,- });-}--export async function readAssetMetadata({- userId,- assetId,-}: {- userId: string;- assetId: string;-}) {- return defaultAssetStore.readAssetMetadata({ userId, assetId });-}--export async function getAssetSize({- userId,- assetId,-}: {- userId: string;- assetId: string;-}) {- return defaultAssetStore.getAssetSize({ userId, assetId });-}--/**- * Deletes the passed in asset if it exists and ignores any errors- * @param userId the id of the user the asset belongs to- * @param assetId the id of the asset to delete- */-export async function silentDeleteAsset(- userId: string,- assetId: string | undefined,-) {- if (assetId) {- await deleteAsset({ userId, assetId }).catch(() => ({}));- }-}--export async function deleteAsset({- userId,- assetId,-}: {- userId: string;- assetId: string;-}) {- return defaultAssetStore.deleteAsset({ userId, assetId });-}--export async function deleteUserAssets({ userId }: { userId: string }) {- return defaultAssetStore.deleteUserAssets({ userId });-}--export async function* getAllAssets() {- yield* defaultAssetStore.getAllAssets();-}diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 9be91c3a..d9eceb33 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -5,9 +5,7 @@"private": true,"type": "module","dependencies": {- "@aws-sdk/client-s3": "^3.1014.0","cheerio": "^1.2.0",- "glob": "^11.0.0","html-to-text": "^9.0.5","js-tiktoken": "^1.0.20","nodemailer": "^7.0.13",diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.tsindex 974b9215..efe8853f 100644--- a/packages/shared/plugins.ts+++ b/packages/shared/plugins.ts@@ -1,5 +1,6 @@// Implementation inspired from Outline+import type { AssetStore } from "./assetdb";import type { QueueClient } from "./queueing";import type { RateLimitClient } from "./ratelimiting";import type { VectorStoreClient } from "./vectorStore";@@ -7,6 +8,7 @@ import logger from "./logger";import { SearchIndexClient } from "./search";export enum PluginType {+ AssetStore = "assetstore",Search = "search",Queue = "queue",RateLimit = "ratelimit",@@ -14,6 +16,7 @@ export enum PluginType {}interface PluginTypeMap {+ [PluginType.AssetStore]: AssetStore;[PluginType.Search]: SearchIndexClient;[PluginType.Queue]: QueueClient;[PluginType.RateLimit]: RateLimitClient;@@ -37,6 +40,7 @@ const pluginProvidersKey = "__karakeep_plugins_providers__";function createProviderMap(): ProviderMap {return {+ [PluginType.AssetStore]: [],[PluginType.Search]: [],[PluginType.Queue]: [],[PluginType.RateLimit]: [],diff --git a/packages/trpc/models/assets.ts b/packages/trpc/models/assets.tsindex 44366a4f..8fb4842b 100644--- a/packages/trpc/models/assets.ts+++ b/packages/trpc/models/assets.ts@@ -3,7 +3,7 @@ import { and, desc, eq, sql } from "drizzle-orm";import { z } from "zod";import { assets } from "@karakeep/db/schema";-import { deleteAsset } from "@karakeep/shared/assetdb";+import { deleteAsset } from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import { createSignedToken } from "@karakeep/shared/signedTokens";import { zAssetSignedTokenSchema } from "@karakeep/shared/types/assets";diff --git a/packages/trpc/models/backups.ts b/packages/trpc/models/backups.tsindex 7f0d0598..82f7b91f 100644--- a/packages/trpc/models/backups.ts+++ b/packages/trpc/models/backups.ts@@ -3,8 +3,7 @@ import { and, desc, eq, lt } from "drizzle-orm";import { z } from "zod";import { assets, backupsTable } from "@karakeep/db/schema";-import { BackupQueue } from "@karakeep/shared-server";-import { deleteAsset } from "@karakeep/shared/assetdb";+import { BackupQueue, deleteAsset } from "@karakeep/shared-server";import { zBackupSchema } from "@karakeep/shared/types/backups";import { AuthedContext } from "..";diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.tsindex dbf1ca63..0fe06904 100644--- a/packages/trpc/models/bookmarks.ts+++ b/packages/trpc/models/bookmarks.ts@@ -31,10 +31,14 @@ import {rssFeedImportsTable,tagsOnBookmarks,} from "@karakeep/db/schema";-import { EmbeddingsQueue, SearchIndexingQueue } from "@karakeep/shared-server";+import {+ deleteAsset,+ EmbeddingsQueue,+ readAsset,+ SearchIndexingQueue,+} from "@karakeep/shared-server";import { WebhooksService } from "./webhooks.service";-import { deleteAsset, readAsset } from "@karakeep/shared/assetdb";import { getAlignedExpiry } from "@karakeep/shared/signedTokens";import {BookmarkTypes,diff --git a/packages/trpc/models/users.ts b/packages/trpc/models/users.tsindex 7cf23c97..a0d52010 100644--- a/packages/trpc/models/users.ts+++ b/packages/trpc/models/users.ts@@ -19,7 +19,7 @@ import {users,verificationTokens,} from "@karakeep/db/schema";-import { deleteAsset, deleteUserAssets } from "@karakeep/shared/assetdb";+import { deleteAsset, deleteUserAssets } from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import {zResetPasswordSchema,diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.tsindex 20f3e3c7..7269560b 100644--- a/packages/trpc/routers/bookmarks.ts+++ b/packages/trpc/routers/bookmarks.ts@@ -26,9 +26,9 @@ import {OpenAIQueue,QueuePriority,QuotaService,+ SUPPORTED_BOOKMARK_ASSET_TYPES,triggerSearchReindex,} from "@karakeep/shared-server";-import { SUPPORTED_BOOKMARK_ASSET_TYPES } from "@karakeep/shared/assetdb";import serverConfig from "@karakeep/shared/config";import {EmbeddingClientFactory,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 2c166e30..df2cbb98 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1277,6 +1277,9 @@ importers:'@aws-sdk/client-s3':specifier: ^3.1014.0version: 3.1014.0+ '@karakeep/plugins':+ specifier: workspace:^0.1.0+ version: link:../plugins'@karakeep/sdk':specifier: workspace:*version: link:../sdk@@ -1333,6 +1336,9 @@ importers:packages/plugins:dependencies:+ '@aws-sdk/client-s3':+ specifier: ^3.1014.0+ version: 3.1014.0'@karakeep/shared':specifier: workspace:*version: link:../shared@@ -1345,6 +1351,9 @@ importers:async-mutex:specifier: ^0.4.1version: 0.4.1+ glob:+ specifier: ^11.0.0+ version: 11.0.2liteque:specifier: ^0.9.0version: 0.9.0(@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])@@ -1392,15 +1401,9 @@ importers:packages/shared:dependencies:- '@aws-sdk/client-s3':- specifier: ^3.1014.0- version: 3.1014.0cheerio:specifier: ^1.2.0version: 1.2.0- glob:- specifier: ^11.0.0- version: 11.0.2html-to-text:specifier: ^9.0.5version: 9.0.5