A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
refactor: move assetdb to shared-server
Mohamed Bassema04d3c35fc90
diff --git a/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts b/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.tsindex a9ae54a6..f76a7dbc 100644--- a/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts+++ b/apps/workers/workers/adminMaintenance/tasks/migrateLinkHtmlContent.ts@@ -10,7 +10,7 @@ import {deleteAsset,newAssetId,saveAsset,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { tryCatch } from "@karakeep/shared/tryCatch";diff --git a/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts b/apps/workers/workers/adminMaintenance/tasks/tidyAssets.tsindex 792f9417..eb3cd5f5 100644--- a/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts+++ b/apps/workers/workers/adminMaintenance/tasks/tidyAssets.ts@@ -7,7 +7,7 @@ import {ZTidyAssetsRequest,zTidyAssetsRequestSchema,} from "@karakeep/shared-server";-import { deleteAsset, getAllAssets } from "@karakeep/shared/assetdb";+import { deleteAsset, getAllAssets } from "@karakeep/shared-server/assetdb";import logger from "@karakeep/shared/logger";import { DequeuedJob } from "@karakeep/shared/queueing";diff --git a/apps/workers/workers/assetPreprocessingWorker.ts b/apps/workers/workers/assetPreprocessingWorker.tsindex d12457d3..bf563abd 100644--- a/apps/workers/workers/assetPreprocessingWorker.ts+++ b/apps/workers/workers/assetPreprocessingWorker.ts@@ -21,7 +21,11 @@ import {StorageQuotaError,triggerSearchReindex,} from "@karakeep/shared-server";-import { newAssetId, readAsset, saveAsset } from "@karakeep/shared/assetdb";+import {+ newAssetId,+ readAsset,+ saveAsset,+} from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import { InferenceClientFactory } from "@karakeep/shared/inference";import logger from "@karakeep/shared/logger";diff --git a/apps/workers/workers/backupWorker.ts b/apps/workers/workers/backupWorker.tsindex 01f54b28..03400bfb 100644--- a/apps/workers/workers/backupWorker.ts+++ b/apps/workers/workers/backupWorker.ts@@ -14,7 +14,7 @@ import type { ZBackupRequest } from "@karakeep/shared-server";import { db } from "@karakeep/db";import { assets, AssetTypes, users } from "@karakeep/db/schema";import { BackupQueue, QuotaService } from "@karakeep/shared-server";-import { saveAssetFromFile } from "@karakeep/shared/assetdb";+import { saveAssetFromFile } from "@karakeep/shared-server/assetdb";import { toExportFormat } from "@karakeep/shared/import-export";import logger from "@karakeep/shared/logger";import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 9815571e..24fe6ad2 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -78,7 +78,7 @@ import {saveAssetFromFile,silentDeleteAsset,SUPPORTED_UPLOAD_ASSET_TYPES,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import {diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.tsindex 668c1d5e..be04f1b9 100644--- a/apps/workers/workers/inference/tagging.ts+++ b/apps/workers/workers/inference/tagging.ts@@ -23,7 +23,7 @@ import {triggerSearchReindex,triggerWebhook,} from "@karakeep/shared-server";-import { ASSET_TYPES, readAsset } from "@karakeep/shared/assetdb";+import { ASSET_TYPES, readAsset } from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { buildImagePrompt } from "@karakeep/shared/prompts";diff --git a/apps/workers/workers/videoWorker.ts b/apps/workers/workers/videoWorker.tsindex 1ffbf674..32145c58 100644--- a/apps/workers/workers/videoWorker.ts+++ b/apps/workers/workers/videoWorker.ts@@ -20,7 +20,7 @@ import {newAssetId,saveAssetFromFile,silentDeleteAsset,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { DequeuedJob, getQueueClient } from "@karakeep/shared/queueing";diff --git a/packages/api/utils/assets.ts b/packages/api/utils/assets.tsindex 7936f4ba..2ebd691d 100644--- a/packages/api/utils/assets.ts+++ b/packages/api/utils/assets.ts@@ -5,7 +5,7 @@ import {createAssetReadStream,getAssetSize,readAssetMetadata,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server/assetdb";import { toWebReadableStream } from "./upload";diff --git a/packages/api/utils/upload.ts b/packages/api/utils/upload.tsindex b82bc855..f532fc66 100644--- a/packages/api/utils/upload.ts+++ b/packages/api/utils/upload.ts@@ -11,7 +11,7 @@ import {newAssetId,saveAssetFromFile,SUPPORTED_UPLOAD_ASSET_TYPES,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import { AuthedContext } from "@karakeep/trpc";diff --git a/packages/e2e_tests/package.json b/packages/e2e_tests/package.jsonindex d93318aa..b1c472f4 100644--- a/packages/e2e_tests/package.json+++ b/packages/e2e_tests/package.json@@ -18,6 +18,7 @@"@aws-sdk/client-s3": "^3.842.0","@karakeep/sdk": "workspace:*","@karakeep/shared": "workspace:^0.1.0",+ "@karakeep/shared-server": "workspace:^0.1.0","@karakeep/trpc": "workspace:^0.1.0","@trpc/client": "^11.9.0","superjson": "^2.2.1",diff --git a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts b/packages/e2e_tests/tests/assetdb/assetdb-utils.tsindex a8e29ab4..85a3f307 100644--- a/packages/e2e_tests/tests/assetdb/assetdb-utils.ts+++ b/packages/e2e_tests/tests/assetdb/assetdb-utils.ts@@ -15,7 +15,7 @@ import {AssetStore,LocalFileSystemAssetStore,S3AssetStore,-} from "@karakeep/shared/assetdb";+} from "@karakeep/shared-server/assetdb";export interface TestAssetData {userId: string;diff --git a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts b/packages/e2e_tests/tests/assetdb/interface-compliance.test.tsindex d5288c7a..c9efaa5b 100644--- a/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts+++ b/packages/e2e_tests/tests/assetdb/interface-compliance.test.ts@@ -1,7 +1,7 @@import * as fs from "fs";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { ASSET_TYPES, AssetStore } from "@karakeep/shared/assetdb";+import { ASSET_TYPES, AssetStore } from "@karakeep/shared-server/assetdb";import {assertAssetExists,diff --git a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.tsindex 36ff837f..a6a7e9ca 100644--- a/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts+++ b/packages/e2e_tests/tests/assetdb/local-filesystem-store.test.ts@@ -2,7 +2,7 @@ import * as fs from "fs";import * as path from "path";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { LocalFileSystemAssetStore } from "@karakeep/shared/assetdb";+import { LocalFileSystemAssetStore } from "@karakeep/shared-server/assetdb";import {assertAssetNotExists,diff --git a/packages/e2e_tests/tests/assetdb/s3-store.test.ts b/packages/e2e_tests/tests/assetdb/s3-store.test.tsindex c573750e..541b8791 100644--- a/packages/e2e_tests/tests/assetdb/s3-store.test.ts+++ b/packages/e2e_tests/tests/assetdb/s3-store.test.ts@@ -1,7 +1,7 @@import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3";import { afterEach, beforeEach, describe, expect, it } from "vitest";-import { S3AssetStore } from "@karakeep/shared/assetdb";+import { S3AssetStore } from "@karakeep/shared-server/assetdb";import {assertAssetExists,diff --git a/packages/shared/assetdb.ts b/packages/shared-server/assetdb.tssimilarity index 96%rename from packages/shared/assetdb.tsrename to packages/shared-server/assetdb.tsindex 2e22faf7..bb6bb75e 100644--- a/packages/shared/assetdb.ts+++ b/packages/shared-server/assetdb.ts@@ -11,12 +11,12 @@ import {PutObjectCommand,S3Client,} from "@aws-sdk/client-s3";-import { Glob } from "glob";+import glob from "glob";import { z } from "zod";-import serverConfig from "./config";-import logger from "./logger";-import { QuotaApproved } from "./storageQuota";+import serverConfig from "@karakeep/shared/config";+import logger from "@karakeep/shared/logger";+import { QuotaApproved } from "@karakeep/shared/storageQuota";const ROOT_PATH = serverConfig.assetsDir;@@ -283,18 +283,30 @@ class LocalFileSystemAssetStore implements AssetStore {}async *getAllAssets() {- const g = new Glob(`/**/**/asset.bin`, {- maxDepth: 3,- root: this.rootPath,- cwd: this.rootPath,- absolute: false,+ const files = await new Promise<string[]>((resolve, reject) => {+ glob(+ "*/*/asset.bin",+ {+ cwd: this.rootPath,+ nodir: true,+ },+ (err, matches) => {+ if (err) {+ reject(err);+ return;+ }+ resolve(matches);+ },+ );});- for await (const file of g) {++ for (const file of files) {const [userId, assetId] = file.split("/").slice(0, 2);const [size, metadata] = await Promise.all([this.getAssetSize({ userId, assetId }),this.readAssetMetadata({ userId, assetId }),]);+yield {userId,assetId,@@ -605,7 +617,7 @@ export { LocalFileSystemAssetStore, S3AssetStore };* Example usage of S3AssetStore:** import { S3Client } from "@aws-sdk/client-s3";- * import { S3AssetStore } from "@karakeep/shared/assetdb";+ * import { S3AssetStore } from "@karakeep/shared-server/assetdb";** const s3Client = new S3Client({* region: "us-east-1",diff --git a/packages/shared-server/package.json b/packages/shared-server/package.jsonindex 357248b4..0afc2bf2 100644--- a/packages/shared-server/package.json+++ b/packages/shared-server/package.json@@ -5,6 +5,7 @@"private": true,"type": "module","dependencies": {+ "@aws-sdk/client-s3": "^3.842.0","@karakeep/db": "workspace:^0.1.0","@karakeep/plugins": "workspace:^0.1.0","@karakeep/shared": "workspace:^0.1.0",@@ -13,7 +14,9 @@"@opentelemetry/resources": "^2.2.0","@opentelemetry/sdk-trace-base": "^2.2.0","@opentelemetry/sdk-trace-node": "^2.2.0",- "@opentelemetry/semantic-conventions": "^1.38.0"+ "@opentelemetry/semantic-conventions": "^1.38.0",+ "glob": "^11.0.0",+ "zod": "^3.24.2"},"devDependencies": {"@karakeep/prettier-config": "workspace:^0.1.0",@@ -29,7 +32,8 @@},"main": "index.ts","exports": {- ".": "./index.ts"+ ".": "./index.ts",+ "./assetdb": "./assetdb.ts"},"prettier": "@karakeep/prettier-config"}diff --git a/packages/shared/package.json b/packages/shared/package.jsonindex 93739354..a0f5608e 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -5,8 +5,6 @@"private": true,"type": "module","dependencies": {- "@aws-sdk/client-s3": "^3.842.0",- "glob": "^11.0.0","html-to-text": "^9.0.5","js-tiktoken": "^1.0.20","nodemailer": "^7.0.4",diff --git a/packages/trpc/models/assets.ts b/packages/trpc/models/assets.tsindex f97cfffb..63342896 100644--- a/packages/trpc/models/assets.ts+++ b/packages/trpc/models/assets.ts@@ -3,7 +3,7 @@ import { and, desc, eq, sql } from "drizzle-orm";import { z } from "zod";import { assets } from "@karakeep/db/schema";-import { deleteAsset } from "@karakeep/shared/assetdb";+import { deleteAsset } from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import { createSignedToken } from "@karakeep/shared/signedTokens";import { zAssetSignedTokenSchema } from "@karakeep/shared/types/assets";diff --git a/packages/trpc/models/backups.ts b/packages/trpc/models/backups.tsindex c7ab99ba..2fd17301 100644--- a/packages/trpc/models/backups.ts+++ b/packages/trpc/models/backups.ts@@ -4,7 +4,7 @@ import { z } from "zod";import { assets, backupsTable } from "@karakeep/db/schema";import { BackupQueue } from "@karakeep/shared-server";-import { deleteAsset } from "@karakeep/shared/assetdb";+import { deleteAsset } from "@karakeep/shared-server/assetdb";import { zBackupSchema } from "@karakeep/shared/types/backups";import { AuthedContext } from "..";diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.tsindex c8cd1f00..e0669787 100644--- a/packages/trpc/models/bookmarks.ts+++ b/packages/trpc/models/bookmarks.ts@@ -30,7 +30,7 @@ import {tagsOnBookmarks,} from "@karakeep/db/schema";import { SearchIndexingQueue, triggerWebhook } from "@karakeep/shared-server";-import { deleteAsset, readAsset } from "@karakeep/shared/assetdb";+import { deleteAsset, readAsset } from "@karakeep/shared-server/assetdb";import { getAlignedExpiry } from "@karakeep/shared/signedTokens";import {BookmarkTypes,diff --git a/packages/trpc/models/users.ts b/packages/trpc/models/users.tsindex 3340956a..c799f390 100644--- a/packages/trpc/models/users.ts+++ b/packages/trpc/models/users.ts@@ -18,7 +18,7 @@ import {users,verificationTokens,} from "@karakeep/db/schema";-import { deleteAsset, deleteUserAssets } from "@karakeep/shared/assetdb";+import { deleteAsset, deleteUserAssets } from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import {zResetPasswordSchema,diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.tsindex 782566cf..db29bf02 100644--- a/packages/trpc/routers/bookmarks.ts+++ b/packages/trpc/routers/bookmarks.ts@@ -27,7 +27,7 @@ import {triggerSearchReindex,triggerWebhook,} from "@karakeep/shared-server";-import { SUPPORTED_BOOKMARK_ASSET_TYPES } from "@karakeep/shared/assetdb";+import { SUPPORTED_BOOKMARK_ASSET_TYPES } from "@karakeep/shared-server/assetdb";import serverConfig from "@karakeep/shared/config";import { InferenceClientFactory } from "@karakeep/shared/inference";import { buildSummaryPrompt } from "@karakeep/shared/prompts.server";diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex e6e5c338..1f569a16 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1209,6 +1209,9 @@ importers:'@karakeep/shared':specifier: workspace:^0.1.0version: link:../shared+ '@karakeep/shared-server':+ specifier: workspace:^0.1.0+ version: link:../shared-server'@karakeep/trpc':specifier: workspace:^0.1.0version: link:../trpc@@ -1327,12 +1330,6 @@ importers:packages/shared:dependencies:- '@aws-sdk/client-s3':- specifier: ^3.842.0- version: 3.842.0- glob:- specifier: ^11.0.0- version: 11.0.2html-to-text:specifier: ^9.0.5version: 9.0.5@@ -1416,6 +1413,9 @@ importers:packages/shared-server:dependencies:+ '@aws-sdk/client-s3':+ specifier: ^3.842.0+ version: 3.842.0'@karakeep/db':specifier: workspace:^0.1.0version: link:../db@@ -1443,6 +1443,12 @@ importers:'@opentelemetry/semantic-conventions':specifier: ^1.38.0version: 1.38.0+ glob:+ specifier: ^11.0.0+ version: 11.1.0+ zod:+ specifier: ^3.24.2+ version: 3.24.2devDependencies:'@karakeep/prettier-config':specifier: workspace:^0.1.0@@ -4008,6 +4014,10 @@ packages:resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}engines: {node: '>=12'}+ '@isaacs/[email protected]':+ resolution: {integrity: sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==}+ engines: {node: '>=18'}+'@isaacs/[email protected]':resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==}engines: {node: '>=18.0.0'}@@ -9322,8 +9332,8 @@ packages:deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]hasBin: true- resolution: {integrity: sha512-YT7U7Vye+t5fZ/QMkBFrTJ7ZQxInIUjwyAjVj84CYXqgBdv30MFUPGnBR6sQaVq6Is15wYJUsnzTuWaGRBhBAQ==}+ resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==}engines: {node: 20 || >=22}deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]hasBin: true@@ -10135,8 +10145,8 @@ packages:resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}- resolution: {integrity: sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ==}+ resolution: {integrity: sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==}engines: {node: 20 || >=22}@@ -11312,10 +11322,6 @@ packages:resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==}- resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==}- engines: {node: 20 || >=22}-resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==}engines: {node: 20 || >=22}@@ -14412,12 +14418,12 @@ packages:resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}engines: {node: '>=18'}- deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]+ deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting [email protected]resolution: {integrity: sha512-ENg5JUHUm2rDD7IvKNFGzyElLXNjachNLp6RaGf4+JOgxXHkqA+gq81ZAMCUmtMtqBsoU62lcp6S27g1LCYGGQ==}engines: {node: '>=18'}- deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting [email protected]+ deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting [email protected]resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}@@ -15789,7 +15795,7 @@ snapshots:bidi-js: 1.0.3css-tree: 3.1.0is-potential-custom-element-name: 1.0.1- lru-cache: 11.2.2+ lru-cache: 11.2.4'@asamuzakjp/[email protected]':dependencies:@@ -19559,6 +19565,8 @@ snapshots:wrap-ansi: 8.1.0wrap-ansi-cjs: [email protected]+ '@isaacs/[email protected]': {}+'@isaacs/[email protected]':dependencies:minipass: 7.1.2@@ -25810,11 +25818,11 @@ snapshots:package-json-from-dist: 1.0.1path-scurry: 1.11.1dependencies:foreground-child: 3.3.1- jackspeak: 4.1.1- minimatch: 10.0.1+ jackspeak: 4.2.3+ minimatch: 10.1.1minipass: 7.1.2package-json-from-dist: 1.0.1path-scurry: 2.0.0@@ -26717,9 +26725,9 @@ snapshots:optionalDependencies:'@pkgjs/parseargs': 0.11.0dependencies:- '@isaacs/cliui': 8.0.2+ '@isaacs/cliui': 9.0.0dependencies:@@ -28742,10 +28750,6 @@ snapshots:- dependencies:- brace-expansion: 2.0.1-dependencies:'@isaacs/brace-expansion': 5.0.0@@ -29524,7 +29528,7 @@ snapshots:dependencies:- lru-cache: 11.2.2+ lru-cache: 11.2.4minipass: 7.1.2