A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
deps: upgrade liteque to 0.9.1 and move more transactions to begin immediate
Mohamed Bassemcc9328484042
diff --git a/apps/workers/package.json b/apps/workers/package.jsonindex 1f2c8eee..43d3804e 100644--- a/apps/workers/package.json+++ b/apps/workers/package.json@@ -30,7 +30,7 @@"https-proxy-agent": "^7.0.6","ipaddr.js": "^2.2.0","jsdom": "^24.0.0",- "liteque": "^0.9.0",+ "liteque": "^0.9.1","lru-cache": "^11.2.2","metascraper": "^5.50.0","metascraper-amazon": "^5.50.0",diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.tsindex 7b049b68..a6349f37 100644--- a/apps/workers/workers/inference/tagging.ts+++ b/apps/workers/workers/inference/tagging.ts@@ -420,90 +420,95 @@ async function connectTags(return;}- const res = await db.transaction((tx) => {- // Attempt to match exiting tags with the new ones- const { matchedTagIds, notFoundTagNames } = (() => {- const { normalizeTag } = tagNormalizer();- const normalizedInferredTags = inferredTags.map((t) => ({- originalTag: t,- normalizedTag: normalizeTag(t),- }));-- const matchedTags = tx.query.bookmarkTags- .findMany({- where: and(- eq(bookmarkTags.userId, userId),- inArray(- bookmarkTags.normalizedName,- normalizedInferredTags.map((t) => t.normalizedTag),+ // This transaction reads before writing, so reserve the writer slot before+ // taking a WAL snapshot that another connection could invalidate.+ const res = await db.transaction(+ (tx) => {+ // Attempt to match exiting tags with the new ones+ const { matchedTagIds, notFoundTagNames } = (() => {+ const { normalizeTag } = tagNormalizer();+ const normalizedInferredTags = inferredTags.map((t) => ({+ originalTag: t,+ normalizedTag: normalizeTag(t),+ }));++ const matchedTags = tx.query.bookmarkTags+ .findMany({+ where: and(+ eq(bookmarkTags.userId, userId),+ inArray(+ bookmarkTags.normalizedName,+ normalizedInferredTags.map((t) => t.normalizedTag),+ ),),- ),- })- .sync();-- const matchedTagIds = matchedTags.map((r) => r.id);- const notFoundTagNames = normalizedInferredTags- .filter(- (t) =>- !matchedTags.some(- (mt) => normalizeTag(mt.name) === t.normalizedTag,- ),- )- .map((t) => t.originalTag);-- return { matchedTagIds, notFoundTagNames };- })();-- // Create tags that didn't exist previously- let newTagIds: string[] = [];- if (notFoundTagNames.length > 0) {- newTagIds = tx- .insert(bookmarkTags)- .values(- notFoundTagNames.map((t) => ({- name: t,- userId,- })),- )- .onConflictDoNothing()- .returning()- .all()- .map((t) => t.id);- }+ })+ .sync();++ const matchedTagIds = matchedTags.map((r) => r.id);+ const notFoundTagNames = normalizedInferredTags+ .filter(+ (t) =>+ !matchedTags.some(+ (mt) => normalizeTag(mt.name) === t.normalizedTag,+ ),+ )+ .map((t) => t.originalTag);++ return { matchedTagIds, notFoundTagNames };+ })();++ // Create tags that didn't exist previously+ let newTagIds: string[] = [];+ if (notFoundTagNames.length > 0) {+ newTagIds = tx+ .insert(bookmarkTags)+ .values(+ notFoundTagNames.map((t) => ({+ name: t,+ userId,+ })),+ )+ .onConflictDoNothing()+ .returning()+ .all()+ .map((t) => t.id);+ }- // Delete old AI tags- const detachedTags = tx- .delete(tagsOnBookmarks)- .where(- and(- eq(tagsOnBookmarks.attachedBy, "ai"),- eq(tagsOnBookmarks.bookmarkId, bookmarkId),- ),- )- .returning()- .all();-- const allTagIds = new Set([...matchedTagIds, ...newTagIds]);-- // Attach new ones- let attachedTags: { tagId: string; bookmarkId: string }[] = [];- if (allTagIds.size > 0) {- attachedTags = tx- .insert(tagsOnBookmarks)- .values(- [...allTagIds].map((tagId) => ({- tagId,- bookmarkId,- attachedBy: "ai" as const,- })),+ // Delete old AI tags+ const detachedTags = tx+ .delete(tagsOnBookmarks)+ .where(+ and(+ eq(tagsOnBookmarks.attachedBy, "ai"),+ eq(tagsOnBookmarks.bookmarkId, bookmarkId),+ ),)- .onConflictDoNothing().returning().all();- }- return { detachedTags, attachedTags };- });+ const allTagIds = new Set([...matchedTagIds, ...newTagIds]);++ // Attach new ones+ let attachedTags: { tagId: string; bookmarkId: string }[] = [];+ if (allTagIds.size > 0) {+ attachedTags = tx+ .insert(tagsOnBookmarks)+ .values(+ [...allTagIds].map((tagId) => ({+ tagId,+ bookmarkId,+ attachedBy: "ai" as const,+ })),+ )+ .onConflictDoNothing()+ .returning()+ .all();+ }++ return { detachedTags, attachedTags };+ },+ { behavior: "immediate" },+ );await RuleEngine.triggerOnEvent(userId, bookmarkId, [...res.detachedTags.map((t) => ({diff --git a/packages/plugins/package.json b/packages/plugins/package.jsonindex c39f864e..b068b015 100644--- a/packages/plugins/package.json+++ b/packages/plugins/package.json@@ -31,7 +31,7 @@"@restatedev/restate-sdk-clients": "^1.16.2","async-mutex": "^0.4.1","glob": "^11.0.0",- "liteque": "^0.9.0",+ "liteque": "^0.9.1","meilisearch": "^0.58.0","redis": "^5.11.0"},diff --git a/packages/trpc/models/importSessions.repo.ts b/packages/trpc/models/importSessions.repo.tsindex 09d0d604..a9b9d37b 100644--- a/packages/trpc/models/importSessions.repo.ts+++ b/packages/trpc/models/importSessions.repo.ts@@ -110,68 +110,73 @@ export class ImportSessionsRepo {// One transaction per session to keep write locks short; a large backlog// (e.g. the first sweep after deploy) shouldn't block other writers.+ // Each transaction reads before writing, so reserve the writer slot before+ // taking a WAL snapshot that another connection could invalidate.let archivedCount = 0;for (const session of sessions) {- const archived = await this.db.transaction((tx) => {- const statusCounts = tx- .select({- status: importStagingBookmarks.status,- count: count(),- })- .from(importStagingBookmarks)- .where(eq(importStagingBookmarks.importSessionId, session.id))- .groupBy(importStagingBookmarks.status)- .all();-- const stats = {- totalBookmarks: 0,- completedBookmarks: 0,- failedBookmarks: 0,- pendingBookmarks: 0,- processingBookmarks: 0,- };-- for (const { status, count: itemCount } of statusCounts) {- stats.totalBookmarks += itemCount;- switch (status) {- case "pending":- stats.pendingBookmarks += itemCount;- break;- case "processing":- stats.processingBookmarks += itemCount;- break;- case "completed":- stats.completedBookmarks += itemCount;- break;- case "failed":- stats.failedBookmarks += itemCount;- break;+ const archived = await this.db.transaction(+ (tx) => {+ const statusCounts = tx+ .select({+ status: importStagingBookmarks.status,+ count: count(),+ })+ .from(importStagingBookmarks)+ .where(eq(importStagingBookmarks.importSessionId, session.id))+ .groupBy(importStagingBookmarks.status)+ .all();++ const stats = {+ totalBookmarks: 0,+ completedBookmarks: 0,+ failedBookmarks: 0,+ pendingBookmarks: 0,+ processingBookmarks: 0,+ };++ for (const { status, count: itemCount } of statusCounts) {+ stats.totalBookmarks += itemCount;+ switch (status) {+ case "pending":+ stats.pendingBookmarks += itemCount;+ break;+ case "processing":+ stats.processingBookmarks += itemCount;+ break;+ case "completed":+ stats.completedBookmarks += itemCount;+ break;+ case "failed":+ stats.failedBookmarks += itemCount;+ break;+ }}- }-- const result = tx- .update(importSessions)- .set({ status: "archived", ...stats })- .where(- and(- eq(importSessions.id, session.id),- eq(importSessions.status, "completed"),- ),- )- .run();-- if (result.changes === 0) {- return false;- }-- tx.delete(importStagingBookmarks)- .where(eq(importStagingBookmarks.importSessionId, session.id))- .run();- tx.delete(importSessionBookmarks)- .where(eq(importSessionBookmarks.importSessionId, session.id))- .run();- return true;- });++ const result = tx+ .update(importSessions)+ .set({ status: "archived", ...stats })+ .where(+ and(+ eq(importSessions.id, session.id),+ eq(importSessions.status, "completed"),+ ),+ )+ .run();++ if (result.changes === 0) {+ return false;+ }++ tx.delete(importStagingBookmarks)+ .where(eq(importStagingBookmarks.importSessionId, session.id))+ .run();+ tx.delete(importSessionBookmarks)+ .where(eq(importSessionBookmarks.importSessionId, session.id))+ .run();+ return true;+ },+ { behavior: "immediate" },+ );if (archived) {archivedCount++;diff --git a/packages/trpc/models/users.ts b/packages/trpc/models/users.tsindex a0d52010..0ad1e926 100644--- a/packages/trpc/models/users.ts+++ b/packages/trpc/models/users.ts@@ -102,48 +102,53 @@ export class User {emailVerified?: Date | null;},) {- return await db.transaction((trx) => {- let userRole = input.role;- if (!userRole) {- const [{ count: userCount }] = trx- .select({ count: count() })- .from(users)- .all();- userRole = userCount === 0 ? "admin" : "user";- }+ // This transaction reads before writing, so reserve the writer slot before+ // taking a WAL snapshot that another connection could invalidate.+ return await db.transaction(+ (trx) => {+ let userRole = input.role;+ if (!userRole) {+ const [{ count: userCount }] = trx+ .select({ count: count() })+ .from(users)+ .all();+ userRole = userCount === 0 ? "admin" : "user";+ }- try {- const [result] = trx- .insert(users)- .values({- name: input.name,- email: input.email,- password: input.password,- salt: input.salt,- role: userRole,- emailVerified: input.emailVerified,- bookmarkQuota: serverConfig.quotas.free.bookmarkLimit,- storageQuota: serverConfig.quotas.free.assetSizeBytes,- })- .returning()- .all();-- return result;- } catch (e) {- if (e instanceof SqliteError) {- if (e.code === "SQLITE_CONSTRAINT_UNIQUE") {- throw new TRPCError({- code: "BAD_REQUEST",- message: "Email is already taken",- });+ try {+ const [result] = trx+ .insert(users)+ .values({+ name: input.name,+ email: input.email,+ password: input.password,+ salt: input.salt,+ role: userRole,+ emailVerified: input.emailVerified,+ bookmarkQuota: serverConfig.quotas.free.bookmarkLimit,+ storageQuota: serverConfig.quotas.free.assetSizeBytes,+ })+ .returning()+ .all();++ return result;+ } catch (e) {+ if (e instanceof SqliteError) {+ if (e.code === "SQLITE_CONSTRAINT_UNIQUE") {+ throw new TRPCError({+ code: "BAD_REQUEST",+ message: "Email is already taken",+ });+ }}+ throw new TRPCError({+ code: "INTERNAL_SERVER_ERROR",+ message: "Something went wrong",+ });}- throw new TRPCError({- code: "INTERNAL_SERVER_ERROR",- message: "Something went wrong",- });- }- });+ },+ { behavior: "immediate" },+ );}static async getAll(ctx: AuthedContext): Promise<User[]> {diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 7a27d056..618a47ae 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -973,8 +973,8 @@ importers:specifier: ^24.0.0version: 24.1.3liteque:- specifier: ^0.9.0- version: 0.9.0(@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])+ specifier: ^0.9.1+ version: 0.9.1(@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])lru-cache:specifier: ^11.2.2version: 11.2.2@@ -1355,8 +1355,8 @@ importers:specifier: ^11.0.0version: 11.0.2liteque:- specifier: ^0.9.0- version: 0.9.0(@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])+ specifier: ^0.9.1+ version: 0.9.1(@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])meilisearch:specifier: ^0.58.0version: 0.58.0@@ -13027,8 +13027,8 @@ packages:resolution: {integrity: sha512-wUayTU8MS827Dam6MxgD72Ui+KOSF+u/eIqpatOtjnvgJ0+mnDq33uC2M7J0tPK+upe/DpUAuK4JUU89iBoNKQ==}engines: {node: '>=4'}- resolution: {integrity: sha512-iHJbsj/IdjicdstEx6yq3IgZnD+zl+KXGguMxq2XDjg6fuPns87hbeNrMIId4V0lcDDLMGnJLMj5b5Y9VLls8A==}+ resolution: {integrity: sha512-llCytWbhLfrUtljGgEhOuMpK/+PdI4b8seMesJfC4xvH7rs6vhEgkuI6HXkIFR4HP8uoMb1I9qEHoHqSfLakiw==}peerDependencies:better-sqlite3: ^13.0.3@@ -32433,7 +32433,7 @@ snapshots:- [email protected](@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected]):+ [email protected](@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected]):dependencies:async-mutex: 0.4.1better-sqlite3: 13.0.3diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex c17328bd..b0ba3584 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -48,7 +48,7 @@ minimumReleaseAgeExclude:"@next/[email protected]","@next/[email protected]",]# Pin vite so adding vitest as a direct devDep to apps/mcp doesn't hoist