Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

deps: upgrade liteque to 0.9.1 and move more transactions to begin immediate

Mohamed Bassemcc9328484042
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 1f2c8eee..43d3804e 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -30,7 +30,7 @@
"https-proxy-agent": "^7.0.6",
"ipaddr.js": "^2.2.0",
"jsdom": "^24.0.0",
- "liteque": "^0.9.0",
+ "liteque": "^0.9.1",
"lru-cache": "^11.2.2",
"metascraper": "^5.50.0",
"metascraper-amazon": "^5.50.0",
diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.ts
index 7b049b68..a6349f37 100644
--- a/apps/workers/workers/inference/tagging.ts
+++ b/apps/workers/workers/inference/tagging.ts
@@ -420,90 +420,95 @@ async function connectTags(
return;
}
- const res = await db.transaction((tx) => {
- // Attempt to match exiting tags with the new ones
- const { matchedTagIds, notFoundTagNames } = (() => {
- const { normalizeTag } = tagNormalizer();
- const normalizedInferredTags = inferredTags.map((t) => ({
- originalTag: t,
- normalizedTag: normalizeTag(t),
- }));
-
- const matchedTags = tx.query.bookmarkTags
- .findMany({
- where: and(
- eq(bookmarkTags.userId, userId),
- inArray(
- bookmarkTags.normalizedName,
- normalizedInferredTags.map((t) => t.normalizedTag),
+ // This transaction reads before writing, so reserve the writer slot before
+ // taking a WAL snapshot that another connection could invalidate.
+ const res = await db.transaction(
+ (tx) => {
+ // Attempt to match exiting tags with the new ones
+ const { matchedTagIds, notFoundTagNames } = (() => {
+ const { normalizeTag } = tagNormalizer();
+ const normalizedInferredTags = inferredTags.map((t) => ({
+ originalTag: t,
+ normalizedTag: normalizeTag(t),
+ }));
+
+ const matchedTags = tx.query.bookmarkTags
+ .findMany({
+ where: and(
+ eq(bookmarkTags.userId, userId),
+ inArray(
+ bookmarkTags.normalizedName,
+ normalizedInferredTags.map((t) => t.normalizedTag),
+ ),
),
- ),
- })
- .sync();
-
- const matchedTagIds = matchedTags.map((r) => r.id);
- const notFoundTagNames = normalizedInferredTags
- .filter(
- (t) =>
- !matchedTags.some(
- (mt) => normalizeTag(mt.name) === t.normalizedTag,
- ),
- )
- .map((t) => t.originalTag);
-
- return { matchedTagIds, notFoundTagNames };
- })();
-
- // Create tags that didn't exist previously
- let newTagIds: string[] = [];
- if (notFoundTagNames.length > 0) {
- newTagIds = tx
- .insert(bookmarkTags)
- .values(
- notFoundTagNames.map((t) => ({
- name: t,
- userId,
- })),
- )
- .onConflictDoNothing()
- .returning()
- .all()
- .map((t) => t.id);
- }
+ })
+ .sync();
+
+ const matchedTagIds = matchedTags.map((r) => r.id);
+ const notFoundTagNames = normalizedInferredTags
+ .filter(
+ (t) =>
+ !matchedTags.some(
+ (mt) => normalizeTag(mt.name) === t.normalizedTag,
+ ),
+ )
+ .map((t) => t.originalTag);
+
+ return { matchedTagIds, notFoundTagNames };
+ })();
+
+ // Create tags that didn't exist previously
+ let newTagIds: string[] = [];
+ if (notFoundTagNames.length > 0) {
+ newTagIds = tx
+ .insert(bookmarkTags)
+ .values(
+ notFoundTagNames.map((t) => ({
+ name: t,
+ userId,
+ })),
+ )
+ .onConflictDoNothing()
+ .returning()
+ .all()
+ .map((t) => t.id);
+ }
- // Delete old AI tags
- const detachedTags = tx
- .delete(tagsOnBookmarks)
- .where(
- and(
- eq(tagsOnBookmarks.attachedBy, "ai"),
- eq(tagsOnBookmarks.bookmarkId, bookmarkId),
- ),
- )
- .returning()
- .all();
-
- const allTagIds = new Set([...matchedTagIds, ...newTagIds]);
-
- // Attach new ones
- let attachedTags: { tagId: string; bookmarkId: string }[] = [];
- if (allTagIds.size > 0) {
- attachedTags = tx
- .insert(tagsOnBookmarks)
- .values(
- [...allTagIds].map((tagId) => ({
- tagId,
- bookmarkId,
- attachedBy: "ai" as const,
- })),
+ // Delete old AI tags
+ const detachedTags = tx
+ .delete(tagsOnBookmarks)
+ .where(
+ and(
+ eq(tagsOnBookmarks.attachedBy, "ai"),
+ eq(tagsOnBookmarks.bookmarkId, bookmarkId),
+ ),
)
- .onConflictDoNothing()
.returning()
.all();
- }
- return { detachedTags, attachedTags };
- });
+ const allTagIds = new Set([...matchedTagIds, ...newTagIds]);
+
+ // Attach new ones
+ let attachedTags: { tagId: string; bookmarkId: string }[] = [];
+ if (allTagIds.size > 0) {
+ attachedTags = tx
+ .insert(tagsOnBookmarks)
+ .values(
+ [...allTagIds].map((tagId) => ({
+ tagId,
+ bookmarkId,
+ attachedBy: "ai" as const,
+ })),
+ )
+ .onConflictDoNothing()
+ .returning()
+ .all();
+ }
+
+ return { detachedTags, attachedTags };
+ },
+ { behavior: "immediate" },
+ );
await RuleEngine.triggerOnEvent(userId, bookmarkId, [
...res.detachedTags.map((t) => ({
diff --git a/packages/plugins/package.json b/packages/plugins/package.json
index c39f864e..b068b015 100644
--- a/packages/plugins/package.json
+++ b/packages/plugins/package.json
@@ -31,7 +31,7 @@
"@restatedev/restate-sdk-clients": "^1.16.2",
"async-mutex": "^0.4.1",
"glob": "^11.0.0",
- "liteque": "^0.9.0",
+ "liteque": "^0.9.1",
"meilisearch": "^0.58.0",
"redis": "^5.11.0"
},
diff --git a/packages/trpc/models/importSessions.repo.ts b/packages/trpc/models/importSessions.repo.ts
index 09d0d604..a9b9d37b 100644
--- a/packages/trpc/models/importSessions.repo.ts
+++ b/packages/trpc/models/importSessions.repo.ts
@@ -110,68 +110,73 @@ export class ImportSessionsRepo {
// One transaction per session to keep write locks short; a large backlog
// (e.g. the first sweep after deploy) shouldn't block other writers.
+ // Each transaction reads before writing, so reserve the writer slot before
+ // taking a WAL snapshot that another connection could invalidate.
let archivedCount = 0;
for (const session of sessions) {
- const archived = await this.db.transaction((tx) => {
- const statusCounts = tx
- .select({
- status: importStagingBookmarks.status,
- count: count(),
- })
- .from(importStagingBookmarks)
- .where(eq(importStagingBookmarks.importSessionId, session.id))
- .groupBy(importStagingBookmarks.status)
- .all();
-
- const stats = {
- totalBookmarks: 0,
- completedBookmarks: 0,
- failedBookmarks: 0,
- pendingBookmarks: 0,
- processingBookmarks: 0,
- };
-
- for (const { status, count: itemCount } of statusCounts) {
- stats.totalBookmarks += itemCount;
- switch (status) {
- case "pending":
- stats.pendingBookmarks += itemCount;
- break;
- case "processing":
- stats.processingBookmarks += itemCount;
- break;
- case "completed":
- stats.completedBookmarks += itemCount;
- break;
- case "failed":
- stats.failedBookmarks += itemCount;
- break;
+ const archived = await this.db.transaction(
+ (tx) => {
+ const statusCounts = tx
+ .select({
+ status: importStagingBookmarks.status,
+ count: count(),
+ })
+ .from(importStagingBookmarks)
+ .where(eq(importStagingBookmarks.importSessionId, session.id))
+ .groupBy(importStagingBookmarks.status)
+ .all();
+
+ const stats = {
+ totalBookmarks: 0,
+ completedBookmarks: 0,
+ failedBookmarks: 0,
+ pendingBookmarks: 0,
+ processingBookmarks: 0,
+ };
+
+ for (const { status, count: itemCount } of statusCounts) {
+ stats.totalBookmarks += itemCount;
+ switch (status) {
+ case "pending":
+ stats.pendingBookmarks += itemCount;
+ break;
+ case "processing":
+ stats.processingBookmarks += itemCount;
+ break;
+ case "completed":
+ stats.completedBookmarks += itemCount;
+ break;
+ case "failed":
+ stats.failedBookmarks += itemCount;
+ break;
+ }
}
- }
-
- const result = tx
- .update(importSessions)
- .set({ status: "archived", ...stats })
- .where(
- and(
- eq(importSessions.id, session.id),
- eq(importSessions.status, "completed"),
- ),
- )
- .run();
-
- if (result.changes === 0) {
- return false;
- }
-
- tx.delete(importStagingBookmarks)
- .where(eq(importStagingBookmarks.importSessionId, session.id))
- .run();
- tx.delete(importSessionBookmarks)
- .where(eq(importSessionBookmarks.importSessionId, session.id))
- .run();
- return true;
- });
+
+ const result = tx
+ .update(importSessions)
+ .set({ status: "archived", ...stats })
+ .where(
+ and(
+ eq(importSessions.id, session.id),
+ eq(importSessions.status, "completed"),
+ ),
+ )
+ .run();
+
+ if (result.changes === 0) {
+ return false;
+ }
+
+ tx.delete(importStagingBookmarks)
+ .where(eq(importStagingBookmarks.importSessionId, session.id))
+ .run();
+ tx.delete(importSessionBookmarks)
+ .where(eq(importSessionBookmarks.importSessionId, session.id))
+ .run();
+ return true;
+ },
+ { behavior: "immediate" },
+ );
if (archived) {
archivedCount++;
diff --git a/packages/trpc/models/users.ts b/packages/trpc/models/users.ts
index a0d52010..0ad1e926 100644
--- a/packages/trpc/models/users.ts
+++ b/packages/trpc/models/users.ts
@@ -102,48 +102,53 @@ export class User {
emailVerified?: Date | null;
},
) {
- return await db.transaction((trx) => {
- let userRole = input.role;
- if (!userRole) {
- const [{ count: userCount }] = trx
- .select({ count: count() })
- .from(users)
- .all();
- userRole = userCount === 0 ? "admin" : "user";
- }
+ // This transaction reads before writing, so reserve the writer slot before
+ // taking a WAL snapshot that another connection could invalidate.
+ return await db.transaction(
+ (trx) => {
+ let userRole = input.role;
+ if (!userRole) {
+ const [{ count: userCount }] = trx
+ .select({ count: count() })
+ .from(users)
+ .all();
+ userRole = userCount === 0 ? "admin" : "user";
+ }
- try {
- const [result] = trx
- .insert(users)
- .values({
- name: input.name,
- email: input.email,
- password: input.password,
- salt: input.salt,
- role: userRole,
- emailVerified: input.emailVerified,
- bookmarkQuota: serverConfig.quotas.free.bookmarkLimit,
- storageQuota: serverConfig.quotas.free.assetSizeBytes,
- })
- .returning()
- .all();
-
- return result;
- } catch (e) {
- if (e instanceof SqliteError) {
- if (e.code === "SQLITE_CONSTRAINT_UNIQUE") {
- throw new TRPCError({
- code: "BAD_REQUEST",
- message: "Email is already taken",
- });
+ try {
+ const [result] = trx
+ .insert(users)
+ .values({
+ name: input.name,
+ email: input.email,
+ password: input.password,
+ salt: input.salt,
+ role: userRole,
+ emailVerified: input.emailVerified,
+ bookmarkQuota: serverConfig.quotas.free.bookmarkLimit,
+ storageQuota: serverConfig.quotas.free.assetSizeBytes,
+ })
+ .returning()
+ .all();
+
+ return result;
+ } catch (e) {
+ if (e instanceof SqliteError) {
+ if (e.code === "SQLITE_CONSTRAINT_UNIQUE") {
+ throw new TRPCError({
+ code: "BAD_REQUEST",
+ message: "Email is already taken",
+ });
+ }
}
+ throw new TRPCError({
+ code: "INTERNAL_SERVER_ERROR",
+ message: "Something went wrong",
+ });
}
- throw new TRPCError({
- code: "INTERNAL_SERVER_ERROR",
- message: "Something went wrong",
- });
- }
- });
+ },
+ { behavior: "immediate" },
+ );
}
static async getAll(ctx: AuthedContext): Promise<User[]> {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7a27d056..618a47ae 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -973,8 +973,8 @@ importers:
specifier: ^24.0.0
version: 24.1.3
liteque:
- specifier: ^0.9.0
+ specifier: ^0.9.1
lru-cache:
specifier: ^11.2.2
version: 11.2.2
@@ -1355,8 +1355,8 @@ importers:
specifier: ^11.0.0
version: 11.0.2
liteque:
- specifier: ^0.9.0
+ specifier: ^0.9.1
meilisearch:
specifier: ^0.58.0
version: 0.58.0
@@ -13027,8 +13027,8 @@ packages:
resolution: {integrity: sha512-wUayTU8MS827Dam6MxgD72Ui+KOSF+u/eIqpatOtjnvgJ0+mnDq33uC2M7J0tPK+upe/DpUAuK4JUU89iBoNKQ==}
engines: {node: '>=4'}
- resolution: {integrity: sha512-iHJbsj/IdjicdstEx6yq3IgZnD+zl+KXGguMxq2XDjg6fuPns87hbeNrMIId4V0lcDDLMGnJLMj5b5Y9VLls8A==}
+ resolution: {integrity: sha512-llCytWbhLfrUtljGgEhOuMpK/+PdI4b8seMesJfC4xvH7rs6vhEgkuI6HXkIFR4HP8uoMb1I9qEHoHqSfLakiw==}
peerDependencies:
better-sqlite3: ^13.0.3
@@ -32433,7 +32433,7 @@ snapshots:
dependencies:
async-mutex: 0.4.1
better-sqlite3: 13.0.3
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index c17328bd..b0ba3584 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -48,7 +48,7 @@ minimumReleaseAgeExclude:
]
# Pin vite so adding vitest as a direct devDep to apps/mcp doesn't hoist