A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
Reapply "fix(restate): use queue size from semaphore instead of admin API"
Mohamed Bassembf1963d5f2f5
diff --git a/packages/plugins/queue-restate/src/admin.ts b/packages/plugins/queue-restate/src/admin.tsindex dddc8f00..9c473144 100644--- a/packages/plugins/queue-restate/src/admin.ts+++ b/packages/plugins/queue-restate/src/admin.ts@@ -1,5 +1,18 @@import { z } from "zod";+const zStatus = z.enum([+ "pending",+ "scheduled",+ "ready",+ "running",+ "paused",+ "backing-off",+ "suspended",+ "completed",+]);++type InvocationStatus = z.infer<typeof zStatus>;+export class AdminClient {constructor(private addr: string) {}@@ -20,7 +33,9 @@ export class AdminClient {}}- async getStats(serviceName: string) {+ async getStats(+ serviceName: string,+ ): Promise<Record<InvocationStatus, number>> {const query = `select status, count(*) as count from sys_invocation where target_service_name='${serviceName}' group by status`;const res = await fetch(`${this.addr}/query`, {method: "POST",@@ -36,16 +51,7 @@ export class AdminClient {if (!res.ok) {throw new Error(`Failed to get stats: ${res.status}`);}- const zStatus = z.enum([- "pending",- "scheduled",- "ready",- "running",- "paused",- "backing-off",- "suspended",- "completed",- ]);+const zSchema = z.object({rows: z.array(z.object({@@ -69,7 +75,7 @@ export class AdminClient {"backing-off": 0,suspended: 0,completed: 0,- } as Record<z.infer<typeof zStatus>, number>,+ } as Record<InvocationStatus, number>,);}}diff --git a/packages/plugins/queue-restate/src/index.ts b/packages/plugins/queue-restate/src/index.tsindex f8761291..33c86fa7 100644--- a/packages/plugins/queue-restate/src/index.ts+++ b/packages/plugins/queue-restate/src/index.ts@@ -13,7 +13,6 @@ import type {} from "@karakeep/shared/queueing";import logger from "@karakeep/shared/logger";-import { AdminClient } from "./admin";import { envConfig } from "./env";import { idProvider } from "./idProvider";import { semaphore } from "./semaphore";@@ -23,7 +22,6 @@ class RestateQueueWrapper<T> implements Queue<T> {constructor(private readonly _name: string,private readonly client: restateClient.Ingress,- private readonly adminClient: AdminClient,public readonly opts: QueueOptions,) {}@@ -75,10 +73,15 @@ class RestateQueueWrapper<T> implements Queue<T> {running: number;failed: number;}> {- const res = await this.adminClient.getStats(this.name());+ const semaphoreId = `queue:${this.name()}`;+ const client = this.client.objectClient<typeof semaphore>(+ { name: "Semaphore" },+ semaphoreId,+ );+ const res = await client.queueSize();return {- pending: res.pending + res.ready,- pending_retry: res["backing-off"] + res.paused + res.suspended,+ pending: res.pending,+ pending_retry: 0,running: res.running,failed: 0,};@@ -123,7 +126,6 @@ class RestateRunnerWrapper<T> implements Runner<T> {class RestateQueueClient implements QueueClient {private client: restateClient.Ingress;- private adminClient: AdminClient;private queues = new Map<string, RestateQueueWrapper<unknown>>();private services = new Map<string, RestateRunnerWrapper<unknown>>();@@ -131,7 +133,6 @@ class RestateQueueClient implements QueueClient {this.client = restateClient.connect({url: envConfig.RESTATE_INGRESS_ADDR,});- this.adminClient = new AdminClient(envConfig.RESTATE_ADMIN_ADDR);}async prepare(): Promise<void> {@@ -175,12 +176,7 @@ class RestateQueueClient implements QueueClient {if (this.queues.has(name)) {throw new Error(`Queue ${name} already exists`);}- const wrapper = new RestateQueueWrapper<T>(- name,- this.client,- this.adminClient,- opts,- );+ const wrapper = new RestateQueueWrapper<T>(name, this.client, opts);this.queues.set(name, wrapper);return wrapper;}diff --git a/packages/plugins/queue-restate/src/semaphore.ts b/packages/plugins/queue-restate/src/semaphore.tsindex 72fa938a..0227e8f0 100644--- a/packages/plugins/queue-restate/src/semaphore.ts+++ b/packages/plugins/queue-restate/src/semaphore.ts@@ -1,7 +1,12 @@// Inspired from https://github.com/restatedev/examples/blob/main/typescript/patterns-use-cases/src/priorityqueue/queue.tsimport * as restate from "@restatedev/restate-sdk";-import { Context, object, ObjectContext } from "@restatedev/restate-sdk";+import {+ Context,+ object,+ ObjectContext,+ ObjectSharedContext,+} from "@restatedev/restate-sdk";interface QueueItem {awakeable: string;@@ -130,6 +135,39 @@ export const semaphore = object({setState(ctx, state);},),+ queueSize: restate.handlers.object.shared(+ {},+ async (+ ctx: ObjectSharedContext<LegacyQueueState>,+ ): Promise<{+ pending: number;+ running: number;+ }> => {+ const groups = (await ctx.get("itemsv2")) as Record<+ string,+ GroupState+ > | null;+ const leases = (await ctx.get("leases")) as Record<+ string,+ number+ > | null;++ let pending = 0;+ for (const group of Object.values(groups ?? {})) {+ pending += group.items.length;+ }++ const now = Date.now();+ let running = 0;+ for (const expiry of Object.values(leases ?? {})) {+ if (expiry > now) {+ running++;+ }+ }++ return { pending, running };+ },+ ),},options: {journalRetention: 0,diff --git a/packages/plugins/queue-restate/src/tests/queue.test.ts b/packages/plugins/queue-restate/src/tests/queue.test.tsindex 3c4cf254..fafde598 100644--- a/packages/plugins/queue-restate/src/tests/queue.test.ts+++ b/packages/plugins/queue-restate/src/tests/queue.test.ts@@ -82,8 +82,16 @@ describe("Restate Queue Provider", () => {async function waitUntilQueueEmpty() {await waitUntil(async () => {- const stats = await queue.stats();- return stats.pending + stats.pending_retry + stats.running === 0;+ const stats = await adminClient.getStats(queue.name());+ return (+ stats.pending ++ stats.ready ++ stats.running ++ stats["backing-off"] ++ stats.paused ++ stats.suspended ===+ 0+ );},"Queue to be empty",60000,