Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

Refactor sync server with async iterators (#1913)

Bogdan Chadkin6440b79369a8
diff --git a/apps/builder/app/builder/shared/sync/index.ts b/apps/builder/app/builder/shared/sync/index.ts
index 118a3473fe..e60e4334c1 100644
--- a/apps/builder/app/builder/shared/sync/index.ts
+++ b/apps/builder/app/builder/shared/sync/index.ts
@@ -1,2 +1 @@
-export { useSyncServer } from "./sync-server";
-export { queueStatus } from "./queue";
+export { useSyncServer, queueStatus } from "./sync-server";
diff --git a/apps/builder/app/builder/shared/sync/queue.test.ts b/apps/builder/app/builder/shared/sync/queue.test.ts
deleted file mode 100644
index 81e0a5172d..0000000000
--- a/apps/builder/app/builder/shared/sync/queue.test.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-import { describe, afterEach, test, expect } from "@jest/globals";
-import { dequeue, enqueue, state, queueStatus } from "./queue";
-
-describe("queue", () => {
- afterEach(() => {
- state.queue = [];
- state.failedAttempts = 0;
- queueStatus.set("idle");
- });
-
- test("dequeue empty", () => {
- dequeue();
- expect(queueStatus.get()).toBe("idle");
- });
-
- test("enqueue with success", async () => {
- enqueue(() => Promise.resolve({ ok: true }));
- const jobPromise = dequeue();
- expect(queueStatus.get()).toBe("running");
- expect(state.failedAttempts).toBe(0);
- expect(state.queue.length).toBe(0);
- await jobPromise;
- expect(queueStatus.get()).toBe("idle");
- expect(state.failedAttempts).toBe(0);
- expect(state.queue.length).toBe(0);
- });
-
- test("enqueue with failure", async () => {
- enqueue(() => Promise.resolve({ ok: false, retry: true }));
- const jobPromise = dequeue();
- expect(queueStatus.get()).toBe("running");
- expect(state.failedAttempts).toBe(0);
- expect(state.queue.length).toBe(0);
-
- await Promise.allSettled([jobPromise]);
- expect(queueStatus.get()).toBe("recovering");
- expect(state.failedAttempts).toBe(1);
- expect(state.queue.length).toBe(1);
- });
-
- test("recovering > failed > idle", async () => {
- let response = Promise.resolve({ ok: false });
- await enqueue(async () => {
- const { ok } = await response;
- if (ok) {
- return { ok: true };
- }
- return { ok: false, retry: true };
- });
-
- expect(queueStatus.get()).toBe("recovering");
-
- await dequeue();
- await dequeue();
- await dequeue();
- await dequeue();
-
- expect(state.queue.length).toBe(1);
- expect(queueStatus.get()).toBe("failed");
- expect(state.failedAttempts).toBe(5);
-
- response = Promise.resolve({ ok: true });
- await dequeue();
-
- expect(state.queue.length).toBe(0);
- expect(queueStatus.get()).toBe("idle");
- expect(state.failedAttempts).toBe(0);
- });
-});
diff --git a/apps/builder/app/builder/shared/sync/queue.ts b/apps/builder/app/builder/shared/sync/queue.ts
deleted file mode 100644
index 3a307cfdfd..0000000000
--- a/apps/builder/app/builder/shared/sync/queue.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-import { atom } from "nanostores";
-
-// After this amount of retries without success, we consider connection status error.
-const MAX_RETRY_RECOVERY = 5;
-
-export type QueueStatus = "running" | "idle" | "recovering" | "failed";
-
-type BaseResult = { ok: true } | { ok: false; retry: boolean };
-
-export const queueStatus = atom<QueueStatus>("idle");
-
-export const state: {
- queue: Array<Job<BaseResult>>;
- failedAttempts: number;
-} = {
- queue: [],
- failedAttempts: 0,
-};
-
-type Job<Result extends BaseResult> = () => Promise<Result>;
-
-export const enqueue = <Result extends BaseResult>(job: Job<Result>) => {
- state.queue.push(job);
- if (queueStatus.get() === "idle") {
- return dequeue();
- }
-};
-
-// We can't change to running or idle status from error or recovering status until
-// we have one successful attempt.
-const getStatus = (phase: "start" | "end"): QueueStatus => {
- if (state.failedAttempts > 0) {
- if (state.failedAttempts < MAX_RETRY_RECOVERY) {
- return "recovering";
- }
- return "failed";
- }
- return phase === "start" ? "running" : "idle";
-};
-
-export const dequeue = () => {
- const job = state.queue.shift();
-
- if (job === undefined) {
- return;
- }
-
- queueStatus.set(getStatus("start"));
-
- return job()
- .then((result) => {
- if (result.ok === false) {
- if (result.retry) {
- throw Error("Bad response");
- }
- }
- state.failedAttempts = 0;
- queueStatus.set(getStatus("end"));
- // When the job was successful, we don't need to wait, we can attempt running another job immediately.
- dequeue();
- })
- .catch(() => {
- state.failedAttempts++;
- queueStatus.set(getStatus("end"));
- // Returning the job to the queue allows us to retry it later,
- // from now on, recovery intervals will try to execute the job.
- state.queue.unshift(job);
- });
-};
diff --git a/apps/builder/app/builder/shared/sync/sync-server.ts b/apps/builder/app/builder/shared/sync/sync-server.ts
index 2c55c3df16..7a5704577c 100644
--- a/apps/builder/app/builder/shared/sync/sync-server.ts
+++ b/apps/builder/app/builder/shared/sync/sync-server.ts
@@ -1,11 +1,11 @@
import { useEffect, useRef } from "react";
import { useBeforeUnload } from "react-use";
+import { atom } from "nanostores";
import { sync } from "immerhin";
import type { Project } from "@webstudio-is/project";
import type { Build } from "@webstudio-is/project-build";
import type { AuthPermit } from "@webstudio-is/trpc-interface/index.server";
import { restPatchPath } from "~/shared/router-utils";
-import { enqueue, dequeue, queueStatus } from "./queue";
// Periodic check for new entries to group them into one job/call in sync queue.
const NEW_ENTRIES_INTERVAL = 1000;
@@ -13,60 +13,64 @@ const NEW_ENTRIES_INTERVAL = 1000;
// First attempts we will simply retry without changing the state or notifying anyone.
const INTERVAL_RECOVERY = 2000;
+// After this amount of retries without success, we consider connection status error.
+const MAX_RETRY_RECOVERY = 5;
+
// When we reached max failed attempts we will slow down the attempts interval.
const INTERVAL_ERROR = 5000;
-const useRecoveryCheck = () => {
- useEffect(() => {
- const intervalId = setInterval(() => {
- if (queueStatus.get() === "recovering") {
- dequeue();
- }
- }, INTERVAL_RECOVERY);
-
- return () => clearInterval(intervalId);
- }, []);
+const pause = (timeout: number) => {
+ return new Promise((resolve) => setTimeout(resolve, timeout));
};
-const useErrorCheck = () => {
- useEffect(() => {
- const intervalId = setInterval(() => {
- if (queueStatus.get() === "failed") {
- dequeue();
- }
- }, INTERVAL_ERROR);
+export type QueueStatus = "running" | "idle" | "recovering" | "failed";
- return () => clearInterval(intervalId);
- }, []);
-};
+export const queueStatus = atom<QueueStatus>("idle");
-const useNewEntriesCheck = ({
- buildId,
- projectId,
- authToken,
- authPermit,
- version,
-}: UserSyncServerProps) => {
- // save the initial version from loaded build
- const lastVersion = useRef<number>(version);
-
- useEffect(() => {
- if (authPermit === "view") {
- return;
+// polling is important to queue new transactions independently
+// from async iterator and batch them into single job
+const pollTransactions = async function* () {
+ while (true) {
+ const transactions = sync();
+ if (transactions.length > 0) {
+ queueStatus.set("running");
+ yield transactions;
+ queueStatus.set("idle");
}
+ await pause(NEW_ENTRIES_INTERVAL);
+ }
+};
- // @todo setInterval can be completely avoided.
- // Right now prisma can't do atomic updates yet with sandbox documents
- // and backend fetches and updates big objects, so if we send quickly,
- // we end up overwriting things
- const intervalId = setInterval(() => {
- const transactions = sync();
-
- if (transactions.length === 0) {
- return;
- }
+const retry = async function* () {
+ let failedAttempts = 0;
+ while (true) {
+ yield;
+ failedAttempts += 1;
+ if (failedAttempts < MAX_RETRY_RECOVERY) {
+ queueStatus.set("recovering");
+ await pause(INTERVAL_RECOVERY);
+ } else {
+ queueStatus.set("failed");
+ await pause(INTERVAL_ERROR);
+ }
+ }
+};
- enqueue(async () => {
+const syncServer = async function ({
+ authToken,
+ buildId,
+ projectId,
+ lastVersion,
+}: {
+ buildId: Build["id"];
+ projectId: Project["id"];
+ authToken: string | undefined;
+ lastVersion: number;
+}) {
+ polling: for await (const transactions of pollTransactions()) {
+ for await (const _ of retry()) {
+ // in case of any error continue retrying
+ try {
const response = await fetch(restPatchPath({ authToken }), {
method: "post",
body: JSON.stringify({
@@ -74,14 +78,15 @@ const useNewEntriesCheck = ({
buildId,
projectId,
// provide latest stored version to server
- version: lastVersion.current,
+ version: lastVersion,
}),
});
if (response.ok) {
const result = await response.json();
if (result.status === "ok") {
- lastVersion.current += 1;
- return { ok: true };
+ lastVersion += 1;
+ // stop retrying and wait next transactions
+ continue polling;
}
// when versions mismatched ask user to reload
// user may cancel to copy own state before reloading
@@ -94,18 +99,18 @@ const useNewEntriesCheck = ({
if (shouldReload) {
location.reload();
}
- return { ok: false, retry: false };
+ // stop synchronization and wait til user reload
+ break polling;
}
}
- return { ok: false, retry: true };
- });
- }, NEW_ENTRIES_INTERVAL);
-
- return () => clearInterval(intervalId);
- }, [buildId, projectId, authToken, authPermit, version]);
+ } catch {
+ //
+ }
+ }
+ }
};
-type UserSyncServerProps = {
+type SyncServerProps = {
buildId: Build["id"];
projectId: Project["id"];
authToken: string | undefined;
@@ -113,10 +118,23 @@ type UserSyncServerProps = {
version: number;
};
-export const useSyncServer = (props: UserSyncServerProps) => {
- useNewEntriesCheck(props);
- useRecoveryCheck();
- useErrorCheck();
+export const useSyncServer = ({
+ buildId,
+ projectId,
+ authToken,
+ authPermit,
+ version,
+}: SyncServerProps) => {
+ // save the initial version from loaded build
+ const initialized = useRef(false);
+ useEffect(() => {
+ if (authPermit === "view" || initialized.current === true) {
+ return;
+ }
+ initialized.current = true;
+ syncServer({ buildId, projectId, authToken, lastVersion: version });
+ }, [buildId, projectId, authToken, authPermit, version]);
+
useBeforeUnload(
() => queueStatus.get() !== "idle",
"You have unsaved changes. Are you sure you want to leave?"