Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.
TypeScript accessibilityalternativecloudflarecloudflare-workersdesignfreefrontendopen-sourceradix-uireactremixweb-developmentwebflowwebsite-builderwebsite-developmentwebsiteswebstudio
refactor: add server as storage to sync engine (#4475)
Bogdan Chadkine06de5322bcf
diff --git a/apps/builder/app/builder/builder.tsx b/apps/builder/app/builder/builder.tsxindex 47435bb6ea..2d0f853657 100644--- a/apps/builder/app/builder/builder.tsx+++ b/apps/builder/app/builder/builder.tsx@@ -7,12 +7,12 @@ import type { Project } from "@webstudio-is/project";import { theme, Box, type CSS, Flex, Grid } from "@webstudio-is/design-system";import type { AuthPermit } from "@webstudio-is/trpc-interface/index.server";import { createImageLoader } from "@webstudio-is/image";+import { registerContainers, createObjectPool } from "~/shared/sync";import {- builderClient,- registerContainers,- useBuilderStore,-} from "~/shared/sync";-import { startProjectSync, useSyncServer } from "./shared/sync/sync-server";+ ServerSyncStorage,+ startProjectSync,+ useSyncServer,+} from "./shared/sync/sync-server";import { SidebarLeft } from "./sidebar-left";import { Inspector } from "./features/inspector";import { Topbar } from "./features/topbar";@@ -57,7 +57,6 @@ import {import { CloneProjectDialog } from "~/shared/clone-project";import type { TokenPermissions } from "@webstudio-is/authorization-token";import { useToastErrors } from "~/shared/error/toast-error";-import { loadBuilderData, setBuilderData } from "~/shared/builder-data";import { initBuilderApi } from "~/shared/builder-api";import { updateWebstudioData } from "~/shared/instance-utils";import { migrateWebstudioDataMutable } from "~/shared/webstudio-data-migrator";@@ -71,6 +70,7 @@ import {} from "~/shared/copy-paste/init-copy-paste";import { useInertHandlers } from "./shared/inert-handlers";import { TextToolbar } from "./features/workspace/canvas-tools/text-toolbar";+import { SyncClient } from "~/shared/sync-client";registerContainers();@@ -206,6 +206,12 @@ const ChromeWrapper = ({);};+const builderClient = new SyncClient({+ role: "leader",+ object: createObjectPool(),+ storages: [new ServerSyncStorage()],+});+export type BuilderProps = {project: Project;publisherHost: string;@@ -242,9 +248,9 @@ export const Builder = ({const controller = new AbortController();$dataLoadingState.set("loading");- loadBuilderData({ projectId: project.id, signal: controller.signal })- .then((data) => {- setBuilderData(data);+ builderClient.connect({+ signal: controller.signal,+ onReady() {startProjectSync({projectId: project.id,buildId: build.id,@@ -260,12 +266,10 @@ export const Builder = ({// so builder is started listening for connect event// when canvas is rendered$dataLoadingState.set("loaded");- })- .catch((error) => {- console.error(error);+// @todo make needs error handling and error state? e.g. a toast- $dataLoadingState.set("idle");- });+ },+ });return () => {$dataLoadingState.set("idle");controller.abort("unmount");@@ -287,7 +291,6 @@ export const Builder = ({$publisher.set({ publish });}, [publish]);- useBuilderStore();useSyncServer({projectId: project.id,authPermit,diff --git a/apps/builder/app/builder/shared/sync/command-queue.ts b/apps/builder/app/builder/shared/sync/command-queue.tsindex 77ca711b43..695c2db930 100644--- a/apps/builder/app/builder/shared/sync/command-queue.ts+++ b/apps/builder/app/builder/shared/sync/command-queue.ts@@ -1,12 +1,13 @@import type { Project } from "@webstudio-is/project";import type { Build } from "@webstudio-is/project-build";-import type { SyncItem } from "immerhin";+import type { Change } from "immerhin";+import type { Transaction } from "~/shared/sync-client";type Command =| {type: "transactions";projectId: Project["id"];- transactions: SyncItem[];+ transactions: Transaction<Change[]>[];}| {type: "setDetails";diff --git a/apps/builder/app/builder/shared/sync/sync-server.ts b/apps/builder/app/builder/shared/sync/sync-server.tsindex c297fe7f41..a39fd35187 100644--- a/apps/builder/app/builder/shared/sync/sync-server.ts+++ b/apps/builder/app/builder/shared/sync/sync-server.ts@@ -1,13 +1,16 @@import { useEffect } from "react";import { atom } from "nanostores";+import type { Change } from "immerhin";import type { Project } from "@webstudio-is/project";import type { Build } from "@webstudio-is/project-build";import type { AuthPermit } from "@webstudio-is/trpc-interface/index.server";import * as commandQueue from "./command-queue";import { restPatchPath } from "~/shared/router-utils";import { toast } from "@webstudio-is/design-system";-import { serverSyncStore } from "~/shared/sync";import { fetch } from "~/shared/fetch.client";+import type { SyncStorage, Transaction } from "~/shared/sync-client";+import { $project } from "~/shared/nano-states";+import { loadBuilderData } from "~/shared/builder-data";// Periodic check for new entries to group them into one job/call in sync queue.const NEW_ENTRIES_INTERVAL = 1000;@@ -284,27 +287,30 @@ const useSyncProject = ({return;}syncServer();-- const updateProjectTransactions = () => {- const transactions = serverSyncStore.popAll();- if (transactions.length === 0) {- return;- }- commandQueue.enqueue({ type: "transactions", transactions, projectId });- };-- const intervalHandle = setInterval(- updateProjectTransactions,- NEW_ENTRIES_INTERVAL- );-- return () => {- updateProjectTransactions();- clearInterval(intervalHandle);- };}, [projectId, authPermit]);};+export class ServerSyncStorage implements SyncStorage {+ name = "server";+ sendTransaction(transaction: Transaction<Change[]>) {+ if (transaction.object === "server") {+ const projectId = $project.get()?.id ?? "";+ commandQueue.enqueue({+ type: "transactions",+ transactions: [transaction],+ projectId,+ });+ }+ }+ subscribe(setState: (state: unknown) => void, signal: AbortSignal) {+ const projectId = $project.get()?.id ?? "";+ loadBuilderData({ projectId, signal }).then((data) => {+ const serverData = new Map(Object.entries(data));+ setState(new Map([["server", serverData]]));+ });+ }+}+type SyncServerProps = {projectId: Project["id"];authPermit: AuthPermit;diff --git a/apps/builder/app/routes/rest.patch.ts b/apps/builder/app/routes/rest.patch.tsindex 052dd23154..8989281764 100644--- a/apps/builder/app/routes/rest.patch.ts+++ b/apps/builder/app/routes/rest.patch.ts@@ -1,6 +1,6 @@import { applyPatches, enableMapSet, enablePatches } from "immer";import type { ActionFunctionArgs } from "@remix-run/server-runtime";-import type { SyncItem } from "immerhin";+import type { Change } from "immerhin";import {Breakpoints,Breakpoint,@@ -49,9 +49,10 @@ import type { Database } from "@webstudio-is/postrest/index.server";import { publicStaticEnv } from "~/env/env.static";import { preventCrossOriginCookie } from "~/services/no-cross-origin-cookie";import { checkCsrf } from "~/services/csrf-session.server";+import type { Transaction } from "~/shared/sync-client";type PatchData = {- transactions: Array<SyncItem>;+ transactions: Transaction<Change[]>[];buildId: Build["id"];projectId: Project["id"];version: number;@@ -96,7 +97,7 @@ export const action = async ({return { status: "error", errors: "Project id required" };}- const lastTransactionId = transactions.at(-1)?.transactionId;+ const lastTransactionId = transactions.at(-1)?.id;if (lastTransactionId === undefined) {return {@@ -175,7 +176,7 @@ export const action = async ({const patchedStyleDeclKeysSet = new Set<string>();for await (const transaction of transactions) {- for await (const change of transaction.changes) {+ for await (const change of transaction.payload) {const { namespace, patches } = change;if (patches.length === 0) {continue;diff --git a/apps/builder/app/shared/builder-data.ts b/apps/builder/app/shared/builder-data.tsindex f4b3c04ca4..872bfeae82 100644--- a/apps/builder/app/shared/builder-data.ts+++ b/apps/builder/app/shared/builder-data.ts@@ -40,21 +40,6 @@ export const getBuilderData = (): BuilderData => {};};-export const setBuilderData = (data: BuilderData) => {- $assets.set(data.assets);- $instances.set(data.instances);- $dataSources.set(data.dataSources);- $resources.set(data.resources);- // props should be after data sources to compute logic- $props.set(data.props);- $pages.set(data.pages);- $styleSources.set(data.styleSources);- $styleSourceSelections.set(data.styleSourceSelections);- $breakpoints.set(data.breakpoints);- $styles.set(data.styles);- $marketplaceProduct.set(data.marketplaceProduct);-};-const getPair = <Item extends { id: string }>(item: Item) =>[item.id, item] as const;diff --git a/apps/builder/app/shared/sync-client.test.ts b/apps/builder/app/shared/sync-client.test.tsindex c719a9a27a..b6b54932f5 100644--- a/apps/builder/app/shared/sync-client.test.ts+++ b/apps/builder/app/shared/sync-client.test.ts@@ -8,6 +8,7 @@ import {NanostoresSyncObject,SyncClient,SyncObjectPool,+ type SyncStorage,} from "./sync-client";enableMapSet();@@ -217,6 +218,18 @@ test("support pool of objects", () => {);});+test("merge state in pool object to partially restore from storage", () => {+ const store1 = atom(0);+ const store2 = atom(1);+ const objectPool = new SyncObjectPool([+ new NanostoresSyncObject("store1", store1),+ new NanostoresSyncObject("store2", store2),+ ]);+ objectPool.setState(new Map([["store1", 2]]));+ expect(store1.get()).toEqual(2);+ expect(store2.get()).toEqual(1);+});+describe("nanostores sync object", () => {test("sync initial state and exchange transactions", () => {const emitter = createNanoEvents();@@ -290,3 +303,79 @@ describe("nanostores sync object", () => {expect(sendTransaction).toBeCalledTimes(0);});});++describe("storages", () => {+ class TestStorage implements SyncStorage {+ name = "TestStorage";+ value: undefined | number;+ constructor(value: undefined | number) {+ this.value = value;+ }+ sendTransaction = vi.fn();+ subscribe(setState: (state: unknown) => void) {+ setState(this.value);+ }+ }++ test("get initial state from storage", () => {+ const $store = atom(0);+ const client = new SyncClient({+ role: "leader",+ object: new NanostoresSyncObject("nanostores", $store),+ storages: [new TestStorage(1)],+ });+ client.connect({ signal: new AbortController().signal });+ expect($store.get()).toEqual(1);+ });++ test("fallback to current state when storage is empty", () => {+ const $store = atom(0);+ const client = new SyncClient({+ role: "leader",+ object: new NanostoresSyncObject("nanostores", $store),+ storages: [new TestStorage(undefined)],+ });+ client.connect({ signal: new AbortController().signal });+ expect($store.get()).toEqual(0);+ });++ test("get state from the first non-empty storage", () => {+ const $store = atom(0);+ const client = new SyncClient({+ role: "leader",+ object: new NanostoresSyncObject("nanostores", $store),+ storages: [+ new TestStorage(undefined),+ new TestStorage(1),+ new TestStorage(2),+ ],+ });+ client.connect({ signal: new AbortController().signal });+ expect($store.get()).toEqual(1);+ });++ test("send transactions to all provided storages", () => {+ const $store = atom(0);+ const storage1 = new TestStorage(undefined);+ const storage2 = new TestStorage(undefined);+ const client = new SyncClient({+ role: "leader",+ object: new NanostoresSyncObject("nanostores", $store),+ storages: [storage1, storage2],+ });+ client.connect({ signal: new AbortController().signal });+ $store.set(1);+ expect(storage1.sendTransaction).toBeCalledTimes(1);+ expect(storage1.sendTransaction).toHaveBeenCalledWith({+ id: expect.any(String),+ object: "nanostores",+ payload: 1,+ });+ expect(storage2.sendTransaction).toBeCalledTimes(1);+ expect(storage2.sendTransaction).toHaveBeenCalledWith({+ id: expect.any(String),+ object: "nanostores",+ payload: 1,+ });+ });+});diff --git a/apps/builder/app/shared/sync-client.ts b/apps/builder/app/shared/sync-client.tsindex 3634ba931d..8b730b846f 100644--- a/apps/builder/app/shared/sync-client.ts+++ b/apps/builder/app/shared/sync-client.ts@@ -3,17 +3,42 @@ import { createNanoEvents, type Emitter } from "nanoevents";import type { Change, Store } from "immerhin";import type { WritableAtom } from "nanostores";-type Transaction<Payload = unknown> = {+export type Transaction<Payload = unknown> = {id: string;object: string;payload: Payload;};-type RevertedTransaction = {+export type RevertedTransaction = {id: string;object: string;};+export interface SyncStorage {+ name: string;+ sendTransaction(transaction: Transaction): void;+ subscribe(setState: (state: unknown) => void, signal: AbortSignal): void;+}++const subscribeStorages = (+ storages: SyncStorage[],+ setState: (state: unknown) => void,+ signal: AbortSignal+) => {+ if (storages.length === 0) {+ setState(undefined);+ return;+ }+ storages[0].subscribe((value) => {+ if (value === undefined) {+ // invoke the next storage+ subscribeStorages(storages.slice(1), setState, signal);+ return;+ }+ setState(value);+ }, signal);+};+interface SyncObject {name: string;getState(): unknown;@@ -128,7 +153,10 @@ export class SyncObjectPool implements SyncObject {}setState(state: Map<string, unknown>) {for (const [name, object] of this.objects) {- object.setState(state.get(name) as never);+ // merge into existing state to partially restore from storage+ if (state.has(name)) {+ object.setState(state.get(name) as never);+ }}}applyTransaction(transaction: Transaction) {@@ -161,18 +189,21 @@ type SyncClientOptions = {role: "leader" | "follower";object: SyncObject;emitter?: SyncEmitter;+ storages?: SyncStorage[];};export class SyncClient {clientId = nanoid();role: SyncClientOptions["role"];object: SyncObject;+ storages: SyncStorage[];emitter: SyncEmitter;connection: "disconnected" | "connecting" | "connected" = "disconnected";constructor(options: SyncClientOptions) {this.role = options.role;this.object = options.object;+ this.storages = options.storages ?? [];this.emitter = options.emitter ?? createNanoEvents();}@@ -185,7 +216,7 @@ export class SyncClient {});}- connect({ signal }: { signal: AbortSignal }) {+ connect({ signal, onReady }: { signal: AbortSignal; onReady?: () => void }) {const off = this.emitter.on("message", (message) => {// ignore own messagesif (this.clientId === message.clientId) {@@ -218,6 +249,11 @@ export class SyncClient {}*/this.object.applyTransaction(message.transaction);+ if (this.role === "leader") {+ for (const storage of this.storages) {+ storage.sendTransaction(message.transaction);+ }+ }}if (message.type === "revert") {this.object.revertTransaction(message.transaction);@@ -231,6 +267,11 @@ export class SyncClient {type: "apply",transaction,});+ if (this.role === "leader") {+ for (const storage of this.storages) {+ storage.sendTransaction(transaction);+ }+ }}, signal);this.emitter.emit("message", {@@ -241,11 +282,21 @@ export class SyncClient {this.connection = "connecting";}if (this.role === "leader") {- this.emitter.emit("message", {- clientId: this.clientId,- type: "state",- state: this.object.getState(),- });+ subscribeStorages(+ this.storages,+ (state) => {+ // fallback to default object state+ state ??= this.object.getState();+ this.object.setState(state);+ this.emitter.emit("message", {+ clientId: this.clientId,+ type: "state",+ state,+ });+ onReady?.();+ },+ signal+ );}}}diff --git a/apps/builder/app/shared/sync/sync-stores.ts b/apps/builder/app/shared/sync/sync-stores.tsindex cf744a5ba8..0715c6ed3e 100644--- a/apps/builder/app/shared/sync/sync-stores.ts+++ b/apps/builder/app/shared/sync/sync-stores.ts@@ -79,7 +79,7 @@ export const registerContainers = () => {serverSyncStore.register("marketplaceProduct", $marketplaceProduct);};-const createObjectPool = () => {+export const createObjectPool = () => {return new SyncObjectPool([new ImmerhinSyncObject("server", serverSyncStore),new ImmerhinSyncObject("client", clientSyncStore),@@ -97,15 +97,15 @@ const createObjectPool = () => {$selectedInstanceBrowserStyle),new NanostoresSyncObject(- "$selectedInstanceIntanceToTag",+ "selectedInstanceIntanceToTag",$selectedInstanceIntanceToTag),new NanostoresSyncObject(- "$selectedInstanceUnitSizes",+ "selectedInstanceUnitSizes",$selectedInstanceUnitSizes),new NanostoresSyncObject(- "$selectedInstanceRenderState",+ "selectedInstanceRenderState",$selectedInstanceRenderState),new NanostoresSyncObject(@@ -141,7 +141,6 @@ const createObjectPool = () => {new NanostoresSyncObject("hoveredInstanceOutline", $hoveredInstanceOutline),new NanostoresSyncObject("blockChildOutline", $blockChildOutline),new NanostoresSyncObject("modifierKeys", $modifierKeys),-new NanostoresSyncObject("collaborativeInstanceSelector",$collaborativeInstanceSelector@@ -175,6 +174,9 @@ const sharedSyncEmitter =typeof window === "undefined"? undefined: window.__webstudioSharedSyncEmitter__;+if (typeof window !== "undefined") {+ delete window.__webstudioSharedSyncEmitter__;+}export const useCanvasStore = () => {useEffect(() => {@@ -191,19 +193,3 @@ export const useCanvasStore = () => {};}, []);};--export const builderClient = new SyncClient({- role: "leader",- object: createObjectPool(),-});--export const useBuilderStore = () => {- useEffect(() => {- const controller = new AbortController();- builderClient.connect({ signal: controller.signal });-- return () => {- controller.abort();- };- }, []);-};