Skip to content

Open source website builder and Webflow alternative. Webstudio is an advanced visual builder that connects to any headless CMS, supports all CSS properties, and can be hosted anywhere, including with us.

perf: Diagnose and reduce Assets resource latency (#6095)

Oleg Isonen11d76eb43d96
diff --git a/apps/builder/app/builder/features/settings-panel/content-database-diagnostics-view.test.tsx b/apps/builder/app/builder/features/settings-panel/content-database-diagnostics-view.test.tsx
index 4144784d08..e5362afa68 100644
--- a/apps/builder/app/builder/features/settings-panel/content-database-diagnostics-view.test.tsx
+++ b/apps/builder/app/builder/features/settings-panel/content-database-diagnostics-view.test.tsx
@@ -3,7 +3,7 @@
*/
import { act } from "react-dom/test-utils";
import { createRoot, type Root } from "react-dom/client";
-import { afterEach, expect, test } from "vitest";
+import { afterEach, beforeEach, expect, test, vi } from "vitest";
import { TooltipProvider } from "@webstudio-is/design-system";
import type { ContentArtifactV1 } from "@webstudio-is/content-engine";
import { ContentDatabaseDiagnostics } from "./content-database-diagnostics";
@@ -14,6 +14,17 @@ import { ContentDatabaseDiagnostics } from "./content-database-diagnostics";
let root: Root | undefined;
+beforeEach(() => {
+ vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ }
+ );
+});
+
const revision = `sha256:${"a".repeat(64)}`;
const artifact: ContentArtifactV1 = {
format: "webstudio-content-database",
@@ -39,9 +50,15 @@ afterEach(() => {
act(() => root?.unmount());
root = undefined;
document.body.innerHTML = "";
+ vi.unstubAllGlobals();
});
-test("shows the unresolved query result in a read-only JSON editor", () => {
+test("orders collapsible sections and opens only database sizes by default", () => {
+ const writeText = vi.fn().mockResolvedValue(undefined);
+ Object.defineProperty(navigator, "clipboard", {
+ configurable: true,
+ value: { writeText },
+ });
const container = document.createElement("div");
document.body.appendChild(container);
root = createRoot(container);
@@ -49,6 +66,31 @@ test("shows the unresolved query result in a read-only JSON editor", () => {
root?.render(
<TooltipProvider>
<ContentDatabaseDiagnostics
+ performance={{
+ serverDurationMs: 125.5,
+ loaderDurationMs: 150.25,
+ responseBytes: 1024,
+ assetQuery: {
+ phases: {
+ buildPlan: 20,
+ indexPreparation: 80,
+ diagnosticsPreparation: 40,
+ compilerContentRead: 12,
+ documentGraph: 30,
+ documentGraphContentRead: 8,
+ assetReferences: 5,
+ sourceValidation: 10,
+ documentResolution: 15,
+ },
+ compilationCache: "miss",
+ compilerContentFetchCount: 3,
+ compilerContentBytes: 3_000,
+ documentGraphContentFetchCount: 2,
+ documentGraphContentBytes: 2_000,
+ resolvedDocumentCount: 2,
+ documentFetchCount: 1,
+ },
+ }}
value={{
scope: "query-preview",
unresolved: {
@@ -87,11 +129,87 @@ test("shows the unresolved query result in a read-only JSON editor", () => {
);
});
+ const sectionLabels = [
+ "Database and sizes",
+ "Timing",
+ "Query work",
+ "Query database",
+ "Published database",
+ "Unresolved query result",
+ ];
+ const triggers = Array.from(
+ container.querySelectorAll<HTMLButtonElement>("button[aria-controls]")
+ );
+ expect(triggers).toHaveLength(sectionLabels.length);
+ expect(triggers.map((trigger) => trigger.parentElement?.textContent)).toEqual(
+ sectionLabels
+ );
+ expect(triggers.map((trigger) => trigger.dataset.state)).toEqual([
+ "open",
+ "closed",
+ "closed",
+ "closed",
+ "closed",
+ "closed",
+ ]);
+ const copyButtons = Array.from(
+ container.querySelectorAll<HTMLButtonElement>('button[aria-label^="Copy "]')
+ );
+ expect(
+ Array.from(
+ new Set(copyButtons.map((button) => button.getAttribute("aria-label")))
+ )
+ ).toEqual(sectionLabels.map((label) => `Copy ${label} as JSON`));
+ expect(container.textContent).not.toContain("Server duration");
+
+ const timingCopyButton = copyButtons
+ .filter(
+ (button) => button.getAttribute("aria-label") === "Copy Timing as JSON"
+ )
+ .at(-1);
+ act(() => timingCopyButton?.click());
+ expect(triggers[1]?.dataset.state).toBe("closed");
+ expect(writeText).toHaveBeenCalledOnce();
+ expect(JSON.parse(writeText.mock.calls[0]?.[0] ?? "")).toEqual({
+ builderRoundTripMs: 150.25,
+ serverDurationMs: 125.5,
+ phases: {
+ buildPlan: 20,
+ indexPreparation: 80,
+ diagnosticsPreparation: 40,
+ compilerContentRead: 12,
+ documentGraph: 30,
+ documentGraphContentRead: 8,
+ assetReferences: 5,
+ sourceValidation: 10,
+ documentResolution: 15,
+ },
+ });
+
+ act(() => {
+ triggers[1]?.click();
+ triggers[2]?.click();
+ triggers[5]?.click();
+ });
+
const editor = container.querySelector(".cm-content");
expect(editor).not.toBeNull();
expect(editor?.getAttribute("aria-readonly")).toBe("true");
- expect(container.textContent).toContain("Unresolved query result");
expect(container.textContent).toContain('"$ref": "./post.md#body"');
- expect(container.textContent).toContain("Query database");
- expect(container.textContent).toContain("Published database");
+ expect(container.textContent).toContain("Server duration");
+ expect(container.textContent).toContain("125.5 ms");
+ expect(container.textContent).toContain("Builder round trip");
+ expect(container.textContent).toContain("150.3 ms");
+ expect(container.textContent).toContain("1.02 kB");
+ expect(container.textContent).toContain("Build plan");
+ expect(container.textContent).toContain("Index preparation");
+ expect(container.textContent).toContain("Published diagnostics");
+ expect(container.textContent).toContain("Compiler storage reads");
+ expect(container.textContent).toContain("Document graph storage reads");
+ expect(container.textContent).toContain("Compilation workCompiled");
+ expect(container.textContent).toContain("Resolved documents2");
+ expect(container.textContent).toContain("Timing");
+ expect(container.textContent).toContain("Query work");
+ expect(container.textContent).toContain("Database and sizes");
+ expect(container.querySelectorAll('svg[tabindex="0"]')).toHaveLength(26);
});
diff --git a/apps/builder/app/builder/features/settings-panel/content-database-diagnostics.tsx b/apps/builder/app/builder/features/settings-panel/content-database-diagnostics.tsx
index 167e91463f..5ce6de451e 100644
--- a/apps/builder/app/builder/features/settings-panel/content-database-diagnostics.tsx
+++ b/apps/builder/app/builder/features/settings-panel/content-database-diagnostics.tsx
@@ -1,33 +1,170 @@
import type { AssetQueryPreviewDiagnostics } from "@webstudio-is/content-engine";
-import { Grid, PanelBanner, Text } from "@webstudio-is/design-system";
+import {
+ PanelBanner,
+ SectionTitle,
+ SectionTitleButton,
+ SectionTitleLabel,
+ Text,
+} from "@webstudio-is/design-system";
+import { CopyIcon } from "@webstudio-is/icons";
+import type { ReactNode } from "react";
import prettyBytes from "pretty-bytes";
+import type { ResourcePerformance } from "~/shared/resource-diagnostics";
import { CodeEditor } from "~/shared/code-editor";
+import { CopyToClipboard } from "~/shared/copy-to-clipboard";
+import {
+ CollapsibleSectionRoot,
+ useOpenState,
+} from "~/builder/shared/collapsible-section";
import {
- RequestDiagnosticsContent,
RequestDiagnosticsRow,
+ RequestDiagnosticsSections,
RequestDiagnosticsTable,
} from "./request-inspector";
const runtimeContentNote =
"Referenced documents fetched from storage at runtime are not included.";
-const ReadonlyJsonEditor = ({
- title,
- value,
+const assetQueryPhaseRows = [
+ [
+ "authorization",
+ "Authorization",
+ "Time spent authenticating the incoming Builder request and checking access to the project.",
+ ],
+ [
+ "buildPlan",
+ "Build plan",
+ "Time spent loading the current build data and deriving the content required by the project.",
+ ],
+ [
+ "repositoryAuthorization",
+ "Repository authorization",
+ "Time spent checking project access at the Assets repository boundary.",
+ ],
+ [
+ "indexPreparation",
+ "Index preparation",
+ "Inclusive time spent preparing the in-memory query database. It contains the source snapshot, canonical metadata, compiler entries, and artifact compilation phases.",
+ ],
+ [
+ "diagnosticsPreparation",
+ "Published diagnostics",
+ "Inclusive time spent compiling the deployment-wide database only for detailed Diagnostics. This work is not part of the normal resource load, and its internal phases are excluded from the rows below.",
+ ],
+ [
+ "sourceSnapshot",
+ "Source snapshot",
+ "Time spent loading canonical asset entries and computing the source revision used by the compilation cache.",
+ ],
+ [
+ "canonicalMetadata",
+ "Canonical metadata",
+ "Time spent synchronizing and loading structured metadata required by the query plan.",
+ ],
+ [
+ "compilerEntries",
+ "Compiler entries",
+ "Time spent reading and parsing selected asset content into compiler input.",
+ ],
+ [
+ "compilerContentRead",
+ "Compiler storage reads",
+ "Cumulative time spent on completed storage reads while preparing compiler entries. Concurrent read durations are summed, so this child measurement can exceed wall-clock phase time.",
+ ],
+ [
+ "documentGraph",
+ "Document graph",
+ "Time spent reading supported documents and compiling their cross-document references.",
+ ],
+ [
+ "documentGraphContentRead",
+ "Document graph storage reads",
+ "Cumulative time spent on completed storage reads while constructing the document graph. Cached request-local bytes are excluded, and concurrent read durations are summed.",
+ ],
+ [
+ "assetReferences",
+ "Asset references",
+ "Time spent discovering asset references in the selected document content.",
+ ],
+ [
+ "sourceValidation",
+ "Source validation",
+ "Time spent reloading the canonical inventory to confirm that the source did not change during compilation.",
+ ],
+ [
+ "artifactCompilation",
+ "Artifact compilation",
+ "Time spent compiling prepared entries into the in-memory content database artifact.",
+ ],
+ [
+ "runtimeAssets",
+ "Runtime assets",
+ "Time spent loading asset metadata needed to execute the query against the prepared database.",
+ ],
+ [
+ "documentResolution",
+ "Document resolution",
+ "Time spent executing the query and resolving referenced documents, including required storage fetches.",
+ ],
+] as const;
+
+const compilationWorkLabels = {
+ hit: "Reused",
+ coalesced: "Joined",
+ miss: "Compiled",
+ disabled: "Reuse disabled",
+} as const;
+
+const DiagnosticsSection = ({
+ label,
+ data,
+ isOpen,
+ children,
}: {
- title: string;
- value: unknown;
-}) => (
- <Grid gap={1}>
- <Text variant="titles">{title}</Text>
- <CodeEditor
- lang="json"
- readOnly
- value={JSON.stringify(value, undefined, 2)}
- onChange={() => {}}
- onChangeComplete={() => {}}
- />
- </Grid>
+ label: string;
+ data: unknown;
+ isOpen?: boolean;
+ children: ReactNode;
+}) => {
+ const [open, setOpen] = useOpenState(label, isOpen);
+ return (
+ <CollapsibleSectionRoot
+ isOpen={open}
+ onOpenChange={setOpen}
+ trigger={
+ <SectionTitle
+ suffix={
+ <CopyToClipboard
+ text={JSON.stringify(data, undefined, 2) ?? "null"}
+ copyText={`Copy ${label} as JSON`}
+ >
+ <SectionTitleButton
+ type="button"
+ tabIndex={0}
+ aria-label={`Copy ${label} as JSON`}
+ prefix={<CopyIcon />}
+ onPointerDown={(event) => event.stopPropagation()}
+ />
+ </CopyToClipboard>
+ }
+ >
+ <SectionTitleLabel>{label}</SectionTitleLabel>
+ </SectionTitle>
+ }
+ >
+ {children}
+ </CollapsibleSectionRoot>
+ );
+};
+
+const ReadonlyJsonEditor = ({ value }: { value: unknown }) => (
+ <CodeEditor
+ lang="json"
+ readOnly
+ value={JSON.stringify(value, undefined, 2)}
+ onChange={() => {}}
+ onChangeComplete={() => {}}
+ />
);
export const getContentDatabaseDiagnosticRows = (
@@ -53,75 +190,295 @@ export const getContentDatabaseDiagnosticRows = (
},
];
+const PerformanceSizeRows = ({ value }: { value: ResourcePerformance }) => (
+ <>
+ {value.responseBytes !== undefined && (
+ <RequestDiagnosticsRow
+ label="Response size"
+ value={prettyBytes(value.responseBytes)}
+ description="Serialized size of the server resource result before performance metadata is attached."
+ />
+ )}
+ {value.assetQuery?.compilerContentBytes !== undefined && (
+ <RequestDiagnosticsRow
+ label="Compiler content read"
+ value={prettyBytes(value.assetQuery.compilerContentBytes)}
+ description="Total bytes read from storage while preparing compiler entries."
+ />
+ )}
+ {value.assetQuery?.documentGraphContentBytes !== undefined && (
+ <RequestDiagnosticsRow
+ label="Document graph content read"
+ value={prettyBytes(value.assetQuery.documentGraphContentBytes)}
+ description="Total bytes read from storage while constructing the document graph."
+ />
+ )}
+ </>
+);
+
+const getPerformanceSizes = (value?: ResourcePerformance) => ({
+ responseBytes: value?.responseBytes,
+ compilerContentBytes: value?.assetQuery?.compilerContentBytes,
+ documentGraphContentBytes: value?.assetQuery?.documentGraphContentBytes,
+});
+
+const ResourcePerformanceSections = ({
+ value,
+ includeSizes = true,
+ openFirst = true,
+}: {
+ value: ResourcePerformance;
+ includeSizes?: boolean;
+ openFirst?: boolean;
+}) => {
+ const hasTiming =
+ value.loaderDurationMs !== undefined ||
+ value.serverDurationMs !== undefined ||
+ assetQueryPhaseRows.some(
+ ([key]) => value.assetQuery?.phases?.[key] !== undefined
+ );
+ const hasQueryWork =
+ value.assetQuery?.compilationCache !== undefined ||
+ value.assetQuery?.compilerContentFetchCount !== undefined ||
+ value.assetQuery?.documentGraphContentFetchCount !== undefined ||
+ value.assetQuery?.resolvedDocumentCount !== undefined ||
+ value.assetQuery?.documentFetchCount !== undefined;
+ const sizes = getPerformanceSizes(value);
+ const hasSizes =
+ includeSizes && Object.values(sizes).some((size) => size !== undefined);
+ const timing = {
+ builderRoundTripMs: value.loaderDurationMs,
+ serverDurationMs: value.serverDurationMs,
+ phases: value.assetQuery?.phases,
+ };
+ const queryWork = {
+ compilationWork: value.assetQuery?.compilationCache,
+ compilerContentFetchCount: value.assetQuery?.compilerContentFetchCount,
+ documentGraphContentFetchCount:
+ value.assetQuery?.documentGraphContentFetchCount,
+ resolvedDocumentCount: value.assetQuery?.resolvedDocumentCount,
+ documentFetchCount: value.assetQuery?.documentFetchCount,
+ };
+ return (
+ <>
+ {hasSizes && (
+ <DiagnosticsSection label="Sizes" data={sizes} isOpen={openFirst}>
+ <RequestDiagnosticsTable>
+ <PerformanceSizeRows value={value} />
+ </RequestDiagnosticsTable>
+ </DiagnosticsSection>
+ )}
+ {hasTiming && (
+ <DiagnosticsSection
+ label="Timing"
+ data={timing}
+ isOpen={openFirst && hasSizes === false}
+ >
+ <RequestDiagnosticsTable>
+ {value.loaderDurationMs !== undefined && (
+ <RequestDiagnosticsRow
+ label="Builder round trip"
+ value={`${value.loaderDurationMs.toFixed(1)} ms`}
+ description="Duration of the complete Builder resource batch request containing this resource."
+ />
+ )}
+ {value.serverDurationMs !== undefined && (
+ <RequestDiagnosticsRow
+ label="Server duration"
+ value={`${value.serverDurationMs.toFixed(1)} ms`}
+ description="Time spent processing this resource on the Builder server, including authorization, loading, and result formatting."
+ />
+ )}
+ {assetQueryPhaseRows.map(([key, label, description]) => {
+ const durationMs = value.assetQuery?.phases?.[key];
+ if (durationMs === undefined) {
+ return;
+ }
+ return (
+ <RequestDiagnosticsRow
+ key={key}
+ label={label}
+ value={`${durationMs.toFixed(1)} ms`}
+ description={description}
+ />
+ );
+ })}
+ </RequestDiagnosticsTable>
+ </DiagnosticsSection>
+ )}
+ {hasQueryWork && (
+ <DiagnosticsSection
+ label="Query work"
+ data={queryWork}
+ isOpen={openFirst && hasSizes === false && hasTiming === false}
+ >
+ <RequestDiagnosticsTable>
+ {value.assetQuery?.compilationCache !== undefined && (
+ <RequestDiagnosticsRow
+ label="Compilation work"
+ value={compilationWorkLabels[value.assetQuery.compilationCache]}
+ description="Whether this Assets request reused an artifact compiled earlier in the same request, joined matching in-progress work, compiled it on a miss, or ran with reuse disabled."
+ />
+ )}
+ {value.assetQuery?.compilerContentFetchCount !== undefined && (
+ <RequestDiagnosticsRow
+ label="Compiler content fetches"
+ value={value.assetQuery.compilerContentFetchCount}
+ description="Number of asset contents read from storage while preparing compiler entries."
+ />
+ )}
+ {value.assetQuery?.documentGraphContentFetchCount !== undefined && (
+ <RequestDiagnosticsRow
+ label="Document graph content fetches"
+ value={value.assetQuery.documentGraphContentFetchCount}
+ description="Number of document contents read from storage while constructing the document graph."
+ />
+ )}
+ {value.assetQuery?.resolvedDocumentCount !== undefined && (
+ <RequestDiagnosticsRow
+ label="Resolved documents"
+ value={value.assetQuery.resolvedDocumentCount}
+ description="Number of query result documents passed through document-reference resolution."
+ />
+ )}
+ {value.assetQuery?.documentFetchCount !== undefined && (
+ <RequestDiagnosticsRow
+ label="Document fetches"
+ value={value.assetQuery.documentFetchCount}
+ description="Number of referenced documents loaded while resolving the query result, including request-local byte reuse."
+ />
+ )}
+ </RequestDiagnosticsTable>
+ </DiagnosticsSection>
+ )}
+ </>
+ );
+};
+
+export const ResourcePerformanceDiagnostics = ({
+ value,
+}: {
+ value: ResourcePerformance;
+}) => (
+ <RequestDiagnosticsSections>
+ <ResourcePerformanceSections value={value} />
+ </RequestDiagnosticsSections>
+);
+
export const ContentDatabaseDiagnostics = ({
value,
+ performance,
}: {
value: AssetQueryPreviewDiagnostics;
+ performance?: ResourcePerformance;
}) => {
const totalDocumentCount =
value.database.includedDocumentCount + value.database.omittedDocumentCount;
- const candidateFilesLabel = `${totalDocumentCount} candidate ${totalDocumentCount === 1 ? "file" : "files"}`;
- const omittedFilesLabel = `${value.database.omittedDocumentCount} ${value.database.omittedDocumentCount === 1 ? "file" : "files"}`;
+ const candidateFilesLabel = `${totalDocumentCount} candidate ${
+ totalDocumentCount === 1 ? "file" : "files"
+ }`;
+ const omittedFilesLabel = `${value.database.omittedDocumentCount} ${
+ value.database.omittedDocumentCount === 1 ? "file" : "files"
+ }`;
const rows = getContentDatabaseDiagnosticRows(value);
+ const databaseAndSizes = {
+ ...getPerformanceSizes(performance),
+ scope: value.scope,
+ query: value.query,
+ database: value.database,
+ };
return (
- <RequestDiagnosticsContent>
- <PanelBanner variant={value.database.truncated ? "warning" : "success"}>
- <Text>
- {value.database.truncated
- ? `${value.database.includedDocumentCount} of ${candidateFilesLabel} fit in the merged published content database. ${omittedFilesLabel} may be omitted from published query results.`
- : totalDocumentCount === 1
- ? "The candidate file fits in the merged published content database."
- : `All ${candidateFilesLabel} fit in the merged published content database.`}
- </Text>
- </PanelBanner>
- <RequestDiagnosticsTable>
- <RequestDiagnosticsRow label="Scope" value="Query preview" />
- <RequestDiagnosticsRow
- label="Published database status"
- value={value.database.truncated ? "Truncated" : "Complete"}
- />
- {rows.map((row) => (
- <RequestDiagnosticsRow key={row.label} {...row} />
- ))}
- <RequestDiagnosticsRow
- label="Database limit"
- value={prettyBytes(value.database.maxBytes)}
- />
- <RequestDiagnosticsRow
- label="Published included files"
- value={value.database.includedDocumentCount}
- />
- <RequestDiagnosticsRow
- label="Published omitted files"
- value={value.database.omittedDocumentCount}
+ <RequestDiagnosticsSections>
+ <DiagnosticsSection
+ label="Database and sizes"
+ data={databaseAndSizes}
+ isOpen
+ >
+ <PanelBanner variant={value.database.truncated ? "warning" : "success"}>
+ <Text>
+ {value.database.truncated
+ ? `${value.database.includedDocumentCount} of ${candidateFilesLabel} fit in the merged published content database. ${omittedFilesLabel} may be omitted from published query results.`
+ : totalDocumentCount === 1
+ ? "The candidate file fits in the merged published content database."
+ : `All ${candidateFilesLabel} fit in the merged published content database.`}
+ </Text>
+ </PanelBanner>
+ <RequestDiagnosticsTable>
+ {performance !== undefined && (
+ <PerformanceSizeRows value={performance} />
+ )}
+ <RequestDiagnosticsRow
+ label="Scope"
+ value="Query preview"
+ description="These database measurements describe the current Assets query preview and its published-database context."
+ />
+ <RequestDiagnosticsRow
+ label="Published database status"
+ value={value.database.truncated ? "Truncated" : "Complete"}
+ description="Whether every candidate document fits within the published content database limit."
+ />
+ {rows.map((row) => (
+ <RequestDiagnosticsRow key={row.label} {...row} />
+ ))}
+ <RequestDiagnosticsRow
+ label="Database limit"
+ value={prettyBytes(value.database.maxBytes)}
+ description="Maximum serialized size allowed for the merged published content database."
+ />
+ <RequestDiagnosticsRow
+ label="Published included files"
+ value={value.database.includedDocumentCount}
+ description="Number of candidate files included in the published content database within the size limit."
+ />
+ <RequestDiagnosticsRow
+ label="Published omitted files"
+ value={value.database.omittedDocumentCount}
+ description="Number of candidate files omitted from the published content database because of its size limit."
+ />
+ </RequestDiagnosticsTable>
+ </DiagnosticsSection>
+ {performance !== undefined && (
+ <ResourcePerformanceSections
+ value={performance}
+ includeSizes={false}
+ openFirst={false}
/>
- </RequestDiagnosticsTable>
+ )}
{value.artifacts !== undefined && (
<>
- <ReadonlyJsonEditor
- title={
+ <DiagnosticsSection
+ label={
value.query.truncated
? "Included query database"
: "Query database"
}
- value={value.artifacts.query}
- />
- <ReadonlyJsonEditor
- title={
+ data={value.artifacts.query}
+ isOpen={false}
+ >
+ <ReadonlyJsonEditor value={value.artifacts.query} />
+ </DiagnosticsSection>
+ <DiagnosticsSection
+ label={
value.database.truncated
? "Included published database"
: "Published database"
}
- value={value.artifacts.database}
- />
+ data={value.artifacts.database}
+ isOpen={false}
+ >
+ <ReadonlyJsonEditor value={value.artifacts.database} />
+ </DiagnosticsSection>
</>
)}
{value.unresolved !== undefined && (
- <ReadonlyJsonEditor
- title="Unresolved query result"
- value={value.unresolved}
- />
+ <DiagnosticsSection
+ label="Unresolved query result"
+ data={value.unresolved}
+ isOpen={false}
+ >
+ <ReadonlyJsonEditor value={value.unresolved} />
+ </DiagnosticsSection>
)}
- </RequestDiagnosticsContent>
+ </RequestDiagnosticsSections>
);
};
diff --git a/apps/builder/app/builder/features/settings-panel/request-inspector.tsx b/apps/builder/app/builder/features/settings-panel/request-inspector.tsx
index 0ff698b75b..08cee41318 100644
--- a/apps/builder/app/builder/features/settings-panel/request-inspector.tsx
+++ b/apps/builder/app/builder/features/settings-panel/request-inspector.tsx
@@ -7,6 +7,7 @@ import {
PanelTabsList,
PanelTabsTrigger,
ScrollAreaNative,
+ styled,
Text,
Tooltip,
theme,
@@ -30,6 +31,16 @@ export const RequestDiagnosticsContent = ({
</ScrollAreaNative>
);
+export const RequestDiagnosticsSections = ({
+ children,
+}: {
+ children: ReactNode;
+}) => (
+ <ScrollAreaNative css={{ height: "100%", overflow: "auto" }}>
+ {children}
+ </ScrollAreaNative>
+);
+
export const RequestDiagnosticsTable = ({
children,
}: {
@@ -46,6 +57,15 @@ export const RequestDiagnosticsTable = ({
</Grid>
);
+const DiagnosticsLabel = styled(Flex, {
+ "& [data-diagnostics-info]": {
+ opacity: 0,
+ },
+ "&:hover [data-diagnostics-info], &:focus-within [data-diagnostics-info]": {
+ opacity: 1,
+ },
+});
+
export const RequestDiagnosticsRow = ({
label,
value,
@@ -66,14 +86,14 @@ export const RequestDiagnosticsRow = ({
"&:last-child": { borderBottom: 0 },
}}
>
- <Flex align="center" gap={1}>
+ <DiagnosticsLabel align="center" gap={1}>
<Text color="moreSubtle">{label}</Text>
{description !== undefined && (
<Tooltip variant="wrapped" content={description}>
- <InfoCircleIcon tabIndex={0} />
+ <InfoCircleIcon data-diagnostics-info tabIndex={0} />
</Tooltip>
)}
- </Flex>
+ </DiagnosticsLabel>
<Text
color={valueColor}
userSelect="text"
diff --git a/apps/builder/app/builder/features/settings-panel/variable-popover.tsx b/apps/builder/app/builder/features/settings-panel/variable-popover.tsx
index c703ebe4a6..7c84e09341 100644
--- a/apps/builder/app/builder/features/settings-panel/variable-popover.tsx
+++ b/apps/builder/app/builder/features/settings-panel/variable-popover.tsx
@@ -90,6 +90,7 @@ import { generateCurl } from "./curl";
import {
$hasPendingResources,
$resourceDiagnosticsCache,
+ $resourcePerformanceCache,
$resourcesCache,
computeResourceRequest,
getResourceKey,
@@ -102,11 +103,15 @@ import {
clearSettledDiagnosticsKey,
RequestInspector,
} from "./request-inspector";
-import { ContentDatabaseDiagnostics } from "./content-database-diagnostics";
+import {
+ ContentDatabaseDiagnostics,
+ ResourcePerformanceDiagnostics,
+} from "./content-database-diagnostics";
import {
getRequestErrorDiagnostics,
RequestErrorDiagnostics,
} from "./request-error-diagnostics";
+import type { ResourcePerformance } from "~/shared/resource-diagnostics";
const NameField = ({
variable,
@@ -700,9 +705,11 @@ const VariablePreview = ({
const variableValues = useStore($instanceVariableValues);
const resourcesCache = useStore($resourcesCache);
const resourceDiagnosticsCache = useStore($resourceDiagnosticsCache);
+ const resourcePerformanceCache = useStore($resourcePerformanceCache);
const resourceScope = useResourceScope({ variable });
let computedValue: unknown;
let resourceDiagnostics: AssetQueryPreviewDiagnostics | undefined;
+ let resourcePerformance: ResourcePerformance | undefined;
let computedResourceRequest: ResourceRequest | undefined;
let computedResourceKey: string | undefined;
if (variableType === "string" || variableType === "boolean") {
@@ -734,6 +741,7 @@ const VariablePreview = ({
computedResourceKey = resourceKey;
computedValue = resourcesCache.get(resourceKey);
resourceDiagnostics = resourceDiagnosticsCache.get(resourceKey);
+ resourcePerformance = resourcePerformanceCache.get(resourceKey);
}
}
const extensions = useMemo(() => [javascript({}), foldGutterExtension], []);
@@ -810,7 +818,12 @@ const VariablePreview = ({
requestErrorDiagnostics !== undefined ? (
<RequestErrorDiagnostics value={requestErrorDiagnostics} />
) : resourceDiagnostics !== undefined ? (
- <ContentDatabaseDiagnostics value={resourceDiagnostics} />
+ <ContentDatabaseDiagnostics
+ value={resourceDiagnostics}
+ performance={resourcePerformance}
+ />
+ ) : resourcePerformance !== undefined ? (
+ <ResourcePerformanceDiagnostics value={resourcePerformance} />
) : undefined
}
/>
diff --git a/apps/builder/app/services/asset-query-preview.server.ts b/apps/builder/app/services/asset-query-preview.server.ts
index b75d775ef8..ebc5c101d3 100644
--- a/apps/builder/app/services/asset-query-preview.server.ts
+++ b/apps/builder/app/services/asset-query-preview.server.ts
@@ -1,4 +1,8 @@
import type { AssetQueryRequestInput } from "@webstudio-is/content-engine";
+import type {
+ AssetQueryDocumentGraphObserver,
+ AssetQueryPerformanceObserver,
+} from "@webstudio-is/asset-uploader/server";
import {
previewProjectAssetQueries as previewSharedProjectAssetQueries,
previewProjectAssetQuery as previewSharedProjectAssetQuery,
@@ -25,6 +29,8 @@ export const previewProjectAssetQueries = async (
requests: readonly AssetQueryRequestInput[];
context: AppContext;
signal?: AbortSignal;
+ onPerformanceEvent?: AssetQueryPerformanceObserver;
+ onDocumentGraphEvent?: AssetQueryDocumentGraphObserver;
},
dependencies = defaultBatchDependencies
) => {
@@ -44,6 +50,8 @@ export const previewProjectAssetQuery = async (
includeDiagnostics?: boolean;
includeUnresolvedDiagnostics?: boolean;
signal?: AbortSignal;
+ onPerformanceEvent?: AssetQueryPerformanceObserver;
+ onDocumentGraphEvent?: AssetQueryDocumentGraphObserver;
},
dependencies = defaultQueryDependencies
) => {
diff --git a/apps/builder/app/services/resource-list-loader.server.ts b/apps/builder/app/services/resource-list-loader.server.ts
index 556ee54b05..79216c78c1 100644
--- a/apps/builder/app/services/resource-list-loader.server.ts
+++ b/apps/builder/app/services/resource-list-loader.server.ts
@@ -10,6 +10,29 @@ import { getResourceKey } from "~/shared/resource-utils";
const defaultDependencies = {
executeAssetQueries,
loadResource,
+ now: () => performance.now(),
+};
+
+const getResponseBytes = (value: unknown) => {
+ try {
+ return new TextEncoder().encode(JSON.stringify(value)).byteLength;
+ } catch {
+ return 0;
+ }
+};
+
+const separateInternalPerformance = (value: unknown) => {
+ if (typeof value !== "object" || value === null) {
+ return { result: value, performance: {} };
+ }
+ const { __performance__, ...result } = value as Record<string, unknown>;
+ return {
+ result,
+ performance:
+ typeof __performance__ === "object" && __performance__ !== null
+ ? __performance__
+ : {},
+ };
};
export const loadResourceRequestList = async (
@@ -26,8 +49,9 @@ export const loadResourceRequestList = async (
includeDiagnostics: boolean;
customFetch: typeof fetch;
},
- dependencies = defaultDependencies
+ dependencies: Partial<typeof defaultDependencies> = {}
) => {
+ const resolvedDependencies = { ...defaultDependencies, ...dependencies };
const assetProvider = includeDiagnostics
? undefined
: createResourceFetchBatchProvider({
@@ -35,7 +59,10 @@ export const loadResourceRequestList = async (
shouldBatch: (input) =>
typeof input === "string" && isLocalResource(input, "assets"),
execute: (resourceRequests) =>
- dependencies.executeAssetQueries({ request, resourceRequests }),
+ resolvedDependencies.executeAssetQueries({
+ request,
+ resourceRequests,
+ }),
});
const providerFetch: typeof fetch = (input, init) =>
assetProvider?.fetch(input, init) ?? customFetch(input, init);
@@ -52,14 +79,24 @@ export const loadResourceRequestList = async (
},
];
}
+ const startedAt = resolvedDependencies.now();
+ const result = await resolvedDependencies.loadResource(
+ providerFetch,
+ resource.data,
+ sourceOrigin,
+ { signal: request.signal }
+ );
+ const separated = separateInternalPerformance(result);
return [
getResourceKey(resource.data),
- await dependencies.loadResource(
- providerFetch,
- resource.data,
- sourceOrigin,
- { signal: request.signal }
- ),
+ {
+ ...result,
+ __performance__: {
+ ...separated.performance,
+ serverDurationMs: Math.max(0, resolvedDependencies.now() - startedAt),
+ responseBytes: getResponseBytes(separated.result),
+ },
+ },
];
});
await assetProvider?.flush();
diff --git a/apps/builder/app/shared/$resources/assets-query.server.test.ts b/apps/builder/app/shared/$resources/assets-query.server.test.ts
index 7c56786425..122ca6fd1e 100644
--- a/apps/builder/app/shared/$resources/assets-query.server.test.ts
+++ b/apps/builder/app/shared/$resources/assets-query.server.test.ts
@@ -38,11 +38,56 @@ describe("configured Assets system resource", () => {
const last = {
data: { items: [{ id: "updates" }], totalCount: 1, hasMore: false },
};
- dependencies.previewProjectAssetQueries.mockResolvedValue([
- { status: "fulfilled", value: first },
- { status: "rejected", reason: new AssetIndexRevisionError() },
- { status: "fulfilled", value: last },
- ]);
+ dependencies.previewProjectAssetQueries.mockImplementation(
+ async ({ onPerformanceEvent, onDocumentGraphEvent }) => {
+ onPerformanceEvent?.({
+ type: "phase-completed",
+ phase: "build-plan",
+ durationMs: 20,
+ });
+ onPerformanceEvent?.({
+ type: "compilation-cache",
+ status: "miss",
+ });
+ onPerformanceEvent?.({
+ type: "phase-completed",
+ phase: "document-graph",
+ durationMs: 30,
+ });
+ onPerformanceEvent?.({
+ type: "phase-completed",
+ phase: "diagnostics-preparation",
+ durationMs: 40,
+ });
+ onPerformanceEvent?.({
+ type: "content-read",
+ purpose: "compiler-entry",
+ byteLength: 1000,
+ durationMs: 12,
+ });
+ onPerformanceEvent?.({
+ type: "content-read",
+ purpose: "document-graph",
+ byteLength: 750,
+ durationMs: 8,
+ });
+ onDocumentGraphEvent?.({
+ type: "resolution-started",
+ rootCount: 1,
+ documentCount: 2,
+ });
+ onDocumentGraphEvent?.({
+ type: "document-fetch-started",
+ documentId: "post",
+ revision: "revision",
+ });
+ return [
+ { status: "fulfilled", value: first },
+ { status: "rejected", reason: new AssetIndexRevisionError() },
+ { status: "fulfilled", value: last },
+ ];
+ }
+ );
const controller = new AbortController();
const requests = [
innerRequest({ query: { limit: 1 } }),
@@ -63,7 +108,28 @@ describe("configured Assets system resource", () => {
);
expect(responses.map(({ status }) => status)).toEqual([200, 400, 409, 200]);
- await expect(responses[0].json()).resolves.toEqual(first);
+ await expect(responses[0].json()).resolves.toEqual({
+ ...first,
+ __performance__: {
+ assetQuery: {
+ phases: {
+ authorization: expect.any(Number),
+ buildPlan: 20,
+ documentGraph: 30,
+ compilerContentRead: 12,
+ documentGraphContentRead: 8,
+ diagnosticsPreparation: 40,
+ },
+ compilationCache: "miss",
+ resolvedDocumentCount: 2,
+ documentFetchCount: 1,
+ compilerContentFetchCount: 1,
+ compilerContentBytes: 1000,
+ documentGraphContentFetchCount: 1,
+ documentGraphContentBytes: 750,
+ },
+ },
+ });
await expect(responses[1].json()).resolves.toMatchObject({
ok: false,
error: { code: "INVALID_REQUEST" },
@@ -72,7 +138,10 @@ describe("configured Assets system resource", () => {
ok: false,
error: { code: "STALE_INDEX" },
});
- await expect(responses[3].json()).resolves.toEqual(last);
+ await expect(responses[3].json()).resolves.toEqual({
+ ...last,
+ __performance__: expect.any(Object),
+ });
expect(dependencies.preventCrossOriginCookie).toHaveBeenCalledOnce();
expect(dependencies.authorizeApiProject).toHaveBeenCalledOnce();
expect(dependencies.previewProjectAssetQueries).toHaveBeenCalledWith({
@@ -90,6 +159,8 @@ describe("configured Assets system resource", () => {
],
context: expect.anything(),
signal: expect.any(AbortSignal),
+ onPerformanceEvent: expect.any(Function),
+ onDocumentGraphEvent: expect.any(Function),
});
});
@@ -169,7 +240,18 @@ describe("configured Assets system resource", () => {
);
expect(response.status).toBe(200);
- await expect(response.json()).resolves.toEqual(responseBody);
+ await expect(response.json()).resolves.toMatchObject({
+ ...responseBody,
+ __performance__: {
+ assetQuery: {
+ phases: { authorization: expect.any(Number) },
+ compilerContentFetchCount: 0,
+ compilerContentBytes: 0,
+ documentGraphContentFetchCount: 0,
+ documentGraphContentBytes: 0,
+ },
+ },
+ });
expect(dependencies.authorizeApiProject).toHaveBeenCalledWith(
expect.any(Request),
projectId,
@@ -189,6 +271,8 @@ describe("configured Assets system resource", () => {
],
context: expect.anything(),
signal: expect.any(AbortSignal),
+ onPerformanceEvent: expect.any(Function),
+ onDocumentGraphEvent: expect.any(Function),
});
});
diff --git a/apps/builder/app/shared/$resources/assets-query.server.ts b/apps/builder/app/shared/$resources/assets-query.server.ts
index 6065b4e37a..8b993e9ff6 100644
--- a/apps/builder/app/shared/$resources/assets-query.server.ts
+++ b/apps/builder/app/shared/$resources/assets-query.server.ts
@@ -2,7 +2,9 @@ import { json } from "@remix-run/server-runtime";
import {
getAssetResourceQueryError,
readAssetQueryRequest,
+ type DocumentGraphRuntimeObserver,
} from "@webstudio-is/content-engine";
+import type { AssetQueryPerformanceObserver } from "@webstudio-is/asset-uploader/server";
import { privateNoStoreResponseHeaders } from "~/services/cache-control.server";
import {
authorizeApiProject,
@@ -21,6 +23,7 @@ type Dependencies = {
previewProjectAssetQueries: typeof previewProjectAssetQueries;
previewProjectAssetQuery: typeof previewProjectAssetQuery;
preventCrossOriginCookie: typeof preventCrossOriginCookie;
+ now: () => number;
};
const defaultDependencies: Dependencies = {
@@ -28,6 +31,100 @@ const defaultDependencies: Dependencies = {
previewProjectAssetQueries,
previewProjectAssetQuery,
preventCrossOriginCookie,
+ now: () => performance.now(),
+};
+
+const phaseKeys = {
+ "build-plan": "buildPlan",
+ "repository-authorization": "repositoryAuthorization",
+ "source-snapshot": "sourceSnapshot",
+ "canonical-metadata": "canonicalMetadata",
+ "compiler-entries": "compilerEntries",
+ "document-graph": "documentGraph",
+ "asset-references": "assetReferences",
+ "source-validation": "sourceValidation",
+ "artifact-compilation": "artifactCompilation",
+ "index-preparation": "indexPreparation",
+ "diagnostics-preparation": "diagnosticsPreparation",
+ "runtime-assets": "runtimeAssets",
+ "document-resolution": "documentResolution",
+} as const;
+
+type PerformancePhaseKey =
+ | (typeof phaseKeys)[keyof typeof phaseKeys]
+ | "authorization"
+ | "compilerContentRead"
+ | "documentGraphContentRead";
+
+const compilationCacheRank = {
+ hit: 0,
+ coalesced: 1,
+ disabled: 2,
+ miss: 3,
+} as const;
+
+const createPerformanceCollector = () => {
+ const phases: Partial<Record<PerformancePhaseKey, number>> = {};
+ let compilationCache: "hit" | "coalesced" | "miss" | "disabled" | undefined;
+ let resolvedDocumentCount = 0;
+ let documentFetchCount = 0;
+ let compilerContentFetchCount = 0;
+ let compilerContentBytes = 0;
+ let documentGraphContentFetchCount = 0;
+ let documentGraphContentBytes = 0;
+ const onPerformanceEvent: AssetQueryPerformanceObserver = (event) => {
+ if (event.type === "phase-completed") {
+ const key = phaseKeys[event.phase];
+ phases[key] = (phases[key] ?? 0) + event.durationMs;
+ return;
+ }
+ if (event.type === "content-read") {
+ if (event.purpose === "compiler-entry") {
+ compilerContentFetchCount += 1;
+ compilerContentBytes += event.byteLength;
+ phases.compilerContentRead =
+ (phases.compilerContentRead ?? 0) + event.durationMs;
+ } else {
+ documentGraphContentFetchCount += 1;
+ documentGraphContentBytes += event.byteLength;
+ phases.documentGraphContentRead =
+ (phases.documentGraphContentRead ?? 0) + event.durationMs;
+ }
+ return;
+ }
+ if (
+ compilationCache === undefined ||
+ compilationCacheRank[event.status] >
+ compilationCacheRank[compilationCache]
+ ) {
+ compilationCache = event.status;
+ }
+ };
+ const onDocumentGraphEvent: DocumentGraphRuntimeObserver = (event) => {
+ if (event.type === "resolution-started") {
+ resolvedDocumentCount += event.documentCount;
+ }
+ if (event.type === "document-fetch-started") {
+ documentFetchCount += 1;
+ }
+ };
+ return {
+ onPerformanceEvent,
+ onDocumentGraphEvent,
+ addPhase: (key: PerformancePhaseKey, durationMs: number) => {
+ phases[key] = (phases[key] ?? 0) + durationMs;
+ },
+ getMetrics: () => ({
+ phases,
+ ...(compilationCache === undefined ? {} : { compilationCache }),
+ resolvedDocumentCount,
+ documentFetchCount,
+ compilerContentFetchCount,
+ compilerContentBytes,
+ documentGraphContentFetchCount,
+ documentGraphContentBytes,
+ }),
+ };
};
const createFailureResponse = (error: unknown, request: Request) => {
@@ -77,9 +174,11 @@ export const executeAssetQueries = async (
resourceRequests: readonly Request[];
includeDiagnostics?: boolean;
},
- dependencies = defaultDependencies
+ dependencies: Partial<Dependencies> = {}
) => {
- dependencies.preventCrossOriginCookie(request);
+ const resolvedDependencies = { ...defaultDependencies, ...dependencies };
+ const performanceCollector = createPerformanceCollector();
+ resolvedDependencies.preventCrossOriginCookie(request);
if (request.signal.aborted) {
return resourceRequests.map(() =>
createFailureResponse(request.signal.reason, request)
@@ -128,11 +227,19 @@ export const executeAssetQueries = async (
let context: Awaited<ReturnType<typeof authorizeApiProject>>;
try {
- context = await dependencies.authorizeApiProject(
- request,
- projectId,
- "view"
- );
+ const startedAt = resolvedDependencies.now();
+ try {
+ context = await resolvedDependencies.authorizeApiProject(
+ request,
+ projectId,
+ "view"
+ );
+ } finally {
+ performanceCollector.addPhase(
+ "authorization",
+ Math.max(0, resolvedDependencies.now() - startedAt)
+ );
+ }
} catch (error) {
for (const { index } of parsedRequests) {
responses[index] = createFailureResponse(error, request);
@@ -145,22 +252,26 @@ export const executeAssetQueries = async (
if (includeDiagnostics) {
results = await Promise.allSettled(
parsedRequests.map(({ request: parsed }) =>
- dependencies.previewProjectAssetQuery({
+ resolvedDependencies.previewProjectAssetQuery({
projectId,
request: parsed,
context,
includeDiagnostics: true,
includeUnresolvedDiagnostics: true,
signal: request.signal,
+ onPerformanceEvent: performanceCollector.onPerformanceEvent,
+ onDocumentGraphEvent: performanceCollector.onDocumentGraphEvent,
})
)
);
} else {
- results = await dependencies.previewProjectAssetQueries({
+ results = await resolvedDependencies.previewProjectAssetQueries({
projectId,
requests: parsedRequests.map(({ request }) => request),
context,
signal: request.signal,
+ onPerformanceEvent: performanceCollector.onPerformanceEvent,
+ onDocumentGraphEvent: performanceCollector.onDocumentGraphEvent,
});
}
} catch (error) {
@@ -174,7 +285,16 @@ export const executeAssetQueries = async (
request
);
} else if (result.status === "fulfilled") {
- responses[index] = json(result.value, {
+ const value =
+ typeof result.value === "object" && result.value !== null
+ ? {
+ ...result.value,
+ __performance__: {
+ assetQuery: performanceCollector.getMetrics(),
+ },
+ }
+ : result.value;
+ responses[index] = json(value, {
headers: privateNoStoreResponseHeaders,
});
} else {
@@ -194,7 +314,7 @@ export const executeAssetQuery = async (
resourceRequest: Request;
includeDiagnostics?: boolean;
},
- dependencies = defaultDependencies
+ dependencies: Partial<Dependencies> = {}
) =>
(
await executeAssetQueries(
diff --git a/apps/builder/app/shared/resource-diagnostics.test.ts b/apps/builder/app/shared/resource-diagnostics.test.ts
index 9d00b0ed33..1c5e269d27 100644
--- a/apps/builder/app/shared/resource-diagnostics.test.ts
+++ b/apps/builder/app/shared/resource-diagnostics.test.ts
@@ -81,4 +81,40 @@ describe("resource diagnostics", () => {
})
).toEqual({ result, diagnostics: undefined });
});
+
+ test("keeps Builder performance metrics outside resource values", () => {
+ const separated = separateResourceDiagnostics({
+ request: { ...assetsRequest, name: "API", url: "/api/items" },
+ result: {
+ ok: true,
+ data: { items: [] },
+ __performance__: {
+ serverDurationMs: 125.5,
+ loaderDurationMs: 150.25,
+ responseBytes: 1024,
+ },
+ },
+ });
+
+ expect(separated.result).toEqual({ ok: true, data: { items: [] } });
+ expect(separated.result).not.toHaveProperty("__performance__");
+ expect(separated.performance).toEqual({
+ serverDurationMs: 125.5,
+ loaderDurationMs: 150.25,
+ responseBytes: 1024,
+ });
+ });
+
+ test("strips malformed Builder performance metrics", () => {
+ const separated = separateResourceDiagnostics({
+ request: assetsRequest,
+ result: {
+ data: { items: [] },
+ __performance__: { serverDurationMs: "slow" },
+ },
+ });
+
+ expect(separated.result).toEqual({ data: { items: [] } });
+ expect(separated.performance).toBeUndefined();
+ });
});
diff --git a/apps/builder/app/shared/resource-diagnostics.ts b/apps/builder/app/shared/resource-diagnostics.ts
index c90877a68c..5055a2a0cc 100644
--- a/apps/builder/app/shared/resource-diagnostics.ts
+++ b/apps/builder/app/shared/resource-diagnostics.ts
@@ -1,6 +1,48 @@
import { assetQueryPreviewDiagnostics } from "@webstudio-is/content-engine";
import type { ResourceRequest } from "@webstudio-is/sdk";
import { isAssetsResourceRequest } from "@webstudio-is/sdk/runtime";
+import { z } from "zod";
+
+export const resourcePerformance = z.strictObject({
+ serverDurationMs: z.number().nonnegative().optional(),
+ loaderDurationMs: z.number().nonnegative().optional(),
+ responseBytes: z.number().int().nonnegative().optional(),
+ assetQuery: z
+ .strictObject({
+ phases: z
+ .strictObject({
+ authorization: z.number().nonnegative().optional(),
+ buildPlan: z.number().nonnegative().optional(),
+ repositoryAuthorization: z.number().nonnegative().optional(),
+ sourceSnapshot: z.number().nonnegative().optional(),
+ canonicalMetadata: z.number().nonnegative().optional(),
+ compilerEntries: z.number().nonnegative().optional(),
+ compilerContentRead: z.number().nonnegative().optional(),
+ documentGraph: z.number().nonnegative().optional(),
+ documentGraphContentRead: z.number().nonnegative().optional(),
+ assetReferences: z.number().nonnegative().optional(),
+ sourceValidation: z.number().nonnegative().optional(),
+ artifactCompilation: z.number().nonnegative().optional(),
+ indexPreparation: z.number().nonnegative().optional(),
+ diagnosticsPreparation: z.number().nonnegative().optional(),
+ runtimeAssets: z.number().nonnegative().optional(),
+ documentResolution: z.number().nonnegative().optional(),
+ })
+ .optional(),
+ compilationCache: z
+ .enum(["hit", "coalesced", "miss", "disabled"])
+ .optional(),
+ resolvedDocumentCount: z.number().int().nonnegative().optional(),
+ documentFetchCount: z.number().int().nonnegative().optional(),
+ compilerContentFetchCount: z.number().int().nonnegative().optional(),
+ compilerContentBytes: z.number().int().nonnegative().optional(),
+ documentGraphContentFetchCount: z.number().int().nonnegative().optional(),
+ documentGraphContentBytes: z.number().int().nonnegative().optional(),
+ })
+ .optional(),
+});
+
+export type ResourcePerformance = z.infer<typeof resourcePerformance>;
// Assets diagnostics may be shown by Builder, but must never enter the value
// exposed to expressions. Ordinary APIs may legitimately use the same field.
@@ -11,18 +53,29 @@ export const separateResourceDiagnostics = ({
request?: ResourceRequest;
result: unknown;
}) => {
+ if (typeof result !== "object" || result === null) {
+ return { result, diagnostics: undefined, performance: undefined };
+ }
+ const { __performance__, ...resultWithoutPerformance } = result as Record<
+ string,
+ unknown
+ >;
+ const performance = resourcePerformance.safeParse(__performance__).data;
if (
request === undefined ||
isAssetsResourceRequest(request) === false ||
- typeof result !== "object" ||
- result === null ||
- "__diagnostics__" in result === false
+ "__diagnostics__" in resultWithoutPerformance === false
) {
- return { result, diagnostics: undefined };
+ return {
+ result: resultWithoutPerformance,
+ diagnostics: undefined,
+ performance,
+ };
}
- const { __diagnostics__, ...resourceResult } = result;
+ const { __diagnostics__, ...resourceResult } = resultWithoutPerformance;
return {
result: resourceResult,
diagnostics: assetQueryPreviewDiagnostics.safeParse(__diagnostics__).data,
+ performance,
};
};
diff --git a/apps/builder/app/shared/resources.test.ts b/apps/builder/app/shared/resources.test.ts
index a4da94ad56..22a14ad625 100644
--- a/apps/builder/app/shared/resources.test.ts
+++ b/apps/builder/app/shared/resources.test.ts
@@ -4,6 +4,7 @@ import {
__testing__,
$hasPendingResources,
$resourceDiagnosticsCache,
+ $resourcePerformanceCache,
$resourcesCache,
getResourceKey,
loadResourceDiagnostics,
@@ -452,6 +453,40 @@ test("loads detailed Assets diagnostics only on demand", async () => {
expect($resourceDiagnosticsCache.get().get(key)).toEqual(diagnostics);
});
+test("caches performance metrics separately from resource values", async () => {
+ const request: ResourceRequest = {
+ name: "Posts",
+ method: "get",
+ url: "https://example.com/performance",
+ searchParams: [],
+ headers: [],
+ };
+ const key = getResourceKey(request);
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json([
+ [
+ key,
+ {
+ data: { title: "Post" },
+ __performance__: { serverDurationMs: 42, responseBytes: 128 },
+ },
+ ],
+ ])
+ );
+
+ queueResources([request]);
+ await loadResources(fetch);
+
+ expect($resourcesCache.get().get(key)).toEqual({
+ data: { title: "Post" },
+ });
+ expect($resourcePerformanceCache.get().get(key)).toMatchObject({
+ serverDurationMs: 42,
+ responseBytes: 128,
+ loaderDurationMs: expect.any(Number),
+ });
+});
+
test("discards stale diagnostics after invalidation", async () => {
vi.useFakeTimers();
const request: ResourceRequest = {
diff --git a/apps/builder/app/shared/resources.ts b/apps/builder/app/shared/resources.ts
index 05da1a47a5..413b6ef833 100644
--- a/apps/builder/app/shared/resources.ts
+++ b/apps/builder/app/shared/resources.ts
@@ -7,6 +7,7 @@ import { fetch } from "./fetch.client";
import { getResourceKey } from "./resource-utils";
import { type AssetQueryPreviewDiagnostics } from "@webstudio-is/content-engine";
import { separateResourceDiagnostics } from "./resource-diagnostics";
+import type { ResourcePerformance } from "./resource-diagnostics";
const MAX_PENDING_RESOURCES = 5;
@@ -21,6 +22,7 @@ const queue = new Map<string, ResourceRequest>();
const pending = new Map<string, InFlightResourceBatch>();
const cache = new Map<string, unknown>();
const diagnosticsCache = new Map<string, AssetQueryPreviewDiagnostics>();
+const performanceCache = new Map<string, ResourcePerformance>();
const pendingDiagnostics = new Map<string, Promise<void>>();
const knownRequests = new Map<string, ResourceRequest>();
const resourceVersions = new Map<string, number>();
@@ -28,10 +30,12 @@ const inFlightBatches = new Set<InFlightResourceBatch>();
export const $resourcesCache = atom(cache);
export const $resourceDiagnosticsCache = atom(diagnosticsCache);
+export const $resourcePerformanceCache = atom(performanceCache);
const updateCache = () => {
$resourcesCache.set(new Map(cache));
$resourceDiagnosticsCache.set(new Map(diagnosticsCache));
+ $resourcePerformanceCache.set(new Map(performanceCache));
};
const $pendingUpdater = atom({});
@@ -69,6 +73,7 @@ const loadResources = async (requestFetch: typeof fetch = fetch) => {
updatePending();
try {
+ const startedAt = performance.now();
const response = await requestFetch(restResourcesLoader(), {
method: "POST",
body: JSON.stringify(list),
@@ -78,6 +83,7 @@ const loadResources = async (requestFetch: typeof fetch = fetch) => {
return;
}
const results = new Map<string, unknown>(await response.json());
+ const loaderDurationMs = performance.now() - startedAt;
for (const [key, result] of results) {
const request = dispatched.get(key);
if (
@@ -90,6 +96,10 @@ const loadResources = async (requestFetch: typeof fetch = fetch) => {
}
const separated = separateResourceDiagnostics({ request, result });
cache.set(key, separated.result);
+ performanceCache.set(key, {
+ ...separated.performance,
+ loaderDurationMs,
+ });
if (separated.diagnostics === undefined) {
diagnosticsCache.delete(key);
} else {
@@ -152,6 +162,7 @@ const preloadResource = (resource: ResourceRequest) => {
const invalidateRequestState = (key: string) => {
diagnosticsCache.delete(key);
+ performanceCache.delete(key);
pendingDiagnostics.delete(key);
resourceVersions.set(key, (resourceVersions.get(key) ?? 0) + 1);
};
@@ -308,6 +319,7 @@ const reset = () => {
pending.clear();
cache.clear();
diagnosticsCache.clear();
+ performanceCache.clear();
pendingDiagnostics.clear();
knownRequests.clear();
resourceVersions.clear();
diff --git a/apps/builder/app/shared/rest-resources-loader.test.ts b/apps/builder/app/shared/rest-resources-loader.test.ts
index 711a35bcea..d13a59d7c2 100644
--- a/apps/builder/app/shared/rest-resources-loader.test.ts
+++ b/apps/builder/app/shared/rest-resources-loader.test.ts
@@ -81,3 +81,63 @@ test("batches Assets while starting and preserving independent remote resources"
});
expect(output[3][1]).toMatchObject({ ok: true, data: { source: "remote" } });
});
+
+test("records non-bindable server duration and response size", async () => {
+ const resource: ResourceRequest = {
+ name: "Remote",
+ method: "get",
+ url: "https://api.example/posts",
+ searchParams: [],
+ headers: [],
+ };
+ const request = new Request(
+ "https://p-090e6e14-ae50-4b2e-bd22-71733cec05bb.localhost/rest/resources-loader",
+ { method: "POST" }
+ );
+ const result = {
+ ok: true,
+ status: 200,
+ statusText: "OK",
+ data: { title: "Post" },
+ __performance__: {
+ assetQuery: { resolvedDocumentCount: 2 },
+ },
+ };
+ const now = vi.fn().mockReturnValueOnce(10).mockReturnValueOnce(35.5);
+
+ const output = await loadResourceRequestList(
+ {
+ request,
+ requestList: [resource],
+ sourceOrigin: "https://source.example",
+ includeDiagnostics: false,
+ customFetch: vi.fn(),
+ },
+ {
+ executeAssetQueries: vi.fn() as never,
+ loadResource: vi.fn().mockResolvedValue(result),
+ now,
+ }
+ );
+
+ expect(output).toEqual([
+ [
+ getResourceKey(resource),
+ {
+ ...result,
+ __performance__: {
+ ...result.__performance__,
+ serverDurationMs: 25.5,
+ responseBytes: new TextEncoder().encode(
+ JSON.stringify({
+ ok: result.ok,
+ status: result.status,
+ statusText: result.statusText,
+ data: result.data,
+ })
+ ).byteLength,
+ },
+ },
+ ],
+ ]);
+});
diff --git a/packages/asset-uploader/src/asset-repository.test.ts b/packages/asset-uploader/src/asset-repository.test.ts
index d107ddc03e..fdf62bba29 100644
--- a/packages/asset-uploader/src/asset-repository.test.ts
+++ b/packages/asset-uploader/src/asset-repository.test.ts
@@ -6,7 +6,10 @@ import {
loadCanonicalAssetBaseEntries,
synchronizeCanonicalAssets,
} from "./canonical-metadata-backfill";
-import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence";
+import {
+ loadCanonicalAssetFileEntries,
+ loadCanonicalAssetFileEntriesForRecovery,
+} from "./canonical-metadata-persistence";
import {
createContentCompilationPlan,
createContentRuntimeArtifact,
@@ -76,40 +79,52 @@ const createCompilationPlan = (query: AssetQueryInput) => {
return plan;
};
-const createDependencies = () => ({
- hasProjectPermit: vi.fn().mockResolvedValue(true),
- createUploadTicket: vi.fn<typeof createUploadTicket>(),
- uploadFile: vi.fn<typeof uploadFile>(),
- updateAssetContent: vi.fn<typeof updateAssetContent>(),
- deleteAssetsWithClient: vi.fn<typeof deleteAssetsWithClient>(),
- synchronizeCanonicalAssets: vi
- .fn<typeof synchronizeCanonicalAssets>()
- .mockResolvedValue({
- scanned: 0,
- indexed: 0,
- metadataUpdated: 0,
- unchanged: 0,
- removed: 0,
- skipped: 0,
- inconsistent: 0,
- issues: [],
- }),
- loadCanonicalAssetBaseEntries: vi
- .fn<typeof loadCanonicalAssetBaseEntries>()
- .mockResolvedValue([]),
- loadCanonicalAssetFileEntries: vi.fn<typeof loadCanonicalAssetFileEntries>(),
- createAssetIndex: vi.fn<typeof createAssetIndex>(),
- updateAssetMetadataWithClient: vi.fn<typeof updateAssetMetadataWithClient>(),
- loadAssetsByProjectWithClient: vi
- .fn<typeof loadAssetsByProjectWithClient>()
- .mockResolvedValue([]),
- loadAssetFoldersByProjectWithClient:
- vi.fn<typeof loadAssetFoldersByProjectWithClient>(),
- upsertAssetFolderWithClient: vi.fn<typeof upsertAssetFolderWithClient>(),
- deleteAssetFoldersWithClient: vi.fn<typeof deleteAssetFoldersWithClient>(),
- createId: vi.fn(() => "folder-1"),
- now: vi.fn(() => new Date("2026-07-25T00:00:00.000Z")),
-});
+const createDependencies = () => {
+ const loadFileEntries = vi
+ .fn<typeof loadCanonicalAssetFileEntries>()
+ .mockResolvedValue([]);
+ return {
+ hasProjectPermit: vi.fn().mockResolvedValue(true),
+ createUploadTicket: vi.fn<typeof createUploadTicket>(),
+ uploadFile: vi.fn<typeof uploadFile>(),
+ updateAssetContent: vi.fn<typeof updateAssetContent>(),
+ deleteAssetsWithClient: vi.fn<typeof deleteAssetsWithClient>(),
+ synchronizeCanonicalAssets: vi
+ .fn<typeof synchronizeCanonicalAssets>()
+ .mockResolvedValue({
+ scanned: 0,
+ indexed: 0,
+ metadataUpdated: 0,
+ unchanged: 0,
+ removed: 0,
+ skipped: 0,
+ inconsistent: 0,
+ issues: [],
+ }),
+ loadCanonicalAssetBaseEntries: vi
+ .fn<typeof loadCanonicalAssetBaseEntries>()
+ .mockResolvedValue([]),
+ loadCanonicalAssetFileEntries: loadFileEntries,
+ loadCanonicalAssetFileEntriesForRecovery: vi
+ .fn<typeof loadCanonicalAssetFileEntriesForRecovery>()
+ .mockImplementation(async (input) => ({
+ entries: await loadFileEntries(input),
+ inconsistentRows: [],
+ })),
+ createAssetIndex: vi.fn<typeof createAssetIndex>(),
+ updateAssetMetadataWithClient:
+ vi.fn<typeof updateAssetMetadataWithClient>(),
+ loadAssetsByProjectWithClient: vi
+ .fn<typeof loadAssetsByProjectWithClient>()
+ .mockResolvedValue([]),
+ loadAssetFoldersByProjectWithClient:
+ vi.fn<typeof loadAssetFoldersByProjectWithClient>(),
+ upsertAssetFolderWithClient: vi.fn<typeof upsertAssetFolderWithClient>(),
+ deleteAssetFoldersWithClient: vi.fn<typeof deleteAssetFoldersWithClient>(),
+ createId: vi.fn(() => "folder-1"),
+ now: vi.fn(() => new Date("2026-07-25T00:00:00.000Z")),
+ };
+};
describe("PostgresAssetRepository", () => {
test("lists, gets, and range-reads assets through the authorized object store", async () => {
@@ -488,7 +503,18 @@ describe("PostgresAssetRepository", () => {
];
const index = { integrity: { checksum: `sha256:${"b".repeat(64)}` } };
dependencies.loadCanonicalAssetBaseEntries.mockResolvedValue(entries);
- dependencies.loadCanonicalAssetFileEntries.mockResolvedValue(entries);
+ dependencies.loadCanonicalAssetFileEntries
+ .mockResolvedValueOnce([
+ {
+ ...entries[0],
+ revision: "stale-revision",
+ document: {
+ ...entries[0].document,
+ revision: "stale-revision",
+ },
+ },
+ ])
+ .mockResolvedValue(entries);
dependencies.createAssetIndex.mockResolvedValue(index as never);
const repository = new PostgresAssetRepository({
projectId: "project-1",
@@ -511,6 +537,7 @@ describe("PostgresAssetRepository", () => {
client: context.postgrest.client,
projectId: "project-1",
});
+ expect(dependencies.loadCanonicalAssetFileEntries).toHaveBeenCalledTimes(2);
expect(dependencies.createAssetIndex).toHaveBeenCalledWith({
projectId: "project-1",
entries,
@@ -651,7 +678,7 @@ describe("PostgresAssetRepository", () => {
expect(readFile).not.toHaveBeenCalled();
});
- test("reuses one compiled database across dynamic query values", async () => {
+ test("compiles concrete databases across dynamic query values", async () => {
const dependencies = createDependencies();
const entries = ["first", "second"].map((slug) => ({
projectId: "project-1",
@@ -677,7 +704,12 @@ describe("PostgresAssetRepository", () => {
document: { ...entry.document, properties: {} },
}))
);
- dependencies.loadCanonicalAssetFileEntries.mockResolvedValue(entries);
+ dependencies.loadCanonicalAssetFileEntries.mockImplementation(
+ async ({ assetIds }) =>
+ assetIds === undefined
+ ? entries
+ : entries.filter((entry) => assetIds.includes(entry.assetId))
+ );
dependencies.createAssetIndex.mockImplementation(createAssetIndex);
const databasePlan = createContentCompilationPlan([
{
@@ -788,7 +820,10 @@ describe("PostgresAssetRepository", () => {
data: { items: [{ properties: { title: "second post" } }] },
});
- expect(dependencies.createAssetIndex).toHaveBeenCalledOnce();
+ expect(dependencies.createAssetIndex).toHaveBeenCalledTimes(4);
+ expect(
+ dependencies.createAssetIndex.mock.calls.map(([input]) => input.plan)
+ ).not.toContain(databasePlan);
});
test("stops diagnostics work after cancellation during index preparation", async () => {
@@ -852,6 +887,7 @@ describe("PostgresAssetRepository", () => {
test("executes an Assets request batch with one authorization and union index", async () => {
const dependencies = createDependencies();
+ const performanceEvents: unknown[] = [];
const categories = ["Tools", "Strategy", "Guide", "Updates"];
const entries = categories.map((category, index) => ({
projectId: "project-1",
@@ -877,7 +913,12 @@ describe("PostgresAssetRepository", () => {
document: { ...entry.document, properties: {} },
}))
);
- dependencies.loadCanonicalAssetFileEntries.mockResolvedValue(entries);
+ dependencies.loadCanonicalAssetFileEntries.mockImplementation(
+ async ({ assetIds }) =>
+ assetIds === undefined
+ ? entries
+ : entries.filter((entry) => assetIds.includes(entry.assetId))
+ );
dependencies.createAssetIndex.mockImplementation(createAssetIndex);
const repository = new PostgresAssetRepository({
projectId: "project-1",
@@ -885,6 +926,7 @@ describe("PostgresAssetRepository", () => {
assetStore: assetClient,
dependencies,
compilationCache: createContentCompilationCache(),
+ onPerformanceEvent: (event: unknown) => performanceEvents.push(event),
});
const validRequests = categories.map((category) => ({
query: {
@@ -928,12 +970,148 @@ describe("PostgresAssetRepository", () => {
});
expect(dependencies.hasProjectPermit).toHaveBeenCalledOnce();
expect(dependencies.loadCanonicalAssetBaseEntries).toHaveBeenCalledTimes(2);
- expect(dependencies.synchronizeCanonicalAssets).toHaveBeenCalledOnce();
+ expect(dependencies.synchronizeCanonicalAssets).not.toHaveBeenCalled();
expect(dependencies.loadCanonicalAssetFileEntries).toHaveBeenCalledOnce();
expect(dependencies.createAssetIndex).toHaveBeenCalledOnce();
expect(
dependencies.createAssetIndex.mock.calls[0][0].plan?.queries
).toHaveLength(4);
+ expect(performanceEvents).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: "phase-completed",
+ phase: "repository-authorization",
+ durationMs: expect.any(Number),
+ }),
+ expect.objectContaining({
+ type: "phase-completed",
+ phase: "index-preparation",
+ durationMs: expect.any(Number),
+ }),
+ expect.objectContaining({
+ type: "compilation-cache",
+ status: "miss",
+ }),
+ ])
+ );
+ });
+
+ test("hydrates concrete preview content instead of the saved dynamic plan", async () => {
+ const dependencies = createDependencies();
+ const sources = new Map([
+ ["storage:post-a", "---\nslug: post-a\ntitle: First\n---\nFirst body\n"],
+ [
+ "storage:post-b",
+ "---\nslug: post-b\ntitle: Second\n---\nSecond body\n",
+ ],
+ ]);
+ const entries: CanonicalAssetFileEntry[] = [...sources].map(
+ ([contentRef, source]) => {
+ const slug = contentRef.slice("storage:".length);
+ return {
+ projectId: "project-1",
+ assetId: slug,
+ revision: `${slug}-r1`,
+ document: {
+ _id: slug,
+ _type: "asset.file",
+ name: `${slug}.md`,
+ path: `content/${slug}.md`,
+ key: slug,
+ extension: "md",
+ mimeType: "text/markdown; charset=utf-8",
+ size: new TextEncoder().encode(source).byteLength,
+ revision: `${slug}-r1`,
+ contentRef,
+ properties: {
+ slug,
+ title: slug === "post-a" ? "First" : "Second",
+ },
+ },
+ };
+ }
+ );
+ dependencies.loadCanonicalAssetBaseEntries.mockResolvedValue(entries);
+ dependencies.loadCanonicalAssetFileEntries.mockResolvedValue(entries);
+ dependencies.createAssetIndex.mockImplementation(createAssetIndex);
+ const readFile = vi.fn(async (contentRef: string) => {
+ const source = sources.get(contentRef);
+ if (source === undefined) {
+ throw new Error(`Missing source for ${contentRef}`);
+ }
+ return {
+ data: new Blob([source]).stream(),
+ contentLength: new TextEncoder().encode(source).byteLength,
+ };
+ });
+ const repository = new PostgresAssetRepository({
+ projectId: "project-1",
+ context,
+ assetStore: { readFile },
+ dependencies,
+ });
+ const databasePlan = createContentCompilationPlan([
+ {
+ id: "post-page",
+ where: {
+ field: ["properties", "slug"],
+ operator: "eq",
+ value: { type: "dynamic" },
+ },
+ sort: [],
+ limit: { type: "literal", value: 1 },
+ offset: { type: "literal", value: 0 },
+ output: {
+ mode: "fields",
+ includeMetadata: false,
+ fields: [["properties", "title"]],
+ },
+ content: { mode: "markdown-body-ref" },
+ },
+ ]);
+ if (databasePlan === undefined) {
+ throw new Error("Expected a build database plan");
+ }
+
+ const [result] = await repository.queryMany(
+ [
+ {
+ query: {
+ where: {
+ field: ["properties", "slug"],
+ operator: "eq",
+ value: "post-a",
+ },
+ limit: 1,
+ output: {
+ mode: "fields",
+ includeMetadata: false,
+ fields: [["properties", "title"]],
+ },
+ content: { mode: "markdown-body-ref" },
+ },
+ },
+ ],
+ { databasePlan }
+ );
+
+ expect(result).toMatchObject({
+ status: "fulfilled",
+ value: {
+ data: {
+ items: [
+ {
+ id: "post-a",
+ properties: { title: "First" },
+ content: { text: "First body\n" },
+ },
+ ],
+ },
+ },
+ });
+ expect(readFile.mock.calls.map(([contentRef]) => contentRef)).toEqual([
+ "storage:post-a",
+ ]);
});
test("loads a shared referenced document once across batch plans", async () => {
@@ -1275,7 +1453,7 @@ describe("PostgresAssetRepository", () => {
).toEqual(["preview", "preview"]);
});
- test("preserves truncation from the build database plan", async () => {
+ test("uses a concrete union instead of a truncated build plan", async () => {
const dependencies = createDependencies();
const entries = ["Tools", "Updates"].map((category, index) => ({
projectId: "project-1",
@@ -1337,10 +1515,13 @@ describe("PostgresAssetRepository", () => {
"fulfilled",
"fulfilled",
]);
- expect(dependencies.createAssetIndex).toHaveBeenCalledOnce();
- expect(dependencies.createAssetIndex.mock.calls[0][0].plan).toBe(
+ expect(dependencies.createAssetIndex).toHaveBeenCalledTimes(3);
+ expect(dependencies.createAssetIndex.mock.calls[0][0].plan).not.toBe(
databasePlan
);
+ expect(
+ dependencies.createAssetIndex.mock.calls[0][0].plan?.queries
+ ).toHaveLength(2);
});
test("falls back to independent queries when union preparation fails", async () => {
@@ -1440,13 +1621,7 @@ describe("PostgresAssetRepository", () => {
await expect(repository.readFieldCatalog()).resolves.toMatchObject({
fields: { "properties.title": expect.any(Object) },
});
- expect(dependencies.synchronizeCanonicalAssets).toHaveBeenCalledWith({
- client: context.postgrest.client,
- assetClient: assetStore,
- projectId: "project-1",
- assetIds: ["asset-1"],
- requirements: { structuredProperties: true, excerpt: false },
- });
+ expect(dependencies.synchronizeCanonicalAssets).not.toHaveBeenCalled();
expect(dependencies.createAssetIndex).not.toHaveBeenCalled();
expect(readFile).not.toHaveBeenCalled();
});
@@ -1479,7 +1654,6 @@ describe("PostgresAssetRepository", () => {
context,
assetStore: assetClient,
dependencies,
- compilationCache: createContentCompilationCache(),
});
const [first, second] = await Promise.all([
@@ -1491,7 +1665,7 @@ describe("PostgresAssetRepository", () => {
expect(first).toBe(second);
expect(second).toBe(third);
expect(dependencies.createAssetIndex).toHaveBeenCalledOnce();
- expect(dependencies.synchronizeCanonicalAssets).toHaveBeenCalledOnce();
+ expect(dependencies.synchronizeCanonicalAssets).not.toHaveBeenCalled();
expect(dependencies.loadCanonicalAssetFileEntries).toHaveBeenCalledOnce();
const updatedEntry = {
@@ -1568,13 +1742,7 @@ describe("PostgresAssetRepository", () => {
expect(index.documents[0].properties).toEqual({ title: "Post" });
expect(index.documents[0]).not.toHaveProperty("excerpt");
- expect(dependencies.synchronizeCanonicalAssets).toHaveBeenCalledWith({
- client: context.postgrest.client,
- assetClient: assetStore,
- projectId: "project-1",
- assetIds: ["asset-1"],
- requirements: { structuredProperties: true, excerpt: false },
- });
+ expect(dependencies.synchronizeCanonicalAssets).not.toHaveBeenCalled();
expect(uploadFile).not.toHaveBeenCalled();
});
@@ -1629,7 +1797,12 @@ describe("PostgresAssetRepository", () => {
document: { ...entry.document, properties: {} },
}))
);
- dependencies.loadCanonicalAssetFileEntries.mockResolvedValue(entries);
+ dependencies.loadCanonicalAssetFileEntries.mockImplementation(
+ async ({ assetIds }) =>
+ assetIds === undefined
+ ? entries
+ : entries.filter((entry) => assetIds.includes(entry.assetId))
+ );
dependencies.createAssetIndex.mockImplementation(createAssetIndex);
const repository = new PostgresAssetRepository({
projectId: "project-1",
@@ -1654,13 +1827,7 @@ describe("PostgresAssetRepository", () => {
})
);
- expect(dependencies.synchronizeCanonicalAssets).toHaveBeenCalledWith({
- client: context.postgrest.client,
- assetClient,
- projectId: "project-1",
- assetIds: ["published", "draft"],
- requirements: { structuredProperties: true, excerpt: false },
- });
+ expect(dependencies.synchronizeCanonicalAssets).not.toHaveBeenCalled();
expect(dependencies.loadCanonicalAssetFileEntries).toHaveBeenCalledWith({
client: context.postgrest.client,
projectId: "project-1",
@@ -1784,6 +1951,30 @@ describe("PostgresAssetRepository", () => {
message: "Object is missing",
},
];
+ dependencies.loadCanonicalAssetBaseEntries.mockResolvedValue([
+ {
+ projectId: "project-1",
+ assetId: "broken",
+ revision: "file:broken.md:now:4",
+ document: {
+ _id: "broken",
+ _type: "asset.file",
+ name: "broken.md",
+ path: "broken.md",
+ key: "broken",
+ extension: "md",
+ mimeType: "text/markdown",
+ size: 4,
+ revision: "file:broken.md:now:4",
+ contentRef: "broken.md",
+ properties: {},
+ },
+ },
+ ]);
+ dependencies.loadCanonicalAssetFileEntriesForRecovery.mockResolvedValue({
+ entries: [],
+ inconsistentRows: [],
+ });
dependencies.synchronizeCanonicalAssets.mockResolvedValue({
scanned: 1,
indexed: 0,
@@ -2002,6 +2193,10 @@ describe("PostgresAssetRepository", () => {
entry,
otherEntry,
]);
+ dependencies.loadCanonicalAssetFileEntriesForRecovery.mockResolvedValue({
+ entries: [],
+ inconsistentRows: [],
+ });
dependencies.createAssetIndex.mockImplementation(createAssetIndex);
const repository = new PostgresAssetRepository({
projectId: "project-1",
@@ -2286,6 +2481,7 @@ describe("PostgresAssetRepository", () => {
format: "webstudio-content-database",
}),
});
+ expect(readFile).toHaveBeenCalledTimes(2);
const artifacts = result.__diagnostics__.artifacts;
if (artifacts === undefined) {
throw new Error("Expected database artifacts in diagnostics");
@@ -2345,11 +2541,13 @@ describe("PostgresAssetRepository", () => {
data: new Blob([source]).stream(),
contentLength: entry.document.size,
}));
+ const performanceEvents: unknown[] = [];
const repository = new PostgresAssetRepository({
projectId: "project-1",
context,
assetStore: { readFile },
dependencies,
+ onPerformanceEvent: (event) => performanceEvents.push(event),
});
const result = await repository.query(
@@ -2390,5 +2588,26 @@ describe("PostgresAssetRepository", () => {
expect(result.__diagnostics__.artifacts?.query.documentGraph).toMatchObject(
{ nodes: [{ id: "post", format: "markdown" }], edges: [] }
);
+ expect(readFile).toHaveBeenCalledOnce();
+ expect(performanceEvents).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: "phase-completed",
+ phase: "document-graph",
+ durationMs: expect.any(Number),
+ }),
+ expect.objectContaining({
+ type: "phase-completed",
+ phase: "source-validation",
+ durationMs: expect.any(Number),
+ }),
+ {
+ type: "content-read",
+ purpose: "compiler-entry",
+ byteLength: entry.document.size,
+ durationMs: expect.any(Number),
+ },
+ ])
+ );
});
});
diff --git a/packages/asset-uploader/src/asset-repository.ts b/packages/asset-uploader/src/asset-repository.ts
index 72334468fa..bf919a66cc 100644
--- a/packages/asset-uploader/src/asset-repository.ts
+++ b/packages/asset-uploader/src/asset-repository.ts
@@ -31,6 +31,7 @@ import {
createContentSourceFile,
computeCanonicalAssetRevision,
decodeUtf8,
+ fullCanonicalAssetMetadataRequirements,
materializeContentSource,
materializeContentSnapshot,
ContentSourceChangedError,
@@ -75,29 +76,73 @@ import {
} from "./asset-patch-core";
import type { UploadTicket } from "./types";
import {
+ areCanonicalAssetFileEntriesCurrent,
loadCanonicalAssetBaseEntries,
synchronizeCanonicalAssets,
} from "./canonical-metadata-backfill";
-import { loadCanonicalAssetFileEntries } from "./canonical-metadata-persistence";
+import {
+ loadCanonicalAssetFileEntries,
+ loadCanonicalAssetFileEntriesForRecovery,
+} from "./canonical-metadata-persistence";
import {
deleteAssetFoldersWithClient,
loadAssetFoldersByProjectWithClient,
upsertAssetFolderWithClient,
} from "./folder-persistence";
import {
+ createContentCompilationCache,
createContentCompilationCacheKey,
getContentDatabaseForArtifact,
- sharedContentCompilationCache,
type ContentCompilationCache,
} from "./content-compilation-cache";
import {
AssetRepositoryConflictError,
AssetRepositoryNotFoundError,
} from "./asset-repository-errors";
+import {
+ emitAssetQueryPerformanceEvent,
+ measureAssetQueryPerformance,
+ type AssetQueryPerformanceObserver,
+ type AssetQueryPerformancePhase,
+} from "./query-performance";
type CreateId = () => Asset["id"];
type RepositoryObjectStore = AssetObjectReader & Partial<AssetObjectWriter>;
+type ContentBytesReference = {
+ contentRef: string;
+ revision: string;
+};
+
+class RequestContentBytesCache {
+ private values = new Map<string, Uint8Array>();
+ private byteLength = 0;
+
+ private getKey({ contentRef, revision }: ContentBytesReference) {
+ return JSON.stringify([contentRef, revision]);
+ }
+
+ get(reference: ContentBytesReference) {
+ return this.values.get(this.getKey(reference));
+ }
+
+ set({
+ contentRef,
+ revision,
+ bytes,
+ }: ContentBytesReference & { bytes: Uint8Array }) {
+ const key = this.getKey({ contentRef, revision });
+ const previous = this.values.get(key);
+ const nextByteLength =
+ this.byteLength - (previous?.byteLength ?? 0) + bytes.byteLength;
+ if (nextByteLength > contentEngineLimits.hydratedTotalBytes) {
+ return;
+ }
+ this.values.set(key, bytes);
+ this.byteLength = nextByteLength;
+ }
+}
+
const defaultDependencies = {
hasProjectPermit: authorizeProject.hasProjectPermit,
createUploadTicket,
@@ -114,7 +159,9 @@ const defaultDependencies = {
loadCanonicalAssetBaseEntries,
synchronizeCanonicalAssets,
loadCanonicalAssetFileEntries,
+ loadCanonicalAssetFileEntriesForRecovery,
createAssetIndex,
+ performanceNow: () => performance.now(),
};
type AssetRepositoryDependencies = typeof defaultDependencies;
@@ -128,7 +175,14 @@ export class AssetIndexPreparationError extends Error {
issues: Awaited<ReturnType<typeof synchronizeCanonicalAssets>>["issues"]
) {
super(
- `Asset index preparation failed for ${issues.length} asset${issues.length === 1 ? "" : "s"}: ${issues.map(({ assetId, storageName, message }) => `${assetId} (${storageName}): ${message}`).join("; ")}`
+ `Asset index preparation failed for ${issues.length} asset${
+ issues.length === 1 ? "" : "s"
+ }: ${issues
+ .map(
+ ({ assetId, storageName, message }) =>
+ `${assetId} (${storageName}): ${message}`
+ )
+ .join("; ")}`
);
this.name = "AssetIndexPreparationError";
this.issues = issues;
@@ -251,6 +305,7 @@ export class PostgresAssetRepository implements AssetRepository {
private readonly contentDatabaseMaxBytes: number;
private readonly compilationCache: ContentCompilationCache | undefined;
private readonly onDocumentGraphEvent?: DocumentGraphRuntimeObserver;
+ private readonly onPerformanceEvent?: AssetQueryPerformanceObserver;
constructor({
projectId,
@@ -260,14 +315,16 @@ export class PostgresAssetRepository implements AssetRepository {
contentDatabaseMaxBytes = contentEngineLimits.databaseBytes,
compilationCache,
onDocumentGraphEvent,
+ onPerformanceEvent,
}: {
projectId: string;
context: AppContext;
assetStore: RepositoryObjectStore;
dependencies?: Partial<AssetRepositoryDependencies>;
contentDatabaseMaxBytes?: number;
- compilationCache?: ContentCompilationCache;
+ compilationCache?: ContentCompilationCache | false;
onDocumentGraphEvent?: DocumentGraphRuntimeObserver;
+ onPerformanceEvent?: AssetQueryPerformanceObserver;
}) {
this.projectId = projectId;
this.context = context;
@@ -275,9 +332,23 @@ export class PostgresAssetRepository implements AssetRepository {
this.dependencies = { ...defaultDependencies, ...dependencies };
this.contentDatabaseMaxBytes = contentDatabaseMaxBytes;
this.compilationCache =
- compilationCache ??
- (dependencies === undefined ? sharedContentCompilationCache : undefined);
+ compilationCache === false
+ ? undefined
+ : (compilationCache ?? createContentCompilationCache());
this.onDocumentGraphEvent = onDocumentGraphEvent;
+ this.onPerformanceEvent = onPerformanceEvent;
+ }
+
+ private measurePerformance<Value>(
+ phase: AssetQueryPerformancePhase,
+ operation: () => Promise<Value>
+ ) {
+ return measureAssetQueryPerformance({
+ phase,
+ operation,
+ observer: this.onPerformanceEvent,
+ now: this.dependencies.performanceNow,
+ });
}
static async forUpload({
@@ -334,7 +405,9 @@ export class PostgresAssetRepository implements AssetRepository {
if (this.context.authorization?.type === "service") {
return;
}
- await this.assertCanView();
+ await this.measurePerformance("repository-authorization", () =>
+ this.assertCanView()
+ );
}
private getWritableStore(): AssetObjectStore {
@@ -604,9 +677,10 @@ export class PostgresAssetRepository implements AssetRepository {
private async prepareIndexAfterAuthorization(
requirements: ContentCompilationPlan | undefined,
- strict: boolean
+ strict: boolean,
+ contentBytesCache = new RequestContentBytesCache()
) {
- const source = this.createContentSource(strict);
+ const source = this.createContentSource(strict, contentBytesCache);
const compile = async (
entries: Parameters<typeof createAssetIndex>[0]["entries"],
assetReferences: Parameters<
@@ -617,24 +691,34 @@ export class PostgresAssetRepository implements AssetRepository {
>[0]["assetValueReferences"],
documentGraph: Parameters<typeof createAssetIndex>[0]["documentGraph"]
) =>
- await this.dependencies.createAssetIndex({
- projectId: this.projectId,
- entries,
- assetReferences,
- ...(assetValueReferences === undefined ||
- Object.keys(assetValueReferences).length === 0
- ? {}
- : { assetValueReferences }),
- documentGraph,
- maxBytes: this.contentDatabaseMaxBytes,
- ...(requirements === undefined ? {} : { plan: requirements }),
- });
+ await this.measurePerformance(
+ "artifact-compilation",
+ async () =>
+ await this.dependencies.createAssetIndex({
+ projectId: this.projectId,
+ entries,
+ assetReferences,
+ ...(assetValueReferences === undefined ||
+ Object.keys(assetValueReferences).length === 0
+ ? {}
+ : { assetValueReferences }),
+ documentGraph,
+ maxBytes: this.contentDatabaseMaxBytes,
+ ...(requirements === undefined ? {} : { plan: requirements }),
+ })
+ );
if (this.compilationCache === undefined) {
+ emitAssetQueryPerformanceEvent(this.onPerformanceEvent, {
+ type: "compilation-cache",
+ status: "disabled",
+ });
const { entries, assetReferences, assetValueReferences, documentGraph } =
await materializeContentSource({
source,
plan: requirements,
maximumContentBytes: this.contentDatabaseMaxBytes,
+ onPerformanceEvent: this.onPerformanceEvent,
+ performanceNow: this.dependencies.performanceNow,
});
return await compile(
entries,
@@ -644,7 +728,10 @@ export class PostgresAssetRepository implements AssetRepository {
);
}
for (let attempt = 0; attempt < 2; attempt += 1) {
- const snapshot = await source.openSnapshot();
+ const snapshot = await this.measurePerformance(
+ "source-snapshot",
+ source.openSnapshot
+ );
const key = createContentCompilationCacheKey({
projectId: this.projectId,
sourceRevision: snapshot.revision,
@@ -653,24 +740,34 @@ export class PostgresAssetRepository implements AssetRepository {
maxBytes: this.contentDatabaseMaxBytes,
});
try {
- return await this.compilationCache.getOrCreate(key, async () => {
- const {
- entries,
- assetReferences,
- assetValueReferences,
- documentGraph,
- } = await materializeContentSnapshot({
- snapshot,
- plan: requirements,
- maximumContentBytes: this.contentDatabaseMaxBytes,
- });
- return await compile(
- entries,
- assetReferences,
- assetValueReferences,
- documentGraph
- );
+ const cached = this.compilationCache.getOrCreateWithStatus(
+ key,
+ async () => {
+ const {
+ entries,
+ assetReferences,
+ assetValueReferences,
+ documentGraph,
+ } = await materializeContentSnapshot({
+ snapshot,
+ plan: requirements,
+ maximumContentBytes: this.contentDatabaseMaxBytes,
+ onPerformanceEvent: this.onPerformanceEvent,
+ performanceNow: this.dependencies.performanceNow,
+ });
+ return await compile(
+ entries,
+ assetReferences,
+ assetValueReferences,
+ documentGraph
+ );
+ }
+ );
+ emitAssetQueryPerformanceEvent(this.onPerformanceEvent, {
+ type: "compilation-cache",
+ status: cached.status,
});
+ return await cached.promise;
} catch (error) {
if (error instanceof ContentSourceChangedError === false) {
throw error;
@@ -680,8 +777,13 @@ export class PostgresAssetRepository implements AssetRepository {
throw new ContentSourceChangedError();
}
- private createContentSource(strict: boolean): ContentSource {
+ private createContentSource(
+ strict: boolean,
+ contentBytesCache = new RequestContentBytesCache()
+ ): ContentSource {
const readFile = this.assetStore.readFile;
+ const onPerformanceEvent = this.onPerformanceEvent;
+ const performanceNow = this.dependencies.performanceNow;
const loadBaseEntries = () =>
this.dependencies.loadCanonicalAssetBaseEntries({
client: this.context.postgrest.client,
@@ -707,22 +809,42 @@ export class PostgresAssetRepository implements AssetRepository {
id: entry.assetId,
source: {
[Symbol.asyncIterator]: async function* () {
+ const cached = contentBytesCache.get({
+ contentRef: entry.document.contentRef,
+ revision: entry.revision,
+ });
+ if (cached !== undefined) {
+ yield cached;
+ return;
+ }
+ const startedAt = performanceNow();
const response = await readFile(
entry.document.contentRef,
entry.document.size === 0
? undefined
: { offset: 0, length: entry.document.size }
);
- let readBytes = 0;
- for await (const chunk of response.data) {
- readBytes += chunk.byteLength;
- yield chunk;
- }
- if (readBytes !== entry.document.size) {
+ const bytes = await readBoundedBytes(
+ response.data,
+ entry.document.size
+ );
+ if (bytes.byteLength !== entry.document.size) {
throw new Error(
"Asset content does not match its canonical size"
);
}
+ contentBytesCache.set({
+ contentRef: entry.document.contentRef,
+ revision: entry.revision,
+ bytes,
+ });
+ emitAssetQueryPerformanceEvent(onPerformanceEvent, {
+ type: "content-read",
+ purpose: "document-graph",
+ byteLength: bytes.byteLength,
+ durationMs: Math.max(0, performanceNow() - startedAt),
+ });
+ yield bytes;
},
},
},
@@ -734,6 +856,7 @@ export class PostgresAssetRepository implements AssetRepository {
requirements: plan,
strict,
maximumContentBytes: options?.maximumContentBytes,
+ contentBytesCache,
}),
isCurrent: async () =>
(await computeCanonicalAssetRevision(await loadBaseEntries())) ===
@@ -748,11 +871,13 @@ export class PostgresAssetRepository implements AssetRepository {
requirements,
strict,
maximumContentBytes,
+ contentBytesCache,
}: {
baseEntries: Awaited<ReturnType<typeof loadCanonicalAssetBaseEntries>>;
requirements?: ContentCompilationPlan;
strict: boolean;
maximumContentBytes?: number;
+ contentBytesCache: RequestContentBytesCache;
}) {
const candidateBaseEntries =
requirements === undefined
@@ -774,54 +899,104 @@ export class PostgresAssetRepository implements AssetRepository {
requiresStructuredProperties(requirements) ||
requirements.excerpt
) {
- const result = await this.synchronizeTrusted(
- requirements,
- candidateAssetIds
- );
- if (strict && result.issues.length > 0) {
- throw new AssetIndexPreparationError(result.issues);
- }
- entries = await this.dependencies.loadCanonicalAssetFileEntries({
- client: this.context.postgrest.client,
- projectId: this.projectId,
- assetIds: candidateAssetIds,
- });
- }
- return await prepareContentCompilerEntries({
- entries,
- plan: requirements,
- loadContent: async (entry) => {
- const response = await this.assetStore.readFile(
- entry.document.contentRef,
- { offset: 0, length: entry.document.size }
- );
- const bytes = await readBoundedBytes(
- response.data,
- entry.document.size
- );
- if (bytes.byteLength !== entry.document.size) {
- throw new Error("Asset content does not match its canonical size");
- }
- let content: string;
- try {
- content = decodeUtf8(bytes);
- } catch {
- if (strict) {
- throw new AssetIndexPreparationError([
- {
- assetId: entry.assetId,
- storageName: entry.document.contentRef,
- revision: entry.revision,
- message: "Selected asset content is not valid UTF-8",
- },
- ]);
+ entries = await this.measurePerformance(
+ "canonical-metadata",
+ async () => {
+ const metadataRequirements =
+ requirements === undefined
+ ? fullCanonicalAssetMetadataRequirements
+ : {
+ structuredProperties:
+ requiresStructuredProperties(requirements),
+ excerpt: requirements.excerpt,
+ };
+ const current =
+ await this.dependencies.loadCanonicalAssetFileEntriesForRecovery({
+ client: this.context.postgrest.client,
+ projectId: this.projectId,
+ assetIds: candidateAssetIds,
+ });
+ if (
+ current.inconsistentRows.length === 0 &&
+ areCanonicalAssetFileEntriesCurrent({
+ baseEntries: candidateBaseEntries,
+ currentEntries: current.entries,
+ requirements: metadataRequirements,
+ })
+ ) {
+ return current.entries;
}
- return;
+ const result = await this.synchronizeTrusted(
+ requirements,
+ candidateAssetIds
+ );
+ if (strict && result.issues.length > 0) {
+ throw new AssetIndexPreparationError(result.issues);
+ }
+ return await this.dependencies.loadCanonicalAssetFileEntries({
+ client: this.context.postgrest.client,
+ projectId: this.projectId,
+ assetIds: candidateAssetIds,
+ });
}
- return content;
- },
- maximumContentBytes,
- });
+ );
+ }
+ return await this.measurePerformance(
+ "compiler-entries",
+ async () =>
+ await prepareContentCompilerEntries({
+ entries,
+ plan: requirements,
+ loadContent: async (entry) => {
+ const startedAt = this.dependencies.performanceNow();
+ const response = await this.assetStore.readFile(
+ entry.document.contentRef,
+ { offset: 0, length: entry.document.size }
+ );
+ const bytes = await readBoundedBytes(
+ response.data,
+ entry.document.size
+ );
+ if (bytes.byteLength !== entry.document.size) {
+ throw new Error(
+ "Asset content does not match its canonical size"
+ );
+ }
+ contentBytesCache.set({
+ contentRef: entry.document.contentRef,
+ revision: entry.revision,
+ bytes,
+ });
+ emitAssetQueryPerformanceEvent(this.onPerformanceEvent, {
+ type: "content-read",
+ purpose: "compiler-entry",
+ byteLength: bytes.byteLength,
+ durationMs: Math.max(
+ 0,
+ this.dependencies.performanceNow() - startedAt
+ ),
+ });
+ let content: string;
+ try {
+ content = decodeUtf8(bytes);
+ } catch {
+ if (strict) {
+ throw new AssetIndexPreparationError([
+ {
+ assetId: entry.assetId,
+ storageName: entry.document.contentRef,
+ revision: entry.revision,
+ message: "Selected asset content is not valid UTF-8",
+ },
+ ]);
+ }
+ return;
+ }
+ return content;
+ },
+ maximumContentBytes,
+ })
+ );
}
async readFieldCatalog() {
@@ -862,7 +1037,9 @@ export class PostgresAssetRepository implements AssetRepository {
);
}
- private createQueryDocumentLoader(): DocumentSourceLoader {
+ private createQueryDocumentLoader(
+ contentBytesCache?: RequestContentBytesCache
+ ): DocumentSourceLoader {
const onEvent = this.onDocumentGraphEvent;
return observeDocumentSourceLoader({
onEvent,
@@ -871,6 +1048,21 @@ export class PostgresAssetRepository implements AssetRepository {
if (node.format === undefined) {
throw new Error(`Document ${node.id} format is unavailable`);
}
+ const cached = contentBytesCache?.get({
+ contentRef: node.contentRef,
+ revision: node.revision,
+ });
+ if (cached !== undefined) {
+ return {
+ format: node.format,
+ revision: node.revision,
+ source: {
+ [Symbol.asyncIterator]: async function* () {
+ yield cached;
+ },
+ },
+ };
+ }
const response = await this.assetStore.readFile(node.contentRef);
signal?.throwIfAborted();
return {
@@ -889,7 +1081,9 @@ export class PostgresAssetRepository implements AssetRepository {
if (requests.length === 0) {
return [];
}
- await this.assertCanView();
+ await this.measurePerformance("repository-authorization", () =>
+ this.assertCanView()
+ );
signal?.throwIfAborted();
const executeIndividually = (request: AssetQueryRequestInput) =>
this.queryAfterAuthorization(request, {
@@ -908,11 +1102,12 @@ export class PostgresAssetRepository implements AssetRepository {
const query = queries[index];
return query === undefined ? [] : [query];
});
- // Queries covered by the saved plan must use its artifact so published
- // truncation and revision semantics remain unchanged. Only unpinned,
- // uncovered queries may use a temporary literal union.
- const databaseIndexes = requests.flatMap((_request, index) =>
+ // Revision-pinned requests must use the saved plan that produced the
+ // expected artifact. Ordinary Builder previews compile their concrete
+ // query values so a detail page hydrates only the documents it needs.
+ const databaseIndexes = requests.flatMap((request, index) =>
queries[index] !== undefined &&
+ request.indexRevision !== undefined &&
databasePlan !== undefined &&
isAssetQueryCoveredByCompilationPlan({
plan: databasePlan,
@@ -921,15 +1116,13 @@ export class PostgresAssetRepository implements AssetRepository {
? [index]
: []
);
- const databaseIndexSet = new Set(databaseIndexes);
const literalIndexes = requests.flatMap((request, index) =>
- databaseIndexSet.has(index) === false &&
- queries[index] !== undefined &&
- request.indexRevision === undefined
+ queries[index] !== undefined && request.indexRevision === undefined
? [index]
: []
);
- const load = this.createQueryDocumentLoader();
+ const contentBytesCache = new RequestContentBytesCache();
+ const load = this.createQueryDocumentLoader(contentBytesCache);
const resolutionSession = createDocumentResolutionSession({
load,
concurrency: contentEngineLimits.concurrentContentReads,
@@ -951,32 +1144,45 @@ export class PostgresAssetRepository implements AssetRepository {
}
try {
signal?.throwIfAborted();
- const artifact = await this.prepareIndexAfterAuthorization(
- artifactPlan,
- false
+ const artifact = await this.measurePerformance(
+ "index-preparation",
+ () =>
+ this.prepareIndexAfterAuthorization(
+ artifactPlan,
+ false,
+ contentBytesCache
+ )
);
signal?.throwIfAborted();
const database = getContentDatabaseForArtifact(artifact);
if (preserveTruncation || database.getStats().truncated === false) {
const runtimePlan = createAssetQueryBatchPlan(selectQueries(indexes));
- const runtimeAssets = await this.loadQueryRuntimeAssets({
- artifact,
- plan: runtimePlan,
- });
+ const runtimeAssets = await this.measurePerformance(
+ "runtime-assets",
+ () =>
+ this.loadQueryRuntimeAssets({
+ artifact,
+ plan: runtimePlan,
+ })
+ );
signal?.throwIfAborted();
let batchExecution:
| Promise<PromiseSettledResult<AssetQueryExecutionResult>[]>
| undefined;
const executeBatch = () => {
- batchExecution ??= database.queryManyWithDocumentGraph({
- requests: indexes.map((index) => requests[index]),
- readContent: this.assetStore.readFile,
- runtimeAssets,
- load,
- resolutionSession,
- signal,
- onEvent: this.onDocumentGraphEvent,
- });
+ batchExecution ??= this.measurePerformance(
+ "document-resolution",
+ () =>
+ database.queryManyWithDocumentGraph({
+ requests: indexes.map((index) => requests[index]),
+ readContent: this.assetStore.readFile,
+ runtimeAssets,
+ load,
+ resolutionSession,
+ signal,
+ onEvent: this.onDocumentGraphEvent,
+ })
+ );
return batchExecution;
};
for (const [position, index] of indexes.entries()) {
@@ -1040,7 +1246,9 @@ export class PostgresAssetRepository implements AssetRepository {
request: AssetQueryRequestInput,
options: AssetQueryPreviewOptions = {}
): Promise<AssetQueryExecutionPreviewResult | AssetQueryResultOnly> {
- await this.assertCanView();
+ await this.measurePerformance("repository-authorization", () =>
+ this.assertCanView()
+ );
return await this.queryAfterAuthorization(request, options);
}
@@ -1055,40 +1263,40 @@ export class PostgresAssetRepository implements AssetRepository {
}: AssetQueryPreviewOptions = {}
): Promise<AssetQueryExecutionPreviewResult | AssetQueryResultOnly> {
signal?.throwIfAborted();
+ const contentBytesCache = new RequestContentBytesCache();
const query = assetQuery.parse(request.query);
const plan = createContentCompilationPlan([
createLiteralContentCompilationQuery({ id: "preview", query }),
]);
- // Public REST queries are not necessarily represented by the Builder's
- // saved compilation plan. Reuse that database only when one of its queries
- // structurally covers the current runtime values.
+ // Revision-pinned requests must use the saved plan that produced the
+ // expected artifact. Unpinned previews compile the concrete query and only
+ // prepare the saved deployment plan for explicit diagnostics.
const coveredByDatabasePlan =
databasePlan !== undefined &&
isAssetQueryCoveredByCompilationPlan({ plan: databasePlan, query });
- const index = await this.prepareIndexAfterAuthorization(
- coveredByDatabasePlan ? databasePlan : plan,
- false
+ const usePublishedIndex =
+ request.indexRevision !== undefined && coveredByDatabasePlan;
+ const index = await this.measurePerformance("index-preparation", () =>
+ this.prepareIndexAfterAuthorization(
+ usePublishedIndex ? databasePlan : plan,
+ false,
+ contentBytesCache
+ )
);
signal?.throwIfAborted();
const database = getContentDatabaseForArtifact(index);
const queryIndex =
- includeDiagnostics && coveredByDatabasePlan
- ? await this.prepareIndexAfterAuthorization(plan, false)
+ includeDiagnostics && usePublishedIndex
+ ? await this.prepareIndexAfterAuthorization(
+ plan,
+ false,
+ contentBytesCache
+ )
: index;
signal?.throwIfAborted();
- let publishedIndex: typeof index | undefined;
- if (includeDiagnostics) {
- const publishedPlan = diagnosticsPlan ?? databasePlan;
- publishedIndex =
- publishedPlan === undefined
- ? queryIndex
- : await this.prepareIndexAfterAuthorization(publishedPlan, false);
- }
- signal?.throwIfAborted();
- const runtimeAssets = await this.loadQueryRuntimeAssets({
- artifact: index,
- plan,
- });
+ const runtimeAssets = await this.measurePerformance("runtime-assets", () =>
+ this.loadQueryRuntimeAssets({ artifact: index, plan })
+ );
signal?.throwIfAborted();
const unresolved = includeUnresolvedDiagnostics
? await getContentDatabaseForArtifact(queryIndex).query(
@@ -1100,14 +1308,40 @@ export class PostgresAssetRepository implements AssetRepository {
)
: undefined;
signal?.throwIfAborted();
- const data = await database.queryWithDocumentGraph({
- request,
- readContent: this.assetStore.readFile,
- runtimeAssets,
- load: this.createQueryDocumentLoader(),
- signal,
- onEvent: this.onDocumentGraphEvent,
- });
+ const data = await this.measurePerformance("document-resolution", () =>
+ database.queryWithDocumentGraph({
+ request,
+ readContent: this.assetStore.readFile,
+ runtimeAssets,
+ load: this.createQueryDocumentLoader(contentBytesCache),
+ signal,
+ onEvent: this.onDocumentGraphEvent,
+ })
+ );
+ signal?.throwIfAborted();
+ let publishedIndex: typeof index | undefined;
+ if (includeDiagnostics) {
+ const publishedPlan = diagnosticsPlan ?? databasePlan;
+ if (publishedPlan === undefined) {
+ publishedIndex = queryIndex;
+ } else {
+ const diagnosticsRepository = new PostgresAssetRepository({
+ projectId: this.projectId,
+ context: this.context,
+ assetStore: this.assetStore,
+ dependencies: this.dependencies,
+ contentDatabaseMaxBytes: this.contentDatabaseMaxBytes,
+ });
+ publishedIndex = await this.measurePerformance(
+ "diagnostics-preparation",
+ () =>
+ diagnosticsRepository.prepareIndexAfterAuthorization(
+ publishedPlan,
+ false
+ )
+ );
+ }
+ }
signal?.throwIfAborted();
const toCapacityStats = ({
usedBytes,
diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts
index a07b762469..833991a310 100644
--- a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts
+++ b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts
@@ -12,6 +12,7 @@ import {
} from "@webstudio-is/postgrest/testing";
import type { AssetObjectStore } from "./client";
import {
+ areCanonicalAssetFileEntriesCurrent,
loadCanonicalAssetBaseEntries,
synchronizeCanonicalAssets,
} from "./canonical-metadata-backfill";
@@ -36,6 +37,70 @@ const entryFromReplaceRpc = (value: ReplaceMetadataRpcArgs) => ({
});
describe("canonical asset metadata synchronization", () => {
+ test("recognizes metadata that already matches the source snapshot", () => {
+ const document = {
+ _id: "post",
+ _type: "asset.file" as const,
+ name: "post.md",
+ path: "post.md",
+ key: "post",
+ extension: "md",
+ mimeType: "text/markdown",
+ size: 20,
+ revision: "revision-1",
+ contentRef: "post.md",
+ properties: {},
+ };
+ const base = createCanonicalAssetFileEntry({
+ projectId: "project-1",
+ document,
+ metadataRequirements: { structuredProperties: false, excerpt: false },
+ });
+ const current = createCanonicalAssetFileEntry({
+ projectId: "project-1",
+ document: {
+ ...document,
+ properties: { title: "Post" },
+ excerpt: "Post body",
+ },
+ metadataRequirements: { structuredProperties: true, excerpt: true },
+ });
+
+ expect(
+ areCanonicalAssetFileEntriesCurrent({
+ baseEntries: [base],
+ currentEntries: [current],
+ requirements: { structuredProperties: true, excerpt: false },
+ })
+ ).toBe(true);
+ expect(
+ areCanonicalAssetFileEntriesCurrent({
+ baseEntries: [{ ...base, revision: "revision-2" }],
+ currentEntries: [current],
+ requirements: { structuredProperties: true, excerpt: false },
+ })
+ ).toBe(false);
+ expect(
+ areCanonicalAssetFileEntriesCurrent({
+ baseEntries: [
+ {
+ ...base,
+ document: { ...base.document, path: "archive/post.md" },
+ },
+ ],
+ currentEntries: [current],
+ requirements: { structuredProperties: true, excerpt: false },
+ })
+ ).toBe(false);
+ expect(
+ areCanonicalAssetFileEntriesCurrent({
+ baseEntries: [base],
+ currentEntries: [],
+ requirements: { structuredProperties: true, excerpt: false },
+ })
+ ).toBe(false);
+ });
+
test("builds base documents from records without canonical writes or object reads", async () => {
server.use(
db.get("Asset", () =>
diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.ts b/packages/asset-uploader/src/canonical-metadata-backfill.ts
index 56d32300d5..ddf763acda 100644
--- a/packages/asset-uploader/src/canonical-metadata-backfill.ts
+++ b/packages/asset-uploader/src/canonical-metadata-backfill.ts
@@ -219,6 +219,54 @@ const hasMatchingCanonicalDocument = (
);
};
+export const areCanonicalAssetFileEntriesCurrent = ({
+ baseEntries,
+ currentEntries,
+ requirements,
+}: {
+ baseEntries: readonly CanonicalAssetFileEntry[];
+ currentEntries: readonly CanonicalAssetFileEntry[];
+ requirements: CanonicalAssetMetadataRequirements;
+}) => {
+ if (baseEntries.length !== currentEntries.length) {
+ return false;
+ }
+ const currentByAssetId = new Map(
+ currentEntries.map((entry) => [entry.assetId, entry])
+ );
+ return baseEntries.every((base) => {
+ const current = currentByAssetId.get(base.assetId);
+ if (
+ current === undefined ||
+ current.revision !== base.revision ||
+ satisfiesCanonicalAssetMetadataRequirements({
+ cached: getCanonicalAssetMetadataRequirements(current),
+ required: requirements,
+ }) === false
+ ) {
+ return false;
+ }
+ const {
+ properties: _properties,
+ excerpt: _excerpt,
+ metadataError: _metadataError,
+ ...baseDocument
+ } = base.document;
+ const expected = {
+ ...baseDocument,
+ properties: current.document.properties,
+ ...(base.document.extension === "md" &&
+ current.document.excerpt !== undefined
+ ? { excerpt: current.document.excerpt }
+ : {}),
+ ...(current.document.metadataError === undefined
+ ? {}
+ : { metadataError: current.document.metadataError }),
+ };
+ return hasMatchingCanonicalDocument(current.document, expected);
+ });
+};
+
const indexCanonicalAsset = async ({
asset,
client,
diff --git a/packages/asset-uploader/src/content-compilation-cache.ts b/packages/asset-uploader/src/content-compilation-cache.ts
index c6623ba4a3..746d1ae6ac 100644
--- a/packages/asset-uploader/src/content-compilation-cache.ts
+++ b/packages/asset-uploader/src/content-compilation-cache.ts
@@ -54,36 +54,47 @@ export const createContentCompilationCache = (maximumEntries = 32) => {
entries.delete(completed[0]);
}
};
+ const getOrCreateWithStatus = (
+ key: string,
+ create: () => Promise<ContentArtifactV1>
+ ): {
+ promise: Promise<ContentArtifactV1>;
+ status: "hit" | "coalesced" | "miss";
+ } => {
+ const existing = entries.get(key);
+ if (existing !== undefined) {
+ entries.delete(key);
+ entries.set(key, existing);
+ return {
+ promise: existing.promise,
+ status: existing.settled ? "hit" : "coalesced",
+ };
+ }
+ let entry: CacheEntry;
+ const promise = create().then(
+ (value) => {
+ entry.settled = true;
+ trim();
+ return value;
+ },
+ (error) => {
+ if (entries.get(key) === entry) {
+ entries.delete(key);
+ }
+ throw error;
+ }
+ );
+ entry = { promise, settled: false };
+ entries.set(key, entry);
+ trim();
+ return { promise: entry.promise, status: "miss" };
+ };
return {
getOrCreate: (
key: string,
create: () => Promise<ContentArtifactV1>
- ): Promise<ContentArtifactV1> => {
- const existing = entries.get(key);
- if (existing !== undefined) {
- entries.delete(key);
- entries.set(key, existing);
- return existing.promise;
- }
- let entry: CacheEntry;
- const promise = create().then(
- (value) => {
- entry.settled = true;
- trim();
- return value;
- },
- (error) => {
- if (entries.get(key) === entry) {
- entries.delete(key);
- }
- throw error;
- }
- );
- entry = { promise, settled: false };
- entries.set(key, entry);
- trim();
- return entry.promise;
- },
+ ): Promise<ContentArtifactV1> => getOrCreateWithStatus(key, create).promise,
+ getOrCreateWithStatus,
clear: () => entries.clear(),
get size() {
return entries.size;
@@ -91,8 +102,6 @@ export const createContentCompilationCache = (maximumEntries = 32) => {
};
};
-export const sharedContentCompilationCache = createContentCompilationCache();
-
const databaseByArtifact = new WeakMap<
ContentArtifactV1,
ReturnType<typeof createContentDatabase>
diff --git a/packages/asset-uploader/src/query-performance.ts b/packages/asset-uploader/src/query-performance.ts
new file mode 100644
index 0000000000..a0900407aa
--- /dev/null
+++ b/packages/asset-uploader/src/query-performance.ts
@@ -0,0 +1,73 @@
+import type { DocumentGraphRuntimeObserver } from "@webstudio-is/content-engine";
+
+export type AssetQueryPerformancePhase =
+ | "build-plan"
+ | "repository-authorization"
+ | "source-snapshot"
+ | "canonical-metadata"
+ | "compiler-entries"
+ | "document-graph"
+ | "asset-references"
+ | "source-validation"
+ | "artifact-compilation"
+ | "index-preparation"
+ | "diagnostics-preparation"
+ | "runtime-assets"
+ | "document-resolution";
+
+export type AssetQueryPerformanceEvent =
+ | Readonly<{
+ type: "phase-completed";
+ phase: AssetQueryPerformancePhase;
+ durationMs: number;
+ }>
+ | Readonly<{
+ type: "compilation-cache";
+ status: "hit" | "coalesced" | "miss" | "disabled";
+ }>
+ | Readonly<{
+ type: "content-read";
+ purpose: "compiler-entry" | "document-graph";
+ byteLength: number;
+ durationMs: number;
+ }>;
+
+export type AssetQueryPerformanceObserver = (
+ event: AssetQueryPerformanceEvent
+) => void;
+
+export const emitAssetQueryPerformanceEvent = (
+ observer: AssetQueryPerformanceObserver | undefined,
+ event: AssetQueryPerformanceEvent
+) => {
+ try {
+ observer?.(event);
+ } catch {
+ // Observability must not change query behavior.
+ }
+};
+
+export const measureAssetQueryPerformance = async <Value>({
+ phase,
+ operation,
+ observer,
+ now = () => performance.now(),
+}: {
+ phase: AssetQueryPerformancePhase;
+ operation: () => Promise<Value>;
+ observer?: AssetQueryPerformanceObserver;
+ now?: () => number;
+}) => {
+ const startedAt = now();
+ try {
+ return await operation();
+ } finally {
+ emitAssetQueryPerformanceEvent(observer, {
+ type: "phase-completed",
+ phase,
+ durationMs: Math.max(0, now() - startedAt),
+ });
+ }
+};
+
+export type AssetQueryDocumentGraphObserver = DocumentGraphRuntimeObserver;
diff --git a/packages/asset-uploader/src/query-preview.ts b/packages/asset-uploader/src/query-preview.ts
index 4e33493783..a76377cd50 100644
--- a/packages/asset-uploader/src/query-preview.ts
+++ b/packages/asset-uploader/src/query-preview.ts
@@ -6,6 +6,7 @@ import {
import type { AssetObjectStore } from "./client";
import type { AppContext } from "@webstudio-is/trpc-interface/index.server";
import { PostgresAssetRepository } from "./asset-repository";
+import type { AssetQueryPerformanceObserver } from "./query-performance";
export const previewAssetResourceQuery = async ({
projectId,
@@ -18,6 +19,7 @@ export const previewAssetResourceQuery = async ({
includeDiagnostics,
includeUnresolvedDiagnostics,
onDocumentGraphEvent,
+ onPerformanceEvent,
signal,
}: {
projectId: string;
@@ -30,6 +32,7 @@ export const previewAssetResourceQuery = async ({
includeDiagnostics?: boolean;
includeUnresolvedDiagnostics?: boolean;
onDocumentGraphEvent?: DocumentGraphRuntimeObserver;
+ onPerformanceEvent?: AssetQueryPerformanceObserver;
signal?: AbortSignal;
}) => {
const repository = new PostgresAssetRepository({
@@ -38,6 +41,7 @@ export const previewAssetResourceQuery = async ({
assetStore: assetClient,
contentDatabaseMaxBytes,
onDocumentGraphEvent,
+ onPerformanceEvent,
});
const options = {
databasePlan,
@@ -65,6 +69,7 @@ export const previewAssetResourceQueries = async ({
contentDatabaseMaxBytes,
databasePlan,
onDocumentGraphEvent,
+ onPerformanceEvent,
signal,
}: {
projectId: string;
@@ -74,6 +79,7 @@ export const previewAssetResourceQueries = async ({
contentDatabaseMaxBytes?: number;
databasePlan?: ContentCompilationPlan;
onDocumentGraphEvent?: DocumentGraphRuntimeObserver;
+ onPerformanceEvent?: AssetQueryPerformanceObserver;
signal?: AbortSignal;
}) => {
const repository = new PostgresAssetRepository({
@@ -82,6 +88,7 @@ export const previewAssetResourceQueries = async ({
assetStore: assetClient,
contentDatabaseMaxBytes,
onDocumentGraphEvent,
+ onPerformanceEvent,
});
return await repository.queryMany(requests, { databasePlan, signal });
};
diff --git a/packages/asset-uploader/src/server.ts b/packages/asset-uploader/src/server.ts
index 7fce448c56..3bb26c4d70 100644
--- a/packages/asset-uploader/src/server.ts
+++ b/packages/asset-uploader/src/server.ts
@@ -11,6 +11,7 @@ export * from "./folder-persistence";
export * from "./revision";
export * from "./field-catalog";
export * from "./query-preview";
+export * from "./query-performance";
export * from "./publication";
export * from "./clients/fs/fs";
export * from "./clients/s3/s3";
diff --git a/packages/content-engine/src/content-source.ts b/packages/content-engine/src/content-source.ts
index f1223c9eb7..0b72b66d6e 100644
--- a/packages/content-engine/src/content-source.ts
+++ b/packages/content-engine/src/content-source.ts
@@ -57,6 +57,44 @@ export interface ContentSource {
openSnapshot(): Promise<ContentSourceSnapshot>;
}
+export type ContentSourcePerformancePhase =
+ | "document-graph"
+ | "asset-references"
+ | "source-validation";
+
+export type ContentSourcePerformanceObserver = (event: {
+ type: "phase-completed";
+ phase: ContentSourcePerformancePhase;
+ durationMs: number;
+}) => void;
+
+const measureContentSourcePerformance = async <Value>({
+ phase,
+ operation,
+ observer,
+ now,
+}: {
+ phase: ContentSourcePerformancePhase;
+ operation: () => Promise<Value> | Value;
+ observer?: ContentSourcePerformanceObserver;
+ now: () => number;
+}) => {
+ const startedAt = now();
+ try {
+ return await operation();
+ } finally {
+ try {
+ observer?.({
+ type: "phase-completed",
+ phase,
+ durationMs: Math.max(0, now() - startedAt),
+ });
+ } catch {
+ // Observability must not change compilation behavior.
+ }
+ }
+};
+
export const createContentSourceFile = ({
assetId,
revision,
@@ -295,30 +333,44 @@ export const materializeContentSnapshot = async ({
snapshot,
plan,
maximumContentBytes = contentEngineLimits.databaseBytes,
+ onPerformanceEvent,
+ performanceNow = () => performance.now(),
}: {
snapshot: ContentSourceSnapshot;
plan?: ContentCompilationPlan;
maximumContentBytes?: number;
+ onPerformanceEvent?: ContentSourcePerformanceObserver;
+ performanceNow?: () => number;
}) => {
validateSnapshot(snapshot);
try {
const entries = await snapshot.loadEntries(plan, { maximumContentBytes });
validateEntries({ snapshot, entries });
- const documentGraph = await discoverSnapshotDocumentGraph(
- snapshot,
- entries,
- plan
- );
- const assetReferences = await discoverSnapshotAssetReferences({
- snapshot,
- entries,
- plan,
+ const documentGraph = await measureContentSourcePerformance({
+ phase: "document-graph",
+ observer: onPerformanceEvent,
+ now: performanceNow,
+ operation: () => discoverSnapshotDocumentGraph(snapshot, entries, plan),
+ });
+ const assetReferences = await measureContentSourcePerformance({
+ phase: "asset-references",
+ observer: onPerformanceEvent,
+ now: performanceNow,
+ operation: () =>
+ discoverSnapshotAssetReferences({ snapshot, entries, plan }),
});
const assetValueReferences = discoverSnapshotAssetValueReferences({
snapshot,
entries,
});
- if (await snapshot.isCurrent()) {
+ if (
+ await measureContentSourcePerformance({
+ phase: "source-validation",
+ observer: onPerformanceEvent,
+ now: performanceNow,
+ operation: () => snapshot.isCurrent(),
+ })
+ ) {
return {
sourceRevision: snapshot.revision,
entries,
@@ -328,7 +380,14 @@ export const materializeContentSnapshot = async ({
};
}
} catch (error) {
- if (await snapshot.isCurrent()) {
+ if (
+ await measureContentSourcePerformance({
+ phase: "source-validation",
+ observer: onPerformanceEvent,
+ now: performanceNow,
+ operation: () => snapshot.isCurrent(),
+ })
+ ) {
throw error;
}
}
@@ -340,11 +399,15 @@ export const compileContentSource = async ({
projectId,
plan,
maxBytes,
+ onPerformanceEvent,
+ performanceNow,
}: {
source: ContentSource;
projectId: string;
plan?: ContentCompilationPlan;
maxBytes?: number;
+ onPerformanceEvent?: ContentSourcePerformanceObserver;
+ performanceNow?: () => number;
}): Promise<{
sourceRevision: string;
documentGraph?: DocumentGraph;
@@ -361,6 +424,8 @@ export const compileContentSource = async ({
source,
plan,
maximumContentBytes: maxBytes,
+ onPerformanceEvent,
+ performanceNow,
});
const compiled = await compileContentArtifact({
projectId,
@@ -378,10 +443,14 @@ export const materializeContentSource = async ({
source,
plan,
maximumContentBytes,
+ onPerformanceEvent,
+ performanceNow,
}: {
source: ContentSource;
plan?: ContentCompilationPlan;
maximumContentBytes?: number;
+ onPerformanceEvent?: ContentSourcePerformanceObserver;
+ performanceNow?: () => number;
}): Promise<{
sourceRevision: string;
entries: readonly ContentCompilerInput[];
@@ -396,6 +465,8 @@ export const materializeContentSource = async ({
snapshot,
plan,
maximumContentBytes,
+ onPerformanceEvent,
+ performanceNow,
});
} catch (error) {
if (error instanceof ContentSourceChangedError === false) {
diff --git a/packages/project/src/asset-query-preview.server.test.ts b/packages/project/src/asset-query-preview.server.test.ts
index 62383b7b0b..8afa9ddab9 100644
--- a/packages/project/src/asset-query-preview.server.test.ts
+++ b/packages/project/src/asset-query-preview.server.test.ts
@@ -70,6 +70,9 @@ describe("project asset query preview", () => {
previewAssetResourceQueries: vi.fn().mockResolvedValue(result),
};
const signal = new AbortController().signal;
+ const performanceEvents: unknown[] = [];
+ const onPerformanceEvent = (event: unknown) =>
+ performanceEvents.push(event);
await expect(
previewProjectAssetQueries(
@@ -80,6 +83,7 @@ describe("project asset query preview", () => {
assetClient,
contentDatabaseMaxBytes: 42,
signal,
+ onPerformanceEvent,
},
dependencies
)
@@ -98,7 +102,15 @@ describe("project asset query preview", () => {
queries: [expect.objectContaining({ id: "posts-resource" })],
}),
signal,
+ onPerformanceEvent,
});
+ expect(performanceEvents).toEqual([
+ expect.objectContaining({
+ type: "phase-completed",
+ phase: "build-plan",
+ durationMs: expect.any(Number),
+ }),
+ ]);
});
test("stops a cancelled batch before loading Build data", async () => {
@@ -136,7 +148,9 @@ describe("project asset query preview", () => {
const assetClient = { readFile: vi.fn() } as never;
const result = { data: { items: [] } } as never;
const dependencies = {
- loadDevBuildByProjectId: vi.fn().mockResolvedValue(build),
+ loadDevBuildContentEngineDataByProjectId: vi
+ .fn()
+ .mockResolvedValue(build),
previewAssetResourceQuery: vi.fn().mockResolvedValue(result),
};
@@ -153,11 +167,9 @@ describe("project asset query preview", () => {
)
).resolves.toBe(result);
- expect(dependencies.loadDevBuildByProjectId).toHaveBeenCalledWith(
- context,
- "project-1",
- undefined
- );
+ expect(
+ dependencies.loadDevBuildContentEngineDataByProjectId
+ ).toHaveBeenCalledWith(context, "project-1", undefined);
expect(dependencies.previewAssetResourceQuery).toHaveBeenCalledWith({
projectId: "project-1",
request: { query: { limit: 5 } },
@@ -176,11 +188,11 @@ describe("project asset query preview", () => {
});
});
- test("stops a cancelled query before loading the full Build", async () => {
+ test("stops a cancelled query before loading Build data", async () => {
const controller = new AbortController();
controller.abort();
const dependencies = {
- loadDevBuildByProjectId: vi
+ loadDevBuildContentEngineDataByProjectId: vi
.fn()
.mockResolvedValue({ props: [], dataSources: [], resources: [] }),
previewAssetResourceQuery: vi.fn().mockResolvedValue({}),
@@ -199,7 +211,9 @@ describe("project asset query preview", () => {
)
).rejects.toMatchObject({ name: "AbortError" });
- expect(dependencies.loadDevBuildByProjectId).not.toHaveBeenCalled();
+ expect(
+ dependencies.loadDevBuildContentEngineDataByProjectId
+ ).not.toHaveBeenCalled();
expect(dependencies.previewAssetResourceQuery).not.toHaveBeenCalled();
});
@@ -216,7 +230,7 @@ describe("project asset query preview", () => {
includeUnresolvedDiagnostics: true,
},
{
- loadDevBuildByProjectId: vi
+ loadDevBuildContentEngineDataByProjectId: vi
.fn()
.mockResolvedValue({ props: [], dataSources: [], resources: [] }),
previewAssetResourceQuery,
diff --git a/packages/project/src/asset-query-preview.server.ts b/packages/project/src/asset-query-preview.server.ts
index dedbdc47f4..c29704bc91 100644
--- a/packages/project/src/asset-query-preview.server.ts
+++ b/packages/project/src/asset-query-preview.server.ts
@@ -1,16 +1,16 @@
import {
previewAssetResourceQueries,
previewAssetResourceQuery,
+ measureAssetQueryPerformance,
type AssetObjectStore,
+ type AssetQueryDocumentGraphObserver,
+ type AssetQueryPerformanceObserver,
} from "@webstudio-is/asset-uploader/server";
import {
createAssetQueryPreviewCompilationPlan,
createBuildContentCompilationPlan,
} from "@webstudio-is/project-build";
-import {
- loadDevBuildByProjectId,
- loadDevBuildContentEngineDataByProjectId,
-} from "@webstudio-is/project-build/server";
+import { loadDevBuildContentEngineDataByProjectId } from "@webstudio-is/project-build/server";
import type { AppContext } from "@webstudio-is/trpc-interface/index.server";
const defaultBatchDependencies = {
@@ -19,7 +19,7 @@ const defaultBatchDependencies = {
};
const defaultQueryDependencies = {
- loadDevBuildByProjectId,
+ loadDevBuildContentEngineDataByProjectId,
previewAssetResourceQuery,
};
@@ -29,6 +29,8 @@ type ProjectAssetQueryContext = {
assetClient: Pick<AssetObjectStore, "readFile">;
contentDatabaseMaxBytes?: number;
signal?: AbortSignal;
+ onPerformanceEvent?: AssetQueryPerformanceObserver;
+ onDocumentGraphEvent?: AssetQueryDocumentGraphObserver;
};
export const previewProjectAssetQueries = async (
@@ -39,17 +41,24 @@ export const previewProjectAssetQueries = async (
assetClient,
contentDatabaseMaxBytes,
signal,
+ onPerformanceEvent,
+ onDocumentGraphEvent,
}: ProjectAssetQueryContext & {
requests: Parameters<typeof previewAssetResourceQueries>[0]["requests"];
},
dependencies = defaultBatchDependencies
) => {
signal?.throwIfAborted();
- const build = await dependencies.loadDevBuildContentEngineDataByProjectId(
- context,
- projectId,
- signal
- );
+ const build = await measureAssetQueryPerformance({
+ phase: "build-plan",
+ observer: onPerformanceEvent,
+ operation: () =>
+ dependencies.loadDevBuildContentEngineDataByProjectId(
+ context,
+ projectId,
+ signal
+ ),
+ });
signal?.throwIfAborted();
return await dependencies.previewAssetResourceQueries({
projectId,
@@ -59,6 +68,8 @@ export const previewProjectAssetQueries = async (
contentDatabaseMaxBytes,
databasePlan: createBuildContentCompilationPlan(build),
signal,
+ onPerformanceEvent,
+ onDocumentGraphEvent,
});
};
@@ -72,6 +83,8 @@ export const previewProjectAssetQuery = async (
includeDiagnostics,
includeUnresolvedDiagnostics,
signal,
+ onPerformanceEvent,
+ onDocumentGraphEvent,
}: ProjectAssetQueryContext & {
request: Parameters<typeof previewAssetResourceQuery>[0]["request"];
includeDiagnostics?: boolean;
@@ -80,11 +93,16 @@ export const previewProjectAssetQuery = async (
dependencies = defaultQueryDependencies
) => {
signal?.throwIfAborted();
- const build = await dependencies.loadDevBuildByProjectId(
- context,
- projectId,
- signal
- );
+ const build = await measureAssetQueryPerformance({
+ phase: "build-plan",
+ observer: onPerformanceEvent,
+ operation: () =>
+ dependencies.loadDevBuildContentEngineDataByProjectId(
+ context,
+ projectId,
+ signal
+ ),
+ });
signal?.throwIfAborted();
const databasePlan = createBuildContentCompilationPlan(build);
return await dependencies.previewAssetResourceQuery({
@@ -95,6 +113,8 @@ export const previewProjectAssetQuery = async (
contentDatabaseMaxBytes,
databasePlan,
signal,
+ onPerformanceEvent,
+ onDocumentGraphEvent,
...(includeDiagnostics === false
? {}
: {
diff --git a/packages/sdk/src/resource-loader.test.ts b/packages/sdk/src/resource-loader.test.ts
index 1a069c1e4a..2c60423ff9 100644
--- a/packages/sdk/src/resource-loader.test.ts
+++ b/packages/sdk/src/resource-loader.test.ts
@@ -245,6 +245,7 @@ describe("loadResource", () => {
hasMore: true,
},
__diagnostics__: { scope: "query-preview" },
+ __performance__: { serverDurationMs: 125.5, responseBytes: 1024 },
})
)
);
@@ -268,6 +269,7 @@ describe("loadResource", () => {
},
meta: { totalCount: 5, hasMore: true },
__diagnostics__: { scope: "query-preview" },
+ __performance__: { serverDurationMs: 125.5, responseBytes: 1024 },
});
expect(Object.keys(result.data as Record<string, unknown>)).toEqual([
"second",
diff --git a/packages/sdk/src/resource-loader.ts b/packages/sdk/src/resource-loader.ts
index 60a7c418ad..6b25909f47 100644
--- a/packages/sdk/src/resource-loader.ts
+++ b/packages/sdk/src/resource-loader.ts
@@ -157,6 +157,13 @@ const formatAssetsResourceResult = (value: unknown, body: unknown) => {
? body.query.result
: "many";
const diagnostics = isRecord(value) ? value.__diagnostics__ : undefined;
+ const performance = isRecord(value) ? value.__performance__ : undefined;
+ const internalMetadata = {
+ ...(preview === undefined || diagnostics === undefined
+ ? {}
+ : { __diagnostics__: diagnostics }),
+ ...(performance === undefined ? {} : { __performance__: performance }),
+ };
if (
resultMode !== "many" &&
isRecord(collection) &&
@@ -167,9 +174,7 @@ const formatAssetsResourceResult = (value: unknown, body: unknown) => {
return {
data: collection.item,
meta: { totalCount: collection.totalCount },
- ...(preview === undefined || diagnostics === undefined
- ? {}
- : { __diagnostics__: diagnostics }),
+ ...internalMetadata,
};
}
if (
@@ -195,9 +200,7 @@ const formatAssetsResourceResult = (value: unknown, body: unknown) => {
totalCount: collection.totalCount,
hasMore: collection.hasMore,
},
- ...(preview === undefined || diagnostics === undefined
- ? {}
- : { __diagnostics__: diagnostics }),
+ ...internalMetadata,
};
};