Skip to content

AI-native design editor. Open-source Figma alternative.

perf(fig): keep archive records as headers and decode them when read (#997)

Danila Poyarkov6afff42469a6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0e4f06bb..3ae4809ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -244,6 +244,7 @@
### Performance
+- Open large `.fig` files with a fraction of the memory: records stay in the file's bytes except for the few fields indexing needs, and are decoded whole only for the pages being read and the components they use. Opening the Nuxt UI design kit takes 0.7 GB instead of 2.2 GB, and its first twelve pages 1.2 GB instead of 2.6 GB.
- Save an edited `.fig` file by rewriting only the layers that changed and copying the rest from the file it was opened from, as it is stored, without decoding it; in the browser that work runs off the main thread. Saving the 109 MB Preline UI kit after an edit takes about 3 seconds instead of over two minutes, and crash recovery no longer freezes Safari after the first edit to a large file. Editing a component also rewrites its instances on pages that were not opened yet.
- Open `.fig` design kits with less memory: layers inside instances hold their component layers' paints, glyphs, and other unchanged values instead of a copy each, about a third fewer objects and an eighth less memory per loaded page.
- Save large `.fig` documents with less memory: once every page is loaded, the first save releases the decoded original file, and saves write from the document instead of a copy of it, about 1.5 GB less on a 74 MB design kit.
diff --git a/packages/fig/docs/source-model.md b/packages/fig/docs/source-model.md
index 949e8dc97..e2d34d85e 100644
--- a/packages/fig/docs/source-model.md
+++ b/packages/fig/docs/source-model.md
@@ -69,12 +69,22 @@ component sets. Current hierarchy and reference semantics must be considered ind
## Ownership
- Record-based reading copies caller-supplied records.
-- Archive-owned reading takes ownership of records it parses, avoiding a second full tree.
+- Archive-owned reading keeps each record as a header: its identity, place, type, and asset key
+ (`RECORD_HEADER_FIELDS`). The whole record is decoded from the archive bytes when a read
+ needs it and prepared as a record read whole is: bindings resolved, property definitions
+ inherited, style references applied. Opening decodes every record once to check its
+ bindings and keeps none; a page read keeps the records it decoded only while it runs, and
+ the components and styles the shared interpreter reads stay for the document.
- Exposed source/resource snapshots are copied.
- Evaluations produce independently mutable occurrence data.
- Blob and image references must remain valid across transfer and export.
-Do not replace copying with shared mutable objects merely to improve memory measurements.
+Do not replace copying of source records or occurrence data with shared mutable objects merely
+to improve memory measurements: the interpreter and the materializer write into what they are
+given. The materialized SceneGraph is different. A layer inside an instance holds its component
+layer's own value for every field no override changed (`shareUnchanged` in
+`../src/instance-overrides/materialize-instance.ts`), so a layer's values are replaced, never
+written into, once it exists.
## Implementation and tests
diff --git a/packages/fig/src/archive.ts b/packages/fig/src/archive.ts
index e00210b5f..f724e284a 100644
--- a/packages/fig/src/archive.ts
+++ b/packages/fig/src/archive.ts
@@ -4,6 +4,7 @@ import type { FigPageManifestEntry } from '@open-pencil/kiwi/fig'
import type { NodeChange } from '@open-pencil/kiwi/fig/codec'
import { buildFigKiwi, parseFigKiwiChunks } from '@open-pencil/kiwi/fig/container'
import { decodeFigKiwiCanvas, parseFigKiwiContainer } from '@open-pencil/kiwi/fig/parse'
+import type { HeaderFields } from '@open-pencil/kiwi/schema-runtime'
import { hasPNGSignature } from './thumbnail'
@@ -55,23 +56,28 @@ function isCanonicalCanvasEntry(name: string): boolean {
function parseRawFigKiwi(
bytes: Uint8Array,
- onPages?: (pages: FigPageManifestEntry[]) => void
+ onPages?: (pages: FigPageManifestEntry[]) => void,
+ headers?: HeaderFields
): FigParseResult | null {
const chunks = parseFigKiwiChunks(bytes)
if (!chunks) return null
- const decoded = decodeFigKiwiCanvas(bytes, onPages)
+ const decoded = decodeFigKiwiCanvas(bytes, onPages, headers)
const thumbnailPNG = chunks.slice(2).find(hasPNGSignature) ?? null
return { ...decoded, images: [], thumbnailPNG, metaJSON: null }
}
-/** Parse a complete zipped or legacy raw `.fig` file into its protocol payload and resources. */
+/**
+ * Parse a complete zipped or legacy raw `.fig` file into its protocol payload and resources.
+ * Records decoded as `headers` keep only the named fields; `decodeWhole` reads one in full.
+ */
export function parseFigBuffer(
buffer: ArrayBuffer,
- onPages?: (pages: FigPageManifestEntry[]) => void
+ onPages?: (pages: FigPageManifestEntry[]) => void,
+ headers?: HeaderFields
): FigParseResult {
const bytes = new Uint8Array(buffer)
- const raw = parseRawFigKiwi(bytes, onPages)
+ const raw = parseRawFigKiwi(bytes, onPages, headers)
if (raw) return raw
const canvasArchive = unzipSync(bytes, { filter: ({ name }) => isCanonicalCanvasEntry(name) })
@@ -79,7 +85,7 @@ export function parseFigBuffer(
let archive: Unzipped
let decoded: ReturnType<typeof decodeFigKiwiCanvas>
if (canvasData) {
- decoded = decodeFigKiwiCanvas(canvasData, onPages)
+ decoded = decodeFigKiwiCanvas(canvasData, onPages, headers)
archive = unzipSync(bytes, { filter: ({ name }) => !isCanonicalCanvasEntry(name) })
} else {
archive = unzipSync(bytes)
@@ -89,7 +95,7 @@ export function parseFigBuffer(
`No canvas data found in .fig file. Entries: ${Object.keys(archive).join(', ')}`
)
}
- decoded = decodeFigKiwiCanvas(canvasData, onPages)
+ decoded = decodeFigKiwiCanvas(canvasData, onPages, headers)
}
const metaBytes = archive['meta.json']
diff --git a/packages/fig/src/document/bindings/references.ts b/packages/fig/src/document/bindings/references.ts
index 6dfc083a5..db1b221d1 100644
--- a/packages/fig/src/document/bindings/references.ts
+++ b/packages/fig/src/document/bindings/references.ts
@@ -29,16 +29,11 @@ function visitChildren(
}
}
-/** Normalize supported binding references without mutating archive records or effective values. */
-export function resolveDocumentBindingReferences(
- changes: readonly NodeChange[],
- report: (diagnostic: BindingReferenceDiagnostic) => void,
- ownership: 'copy' | 'transfer' = 'copy'
-): NodeChange[] {
- const resolve = createResourceResolver(changes)
- return changes.map((source) => {
- const node = ownership === 'transfer' ? source : structuredClone(source)
- const sourceId = source.guid ? guidToString(source.guid) : 'unknown'
+type Report = (diagnostic: BindingReferenceDiagnostic) => void
+
+/** Resolves a record's binding references in place, reporting the ones that name nothing. */
+function createRecordNormalizer(resolve: ReturnType<typeof createResourceResolver>) {
+ return (record: NodeChange, sourceId: string, report?: Report): void => {
const normalize = (
reference: NodeChange['variableSetID'],
field: string,
@@ -47,7 +42,7 @@ export function resolveDocumentBindingReferences(
if (!reference?.assetRef || reference.guid) return
const id = resolve(reference)
if (!id) {
- report({ sourceId, field, key: reference.assetRef.key, path: structuredClone(path) })
+ report?.({ sourceId, field, key: reference.assetRef.key, path: structuredClone(path) })
return
}
reference.guid = stringToGuid(id)
@@ -78,7 +73,29 @@ export function resolveDocumentBindingReferences(
}
visitChildren(node, path, visit)
}
- visit(node, [])
+ visit(record, [])
+ }
+}
+
+/**
+ * Resolves the binding references of one whole record in place, as the document pass resolves
+ * every record: against the given records' variables, reporting what names nothing when asked.
+ */
+export function createBindingNormalizer(resources: readonly NodeChange[]) {
+ const normalizeRecord = createRecordNormalizer(createResourceResolver(resources))
+ return (record: NodeChange, report?: Report): void =>
+ normalizeRecord(record, record.guid ? guidToString(record.guid) : 'unknown', report)
+}
+
+/** Normalize supported binding references without mutating archive records or effective values. */
+export function resolveDocumentBindingReferences(
+ changes: readonly NodeChange[],
+ report: Report
+): NodeChange[] {
+ const normalize = createBindingNormalizer(changes)
+ return changes.map((source) => {
+ const node = structuredClone(source)
+ normalize(node, report)
return node
})
}
diff --git a/packages/fig/src/document/decoded-map.ts b/packages/fig/src/document/decoded-map.ts
new file mode 100644
index 000000000..c8d367842
--- /dev/null
+++ b/packages/fig/src/document/decoded-map.ts
@@ -0,0 +1,44 @@
+/**
+ * A read-only view of a map whose values are turned into something else as they are read, such
+ * as record headers into whole records. Reading by key decodes one value; iterating values
+ * decodes every one, so readers look values up by key.
+ */
+export class DecodedMap<K, V, W> implements ReadonlyMap<K, W> {
+ constructor(
+ private readonly source: ReadonlyMap<K, V>,
+ private readonly decode: (value: V) => W
+ ) {}
+
+ get size(): number {
+ return this.source.size
+ }
+
+ get(key: K): W | undefined {
+ const value = this.source.get(key)
+ return value === undefined ? undefined : this.decode(value)
+ }
+
+ has(key: K): boolean {
+ return this.source.has(key)
+ }
+
+ keys(): MapIterator<K> {
+ return this.source.keys()
+ }
+
+ *values(): MapIterator<W> {
+ for (const value of this.source.values()) yield this.decode(value)
+ }
+
+ *entries(): MapIterator<[K, W]> {
+ for (const [key, value] of this.source) yield [key, this.decode(value)]
+ }
+
+ forEach(callback: (value: W, key: K, map: ReadonlyMap<K, W>) => void): void {
+ for (const [key, value] of this.entries()) callback(value, key, this)
+ }
+
+ [Symbol.iterator](): MapIterator<[K, W]> {
+ return this.entries()
+ }
+}
diff --git a/packages/fig/src/document/dependency-closure.ts b/packages/fig/src/document/dependency-closure.ts
index 5ca655568..7587d66bf 100644
--- a/packages/fig/src/document/dependency-closure.ts
+++ b/packages/fig/src/document/dependency-closure.ts
@@ -58,7 +58,9 @@ function collectAncestors(
export function collectSceneDependencies(
changes: readonly NodeChange[],
pageIds: ReadonlySet<string> | undefined,
- index: SourceIndex
+ index: SourceIndex,
+ /** The whole record for one of `changes`, when they are headers that leave fields out. */
+ whole: (record: NodeChange) => NodeChange = (record) => record
): SceneDependencyClosure {
const resolveReference = createResourceResolver(changes)
const { sources, children } = index
@@ -90,15 +92,16 @@ export function collectSceneDependencies(
continue
}
contentIds.add(id)
- const components = componentDependencies(node, resolveReference, (key) =>
+ const record = whole(node)
+ const components = componentDependencies(record, resolveReference, (key) =>
externalPreferredKeys.add(key)
)
for (const component of components) componentReferences.add(component)
- const slotContent = slotContentDependencies(node)
+ const slotContent = slotContentDependencies(record)
for (const content of slotContent) slotContentReferences.add(content)
pending.push(
...(children.get(id) ?? []).flatMap((child) => idOf(child) ?? []),
- ...styleDependencies(node, resolveReference, availableIds),
+ ...styleDependencies(record, resolveReference, availableIds),
...components,
...slotContent
)
diff --git a/packages/fig/src/document/property-inheritance.ts b/packages/fig/src/document/property-inheritance.ts
index 50a1a2b49..f3776f9be 100644
--- a/packages/fig/src/document/property-inheritance.ts
+++ b/packages/fig/src/document/property-inheritance.ts
@@ -9,6 +9,27 @@ interface LinkedPropertyDefinition {
[field: string]: unknown
}
+/**
+ * A record's property definitions with each that names a parent definition completed from the
+ * nearest ancestor holding it. `ancestors` run outward and have their own inheritance applied.
+ */
+export function inheritFromAncestors(node: NodeChange, ancestors: readonly NodeChange[]): void {
+ const definitions = node.componentPropDefs as LinkedPropertyDefinition[] | undefined
+ if (!definitions) return
+ node.componentPropDefs = definitions.map((definition) => {
+ if (!definition.parentPropDefId) return definition
+ const id = guidToString(definition.parentPropDefId)
+ for (const ancestor of ancestors) {
+ const matches = (
+ (ancestor.componentPropDefs as LinkedPropertyDefinition[] | undefined) ?? []
+ ).filter((candidate) => candidate.id && guidToString(candidate.id) === id)
+ if (matches.length > 1) throw new Error(`Ambiguous parent property ${id}`)
+ if (matches.length === 1) return { ...structuredClone(matches[0]), ...definition }
+ }
+ throw new Error(`Missing parent property ${id}`)
+ })
+}
+
/** Resolve definition inheritance only within the source node's ancestry. */
export function inheritComponentPropertyDefinitions(
changes: readonly NodeChange[],
@@ -31,20 +52,7 @@ export function inheritComponentPropertyDefinitions(
ancestors.push(parent)
parent = sources.get(parentIdOf(parent) ?? '')
}
- const definitions = node.componentPropDefs as LinkedPropertyDefinition[] | undefined
- if (definitions)
- node.componentPropDefs = definitions.map((definition) => {
- if (!definition.parentPropDefId) return definition
- const id = guidToString(definition.parentPropDefId)
- for (const ancestor of ancestors) {
- const matches = (
- (ancestor.componentPropDefs as LinkedPropertyDefinition[] | undefined) ?? []
- ).filter((candidate) => candidate.id && guidToString(candidate.id) === id)
- if (matches.length > 1) throw new Error(`Ambiguous parent property ${id}`)
- if (matches.length === 1) return { ...structuredClone(matches[0]), ...definition }
- }
- throw new Error(`Missing parent property ${id}`)
- })
+ inheritFromAncestors(node, ancestors)
pending.delete(node)
complete.add(node)
}
diff --git a/packages/fig/src/document/read.ts b/packages/fig/src/document/read.ts
index 1c3c0e0e0..b5ef758f8 100644
--- a/packages/fig/src/document/read.ts
+++ b/packages/fig/src/document/read.ts
@@ -1,78 +1,219 @@
import type { NodeChange } from '@open-pencil/kiwi/fig/codec'
import { guidToString } from '@open-pencil/kiwi/fig/guid'
+import { deduplicateNodeChangePluginData } from '@open-pencil/kiwi/fig/parse'
+import { decodeWhole } from '@open-pencil/kiwi/schema-runtime'
import { parseFigBuffer } from '../archive'
-import { createOccurrenceInterpreter } from '../instance-overrides/interpret'
+import { createIndexedOccurrenceInterpreter } from '../instance-overrides/interpret'
import type { InterpretInstanceOptions } from '../instance-overrides/occurrence/types'
import {
bySavedPosition,
createSourceIndex,
+ parentIdOf,
type SourceIndex
} from '../instance-overrides/source-index'
import { symbolOverridesOf } from '../instance-overrides/types'
import { applyStyleRefsToFields } from '../node-change/style/refs'
import {
+ createBindingNormalizer,
resolveDocumentBindingReferences,
type BindingReferenceDiagnostic
} from './bindings/references'
import { planComponentConstruction } from './components'
+import { DecodedMap } from './decoded-map'
import { collectSceneDependencies } from './dependency-closure'
-import { inheritComponentPropertyDefinitions } from './property-inheritance'
+import { inheritComponentPropertyDefinitions, inheritFromAncestors } from './property-inheritance'
/** Indexed source document. Resources remain separate from scene occurrences. */
export function createDocumentReader(source: readonly NodeChange[], pageIds?: ReadonlySet<string>) {
- return createReader(source, 'copy', pageIds)
+ return createScopedReader(wholeRecords(source), pageIds)
}
+/**
+ * The fields a record keeps when an archive is read: what indexing, the dependency closure's
+ * walk, page listing, and resource lookup read. Every other field stays in the archive's bytes
+ * until something needs the whole record, which most records of a design kit never are.
+ */
+export const RECORD_HEADER_FIELDS = [
+ 'guid',
+ 'parentIndex',
+ 'type',
+ 'phase',
+ 'name',
+ 'internalOnly',
+ 'key',
+ 'version',
+ 'styleType',
+ 'overrideKey'
+] as const
+
/** Parse into exclusively owned records; callers never receive the mutable source index. */
export function createArchiveDocumentReader(bytes: ArrayBuffer, pageIds?: ReadonlySet<string>) {
- const parsed = parseFigBuffer(bytes)
+ const parsed = parseFigBuffer(bytes, undefined, { NodeChange: RECORD_HEADER_FIELDS })
return {
figKiwiVersion: parsed.figKiwiVersion,
figSchemaDeflated: parsed.figSchemaDeflated,
- reader: createReader(parsed.nodeChanges, 'transfer', pageIds),
+ reader: createScopedReader(archiveRecords(parsed.nodeChanges), pageIds),
blobs: parsed.blobs,
images: parsed.images,
- /** The archive's records; the reader shares them, so keeping them costs nothing more. */
+ /** The archive's record headers, which carry the identities patching numbers from. */
records: parsed.nodeChanges
}
}
-function createReader(
- source: readonly NodeChange[],
- ownership: 'copy' | 'transfer',
- pageIds?: ReadonlySet<string>
-) {
- const bindingDiagnostics: BindingReferenceDiagnostic[] = []
- const liveSource = source.filter((node) => node.phase !== 'REMOVED')
- const changes = resolveDocumentBindingReferences(
- liveSource,
- (diagnostic) => bindingDiagnostics.push(diagnostic),
- ownership
- )
- // One index over these records serves inheritance, style lookup, the dependency closure
- // and component planning.
- const index = createSourceIndex(changes)
- inheritComponentPropertyDefinitions(changes, index.sources)
- const styles = index.sources
+/** Whole records, prepared, by the record or header that stands for them. */
+type WholeRecord = (record: NodeChange) => NodeChange
+
+/**
+ * What a reader reads records through: every live record, whole or as a header, one index
+ * over them, and the whole records behind them, prepared as a document read whole prepares
+ * every record.
+ */
+interface ReaderRecords {
+ records: readonly NodeChange[]
+ /** Serves the dependency closure, component planning, and pages. */
+ index: SourceIndex
+ bindingDiagnostics: BindingReferenceDiagnostic[]
+ /** Whole records for one read of pages, each decoded at most once while it is held. */
+ scope(): WholeRecord
+ /** Whole records kept for the document: components the interpreter reads, and styles. */
+ shared: WholeRecord
+ /** Indexes whole records, for the interpreter that reads components on demand. */
+ interpreterIndex(): SourceIndex
+}
+
+function assetIds(records: readonly NodeChange[]): Map<string, string> {
const assets = new Map<string, string>()
- for (const node of changes)
+ for (const node of records)
if (node.guid && typeof node.key === 'string') {
const id = guidToString(node.guid)
assets.set(node.key, id)
if (typeof node.version === 'string') assets.set(`${node.key}@${node.version}`, id)
}
+ return assets
+}
+
+function isResource(record: NodeChange): boolean {
+ return record.type === 'VARIABLE' || record.type === 'VARIABLE_SET'
+}
+
+/** Records decoded whole up front, as clipboard fragments and in-memory documents are. */
+function wholeRecords(source: readonly NodeChange[]): ReaderRecords {
+ const bindingDiagnostics: BindingReferenceDiagnostic[] = []
+ const records = resolveDocumentBindingReferences(
+ source.filter((node) => node.phase !== 'REMOVED'),
+ (diagnostic) => bindingDiagnostics.push(diagnostic)
+ )
+ const index = createSourceIndex(records)
+ inheritComponentPropertyDefinitions(records, index.sources)
+ const assets = assetIds(records)
const resolveStyles = (node: NodeChange): void => {
- applyStyleRefsToFields(styles, node, assets)
+ applyStyleRefsToFields(index.sources, node, assets)
for (const override of symbolOverridesOf(node)) resolveStyles(override as NodeChange)
}
- for (const node of changes) resolveStyles(node)
- return createScopedReader(
- changes,
+ for (const node of records) resolveStyles(node)
+ const same: WholeRecord = (record) => record
+ return {
+ records,
+ index,
bindingDiagnostics,
- pageIds,
- createSharedReaderState(changes, index)
+ scope: () => same,
+ shared: same,
+ interpreterIndex: () => createSourceIndex(records.filter((record) => !isResource(record)))
+ }
+}
+
+/** A header's whole record, read again from the archive bytes, its plugin data deduplicated. */
+function decodeRecord(header: NodeChange): NodeChange {
+ const record = decodeWhole(header) as NodeChange | undefined
+ if (!record) return header
+ deduplicateNodeChangePluginData([record])
+ return record
+}
+
+/**
+ * Records kept as headers, each decoded whole when a read needs it. Opening checks every
+ * record's bindings, as a document read whole does, and keeps none of what it decodes; a whole
+ * record is then prepared as that read prepares it: bindings resolved, property definitions
+ * inherited from its ancestors, and style references applied.
+ */
+function archiveRecords(headers: readonly NodeChange[]): ReaderRecords {
+ const records = headers.filter((node) => node.phase !== 'REMOVED')
+ const index = createSourceIndex(records)
+ const normalize = createBindingNormalizer(records)
+ const assets = assetIds(records)
+ const bindingDiagnostics: BindingReferenceDiagnostic[] = []
+ for (const header of records)
+ normalize(decodeRecord(header), (diagnostic) => bindingDiagnostics.push(diagnostic))
+
+ const kept = new Map<NodeChange, NodeChange>()
+ /** Styles resolve against style records kept whole; other records only need their header. */
+ const styles = new DecodedMap(index.sources, (header) =>
+ header.styleType ? shared(header) : header
)
+ const prepare = (header: NodeChange, whole: WholeRecord): NodeChange => {
+ const record = decodeRecord(header)
+ normalize(record)
+ const ancestors: NodeChange[] = []
+ const visited = new Set<NodeChange>([header])
+ for (
+ let parent = index.sources.get(parentIdOf(header) ?? '');
+ parent;
+ parent = index.sources.get(parentIdOf(parent) ?? '')
+ ) {
+ if (visited.has(parent)) throw new Error('Cyclic component-property ancestry')
+ visited.add(parent)
+ ancestors.push(whole(parent))
+ }
+ inheritFromAncestors(record, ancestors)
+ const resolveStyles = (node: NodeChange): void => {
+ applyStyleRefsToFields(styles, node, assets)
+ for (const override of symbolOverridesOf(node)) resolveStyles(override as NodeChange)
+ }
+ resolveStyles(record)
+ return record
+ }
+ const shared: WholeRecord = (header) => {
+ let record = kept.get(header)
+ if (!record) {
+ record = prepare(header, shared)
+ kept.set(header, record)
+ }
+ return record
+ }
+ const interpreterIndex = (): SourceIndex => {
+ const { sources, children } = createSourceIndex(records.filter((record) => !isResource(record)))
+ const lists = new WeakMap<readonly NodeChange[], NodeChange[]>()
+ return {
+ sources: new DecodedMap(sources, shared),
+ children: new DecodedMap(children, (list) => {
+ let whole = lists.get(list)
+ if (!whole) {
+ whole = list.map(shared)
+ lists.set(list, whole)
+ }
+ return whole
+ })
+ }
+ }
+ return {
+ records,
+ index,
+ bindingDiagnostics,
+ scope: () => {
+ const decoded = new Map<NodeChange, NodeChange>()
+ const whole: WholeRecord = (header) => {
+ const record = kept.get(header) ?? decoded.get(header)
+ if (record) return record
+ const prepared = prepare(header, whole)
+ decoded.set(header, prepared)
+ return prepared
+ }
+ return whole
+ },
+ shared,
+ interpreterIndex
+ }
}
/**
@@ -80,47 +221,46 @@ function createReader(
* the page's own subset varies, so the whole-document work happens once per document.
*/
interface SharedReaderState {
- index: SourceIndex
- sourceInterpreter: ReturnType<typeof createOccurrenceInterpreter>
+ records: ReaderRecords
+ sourceInterpreter: ReturnType<typeof createIndexedOccurrenceInterpreter>
}
-function createSharedReaderState(changes: readonly NodeChange[], index: SourceIndex) {
+function createSharedReaderState(records: ReaderRecords): SharedReaderState {
let sourceInterpreter: SharedReaderState['sourceInterpreter'] | undefined
return {
- index,
+ records,
get sourceInterpreter() {
- sourceInterpreter ??= createOccurrenceInterpreter(
- changes.filter((change) => change.type !== 'VARIABLE' && change.type !== 'VARIABLE_SET')
- )
+ sourceInterpreter ??= createIndexedOccurrenceInterpreter(records.interpreterIndex())
return sourceInterpreter
}
}
}
function createScopedReader(
- changes: NodeChange[],
- bindingDiagnostics: BindingReferenceDiagnostic[],
- pageIds: ReadonlySet<string> | undefined,
- shared: SharedReaderState
+ source: ReaderRecords | SharedReaderState,
+ pageIds: ReadonlySet<string> | undefined
) {
- const resources = changes.filter(
- (change) => change.type === 'VARIABLE' || change.type === 'VARIABLE_SET'
- )
- const closure = collectSceneDependencies(changes, pageIds, shared.index)
+ const shared = 'sourceInterpreter' in source ? source : createSharedReaderState(source)
+ const { records, index, bindingDiagnostics } = shared.records
+ const changes = records
+ const whole = shared.records.scope()
+ const resources = changes.filter(isResource)
+ const closure = collectSceneDependencies(changes, pageIds, index, whole)
// Deleted components are interpreted per instance; broken hierarchy is not recoverable.
if (closure.missingIds.size)
throw new Error(`Missing reachable sources: ${[...closure.missingIds].join(', ')}`)
- const sceneChanges = changes.filter(
- (change) =>
- change.type !== 'VARIABLE' &&
- change.type !== 'VARIABLE_SET' &&
- (change.type === 'CANVAS' ||
- (change.guid &&
- (closure.contentIds.has(guidToString(change.guid)) ||
- closure.ancestorIds.has(guidToString(change.guid)))))
- )
+ const sceneChanges = changes
+ .filter(
+ (change) =>
+ !isResource(change) &&
+ (change.type === 'CANVAS' ||
+ (change.guid &&
+ (closure.contentIds.has(guidToString(change.guid)) ||
+ closure.ancestorIds.has(guidToString(change.guid)))))
+ )
+ .map(whole)
const sourceInterpreter = shared.sourceInterpreter
- const interpreter = createOccurrenceInterpreter(sceneChanges)
+ const interpreter = createIndexedOccurrenceInterpreter(createSourceIndex(sceneChanges))
const pages = changes
.filter((change) => change.type === 'CANVAS')
.toSorted(bySavedPosition)
@@ -136,18 +276,19 @@ function createScopedReader(
const knownPageIds = new Set(pages.map((page) => page.id))
return {
selectPages(ids: ReadonlySet<string>) {
- return createScopedReader(changes, bindingDiagnostics, ids, shared)
+ return createScopedReader(shared, ids)
},
get sourceRecords() {
- return structuredClone(changes)
+ return structuredClone(changes.map(whole))
},
get documentRecord() {
- return structuredClone(changes.find((change) => change.type === 'DOCUMENT'))
+ const document = changes.find((change) => change.type === 'DOCUMENT')
+ return structuredClone(document && whole(document))
},
dependencyClosure: closure,
pages,
get resources() {
- return structuredClone(resources)
+ return structuredClone(resources.map(whole))
},
bindingDiagnostics,
readPage(id: string, options: InterpretInstanceOptions = {}) {
@@ -164,7 +305,7 @@ function createScopedReader(
return planComponentConstruction(
roots,
(id) => sourceInterpreter.component(id, options),
- shared.index.sources
+ index.sources
)
},
readComponent: sourceInterpreter.component
diff --git a/packages/fig/src/instance-overrides/interpret.ts b/packages/fig/src/instance-overrides/interpret.ts
index 199bb116b..27c6b87bd 100644
--- a/packages/fig/src/instance-overrides/interpret.ts
+++ b/packages/fig/src/instance-overrides/interpret.ts
@@ -83,7 +83,11 @@ export function interpretComponent(
/** One source index per document; evaluation state remains local to each call. */
export function createOccurrenceInterpreter(changes: readonly NodeChange[]) {
- const index = createSourceIndex(changes)
+ return createIndexedOccurrenceInterpreter(createSourceIndex(changes))
+}
+
+/** An interpreter over an index built elsewhere, such as one that decodes records as read. */
+export function createIndexedOccurrenceInterpreter(index: SourceIndex) {
return {
instance: (id: string, options: InterpretInstanceOptions = {}) =>
interpretRoot(index, id, 'INSTANCE', options),
diff --git a/packages/fig/tests/document/bindings/references.test.ts b/packages/fig/tests/document/bindings/references.test.ts
index 8ee58b58c..bcdf46843 100644
--- a/packages/fig/tests/document/bindings/references.test.ts
+++ b/packages/fig/tests/document/bindings/references.test.ts
@@ -1,7 +1,10 @@
import { expect, test } from 'bun:test'
import { guid } from '#fig-tests/helpers/guid'
-import { resolveDocumentBindingReferences } from '#fig/document/bindings/references'
+import {
+ createBindingNormalizer,
+ resolveDocumentBindingReferences
+} from '#fig/document/bindings/references'
import { nodeChangeToProps } from '@open-pencil/fig/node-change'
import type { NodeChange } from '@open-pencil/kiwi/fig/codec'
@@ -107,3 +110,42 @@ for (const map of ['variableConsumptionMap', 'parameterConsumptionMap'] as const
})
}
}
+
+// A record decoded whole later resolves as the document pass resolved it: a legacy entry a
+// parameter entry shadows stays as saved, and the rest resolve against the document's variables.
+test('a record normalized on its own resolves as the document pass resolves it', () => {
+ const variable = { guid: guid(9), type: 'VARIABLE', key: 'width', version: 'v1' } as NodeChange
+ const record = () =>
+ ({
+ guid: guid(1),
+ type: 'RECTANGLE',
+ variableConsumptionMap: {
+ entries: [
+ {
+ variableField: 'WIDTH',
+ variableData: {
+ dataType: 'ALIAS',
+ value: { alias: { assetRef: { key: 'width', version: 'v1' } } }
+ }
+ },
+ {
+ variableField: 'HEIGHT',
+ variableData: {
+ dataType: 'ALIAS',
+ value: { alias: { assetRef: { key: 'width', version: 'v1' } } }
+ }
+ }
+ ]
+ },
+ parameterConsumptionMap: { entries: [{ variableField: 'WIDTH' }] }
+ }) as NodeChange
+ const [, passed] = resolveDocumentBindingReferences([variable, record()], () => undefined)
+ const alone = record()
+
+ createBindingNormalizer([variable, record()])(alone)
+
+ expect(alone).toEqual(passed)
+ const entries = alone.variableConsumptionMap?.entries ?? []
+ expect(entries[0]?.variableData?.value?.alias).not.toHaveProperty('guid')
+ expect(entries[1]?.variableData?.value?.alias).toHaveProperty('guid', guid(9))
+})
diff --git a/packages/fig/tests/document/record-headers.test.ts b/packages/fig/tests/document/record-headers.test.ts
new file mode 100644
index 000000000..1f9cbafd8
--- /dev/null
+++ b/packages/fig/tests/document/record-headers.test.ts
@@ -0,0 +1,68 @@
+import { expect, test } from 'bun:test'
+
+import { readFixtureArrayBuffer } from '#fig-tests/helpers/fig-fixtures'
+import { parseFigBuffer } from '#fig/archive'
+import { materializeDocument, materializeFigArchive } from '#fig/document/materialize'
+import { RECORD_HEADER_FIELDS } from '#fig/document/read'
+
+import { deduplicateNodeChangePluginData } from '@open-pencil/kiwi/fig/parse'
+import { decodeWhole } from '@open-pencil/kiwi/schema-runtime'
+import { setIdSession, type SceneGraph } from '@open-pencil/scene-graph'
+
+const FIXTURE = 'gold-preview.fig'
+
+/** A session no Figma file's GUIDs use, so the IDs this test mints stand apart from theirs. */
+const TEST_SESSION = 7_777
+
+/**
+ * A document's layers as JSON with every ID this test minted renamed by first appearance: IDs
+ * come from one process-wide counter, so two documents built in turn differ only in them.
+ */
+function canonical(graph: SceneGraph): string {
+ const json = JSON.stringify([...graph.nodes.values()], (_key, value: unknown) => {
+ if (value instanceof Map) return [...value]
+ if (value instanceof Set) return [...value]
+ if (value instanceof Uint8Array) return [...value]
+ return value
+ })
+ const order = new Map<string, number>()
+ return json.replace(new RegExp(`"${TEST_SESSION}:\\d+"`, 'g'), (id) => {
+ if (!order.has(id)) order.set(id, order.size)
+ return `"#${order.get(id)}"`
+ })
+}
+
+test('record headers keep only their header fields and decode whole into the eager records', () => {
+ const bytes = readFixtureArrayBuffer(FIXTURE)
+ const eager = parseFigBuffer(bytes.slice(0)).nodeChanges
+ const headers = parseFigBuffer(bytes.slice(0), undefined, {
+ NodeChange: RECORD_HEADER_FIELDS
+ }).nodeChanges
+ const header = new Set<string>(RECORD_HEADER_FIELDS)
+
+ expect(headers.flatMap(Object.keys).filter((field) => !header.has(field))).toEqual([])
+ const whole = headers.map((record) => decodeWhole(record) as (typeof eager)[number])
+ deduplicateNodeChangePluginData(whole)
+ expect(whole).toEqual(eager)
+})
+
+// The archive reader keeps records as headers and decodes each whole when a read needs it; the
+// document it builds must match one built from records decoded whole up front.
+test('an archive reader with record headers builds the document records decoded whole build', () => {
+ const bytes = readFixtureArrayBuffer(FIXTURE)
+ const parsed = parseFigBuffer(bytes.slice(0))
+ const options = { derivedBounds: true, onUnresolvedProperty: () => undefined }
+
+ setIdSession(TEST_SESSION)
+ try {
+ const whole = materializeDocument(parsed.nodeChanges, parsed.blobs, {
+ ...options,
+ images: new Map(parsed.images)
+ })
+ const lazy = materializeFigArchive(bytes.slice(0), options)
+
+ expect(canonical(lazy.graph)).toBe(canonical(whole.graph))
+ } finally {
+ setIdSession(0)
+ }
+})
diff --git a/packages/kiwi/src/fig/parse.ts b/packages/kiwi/src/fig/parse.ts
index 1113035e8..54115ac5e 100644
--- a/packages/kiwi/src/fig/parse.ts
+++ b/packages/kiwi/src/fig/parse.ts
@@ -1,7 +1,7 @@
import { inflateSync } from 'fflate'
import { decompress as zstdDecompress } from 'fzstd'
-import { decodeBinarySchema, compileSchema, ByteBuffer } from '../schema-runtime'
+import { decodeBinarySchema, compileSchema, ByteBuffer, type HeaderFields } from '../schema-runtime'
import type { FigmaMessage, NodeChange } from './codec'
import { extractFigPageManifest, type FigPageManifestEntry } from './page-manifest'
import { isZstdCompressed } from './protocol'
@@ -96,7 +96,8 @@ export interface FigKiwiDecodeResult {
/** Decode one raw `fig-kiwi` canvas payload. Outer `.fig` archive handling lives in `@open-pencil/fig`. */
export function decodeFigKiwiCanvas(
data: Uint8Array,
- onPages?: (pages: FigPageManifestEntry[]) => void
+ onPages?: (pages: FigPageManifestEntry[]) => void,
+ headers?: HeaderFields
): FigKiwiDecodeResult {
const payload = parseFigKiwiContainer(data)
if (!payload) throw new Error('Invalid fig-kiwi container')
@@ -111,7 +112,7 @@ export function decodeFigKiwiCanvas(
console.warn('Failed to scan FIG page manifest; continuing with full decode:', error)
}
}
- const compiled = compileSchema(schema) as CompiledKiwiSchema
+ const compiled = compileSchema(schema, headers) as CompiledKiwiSchema
const message = compiled.decodeMessage(payload.dataRaw) as FigmaMessage
const nodeChanges = message.nodeChanges
diff --git a/packages/kiwi/src/schema-runtime/bb.ts b/packages/kiwi/src/schema-runtime/bb.ts
index ab3367773..2af50d258 100644
--- a/packages/kiwi/src/schema-runtime/bb.ts
+++ b/packages/kiwi/src/schema-runtime/bb.ts
@@ -24,6 +24,11 @@ export class ByteBuffer {
this._index = value
}
+ /** The bytes this buffer reads, as a view rather than a copy. */
+ get bytes(): Uint8Array {
+ return this._data
+ }
+
/**
* Returns a view into the internal buffer, not a copy.
*
diff --git a/packages/kiwi/src/schema-runtime/index.ts b/packages/kiwi/src/schema-runtime/index.ts
index 18cdf264a..73ebda4a3 100644
--- a/packages/kiwi/src/schema-runtime/index.ts
+++ b/packages/kiwi/src/schema-runtime/index.ts
@@ -1,6 +1,10 @@
export type { Schema, Definition, Field } from './schema'
export { ByteBuffer } from './bb'
-export { compileSchemaRuntime as compileSchema } from './interpreter'
+export {
+ compileSchemaRuntime as compileSchema,
+ decodeWhole,
+ type HeaderFields
+} from './interpreter'
export { decodeBinarySchema, encodeBinarySchema } from './binary'
export { parseSchema } from './parser'
export { createSchemaSkipper, type SchemaSkipper } from './skip'
diff --git a/packages/kiwi/src/schema-runtime/interpreter.ts b/packages/kiwi/src/schema-runtime/interpreter.ts
index 69debb4a1..f97416c74 100644
--- a/packages/kiwi/src/schema-runtime/interpreter.ts
+++ b/packages/kiwi/src/schema-runtime/interpreter.ts
@@ -1,6 +1,7 @@
import { ByteBuffer } from './bb'
import { nativeTypes } from './parser'
import type { Definition, Field, Schema } from './schema'
+import { createSchemaSkipper, type SchemaSkipper } from './skip'
import { error, quote } from './util'
type RuntimeEnum = { [name: string]: string | number }
@@ -32,6 +33,67 @@ export type RuntimeCodec = {
}
type Definitions = { [name: string]: Definition }
+/**
+ * Message types decoded as headers: only the named fields are read, and the rest stay in the
+ * message bytes until `decodeWhole` reads the message in full. A large message of many such
+ * messages then holds a little of each rather than every value of every one.
+ */
+export type HeaderFields = Readonly<Record<string, readonly string[]>>
+
+/**
+ * What a header needs to be read whole: the bytes it came from, where it starts in them, and
+ * the decoder that reads every field. They are hidden properties of the header, so each costs a
+ * slot on the object; neither spread nor `structuredClone` copies them.
+ */
+const HEADER_BYTES = Symbol('header bytes')
+const HEADER_START = Symbol('header start')
+const HEADER_WHOLE = Symbol('header whole decoder')
+
+/**
+ * How many whole decodes are running. A message type can nest itself, as instance overrides
+ * are records too, and a message read whole is read whole all the way down.
+ */
+let wholeDepth = 0
+
+/**
+ * A header read again in full from the bytes it came from, as a new message the caller owns,
+ * or undefined for a message that was decoded whole.
+ */
+export function decodeWhole(message: object): RuntimeMessage | undefined {
+ const bytes: unknown = Reflect.get(message, HEADER_BYTES)
+ const start: unknown = Reflect.get(message, HEADER_START)
+ const whole: unknown = Reflect.get(message, HEADER_WHOLE)
+ if (!(bytes instanceof Uint8Array) || typeof start !== 'number' || typeof whole !== 'function')
+ return undefined
+ const buffer = new ByteBuffer(bytes)
+ buffer.offset = start
+ return (whole as (bb: ByteBuffer) => RuntimeMessage)(buffer)
+}
+
+function hide(message: RuntimeMessage, key: symbol, value: unknown): void {
+ Reflect.defineProperty(message, key, { value, enumerable: false })
+}
+
+interface HeaderDecoding {
+ keep: ReadonlySet<string>
+ skipper: SchemaSkipper
+ /** Reads the message in full, for `decodeWhole`. */
+ whole: (bb: ByteBuffer) => RuntimeMessage
+}
+
+function wholeDecoder(
+ decode: (bb: ByteBuffer) => RuntimeMessage
+): (bb: ByteBuffer) => RuntimeMessage {
+ return (bb) => {
+ wholeDepth++
+ try {
+ return decode(bb)
+ } finally {
+ wholeDepth--
+ }
+ }
+}
+
function asDecodeFunction(value: unknown): DecodeFunction {
return value as DecodeFunction
}
@@ -220,7 +282,12 @@ function writeFrom(
writeField(self, definitions, type, value, bb)
}
-function interpretDecode(self: RuntimeCodec, definitions: Definitions, definition: Definition) {
+function interpretDecode(
+ self: RuntimeCodec,
+ definitions: Definitions,
+ definition: Definition,
+ header?: HeaderDecoding
+) {
let fieldsById = new Map<number, Field>()
for (let i = 0; i < definition.fields.length; i++)
fieldsById.set(definition.fields[i].value, definition.fields[i])
@@ -229,12 +296,19 @@ function interpretDecode(self: RuntimeCodec, definitions: Definitions, definitio
let result: RuntimeMessage = {}
if (definition.kind === 'MESSAGE') {
+ const asHeader = header && wholeDepth === 0 ? header : undefined
+ if (asHeader) {
+ hide(result, HEADER_BYTES, buffer.bytes)
+ hide(result, HEADER_START, buffer.offset)
+ hide(result, HEADER_WHOLE, asHeader.whole)
+ }
while (true) {
const id = buffer.readVarUint()
if (id === 0) return result
const field = fieldsById.get(id)
if (!field) throw new Error('Attempted to parse invalid message')
- readInto(self, definitions, field, buffer, result)
+ if (asHeader && !asHeader.keep.has(field.name)) asHeader.skipper.skipField(field, buffer)
+ else readInto(self, definitions, field, buffer, result)
}
} else {
for (let i = 0; i < definition.fields.length; i++) {
@@ -268,7 +342,7 @@ function interpretEncode(self: RuntimeCodec, definitions: Definitions, definitio
}
}
-export function compileSchemaRuntime(schema: Schema): RuntimeCodec {
+export function compileSchemaRuntime(schema: Schema, headers?: HeaderFields): RuntimeCodec {
let definitions: Definitions = Object.create(null) as Definitions
for (let i = 0; i < schema.definitions.length; i++) {
definitions[schema.definitions[i].name] = schema.definitions[i]
@@ -279,6 +353,7 @@ export function compileSchemaRuntime(schema: Schema): RuntimeCodec {
let result: RuntimeCodec = {
ByteBuffer: ByteBuffer
}
+ let skipper: SchemaSkipper | undefined
for (let i = 0; i < schema.definitions.length; i++) {
let definition = schema.definitions[i]
@@ -297,7 +372,18 @@ export function compileSchemaRuntime(schema: Schema): RuntimeCodec {
case 'STRUCT':
case 'MESSAGE': {
- result['decode' + definition.name] = interpretDecode(result, definitions, definition)
+ const keep = definition.kind === 'MESSAGE' ? headers?.[definition.name] : undefined
+ const header: HeaderDecoding | undefined = keep && {
+ keep: new Set(keep),
+ skipper: (skipper ??= createSchemaSkipper(schema)),
+ whole: wholeDecoder(interpretDecode(result, definitions, definition))
+ }
+ result['decode' + definition.name] = interpretDecode(
+ result,
+ definitions,
+ definition,
+ header
+ )
result['encode' + definition.name] = interpretEncode(result, definitions, definition)
break
}
diff --git a/packages/kiwi/tests/header-fields.test.ts b/packages/kiwi/tests/header-fields.test.ts
new file mode 100644
index 000000000..269f88e88
--- /dev/null
+++ b/packages/kiwi/tests/header-fields.test.ts
@@ -0,0 +1,69 @@
+import { describe, expect, test } from 'bun:test'
+
+import { compileSchema, decodeWhole, parseSchema, type HeaderFields } from '../src/schema-runtime'
+import type { RuntimeMessage } from '../src/schema-runtime/interpreter'
+
+const schema = parseSchema(`
+package Example;
+
+struct Point {
+ float x;
+ float y;
+}
+
+message Item {
+ uint id = 1;
+ Point[] outline = 2;
+ string note = 3;
+ Item[] children = 4;
+}
+
+message Document {
+ Item[] items = 1;
+}
+`)
+
+const document = {
+ items: [
+ {
+ id: 1,
+ outline: [{ x: 1, y: 2 }],
+ note: 'first',
+ children: [{ id: 3, note: 'nested', children: [{ id: 4, note: 'deeper' }] }]
+ },
+ { id: 2, note: 'second' }
+ ]
+}
+
+function decode(headers?: HeaderFields): RuntimeMessage[] {
+ const encode = compileSchema(schema).encodeDocument as (message: RuntimeMessage) => Uint8Array
+ const codec = compileSchema(schema, headers)
+ const decodeDocument = codec.decodeDocument as (bytes: Uint8Array) => { items: RuntimeMessage[] }
+ return decodeDocument(encode(document as RuntimeMessage)).items
+}
+
+describe('header fields', () => {
+ test('a header keeps the named fields, and reads whole into every field', () => {
+ const items = decode({ Item: ['id'] })
+
+ expect(items).toEqual([{ id: 1 }, { id: 2 }])
+ expect<unknown>(items.map((item) => decodeWhole(item))).toEqual(document.items)
+ })
+
+ test('a message nesting its own type reads whole all the way down', () => {
+ const [first] = decode({ Item: ['id'] })
+
+ const whole = decodeWhole(first)
+
+ expect<unknown>(whole?.children).toEqual(document.items[0].children)
+ })
+
+ test('a message decoded whole, or a copy of a header, is no header', () => {
+ const [eager] = decode()
+ const [header] = decode({ Item: ['id'] })
+
+ expect(decodeWhole(eager)).toBeUndefined()
+ expect(decodeWhole({ ...header })).toBeUndefined()
+ expect(decodeWhole(structuredClone(header))).toBeUndefined()
+ })
+})